From cb0cf380478608339bde42f12e8eb4dc9dcd82da Mon Sep 17 00:00:00 2001 From: Stephen Jia Date: Sat, 31 Oct 2026 00:31:02 -0700 Subject: [PATCH] [ET-VK][benchmark] Add JSON operator and shader benchmark cases Pull Request resolved: https://github.com/pytorch/executorch/pull/24344 ## Ulterior Motive Replace C++ benchmark case definitions with JSON specifications. ## Rationale Introduce `etvk_ops_bench` with inline tensor arguments, deterministic initialization, dtype/storage and shader/workgroup sweeps, operator routes, independent CPU references, and registered throughput estimates. Generate host tensor data once per case and share it between the staged validation graph and a separate benchmark graph using automatic calibration. Select validation scope with `--validation_mode=all|first|none`, defaulting to all cases. Default to compact `json` reports; provide condensed tables and detailed `json_full` reports with per-batch GPU/operator samples and shader statistics. Record Android GPU clock start/end snapshots per measurement pass, adjacent to the corresponding batch timing arrays. Tables show observed minimum/maximum GPU clocks across each case and profiling-only ranges for shader rows. Document measurement boundaries, clock units, and shader percentile sampling. Migrate 64 sets into 4,560 definitions using production operators directly, without a dependency on the old benchmark directory. Keep quantized references, route checks, and scripts together. Case JSONs and reports omit `legacy_case`. Review framework parsing/execution/reporting first, then operator references/routes and JSON cases, then usage documentation. This remains one commit above the five runtime commits. Runtime-specific generator tests accompany their implementation; workgroup tests and their Buck target live in `test/api_abstractions/`. Benchmark validation also covers rejection of unrecognized SPIR-V capabilities before pipeline creation. Non-constant tensor inputs display their configured memory layout after the dtype; constant tensors omit this suffix because operator prepacking selects their GPU layout. `json` uses untruncated input strings, min/median/max batch timing summaries, and per-pass clock extrema. Float arguments and list elements in input strings use three significant figures. `json_full` preserves tensor descriptions and individual batch samples. Both retain shader timing statistics. The public README describes the binary, layout, JSON schema, validation, and timing. Internal helper scripts and Buck build instructions are documented under `fb/README.md`; migration documents and the TODO file are omitted. CI workflow integration will follow separately. Authored with Codex (AI). ghstack-source-id: 461636577 Differential Revision: [D128509578](https://our.internmc.facebook.com/intern/diff/D128509578/) --- backends/vulkan/test/ops/benchmark_v2/BUCK | 9 + .../vulkan/test/ops/benchmark_v2/README.md | 995 +++ .../test/ops/benchmark_v2/etvk_ops_bench.cpp | 287 + .../benchmark_v2/examples/buffer_shader.json | 29 + .../ops/benchmark_v2/examples/conv2d.json | 62 + .../benchmark_v2/examples/conv2d_routes.json | 110 + .../ops/benchmark_v2/examples/linear.json | 101 + .../benchmark_v2/examples/q4gsw_linear.json | 67 + .../benchmark_v2/examples/q8ta_conv2d.json | 83 + .../examples/q8ta_conv2d_routes.json | 116 + .../examples/shader_workgroup_sweep.json | 33 + .../ops/benchmark_v2/examples/shaders.json | 79 + .../benchmark_v2/examples/texture_shader.json | 47 + .../examples/texture_shader_lwg_range.json | 75 + .../examples/texture_shader_lwg_sweep.json | 76 + .../framework/CaseCheckRegistry.cpp | 66 + .../framework/CaseCheckRegistry.h | 36 + .../ops/benchmark_v2/framework/Config.cpp | 531 ++ .../test/ops/benchmark_v2/framework/Config.h | 28 + .../framework/DataInitializer.cpp | 433 ++ .../benchmark_v2/framework/DataInitializer.h | 25 + .../ops/benchmark_v2/framework/GpuClock.cpp | 78 + .../ops/benchmark_v2/framework/GpuClock.h | 23 + .../framework/OperatorBenchmark.cpp | 161 + .../framework/OperatorBenchmark.h | 26 + .../ops/benchmark_v2/framework/Reference.cpp | 307 + .../ops/benchmark_v2/framework/Reference.h | 51 + .../framework/ReferenceRegistry.cpp | 112 + .../framework/ReferenceRegistry.h | 73 + .../ops/benchmark_v2/framework/Report.cpp | 823 +++ .../test/ops/benchmark_v2/framework/Report.h | 61 + .../benchmark_v2/framework/RouteRegistry.cpp | 95 + .../benchmark_v2/framework/RouteRegistry.h | 77 + .../framework/ShaderBenchmark.cpp | 302 + .../benchmark_v2/framework/ShaderBenchmark.h | 19 + .../test/ops/benchmark_v2/framework/Sweep.cpp | 268 + .../test/ops/benchmark_v2/framework/Sweep.h | 16 + .../benchmark_v2/framework/TensorUtils.cpp | 181 + .../ops/benchmark_v2/framework/TensorUtils.h | 40 + .../framework/ThroughputRegistry.cpp | 280 + .../framework/ThroughputRegistry.h | 105 + .../ops/benchmark_v2/framework/Timestamps.cpp | 82 + .../ops/benchmark_v2/framework/Timestamps.h | 37 + .../ops/benchmark_v2/framework/Timing.cpp | 275 + .../test/ops/benchmark_v2/framework/Timing.h | 23 + .../ops/benchmark_v2/framework/Validation.cpp | 148 + .../ops/benchmark_v2/framework/Validation.h | 33 + .../benchmark_v2/framework/WorkgroupUtils.cpp | 110 + .../benchmark_v2/framework/WorkgroupUtils.h | 28 + .../test/case_check_registry_test.cpp | 92 + .../framework/test/config_test.cpp | 937 +++ .../framework/test/data_initializer_test.cpp | 255 + .../framework/test/gpu_clock_test.cpp | 112 + .../framework/test/operator_route_test.cpp | 426 ++ .../test/reference_registry_test.cpp | 251 + .../framework/test/reference_test.cpp | 362 + .../framework/test/report_test.cpp | 980 +++ .../framework/test/route_registry_test.cpp | 111 + .../test/throughput_registry_test.cpp | 510 ++ .../framework/test/throughput_report_test.cpp | 135 + .../framework/test/validation_test.cpp | 254 + .../operators/MigrationReferenceTest.cpp | 221 + .../attention/sdpa/SdpaReference.cpp | 104 + .../operators/attention/sdpa/SdpaReference.h | 16 + .../operators/attention/sdpa/SdpaRoutes.cpp | 51 + .../attention/sdpa/SdpaThroughput.cpp | 37 + .../attention/sdpa/cases/correctness.json | 1522 ++++ .../attention/sdpa/cases/decode.json | 494 ++ .../attention/sdpa/cases/performance.json | 160 + .../operators/binary/BinaryReference.cpp | 111 + .../operators/binary/BinaryReference.h | 19 + .../operators/binary/BinaryThroughput.cpp | 29 + .../operators/binary/add/AddReference.cpp | 30 + .../operators/binary/add/AddReference.h | 18 + .../operators/binary/add/AddThroughput.cpp | 19 + .../binary/add/cases/correctness.json | 72 + .../binary/q8ta/Q8taBinaryReference.cpp | 35 + .../binary/q8ta/Q8taBinaryThroughput.cpp | 20 + .../binary/q8ta/cases/correctness.json | 4087 +++++++++++ .../operators/binary/q8ta/cases/debug.json | 483 ++ .../clone/q8ta/Q8taCloneReference.cpp | 18 + .../clone/q8ta/Q8taCloneThroughput.cpp | 20 + .../clone/q8ta/cases/correctness.json | 94 + .../operators/clone/q8ta/cases/debug.json | 74 + .../operators/convolution/Conv2dReference.cpp | 68 + .../operators/convolution/Conv2dReference.h | 15 + .../convolution/Conv2dReferenceCommon.cpp | 58 + .../convolution/Conv2dReferenceCommon.h | 97 + .../convolution/Conv2dRouteCommon.cpp | 135 + .../operators/convolution/Conv2dRouteCommon.h | 47 + .../operators/convolution/Conv2dRoutes.cpp | 235 + .../operators/convolution/Conv2dRoutes.h | 26 + .../convolution/Conv2dThroughput.cpp | 43 + .../convolution/Conv2dThroughputCommon.cpp | 76 + .../convolution/Conv2dThroughputCommon.h | 21 + .../convolution/conv1d/Conv1dReference.cpp | 56 + .../convolution/conv1d/Conv1dReference.h | 11 + .../conv1d/cases/dw_correctness.json | 796 +++ .../conv1d/cases/dw_performance.json | 474 ++ .../conv1d/cases/pw_correctness.json | 1129 +++ .../conv1d/cases/pw_performance.json | 646 ++ .../convolution/conv2d/DepthwiseRoutes.cpp | 300 + .../convolution/conv2d/cases/correctness.json | 1464 ++++ .../conv2d/cases/dw_correctness.json | 698 ++ .../conv2d/cases/dw_performance.json | 1826 +++++ .../convolution/conv2d/cases/performance.json | 1786 +++++ .../conv2d/cases/pw_correctness.json | 378 + .../conv2d/cases/pw_performance.json | 1900 +++++ .../convolution/q8ta/Q8taConv2dReference.cpp | 146 + .../convolution/q8ta/Q8taConv2dReference.h | 21 + .../convolution/q8ta/Q8taConv2dRoutes.cpp | 298 + .../convolution/q8ta/Q8taConv2dRoutes.h | 26 + .../convolution/q8ta/Q8taConv2dThroughput.cpp | 122 + .../convolution/q8ta/Q8taValidationChecks.cpp | 158 + .../convolution/q8ta/Q8taValidationChecks.h | 13 + .../convolution/q8ta/cases/correctness.json | 3474 +++++++++ .../convolution/q8ta/cases/debug.json | 368 + .../q8ta/cases/dw_correctness.json | 1826 +++++ .../convolution/q8ta/cases/dw_debug.json | 245 + .../q8ta/cases/dw_narrow_workgroup.json | 248 + .../q8ta/cases/dw_performance.json | 1214 ++++ .../convolution/q8ta/cases/im2col.json | 961 +++ .../q8ta/cases/narrow_workgroup.json | 242 + .../convolution/q8ta/cases/performance.json | 4074 +++++++++++ .../q8ta/cases/pw_correctness.json | 3355 +++++++++ .../q8ta/cases/pw_performance.json | 2178 ++++++ .../q8ta/cases/scenex_grouped.json | 371 + .../q8ta/cases/scenex_regular.json | 248 + .../q8ta/cases/streaming_dynamic_shrink.json | 200 + .../q8ta/cases/streaming_im2col.json | 725 ++ .../q8ta/cases/unsigned_dot_selection.json | 8 + .../q8ta/test/q8ta_native_reference_test.cpp | 159 + .../transposed/Q8taTransposedReference.cpp | 115 + .../q8ta/transposed/cases/correctness.json | 627 ++ .../q8ta/transposed/cases/debug.json | 127 + .../q8ta/transposed/cases/performance.json | 377 + .../qw/q8csw/Q8cswConv2dReference.cpp | 78 + .../qw/q8csw/cases/correctness.json | 2396 +++++++ .../qw/q8csw/cases/performance.json | 608 ++ .../embedding/qw/q4gsw/EmbeddingReference.cpp | 45 + .../embedding/qw/q4gsw/EmbeddingReference.h | 18 + .../embedding/qw/q4gsw/cases/correctness.json | 922 +++ .../embedding/qw/q4gsw/cases/performance.json | 154 + .../operators/gemm/GemmReference.cpp | 68 + .../operators/gemm/GemmReference.h | 18 + .../operators/gemm/GemmThroughputCommon.cpp | 44 + .../operators/gemm/GemmThroughputCommon.h | 13 + .../operators/gemm/linear/LinearReference.cpp | 29 + .../operators/gemm/linear/LinearReference.h | 15 + .../gemm/linear/LinearThroughput.cpp | 62 + .../gemm/linear/q8ta/Q8taLinearReference.cpp | 75 + .../gemm/linear/q8ta/Q8taLinearRoutes.cpp | 24 + .../gemm/linear/q8ta/Q8taLinearThroughput.cpp | 26 + .../gemm/linear/q8ta/cases/correctness.json | 1638 +++++ .../gemm/linear/q8ta/cases/performance.json | 947 +++ .../qw/dq8ca_q4gsw/Dq8caQ4gswReference.cpp | 59 + .../qw/dq8ca_q4gsw/Dq8caQ4gswReference.h | 18 + .../qw/dq8ca_q4gsw/Dq8caQ4gswThroughput.cpp | 26 + .../linear/qw/q4gsw/Q4gswLinearReference.cpp | 56 + .../linear/qw/q4gsw/Q4gswLinearReference.h | 17 + .../linear/qw/q4gsw/Q4gswLinearThroughput.cpp | 91 + .../gemm/linear/qw/q4gsw/Q4gswRoutes.cpp | 766 ++ .../qw/q4gsw/cases/coopmat_correctness.json | 1466 ++++ .../qw/q4gsw/cases/coopmat_performance.json | 734 ++ .../linear/qw/q4gsw/cases/correctness.json | 6290 +++++++++++++++++ .../gemm/linear/qw/q4gsw/cases/debug.json | 124 + .../qw/q4gsw/cases/fpa_correctness.json | 1458 ++++ .../qw/q4gsw/cases/fpa_performance.json | 1897 +++++ .../linear/qw/q4gsw/cases/performance.json | 734 ++ .../q4gsw/glsl/q4gsw_linear_gemv__w_4x8.glsl | 371 + .../q4gsw/glsl/q4gsw_linear_gemv__w_4x8.yaml | 40 + .../linear/qw/q8csw/Q8cswLinearReference.cpp | 83 + .../linear/qw/q8csw/cases/correctness.json | 1382 ++++ .../gemm/linear/qw/q8csw/cases/debug.json | 100 + .../linear/qw/q8csw/cases/performance.json | 190 + .../operators/gemm/matmul/MatmulReference.cpp | 61 + .../operators/gemm/matmul/MatmulReference.h | 16 + .../operators/gemm/matmul/MatmulRoutes.cpp | 84 + .../gemm/matmul/MatmulThroughput.cpp | 32 + .../gemm/matmul/cases/correctness.json | 3681 ++++++++++ .../gemm/matmul/cases/performance.json | 5502 ++++++++++++++ .../q8ta/Q8taPixelShuffleReference.cpp | 49 + .../q8ta/Q8taPixelShuffleThroughput.cpp | 30 + .../pixel_shuffle/q8ta/cases/correctness.json | 482 ++ .../pixel_shuffle/q8ta/cases/performance.json | 242 + .../ChooseQparamsReference.cpp | 67 + .../ChooseQparamsReference.h | 18 + .../ChooseQparamsThroughput.cpp | 25 + .../cases/correctness.json | 564 ++ .../choose_qparams_per_row/cases/debug.json | 64 + .../cases/performance.json | 64 + .../quantize/q8ta/Q8taQdqReference.cpp | 49 + .../operators/quantize/q8ta/Q8taQdqRoutes.cpp | 67 + .../quantize/q8ta/Q8taQdqThroughput.cpp | 30 + .../quantize/q8ta/Q8taReferenceCommon.h | 90 + .../quantize/q8ta/cases/correctness.json | 710 ++ .../operators/quantize/q8ta/cases/debug.json | 274 + .../q8ta/test/q8_benchmark_reference_test.cpp | 333 + .../unary/q8ta/Q8taUnaryReference.cpp | 30 + .../unary/q8ta/Q8taUnaryThroughput.cpp | 20 + .../unary/q8ta/cases/correctness.json | 115 + .../operators/unary/q8ta/cases/debug.json | 96 + .../vulkan/test/ops/benchmark_v2/targets.bzl | 358 + 203 files changed, 89727 insertions(+) create mode 100644 backends/vulkan/test/ops/benchmark_v2/BUCK create mode 100644 backends/vulkan/test/ops/benchmark_v2/README.md create mode 100644 backends/vulkan/test/ops/benchmark_v2/etvk_ops_bench.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/examples/buffer_shader.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/examples/conv2d.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/examples/conv2d_routes.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/examples/linear.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/examples/q4gsw_linear.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/examples/q8ta_conv2d.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/examples/q8ta_conv2d_routes.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/examples/shader_workgroup_sweep.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/examples/shaders.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/examples/texture_shader.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/examples/texture_shader_lwg_range.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/examples/texture_shader_lwg_sweep.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/CaseCheckRegistry.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/CaseCheckRegistry.h create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/Config.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/Config.h create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/DataInitializer.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/DataInitializer.h create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/GpuClock.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/GpuClock.h create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/OperatorBenchmark.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/OperatorBenchmark.h create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/Reference.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/Reference.h create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/ReferenceRegistry.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/ReferenceRegistry.h create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/Report.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/Report.h create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/RouteRegistry.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/RouteRegistry.h create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/ShaderBenchmark.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/ShaderBenchmark.h create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/Sweep.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/Sweep.h create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/TensorUtils.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/TensorUtils.h create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/ThroughputRegistry.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/ThroughputRegistry.h create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/Timestamps.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/Timestamps.h create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/Timing.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/Timing.h create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/Validation.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/Validation.h create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/WorkgroupUtils.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/WorkgroupUtils.h create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/test/case_check_registry_test.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/test/config_test.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/test/data_initializer_test.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/test/gpu_clock_test.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/test/operator_route_test.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/test/reference_registry_test.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/test/reference_test.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/test/report_test.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/test/route_registry_test.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/test/throughput_registry_test.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/test/throughput_report_test.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/framework/test/validation_test.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/MigrationReferenceTest.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/SdpaReference.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/SdpaReference.h create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/SdpaRoutes.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/SdpaThroughput.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/cases/correctness.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/cases/decode.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/cases/performance.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/binary/BinaryReference.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/binary/BinaryReference.h create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/binary/BinaryThroughput.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/binary/add/AddReference.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/binary/add/AddReference.h create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/binary/add/AddThroughput.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/binary/add/cases/correctness.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/binary/q8ta/Q8taBinaryReference.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/binary/q8ta/Q8taBinaryThroughput.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/binary/q8ta/cases/correctness.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/binary/q8ta/cases/debug.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/clone/q8ta/Q8taCloneReference.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/clone/q8ta/Q8taCloneThroughput.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/clone/q8ta/cases/correctness.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/clone/q8ta/cases/debug.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dReference.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dReference.h create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dReferenceCommon.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dReferenceCommon.h create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dRouteCommon.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dRouteCommon.h create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dRoutes.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dRoutes.h create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dThroughput.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dThroughputCommon.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dThroughputCommon.h create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv1d/Conv1dReference.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv1d/Conv1dReference.h create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv1d/cases/dw_correctness.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv1d/cases/dw_performance.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv1d/cases/pw_correctness.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv1d/cases/pw_performance.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/DepthwiseRoutes.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/cases/correctness.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/cases/dw_correctness.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/cases/dw_performance.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/cases/performance.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/cases/pw_correctness.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/cases/pw_performance.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taConv2dReference.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taConv2dReference.h create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taConv2dRoutes.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taConv2dRoutes.h create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taConv2dThroughput.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taValidationChecks.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taValidationChecks.h create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/correctness.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/debug.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/dw_correctness.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/dw_debug.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/dw_narrow_workgroup.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/dw_performance.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/im2col.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/narrow_workgroup.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/performance.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/pw_correctness.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/pw_performance.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/scenex_grouped.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/scenex_regular.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/streaming_dynamic_shrink.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/streaming_im2col.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/unsigned_dot_selection.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/test/q8ta_native_reference_test.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/transposed/Q8taTransposedReference.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/transposed/cases/correctness.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/transposed/cases/debug.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/transposed/cases/performance.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/qw/q8csw/Q8cswConv2dReference.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/qw/q8csw/cases/correctness.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/convolution/qw/q8csw/cases/performance.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/embedding/qw/q4gsw/EmbeddingReference.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/embedding/qw/q4gsw/EmbeddingReference.h create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/embedding/qw/q4gsw/cases/correctness.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/embedding/qw/q4gsw/cases/performance.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/gemm/GemmReference.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/gemm/GemmReference.h create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/gemm/GemmThroughputCommon.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/gemm/GemmThroughputCommon.h create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/LinearReference.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/LinearReference.h create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/LinearThroughput.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/q8ta/Q8taLinearReference.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/q8ta/Q8taLinearRoutes.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/q8ta/Q8taLinearThroughput.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/q8ta/cases/correctness.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/q8ta/cases/performance.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/dq8ca_q4gsw/Dq8caQ4gswReference.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/dq8ca_q4gsw/Dq8caQ4gswReference.h create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/dq8ca_q4gsw/Dq8caQ4gswThroughput.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/Q4gswLinearReference.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/Q4gswLinearReference.h create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/Q4gswLinearThroughput.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/Q4gswRoutes.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/coopmat_correctness.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/coopmat_performance.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/correctness.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/debug.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/fpa_correctness.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/fpa_performance.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/performance.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/glsl/q4gsw_linear_gemv__w_4x8.glsl create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/glsl/q4gsw_linear_gemv__w_4x8.yaml create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q8csw/Q8cswLinearReference.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q8csw/cases/correctness.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q8csw/cases/debug.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q8csw/cases/performance.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/gemm/matmul/MatmulReference.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/gemm/matmul/MatmulReference.h create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/gemm/matmul/MatmulRoutes.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/gemm/matmul/MatmulThroughput.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/gemm/matmul/cases/correctness.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/gemm/matmul/cases/performance.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/pixel_shuffle/q8ta/Q8taPixelShuffleReference.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/pixel_shuffle/q8ta/Q8taPixelShuffleThroughput.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/pixel_shuffle/q8ta/cases/correctness.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/pixel_shuffle/q8ta/cases/performance.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/quantization/choose_qparams_per_row/ChooseQparamsReference.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/quantization/choose_qparams_per_row/ChooseQparamsReference.h create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/quantization/choose_qparams_per_row/ChooseQparamsThroughput.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/quantization/choose_qparams_per_row/cases/correctness.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/quantization/choose_qparams_per_row/cases/debug.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/quantization/choose_qparams_per_row/cases/performance.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/Q8taQdqReference.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/Q8taQdqRoutes.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/Q8taQdqThroughput.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/Q8taReferenceCommon.h create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/cases/correctness.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/cases/debug.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/test/q8_benchmark_reference_test.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/unary/q8ta/Q8taUnaryReference.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/unary/q8ta/Q8taUnaryThroughput.cpp create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/unary/q8ta/cases/correctness.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/operators/unary/q8ta/cases/debug.json create mode 100644 backends/vulkan/test/ops/benchmark_v2/targets.bzl diff --git a/backends/vulkan/test/ops/benchmark_v2/BUCK b/backends/vulkan/test/ops/benchmark_v2/BUCK new file mode 100644 index 00000000000..f559a6f1cfe --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/BUCK @@ -0,0 +1,9 @@ +# Any targets that should be shared between fbcode and xplat must be defined in +# targets.bzl. + +load("@fbsource//tools/build_defs:fbsource_utils.bzl", "is_fbcode") +load(":targets.bzl", "define_common_targets") + +oncall("executorch") + +define_common_targets(is_fbcode = is_fbcode()) diff --git a/backends/vulkan/test/ops/benchmark_v2/README.md b/backends/vulkan/test/ops/benchmark_v2/README.md new file mode 100644 index 00000000000..b745d451c54 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/README.md @@ -0,0 +1,995 @@ +# benchmark_v2: JSON shader and operator benchmarks + +`etvk_ops_bench` replays a registered ET-VK shader or operator from a JSON config. +Pass one object or an array mixing both case types. Each shader config or operator +sweep entry gets its own tensors, calibration, and measurements. Results default +to compact JSON on stdout, with argument strings and timing summaries. Runtime +failures appear as error objects and execution continues with the next config. +Use `--output_format=json_full` for full ordered JSON reports or +`--output_format=table` for a condensed table. + +## Layout + +Define inputs, tensor shapes, initialization, sweeps, and measurement settings in +JSON. The binary validates outputs against registered CPU references, calibrates +the repetition count, and reports GPU timestamps, wall time, and optional +throughput estimates. It executes shaders and operators linked into the ET-VK +runtime; raw shader configurations must match the selected shader's interface. + +Paths below are relative to `test/ops/benchmark_v2/`: + +```text +etvk_ops_bench.cpp CLI and case execution +framework/ Parsing, graph setup, validation, timing, and reporting + test/ Framework and registry tests +operators/ CPU references, routes, throughput counters, and checks + /.../cases/ JSON case collections by operator and quantization scheme +examples/ Small shader and operator configuration examples +``` + +Each expanded case runs separately. Shader variants can share one binding +interface through `shader_sweep`; operators can sweep argument sets, dtype and +storage variables, and execution routes. + +## Running the binary + +The commands below assume `etvk_ops_bench` is on `PATH` and the current directory +is `test/ops/benchmark_v2/`. A host run requires a Vulkan loader and a compute-capable +Vulkan implementation. Select a hardware GPU or configure a software implementation +such as SwiftShader in the execution environment. + +```sh +etvk_ops_bench --config=examples/shaders.json +etvk_ops_bench --config=examples/linear.json --output_format=table +etvk_ops_bench --config=examples/conv2d.json --validation_only --validation_mode=first +``` + +For Android, stage an Android build of the executable and the JSON file with ADB. +Replace `SERIAL` with the connected device's serial and `/path/to/etvk_ops_bench` +with the built executable: + +```sh +adb -s SERIAL push /path/to/etvk_ops_bench /data/local/tmp/etvk_ops_bench +adb -s SERIAL push examples/shaders.json /data/local/tmp/benchmark_config.json +adb -s SERIAL shell chmod +x /data/local/tmp/etvk_ops_bench +adb -s SERIAL shell /data/local/tmp/etvk_ops_bench --config=/data/local/tmp/benchmark_config.json +``` + +## Command-line options + +The binary requires `--config=`. All options follow the executable +name; tensor and benchmark settings belong in the JSON file. + +| Option | Default | Behavior | +| --- | --- | --- | +| `--config=` | Required | Read one case or an array of case definitions. | +| `--output_format=json\|json_full\|table` | `json` | Compact JSON, full JSON, or human-readable table. | +| `--validation_mode=all\|first\|none` | `all` | Validate every case, only the first expanded case, or no cases. | +| `--error_mode=report\|throw\|skip` | `report` | Report failures and continue, stop on the first failure, or disable validation. Unsupported cases report `SKIP`. | +| `--validate_specs` | `false` | Audit schemas and registrations without allocating case tensors or running GPU work. | +| `--validation_only` | `false` | Run correctness checks without calibration or timing passes. Requires validation enabled. | +| `--table_name_max_chars=` | `32` | Limit shader/operator names in table output; `0` disables truncation. | +| `--table_inputs_max_chars=` | `64` | Limit table input strings; `0` disables truncation. | +| `--helpshort` | — | Show the binary's flags and defaults. | + +For a quick correctness check, pass `--validation_only --validation_mode=first`. +For timing without CPU reference work, pass `--validation_mode=none`. +Benchmark conditions such as `target_ms` and `benchmark_runs` belong in the JSON. + +## Config + +| Example | Cases | +| --- | --- | +| `examples/buffer_shader.json` | Raw buffer copy. | +| `examples/texture_shader.json` | Raw 3D texture addition. | +| `examples/texture_shader_lwg_sweep.json` | Texture addition with explicit LWG candidates, including `[10,10,1]`. | +| `examples/texture_shader_lwg_range.json` | Texture addition with a Cartesian LWG range. | +| `examples/shader_workgroup_sweep.json` | Two compatible 32-bit buffer-copy shaders crossed with four LWG sizes. | +| `examples/shaders.json` | Raw buffer copy and texture addition. | +| `examples/conv2d.json` | FP32 and FP16 convolution. | +| `examples/conv2d_routes.json` | FP32/FP16 direct and im2col convolution, including intermediate-storage variants. | +| `examples/q8ta_conv2d_routes.json` | Q8TA automatic, direct, and im2col convolution. | +| `examples/q8ta_conv2d.json` | Quantized int8 activations, per-channel int8 weights, and int8 output. | +| `examples/linear.json` | FP32/FP16 linear projections and feed-forward expansion, with decode and batched/prefill shapes. | +| `examples/q4gsw_linear.json` | Weight-only Q4GSW linear with FP32 and FP16 activations. | + +| Field | Meaning | +| --- | --- | +| `shader` | Compiled registry name, e.g. `buffer_to_buffer_float`. | +| `operator` | Operator registry name, e.g. `aten.linear.default`. Specify exactly one of `shader`, `shader_sweep`, `operator`, or `check`. | +| `shader_sweep` | Nonempty shader-name list; every shader uses the shared bindings, initialization, parameters, and benchmark settings. | +| `name` | Optional result label; defaults to the registry name. | +| `gwg_size` | Three positive shader invocation counts or a global picker name. | +| `lwg_size` | Three positive workgroup dimensions or a local picker name. | +| `lwg_size_sweep` | Shader-only Cartesian search: `x`, `y`, and `z` ranges `[start, stop, step]`. | +| `bindings` | Shader bindings in binding order, with tensor properties declared inline. | +| `sweep_variables` | Operator variable names mapped to nonempty choice arrays; all Cartesian combinations are tested. | +| `route` | Operator-only route name or `{"name":"im2col","options":{...}}`; omitted means normal automatic routing. | +| `route_sweep` | Operator-only nonempty list of route selections, crossed with argument sets and variables. | +| `args_sweep` | Nonempty list of full operator argument arrays; each set is benchmarked separately. | +| `specialization_constants` | Optional array for IDs 3 onward; local dimensions occupy IDs 0–2. | +| `push_constants` | Optional array of typed data blocks, concatenated into at most 128 bytes. | +| `outputs` | Shader-only optional logical output names. | +| `target_ms` | Calibration target per batch, default 50. | +| `max_dispatches` | Invocation count cap, default 200 for operators and 65536 for shaders. | +| `warmup_runs` | Unmeasured batches per timing mode, default 3. | +| `benchmark_runs` | Measured batches per timing mode, default 10. | +| `check` | Registered validation-only check, e.g. capability or dynamic-resize checks. | +| `reference` | Optional registered CPU reference name; inferred for supported operators/shaders. | +| `validation_method` | Comparison function and tolerances, e.g. `{"function":"is_close","atol":0.01,"rtol":0.01}`. Optional `outputs` selects named tensors to check. | +| `show_memory_throughput` | Show registered memory-throughput estimates for each measurement; default false. | +| `show_compute_throughput` | Show registered compute-throughput estimates for each measurement; default false. | + +Tensor descriptions require `sizes` in logical NCHW order. For shaders, put these +properties inside the `buffer`, `texture2d`, or `texture3d` binding object; +for operators, use the `tensor` object inside each argument set in `args_sweep`. Optional fields: + +- `name`: optional tensor name; defaults to `arg_` + for operators and `binding_` for shaders. +- `dtype`: `float32` (default), `float16`, `int32`, `int8`, `uint8`, or `int8x4`. +- `storage`: operator-only; `buffer` (default), `texture2d`, `texture3d`, or + `constant`. Constant storage passes a CPU tensor reference to the operator's + prepack route; the operator chooses its GPU storage. + For shader bindings, the wrapper key determines storage. +- `layout`: `width` (default), `height`, or `channels` packing. +- `data`: initialization descriptor, described below; defaults to zero. +- `access`: optional operator role override, `read`, `write`, or `read_write`. + Roles are inferred from the operator's dispatch nodes when omitted. + Writable tensors cannot be constant. + +| `data` | Initialization | +| --- | --- | +| `{"fill":0.12}` | Fill every logical element with a value representable by the dtype. | +| `{"rand_uniform_float":[0,1]}` | Uniform random values for FP32/FP16 tensors. | +| `{"rand_uniform_float":[0,1],"seed":42}` | Uniform values using an explicit uint32 seed. | +| `{"rand_uniform_int":[-8,8]}` | Uniform integers, including both bounds; also supports integer-valued FP32/FP16 inputs. Optional `seed` is supported. | +| `{"repeat":[1,2,3]}` | Repeat a fixed sequence to fill the tensor. | +| `{"abs_mod":{"bounds":[-128,127],"mod":5}}` | Generate uniform integers, then take `abs(value) % mod`. | +| `{"pad_columns":{"source":{"rand_uniform_int":[-8,8]},"valid_columns":12}}` | Generate the entire padded tensor, then zero columns after the valid prefix. | +| `{"row_sum":"weight"}` | Sum flattened source rows into int32; trailing destination rows are zero. | +| `{"column_sum":"weight"}` | Sum the source's leading dimensions into its last dimension. | +| `{"q4_group_sum":{"tensor":"weight","group_size":32}}` | Sum signed packed Q4 weights per group into group-major `[K/group_size,N]` data. | + +Random data is reproducible. By default, each tensor name determines its seed, +so dtype/storage variants use the same source values and different tensors use +independent streams. FP16 conversion happens during tensor preparation. Flat +`data` arrays and a tensor-level `fill` field are also accepted; use either `data` +or `fill`. The tensor's `storage: "constant"` selects CPU prepacking inputs; +`data.fill` only controls initialization. +Example inputs use random initialization; `fill` is used only for output tensors. +`row_sum` resolves named tensor dependencies, including forward references, and +rejects missing sources, cycles, noninteger sources, and int32 overflow. + +For `int8x4`, use buffer storage and a packed layout: `4w`, `4c`, `4w4c`, +`4h4w`, or `4c1w`. Supply logical int8 values; GPU prepacking packs them into +int32 storage. Reports contain tensor properties and timing data without tensor contents. + +### Operator arguments + +Operators use `args_sweep`, a list of ordered argument arrays; workgroup dimensions and shader bindings +are selected by their runtime implementations. Arguments support: + +| JSON | Graph value | +| --- | --- | +| `{"tensor":{"name":"input","sizes":[4,32],"dtype":"float32"}}` | Inline tensor definition. Use `storage: "constant"` for a CPU tensor reference. | +| `{"tensor":{"ref":"input"}}` | Reuse an inline tensor definition. | +| `{"int":32}` | Integer. | +| `{"float":0.25}` | Floating point scalar. | +| `{"bool":false}` | Boolean. | +| `{"int_list":[1,1]}` | Integer list. | +| `{"string":"none"}` | String. | +| `null` | None, e.g. absent bias. | + +Each sweep entry contains the full registry argument list, including outputs, in signature order. +Arguments are passed directly without reordering or appending output arguments. +Output tensors do not require an `access` annotation. The runner infers input, +output, and read/write roles from the dispatch nodes for reporting. + +For example, each linear argument set has the order input, weight, bias, output. +To reuse an input as the output, pass `{"tensor":{"ref":"input"}}` at the output position. +References can precede definitions; names must be unique within a set and can +be reused in other sets. References cannot cross sweep entries. References accept `ref` +and optional `access`; other tensor properties belong in the definition. +Operator configs reject separate `tensors`, `args`, `inputs`, and `outputs` fields. +Weight and bias tensors usually require `storage: "constant"`. An operator +case cannot specify `gwg_size`, `lwg_size`, `bindings`, push constants, +or specialization constants. + +One case can benchmark both precisions: + +```json +{ + "name": "relu", + "operator": "aten.relu.default", + "target_ms": 10, + "benchmark_runs": 5, + "sweep_variables": {"v_DTYPE": ["float32", "float16"]}, + "args_sweep": [ + [ + {"tensor": {"sizes": [8], "dtype": "v_DTYPE", "data": {"rand_uniform_float": [0, 1]}}}, + {"tensor": {"sizes": [8], "dtype": "v_DTYPE", "data": {"fill": 0}}} + ] + ] +} +``` + +Case-level timing settings apply to every entry. Results follow definition and +sweep order, with separate calibration and measurements for each argument set. +Variable names start with `v_`. An entire matching string value is replaced by +the selected JSON value, including numbers, arrays, or objects. Variables can +appear in arguments or case-level settings. Choices must be concrete values; +undefined placeholders are rejected. For each argument template, variables are +expanded in lexical name order, with the last variable varying fastest. + +The linear example sweeps 1, 16, and 128 input rows for 2048→2048 projections +and 2048→8192 feed-forward expansion. Each shape runs in FP32 and FP16 with +buffer and texture3d activations, for 24 cases total. +Weights use uniform `[-0.02, 0.02]` values. Its explicit `atol=0.05`, `rtol=0.01` +allows FP16 accumulation error in long reductions; tighter tolerances can be used +when studying numerical accuracy. + +The convolution examples use 3x3 kernels, stride/dilation 1, and padding 1. +Q8TA weights use the flattened `[C_out, C_in * K_h * K_w]` layout. Weight sums +are derived from the random weights; the unsigned-dot path requires these sums +even when the input zero point is zero. Q4GSW weights use `[N, K/2]` uniform random +packed bytes; each byte stores two signed Q4 values with zero point 8. +Random scales remain positive and have shape `[K/group_size, N]`. + +### Operator routes + +Use `route_sweep` to compare execution routes with the same canonical +operator arguments and initialized data: + +```json +{ + "operator": "aten.convolution.default", + "route_sweep": [ + "auto", + "direct", + "im2col", + {"name": "im2col", "options": {"intermediate_storage": "buffer"}}, + {"name": "im2col", "options": {"intermediate_storage": "texture2d"}}, + {"name": "im2col", "options": {"intermediate_storage": "texture3d"}} + ], + "sweep_variables": {"v_DTYPE": ["float32", "float16"]}, + "args_sweep": [""] +} +``` + +For one selection, use `"route": "direct"` or an object with `name` +and optional `options`. The two fields are mutually exclusive and are rejected +for raw shaders. Variables can appear in route names and option values. Expansion +runs argument templates first, variable combinations next, and route +selections in listed order last. The combined expansion is capped at 65536 cases +per definition. Each case gets a separate graph, validation, calibration, and +measurements; route selectors are never appended to the operator argument list. + +`auto` calls the ordinary production operator registry, including its performance +heuristics. Forced routes call production route entry points directly, subject +to correctness and device-capability checks. A forced route never silently falls +back to another route. Validation uses the same selection in a separate +graph and the operator's existing CPU reference. + +Registered convolution routes: + +| Operator | Routes and options | +| --- | --- | +| `aten.convolution.default` | `auto`, `direct`, `im2col`. Only `im2col` accepts `intermediate_storage`: `buffer`, `texture2d`, or `texture3d`; omission uses the production choice. | +| `et_vk.q8ta_conv2d.default` | `auto`, `direct` (general convolution), `im2col`. No route options. | + +The floating-point forced routes currently support forward 2D convolution with +batch 1 and channels-packed texture3d input/output. Direct grouped convolution +requires group channel boundaries aligned to four; im2col requires groups 1. +The Q8TA routes use packed int8 buffers and the production streaming feasibility +checks. Route eligibility is separate from automatic performance heuristics: +for example, forcing floating-point im2col can test dilated convolutions even +when automatic routing prefers direct. Constants use `storage: "constant"`. Floating-point weight and bias constants +must share a declared dtype, whose effective GPU dtype matches the input/output +dtype; devices that promote FP16 storage to FP32 keep that promotion supported. + +Unsupported forced cases report `status: "SKIP"` and `skip_reason`, with no timing +measurements. They do not fail the run and are skipped in every `error_mode`, +including `throw`. Unknown route names, malformed options, and unexpected runtime +failures remain errors. `--validation_mode=none` skips correctness validation; +it does not disable route eligibility checks. Since unsupported cases produce no +nodes from which to infer tensor roles, their table row may include all arguments. + +Explicit selections add requested/resolved `route` metadata to JSON. +An `auto` result identifies the actual production route when an inspector is +registered; otherwise `resolved` is null. The table adds a `Route` +column, e.g. `auto->im2col` or `im2col[buffer]`. Shader breakdowns still show the +actual shaders, and the operator timing covers the complete execute chain. +Prepacking remains outside benchmark timing. + +### Registering routes + +`framework/RouteRegistry.h` registers a builder function directly under +an operator/route pair, plus an optional inspector for normal automatic routing: + +```cpp +REGISTER_OPERATOR_ROUTES { + REGISTER_OPERATOR_ROUTE(aten.my_op.default, direct, build_my_direct); + REGISTER_ROUTE_INSPECTOR(aten.my_op.default, inspect_my_route); +} +``` + +Builders live under `operators//`. Their signature is +`Json(ComputeGraph&, const std::vector&, const Json& options)`. +They receive the canonical argument list and return resolved metadata +`{"name":"direct","options":{}}`. Reject unknown options with +`std::invalid_argument`; throw `UnsupportedRoute` for expected shape, +storage, or capability restrictions before adding nodes. Preflight entry points +that can fall back internally so a forced result always describes the requested +route. The registry reserves `auto` for ordinary operator dispatch. + +An inspector receives the graph, canonical arguments, and the first/end execute +node indices for that operator. It returns resolved metadata or null when the +route cannot be identified. Route names are scoped to each operator; duplicate +operator/route registrations and null functions are rejected. Multiple pairs +can register the same builder function. The route library uses +whole-archive linking to retain static registrations. + +### Workgroup pickers + +Shader workgroup settings accept vectors or picker names, independently: + +```json +{ + "gwg_size": "default_pick_gwg", + "lwg_size": "default_pick_lwg" +} +``` + +Use `shader_sweep` instead of `shader` to test variants with the same binding +interface and tensor representation: + +```json +{ + "shader_sweep": ["buffer_to_buffer_float", "buffer_to_buffer_int32"], + "gwg_size": "default_pick_gwg", + "lwg_size_sweep": {"x": [32, 128, 32]} +} +``` + +Shader names run in listed order, with every workgroup candidate tested before +moving to the next shader. Each combination gets a fresh graph and its own +`sweep_index`, calibration, validation, and measurements. The combined Cartesian +sweep is capped at 65536 cases. `shader_sweep` also combines with an explicit LWG +list; omitting a workgroup sweep tests each shader once with the selected LWG. +Each shader is checked against the bindings when executed; incompatible variants +report case errors using the configured `error_mode`. + +The combined example keeps FP32 buffer inputs unchanged. Its int32 copy variant +loads and stores the same 32-bit words, preserving the floating-point bits. +The shared `reference: "copy"` checks identical logical outputs for both shaders. + +The global picker is `default_pick_gwg`. It uses the first tensor binding whose +`access` is `write` or `read_write`: logical element count for buffers, texture +extents for textures. Local pickers are `default_pick_lwg`, `pick_required_lwg`, +`pick_xy_square_lwg`, `pick_xz_square_lwg`, and `pick_120shape_lwg`. +`pick_required_lwg` needs a global grid with a wrapping hint, such as the default +buffer grid. Unsupported names produce a case error. + +For a shader, `lwg_size` also accepts a nonempty list of vectors: + +```json +"lwg_size": [[8, 8, 1], [10, 10, 1]] +``` + +Alternatively, omit `lwg_size` and specify ranges: + +```json +"lwg_size_sweep": {"x": [4, 12, 4], "y": [4, 12, 4], "z": [1, 1, 1]} +``` + +Ranges include the stop when reached by the step. Omitted axes default to 1. +The Cartesian order is x outermost, then y, then z; each candidate has its own +`sweep_index`, graph, calibration, validation, and measurements. A definition +can expand to at most 65536 cases. Invalid device configurations report case +errors; they are not silently filtered. `lwg_size` and `lwg_size_sweep` are +mutually exclusive. + +Explicit local dimensions must be positive and fit device limits. For a +linear buffer grid, an explicit or picked local size also sets the wrapping +geometry. Explicit global vectors are rounded up to local workgroup multiples, +with no output tile scaling. Results report resolved `gwg_size` and `lwg_size` +vectors and, when used, `gwg_picker` and `lwg_picker` names. +Shaders using generic linear buffer indexing require `lwg_size.z=1`; larger z +dimensions need a shader that explicitly handles those invocations. + +### Raw shader bindings + +Binding forms: + +```json +[ + {"buffer": {"name": "output", "sizes": [8], "access": "write"}}, + {"buffer": {"name": "input", "sizes": [8], "dtype": "float32", "data": {"rand_uniform_float": [-1, 1]}}}, + {"meta_ubo": "output", "field": "meta"}, + {"uniform": [ + {"dtype": "int32", "data": [8, 0, 0, 0]}, + {"dtype": "float32", "data": [0.5, 0, 0, 0]} + ]} +] +``` + +Tensor bindings use a `buffer`, `texture2d`, or `texture3d` wrapper. Put tensor +properties and `access` inside it, with `sizes` defining a new tensor. Omit +`storage` because the wrapper determines it. `name` is optional and defaults to +`binding_`; names must be unique. `meta_ubo`, specialization layout +constants, and `outputs` refer to these names. References can precede definitions. +Use `{"buffer":{"ref":"input","access":"read"}}` to bind the same buffer again +without allocating another resource. A reference's wrapper type must match its +tensor. Shader configs do not accept a separate `tensors` object. + +`access` defaults to `read`; `read_write` is also supported. `meta_ubo` fields are +`meta` (default; `BufferMetadata` or `TextureMetadata`), `sizes`, `strides`, +`numel`, and `logical_limits`. Bindings must match the shader signature. + +UBOs and push constants use arrays of typed blocks. Blocks are concatenated +without implicit padding: include the shader's alignment/padding explicitly. +Supported block dtypes: `float32`, `int32`, `uint32`, `int8`, and `uint8`. +Total byte size must be a multiple of four. + +Specialization constants use `{"dtype":"int32","value":8}` or +`{"layout":"input"}` for a tensor's hashed layout. Supported scalar dtypes are +`int32` (default), `uint32`, `float32`, and `bool`. + +The binary executes shaders and operators already linked into the runtime. +It does not compile GLSL or load external SPIR-V. Raw shader cases require +the selected shader's tensor layouts and parameters. Operator cases invoke +the operator's prepack and dispatch routes. + +## Output and timing + +The default output is compact JSON. Use `--output_format=table` for a condensed +table: + +```sh +etvk_ops_bench --config=examples/shaders.json --output_format=table +``` + +Columns show shader/operator name, resolved GWG/LWG, inputs, and mean/p90 GPU +timestamp latency in microseconds. Tensor prefixes indicate storage: `b` buffer, +`t2` texture2d, `t3` texture3d, and `c` CPU constant. Non-constant tensors append +their layout name after the dtype, e.g. `b[1,1,1,8]f32.width` or +`b[60,64,40,51]i8x4.4w4c`. Constants omit the layout suffix because the operator +determines their prepacked GPU layout, e.g. `c[128,576]i8`. +Shader inputs include `read` and `read_write` tensor bindings in binding order; +operator inputs follow `args` order and include constant tensors and `read_write` +arguments. Scalar, list, string, and null operator arguments are included in +signature order. Raw shader uniform and push constant values are included. +Floating-point arguments use three significant figures, with scientific notation +when needed. This also applies to floating-point list elements in `args_str`. +Output-only +tensors and UBOs are omitted. Dtype suffixes reflect actual precision. Operator +rows use `-` for workgroup sizes and report latency for the full execute chain. +Unsupported timestamps show `N/A`; failed cases include their error message. +Operator labels append the zero-based sweep index, e.g. `aten.linear.default [1]`. +Separators follow the header and separate cases; operator shader breakdowns stay +in the same group. Names default to a 32-character limit and Inputs to 64; +long values end in `...`. Configure `--table_name_max_chars` and +`--table_inputs_max_chars`; `0` disables the corresponding limit. JSON strings +remain complete. + +Use `--output_format=json_full` for full reports. Shader fields follow this order: + +Workgroup dimension values and tensor size arrays are printed on one line. +Other report containers retain their indentation. + +1. `case_name`, `shader_name`. +2. `gwg_size`, `lwg_size`: `{"method": "explicit", "value": [8, 1, 1]}` or + the picker name as `method`, with resolved dimensions as `value`. +3. `bindings`: named tensor/UBO descriptions with physical `binding_index`, + type, sizes, dtype, layout, and access. Shared bindings include `ref`. +4. `specialization_constants`: actual values keyed by constant ID (3 onward; + local dimensions are already shown in `lwg_size`). Layout-derived constants + also identify their tensor. Push constants follow when configured. +5. Run info: device, target duration, repetition cap/count, warmup and benchmark + runs, and timestamp availability/precision. +6. `gpu_timestamp_shader_us`: statistics with `min`, `max`, `mean`, `p50`, + `p90`, and `p95`. +7. `gpu_timestamp_batch_us`, `wall_time_shader_us`, `wall_time_batch_us`: + arrays with one value per measured batch, in execution order. + +Tensor values are excluded. Dtype metadata reports the actual dtype; +`requested_dtype` appears when promotion occurs, such as FP16 to FP32 on +SwiftShader. + +### JSON summaries (`json`) + +The default `--output_format=json` keeps the case grouping and field names of +full JSON, with these substitutions: + +- Operator `args` becomes `args_str`, using the full `Inputs` notation, including + non-tensor arguments, storage, dtype, and non-constant layouts. Output-only + tensors are excluded. Table width limits do not truncate this string. +- `gpu_timestamp_operator_us`, `gpu_timestamp_batch_us`, and wall timing arrays + become objects with `min`, `median`, and `max`. Median is the middle sorted + sample, averaging the two middle samples for even counts. Empty timing arrays + become `null`. +- Each `*_observed_gpu_clock_freq_MHz` array becomes `{min, max}` over that + pass's start/end readings. Unavailable readings are ignored; a pass without + usable readings becomes `null`. Zero remains a valid observation. + +Shader timestamp statistics, shader breakdowns, raw shader bindings, validation, +and other metadata retain their full JSON representation. + +```json +{ + "operator_name": "aten.linear.default", + "args_str": "b[4,32]f32.width, c[16,32]f32, c[16]f32", + "operator_invocations_per_batch": 10, + "gpu_timestamp_operator_us": {"min": 2, "median": 4.5, "max": 9}, + "gpu_timestamp_batch_us": {"min": 20, "median": 45, "max": 90}, + "gpu_batch_observed_gpu_clock_freq_MHz": {"min": 800, "max": 950} +} +``` + +### Batches and calibration + +A batch is one recorded command buffer containing repeated executions of the +case. For a raw shader, each repetition is one shader dispatch; for an operator, +each repetition is the full execute chain, which may dispatch several shaders. +The repetition count is reported as `dispatches_per_batch` or +`operator_invocations_per_batch`, respectively. + +Calibration always warms and probes query-free batches, increasing repetitions +until the measured wall time reaches roughly `target_ms` or `max_dispatches`. +Calibration probes are excluded from reported latency samples. Commands are +recorded once per calibration step, then reused for warmup and measurement. +Every expanded sweep combination is calibrated and measured separately. +Operator graphs always invoke resize functions before benchmarking, including +shader and workgroup selection even when input shapes have not changed. + +### Measurement passes + +The benchmark runs the enabled passes below in the listed order, using separate +submissions. + +| Pass | Measurement boundary | Reported latency | +| --- | --- | --- | +| Wall time | Host monotonic clock from command-buffer submission through fence completion, without timestamp queries. Includes host submission overhead, queue/fence waiting, and GPU execution. | `wall_time_batch_us` contains batch totals. Dividing each total by the repetition count gives `wall_time_shader_us` or `wall_time_operator_us`. | +| Whole-batch GPU timestamps | One timestamp pair brackets the entire repeated dispatch chain, including GPU dependency barriers and inter-dispatch gaps. | `gpu_timestamp_batch_us` describes batch totals. Dividing each total by the repetition count gives `gpu_timestamp_shader_us` or `gpu_timestamp_operator_us`. These are two views of the same capture. | +| Per-dispatch GPU timestamps | Operator-only profiling pass with a timestamp pair around each executed shader, including its associated barriers. | `shader_breakdown[].gpu_timestamp_shader_us` describes each dispatch position, averaged across that profiling batch's operator repetitions. | + +Pass selection depends on case type and timestamp support: + +| Case | GPU timestamps available | Enabled passes | +| --- | --- | --- | +| Raw shader | Yes | Wall time, whole-batch GPU timestamps. | +| Operator | Yes | Wall time, whole-batch GPU timestamps, per-dispatch GPU timestamps. | +| Raw shader or operator | No | Wall time only. GPU batch/operator timings are `null` in `json` and empty arrays in `json_full`; raw shader statistics are `null`, and shader breakdowns are omitted. | + +Each enabled pass performs `warmup_runs` unmeasured replays (default 3), followed +by `benchmark_runs` measured replays (default 10; many examples specify 5). +The wall and whole-batch GPU passes use the calibrated repetition count. The +profiling pass may reduce that count to fit its timestamp query pool; its divisor +is `shader_profile_invocations_per_batch`. + +### Android GPU clock samples + +Android builds query the GPU clock immediately before submission and after fence +completion for every measured replay. Reads are outside the latency intervals; +warmups, calibration, and validation do not produce clock samples. JSON reports +include separate clock series for each pass. `json` summarizes each series as +`{min, max}`; `json_full` preserves arrays of start/end pairs in MHz: + +```json +{ + "gpu_batch_observed_gpu_clock_freq_MHz": [ + [903, 903], + [903, 950] + ], + "wall_time_batch_observed_gpu_clock_freq_MHz": [ + [903, 903], + [903, 903] + ] +} +``` + +| Clock field | Measurement pass | Associated timing | +| --- | --- | --- | +| `gpu_batch_observed_gpu_clock_freq_MHz` | Whole-batch GPU timestamps | `gpu_timestamp_batch_us` and operator batch averages in `gpu_timestamp_operator_us`. | +| `wall_time_batch_observed_gpu_clock_freq_MHz` | Wall time | `wall_time_batch_us` and `wall_time_shader_us`/`wall_time_operator_us`. | +| `shader_profile_batch_observed_gpu_clock_freq_MHz` | Operator shader profiling | Runs summarized by `shader_breakdown[].gpu_timestamp_shader_us`. | + +In `json_full`, each enabled pass contributes `benchmark_runs` pairs in run +order. For GPU-batch and wall-time arrays, entry `i` corresponds to timing sample +`i` from that pass. +The GPU-batch and wall-time clock arrays immediately follow their corresponding +`_batch_us` fields in JSON output. Array indexing applies to `json_full`; compact +`json` reports extrema over each pass's observations. +Shader-profile entry `i` describes profiling batch `i`, whose dispatch latencies +feed the shader-breakdown statistics. The GPU batch array is empty without GPU +timestamp support; the shader profiling array appears only when operator +profiling runs. + +An unreadable clock sample is `null`; a driver-reported zero is retained. These +are driver-reported snapshots, not average clocks during GPU execution. Host +builds omit clock fields. + +When clock observations are present, tables add `Min GPU MHz` and `Max GPU MHz` +after the latency columns. A main case row reports the minimum and maximum +across all start/end observations from its wall, GPU batch, and shader profiling +runs. Indented shader rows share the range from the operator's profiling runs. +Missing, nonnumeric, or nonfinite readings are ignored; rows without usable +observations show `N/A`. Driver-reported zero values remain valid observations. + +The sampler discovers readable KGSL/devfreq, Pixel Mali, and Samsung SGPU sysfs +sources and converts their driver-specific Hz, kHz, or MHz units to MHz. Clock +access is best effort and does not require root or change GPU clock settings. + +### Samples and percentiles + +Each measured batch contributes one sample to each reported latency series. +For whole-batch GPU timing, the sample is the batch timestamp interval. For +amortized shader/operator timing, it is that interval divided by the repetition +count. For a shader breakdown entry, it is the average interval for that dispatch +position across the profiling batch's repetitions. Repeated shader names at +different dispatch positions remain separate series. + +In `json_full`, `gpu_timestamp_batch_us`, `gpu_timestamp_operator_us`, and +wall-time fields contain arrays with one value per measured batch, in execution +order. Compact `json` summarizes these arrays as `min`, `median`, and `max`. +Operator GPU values are batch totals divided by +`operator_invocations_per_batch`; for example, in `json_full`: + +```json +{ + "operator_invocations_per_batch": 10, + "gpu_timestamp_operator_us": [3, 1, 2], + "gpu_timestamp_batch_us": [30, 10, 20] +} +``` + +`gpu_timestamp_shader_us`, both for raw shaders and shader breakdown entries, +reports `min`, `max`, `mean`, `p50`, `p90`, and `p95`. These shader statistics +currently use one batch-average sample per benchmark run. The table derives +mean/p90 from operator sample arrays and reads shader statistics for raw shader +and indented profiling rows. + +Percentiles use linear interpolation. For `R` sorted samples `x[0]` through +`x[R-1]` and percentile fraction `f`, the position is `f * (R - 1)`; the result +interpolates between the adjacent samples at that position. With five sorted +samples `x1 <= x2 <= x3 <= x4 <= x5`: + +| Statistic | Result | +| --- | --- | +| p50 | `x3` | +| p90 | `0.4 * x4 + 0.6 * x5` | +| p95 | `0.2 * x4 + 0.8 * x5` | + +With `benchmark_runs: 5`, even a batch containing hundreds of dispatches yields +only five batch-average samples. High percentiles then largely interpolate +toward the largest observation and provide weak estimates of the tail. +Increasing `benchmark_runs` to 50 or 100 improves sampling, but these remain +percentiles of batch-average latency; averaging within each batch smooths +variation among individual executions. + +### Measurement boundaries and shader breakdowns + +Wall timing samples cover query-free submission through fence completion. GPU +timestamps bracket the repeated dispatch chain, including dependency barriers. Input +preparation, uploads, pipeline creation, command recording, query retrieval, and +readback are outside these intervals. Unsupported timestamps produce `null` GPU +batch/operator timings in `json`, empty arrays in `json_full`, and `null` raw +shader statistics. Conversion preserves fractional `timestampPeriod` and handles +timestamp counter width. + +Operator reports use `operator_name`, `sweep_index`, and the selected inputs: +`args_str` in `json`, or ordered `args` with inline tensor metadata and access +roles in `json_full`. They also include +the runtime-selected `execute_nodes`. Per-operator fields are +`gpu_timestamp_operator_us` and `wall_time_operator_us`; they include the full +execute chain (e.g. im2col, transpose, GEMM), with weight prepacking outside timing. +`operator_invocations_per_batch` is the divisor for both wall and batch GPU +measurements. Nodes may self-gate and issue +no dispatch. Repeats reuse initialized storage, so in-place operators accumulate +updates. +When variables are present, reports include their selected `sweep_variables` +values and the original `args_sweep_index`. `sweep_index` identifies the expanded +run within that definition. + +When GPU timestamps are available, operator reports also include `shader_breakdown`. +Each entry describes an actual dispatch in execution order: `dispatch_index`, +runtime-selected `shader_name`, resolved GWG/LWG, and `gpu_timestamp_shader_us` +statistics. Repeated shader names remain separate entries; skipped nodes are +omitted. Table output shows these shaders indented below their operator. + +The breakdown uses a separate profiling pass with the calibrated invocation count, +capped by timestamp query-pool capacity. `shader_profile_invocations_per_batch` +reports that count. Warmup and benchmark run counts are the same as the aggregate +timing passes. Each shader sample averages that dispatch position over the batch +and includes its dependency barrier. +Per-dispatch timestamp overhead and inter-dispatch gaps mean the sum need not +match the separately measured operator latency. The passes can also run under +different GPU clock/cache conditions, and their repetition counts may differ. +For small workloads, per-dispatch timestamps can substantially increase measured +latency. Compare the batch GPU timestamps and query-free wall measurements +when evaluating that overhead. +Prepacking remains outside timing. +Breakdowns are omitted when timestamps are unavailable, including on this +repository's SwiftShader build. + +## Throughput + +Enable either or both metrics in a case definition: + +```json +{ + "show_memory_throughput": true, + "show_compute_throughput": true +} +``` + +The report's `throughput` object contains `memory_bytes` and/or `compute_flops` +per invocation. `gpu_timestamp` and `gpu_timestamp_batch` contain rate statistics; +`wall_time` and `wall_time_batch` contain one rate per measured batch. Memory +rates are available as `memory_gib_per_s` (GiB/s, 2^30 bytes/s) and +`memory_mib_per_s` (MiB/s, 2^20 bytes/s); compute rates use `compute_gflops` +(10^9 FLOPs/s). Throughput is computed from each sample before aggregation, so +its percentiles rank rates, with higher values indicating faster execution. +Batch rates account for all repeated invocations and match the invocation rates. + +Table output appends `GPU GiB/s` and/or `GPU GFLOPS`, showing mean GPU rates. +Shader breakdown entries include their own GPU throughput when a counter is +registered for the executed shader. No parent operator work is assigned to a +child shader. Unregistered counters produce null rates, table `N/A`, and a +diagnostic; a known zero-work counter produces zero. Unavailable GPU timestamps +produce null GPU rates while wall-time rates remain available. + +Memory counters estimate logical tensor traffic: each distinct read tensor once, +each write tensor once, and read/write tensors twice. Operator counters include +logical input/weight/bias/output tensors; child shader counters estimate their bound tensor footprints, including packed +weights and intermediates. Tiled/streaming dispatches may touch only part of those +footprints. The model excludes +UBOs, staging, tensor padding, prepacking, and additional/cache-dependent traffic. +GEMM/convolution counters use two operations per multiply-accumulate, plus +optional bias additions. Quantized operators report arithmetic-equivalent work, +excluding weight dequantization and activation requantization. Q4 linear +counters can use the model `3*M*N*K + M*N`; FPA counters use the two-operation +multiply-accumulate model. The registered counter determines the model. Custom +counters can implement another work model. + +Counters register beside operator implementations using +`framework/ThroughputRegistry.h`, with independent memory/compute associations: + +```cpp +REGISTER_THROUGHPUT_FUNCTIONS { + REGISTER_OPERATOR_MEMORY_TRAFFIC(aten.my_op.default, my_byte_count); + REGISTER_OPERATOR_COMPUTE_FLOPS(aten.my_op.default, my_flop_count); + REGISTER_SHADER_MEMORY_TRAFFIC(my_shader, my_byte_count); + REGISTER_SHADER_COMPUTE_FLOPS(my_shader, my_flop_count); +} +``` + +Shader counters can register a case-sensitive prefix. For example: + +```cpp +REGISTER_SHADER_MEMORY_TRAFFIC(conv2d, throughput::tensor_traffic); +``` + +This covers every shader whose name starts with `conv2d`, including dtype, +storage, and tile variants. Lookup checks exact registrations first, then uses +the longest matching prefix. `REGISTER_SHADER_MEMORY` and +`REGISTER_SHADER_COMPUTE` register exact names for individual overrides; +operator associations remain exact. Memory and compute prefixes are independent, +and empty/duplicate prefix registrations and null callables are rejected. Each +registration stores the function pointer directly; one callable can serve +multiple operators, shaders, or prefixes. + +Prefixes group variants with the same work model. For example, `linear_vec_bias` +selects the bias-aware compute counter over `linear_vec`, and +`q4gsw_linear_gemm__tin__` selects its binding-aware memory counter over +`q4gsw_linear_gemm__`. Prefix association happens in the benchmark's throughput +registry when the actual dispatched shader name is reported. + +A counter returns a finite, nonnegative `double` from a `const Json&` containing +resolved case metadata. Operators receive `operator_name` and ordered `args` +with tensor descriptions and scalar values. Shaders receive `shader_name`, +resolved GWG/LWG, and named `bindings` with sizes, actual dtype, storage, and +access; shader breakdown counters receive metadata from that dispatch's tensors. +Registration is performed before benchmarking. Duplicate keys in each +association category are rejected. Functions are passed directly to registration +macros and receive resolved case metadata; no separate function-name registry +is required. + +## Correctness validation + +Validation runs before timing by default. It constructs a separate graph with +input/output staging and executes the target shader or operator once. Benchmark +graphs have no execute-stage input/output transfers. CPU reference work and +readback are outside all benchmark intervals. +Tensor data is generated once per expanded case and shared by the validation +and benchmark graphs. Each graph uploads its inputs during setup; benchmarking +does not regenerate input data. When validation is skipped, data is still +generated once to initialize the benchmark graph. +Validation GPU execution latency is not reported. The separate +`validation.reference_wall_time_us` field times CPU reference computation, +including input decoding and precision conversion; it excludes output readback +and comparison and is not a shader/operator benchmark latency. + +Select correctness checks with `--validation_mode`: + +| Value | Behavior | +| --- | --- | +| `all` (default) | Validate every expanded case when supported. | +| `first` | Validate only the first expanded case in the entire input file, for smoke testing. | +| `none` | Skip correctness validation and CPU reference computation. | + +`first` selects the first case after argument, shader, workgroup, and route sweep +expansion. If that case is unsupported, it reports `SKIP`; later cases are not +validated. All cases still run their benchmark passes when supported. Results +include a JSON `validation` object only for selected checks; the table includes +a `Validation` column when any result has validation information. + +`--error_mode=report` (default) reports `PASS` or `FAIL` for selected validation +checks and continues benchmarking. Any validation or execution failure produces +exit status 1. +`--error_mode=throw` stops at the first validation or execution failure with +exit status 1. `--error_mode=skip` disables validation, overriding +`--validation_mode`; prefer `--validation_mode=none`. +Malformed configs and flags produce exit status 2. + +Example reference names include `copy`, `add`, `linear`, `conv2d`, `q8ta_conv2d`, +and `q4gsw_linear`. They are inferred for the example shaders/operators; +`reference` can select one explicitly for a compatible custom shader. Unknown +references report validation failure. New CPU references belong under +`operators//` and register beside their implementations. + +### Registering references + +`framework/ReferenceRegistry.h` provides two registration levels: + +```cpp +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_FN(my_reference, my_reference_impl); + REGISTER_OPERATOR_REFERENCE(aten.my_op.default, my_reference); + REGISTER_SHADER_REFERENCE(my_shader, my_reference); +} +``` + +The first macro registers a callable under a unique name. The remaining macros +associate an operator or shader name with that reference name. Multiple names +may share a reference; operator and shader associations have separate namespaces. +Registration blocks run at static initialization, following the ET-VK operator +registration pattern. Associations may be declared before the function's +registration in another translation unit; lookup verifies that the function +exists. Duplicate names or associations are errors. +Each associated function must understand that case's operator arguments or +shader bindings. Registration is performed before benchmarking starts. + +A reference callable has the signature +`reference::Tensor(const Json&, const reference::TensorMap&, size_t threads)`. +Tensor inputs have already been decoded and rounded to actual GPU precision. +The returned tensor is shape-checked and rounded to output precision by the +framework. A JSON `reference` override resolves through the same function registry. +The reference library uses whole-archive linking to retain registration +initializers from every reference source file. + +Linear references use a tiled, vectorizable CPU GEMM with up to eight threads. +Convolution references parallelize output work. Validation does not skip cases +based on tensor size. Inputs are rounded to their actual GPU precision, +including FP16 constants, before computing reference results. + +The supported comparison is `is_close`: every element must satisfy +`abs(actual - expected) <= atol + rtol * abs(expected)`. Nonfinite values, +shape/count mismatches, and missing/unexpected outputs fail validation. +Defaults are FP32 `atol=rtol=1e-4`, FP16 `atol=rtol=1e-2`, and exact integer +comparison. `validation_method` overrides either tolerance; tolerances must be +finite and nonnegative. FP16 accumulation may require larger tolerances for +long reductions. Reports include per-output mismatch counts and error maxima, +and `reference_wall_time_us`, without tensor values. + +## Operator source organization + +Operator code follows family and quantization-scheme folders, using the production +runtime organization as a guide: + +```text +operators/ + convolution/ + Conv2dReference.{h,cpp} + Conv2dRoutes.{h,cpp} + Conv2dThroughput.cpp + cases/ + q8ta/ + Q8taConv2dReference.{h,cpp} + Q8taConv2dRoutes.{h,cpp} + Q8taConv2dThroughput.cpp + cases/ + gemm/ + linear/ + LinearReference.{h,cpp} + LinearThroughput.cpp + cases/ + qw/q4gsw/ + Q4gswLinearReference.{h,cpp} + Q4gswLinearThroughput.cpp + cases/ +``` + +Scheme-specific arithmetic, argument validation, and registrations live with the +scheme. Shared convolution geometry/traversal and route guards live in the +convolution parent; shared GEMM projection and work-count helpers live in the +GEMM parent. Add other quantized implementations under their family’s `q8ta/` or +`qw//` folder. + +## Framework components + +Shared headers and implementations live in `framework/`: `Config` validates JSON, +`Sweep` expands templates, Cartesian variables, and route selections, `DataInitializer` generates +typed input data, and `TensorUtils` handles tensor setup and dtype metadata. +`Timestamps` owns GPU queries, `Timing` handles calibration and measurement, +`WorkgroupUtils` resolves workgroup vectors and named pickers, and `Report` +formats ordered case metadata and timing statistics. +`RouteRegistry` resolves named operator routes and optional automatic +route inspectors. `ShaderBenchmark` and `OperatorBenchmark` assemble and run their respective cases. +Component tests live under `framework/test/`, including registry, reference, +route, reporting, and validation tests. +`etvk_ops_bench.cpp` handles the CLI and dispatches configs to the runners. + +## Framework tests + +Tests under `framework/test/` cover schemas and sweep expansion, data +initialization, reference and throughput registrations, route lookup, validation, +and report formatting. Registry and reporting tests run without GPU execution. +Operator route tests exercise shader chains, route options, automatic route +inspection, and unsupported-case guards; these require a Vulkan implementation +such as SwiftShader or a hardware GPU. + +## Case collections + +JSON collections live under `operators//.../cases/*.json`, alongside the +references and registrations for that operator family or quantization scheme. +Pass any collection to `--config`. Each definition and its expanded sweeps run +in file order. + +Operator cases call registered production operators directly. Q8TA cases use +packed `int8x4` activations and outputs; weight-only quantized operators use their +native weight, scale, and activation arguments. Quantize and dequantize are +separate operator cases. Quantized comparisons use quantized units with zero +relative tolerance. Use `route` or `route_sweep` to select execution routes. + +## Schema audits and validation-only runs + +`--validate_specs` parses all expanded cases and checks operator, route, reference, +and custom-check registrations without allocating case tensors or executing +operators. It emits a JSON audit summary and exits nonzero on errors. The binary +attempts texture creation without screening dimensions against driver-reported limits, +which can understate usable dimensions. Ordinary execution still validates +outputs according to `--validation_mode`. Use +`--validation_only` to execute GPU correctness checks without calibration or +benchmark passes; it requires `--validation_mode=all` or `first` and +`--error_mode=report` or `throw`. With `first`, remaining cases report `SKIP` +without allocating tensors or executing benchmark passes. + +Device support is determined while constructing and executing each case. The +binary catches ET-VK `ShaderNotSupportedError` and unsupported-route exceptions +and reports `SKIP` with a reason, then continues to the next case. This applies +to validation and benchmarking, including `--error_mode=throw`; that option +still stops on validation failures and other errors. JSON `requirements` gates +are not supported. `--validate_specs` checks schemas and registrations, rather +than executing cases to establish device support. + +Validation-only `check` cases return PASS, FAIL, or SKIP with diagnostic details; +they have no latency measurements. Unselected checks report `SKIP`, including +all checks under `--validation_mode=none` or `--error_mode=skip`. + +To preserve a case's declared validation scope when an operator also updates +auxiliary state, select outputs inside the comparison settings: + +```json +"validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001, + "outputs": ["output"] +} +``` + +The default checks all inferred writable tensors. An explicit list must be +nonempty, unique, and name writable GPU tensors from the case. Each selected +tensor must be produced by the reference and the GPU. Unselected auxiliary +tensors remain in the operator arguments and execute normally. diff --git a/backends/vulkan/test/ops/benchmark_v2/etvk_ops_bench.cpp b/backends/vulkan/test/ops/benchmark_v2/etvk_ops_bench.cpp new file mode 100644 index 00000000000..4d9f8ed8399 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/etvk_ops_bench.cpp @@ -0,0 +1,287 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +DEFINE_string( + config, + "", + "JSON file containing shader/operator configs, singly or as an array."); +DEFINE_bool( + validate_specs, + false, + "Validate case schemas and registry bindings without executing operators."); +DEFINE_bool( + validation_only, + false, + "Execute correctness checks without calibration or latency measurements."); +DEFINE_string( + validation_mode, + "all", + "Cases to validate: all, first, or none."); +DEFINE_string( + output_format, + "json", + "Output format: json (default), json_full, or table."); +DEFINE_string( + error_mode, + "report", + "Error handling: report or throw; skip disables validation for compatibility."); +DEFINE_uint32( + table_name_max_chars, + 32, + "Table name width limit; 0 means unlimited."); +DEFINE_uint32( + table_inputs_max_chars, + 64, + "Table Inputs width limit; 0 means unlimited."); + +namespace { + +vkcompute::shader_profiling::Report skip_report( + const vkcompute::shader_profiling::Json& spec, + const std::string& reason, + bool validation_enabled) { + using namespace vkcompute::shader_profiling; + Json accesses = Json::object(); + for (const auto& tensor : spec.at("tensors").items()) { + accesses[tensor.key()] = "unknown"; + } + Report result = spec.contains("operator") + ? operator_report(spec, Json::object(), accesses) + : shader_report(spec, Json(), Json(), Json::object(), Report::array()); + result["status"] = "SKIP"; + result["skip_reason"] = reason; + if (validation_enabled) { + result["validation"] = {{"status", "SKIP"}}; + } + Report throughput = error_report(spec, reason); + if (throughput.contains("throughput")) { + result["throughput"] = std::move(throughput["throughput"]); + } + return result; +} + +} // namespace + +int main(int argc, char* argv[]) { + gflags::SetUsageMessage( + "Replay registered ET-VK shaders or operators from --config=."); + gflags::ParseCommandLineFlags(&argc, &argv, true); + if (FLAGS_config.empty() || argc != 1) { + std::cerr << "Usage: etvk_ops_bench --config= " + "[--output_format=json|json_full|table] " + "[--validation_mode=all|first|none]\n"; + return 2; + } + using namespace vkcompute::shader_profiling; + try { + if (FLAGS_output_format != "json" && FLAGS_output_format != "table" && + FLAGS_output_format != "json_full") { + throw std::invalid_argument( + "output_format must be json, json_full, or table"); + } + if (FLAGS_validation_mode != "all" && FLAGS_validation_mode != "first" && + FLAGS_validation_mode != "none") { + throw std::invalid_argument( + "validation_mode must be all, first, or none"); + } + if (FLAGS_error_mode != "report" && FLAGS_error_mode != "throw" && + FLAGS_error_mode != "skip") { + throw std::invalid_argument("error_mode must be report, throw, or skip"); + } + if (FLAGS_validation_only && + (FLAGS_validation_mode == "none" || FLAGS_error_mode == "skip")) { + throw std::invalid_argument( + "validation_only requires validation_mode=all or first and " + "error_mode=report or throw"); + } + std::ifstream file(FLAGS_config); + if (!file) { + throw std::invalid_argument("Cannot open config: " + FLAGS_config); + } + Json document; + file >> document; + const auto configs = parse_configs(document); + if (FLAGS_validate_specs) { + Report errors = Report::array(); + for (const auto& config : configs) { + try { + if (config.contains("check")) { + case_check_registry().get_check(config.at("check")); + } else if (config.contains("operator")) { + const std::string op = config.at("operator"); + VK_GET_OP_FN(op); + if (config.contains("route") && + config.at("route").at("name") != "auto") { + route_registry().operator_route( + op, config.at("route").at("name")); + } + const auto reference = reference_name(config); + if (!reference_registry().has_reference(reference)) + throw std::invalid_argument( + "Unknown reference function: " + reference); + } + } catch (const std::exception& error) { + errors.push_back( + {{"case_name", config.value("name", "")}, + {"error", error.what()}}); + } + } + const Report summary{ + {"status", errors.empty() ? "PASS" : "FAIL"}, + {"case_count", configs.size()}, + {"errors", errors}}; + std::cout << summary.dump(2) << std::endl; + return errors.empty() ? 0 : 1; + } + Report results = Report::array(); + bool failed = false; + for (size_t case_index = 0; case_index < configs.size(); ++case_index) { + const Json& config = configs.at(case_index); + const bool validation_enabled = FLAGS_error_mode != "skip" && + (FLAGS_validation_mode == "all" || + (FLAGS_validation_mode == "first" && case_index == 0)); + if (config.contains("check")) { + Report result; + if (!validation_enabled) { + result = { + {"case_name", + config.value("name", config.at("check").get())}, + {"check_name", config.at("check")}, + {"status", "SKIP"}, + {"skip_reason", + FLAGS_error_mode == "skip" + ? "Validation-only case; error_mode=skip" + : "Validation-only case; validation_mode=" + + FLAGS_validation_mode}}; + } else { + result = run_case_check(config); + if (result.at("validation").at("status") == "FAIL") { + failed = true; + if (FLAGS_error_mode == "throw") { + std::cerr << result.at("validation").dump() << std::endl; + return 1; + } + } + } + results.push_back(std::move(result)); + continue; + } + if (FLAGS_validation_only && !validation_enabled) { + results.push_back(skip_report( + config, + "Validation not selected by validation_mode=" + + FLAGS_validation_mode, + true)); + continue; + } + Report validation; + Report validation_result; + TensorData host_data; + try { + host_data = prepare_tensor_data(config); + } catch (const std::exception& e) { + failed = true; + if (FLAGS_error_mode == "throw") { + std::cerr << e.what() << std::endl; + return 1; + } + Report result = error_report(config, e.what()); + if (validation_enabled) { + result["validation"] = {{"status", "FAIL"}, {"error", e.what()}}; + } + results.push_back(std::move(result)); + continue; + } + if (validation_enabled) { + try { + validation_result = config.contains("operator") + ? benchmark_operator(config, host_data, true) + : benchmark_shader(config, host_data, true); + validation = validation_result.at("validation"); + } catch (const UnsupportedRoute& e) { + results.push_back(skip_report(config, e.what(), validation_enabled)); + continue; + } catch (const vkcompute::vkapi::ShaderNotSupportedError& e) { + results.push_back(skip_report(config, e.what(), validation_enabled)); + continue; + } catch (const std::exception& e) { + validation = {{"status", "FAIL"}, {"error", e.what()}}; + } + if (validation.at("status") == "FAIL") { + failed = true; + if (FLAGS_error_mode == "throw") { + std::cerr << "Validation failed for " + << config.value( + "name", + config.value( + "operator", config.value("shader", ""))) + << ": " << validation.dump() << std::endl; + return 1; + } + } + } + if (FLAGS_validation_only) { + if (validation_result.is_null()) { + validation_result = error_report(config, validation.dump()); + } + validation_result["validation"] = std::move(validation); + results.push_back(std::move(validation_result)); + continue; + } + Report result; + try { + result = config.contains("operator") + ? benchmark_operator(config, host_data) + : benchmark_shader(config, host_data); + } catch (const UnsupportedRoute& e) { + results.push_back(skip_report(config, e.what(), validation_enabled)); + continue; + } catch (const vkcompute::vkapi::ShaderNotSupportedError& e) { + results.push_back(skip_report(config, e.what(), validation_enabled)); + continue; + } catch (const std::exception& e) { + failed = true; + if (FLAGS_error_mode == "throw") { + std::cerr << e.what() << std::endl; + return 1; + } + result = error_report(config, e.what()); + } + if (!validation.is_null()) { + result["validation"] = std::move(validation); + } + results.push_back(std::move(result)); + } + const TableOptions table_options = { + FLAGS_table_name_max_chars, FLAGS_table_inputs_max_chars}; + std::cout << (FLAGS_output_format == "table" + ? format_table(results, table_options) + : FLAGS_output_format == "json" + ? format_condensed_report(results) + : format_report(results)) + << std::endl; + return failed ? 1 : 0; + } catch (const std::exception& e) { + std::cerr << e.what() << std::endl; + return 2; + } +} diff --git a/backends/vulkan/test/ops/benchmark_v2/examples/buffer_shader.json b/backends/vulkan/test/ops/benchmark_v2/examples/buffer_shader.json new file mode 100644 index 00000000000..2e5df2180eb --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/examples/buffer_shader.json @@ -0,0 +1,29 @@ +{ + "shader": "buffer_to_buffer_float", + "gwg_size": "default_pick_gwg", + "lwg_size": "default_pick_lwg", + "target_ms": 10, + "benchmark_runs": 5, + "bindings": [ + { + "buffer": { + "name": "output", + "sizes": [8], + "access": "write", + "data": {"fill": 0} + } + }, + { + "buffer": { + "name": "input", + "sizes": [8], + "data": {"rand_uniform_float": [-1, 1]} + } + }, + { + "meta_ubo": "output", + "field": "numel" + } + ], + "outputs": ["output"] +} diff --git a/backends/vulkan/test/ops/benchmark_v2/examples/conv2d.json b/backends/vulkan/test/ops/benchmark_v2/examples/conv2d.json new file mode 100644 index 00000000000..f05511f47ee --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/examples/conv2d.json @@ -0,0 +1,62 @@ +{ + "name": "conv2d", + "operator": "aten.convolution.default", + "target_ms": 10, + "benchmark_runs": 5, + "show_memory_throughput": true, + "show_compute_throughput": true, + "max_dispatches": 512, + "sweep_variables": { + "v_DTYPE": ["float32", "float16"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4, 4, 4], + "dtype": "v_DTYPE", + "storage": "texture3d", + "layout": "channels", + "data": {"rand_uniform_float": [-1, 1]} + } + }, + { + "tensor": { + "name": "weight", + "sizes": [8, 4, 3, 3], + "dtype": "v_DTYPE", + "storage": "constant", + "layout": "width", + "data": {"rand_uniform_float": [-1, 1]} + } + }, + { + "tensor": { + "name": "bias", + "sizes": [8], + "dtype": "v_DTYPE", + "storage": "constant", + "layout": "width", + "data": {"rand_uniform_float": [-1, 1]} + } + }, + {"int_list": [1, 1]}, + {"int_list": [1, 1]}, + {"int_list": [1, 1]}, + {"bool": false}, + {"int_list": [0, 0]}, + {"int": 1}, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 4, 4], + "dtype": "v_DTYPE", + "storage": "texture3d", + "layout": "channels", + "data": {"fill": 0} + } + } + ] + ] +} diff --git a/backends/vulkan/test/ops/benchmark_v2/examples/conv2d_routes.json b/backends/vulkan/test/ops/benchmark_v2/examples/conv2d_routes.json new file mode 100644 index 00000000000..8ea0b12ff1b --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/examples/conv2d_routes.json @@ -0,0 +1,110 @@ +{ + "name": "conv2d routes", + "operator": "aten.convolution.default", + "target_ms": 10, + "benchmark_runs": 5, + "show_memory_throughput": true, + "show_compute_throughput": true, + "max_dispatches": 512, + "route_sweep": [ + "auto", + "direct", + "im2col", + { + "name": "im2col", + "options": { + "intermediate_storage": "buffer" + } + }, + { + "name": "im2col", + "options": { + "intermediate_storage": "texture2d" + } + }, + { + "name": "im2col", + "options": { + "intermediate_storage": "texture3d" + } + } + ], + "validation_method": { + "function": "is_close", + "atol": 0.05, + "rtol": 0.01 + }, + "sweep_variables": { + "v_DTYPE": ["float32", "float16"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 32, 32], + "dtype": "v_DTYPE", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 32, 3, 3], + "dtype": "v_DTYPE", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-0.05, 0.05] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "v_DTYPE", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 32, 32], + "dtype": "v_DTYPE", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] +} diff --git a/backends/vulkan/test/ops/benchmark_v2/examples/linear.json b/backends/vulkan/test/ops/benchmark_v2/examples/linear.json new file mode 100644 index 00000000000..abb5f2217e7 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/examples/linear.json @@ -0,0 +1,101 @@ +{ + "name": "linear", + "operator": "aten.linear.default", + "target_ms": 10, + "benchmark_runs": 5, + "show_memory_throughput": true, + "show_compute_throughput": true, + "max_dispatches": 512, + "validation_method": {"function": "is_close", "atol": 0.05, "rtol": 0.01}, + "sweep_variables": { + "v_DTYPE": ["float32", "float16"], + "v_M": [1, 16, 128], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": ["v_M", 2048], + "dtype": "v_DTYPE", + "storage": "v_STORAGE", + "layout": "width", + "data": {"rand_uniform_float": [-1, 1]} + } + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 2048], + "dtype": "v_DTYPE", + "storage": "constant", + "layout": "width", + "data": {"rand_uniform_float": [-0.02, 0.02]} + } + }, + { + "tensor": { + "name": "bias", + "sizes": [2048], + "dtype": "v_DTYPE", + "storage": "constant", + "layout": "width", + "data": {"rand_uniform_float": [-1, 1]} + } + }, + { + "tensor": { + "name": "output", + "sizes": ["v_M", 2048], + "dtype": "v_DTYPE", + "storage": "v_STORAGE", + "layout": "width", + "data": {"fill": 0} + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": ["v_M", 2048], + "dtype": "v_DTYPE", + "storage": "v_STORAGE", + "layout": "width", + "data": {"rand_uniform_float": [-1, 1]} + } + }, + { + "tensor": { + "name": "weight", + "sizes": [8192, 2048], + "dtype": "v_DTYPE", + "storage": "constant", + "layout": "width", + "data": {"rand_uniform_float": [-0.02, 0.02]} + } + }, + { + "tensor": { + "name": "bias", + "sizes": [8192], + "dtype": "v_DTYPE", + "storage": "constant", + "layout": "width", + "data": {"rand_uniform_float": [-1, 1]} + } + }, + { + "tensor": { + "name": "output", + "sizes": ["v_M", 8192], + "dtype": "v_DTYPE", + "storage": "v_STORAGE", + "layout": "width", + "data": {"fill": 0} + } + } + ] + ] +} diff --git a/backends/vulkan/test/ops/benchmark_v2/examples/q4gsw_linear.json b/backends/vulkan/test/ops/benchmark_v2/examples/q4gsw_linear.json new file mode 100644 index 00000000000..99928a409ce --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/examples/q4gsw_linear.json @@ -0,0 +1,67 @@ +{ + "name": "q4gsw linear", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 10, + "benchmark_runs": 5, + "show_memory_throughput": true, + "show_compute_throughput": true, + "max_dispatches": 512, + "sweep_variables": { + "v_DTYPE": ["float32", "float16"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 32], + "dtype": "v_DTYPE", + "storage": "buffer", + "layout": "width", + "data": {"rand_uniform_float": [-1, 1]} + } + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 16], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": {"rand_uniform_int": [0, 255]} + } + }, + { + "tensor": { + "name": "scales", + "sizes": [1, 16], + "dtype": "v_DTYPE", + "storage": "constant", + "layout": "width", + "data": {"rand_uniform_float": [0.01, 0.1]} + } + }, + {"int": 32}, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "v_DTYPE", + "storage": "constant", + "layout": "width", + "data": {"rand_uniform_float": [-1, 1]} + } + }, + { + "tensor": { + "name": "output", + "sizes": [4, 16], + "dtype": "v_DTYPE", + "storage": "buffer", + "layout": "width", + "data": {"fill": 0} + } + } + ] + ] +} diff --git a/backends/vulkan/test/ops/benchmark_v2/examples/q8ta_conv2d.json b/backends/vulkan/test/ops/benchmark_v2/examples/q8ta_conv2d.json new file mode 100644 index 00000000000..9670ec2ec62 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/examples/q8ta_conv2d.json @@ -0,0 +1,83 @@ +{ + "name": "q8ta conv2d", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 10, + "benchmark_runs": 5, + "show_memory_throughput": true, + "show_compute_throughput": true, + "max_dispatches": 512, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4, 4, 4], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": {"rand_uniform_int": [-8, 8]} + } + }, + {"float": 0.25}, + {"int": 0}, + { + "tensor": { + "name": "weight", + "sizes": [8, 36], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": {"rand_uniform_int": [-8, 8]} + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [8], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": {"row_sum": "weight"} + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [8], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": {"rand_uniform_float": [0.01, 0.1]} + } + }, + {"float": 0.25}, + {"int": 0}, + { + "tensor": { + "name": "bias", + "sizes": [8], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": {"rand_uniform_float": [-1, 1]} + } + }, + {"int_list": [3, 3]}, + {"int_list": [1, 1]}, + {"int_list": [1, 1]}, + {"int_list": [1, 1]}, + {"int": 1}, + {"string": "none"}, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 4, 4], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": {"fill": 0} + } + } + ] + ] +} diff --git a/backends/vulkan/test/ops/benchmark_v2/examples/q8ta_conv2d_routes.json b/backends/vulkan/test/ops/benchmark_v2/examples/q8ta_conv2d_routes.json new file mode 100644 index 00000000000..81f7374097b --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/examples/q8ta_conv2d_routes.json @@ -0,0 +1,116 @@ +{ + "name": "q8ta conv2d routes", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 10, + "benchmark_runs": 5, + "show_memory_throughput": true, + "show_compute_throughput": true, + "max_dispatches": 512, + "route_sweep": ["auto", "direct", "im2col"], + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-8, 8] + } + } + }, + { + "float": 0.25 + }, + { + "int": 0 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-8, 8] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.01, 0.1] + } + } + }, + { + "float": 0.25 + }, + { + "int": 0 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] +} diff --git a/backends/vulkan/test/ops/benchmark_v2/examples/shader_workgroup_sweep.json b/backends/vulkan/test/ops/benchmark_v2/examples/shader_workgroup_sweep.json new file mode 100644 index 00000000000..db9f3816498 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/examples/shader_workgroup_sweep.json @@ -0,0 +1,33 @@ +{ + "name": "buffer copy shader and workgroup sweep", + "shader_sweep": ["buffer_to_buffer_float", "buffer_to_buffer_int32"], + "gwg_size": "default_pick_gwg", + "lwg_size_sweep": {"x": [32, 128, 32]}, + "target_ms": 10, + "benchmark_runs": 5, + "show_memory_throughput": true, + "show_compute_throughput": false, + "max_dispatches": 512, + "reference": "copy", + "bindings": [ + { + "buffer": { + "name": "output", + "sizes": [65536], + "dtype": "float32", + "access": "write", + "data": {"fill": 0} + } + }, + { + "buffer": { + "name": "input", + "sizes": [65536], + "dtype": "float32", + "data": {"rand_uniform_float": [-1, 1]} + } + }, + {"meta_ubo": "output", "field": "numel"} + ], + "outputs": ["output"] +} diff --git a/backends/vulkan/test/ops/benchmark_v2/examples/shaders.json b/backends/vulkan/test/ops/benchmark_v2/examples/shaders.json new file mode 100644 index 00000000000..a9551441f4b --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/examples/shaders.json @@ -0,0 +1,79 @@ +[ + { + "name": "buffer copy", + "shader": "buffer_to_buffer_float", + "gwg_size": [8, 1, 1], + "lwg_size": [8, 1, 1], + "target_ms": 10, + "benchmark_runs": 5, + "bindings": [ + { + "buffer": { + "name": "output", + "sizes": [8], + "access": "write", + "data": {"fill": 0} + } + }, + { + "buffer": { + "name": "input", + "sizes": [8], + "data": {"rand_uniform_float": [-1, 1]} + } + }, + { + "meta_ubo": "output", + "field": "numel" + } + ], + "outputs": ["output"] + }, + { + "name": "texture addition", + "shader": "binary_add_texture3d_float", + "gwg_size": "default_pick_gwg", + "lwg_size": "default_pick_lwg", + "target_ms": 10, + "benchmark_runs": 5, + "bindings": [ + { + "texture3d": { + "name": "out", + "sizes": [1, 1, 1, 8], + "access": "write", + "data": {"fill": 0} + } + }, + { + "texture3d": { + "name": "a", + "sizes": [1, 1, 1, 8], + "data": {"rand_uniform_float": [-1, 1]} + } + }, + { + "texture3d": { + "name": "b", + "sizes": [1, 1, 1, 8], + "data": {"rand_uniform_float": [-1, 1]} + } + }, + {"meta_ubo": "out"}, + {"meta_ubo": "a"}, + {"meta_ubo": "b"} + ], + "specialization_constants": [ + {"layout": "out"}, + {"layout": "a"}, + {"layout": "b"} + ], + "push_constants": [ + { + "dtype": "float32", + "data": [2] + } + ], + "outputs": ["out"] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/examples/texture_shader.json b/backends/vulkan/test/ops/benchmark_v2/examples/texture_shader.json new file mode 100644 index 00000000000..6c83fef1c06 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/examples/texture_shader.json @@ -0,0 +1,47 @@ +{ + "name": "texture addition", + "shader": "binary_add_texture3d_float", + "gwg_size": [2, 1, 1], + "lwg_size": [2, 1, 1], + "target_ms": 10, + "benchmark_runs": 5, + "bindings": [ + { + "texture3d": { + "name": "out", + "sizes": [1, 1, 1, 8], + "access": "write", + "data": {"fill": 0} + } + }, + { + "texture3d": { + "name": "a", + "sizes": [1, 1, 1, 8], + "data": {"rand_uniform_float": [-1, 1]} + } + }, + { + "texture3d": { + "name": "b", + "sizes": [1, 1, 1, 8], + "data": {"rand_uniform_float": [-1, 1]} + } + }, + {"meta_ubo": "out"}, + {"meta_ubo": "a"}, + {"meta_ubo": "b"} + ], + "specialization_constants": [ + {"layout": "out"}, + {"layout": "a"}, + {"layout": "b"} + ], + "push_constants": [ + { + "dtype": "float32", + "data": [2] + } + ], + "outputs": ["out"] +} diff --git a/backends/vulkan/test/ops/benchmark_v2/examples/texture_shader_lwg_range.json b/backends/vulkan/test/ops/benchmark_v2/examples/texture_shader_lwg_range.json new file mode 100644 index 00000000000..818161f0c7f --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/examples/texture_shader_lwg_range.json @@ -0,0 +1,75 @@ +{ + "name": "texture addition LWG range", + "shader": "binary_add_texture3d_float", + "gwg_size": "default_pick_gwg", + "lwg_size_sweep": { + "x": [4, 12, 4], + "y": [4, 12, 4], + "z": [1, 1, 1] + }, + "target_ms": 10, + "benchmark_runs": 5, + "max_dispatches": 512, + "validation_method": { + "function": "is_close", + "atol": 0.0001, + "rtol": 0.0001 + }, + "bindings": [ + { + "texture3d": { + "name": "out", + "sizes": [1, 16, 64, 64], + "access": "write", + "data": { + "fill": 0 + } + } + }, + { + "texture3d": { + "name": "a", + "sizes": [1, 16, 64, 64], + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "texture3d": { + "name": "b", + "sizes": [1, 16, 64, 64], + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "meta_ubo": "out" + }, + { + "meta_ubo": "a" + }, + { + "meta_ubo": "b" + } + ], + "specialization_constants": [ + { + "layout": "out" + }, + { + "layout": "a" + }, + { + "layout": "b" + } + ], + "push_constants": [ + { + "dtype": "float32", + "data": [2] + } + ], + "outputs": ["out"] +} diff --git a/backends/vulkan/test/ops/benchmark_v2/examples/texture_shader_lwg_sweep.json b/backends/vulkan/test/ops/benchmark_v2/examples/texture_shader_lwg_sweep.json new file mode 100644 index 00000000000..1a5a2f5782c --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/examples/texture_shader_lwg_sweep.json @@ -0,0 +1,76 @@ +{ + "name": "texture addition LWG list", + "shader": "binary_add_texture3d_float", + "gwg_size": "default_pick_gwg", + "lwg_size": [ + [8, 8, 1], + [10, 10, 1] + ], + "target_ms": 10, + "benchmark_runs": 5, + "show_memory_throughput": true, + "show_compute_throughput": true, + "max_dispatches": 512, + "validation_method": { + "function": "is_close", + "atol": 0.0001, + "rtol": 0.0001 + }, + "bindings": [ + { + "texture3d": { + "name": "out", + "sizes": [1, 16, 64, 64], + "access": "write", + "data": { + "fill": 0 + } + } + }, + { + "texture3d": { + "name": "a", + "sizes": [1, 16, 64, 64], + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "texture3d": { + "name": "b", + "sizes": [1, 16, 64, 64], + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "meta_ubo": "out" + }, + { + "meta_ubo": "a" + }, + { + "meta_ubo": "b" + } + ], + "specialization_constants": [ + { + "layout": "out" + }, + { + "layout": "a" + }, + { + "layout": "b" + } + ], + "push_constants": [ + { + "dtype": "float32", + "data": [2] + } + ], + "outputs": ["out"] +} diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/CaseCheckRegistry.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/CaseCheckRegistry.cpp new file mode 100644 index 00000000000..bdbdffd82cf --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/CaseCheckRegistry.cpp @@ -0,0 +1,66 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +namespace vkcompute::shader_profiling { + +void CaseCheckRegistry::register_check( + const std::string& name, + CaseCheckFunction function) { + if (name.empty() || !function) + throw std::invalid_argument("Case check requires a name and callable"); + if (!checks_.emplace(name, function).second) + throw std::invalid_argument("Duplicate case check: " + name); +} + +CaseCheckFunction CaseCheckRegistry::get_check(const std::string& name) const { + const auto entry = checks_.find(name); + if (entry == checks_.end()) + throw std::invalid_argument("Unknown case check: " + name); + return entry->second; +} + +CaseCheckRegistry& case_check_registry() { + static CaseCheckRegistry registry; + return registry; +} + +Report run_case_check(const Json& spec) { + const std::string name = spec.at("check"); + Report result{{"case_name", spec.value("name", name)}, {"check_name", name}}; + try { + GraphConfig config; + config.force_resize = true; + ComputeGraph graph(config); + Json validation = case_check_registry().get_check(name)(spec, graph); + const std::string status = validation.value("status", ""); + if (status != "PASS" && status != "FAIL" && status != "SKIP") + throw std::invalid_argument( + "Case check must return a PASS, FAIL, or SKIP status"); + result["validation"] = Report(validation); + if (status == "SKIP") { + result["status"] = "SKIP"; + result["skip_reason"] = + validation.value("reason", "Case check unsupported"); + } + } catch (const UnsupportedRoute& error) { + result["status"] = "SKIP"; + result["skip_reason"] = error.what(); + result["validation"] = {{"status", "SKIP"}}; + } catch (const vkapi::ShaderNotSupportedError& error) { + result["status"] = "SKIP"; + result["skip_reason"] = error.what(); + result["validation"] = {{"status", "SKIP"}}; + } catch (const std::exception& error) { + result["validation"] = {{"status", "FAIL"}, {"error", error.what()}}; + } + return result; +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/CaseCheckRegistry.h b/backends/vulkan/test/ops/benchmark_v2/framework/CaseCheckRegistry.h new file mode 100644 index 00000000000..b24f941a7b0 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/CaseCheckRegistry.h @@ -0,0 +1,36 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include +#include + +#include + +namespace vkcompute::shader_profiling { +using CaseCheckFunction = Json (*)(const Json&, ComputeGraph&); +class CaseCheckRegistry final { + private: + std::unordered_map checks_; + + public: + void register_check(const std::string& name, CaseCheckFunction function); + CaseCheckFunction get_check(const std::string& name) const; +}; +CaseCheckRegistry& case_check_registry(); +Report run_case_check(const Json& spec); +} // namespace vkcompute::shader_profiling + +#define REGISTER_CASE_CHECK(name, function) \ + ::vkcompute::shader_profiling::case_check_registry().register_check( \ + #name, &function) +#define REGISTER_CASE_CHECKS \ + static void register_case_checks(); \ + static const ::vkcompute::shader_profiling::RouteRegisterInit \ + case_checks_reg(®ister_case_checks); \ + static void register_case_checks() diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/Config.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/Config.cpp new file mode 100644 index 00000000000..f967bd50793 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/Config.cpp @@ -0,0 +1,531 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +#include +#include +#include +#include +#include + +namespace vkcompute::shader_profiling { + +namespace { + +void require(bool condition, const std::string& message) { + if (!condition) { + throw std::invalid_argument(message); + } +} + +uint32_t positive_integer(const Json& value, const std::string& name) { + require(value.is_number_integer(), name + " must be an integer"); + const double number = value.get(); + require(number > 0 && number <= UINT32_MAX, name + " is out of range"); + return value.get(); +} + +std::string +normalize_tensor(Json& tensors, Json tensor, const std::string& default_name) { + require(tensor.is_object(), "Tensor properties must be an object"); + if (tensor.contains("ref")) { + require(tensor.size() == 1, "Tensor references accept only ref"); + require( + tensor.at("ref").is_string() && + !tensor.at("ref").get().empty(), + "ref must be a nonempty tensor name"); + return tensor.at("ref").get(); + } + require(tensor.contains("sizes"), "Tensor requires sizes or ref"); + const Json name = tensor.value("name", Json(default_name)); + require( + name.is_string() && !name.get().empty(), + "Tensor name must be a nonempty string"); + const std::string tensor_name = name.get(); + require( + !tensors.contains(tensor_name), "Duplicate tensor name: " + tensor_name); + tensor.erase("name"); + tensors[tensor_name] = std::move(tensor); + return tensor_name; +} + +Json normalize_shader_config(const Json& spec) { + require( + !spec.contains("tensors"), + "Declare shader tensor properties in bindings, not tensors"); + require( + spec.contains("bindings") && spec.at("bindings").is_array(), + "bindings must be an array in shader binding order"); + Json normalized = spec; + normalized["tensors"] = Json::object(); + auto& bindings = normalized["bindings"]; + for (size_t i = 0; i < bindings.size(); ++i) { + Json& binding = bindings.at(i); + require(binding.is_object(), "Each binding must be an object"); + int kinds = binding.contains("meta_ubo") + binding.contains("uniform"); + const char* type = nullptr; + for (const char* candidate : {"buffer", "texture2d", "texture3d"}) { + if (binding.contains(candidate)) { + ++kinds; + type = candidate; + } + } + require( + kinds == 1, + "Each binding requires exactly one of buffer, texture2d, texture3d, meta_ubo, uniform"); + if (type == nullptr) { + continue; + } + require( + binding.size() == 1, + "Put tensor properties inside the binding type object"); + Json tensor = binding.at(type); + require(tensor.is_object(), "Tensor binding properties must be an object"); + require( + !tensor.contains("storage"), + "Binding type determines storage; omit storage"); + const Json access = tensor.value("access", Json("read")); + tensor.erase("access"); + if (!tensor.contains("ref")) { + tensor["storage"] = type; + } + const std::string tensor_name = normalize_tensor( + normalized["tensors"], + std::move(tensor), + "binding_" + std::to_string(i)); + binding = {{"tensor", tensor_name}, {"storage", type}, {"access", access}}; + } + return normalized; +} + +Json normalize_operator_config(const Json& spec) { + require( + !spec.contains("tensors"), + "Declare operator tensor properties in args_sweep, not tensors"); + require( + !spec.contains("inputs") && !spec.contains("outputs"), + "Use args_sweep instead of inputs and outputs for operators"); + require( + spec.contains("args") && spec.at("args").is_array() && + !spec.at("args").empty(), + "Operator args must be a nonempty ordered array"); + Json normalized = spec; + normalized["tensors"] = Json::object(); + auto& args = normalized["args"]; + for (size_t i = 0; i < args.size(); ++i) { + Json& arg = args.at(i); + if (!arg.contains("tensor")) { + continue; + } + require(arg.size() == 1, "Each argument requires a single typed value"); + Json tensor = arg.at("tensor"); + require(tensor.is_object(), "Tensor properties must be an object"); + const bool has_access = tensor.contains("access"); + const Json access = tensor.value("access", Json("read")); + tensor.erase("access"); + const std::string name = normalize_tensor( + normalized["tensors"], std::move(tensor), "arg_" + std::to_string(i)); + arg = {{"tensor", name}}; + if (has_access) { + arg["access"] = access; + } + } + return normalized; +} + +void validate_config(const Json& spec) { + require(spec.is_object(), "Each config must be an object"); + require( + spec.contains("shader") != spec.contains("operator"), + "Specify exactly one of shader or operator"); + const bool is_operator = spec.contains("operator"); + const char* kind = is_operator ? "operator" : "shader"; + require( + !spec.contains("global_size") && !spec.contains("local_size"), + "Use gwg_size and lwg_size for workgroup settings"); + require( + spec.at(kind).is_string(), + std::string(kind) + " must be a registry name"); + if (!is_operator) { + for (const char* key : {"gwg_size", "lwg_size"}) { + require(spec.contains(key), std::string("Missing ") + key); + if (spec.at(key).is_string()) { + require( + !spec.at(key).get().empty(), + std::string(key) + " picker name must not be empty"); + continue; + } + require( + spec.contains(key) && spec.at(key).is_array() && + spec.at(key).size() == 3, + std::string(key) + + " must be a picker name or three positive integers"); + for (const Json& dim : spec.at(key)) { + positive_integer(dim, key); + } + } + } + const Json tensors = spec.value("tensors", Json::object()); + require(tensors.is_object(), "tensors must be an object keyed by name"); + for (const auto& item : tensors.items()) { + const Json& tensor = item.value(); + require( + tensor.is_object() && tensor.contains("sizes"), + "Tensor requires sizes"); + tensor_numel(tensor.at("sizes")); + const std::string dtype = tensor.value("dtype", "float32"); + require( + dtype == "float32" || dtype == "float16" || dtype == "int32" || + dtype == "int8" || dtype == "uint8" || dtype == "int8x4", + "Unsupported tensor dtype: " + dtype); + const std::string storage = tensor.value("storage", "buffer"); + require( + storage == "buffer" || storage == "texture2d" || + storage == "texture3d" || (is_operator && storage == "constant"), + "Unsupported tensor storage: " + storage); + if (storage != "buffer" && storage != "constant") { + require( + tensor.at("sizes").size() <= 4, + "Textures support at most four dimensions"); + } + const std::string layout = tensor.value("layout", "width"); + require( + layout == "width" || layout == "height" || layout == "channels" || + layout == "4w" || layout == "4c" || layout == "4w4c" || + layout == "4h4w" || layout == "4c1w", + "Unsupported tensor layout: " + layout); + const bool packed_layout = + layout != "width" && layout != "height" && layout != "channels"; + require( + (dtype == "int8x4") == packed_layout, + "int8x4 requires a packed int8 layout; other dtypes require width, height, or channels"); + if (dtype == "int8x4") { + require( + (storage == "buffer" || storage == "constant") && + tensor.at("sizes").size() <= 4, + "int8x4 requires buffer or constant storage with at most four dimensions"); + } + require( + !tensor.contains("constant"), + "Use storage: constant instead of the tensor constant flag"); + validate_data_initializer(tensor, tensor_numel(tensor.at("sizes"))); + } + if (is_operator) { + for (const char* key : + {"gwg_size", + "lwg_size", + "lwg_size_sweep", + "bindings", + "specialization_constants", + "push_constants"}) { + require( + !spec.contains(key), + std::string(key) + " is only valid for raw shaders"); + } + require( + spec.contains("args") && spec.at("args").is_array(), + "Operator args must be an ordered argument array"); + for (const Json& arg : spec.at("args")) { + if (arg.is_null()) { + continue; + } + require(arg.is_object(), "Each argument must be null or a typed value"); + if (arg.contains("tensor")) { + require( + arg.size() == 1 || (arg.size() == 2 && arg.contains("access")), + "Tensor argument accepts only tensor and optional access"); + const Json& value = arg.at("tensor"); + require( + value.is_string() && tensors.contains(value.get()), + "Unknown argument tensor"); + const Json access = arg.value("access", Json("read")); + require( + access == "read" || access == "write" || access == "read_write", + "Tensor access must be read, write, or read_write"); + if (access != "read") { + require( + tensors.at(value.get()).value("storage", "buffer") != + "constant", + "Writable tensors cannot be constant"); + } + continue; + } + require( + arg.size() == 1, + "Each argument must be null or a single typed value"); + const auto item = arg.items().begin(); + const std::string key = item.key(); + const Json& value = item.value(); + if (key == "int") { + require( + value.is_number_integer() && value.get() >= INT32_MIN && + value.get() <= INT32_MAX, + "int argument must fit int32"); + } else if (key == "float") { + require( + value.is_number() && std::isfinite(value.get()), + "float argument must be finite"); + } else if (key == "bool") { + require(value.is_boolean(), "bool argument must be boolean"); + } else if (key == "string") { + require(value.is_string(), "string argument must be a string"); + } else if (key == "int_list") { + require(value.is_array(), "int_list must be an array"); + for (const Json& number : value) { + require( + number.is_number_integer() && number.get() >= INT32_MIN && + number.get() <= INT32_MAX, + "int_list values must fit int32"); + } + } else { + require(false, "Unsupported operator argument type: " + key); + } + } + } else { + require( + !spec.contains("route") && !spec.contains("route_sweep"), + "route and route_sweep are only valid for operators"); + require( + !spec.contains("args") && !spec.contains("args_sweep") && + !spec.contains("sweep_variables"), + "args_sweep is only valid for operators"); + require( + spec.contains("bindings") && spec.at("bindings").is_array(), + "bindings must be an array in shader binding order"); + for (const Json& binding : spec.at("bindings")) { + require(binding.is_object(), "Each binding must be an object"); + const int kinds = binding.contains("tensor") + + binding.contains("meta_ubo") + binding.contains("uniform"); + require( + kinds == 1, + "Each binding requires exactly one of tensor, meta_ubo, uniform"); + for (const char* key : {"tensor", "meta_ubo"}) { + if (binding.contains(key)) { + require( + binding.at(key).is_string() && + tensors.contains(binding.at(key).get()), + std::string(key) + " references an unknown tensor"); + } + } + if (binding.contains("tensor")) { + require( + tensors.at(binding.at("tensor").get()).at("storage") == + binding.at("storage"), + "Tensor reference binding type does not match the referenced tensor"); + } + const std::string access = binding.value("access", "read"); + require( + access == "read" || access == "write" || access == "read_write", + "Unsupported binding access: " + access); + } + if (spec.contains("specialization_constants")) { + require( + spec.at("specialization_constants").is_array(), + "specialization_constants must be an array"); + for (const Json& constant : spec.at("specialization_constants")) { + require( + constant.is_object(), + "Each specialization constant must be an object"); + if (constant.contains("layout")) { + require( + constant.at("layout").is_string() && + tensors.contains(constant.at("layout").get()), + "layout references an unknown tensor binding"); + } + } + } + if (spec.contains("outputs")) { + require( + spec.at("outputs").is_array(), + "outputs must be an array of tensor binding names"); + for (const Json& name : spec.at("outputs")) { + require( + name.is_string() && tensors.contains(name.get()), + "Unknown output tensor binding"); + } + } + } + if (spec.contains("dump_outputs")) { + require( + spec.at("dump_outputs").is_boolean(), "dump_outputs must be boolean"); + } + for (const char* key : + {"show_memory_throughput", "show_compute_throughput"}) { + if (spec.contains(key)) { + require(spec.at(key).is_boolean(), std::string(key) + " must be boolean"); + } + } + if (spec.contains("reference")) { + require( + spec.at("reference").is_string() && + !spec.at("reference").get().empty(), + "reference must be a nonempty function name"); + } + if (spec.contains("validation_method")) { + const Json& method = spec.at("validation_method"); + require(method.is_object(), "validation_method must be an object"); + require( + method.contains("function") && method.at("function") == "is_close", + "validation_method.function must be is_close"); + for (const auto& option : method.items()) { + require( + option.key() == "function" || option.key() == "atol" || + option.key() == "rtol" || option.key() == "outputs", + "validation_method accepts only function, atol, rtol, outputs"); + if (option.key() == "atol" || option.key() == "rtol") { + require( + option.value().is_number() && + std::isfinite(option.value().get()) && + option.value().get() >= 0, + "Validation tolerance must be finite and nonnegative"); + } + } + validation_output_names(spec); + } + for (const char* key : {"benchmark_runs", "max_dispatches"}) { + if (spec.contains(key)) { + const uint32_t number = positive_integer(spec.at(key), key); + require(number <= 65536, std::string(key) + " must not exceed 65536"); + } + } + if (spec.contains("warmup_runs")) { + require( + spec.at("warmup_runs").is_number_integer() && + spec.at("warmup_runs").get() >= 0 && + spec.at("warmup_runs").get() <= 65536, + "warmup_runs must be in [0, 65536]"); + } + if (spec.contains("target_ms")) { + require( + spec.at("target_ms").is_number() && + spec.at("target_ms").get() > 0 && + spec.at("target_ms").get() <= 10000, + "target_ms must be in (0, 10000]"); + } +} + +} // namespace + +std::set validation_output_names(const Json& spec) { + const Json method = spec.value("validation_method", Json::object()); + std::set names; + if (!method.contains("outputs")) + return names; + const Json& selected = method.at("outputs"); + require( + selected.is_array() && !selected.empty(), + "validation_method.outputs must be a nonempty list"); + for (const Json& name : selected) { + require( + name.is_string() && !name.get().empty() && + spec.at("tensors").contains(name.get()), + "validation_method.outputs references an unknown tensor"); + const std::string value = name; + require( + names.insert(value).second, "Duplicate validation output: " + value); + require( + spec.at("tensors").at(value).value("storage", "buffer") != "constant", + "Validation output must be a GPU tensor: " + value); + } + return names; +} + +std::vector parse_configs(const Json& document) { + std::vector definitions; + if (document.is_array()) { + require(!document.empty(), "Config list must not be empty"); + require( + document.size() <= 65536, "Config list must not exceed 65536 cases"); + definitions = document.get>(); + } else { + definitions.push_back(document); + } + std::vector configs; + for (const Json& definition : definitions) { + require(definition.is_object(), "Each config must be an object"); + require( + !definition.contains("force_narrow_int4_tile"), + "force_narrow_int4_tile is not supported by benchmark_v2"); + for (const char* key : + {"force_resize", + "invocations_per_batch", + "gpu_timing", + "shader_filter", + "requirements", + "device_variables"}) { + require( + !definition.contains(key), + std::string(key) + " is not supported by benchmark_v2"); + } + require( + !definition.contains("implementation") && + !definition.contains("implementation_sweep"), + "Use route and route_sweep for operator route selection"); + require( + definition.contains("operator") + definition.contains("shader") + + definition.contains("shader_sweep") + + definition.contains("check") == + 1, + "Specify exactly one of operator, shader, shader_sweep, or check"); + if (definition.contains("check")) { + require( + definition.at("check").is_string() && + !definition.at("check").get().empty(), + "check must be a nonempty registry name"); + require( + !definition.contains("args_sweep") && !definition.contains("args") && + !definition.contains("bindings") && + !definition.contains("tensors"), + "Standalone checks define their inputs in parameters or operator_case"); + configs.push_back(definition); + continue; + } + const bool is_operator = definition.contains("operator"); + const auto expanded = is_operator ? expand_operator_sweep(definition) + : expand_shader_sweep(definition); + require( + expanded.size() <= 65536 - configs.size(), + "Config list must not exceed 65536 cases"); + for (Json config : expanded) { + if (is_operator) { + config = normalize_operator_config(config); + } else { + config = normalize_shader_config(config); + } + validate_config(config); + configs.push_back(std::move(config)); + } + } + return configs; +} + +size_t tensor_numel(const Json& sizes) { + require( + sizes.is_array() && !sizes.empty() && sizes.size() <= 8, + "sizes must contain one to eight positive integers"); + size_t count = 1; + for (const Json& size : sizes) { + const uint32_t dim = positive_integer(size, "Tensor dimension"); + require(count <= INT32_MAX / dim, "Tensor element count must fit int32"); + count *= dim; + } + return count; +} + +uint32_t +choose_dispatch_count(double probe_us, double target_us, uint32_t limit) { + require( + limit > 0 && std::isfinite(probe_us) && probe_us >= 0 && + std::isfinite(target_us) && target_us > 0, + "Invalid calibration parameters"); + const double count = std::ceil(target_us / std::max(probe_us, 0.001)); + return static_cast( + std::clamp(count, 1.0, static_cast(limit))); +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/Config.h b/backends/vulkan/test/ops/benchmark_v2/framework/Config.h new file mode 100644 index 00000000000..99c3cb75e60 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/Config.h @@ -0,0 +1,28 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +#include +#include +#include +#include +#include + +namespace vkcompute::shader_profiling { + +using Json = nlohmann::json; + +// Expands sweeps and normalizes inline tensor definitions. +std::vector parse_configs(const Json& document); +std::set validation_output_names(const Json& spec); +size_t tensor_numel(const Json& sizes); +uint32_t +choose_dispatch_count(double probe_us, double target_us, uint32_t limit); + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/DataInitializer.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/DataInitializer.cpp new file mode 100644 index 00000000000..5f068e872f9 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/DataInitializer.cpp @@ -0,0 +1,433 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace vkcompute::shader_profiling { +namespace { +void require(bool condition, const std::string& message) { + if (!condition) + throw std::invalid_argument(message); +} + +template +double numeric_value(const Json& value) { + require(value.is_number(), "Input values must be numbers"); + const double number = value.get(); + require( + std::isfinite(number) && + number >= double(std::numeric_limits::lowest()) && + number <= double(std::numeric_limits::max()), + "Input value is out of dtype range"); + if constexpr (std::is_integral_v) + require( + std::trunc(number) == number, "Integer input must not have a fraction"); + return number; +} +uint32_t random_seed(const Json& value) { + require( + value.is_number_integer() && value.get() >= 0 && + value.get() <= UINT32_MAX, + "seed must fit uint32"); + return value.get(); +} +void integer_bounds(double lower, double upper) { + require( + std::trunc(lower) == lower && std::trunc(upper) == upper, + "Integer random bounds must not have a fraction"); + require( + lower >= INT32_MIN && upper <= UINT32_MAX && upper - lower <= UINT32_MAX, + "Integer random bounds require at most a uint32-sized range"); +} + +template +std::vector +pack_data(const Json& spec, size_t count, uint32_t seed, bool generate) { + require(count <= SIZE_MAX / sizeof(T), "Input byte size overflows"); + const Json data = spec.value("data", Json()); + const Json* values = nullptr; + bool repeat = false, random = false, random_integer = false; + double lower = 0, upper = 0; + uint64_t modulus = 0; + Json fill = spec.value("fill", Json(0)); + if (spec.contains("data")) { + require(!spec.contains("fill"), "Specify either data or fill"); + require( + data.is_array() || data.is_object(), + "data must be an initializer object or array"); + if (data.is_array()) { + require( + data.size() == count, + "data must contain exactly one value per element"); + values = &data; + } else if (data.contains("repeat")) { + require( + data.size() == 1 && data.at("repeat").is_array() && + !data.at("repeat").empty(), + "repeat requires a nonempty array"); + values = &data.at("repeat"); + repeat = true; + } else if ( + data.contains("row_sum") || data.contains("column_sum") || + data.contains("q4_group_sum")) { + require(data.size() == 1, "Derived initializers require a single method"); + if (data.contains("q4_group_sum")) { + const Json& method = data.at("q4_group_sum"); + require( + method.is_object() && method.size() == 2 && + method.contains("tensor") && method.contains("group_size"), + "q4_group_sum requires tensor and group_size"); + require( + method.at("tensor").is_string() && + !method.at("tensor").get().empty(), + "q4_group_sum requires a source tensor name"); + const Json& group = method.at("group_size"); + require( + group.is_number_integer() && group.get() > 0 && + group.get() <= INT32_MAX, + "q4_group_sum group_size must be a positive int32"); + require( + (std::is_same_v || std::is_same_v), + "q4_group_sum requires int32 or floating point output data"); + } else { + const char* method = + data.contains("row_sum") ? "row_sum" : "column_sum"; + require( + data.at(method).is_string() && + !data.at(method).get().empty(), + std::string(method) + " requires a source tensor name"); + require( + (std::is_same_v), + std::string(method) + " requires int32 output data"); + } + require(!generate, "Derived initializers require tensor data context"); + return {}; + } else if (data.contains("fill")) { + require(data.size() == 1, "fill initializer accepts only fill"); + fill = data.at("fill"); + } else if (data.contains("abs_mod")) { + require(data.size() == 1, "abs_mod accepts only its method object"); + const Json& method = data.at("abs_mod"); + require( + method.is_object() && method.contains("bounds") && + method.contains("mod") && + (method.size() == 2 || + (method.size() == 3 && method.contains("seed"))), + "abs_mod requires bounds, mod, and optional seed"); + const Json& bounds = method.at("bounds"); + require( + bounds.is_array() && bounds.size() == 2, + "abs_mod bounds require [min, max]"); + lower = numeric_value(bounds.at(0)); + upper = numeric_value(bounds.at(1)); + const Json& mod = method.at("mod"); + require( + mod.is_number_integer() && mod.get() > 0 && + mod.get() <= UINT32_MAX, + "abs_mod mod must be a positive uint32"); + modulus = mod.get(); + numeric_value(Json(modulus - 1)); + if (method.contains("seed")) + seed = random_seed(method.at("seed")); + random = true; + random_integer = true; + } else { + random_integer = data.contains("rand_uniform_int"); + require( + random_integer || data.contains("rand_uniform_float"), + "Unknown data initializer"); + const char* method = + random_integer ? "rand_uniform_int" : "rand_uniform_float"; + require( + data.size() == 1 || (data.size() == 2 && data.contains("seed")), + "Random initializers accept only bounds and optional seed"); + require( + random_integer || std::is_floating_point_v, + "Floating point random initializer requires floating point dtype"); + const Json& bounds = data.at(method); + require( + bounds.is_array() && bounds.size() == 2, + "Random initializer requires [min, max]"); + lower = numeric_value(bounds.at(0)); + upper = numeric_value(bounds.at(1)); + if (data.contains("seed")) + seed = random_seed(data.at("seed")); + random = true; + } + } + if (values) + for (const Json& value : *values) + numeric_value(value); + else if (!random) + numeric_value(fill); + if (random) { + require(lower <= upper, "Random initializer min must not exceed max"); + if (random_integer) + integer_bounds(lower, upper); + } + if (!generate) + return {}; + std::mt19937 engine(seed); + std::vector bytes(count * sizeof(T)); + for (size_t i = 0; i < count; ++i) { + double number; + if (random_integer) { + const uint64_t span = uint64_t(int64_t(upper) - int64_t(lower)) + 1; + const uint64_t range = uint64_t{1} << 32, limit = range - range % span; + uint32_t draw; + do { + draw = engine(); + } while (draw >= limit); + int64_t value = int64_t(lower) + int64_t(uint64_t(draw) % span); + if (modulus) { + const uint64_t absolute = + value < 0 ? uint64_t(-value) : uint64_t(value); + value = int64_t(absolute % modulus); + } + number = double(value); + } else if (random) { + const double unit = double(engine()) / (double(UINT32_MAX) + 1); + number = lower + (upper - lower) * unit; + } else + number = numeric_value( + values ? values->at(repeat ? i % values->size() : i) : fill); + const T value = static_cast(number); + std::memcpy(bytes.data() + i * sizeof(T), &value, sizeof(T)); + } + return bytes; +} + +std::vector build_data( + const Json& spec, + size_t count, + uint32_t seed, + bool generate, + size_t depth = 0) { + require(depth <= 64, "Data initializer nesting exceeds 64 levels"); + const Json data = spec.value("data", Json()); + if (data.is_object() && data.contains("pad_columns")) { + require(!spec.contains("fill"), "Specify either data or fill"); + require(data.size() == 1, "pad_columns accepts only its method object"); + const Json& method = data.at("pad_columns"); + require( + method.is_object() && method.size() == 2 && method.contains("source") && + method.contains("valid_columns"), + "pad_columns requires source and valid_columns"); + require( + spec.contains("sizes") && tensor_numel(spec.at("sizes")) == count, + "pad_columns requires sizes matching element count"); + const size_t columns = spec.at("sizes").back().get(); + const Json& valid = method.at("valid_columns"); + require( + valid.is_number_integer() && valid.get() >= 0 && + valid.get() <= double(columns), + "pad_columns valid_columns must be between zero and row width"); + const Json& source = method.at("source"); + require( + source.is_array() || source.is_object(), + "pad_columns source must be an initializer"); + require( + !source.is_object() || + (!source.contains("row_sum") && !source.contains("column_sum") && + !source.contains("q4_group_sum")), + "pad_columns source must not require tensor context"); + Json nested = spec; + nested["data"] = source; + auto result = build_data(nested, count, seed, generate, depth + 1); + if (generate) { + const size_t element_size = result.size() / count, + valid_columns = valid.get(); + for (size_t row = 0; row < count / columns; ++row) + std::memset( + result.data() + (row * columns + valid_columns) * element_size, + 0, + (columns - valid_columns) * element_size); + } + return result; + } + const std::string dtype = spec.value("dtype", "float32"); + if (dtype == "float32" || dtype == "float16") + return pack_data(spec, count, seed, generate); + if (dtype == "int32") + return pack_data(spec, count, seed, generate); + if (dtype == "uint32") + return pack_data(spec, count, seed, generate); + if (dtype == "int8" || dtype == "int8x4") + return pack_data(spec, count, seed, generate); + if (dtype == "uint8") + return pack_data(spec, count, seed, generate); + throw std::invalid_argument("Unsupported data dtype: " + dtype); +} + +template +std::vector integer_sums( + const std::vector& source, + size_t segments, + size_t count, + bool columns) { + require( + segments > 0 && count >= segments && + source.size() % (segments * sizeof(T)) == 0, + "Derived sum output is too short or source shape is invalid"); + const size_t contributions = source.size() / (segments * sizeof(T)); + std::vector result(count * sizeof(int32_t)); + for (size_t segment = 0; segment < segments; ++segment) { + int64_t sum = 0; + for (size_t i = 0; i < contributions; ++i) { + const size_t index = + columns ? i * segments + segment : segment * contributions + i; + T value; + std::memcpy(&value, source.data() + index * sizeof(T), sizeof(T)); + sum += value; + } + require( + sum >= INT32_MIN && sum <= INT32_MAX, "Derived sum overflows int32"); + const int32_t value = int32_t(sum); + std::memcpy(result.data() + segment * sizeof(value), &value, sizeof(value)); + } + return result; +} +std::vector sum_integer_tensor( + const Json& spec, + const std::vector& source, + size_t segments, + size_t count, + bool columns) { + const std::string dtype = spec.value("dtype", "float32"); + if (dtype == "int8" || dtype == "int8x4") + return integer_sums(source, segments, count, columns); + if (dtype == "uint8") + return integer_sums(source, segments, count, columns); + if (dtype == "int32") + return integer_sums(source, segments, count, columns); + if (dtype == "uint32") + return integer_sums(source, segments, count, columns); + throw std::invalid_argument("Derived sum source must have integer dtype"); +} +std::vector q4_group_sums( + const Json& spec, + const Json& source_spec, + const std::vector& source) { + const std::string source_dtype = source_spec.value("dtype", "float32"); + require( + source_dtype == "int8" || source_dtype == "uint8", + "q4_group_sum source must contain packed int8/uint8 weights"); + const Json& input_sizes = source_spec.at("sizes"); + const Json& output_sizes = spec.at("sizes"); + require( + input_sizes.size() == 2 && output_sizes.size() == 2, + "q4_group_sum requires [N,K/2] weights and [K/group_size,N] sums"); + const size_t channels = input_sizes.at(0).get(), + packed_width = input_sizes.at(1).get(), + width = packed_width * 2; + const size_t group_size = + spec.at("data").at("q4_group_sum").at("group_size").get(); + require( + width % group_size == 0, + "q4_group_sum group_size must divide unpacked weight width"); + const size_t groups = width / group_size, + output_channels = output_sizes.at(1).get(); + require( + output_sizes.at(0).get() == groups && output_channels >= channels, + "q4_group_sum output shape does not match groups and channels"); + require( + source.size() == channels * packed_width, + "q4_group_sum source byte size does not match weights"); + const bool floating = spec.value("dtype", "float32") != "int32"; + std::vector result(groups * output_channels * sizeof(int32_t)); + for (size_t group = 0; group < groups; ++group) + for (size_t channel = 0; channel < channels; ++channel) { + int64_t sum = 0; + for (size_t k = group * group_size; k < (group + 1) * group_size; ++k) { + const uint8_t value = source.at(channel * packed_width + k / 2); + sum += int((value >> ((k % 2) * 4)) & 15) - 8; + } + const size_t index = group * output_channels + channel; + if (floating) { + const float value = float(sum); + std::memcpy( + result.data() + index * sizeof(value), &value, sizeof(value)); + } else { + require( + sum >= INT32_MIN && sum <= INT32_MAX, + "q4_group_sum overflows int32"); + const int32_t value = int32_t(sum); + std::memcpy( + result.data() + index * sizeof(value), &value, sizeof(value)); + } + } + return result; +} +} // namespace + +void validate_data_initializer(const Json& spec, size_t count) { + build_data(spec, count, 0, false); +} +std::vector make_data(const Json& spec, size_t count, uint32_t seed) { + return build_data(spec, count, seed, true); +} +std::map> make_tensor_data( + const Json& tensors) { + std::map> result; + std::set active; + std::function initialize = + [&](const std::string& name) { + if (result.count(name)) + return; + require(tensors.contains(name), "Unknown data source tensor: " + name); + require( + active.insert(name).second, + "Cyclic data initializer reference: " + name); + const Json& spec = tensors.at(name); + const size_t count = tensor_numel(spec.at("sizes")); + validate_data_initializer(spec, count); + const Json data = spec.value("data", Json()); + if (data.is_object() && + (data.contains("row_sum") || data.contains("column_sum") || + data.contains("q4_group_sum"))) { + const bool q4 = data.contains("q4_group_sum"), + columns = data.contains("column_sum"); + const std::string source = q4 + ? data.at("q4_group_sum").at("tensor").get() + : data.at(columns ? "column_sum" : "row_sum").get(); + initialize(source); + const Json& source_spec = tensors.at(source); + if (q4) + result[name] = q4_group_sums(spec, source_spec, result.at(source)); + else { + const size_t segments = + source_spec.at("sizes") + .at(columns ? source_spec.at("sizes").size() - 1 : 0) + .get(); + result[name] = sum_integer_tensor( + source_spec, result.at(source), segments, count, columns); + } + } else { + // FNV-1a preserves per-tensor seeds across platforms and sweep + // entries. + uint32_t seed = 2166136261u; + for (unsigned char ch : name) + seed = (seed ^ ch) * 16777619u; + result[name] = make_data(spec, count, seed); + } + active.erase(name); + }; + for (const auto& tensor : tensors.items()) + initialize(tensor.key()); + return result; +} +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/DataInitializer.h b/backends/vulkan/test/ops/benchmark_v2/framework/DataInitializer.h new file mode 100644 index 00000000000..fadcd995431 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/DataInitializer.h @@ -0,0 +1,25 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +#include +#include +#include +#include +#include + +namespace vkcompute::shader_profiling { + +void validate_data_initializer(const nlohmann::json& spec, size_t count); +std::vector +make_data(const nlohmann::json& spec, size_t count, uint32_t seed = 0); +std::map> make_tensor_data( + const nlohmann::json& tensors); + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/GpuClock.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/GpuClock.cpp new file mode 100644 index 00000000000..74f8152abb1 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/GpuClock.cpp @@ -0,0 +1,78 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include +#include +#include +#include + +namespace vkcompute::shader_profiling { + +GpuClock::GpuClock(const std::string& sysfs_root) { + const std::filesystem::path root(sysfs_root); + std::vector> sources{ + {root / "class/kgsl/kgsl-3d0/gpuclk", 1e6}}; + std::error_code error; + auto devices = + std::filesystem::directory_iterator(root / "devices/platform", error); + for (; !error && devices != std::filesystem::directory_iterator(); + devices.increment(error)) { + const auto path = devices->path(); + if (path.extension() == ".mali" || path.filename() == "mali") { + // Pixel Mali cur_freq uses kHz; legacy Mali clock uses MHz. + sources.emplace_back(path / "cur_freq", 1000); + sources.emplace_back(path / "clock", 1); + } + } + std::optional kernel_units; + error.clear(); + devices = std::filesystem::directory_iterator(root / "class/devfreq", error); + for (; !error && devices != std::filesystem::directory_iterator(); + devices.increment(error)) { + const auto path = devices->path(); + const auto name = path.filename().string(); + const auto extension = path.extension(); + const bool kgsl = name.find("kgsl-3d0") != std::string::npos; + const bool sgpu = extension == ".sgpu" || name == "sgpu"; + if (kgsl || sgpu || extension == ".gpu" || extension == ".mali" || + name == "gpu" || name == "mali" || name == "pvrsrvkm") { + // Samsung SGPU devfreq uses kHz; standard devfreq and KGSL use Hz. + sources.emplace_back(path / "cur_freq", sgpu ? 1000 : 1e6); + if (kgsl) + kernel_units = 1; + else if (sgpu) + kernel_units = 1000; + } + } + if (kernel_units) { + sources.emplace_back(root / "kernel/gpu/gpu_clock", *kernel_units); + } + for (const auto& source : sources) { + path_ = source.first.string(); + units_per_mhz_ = source.second; + if (read_mhz()) + return; + } + path_.clear(); +} + +std::optional GpuClock::read_mhz() const { + if (path_.empty()) + return std::nullopt; + std::ifstream input(path_); + double frequency; + if (!(input >> frequency) || !std::isfinite(frequency) || frequency < 0) + return std::nullopt; + input >> std::ws; + if (!input.eof()) + return std::nullopt; + return frequency / units_per_mhz_; +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/GpuClock.h b/backends/vulkan/test/ops/benchmark_v2/framework/GpuClock.h new file mode 100644 index 00000000000..7dcfbcde3a7 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/GpuClock.h @@ -0,0 +1,23 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +namespace vkcompute::shader_profiling { + +class GpuClock final { + std::string path_; + double units_per_mhz_ = 1; + + public: + explicit GpuClock(const std::string& sysfs_root = "/sys"); + std::optional read_mhz() const; +}; + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/OperatorBenchmark.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/OperatorBenchmark.cpp new file mode 100644 index 00000000000..3dc72a2cd58 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/OperatorBenchmark.cpp @@ -0,0 +1,161 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace vkcompute::shader_profiling { + +namespace { + +Json tensor_accesses( + ComputeGraph& graph, + size_t first, + size_t end, + const std::map& tensors, + ValueRef primary_output) { + std::map first_access; + std::map all_access; + for (size_t i = first; i < end; ++i) { + std::map node_access; + for (const auto& group : graph.execute_nodes()[i]->args()) { + for (ValueRef ref : group.refs) { + node_access[ref] |= group.access; + } + } + for (const auto& entry : node_access) { + first_access.emplace(entry); + all_access[entry.first] |= entry.second; + } + } + Json result = Json::object(); + for (const auto& tensor : tensors) { + const ValueRef ref = tensor.second; + // Mutable input caches can be written before their unchanged prefix is + // read. + const bool reads_existing = (first_access[ref] & vkapi::kRead) || + (ref != primary_output && (all_access[ref] & vkapi::kRead)); + result[tensor.first] = !(all_access[ref] & vkapi::kWrite) ? "read" + : reads_existing ? "read_write" + : "write"; + } + return result; +} + +ValueRef operator_arg( + const Json& arg, + ComputeGraph& graph, + const std::map& tensors) { + if (arg.is_null()) { + return graph.add_none(); + } + if (arg.contains("tensor")) { + return tensors.at(arg.at("tensor").get()); + } + if (arg.contains("int")) { + return graph.add_scalar(arg.at("int").get()); + } + if (arg.contains("float")) { + return graph.add_scalar(arg.at("float").get()); + } + if (arg.contains("bool")) { + return graph.add_scalar(arg.at("bool").get()); + } + if (arg.contains("int_list")) { + return graph.add_scalar_list( + arg.at("int_list").get>()); + } + return graph.add_string(arg.at("string").get()); +} + +} // namespace + +std::vector build_operator_args( + const Json& spec, + ComputeGraph& graph, + const std::map& tensors) { + std::vector args; + for (const Json& arg : spec.at("args")) + args.push_back(operator_arg(arg, graph, tensors)); + return args; +} + +Report benchmark_operator( + const Json& spec, + const TensorData& host_data, + bool validation_only) { + GraphConfig config; + config.force_resize = true; + if (!validation_only) { + config.descriptor_pool_safety_factor *= spec.value("max_dispatches", 200u); + } + ComputeGraph graph(config); + const auto tensors = create_tensors(spec, graph, host_data, validation_only); + const auto args = build_operator_args(spec, graph, tensors); + const size_t first = graph.execute_nodes().size(); + const Json route = build_operator_route(spec, graph, args); + const size_t end = graph.execute_nodes().size(); + if (first == end) { + throw std::invalid_argument("Operator produced no execute nodes"); + } + const Json accesses = + tensor_accesses(graph, first, end, tensors, args.back()); + std::map outputs; + if (validation_only) { + outputs = add_validation_outputs(spec, graph, tensors, accesses); + } + graph.prepare(); + graph.prepack(); + graph.propagate_resize(); + if (validation_only) { + copy_validation_inputs(spec, graph, tensors, host_data); + graph.execute(); + Report result = + operator_report(spec, actual_tensor_dtypes(graph, tensors), accesses); + if (spec.contains("route")) + result["route"] = route; + result["validation"] = + validate_outputs(spec, graph, tensors, outputs, host_data); + return result; + } + auto encode_batch = [&](uint32_t count) { + for (uint32_t i = 0; i < count; ++i) { + for (size_t j = first; j < end; ++j) { + graph.execute_nodes()[j]->encode(&graph); + } + } + }; + // Warm once to establish the layout state used at each replay boundary. + auto& context = *graph.context(); + context.set_cmd(); + encode_batch(1); + vkapi::VulkanFence fence(context.device()); + context.submit_cmd_to_gpu(fence.get_submit_handle(), true); + fence.wait(); + Report result = + operator_report(spec, actual_tensor_dtypes(graph, tensors), accesses); + if (spec.contains("route")) { + result["route"] = route; + } + result["execute_nodes_per_invocation"] = end - first; + result["execute_nodes"] = Report::array(); + for (size_t i = first; i < end; ++i) { + result["execute_nodes"].push_back(graph.execute_nodes()[i]->name()); + } + result.update(measure(spec, graph, encode_batch, Json(result))); + return result; +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/OperatorBenchmark.h b/backends/vulkan/test/ops/benchmark_v2/framework/OperatorBenchmark.h new file mode 100644 index 00000000000..d55357c0233 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/OperatorBenchmark.h @@ -0,0 +1,26 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include +#include +#include + +namespace vkcompute::shader_profiling { + +std::vector build_operator_args( + const Json& spec, + ComputeGraph& graph, + const std::map& tensors); + +Report benchmark_operator( + const Json& spec, + const TensorData& host_data, + bool validation_only = false); + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/Reference.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/Reference.cpp new file mode 100644 index 00000000000..beaffcab269 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/Reference.cpp @@ -0,0 +1,307 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace vkcompute::shader_profiling { + +namespace reference { + +void require(bool condition, const std::string& message) { + if (!condition) { + throw std::invalid_argument(message); + } +} + +void parallel_for( + size_t count, + size_t threads, + const std::function& function) { + if (threads == 0) { + threads = std::max(1u, std::thread::hardware_concurrency()); + } + threads = std::min({threads, size_t(8), count}); + std::vector workers; + for (size_t worker = 1; worker < threads; ++worker) { + workers.emplace_back([&, worker] { + for (size_t i = count * worker / threads; + i < count * (worker + 1) / threads; + ++i) { + function(i); + } + }); + } + for (size_t i = 0; threads > 0 && i < count / threads; ++i) { + function(i); + } + for (auto& worker : workers) { + worker.join(); + } +} + +const Tensor& +tensor_arg(const Json& spec, size_t index, const TensorMap& tensors) { + return tensors.at(spec.at("args").at(index).at("tensor").get()); +} + +} // namespace reference + +namespace { + +using Half = executorch::runtime::etensor::Half; + +std::string tensor_dtype( + const Json& spec, + const Json& actual_dtypes, + const std::string& name) { + return actual_dtypes.contains(name) + ? actual_dtypes.at(name).get() + : spec.at("tensors").at(name).value("dtype", "float32"); +} + +template +double read_element(const std::vector& bytes, size_t index) { + T value; + std::memcpy(&value, bytes.data() + index * sizeof(T), sizeof(T)); + return double(value); +} + +reference::Tensor decode_tensor( + const Json& description, + const std::string& actual_dtype, + const std::vector& bytes) { + const std::string source_dtype = description.value("dtype", "float32"); + const bool constant_half = source_dtype == "float16" && + description.value("storage", "buffer") == "constant"; + const size_t element_size = constant_half ? sizeof(Half) + : source_dtype == "float32" || source_dtype == "float16" ? sizeof(float) + : source_dtype == "int32" ? sizeof(int32_t) + : sizeof(uint8_t); + const size_t count = tensor_numel(description.at("sizes")); + reference::require( + bytes.size() == count * element_size, + "Reference tensor byte count does not match its shape and dtype"); + reference::Tensor result{ + description.at("sizes").get>(), + std::vector(count)}; + for (size_t i = 0; i < count; ++i) { + double value; + if (constant_half) { + value = read_element(bytes, i); + } else if (source_dtype == "float32" || source_dtype == "float16") { + value = read_element(bytes, i); + } else if (source_dtype == "int32") { + value = read_element(bytes, i); + } else if (source_dtype == "uint8") { + value = read_element(bytes, i); + } else { + value = read_element(bytes, i); + } + result.values[i] = + actual_dtype == "float16" ? float(Half(float(value))) : value; + } + return result; +} + +std::string output_name(const Json& spec) { + if (spec.contains("operator")) { + return spec.at("args").back().at("tensor"); + } + for (const Json& binding : spec.at("bindings")) { + if (binding.contains("tensor")) { + return binding.at("tensor"); + } + } + throw std::invalid_argument("Reference requires an output tensor binding"); +} + +} // namespace + +std::string reference_name(const Json& spec) { + const auto& registry = reference_registry(); + if (spec.contains("reference")) { + const std::string name = spec.at("reference"); + reference::require( + registry.has_reference(name), "Unknown reference function: " + name); + return name; + } + return spec.contains("operator") + ? registry.operator_reference(spec.at("operator").get()) + : registry.shader_reference(spec.at("shader").get()); +} + +ReferenceOutputs reference_outputs( + const Json& spec, + const Json& actual_dtypes, + const std::map>& host_data, + size_t threads) { + const std::string function = reference_name(spec); + reference::TensorMap tensors; + for (const auto& item : host_data) { + const Json& description = spec.at("tensors").at(item.first); + tensors.emplace( + item.first, + decode_tensor( + description, + tensor_dtype(spec, actual_dtypes, item.first), + item.second)); + } + ReferenceOutputs outputs; + if (const auto callback = + reference_registry().get_reference_outputs(function)) { + outputs = callback(spec, tensors, threads); + } else { + reference::Tensor output = + reference_registry().get_reference(function)(spec, tensors, threads); + const std::string name = output_name(spec); + reference::require( + output.sizes == + spec.at("tensors").at(name).at("sizes").get>(), + "Reference output shape does not match declared output tensor"); + outputs.emplace(name, std::move(output.values)); + } + for (auto& output : outputs) { + reference::require( + spec.at("tensors").contains(output.first), + "Reference produced unknown output tensor: " + output.first); + reference::require( + output.second.size() == + tensor_numel(spec.at("tensors").at(output.first).at("sizes")), + "Reference output element count does not match declared shape: " + + output.first); + if (tensor_dtype(spec, actual_dtypes, output.first) == "float16") { + for (double& value : output.second) + value = float(Half(float(value))); + } + } + return outputs; +} + +Report compare_outputs( + const Json& spec, + const ReferenceOutputs& expected, + const ReferenceOutputs& actual, + const Json& actual_dtypes) { + const Json method = spec.value("validation_method", Json::object()); + reference::require( + method.value("function", "is_close") == "is_close", + "Unknown validation method"); + Report result = { + {"status", "PASS"}, + {"reference", reference_name(spec)}, + {"method", "is_close"}, + {"outputs", Report::object()}}; + const auto selected = validation_output_names(spec); + if (!selected.empty()) { + result["output_selection"] = selected; + for (const auto& name : selected) { + if (!expected.count(name)) { + result["status"] = "FAIL"; + result["outputs"][name] = { + {"status", "FAIL"}, {"error", "Missing reference output"}}; + } + } + } + size_t total_mismatches = 0; + if (expected.empty()) { + result["status"] = "FAIL"; + result["error"] = "Reference produced no outputs"; + } + for (const auto& item : expected) { + if (!selected.empty() && !selected.count(item.first)) + continue; + const std::string dtype = tensor_dtype(spec, actual_dtypes, item.first); + const double default_tolerance = dtype == "float16" ? 1e-2 + : dtype == "float32" ? 1e-4 + : 0; + const double atol = method.value("atol", default_tolerance); + const double rtol = method.value("rtol", default_tolerance); + reference::require( + std::isfinite(atol) && atol >= 0 && std::isfinite(rtol) && rtol >= 0, + "Validation tolerances must be finite and nonnegative"); + Report output = { + {"status", "PASS"}, + {"element_count", item.second.size()}, + {"atol", atol}, + {"rtol", rtol}}; + const auto computed = actual.find(item.first); + if (computed == actual.end() || + computed->second.size() != item.second.size() || + item.second.size() != + tensor_numel(spec.at("tensors").at(item.first).at("sizes"))) { + output["status"] = "FAIL"; + output["error"] = computed == actual.end() + ? "Missing output" + : "Output element count does not match declared shape"; + result["status"] = "FAIL"; + result["outputs"][item.first] = std::move(output); + continue; + } + size_t mismatches = 0; + size_t nonfinite = 0; + double max_abs_error = 0; + double max_rel_error = 0; + Report mismatch_indices = Report::array(); + for (size_t i = 0; i < item.second.size(); ++i) { + const double reference_value = item.second[i]; + const double actual_value = computed->second[i]; + const bool finite = + std::isfinite(reference_value) && std::isfinite(actual_value); + const double error = std::abs(actual_value - reference_value); + if (finite) { + max_abs_error = std::max(max_abs_error, error); + const double relative = reference_value != 0 + ? error / std::abs(reference_value) + : error == 0 ? 0 + : std::numeric_limits::infinity(); + max_rel_error = std::max(max_rel_error, relative); + } else { + ++nonfinite; + } + if (!finite || error > atol + rtol * std::abs(reference_value)) { + ++mismatches; + if (mismatch_indices.size() < 8) { + mismatch_indices.push_back(i); + } + } + } + output["status"] = mismatches == 0 ? "PASS" : "FAIL"; + output["mismatch_count"] = mismatches; + output["nonfinite_count"] = nonfinite; + output["max_abs_error"] = max_abs_error; + output["max_rel_error"] = + std::isfinite(max_rel_error) ? Report(max_rel_error) : Report(); + output["first_mismatch_indices"] = std::move(mismatch_indices); + total_mismatches += mismatches; + if (mismatches > 0) { + result["status"] = "FAIL"; + } + result["outputs"][item.first] = std::move(output); + } + for (const auto& item : actual) { + if (!selected.empty() && !selected.count(item.first)) + continue; + if (!expected.count(item.first)) { + result["status"] = "FAIL"; + result["outputs"][item.first] = { + {"status", "FAIL"}, {"error", "Unexpected output"}}; + } + } + result["mismatch_count"] = total_mismatches; + return result; +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/Reference.h b/backends/vulkan/test/ops/benchmark_v2/framework/Reference.h new file mode 100644 index 00000000000..8aa90c4832e --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/Reference.h @@ -0,0 +1,51 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +#include +#include +#include +#include + +namespace vkcompute::shader_profiling { + +using ReferenceOutputs = std::map>; + +namespace reference { + +struct Tensor { + std::vector sizes; + std::vector values; +}; + +using TensorMap = std::map; + +void parallel_for( + size_t count, + size_t threads, + const std::function& function); +const Tensor& +tensor_arg(const Json& spec, size_t index, const TensorMap& tensors); +void require(bool condition, const std::string& message); + +} // namespace reference + +std::string reference_name(const Json& spec); +ReferenceOutputs reference_outputs( + const Json& spec, + const Json& actual_dtypes, + const std::map>& host_data, + size_t threads = 0); +Report compare_outputs( + const Json& spec, + const ReferenceOutputs& expected, + const ReferenceOutputs& actual, + const Json& actual_dtypes); + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/ReferenceRegistry.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/ReferenceRegistry.cpp new file mode 100644 index 00000000000..77219dc5206 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/ReferenceRegistry.cpp @@ -0,0 +1,112 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +namespace vkcompute::shader_profiling { + +namespace { + +void register_association( + std::unordered_map& table, + const std::string& name, + const std::string& reference_name, + const std::string& kind) { + reference::require( + !name.empty() && !reference_name.empty(), + kind + " and reference names must be nonempty"); + reference::require( + table.emplace(name, reference_name).second, + "Duplicate " + kind + " reference association: " + name); +} + +const std::string& lookup_association( + const std::unordered_map& table, + const std::string& name, + const std::string& kind) { + const auto entry = table.find(name); + reference::require( + entry != table.end(), + "No correctness reference registered for " + kind + " " + name); + return entry->second; +} + +} // namespace + +ReferenceFunction ReferenceRegistry::get_reference( + const std::string& name) const { + const auto entry = functions_.find(name); + reference::require( + entry != functions_.end(), "Unknown reference function: " + name); + return entry->second; +} + +ReferenceOutputsFunction ReferenceRegistry::get_reference_outputs( + const std::string& name) const { + const auto entry = output_functions_.find(name); + return entry == output_functions_.end() ? nullptr : entry->second; +} + +bool ReferenceRegistry::has_reference(const std::string& name) const { + return functions_.count(name) || output_functions_.count(name); +} + +const std::string& ReferenceRegistry::operator_reference( + const std::string& name) const { + const auto& reference = lookup_association(operators_, name, "operator"); + reference::require( + has_reference(reference), "Unknown reference function: " + reference); + return reference; +} + +const std::string& ReferenceRegistry::shader_reference( + const std::string& name) const { + const auto& reference = lookup_association(shaders_, name, "shader"); + reference::require( + has_reference(reference), "Unknown reference function: " + reference); + return reference; +} + +void ReferenceRegistry::register_reference( + const std::string& name, + ReferenceFunction function) { + reference::require(!name.empty(), "Reference name must be nonempty"); + reference::require( + function != nullptr, "Reference function must not be null"); + reference::require( + !has_reference(name), "Duplicate reference function: " + name); + functions_.emplace(name, function); +} + +void ReferenceRegistry::register_reference_outputs( + const std::string& name, + ReferenceOutputsFunction function) { + reference::require(!name.empty(), "Reference name must be nonempty"); + reference::require( + function != nullptr, "Reference function must not be null"); + reference::require( + !has_reference(name), "Duplicate reference function: " + name); + output_functions_.emplace(name, function); +} + +void ReferenceRegistry::register_operator_reference( + const std::string& name, + const std::string& reference) { + register_association(operators_, name, reference, "operator"); +} + +void ReferenceRegistry::register_shader_reference( + const std::string& name, + const std::string& reference) { + register_association(shaders_, name, reference, "shader"); +} + +ReferenceRegistry& reference_registry() { + static ReferenceRegistry registry; + return registry; +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/ReferenceRegistry.h b/backends/vulkan/test/ops/benchmark_v2/framework/ReferenceRegistry.h new file mode 100644 index 00000000000..d32f90e886f --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/ReferenceRegistry.h @@ -0,0 +1,73 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +#include + +namespace vkcompute::shader_profiling { + +using ReferenceFunction = + reference::Tensor (*)(const Json&, const reference::TensorMap&, size_t); + +using ReferenceOutputsFunction = + ReferenceOutputs (*)(const Json&, const reference::TensorMap&, size_t); + +class ReferenceRegistry final { + private: + std::unordered_map functions_; + std::unordered_map output_functions_; + std::unordered_map operators_; + std::unordered_map shaders_; + + public: + ReferenceFunction get_reference(const std::string& name) const; + ReferenceOutputsFunction get_reference_outputs(const std::string& name) const; + bool has_reference(const std::string& name) const; + const std::string& operator_reference(const std::string& name) const; + const std::string& shader_reference(const std::string& name) const; + void register_reference(const std::string& name, ReferenceFunction function); + void register_reference_outputs( + const std::string& name, + ReferenceOutputsFunction function); + void register_operator_reference( + const std::string& name, + const std::string& reference); + void register_shader_reference( + const std::string& name, + const std::string& reference); +}; + +class ReferenceRegisterInit final { + public: + explicit ReferenceRegisterInit(void (*initialize)()) { + initialize(); + } +}; + +ReferenceRegistry& reference_registry(); + +} // namespace vkcompute::shader_profiling + +#define REGISTER_REFERENCE_FN(name, function) \ + ::vkcompute::shader_profiling::reference_registry().register_reference( \ + #name, &function) +#define REGISTER_OPERATOR_REFERENCE(name, reference) \ + ::vkcompute::shader_profiling::reference_registry() \ + .register_operator_reference(#name, #reference) +#define REGISTER_SHADER_REFERENCE(name, reference) \ + ::vkcompute::shader_profiling::reference_registry() \ + .register_shader_reference(#name, #reference) +#define REGISTER_REFERENCE_OUTPUTS_FN(name, function) \ + ::vkcompute::shader_profiling::reference_registry() \ + .register_reference_outputs(#name, &function) +#define REGISTER_REFERENCE_FUNCTIONS \ + static void register_references(); \ + static const ::vkcompute::shader_profiling::ReferenceRegisterInit \ + reference_reg(®ister_references); \ + static void register_references() diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/Report.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/Report.cpp new file mode 100644 index 00000000000..58fbdb75317 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/Report.cpp @@ -0,0 +1,823 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace vkcompute::shader_profiling { +namespace { + +void write_report_value( + std::ostream& output, + const Report& value, + size_t depth, + std::string_view field = {}, + bool inline_array = false) { + if ((!value.is_object() && !value.is_array()) || value.empty()) { + output << value.dump(); + return; + } + if (value.is_array() && inline_array) { + output << '['; + for (size_t i = 0; i < value.size(); ++i) { + if (i > 0) { + output << ", "; + } + output << value.at(i).dump(); + } + output << ']'; + return; + } + + const bool object = value.is_object(); + output << (object ? '{' : '[') << '\n'; + for (auto it = value.begin(); it != value.end(); ++it) { + if (it != value.begin()) { + output << ",\n"; + } + output << std::string((depth + 1) * 2, ' '); + const std::string key = object ? it.key() : ""; + if (object) { + output << Report(key).dump() << ": "; + } + constexpr std::string_view clock_suffix = "_observed_gpu_clock_freq_MHz"; + const bool clock_pairs = field.size() >= clock_suffix.size() && + field.substr(field.size() - clock_suffix.size()) == clock_suffix; + const bool compact = key == "sizes" || clock_pairs || + (key == "value" && (field == "gwg_size" || field == "lwg_size")); + write_report_value(output, it.value(), depth + 1, key, compact); + } + output << '\n' << std::string(depth * 2, ' ') << (object ? '}' : ']'); +} + +double percentile(const std::vector& sorted, double fraction) { + const double position = fraction * (sorted.size() - 1); + const size_t lower = static_cast(position); + const size_t upper = std::min(lower + 1, sorted.size() - 1); + return sorted[lower] + (sorted[upper] - sorted[lower]) * (position - lower); +} + +Report case_header(const Json& spec) { + const bool op = spec.contains("operator"); + const std::string name = spec.at(op ? "operator" : "shader"); + Report report = { + {"case_name", spec.value("name", name)}, + {op ? "operator_name" : "shader_name", name}}; + if (spec.contains("sweep_index")) { + report["sweep_index"] = spec.at("sweep_index"); + } + if (op && spec.contains("sweep_variables")) { + report["args_sweep_index"] = spec.at("args_sweep_index"); + report["sweep_variables"] = spec.at("sweep_variables"); + } + if (op && spec.contains("route")) { + report["route"] = {{"requested", spec.at("route")}, {"resolved", nullptr}}; + } + return report; +} + +Report tensor_description(const Json& tensor, const std::string& actual_dtype) { + const std::string requested = tensor.value("dtype", "float32"); + Report info = { + {"type", tensor.value("storage", "buffer")}, + {"sizes", tensor.at("sizes")}, + {"dtype", actual_dtype}, + {"layout", tensor.value("layout", "width")}}; + if (actual_dtype != requested) { + info["requested_dtype"] = requested; + } + return info; +} + +Report describe_bindings(const Json& spec, const Json& actual_dtypes) { + Report result = Report::object(); + const Json& tensors = spec.at("tensors"); + for (size_t i = 0; i < spec.at("bindings").size(); ++i) { + const Json& binding = spec.at("bindings").at(i); + Report info; + std::string name; + if (binding.contains("tensor")) { + name = binding.at("tensor"); + const Json& tensor = tensors.at(name); + info = tensor_description( + tensor, actual_dtypes.value(name, tensor.value("dtype", "float32"))); + info["access"] = binding.value("access", "read"); + if (result.contains(name)) { + info["ref"] = name; + } + } else if (binding.contains("meta_ubo")) { + const std::string tensor = binding.at("meta_ubo"); + const std::string field = binding.value("field", "meta"); + name = tensor + "." + field; + info = {{"type", "meta_ubo"}, {"tensor", tensor}, {"field", field}}; + } else { + name = "binding_" + std::to_string(i); + info = {{"type", "uniform"}, {"values", binding.at("uniform")}}; + } + info["binding_index"] = i; + std::string key = name; + for (size_t suffix = i; result.contains(key); ++suffix) { + key = name + "_" + std::to_string(suffix); + } + result[key] = std::move(info); + } + return result; +} + +Report workgroup_report(const Json& setting, const Json& resolved) { + return { + {"method", setting.is_string() ? setting.get() : "explicit"}, + {"value", resolved}}; +} + +std::string compact_tensor(const Report& tensor) { + const std::string storage = tensor.value("type", "buffer"); + std::string prefix = storage; + if (storage == "buffer") { + prefix = "b"; + } else if (storage == "texture2d") { + prefix = "t2"; + } else if (storage == "texture3d") { + prefix = "t3"; + } else if (storage == "constant") { + prefix = "c"; + } + const std::string dtype = tensor.at("dtype"); + std::string suffix = dtype; + if (dtype == "float32") { + suffix = "f32"; + } else if (dtype == "float16") { + suffix = "f16"; + } else if (dtype == "int32") { + suffix = "i32"; + } else if (dtype == "int8") { + suffix = "i8"; + } else if (dtype == "uint8") { + suffix = "u8"; + } else if (dtype == "int8x4") { + suffix = "i8x4"; + } + if (storage != "constant") { + suffix += "." + tensor.value("layout", "width"); + } + return prefix + tensor.at("sizes").dump() + suffix; +} + +std::string compact_input_value(const Report& value) { + if (value.is_number_float()) { + std::ostringstream output; + output << std::setprecision(3) << value.get(); + return output.str(); + } + if (value.is_array()) { + std::string output = "["; + for (size_t i = 0; i < value.size(); ++i) { + if (i > 0) { + output += ','; + } + output += compact_input_value(value.at(i)); + } + return output + ']'; + } + return value.dump(); +} + +std::string table_inputs(const Report& report, bool op) { + std::string inputs; + auto append = [&](const std::string& value) { + if (!inputs.empty()) { + inputs += ", "; + } + inputs += value; + }; + if (op) { + for (const auto& arg : report.at("args")) { + if (arg.contains("tensor")) { + if (arg.at("tensor").value("access", "read") != "write") { + append(compact_tensor(arg.at("tensor"))); + } + } else { + append( + compact_input_value(arg.is_object() ? arg.begin().value() : arg)); + } + } + } else { + auto append_blocks = [&](const Report& blocks) { + for (const auto& block : blocks) { + append(compact_input_value(block.at("data"))); + } + }; + for (const auto& binding : report.at("bindings")) { + if (binding.contains("sizes") && + binding.value("access", "read") != "write") { + append(compact_tensor(binding)); + } else if (binding.value("type", "") == "uniform") { + append_blocks(binding.at("values")); + } + } + if (report.contains("push_constants")) { + append_blocks(report.at("push_constants")); + } + } + return inputs.empty() ? "-" : inputs; +} + +std::string table_latency(const Report& stats, const char* key) { + if (stats.is_null()) { + return "N/A"; + } + std::ostringstream output; + output << std::fixed << std::setprecision(3) << stats.at(key).get(); + return output.str(); +} + +std::string table_gpu_throughput(const Report& report, const char* metric) { + if (!report.contains("throughput") || + !report.at("throughput").contains("gpu_timestamp")) { + return "N/A"; + } + const Report& rates = report.at("throughput").at("gpu_timestamp"); + return rates.contains(metric) ? table_latency(rates.at(metric), "mean") + : "N/A"; +} + +constexpr std::array kObservedGpuClockFields = { + "gpu_batch_observed_gpu_clock_freq_MHz", + "wall_time_batch_observed_gpu_clock_freq_MHz", + "shader_profile_batch_observed_gpu_clock_freq_MHz"}; + +void observe_gpu_clock_range( + std::array, 2>& range, + const Report& samples) { + if (!samples.is_array()) + return; + for (const auto& pair : samples) { + if (!pair.is_array()) + continue; + for (const auto& observation : pair) { + if (!observation.is_number()) + continue; + const double frequency = observation.get(); + if (!std::isfinite(frequency) || frequency < 0) + continue; + range[0] = range[0] ? std::min(*range[0], frequency) : frequency; + range[1] = range[1] ? std::max(*range[1], frequency) : frequency; + } + } +} + +std::array table_gpu_clock_range( + const Report& report, + bool profile_only = false) { + std::array, 2> range{}; + const size_t first = profile_only ? kObservedGpuClockFields.size() - 1 : 0; + for (size_t i = first; i < kObservedGpuClockFields.size(); ++i) { + const char* field = kObservedGpuClockFields[i]; + if (report.contains(field)) + observe_gpu_clock_range(range, report.at(field)); + } + std::array result{"N/A", "N/A"}; + for (size_t i = 0; i < result.size(); ++i) { + if (range[i]) { + std::ostringstream output; + output << std::setprecision(9) << *range[i]; + result[i] = output.str(); + } + } + return result; +} + +std::string table_text(std::string text, size_t max_chars = 0) { + for (char& ch : text) { + if (ch == '\n' || ch == '\r' || ch == '\t') { + ch = ' '; + } + } + if (max_chars != 0 && text.size() > max_chars) { + const size_t indentation = std::min(text.find_first_not_of(' '), max_chars); + const size_t dots = std::min(3, max_chars - indentation); + text = text.substr(0, max_chars - dots) + std::string(dots, '.'); + } + return text; +} + +std::string table_route(const Report& report) { + if (!report.contains("route")) { + return "-"; + } + const Report& route = report.at("route"); + const Report& requested = route.at("requested"); + const Report& resolved = route.at("resolved"); + const Report& selected = resolved.is_null() ? requested : resolved; + const std::string requested_name = requested.at("name"); + const std::string selected_name = selected.at("name"); + std::string label = requested_name == "auto" && selected_name != "auto" + ? "auto->" + selected_name + : selected_name; + const Report selected_options = selected.value("options", Report::object()); + if (selected_options.contains("intermediate_storage")) { + label += "[" + + selected_options.at("intermediate_storage").get() + "]"; + } + return table_text(label, 32); +} + +} // namespace + +std::string format_report(const Report& report) { + std::ostringstream output; + write_report_value(output, report, 0); + return output.str(); +} + +std::string format_condensed_report(const Report& report) { + auto condense_case = [](const Report& original) { + if (!original.is_object()) + return original; + Report condensed = Report::object(); + for (const auto& item : original.items()) { + const std::string& key = item.key(); + const Report& value = item.value(); + const bool batch_timing = key == "gpu_timestamp_operator_us" || + key == "gpu_timestamp_batch_us" || key == "wall_time_operator_us" || + key == "wall_time_batch_us" || key == "wall_time_shader_us"; + if (key == "args") { + condensed["args_str"] = table_text(table_inputs(original, true)); + } else if (batch_timing && value.is_array()) { + const Report stats = + latency_statistics(value.get>()); + condensed[key] = stats.is_null() ? Report() + : Report{ + {"min", stats.at("min")}, + {"median", stats.at("p50")}, + {"max", stats.at("max")}}; + } else if ( + std::find( + kObservedGpuClockFields.begin(), + kObservedGpuClockFields.end(), + key) != kObservedGpuClockFields.end()) { + std::array, 2> range{}; + observe_gpu_clock_range(range, value); + condensed[key] = range[0] + ? Report{{"min", *range[0]}, {"max", *range[1]}} + : Report(); + } else { + condensed[key] = value; + } + } + return condensed; + }; + if (!report.is_array()) + return format_report(condense_case(report)); + Report condensed = Report::array(); + for (const auto& item : report) + condensed.push_back(condense_case(item)); + return format_report(condensed); +} + +std::string format_table(const Report& reports, const TableOptions& options) { + const bool show_route = + std::any_of(reports.begin(), reports.end(), [](const Report& report) { + return report.contains("route"); + }); + const bool show_validation = + std::any_of(reports.begin(), reports.end(), [](const Report& report) { + return report.contains("validation"); + }); + const bool show_clocks = + std::any_of(reports.begin(), reports.end(), [](const Report& report) { + return std::any_of( + kObservedGpuClockFields.begin(), + kObservedGpuClockFields.end(), + [&](const char* field) { return report.contains(field); }); + }); + auto has_throughput = [&](const char* metric) { + return std::any_of( + reports.begin(), reports.end(), [&](const Report& report) { + return report.contains("throughput") && + report.at("throughput").contains(metric); + }); + }; + std::vector columns = {0}; + if (show_route) { + columns.push_back(9); + } + columns.insert(columns.end(), {1, 2, 3, 4, 5}); + if (show_clocks) { + columns.insert(columns.end(), {10, 11}); + } + if (show_validation) { + columns.push_back(6); + } + if (has_throughput("memory_bytes")) { + columns.push_back(7); + } + if (has_throughput("compute_flops")) { + columns.push_back(8); + } + using Row = std::array; + std::vector rows = { + {"Shader / operator", + "GWG", + "LWG", + "Inputs", + "Mean GPU us", + "P90 GPU us", + "Validation", + "GPU GiB/s", + "GPU GFLOPS", + "Route", + "Min GPU MHz", + "Max GPU MHz"}}; + std::vector case_rows; + std::vector errors; + for (const auto& report : reports) { + const bool op = report.contains("operator_name"); + const bool check = report.contains("check_name"); + const std::string name = report.at( + check ? "check_name" + : op ? "operator_name" + : "shader_name"); + const std::string suffix = report.contains("sweep_index") + ? " [" + std::to_string(report.at("sweep_index").get()) + "]" + : ""; + const bool skipped = report.value("status", "") == "SKIP"; + const std::string label = table_text( + name + suffix + (skipped ? " (SKIP)" : ""), options.name_max_chars); + const std::string route = table_route(report); + const std::string validation = !skipped && report.contains("validation") + ? table_text(report.at("validation").at("status").get()) + : "SKIP"; + case_rows.push_back(rows.size()); + if (skipped) { + const std::string inputs = report.contains(op ? "args" : "bindings") + ? table_inputs(report, op) + : "-"; + rows.push_back( + {label, + "-", + "-", + table_text(inputs, options.inputs_max_chars), + "N/A", + "N/A", + "SKIP", + "N/A", + "N/A", + route, + "N/A", + "N/A"}); + errors.push_back(table_text( + "SKIP " + report.value("case_name", name) + suffix + ": " + + report.value("skip_reason", ""))); + continue; + } + if (report.contains("error")) { + rows.push_back( + {label, + "-", + "-", + "ERROR", + "N/A", + "N/A", + validation, + "N/A", + "N/A", + route, + "N/A", + "N/A"}); + errors.push_back(table_text( + "ERROR " + report.value("case_name", name) + suffix + ": " + + report.at("error").get())); + continue; + } + if (check) { + rows.push_back( + {label, + "-", + "-", + "-", + "N/A", + "N/A", + validation, + "N/A", + "N/A", + "-", + "N/A", + "N/A"}); + if (report.at("validation").value("status", "") == "FAIL") + errors.push_back(table_text( + "FAIL " + report.value("case_name", name) + ": " + + report.at("validation").value("error", "Case check failed"))); + continue; + } + const auto& timing = + report.at(op ? "gpu_timestamp_operator_us" : "gpu_timestamp_shader_us"); + const Report stats = timing.is_array() + ? latency_statistics(timing.get>()) + : timing; + const auto clocks = table_gpu_clock_range(report); + rows.push_back( + {label, + op ? "-" : report.at("gwg_size").at("value").dump(), + op ? "-" : report.at("lwg_size").at("value").dump(), + table_text(table_inputs(report, op), options.inputs_max_chars), + table_latency(stats, "mean"), + table_latency(stats, "p90"), + validation, + table_gpu_throughput(report, "memory_gib_per_s"), + table_gpu_throughput(report, "compute_gflops"), + route, + clocks[0], + clocks[1]}); + if (op && report.contains("shader_breakdown")) { + const auto profile_clocks = table_gpu_clock_range(report, true); + for (const auto& shader : report.at("shader_breakdown")) { + const auto& shader_stats = shader.at("gpu_timestamp_shader_us"); + rows.push_back( + {table_text( + " " + shader.at("shader_name").get(), + options.name_max_chars), + shader.at("gwg_size").at("value").dump(), + shader.at("lwg_size").at("value").dump(), + "-", + table_latency(shader_stats, "mean"), + table_latency(shader_stats, "p90"), + "-", + table_gpu_throughput(shader, "memory_gib_per_s"), + table_gpu_throughput(shader, "compute_gflops"), + "-", + profile_clocks[0], + profile_clocks[1]}); + } + } + } + std::array widths{}; + for (const auto& row : rows) { + for (size_t i : columns) { + widths[i] = std::max(widths[i], row[i].size()); + } + } + const size_t line_width = + std::accumulate(widths.begin(), widths.end(), size_t{0}) + + 2 * (columns.size() - 1); + const std::string separator(line_width, '-'); + std::ostringstream output; + for (size_t r = 0; r < rows.size(); ++r) { + if (r > 0) { + output << '\n'; + } + if (std::binary_search(case_rows.begin(), case_rows.end(), r)) { + output << separator << '\n'; + } + for (size_t i : columns) { + if (i != columns.front()) { + output << " "; + } + output << (i < 4 || i == 6 || i == 9 ? std::left : std::right) + << std::setw(static_cast(widths[i])) << rows[r][i]; + } + } + for (const auto& error : errors) { + output << '\n' << error; + } + return output.str(); +} + +Report latency_statistics(std::vector samples) { + if (samples.empty()) { + return nullptr; + } + std::sort(samples.begin(), samples.end()); + return { + {"min", samples.front()}, + {"max", samples.back()}, + {"mean", + std::accumulate(samples.begin(), samples.end(), 0.0) / samples.size()}, + {"p50", percentile(samples, 0.5)}, + {"p90", percentile(samples, 0.9)}, + {"p95", percentile(samples, 0.95)}}; +} + +Report measurement_report( + uint32_t repetitions, + bool operator_case, + const std::vector& gpu_batches, + const std::vector& wall_batches) { + if (repetitions == 0) { + throw std::invalid_argument("Repetitions must be positive"); + } + std::vector gpu_average = gpu_batches; + std::vector wall_average = wall_batches; + for (double& value : gpu_average) { + value /= repetitions; + } + for (double& value : wall_average) { + value /= repetitions; + } + const Report gpu_latency = operator_case + ? Report(gpu_average) + : latency_statistics(std::move(gpu_average)); + return { + {operator_case ? "gpu_timestamp_operator_us" : "gpu_timestamp_shader_us", + gpu_latency}, + {"gpu_timestamp_batch_us", gpu_batches}, + {operator_case ? "wall_time_operator_us" : "wall_time_shader_us", + wall_average}, + {"wall_time_batch_us", wall_batches}}; +} + +Report throughput_report( + const Json& spec, + const WorkAmounts& work, + uint32_t repetitions, + const std::vector& gpu_batches, + const std::vector& wall_batches) { + const bool memory = + spec.is_object() && spec.value("show_memory_throughput", false); + const bool compute = + spec.is_object() && spec.value("show_compute_throughput", false); + Report result = Report::object(); + if (!memory && !compute) { + return result; + } + if (repetitions == 0) { + throw std::invalid_argument("Repetitions must be positive"); + } + auto set_work = [&](const char* key, const std::optional& amount) { + if (amount && (!std::isfinite(*amount) || *amount < 0)) { + throw std::invalid_argument("Work counts must be finite and nonnegative"); + } + result[key] = amount ? Report(*amount) : Report(); + }; + if (memory) { + set_work("memory_bytes", work.memory_bytes); + } + if (compute) { + set_work("compute_flops", work.compute_flops); + } + auto metrics = [&](const std::vector& batches, bool statistics) { + Report output = Report::object(); + auto add_metric = [&](const char* key, + const std::optional& amount, + double units) { + if (!amount) { + output[key] = nullptr; + return; + } + Report rates = Report::array(); + std::vector samples; + bool valid = true; + for (double batch_us : batches) { + const double invocation_us = batch_us / repetitions; + if (invocation_us <= 0 || !std::isfinite(invocation_us)) { + rates.push_back(nullptr); + valid = false; + continue; + } + const double rate = *amount / units * (1e6 / invocation_us); + if (std::isfinite(rate)) { + rates.push_back(rate); + samples.push_back(rate); + } else { + rates.push_back(nullptr); + valid = false; + } + } + output[key] = statistics + ? (valid ? latency_statistics(std::move(samples)) : Report()) + : std::move(rates); + }; + if (memory) { + add_metric("memory_gib_per_s", work.memory_bytes, 1073741824.0); + add_metric("memory_mib_per_s", work.memory_bytes, 1048576.0); + } + if (compute) { + add_metric("compute_gflops", work.compute_flops, 1e9); + } + return output; + }; + result["gpu_timestamp"] = metrics(gpu_batches, true); + result["gpu_timestamp_batch"] = result.at("gpu_timestamp"); + result["wall_time"] = metrics(wall_batches, false); + result["wall_time_batch"] = result.at("wall_time"); + if (!work.diagnostics.empty()) { + result["diagnostics"] = work.diagnostics; + } + return result; +} + +Report shader_report( + const Json& spec, + const Json& gwg, + const Json& lwg, + const Json& actual_dtypes, + const Report& constants) { + Report report = case_header(spec); + report["gwg_size"] = workgroup_report(spec.at("gwg_size"), gwg); + report["lwg_size"] = workgroup_report(spec.at("lwg_size"), lwg); + report["bindings"] = describe_bindings(spec, actual_dtypes); + report["specialization_constants"] = constants; + if (spec.contains("push_constants")) { + report["push_constants"] = spec.at("push_constants"); + } + return report; +} + +Report shader_breakdown_report( + const Report& dispatches, + const std::vector>& samples, + uint32_t repetitions, + const Json& throughput_flags, + const std::vector& work) { + if (repetitions == 0) { + throw std::invalid_argument("Repetitions must be positive"); + } + for (const auto& sample : samples) { + if (sample.size() != dispatches.size() * repetitions) { + throw std::invalid_argument( + "Shader timestamp count changed between runs"); + } + } + Report result = dispatches; + if (!work.empty() && work.size() != dispatches.size()) { + throw std::invalid_argument( + "Shader work count does not match dispatch count"); + } + for (size_t i = 0; i < dispatches.size(); ++i) { + std::vector durations; + durations.reserve(samples.size()); + for (const auto& sample : samples) { + double total = 0; + for (uint32_t r = 0; r < repetitions; ++r) { + total += sample[r * dispatches.size() + i]; + } + durations.push_back(total / repetitions); + } + result[i]["gpu_timestamp_shader_us"] = latency_statistics(durations); + Report throughput = throughput_report( + throughput_flags, + work.empty() ? WorkAmounts{} : work[i], + 1, + durations, + {}); + if (!throughput.empty()) { + throughput.erase("gpu_timestamp_batch"); + throughput.erase("wall_time"); + throughput.erase("wall_time_batch"); + result[i]["throughput"] = std::move(throughput); + } + } + return result; +} + +Report operator_report( + const Json& spec, + const Json& actual_dtypes, + const Json& actual_accesses) { + Report report = case_header(spec); + report["args"] = Report::array(); + for (const Json& arg : spec.at("args")) { + if (arg.contains("tensor")) { + const std::string name = arg.at("tensor"); + const Json& tensor = spec.at("tensors").at(name); + Report description = {{"name", name}}; + description.update(tensor_description( + tensor, actual_dtypes.value(name, tensor.value("dtype", "float32")))); + description["access"] = + arg.value("access", actual_accesses.value(name, Json("read"))); + report["args"].push_back({{"tensor", std::move(description)}}); + } else { + report["args"].push_back(Report(arg)); + } + } + return report; +} + +Report error_report(const Json& spec, const std::string& error) { + Report report = case_header(spec); + report["error"] = error; + Report throughput = throughput_report( + spec, + {std::nullopt, std::nullopt, {"No timing measurements"}}, + 1, + {}, + {}); + if (!throughput.empty()) { + report["throughput"] = std::move(throughput); + } + return report; +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/Report.h b/backends/vulkan/test/ops/benchmark_v2/framework/Report.h new file mode 100644 index 00000000000..e4751b1f6f9 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/Report.h @@ -0,0 +1,61 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +#include +#include +#include +#include + +namespace vkcompute::shader_profiling { + +using Report = nlohmann::ordered_json; + +struct TableOptions { + size_t name_max_chars = 32; + size_t inputs_max_chars = 64; +}; + +std::string format_report(const Report& report); +std::string format_condensed_report(const Report& report); +std::string format_table( + const Report& reports, + const TableOptions& options = {}); +Report latency_statistics(std::vector samples); +Report measurement_report( + uint32_t repetitions, + bool operator_case, + const std::vector& gpu_batches, + const std::vector& wall_batches); +Report throughput_report( + const Json& spec, + const WorkAmounts& work, + uint32_t repetitions, + const std::vector& gpu_batches, + const std::vector& wall_batches); +Report shader_breakdown_report( + const Report& dispatches, + const std::vector>& samples, + uint32_t repetitions = 1, + const Json& throughput_flags = Json::object(), + const std::vector& work = {}); +Report shader_report( + const Json& spec, + const Json& gwg, + const Json& lwg, + const Json& actual_dtypes, + const Report& constants); +Report operator_report( + const Json& spec, + const Json& actual_dtypes, + const Json& actual_accesses = Json::object()); +Report error_report(const Json& spec, const std::string& error); + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/RouteRegistry.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/RouteRegistry.cpp new file mode 100644 index 00000000000..c1af2b3ac0b --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/RouteRegistry.cpp @@ -0,0 +1,95 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +namespace vkcompute::shader_profiling { + +RouteBuildFunction RouteRegistry::operator_route( + const std::string& op, + const std::string& route) const { + const auto it = operators_.find(op); + if (it == operators_.end() || !it->second.count(route)) { + throw std::invalid_argument("Unknown route for " + op + ": " + route); + } + return it->second.at(route); +} + +RouteInspectFunction RouteRegistry::operator_inspector( + const std::string& op) const { + const auto it = inspectors_.find(op); + return it == inspectors_.end() ? nullptr : it->second; +} + +void RouteRegistry::register_operator_route( + const std::string& op, + const std::string& route, + RouteBuildFunction function) { + if (op.empty() || route.empty() || route == "auto" || !function) { + throw std::invalid_argument( + "Route requires operator/route names and a function; auto is reserved"); + } + if (!operators_[op].emplace(route, function).second) { + throw std::invalid_argument("Duplicate route for " + op + ": " + route); + } +} + +void RouteRegistry::register_operator_inspector( + const std::string& op, + RouteInspectFunction function) { + if (op.empty() || !function) { + throw std::invalid_argument("Route inspector requires a name and function"); + } + if (!inspectors_.emplace(op, function).second) { + throw std::invalid_argument("Duplicate route inspector: " + op); + } +} + +RouteRegistry& route_registry() { + static RouteRegistry registry; + return registry; +} + +Json build_operator_route( + const Json& spec, + ComputeGraph& graph, + const std::vector& args) { + const std::string op = spec.at("operator"); + const Json requested = + spec.value("route", Json{{"name", "auto"}, {"options", Json::object()}}); + const std::string route = requested.at("name"); + const Json& options = requested.at("options"); + auto& registry = route_registry(); + Json resolved; + if (route == "auto") { + if (!options.empty()) { + throw std::invalid_argument("auto route does not accept options"); + } + const size_t first = graph.execute_nodes().size(); + VK_GET_OP_FN(op)(graph, args); + if (const auto inspector = registry.operator_inspector(op)) { + resolved = inspector(graph, args, first, graph.execute_nodes().size()); + } + } else { + const auto builder = registry.operator_route(op, route); + resolved = builder(graph, args, options); + } + if (route != "auto" && resolved.is_null()) { + throw std::invalid_argument("Route builder returned no resolved metadata"); + } + if (!resolved.is_null() && + (!resolved.is_object() || !resolved.contains("name") || + !resolved.at("name").is_string() || + resolved.at("name").get().empty() || + !resolved.contains("options") || !resolved.at("options").is_object())) { + throw std::invalid_argument("Route returned invalid resolved metadata"); + } + return {{"requested", requested}, {"resolved", resolved}}; +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/RouteRegistry.h b/backends/vulkan/test/ops/benchmark_v2/framework/RouteRegistry.h new file mode 100644 index 00000000000..a583bba9075 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/RouteRegistry.h @@ -0,0 +1,77 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +#include +#include +#include +#include +#include + +namespace vkcompute::shader_profiling { + +class UnsupportedRoute : public std::runtime_error { + public: + using std::runtime_error::runtime_error; +}; + +using RouteBuildFunction = + Json (*)(ComputeGraph&, const std::vector&, const Json&); +using RouteInspectFunction = + Json (*)(ComputeGraph&, const std::vector&, size_t, size_t); + +class RouteRegistry final { + private: + std::unordered_map< + std::string, + std::unordered_map> + operators_; + std::unordered_map inspectors_; + + public: + RouteBuildFunction operator_route( + const std::string& op, + const std::string& route) const; + RouteInspectFunction operator_inspector(const std::string& op) const; + void register_operator_route( + const std::string& op, + const std::string& route, + RouteBuildFunction function); + void register_operator_inspector( + const std::string& op, + RouteInspectFunction function); +}; + +class RouteRegisterInit final { + public: + explicit RouteRegisterInit(void (*initialize)()) { + initialize(); + } +}; + +RouteRegistry& route_registry(); +Json build_operator_route( + const Json& spec, + ComputeGraph& graph, + const std::vector& args); + +} // namespace vkcompute::shader_profiling + +#define REGISTER_OPERATOR_ROUTE(op, route, function) \ + ::vkcompute::shader_profiling::route_registry().register_operator_route( \ + #op, #route, &function) +#define REGISTER_ROUTE_INSPECTOR(op, function) \ + ::vkcompute::shader_profiling::route_registry().register_operator_inspector( \ + #op, &function) +#define REGISTER_OPERATOR_ROUTES \ + static void register_routes(); \ + static const ::vkcompute::shader_profiling::RouteRegisterInit route_reg( \ + ®ister_routes); \ + static void register_routes() diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/ShaderBenchmark.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/ShaderBenchmark.cpp new file mode 100644 index 00000000000..84c577a3758 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/ShaderBenchmark.cpp @@ -0,0 +1,302 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace vkcompute::shader_profiling { + +namespace { + +std::vector packed_blocks(const Json& blocks) { + std::vector bytes; + for (const Json& block : blocks) { + if (block.value("dtype", "float32") == "float16") { + throw std::invalid_argument("Parameter blocks do not support float16"); + } + const auto data = make_data(block, block.at("data").size()); + bytes.insert(bytes.end(), data.begin(), data.end()); + } + if (bytes.size() % 4 != 0) { + throw std::invalid_argument( + "Parameter blocks must be padded to a multiple of four bytes"); + } + return bytes; +} + +} // namespace + +Report benchmark_shader( + const Json& spec, + const TensorData& host_data, + bool validation_only) { + GraphConfig config; + config.force_resize = true; + ComputeGraph graph(config); + const auto tensors = create_tensors(spec, graph, host_data, validation_only); + + const std::string shader_name = spec.at("shader"); + vkapi::ShaderInfo shader = VK_KERNEL_FROM_STR(shader_name); + // JSON describes shader invocations, rather than logical output elements. + shader.out_tile_size = LocalWorkGroup(1, 1, 1); + api::Context& context = *graph.context(); + context.check_device_capabilities(shader); + const Json& bindings = spec.at("bindings"); + if (bindings.size() != shader.kernel_layout.size()) { + throw std::invalid_argument( + "Binding count does not match shader signature"); + } + std::vector uniforms; + std::map params; + std::map image_bindings; + for (size_t i = 0; i < bindings.size(); ++i) { + const Json& binding = bindings.at(i); + const VkDescriptorType expected = shader.kernel_layout.at(i); + if (binding.contains("tensor")) { + const ValueRef tensor = + tensors.at(binding.at("tensor").get()); + const bool buffer = graph.is_buffer_storage(tensor); + const std::string access = binding.value("access", "read"); + if ((buffer && expected != VK_DESCRIPTOR_TYPE_STORAGE_BUFFER) || + (!buffer && expected != VK_DESCRIPTOR_TYPE_STORAGE_IMAGE && + expected != VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER) || + (expected == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER && + access != "read")) { + throw std::invalid_argument( + "Tensor storage/access does not match shader binding " + + std::to_string(i)); + } + if (!buffer) { + const auto previous = image_bindings.emplace(tensor, expected); + if (!previous.second && previous.first->second != expected) { + throw std::invalid_argument( + "Sampled and storage bindings must use separate texture tensors"); + } + } + } else { + if (expected != VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER) { + throw std::invalid_argument( + "Expected uniform buffer at binding " + std::to_string(i)); + } + if (binding.contains("meta_ubo")) { + params.emplace( + i, + meta_ubo( + graph, + tensors.at(binding.at("meta_ubo").get()), + binding.value("field", "meta"))); + } else { + const auto bytes = packed_blocks(binding.at("uniform")); + if (bytes.empty()) { + throw std::invalid_argument("Uniform buffer must not be empty"); + } + uniforms.push_back( + context.adapter_ptr()->vma().create_uniform_buffer(bytes.size())); + vkapi::MemoryMap map(uniforms.back(), vkapi::kWrite); + std::memcpy(map.data(), bytes.data(), bytes.size()); + params.emplace(i, vkapi::BufferBindInfo(uniforms.back())); + } + } + } + vkapi::SpecVarList constants; + for (const Json& constant : + spec.value("specialization_constants", Json::array())) { + if (constant.contains("layout")) { + constants.append(vkapi::SpecVar(graph.hashed_layout_of( + tensors.at(constant.at("layout").get())))); + } else { + const std::string dtype = constant.value("dtype", "int32"); + if (dtype == "bool") { + constants.append(vkapi::SpecVar(constant.at("value").get())); + } else { + const auto bytes = + make_data({{"dtype", dtype}, {"data", {constant.at("value")}}}, 1); + if (bytes.size() != 4) { + throw std::invalid_argument( + "Specialization constants must be 32-bit or bool"); + } + vkapi::SpecVar value; + if (dtype == "float32") { + value.type = vkapi::SpecVar::Type::FLOAT; + } else if (dtype == "int32") { + value.type = vkapi::SpecVar::Type::INT; + } else if (dtype == "uint32") { + value.type = vkapi::SpecVar::Type::UINT; + } else { + throw std::invalid_argument( + "Unsupported specialization constant dtype"); + } + std::memcpy(&value.value, bytes.data(), 4); + constants.append(value); + } + } + } + const auto push_constants = + packed_blocks(spec.value("push_constants", Json::array())); + if (push_constants.size() > 128) { + throw std::invalid_argument( + "Prototype supports up to 128 bytes of push constants"); + } + const auto workgroups = pick_workgroup_sizes(spec, graph, shader, tensors); + const GlobalWorkGrid& gwg = workgroups.gwg; + const LocalWorkGroup& lwg = workgroups.lwg; + + const size_t input_nodes = graph.execute_nodes().size(); + std::map outputs; + if (validation_only) { + Json accesses = Json::object(); + for (const auto& binding : bindings) { + if (binding.contains("tensor") && + binding.value("access", "read") != "read") { + accesses[binding.at("tensor").get()] = + binding.at("access"); + } + } + outputs = add_validation_outputs(spec, graph, tensors, accesses); + } + + graph.update_descriptor_counts(shader, true); + graph.prepare(); + graph.prepack(); + graph.propagate_resize(); + if (validation_only) { + copy_validation_inputs(spec, graph, tensors, host_data); + } + + const VkDescriptorSetLayout shader_layout = + context.shader_layout_cache().retrieve(shader.kernel_layout); + const VkPipelineLayout pipeline_layout = + context.pipeline_layout_cache().retrieve( + shader_layout, static_cast(push_constants.size())); + vkapi::SpecVarList pipeline_constants = {SV(lwg[0]), SV(lwg[1]), SV(lwg[2])}; + pipeline_constants.append(constants); + const VkPipeline pipeline = context.pipeline_cache().retrieve( + {pipeline_layout, + context.shader_cache().retrieve(shader), + pipeline_constants, + vkapi::resolve_required_subgroup_size(shader, context.adapter_ptr())}); + + vkapi::VulkanFence fence(context.device()); + auto bind_resources = [&](vkapi::DescriptorSet& descriptors, + vkapi::PipelineBarrier& barrier) { + for (size_t i = 0; i < bindings.size(); ++i) { + const Json& binding = bindings.at(i); + if (binding.contains("tensor")) { + const std::string access = binding.value("access", "read"); + const vkapi::MemoryAccessFlags flags = access == "read" ? vkapi::kRead + : access == "write" ? vkapi::kWrite + : vkapi::kReadWrite; + graph.bind_value_to_descriptor_set( + tensors.at(binding.at("tensor").get()), + barrier, + flags, + descriptors, + static_cast(i)); + } else { + descriptors.bind(static_cast(i), params.at(i)); + } + } + }; + if (!validation_only) { + // Initial image layout transitions must not be replayed. + context.set_cmd(); + auto initial_descriptors = context.descriptor_pool().get_descriptor_set( + shader_layout, shader.kernel_layout); + vkapi::PipelineBarrier transitions{}; + bind_resources(initial_descriptors, transitions); + context.register_barrier(transitions); + context.submit_cmd_to_gpu(fence.get_submit_handle(), true); + fence.wait(); + } + + auto encode = [&](uint32_t count) { + auto descriptors = context.descriptor_pool().get_descriptor_set( + shader_layout, shader.kernel_layout); + vkapi::PipelineBarrier initial{}; + bind_resources(descriptors, initial); + context.register_barrier(initial); + for (uint32_t i = 0; i < count; ++i) { + context.extract_cmd().bind_pipeline(pipeline, pipeline_layout, lwg); + // Serialize repeats, including across submissions of the cached buffer. + vkapi::PipelineBarrier barrier{}; + barrier.stage = { + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_HOST_BIT, + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT}; + bind_resources(descriptors, barrier); + context.register_shader_dispatch( + descriptors, + barrier, + shader, + gwg, + lwg, + push_constants.data(), + static_cast(push_constants.size())); + } + }; + if (validation_only) { + context.set_cmd(); + for (size_t i = 0; i < input_nodes; ++i) { + graph.execute_nodes()[i]->encode(&graph); + } + encode(1); + for (size_t i = input_nodes; i < graph.execute_nodes().size(); ++i) { + graph.execute_nodes()[i]->encode(&graph); + } + context.submit_cmd_to_gpu(fence.get_submit_handle(), true); + fence.wait(); + } + const Json definitions = + spec.value("specialization_constants", Json::array()); + Report specialization = Report::object(); + for (uint32_t i = 0; i < constants.size(); ++i) { + const vkapi::SpecVar& value = constants.at(i); + Report entry; + switch (value.type) { + case vkapi::SpecVar::Type::FLOAT: + entry = {{"dtype", "float32"}, {"value", value.value.as_float}}; + break; + case vkapi::SpecVar::Type::INT: + entry = {{"dtype", "int32"}, {"value", value.value.as_int32}}; + break; + case vkapi::SpecVar::Type::UINT: + entry = {{"dtype", "uint32"}, {"value", value.value.as_uint32}}; + break; + case vkapi::SpecVar::Type::BOOL: + entry = {{"dtype", "bool"}, {"value", value.value.as_bool}}; + break; + } + if (definitions.at(i).contains("layout")) { + entry["layout"] = definitions.at(i).at("layout"); + } + specialization[std::to_string(i + 3)] = std::move(entry); + } + Report result = shader_report( + spec, + {gwg[0], gwg[1], gwg[2]}, + {lwg[0], lwg[1], lwg[2]}, + actual_tensor_dtypes(graph, tensors), + specialization); + if (validation_only) { + result["validation"] = + validate_outputs(spec, graph, tensors, outputs, host_data); + } else { + result.update(measure(spec, graph, encode, Json(result))); + } + return result; +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/ShaderBenchmark.h b/backends/vulkan/test/ops/benchmark_v2/framework/ShaderBenchmark.h new file mode 100644 index 00000000000..681558bc8eb --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/ShaderBenchmark.h @@ -0,0 +1,19 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +namespace vkcompute::shader_profiling { + +Report benchmark_shader( + const Json& spec, + const TensorData& host_data, + bool validation_only = false); + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/Sweep.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/Sweep.cpp new file mode 100644 index 00000000000..017da8b2ac5 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/Sweep.cpp @@ -0,0 +1,268 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include +#include +#include + +namespace vkcompute::shader_profiling { +namespace { + +constexpr size_t kMaxSweepCases = 65536; + +void require(bool condition, const std::string& message) { + if (!condition) { + throw std::invalid_argument(message); + } +} + +Json substitute(const Json& value, const Json& variables) { + if (value.is_string()) { + const std::string name = value.get(); + if (name.compare(0, 2, "v_") == 0) { + require(variables.contains(name), "Undefined sweep variable: " + name); + return variables.at(name); + } + } + Json result = value; + if (result.is_structured()) { + for (auto& item : result) { + item = substitute(item, variables); + } + } + return result; +} + +std::vector combinations(const Json& variables, size_t max_cases) { + require(variables.is_object(), "sweep_variables must be an object"); + std::vector result{Json::object()}; + for (const auto& variable : variables.items()) { + const std::string& name = variable.key(); + require( + name.size() > 2 && name.compare(0, 2, "v_") == 0, + "Sweep variable names must start with v_"); + require( + variable.value().is_array() && !variable.value().empty(), + "Sweep variable choices must be a nonempty array: " + name); + require( + variable.value().size() <= max_cases / result.size(), + "Sweep must not exceed 65536 cases"); + std::vector next; + for (const Json& combination : result) { + for (const Json& value : variable.value()) { + substitute(value, Json::object()); + Json entry = combination; + entry[name] = value; + next.push_back(std::move(entry)); + } + } + result = std::move(next); + } + return result; +} + +Json normalize_route(Json route) { + if (route.is_string()) { + route = {{"name", route}}; + } + require(route.is_object(), "route must be a name or an object"); + for (const auto& field : route.items()) { + require( + field.key() == "name" || field.key() == "options", + "route accepts only name and options"); + } + require( + route.contains("name") && route.at("name").is_string() && + !route.at("name").get().empty(), + "route.name must be a nonempty string"); + route["options"] = route.value("options", Json::object()); + require(route.at("options").is_object(), "route.options must be an object"); + return route; +} + +} // namespace + +std::vector expand_operator_sweep(const Json& definition) { + require(!definition.contains("args"), "Use args_sweep instead of args"); + require( + definition.contains("args_sweep") && + definition.at("args_sweep").is_array() && + !definition.at("args_sweep").empty(), + "args_sweep must be a nonempty list of argument sets"); + require( + !(definition.contains("route") && definition.contains("route_sweep")), + "Specify only one of route and route_sweep"); + const bool has_route = + definition.contains("route") || definition.contains("route_sweep"); + Json routes = Json::array({nullptr}); + if (definition.contains("route_sweep")) { + routes = definition.at("route_sweep"); + require( + routes.is_array() && !routes.empty(), + "route_sweep must be a nonempty list of routes"); + } else if (definition.contains("route")) { + routes = Json::array({definition.at("route")}); + } + const Json& sweep = definition.at("args_sweep"); + require( + routes.size() <= kMaxSweepCases && + sweep.size() <= kMaxSweepCases / routes.size(), + "Sweep must not exceed 65536 cases"); + const auto choices = combinations( + definition.value("sweep_variables", Json::object()), + kMaxSweepCases / (sweep.size() * routes.size())); + Json common = definition; + common.erase("args_sweep"); + common.erase("sweep_variables"); + common.erase("route"); + common.erase("route_sweep"); + std::vector result; + result.reserve(sweep.size() * choices.size() * routes.size()); + for (size_t i = 0; i < sweep.size(); ++i) { + require( + sweep.at(i).is_array(), + "Each args_sweep entry must be an argument array"); + common["args"] = sweep.at(i); + for (const Json& choice : choices) { + Json selected = substitute(common, choice); + if (!choice.empty()) { + selected["args_sweep_index"] = i; + selected["sweep_variables"] = choice; + } + for (const Json& route : routes) { + Json entry = selected; + if (has_route) { + entry["route"] = normalize_route(substitute(route, choice)); + } + entry["sweep_index"] = result.size(); + result.push_back(std::move(entry)); + } + } + } + return result; +} + +namespace { + +std::vector expand_lwg_sweep(const Json& definition, size_t max_cases) { + require( + !(definition.contains("lwg_size") && + definition.contains("lwg_size_sweep")), + "Specify only one of lwg_size and lwg_size_sweep"); + Json sizes; + if (definition.contains("lwg_size_sweep")) { + const Json& sweep = definition.at("lwg_size_sweep"); + require(sweep.is_object(), "lwg_size_sweep must be an object"); + for (const auto& axis : sweep.items()) { + require( + axis.key() == "x" || axis.key() == "y" || axis.key() == "z", + "lwg_size_sweep accepts only x, y, z"); + } + std::array starts{1, 1, 1}; + std::array steps{1, 1, 1}; + std::array counts{1, 1, 1}; + size_t count = 1; + for (size_t i = 0; i < 3; ++i) { + const char* axis = i == 0 ? "x" : i == 1 ? "y" : "z"; + if (!sweep.contains(axis)) { + continue; + } + const Json& range = sweep.at(axis); + require( + range.is_array() && range.size() == 3, + std::string("lwg_size_sweep.") + axis + + " must be [start, stop, step]"); + for (const Json& value : range) { + require( + value.is_number_integer() && value.get() > 0 && + value.get() <= UINT32_MAX, + "Local workgroup range values must be positive uint32 integers"); + } + starts[i] = range.at(0).get(); + const uint32_t stop = range.at(1).get(); + steps[i] = range.at(2).get(); + require(starts[i] <= stop, "Local workgroup range start exceeds stop"); + const uint64_t length = (uint64_t(stop) - starts[i]) / steps[i] + 1; + require(length <= max_cases / count, "Sweep must not exceed 65536 cases"); + counts[i] = static_cast(length); + count *= counts[i]; + } + sizes = Json::array(); + for (size_t x = 0; x < counts[0]; ++x) { + for (size_t y = 0; y < counts[1]; ++y) { + for (size_t z = 0; z < counts[2]; ++z) { + sizes.push_back( + {starts[0] + uint64_t(x) * steps[0], + starts[1] + uint64_t(y) * steps[1], + starts[2] + uint64_t(z) * steps[2]}); + } + } + } + } else if ( + definition.contains("lwg_size") && definition.at("lwg_size").is_array() && + !definition.at("lwg_size").empty() && + definition.at("lwg_size").front().is_array()) { + sizes = definition.at("lwg_size"); + } else { + return {definition}; + } + require(sizes.size() <= max_cases, "Sweep must not exceed 65536 cases"); + std::vector result; + result.reserve(sizes.size()); + for (const Json& size : sizes) { + require( + size.is_array() && size.size() == 3, + "Each lwg_size sweep entry must contain three positive integers"); + Json entry = definition; + entry.erase("lwg_size_sweep"); + entry["lwg_size"] = size; + entry["sweep_index"] = result.size(); + result.push_back(std::move(entry)); + } + return result; +} + +} // namespace + +std::vector expand_shader_sweep(const Json& definition) { + if (!definition.contains("shader_sweep")) { + return expand_lwg_sweep(definition, kMaxSweepCases); + } + require( + !definition.contains("shader"), + "Specify only one of shader and shader_sweep"); + const Json& shaders = definition.at("shader_sweep"); + require( + shaders.is_array() && !shaders.empty(), + "shader_sweep must be a nonempty list of shader names"); + require( + shaders.size() <= kMaxSweepCases, "Sweep must not exceed 65536 cases"); + for (const Json& shader : shaders) { + require( + shader.is_string() && !shader.get().empty(), + "shader_sweep entries must be nonempty shader names"); + } + Json common = definition; + common.erase("shader_sweep"); + const auto workgroups = + expand_lwg_sweep(common, kMaxSweepCases / shaders.size()); + std::vector result; + result.reserve(shaders.size() * workgroups.size()); + for (const Json& shader : shaders) { + for (const Json& workgroup : workgroups) { + Json entry = workgroup; + entry["shader"] = shader; + entry["sweep_index"] = result.size(); + result.push_back(std::move(entry)); + } + } + return result; +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/Sweep.h b/backends/vulkan/test/ops/benchmark_v2/framework/Sweep.h new file mode 100644 index 00000000000..d3fbea7c8c9 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/Sweep.h @@ -0,0 +1,16 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +namespace vkcompute::shader_profiling { + +std::vector expand_operator_sweep(const Json& definition); +std::vector expand_shader_sweep(const Json& definition); + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/TensorUtils.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/TensorUtils.cpp new file mode 100644 index 00000000000..89c526e832e --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/TensorUtils.cpp @@ -0,0 +1,181 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +namespace vkcompute::shader_profiling { + +namespace { + +const std::map kScalarTypes = { + {"float32", vkapi::kFloat}, + {"float16", vkapi::kHalf}, + {"int32", vkapi::kInt}, + {"int8", vkapi::kChar}, + {"uint8", vkapi::kByte}, + {"int8x4", vkapi::kInt8x4}}; + +const std::map kMemoryLayouts = { + {"width", utils::kWidthPacked}, + {"height", utils::kHeightPacked}, + {"channels", utils::kChannelsPacked}, + {"4w", utils::kPackedInt8_4W}, + {"4c", utils::kPackedInt8_4C}, + {"4w4c", utils::kPackedInt8_4W4C}, + {"4h4w", utils::kPackedInt8_4H4W}, + {"4c1w", utils::kPackedInt8_4C1W}}; + +} // namespace + +vkapi::ScalarType scalar_type(const std::string& name) { + return kScalarTypes.at(name); +} + +vkapi::BufferBindInfo +meta_ubo(ComputeGraph& graph, ValueRef tensor, const std::string& field) { + if (field == "meta") { + return graph.meta_ubo(tensor); + } + if (field == "sizes") { + return graph.sizes_ubo(tensor); + } + if (field == "strides") { + return graph.strides_ubo(tensor); + } + if (field == "numel") { + return graph.numel_ubo(tensor); + } + if (field == "logical_limits") { + return graph.logical_limits_ubo(tensor); + } + throw std::invalid_argument("Unknown tensor metadata field: " + field); +} + +TensorData prepare_tensor_data(const Json& spec) { + const Json descriptions = spec.value("tensors", Json::object()); + auto host_data = make_tensor_data(descriptions); + for (const auto& item : descriptions.items()) { + const Json& tensor = item.value(); + if (tensor.value("storage", "buffer") != "constant" || + tensor.value("dtype", "float32") != "float16") { + continue; + } + auto& data = host_data.at(item.key()); + std::vector half_data(data.size() / 2); + for (size_t i = 0; i < data.size() / sizeof(float); ++i) { + float value; + std::memcpy(&value, data.data() + i * sizeof(float), sizeof(float)); + const executorch::runtime::etensor::Half half(value); + std::memcpy(half_data.data() + i * sizeof(half), &half, sizeof(half)); + } + data = std::move(half_data); + } + return host_data; +} + +std::map create_tensors( + const Json& spec, + ComputeGraph& graph, + const TensorData& host_data, + bool with_staging) { + std::map tensors; + const Json descriptions = spec.value("tensors", Json::object()); + for (const auto& item : descriptions.items()) { + const Json& tensor = item.value(); + const auto sizes = tensor.at("sizes").get>(); + const auto dtype = scalar_type(tensor.value("dtype", "float32")); + const auto& data = host_data.at(item.key()); + const std::string storage = tensor.value("storage", "buffer"); + const bool constant = storage == "constant"; + const vkapi::ScalarType source_dtype = dtype == vkapi::kHalf && !constant + ? vkapi::kFloat + : dtype == vkapi::kInt8x4 ? vkapi::kChar + : dtype; + const ValueRef source = + graph.add_tensorref(sizes, source_dtype, data.data()); + if (constant) { + tensors.emplace(item.key(), source); + continue; + } + const auto storage_type = storage == "buffer" ? utils::kBuffer + : storage == "texture2d" ? utils::kTexture2D + : utils::kTexture3D; + const ValueRef ref = graph.add_tensor( + sizes, + dtype, + storage_type, + kMemoryLayouts.at(tensor.value("layout", "width"))); + tensors.emplace(item.key(), ref); + if (with_staging) { + graph.set_input_tensor(ref, source_dtype); + } else if (dtype == vkapi::kInt8x4) { + add_prepack_int8x4_buffer_node(graph, source, ref); + } else { + add_prepack_standard_node(graph, source, ref); + } + } + return tensors; +} + +Json actual_tensor_dtypes( + ComputeGraph& graph, + const std::map& tensors) { + Json result = Json::object(); + for (const auto& tensor : tensors) { + const auto dtype = graph.dtype_of(tensor.second); + std::string name = vkapi::to_string(dtype); + for (const auto& entry : kScalarTypes) { + if (entry.second == dtype) { + name = entry.first; + break; + } + } + result[tensor.first] = name; + } + return result; +} + +Json dispatch_tensor_bindings(ComputeGraph& graph, const ExecuteNode& node) { + Json bindings = Json::object(); + size_t index = 0; + for (const auto& group : node.args()) { + for (ValueRef ref : group.refs) { + if (ref < 0 || !graph.val_is_tensor(ref)) { + ++index; + continue; + } + const std::string name = "value_" + std::to_string(ref); + const std::string storage = graph.is_buffer_storage(ref) ? "buffer" + : graph.storage_type_of(ref) == utils::kTexture2D ? "texture2d" + : "texture3d"; + const auto access = group.access; + bindings["binding_" + std::to_string(index)] = { + {"name", name}, + {"binding_index", index}, + {"type", storage}, + {"sizes", graph.sizes_of(ref)}, + {"dtype", actual_tensor_dtypes(graph, {{name, ref}}).at(name)}, + {"access", + access & vkapi::kWrite + ? (access & vkapi::kRead ? "read_write" : "write") + : "read"}}; + ++index; + } + } + return bindings; +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/TensorUtils.h b/backends/vulkan/test/ops/benchmark_v2/framework/TensorUtils.h new file mode 100644 index 00000000000..1c2c79211ea --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/TensorUtils.h @@ -0,0 +1,40 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +#include +#include +#include +#include +#include + +namespace vkcompute::shader_profiling { + +using TensorData = std::map>; + +TensorData prepare_tensor_data(const Json& spec); + +vkapi::ScalarType scalar_type(const std::string& name); + +vkapi::BufferBindInfo +meta_ubo(ComputeGraph& graph, ValueRef tensor, const std::string& field); + +std::map create_tensors( + const Json& spec, + ComputeGraph& graph, + const TensorData& host_data, + bool with_staging = false); + +Json actual_tensor_dtypes( + ComputeGraph& graph, + const std::map& tensors); +Json dispatch_tensor_bindings(ComputeGraph& graph, const ExecuteNode& node); + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/ThroughputRegistry.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/ThroughputRegistry.cpp new file mode 100644 index 00000000000..4d456dc525e --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/ThroughputRegistry.cpp @@ -0,0 +1,280 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include +#include + +namespace vkcompute::shader_profiling { +namespace { + +void require(bool condition, const std::string& message) { + if (!condition) { + throw std::invalid_argument(message); + } +} + +void associate( + std::unordered_map& table, + const std::string& name, + WorkCountFunction function, + const std::string& kind) { + require( + !name.empty() && function, + "Work count registration requires a nonempty name and callable"); + require( + table.emplace(name, function).second, + "Duplicate " + kind + " work count association: " + name); +} + +WorkCountFunction lookup( + const std::unordered_map& table, + const std::string& name, + const std::string& kind) { + const auto entry = table.find(name); + require( + entry != table.end(), + "No " + kind + " work count registered for " + name); + return entry->second; +} + +WorkCountFunction lookup_shader( + const std::unordered_map& exact, + const std::unordered_map& prefixes, + const std::string& name, + const std::string& kind) { + const auto entry = exact.find(name); + if (entry != exact.end()) { + return entry->second; + } + WorkCountFunction function = nullptr; + size_t length = 0; + for (const auto& prefix : prefixes) { + if (prefix.first.size() > length && + name.compare(0, prefix.first.size(), prefix.first) == 0) { + function = prefix.second; + length = prefix.first.size(); + } + } + require( + function != nullptr, "No " + kind + " work count registered for " + name); + return function; +} + +bool tensor_type(const Json& tensor) { + const std::string type = tensor.value("type", ""); + return type == "buffer" || type == "texture2d" || type == "texture3d" || + type == "constant"; +} + +} // namespace + +WorkCountFunction ThroughputRegistry::operator_memory( + const std::string& name) const { + const auto function = lookup(operator_memory_, name, "operator memory"); + return function; +} +WorkCountFunction ThroughputRegistry::operator_compute( + const std::string& name) const { + const auto function = lookup(operator_compute_, name, "operator compute"); + return function; +} +WorkCountFunction ThroughputRegistry::shader_memory( + const std::string& name) const { + const auto function = lookup_shader( + shader_memory_, shader_memory_prefixes_, name, "shader memory"); + return function; +} +WorkCountFunction ThroughputRegistry::shader_compute( + const std::string& name) const { + const auto function = lookup_shader( + shader_compute_, shader_compute_prefixes_, name, "shader compute"); + return function; +} + +void ThroughputRegistry::register_operator_memory( + const std::string& name, + WorkCountFunction function) { + associate(operator_memory_, name, function, "operator memory"); +} +void ThroughputRegistry::register_operator_compute( + const std::string& name, + WorkCountFunction function) { + associate(operator_compute_, name, function, "operator compute"); +} +void ThroughputRegistry::register_shader_memory( + const std::string& name, + WorkCountFunction function) { + associate(shader_memory_, name, function, "shader memory"); +} +void ThroughputRegistry::register_shader_compute( + const std::string& name, + WorkCountFunction function) { + associate(shader_compute_, name, function, "shader compute"); +} + +void ThroughputRegistry::register_shader_memory_prefix( + const std::string& prefix, + WorkCountFunction function) { + associate(shader_memory_prefixes_, prefix, function, "shader memory prefix"); +} + +void ThroughputRegistry::register_shader_compute_prefix( + const std::string& prefix, + WorkCountFunction function) { + associate( + shader_compute_prefixes_, prefix, function, "shader compute prefix"); +} + +ThroughputRegistry& throughput_registry() { + static ThroughputRegistry registry; + return registry; +} + +WorkAmounts estimate_work(const Json& spec, bool memory, bool compute) { + WorkAmounts amounts; + const auto calculate = [&](bool memory_count) -> std::optional { + try { + require( + spec.contains("operator_name") != spec.contains("shader_name"), + "Work counts require exactly one operator_name or shader_name"); + const bool op = spec.contains("operator_name"); + const std::string name = spec.at(op ? "operator_name" : "shader_name"); + const auto& registry = throughput_registry(); + const auto function = op + ? (memory_count ? registry.operator_memory(name) + : registry.operator_compute(name)) + : (memory_count ? registry.shader_memory(name) + : registry.shader_compute(name)); + const double count = function(spec); + require( + std::isfinite(count) && count >= 0, + "Work count must be finite and nonnegative for " + name); + return count; + } catch (const std::exception& error) { + amounts.diagnostics.push_back( + std::string( + memory_count ? "Memory throughput: " : "Compute throughput: ") + + error.what()); + return std::nullopt; + } + }; + if (memory) { + amounts.memory_bytes = calculate(true); + } + if (compute) { + amounts.compute_flops = calculate(false); + } + return amounts; +} + +namespace throughput { + +double tensor_elements(const Json& tensor) { + return double(tensor_numel(tensor.at("sizes"))); +} + +double tensor_bytes(const Json& tensor) { + static const std::map bytes_per_element = { + {"float32", 4}, + {"float16", 2}, + {"int32", 4}, + {"int8", 1}, + {"uint8", 1}, + {"int8x4", 1}}; + const std::string dtype = tensor.at("dtype"); + const auto element_size = bytes_per_element.find(dtype); + require( + element_size != bytes_per_element.end(), + "Unknown tensor dtype for memory work count: " + dtype); + return tensor_elements(tensor) * element_size->second; +} + +double tensor_traffic(const Json& spec) { + struct Traffic { + double bytes; + unsigned access; + }; + std::map tensors; + const auto add = [&](const Json& tensor, const std::string& fallback_name) { + require( + tensor_type(tensor), "Unknown tensor storage for memory work count"); + const std::string access = tensor.at("access"); + require( + access == "read" || access == "write" || access == "read_write", + "Unknown tensor access for memory work count: " + access); + const unsigned flags = access == "read" ? 1 : access == "write" ? 2 : 3; + const std::string name = + tensor.value("ref", tensor.value("name", fallback_name)); + const double bytes = tensor_bytes(tensor); + auto entry = tensors.emplace(name, Traffic{bytes, flags}); + require( + entry.first->second.bytes == bytes, + "Aliased tensor byte counts disagree: " + name); + entry.first->second.access |= flags; + }; + if (spec.contains("operator_name")) { + const Json& args = spec.at("args"); + for (size_t i = 0; i < args.size(); ++i) { + if (args[i].contains("tensor")) { + add(args[i].at("tensor"), "arg_" + std::to_string(i)); + } + } + } else { + for (const auto& binding : spec.at("bindings").items()) { + if (tensor_type(binding.value()) || binding.value().contains("sizes")) { + add(binding.value(), binding.key()); + } + } + } + require(!tensors.empty(), "Memory work count requires tensor bindings"); + double total = 0; + for (const auto& tensor : tensors) { + const auto& traffic = tensor.second; + total += + traffic.bytes * ((traffic.access & 1) + ((traffic.access >> 1) & 1)); + } + return total; +} + +const Json& tensor_argument(const Json& spec, size_t index) { + const Json& tensor = spec.at("args").at(index).at("tensor"); + require( + tensor.is_object() && tensor.contains("sizes"), + "Work count requires resolved tensor argument metadata"); + return tensor; +} + +const Json& output_tensor(const Json& spec) { + if (spec.contains("operator_name")) { + return tensor_argument(spec, spec.at("args").size() - 1); + } + const Json* result = nullptr; + for (const auto& binding : spec.at("bindings").items()) { + const Json& tensor = binding.value(); + if (!tensor_type(tensor)) { + continue; + } + const std::string access = tensor.at("access"); + if (access == "write" || access == "read_write") { + if (!result || + tensor.value("binding_index", SIZE_MAX) < + result->value("binding_index", SIZE_MAX)) { + result = &tensor; + } + } + } + require( + result != nullptr, + "Compute work count requires an output tensor binding"); + return *result; +} + +} // namespace throughput + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/ThroughputRegistry.h b/backends/vulkan/test/ops/benchmark_v2/framework/ThroughputRegistry.h new file mode 100644 index 00000000000..43fdf3bf50d --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/ThroughputRegistry.h @@ -0,0 +1,105 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +#include +#include +#include +#include + +namespace vkcompute::shader_profiling { + +using WorkCountFunction = double (*)(const Json&); + +class ThroughputRegistry final { + private: + std::unordered_map operator_memory_; + std::unordered_map operator_compute_; + std::unordered_map shader_memory_; + std::unordered_map shader_compute_; + std::unordered_map shader_memory_prefixes_; + std::unordered_map shader_compute_prefixes_; + + public: + WorkCountFunction operator_memory(const std::string& name) const; + WorkCountFunction operator_compute(const std::string& name) const; + WorkCountFunction shader_memory(const std::string& name) const; + WorkCountFunction shader_compute(const std::string& name) const; + void register_operator_memory( + const std::string& name, + WorkCountFunction function); + void register_operator_compute( + const std::string& name, + WorkCountFunction function); + void register_shader_memory( + const std::string& name, + WorkCountFunction function); + void register_shader_compute( + const std::string& name, + WorkCountFunction function); + void register_shader_memory_prefix( + const std::string& prefix, + WorkCountFunction function); + void register_shader_compute_prefix( + const std::string& prefix, + WorkCountFunction function); +}; + +class ThroughputRegisterInit final { + public: + explicit ThroughputRegisterInit(void (*initialize)()) { + initialize(); + } +}; + +struct WorkAmounts { + std::optional memory_bytes; + std::optional compute_flops; + std::vector diagnostics; +}; + +ThroughputRegistry& throughput_registry(); +WorkAmounts estimate_work(const Json& resolved_case, bool memory, bool compute); + +namespace throughput { + +double tensor_elements(const Json& tensor); +double tensor_bytes(const Json& tensor); +double tensor_traffic(const Json& spec); +const Json& tensor_argument(const Json& spec, size_t index); +const Json& output_tensor(const Json& spec); + +} // namespace throughput + +} // namespace vkcompute::shader_profiling + +#define REGISTER_OPERATOR_MEMORY_TRAFFIC(name, function) \ + ::vkcompute::shader_profiling::throughput_registry() \ + .register_operator_memory(#name, &function) +#define REGISTER_OPERATOR_COMPUTE_FLOPS(name, function) \ + ::vkcompute::shader_profiling::throughput_registry() \ + .register_operator_compute(#name, &function) +#define REGISTER_SHADER_MEMORY(name, function) \ + ::vkcompute::shader_profiling::throughput_registry().register_shader_memory( \ + #name, &function) +#define REGISTER_SHADER_COMPUTE(name, function) \ + ::vkcompute::shader_profiling::throughput_registry() \ + .register_shader_compute(#name, &function) +#define REGISTER_SHADER_MEMORY_TRAFFIC(prefix, function) \ + ::vkcompute::shader_profiling::throughput_registry() \ + .register_shader_memory_prefix(#prefix, &function) +#define REGISTER_SHADER_COMPUTE_FLOPS(prefix, function) \ + ::vkcompute::shader_profiling::throughput_registry() \ + .register_shader_compute_prefix(#prefix, &function) + +#define REGISTER_THROUGHPUT_FUNCTIONS \ + static void register_throughput_functions(); \ + static const ::vkcompute::shader_profiling::ThroughputRegisterInit \ + throughput_reg(®ister_throughput_functions); \ + static void register_throughput_functions() diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/Timestamps.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/Timestamps.cpp new file mode 100644 index 00000000000..d85e0800ac5 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/Timestamps.cpp @@ -0,0 +1,82 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +namespace vkcompute::shader_profiling { + +Timestamps::Timestamps(api::Context& context) + : device_(context.device()), + period_ns_(context.adapter_ptr()->timestamp_period()) { + uint32_t count = 0; + const VkPhysicalDevice physical = context.adapter_ptr()->physical_handle(); + vkGetPhysicalDeviceQueueFamilyProperties(physical, &count, nullptr); + std::vector families(count); + vkGetPhysicalDeviceQueueFamilyProperties(physical, &count, families.data()); + valid_bits_ = families.at(context.queue().family_index).timestampValidBits; + if (valid_bits_ > 0 && period_ns_ > 0) { + const VkQueryPoolCreateInfo info{ + VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO, + nullptr, + 0, + VK_QUERY_TYPE_TIMESTAMP, + 2, + 0}; + VK_CHECK(vkCreateQueryPool(device_, &info, nullptr, &pool_)); + } +} + +Timestamps::~Timestamps() { + if (pool_ != VK_NULL_HANDLE) { + vkDestroyQueryPool(device_, pool_, nullptr); + } +} + +bool Timestamps::available() const { + return pool_ != VK_NULL_HANDLE; +} + +double Timestamps::period_ns() const { + return period_ns_; +} + +uint32_t Timestamps::valid_bits() const { + return valid_bits_; +} + +void Timestamps::begin(const vkapi::CommandBuffer& cmd) const { + cmd.reset_querypool(pool_, 0, 2); + cmd.write_timestamp(pool_, 0); +} + +void Timestamps::end(const vkapi::CommandBuffer& cmd) const { + cmd.write_timestamp(pool_, 1); +} + +double Timestamps::elapsed_us() const { + uint64_t ticks[2]; + VK_CHECK(vkGetQueryPoolResults( + device_, + pool_, + 0, + 2, + sizeof(ticks), + ticks, + sizeof(uint64_t), + VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WAIT_BIT)); + return elapsed_us(ticks[0], ticks[1]); +} + +double Timestamps::elapsed_us(uint64_t start_ticks, uint64_t end_ticks) const { + const uint64_t mask = + valid_bits_ == 64 ? UINT64_MAX : (uint64_t(1) << valid_bits_) - 1; + return static_cast((end_ticks - start_ticks) & mask) * period_ns_ / + 1000.0; +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/Timestamps.h b/backends/vulkan/test/ops/benchmark_v2/framework/Timestamps.h new file mode 100644 index 00000000000..774d1a3f40b --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/Timestamps.h @@ -0,0 +1,37 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +#include + +namespace vkcompute::shader_profiling { + +class Timestamps final { + VkDevice device_; + VkQueryPool pool_ = VK_NULL_HANDLE; + double period_ns_; + uint32_t valid_bits_; + + public: + explicit Timestamps(api::Context& context); + Timestamps(const Timestamps&) = delete; + ~Timestamps(); + + Timestamps& operator=(const Timestamps&) = delete; + + bool available() const; + double period_ns() const; + uint32_t valid_bits() const; + void begin(const vkapi::CommandBuffer& cmd) const; + void end(const vkapi::CommandBuffer& cmd) const; + double elapsed_us() const; + double elapsed_us(uint64_t start_ticks, uint64_t end_ticks) const; +}; + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/Timing.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/Timing.cpp new file mode 100644 index 00000000000..9e976a0b678 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/Timing.cpp @@ -0,0 +1,275 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace vkcompute::shader_profiling { + +namespace { + +using Clock = std::chrono::steady_clock; + +Report read_gpu_clock(GpuClock* clock) { + const auto frequency = clock ? clock->read_mhz() : std::nullopt; + return frequency ? Report(*frequency) : Report(); +} + +Report profile_operator_shaders( + const Json& spec, + ComputeGraph& graph, + const Timestamps& timestamps, + const std::function& encode_batch, + uint32_t count, + uint32_t query_limit, + uint32_t warmup, + uint32_t runs, + GpuClock* gpu_clock) { + auto& context = *graph.context(); + context.flush(); + context.initialize_querypool(); + context.set_shader_profiling_enabled(true); + context.set_cmd(true); + context.cmd_reset_querypool(); + const bool memory = spec.value("show_memory_throughput", false); + const bool compute = spec.value("show_compute_throughput", false); + std::vector work; + if (memory || compute) { + for (const auto& node : graph.execute_nodes()) { + const size_t before = + context.querypool().get_shader_timestamp_data().size(); + node->encode(&graph); + const auto records = context.querypool().get_shader_timestamp_data(); + const size_t emitted = + records.size() > before ? records.size() - before : 0; + for (size_t i = before; i < records.size(); ++i) { + if (emitted == 1) { + work.push_back(estimate_work( + {{"shader_name", records[i].kernel_name}, + {"gwg_size", + {{"method", "operator"}, {"value", records[i].metadata.gwg}}}, + {"lwg_size", + {{"method", "operator"}, {"value", records[i].metadata.lwg}}}, + {"bindings", dispatch_tensor_bindings(graph, *node)}}, + memory, + compute)); + } else { + work.push_back( + {std::nullopt, + std::nullopt, + {"Binding metadata unavailable for a node with multiple dispatches"}}); + } + } + } + } else { + encode_batch(1); + } + const size_t shaders_per_invocation = + context.querypool().get_shader_timestamp_data().size(); + if ((memory || compute) && work.size() != shaders_per_invocation) { + work.assign( + shaders_per_invocation, + {std::nullopt, std::nullopt, {"Shader binding metadata unavailable"}}); + } + if (shaders_per_invocation > query_limit / 2) { + throw UnsupportedRoute( + "Operator dispatch chain exceeds timestamp query capacity"); + } + count = shaders_per_invocation == 0 + ? 1u + : std::min( + count, + static_cast(query_limit / (2 * shaders_per_invocation))); + if (count > 1) { + context.flush(); + context.set_cmd(true); + context.cmd_reset_querypool(); + encode_batch(count); + } + vkapi::VulkanFence fence(context.device()); + auto replay = [&]() { + context.submit_cmd_to_gpu(fence.get_submit_handle()); + fence.wait(); + }; + for (uint32_t i = 0; i < warmup; ++i) { + replay(); + } + Report dispatches = Report::array(); + Report clock_samples = Report::array(); + std::vector> samples; + for (uint32_t i = 0; i < runs; ++i) { + const Report clock_start = read_gpu_clock(gpu_clock); + replay(); + if (gpu_clock) { + clock_samples.push_back({clock_start, read_gpu_clock(gpu_clock)}); + } + const auto shaders = context.querypool().get_shader_timestamp_ticks(); + std::vector durations; + for (size_t j = 0; j < shaders.size(); ++j) { + const auto& shader = shaders[j]; + if (i == 0 && j < shaders_per_invocation) { + dispatches.push_back( + {{"dispatch_index", j}, + {"shader_name", shader.kernel_name}, + {"gwg_size", + {{"method", "operator"}, {"value", shader.metadata.gwg}}}, + {"lwg_size", + {{"method", "operator"}, {"value", shader.metadata.lwg}}}}); + } + durations.push_back( + timestamps.elapsed_us(shader.start_ticks, shader.end_ticks)); + } + samples.push_back(std::move(durations)); + } + Report result = { + {"shader_profile_invocations_per_batch", count}, + {"shader_breakdown", + shader_breakdown_report(dispatches, samples, count, spec, work)}}; + if (gpu_clock) { + result["shader_profile_batch_observed_gpu_clock_freq_MHz"] = + std::move(clock_samples); + } + return result; +} + +} // namespace + +Report measure( + const Json& spec, + ComputeGraph& graph, + const std::function& encode_batch, + const Json& work_spec) { + api::Context& context = *graph.context(); + GpuClock* gpu_clock = nullptr; +#if defined(__ANDROID__) + GpuClock android_gpu_clock; + gpu_clock = &android_gpu_clock; +#endif + Report wall_clock_samples = Report::array(); + Report gpu_clock_samples = Report::array(); + Timestamps timestamps(context); + vkapi::VulkanFence fence(context.device()); + const WorkAmounts work = estimate_work( + work_spec, + spec.value("show_memory_throughput", false), + spec.value("show_compute_throughput", false)); + auto encode = [&](uint32_t count, bool timed) { + context.set_shader_profiling_enabled(false); + context.flush(); + context.set_cmd(true); + if (timed) { + timestamps.begin(context.extract_cmd()); + } + encode_batch(count); + if (timed) { + timestamps.end(context.extract_cmd()); + } + }; + auto replay = [&]() { + const auto start = Clock::now(); + context.submit_cmd_to_gpu(fence.get_submit_handle()); + fence.wait(); + return std::chrono::duration(Clock::now() - start) + .count(); + }; + auto measured_replay = [&](Report& clock_samples) { + const Report clock_start = read_gpu_clock(gpu_clock); + const double wall = replay(); + if (gpu_clock) { + clock_samples.push_back({clock_start, read_gpu_clock(gpu_clock)}); + } + return wall; + }; + + uint32_t count = 1; + const uint32_t limit = + spec.value("max_dispatches", spec.contains("operator") ? 200u : 65536u); + const double target_us = spec.value("target_ms", 50.0) * 1000; + for (;;) { + encode(count, false); + replay(); + const double wall = replay(); + if (wall >= target_us * 0.8 || count == limit) + break; + const uint32_t next = choose_dispatch_count(wall / count, target_us, limit); + count = std::min(limit, std::max(count + 1, std::min(next, count * 8))); + } + const uint32_t warmup = spec.value("warmup_runs", 3u); + const uint32_t runs = spec.value("benchmark_runs", 10u); + encode(count, false); + std::vector wall_samples; + std::vector gpu_samples; + for (uint32_t i = 0; i < warmup; ++i) { + replay(); + } + for (uint32_t i = 0; i < runs; ++i) { + wall_samples.push_back(measured_replay(wall_clock_samples)); + } + if (timestamps.available()) { + encode(count, true); + for (uint32_t i = 0; i < warmup; ++i) { + replay(); + } + for (uint32_t i = 0; i < runs; ++i) { + measured_replay(gpu_clock_samples); + gpu_samples.push_back(timestamps.elapsed_us()); + } + } + Report result = { + {"device", graph.device_name()}, + {"target_ms", target_us / 1000}, + {"max_dispatches", limit}, + {spec.contains("operator") ? "operator_invocations_per_batch" + : "dispatches_per_batch", + count}, + {"warmup_runs", warmup}, + {"benchmark_runs", runs}, + {"gpu_timestamps_available", timestamps.available()}, + {"timestamp_period_ns", timestamps.period_ns()}, + {"timestamp_valid_bits", timestamps.valid_bits()}}; + const Report measurements = measurement_report( + count, spec.contains("operator"), gpu_samples, wall_samples); + for (const auto& measurement : measurements.items()) { + result[measurement.key()] = measurement.value(); + if (gpu_clock && measurement.key() == "gpu_timestamp_batch_us") { + result["gpu_batch_observed_gpu_clock_freq_MHz"] = + std::move(gpu_clock_samples); + } + if (gpu_clock && measurement.key() == "wall_time_batch_us") { + result["wall_time_batch_observed_gpu_clock_freq_MHz"] = + std::move(wall_clock_samples); + } + } + Report throughput = + throughput_report(spec, work, count, gpu_samples, wall_samples); + if (!throughput.empty()) { + result["throughput"] = std::move(throughput); + } + if (spec.contains("operator") && timestamps.available()) { + result.update(profile_operator_shaders( + spec, + graph, + timestamps, + encode_batch, + count, + graph.graphconfig().context_config.query_pool_config.max_query_count, + warmup, + runs, + gpu_clock)); + } + return result; +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/Timing.h b/backends/vulkan/test/ops/benchmark_v2/framework/Timing.h new file mode 100644 index 00000000000..8d69d8344c4 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/Timing.h @@ -0,0 +1,23 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +#include +#include + +namespace vkcompute::shader_profiling { + +Report measure( + const Json& spec, + ComputeGraph& graph, + const std::function& encode_batch, + const Json& work_spec = Json::object()); + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/Validation.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/Validation.cpp new file mode 100644 index 00000000000..67ff428f542 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/Validation.cpp @@ -0,0 +1,148 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include +#include + +namespace vkcompute::shader_profiling { + +void copy_validation_inputs( + const Json& spec, + ComputeGraph& graph, + const std::map& tensors, + const std::map>& host_data) { + for (const auto& tensor : tensors) { + const Json& description = spec.at("tensors").at(tensor.first); + if (description.value("storage", "buffer") == "constant") { + continue; + } + auto dtype = scalar_type(description.value("dtype", "float32")); + if (dtype == vkapi::kHalf) { + dtype = vkapi::kFloat; + } else if (dtype == vkapi::kInt8x4) { + dtype = vkapi::kChar; + } + graph.maybe_cast_and_copy_into_staging( + graph.staging_of(tensor.second), + host_data.at(tensor.first).data(), + tensor_numel(description.at("sizes")), + dtype); + } +} + +std::map add_validation_outputs( + const Json& spec, + ComputeGraph& graph, + const std::map& tensors, + const Json& accesses) { + std::set outputs = validation_output_names(spec); + const bool explicitly_selected = !outputs.empty(); + if (outputs.empty() && spec.contains("outputs")) { + for (const Json& name : spec.at("outputs")) { + outputs.insert(name.get()); + } + } else if (outputs.empty()) { + for (const auto& access : accesses.items()) { + if (access.value() != "read") { + outputs.insert(access.key()); + } + } + } + if (outputs.empty()) { + throw std::invalid_argument( + "Validation requires at least one output tensor"); + } + std::map staging; + for (const auto& name : outputs) { + if (explicitly_selected) { + reference::require( + accesses.contains(name) && accesses.at(name) != "read", + "Selected validation tensor is not writable: " + name); + } + const ValueRef tensor = tensors.at(name); + auto dtype = graph.dtype_of(tensor); + if (dtype == vkapi::kHalf) { + dtype = vkapi::kFloat; + } else if (dtype == vkapi::kInt8x4) { + dtype = vkapi::kChar; + } + staging.emplace(name, graph.set_output_tensor(tensor, dtype)); + } + return staging; +} + +namespace { + +template +std::vector read_staging( + ComputeGraph& graph, + ValueRef staging, + size_t count, + vkapi::ScalarType dtype) { + std::vector data(count); + graph.maybe_cast_and_copy_from_staging(staging, data.data(), count, dtype); + return std::vector(data.begin(), data.end()); +} + +} // namespace + +Report validate_outputs( + const Json& spec, + ComputeGraph& graph, + const std::map& tensors, + const std::map& staging, + const std::map>& host_data) { + ReferenceOutputs actual; + for (const auto& output : staging) { + const ValueRef tensor = tensors.at(output.first); + if (graph.sizes_of(tensor) != + spec.at("tensors") + .at(output.first) + .at("sizes") + .get>()) { + throw std::invalid_argument( + "Output tensor shape does not match declared shape: " + output.first); + } + const size_t count = tensor_numel(graph.sizes_of(tensor)); + const auto dtype = graph.dtype_of(tensor); + if (dtype == vkapi::kChar || dtype == vkapi::kInt8x4) { + actual[output.first] = + read_staging(graph, output.second, count, vkapi::kChar); + } else if (dtype == vkapi::kByte) { + actual[output.first] = + read_staging(graph, output.second, count, vkapi::kByte); + } else if (dtype == vkapi::kInt) { + actual[output.first] = + read_staging(graph, output.second, count, vkapi::kInt); + } else { + actual[output.first] = + read_staging(graph, output.second, count, vkapi::kFloat); + } + } + const Json dtypes = actual_tensor_dtypes(graph, tensors); + const auto start = std::chrono::steady_clock::now(); + Json reference_spec = spec; + Json shaders = Json::array(); + for (const auto& node : graph.execute_nodes()) { + shaders.push_back(node->name()); + } + reference_spec["reference_context"] = { + {"int8_dot_product", graph.can_use_int8_dot_product()}, + {"shaders", shaders}}; + const ReferenceOutputs expected = + reference_outputs(reference_spec, dtypes, host_data); + const double reference_us = std::chrono::duration( + std::chrono::steady_clock::now() - start) + .count(); + Report result = compare_outputs(spec, expected, actual, dtypes); + result["reference_wall_time_us"] = reference_us; + return result; +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/Validation.h b/backends/vulkan/test/ops/benchmark_v2/framework/Validation.h new file mode 100644 index 00000000000..15b91f33e79 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/Validation.h @@ -0,0 +1,33 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +namespace vkcompute::shader_profiling { + +void copy_validation_inputs( + const Json& spec, + ComputeGraph& graph, + const std::map& tensors, + const std::map>& host_data); + +std::map add_validation_outputs( + const Json& spec, + ComputeGraph& graph, + const std::map& tensors, + const Json& accesses); + +Report validate_outputs( + const Json& spec, + ComputeGraph& graph, + const std::map& tensors, + const std::map& staging, + const std::map>& host_data); + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/WorkgroupUtils.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/WorkgroupUtils.cpp new file mode 100644 index 00000000000..5f4227cd0bb --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/WorkgroupUtils.cpp @@ -0,0 +1,110 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +#include +#include + +namespace vkcompute::shader_profiling { +namespace { + +GlobalWorkGrid pick_gwg( + const Json& setting, + ComputeGraph& graph, + const vkapi::ShaderInfo& shader, + const std::vector& args) { + if (setting.is_array()) { + const auto dims = setting.get>(); + return GlobalWorkGrid({dims[0], dims[1], dims[2]}, kExplicitWorkGrid); + } + static const std::map pickers = { + {"default_pick_gwg", default_pick_gwg}}; + const std::string name = setting.get(); + const auto picker = pickers.find(name); + if (picker == pickers.end()) { + throw std::invalid_argument("Unknown gwg_size picker: " + name); + } + if (args.empty() || !(args.front().access & vkapi::WRITE)) { + throw std::invalid_argument( + "gwg_size picker requires a write or read_write tensor binding"); + } + return picker->second(&graph, shader, args, {}); +} + +LocalWorkGroup pick_lwg( + const Json& setting, + ComputeGraph& graph, + const vkapi::ShaderInfo& shader, + const GlobalWorkGrid& gwg, + const std::vector& args) { + if (setting.is_array()) { + const auto dims = setting.get>(); + return LocalWorkGroup(dims[0], dims[1], dims[2]); + } + static const std::map pickers = { + {"default_pick_lwg", default_pick_lwg}, + {"pick_required_lwg", pick_required_lwg}, + {"pick_xy_square_lwg", pick_xy_square_lwg}, + {"pick_xz_square_lwg", pick_xz_square_lwg}, + {"pick_120shape_lwg", pick_120shape_lwg}}; + const std::string name = setting.get(); + const auto picker = pickers.find(name); + if (picker == pickers.end()) { + throw std::invalid_argument("Unknown lwg_size picker: " + name); + } + return picker->second(&graph, shader, gwg, args, {}); +} + +} // namespace + +WorkgroupSizes pick_workgroup_sizes( + const Json& spec, + ComputeGraph& graph, + const vkapi::ShaderInfo& shader, + const std::map& tensors) { + std::vector args; + std::vector read_args; + for (const Json& binding : spec.at("bindings")) { + if (!binding.contains("tensor")) { + continue; + } + const ValueRef ref = tensors.at(binding.at("tensor").get()); + const std::string access = binding.value("access", "read"); + if (access == "read") { + read_args.emplace_back(ref, vkapi::kRead); + } else { + args.emplace_back( + ref, access == "write" ? vkapi::kWrite : vkapi::kReadWrite); + } + } + for (const ArgGroup& arg : read_args) { + args.emplace_back(arg.refs, arg.access); + } + GlobalWorkGrid gwg = pick_gwg(spec.at("gwg_size"), graph, shader, args); + const LocalWorkGroup lwg = + pick_lwg(spec.at("lwg_size"), graph, shader, gwg, args); + auto* adapter = graph.context()->adapter_ptr(); + lwg.validate( + adapter->max_compute_workgroup_size(), + adapter->max_compute_workgroup_invocations()); + if (gwg.is_linear() && gwg.required_lwg_size() != lwg) { + // Rewrap the linear workload using the selected local geometry. + gwg = GlobalWorkGrid( + {utils::safe_downcast( + graph.numel_of(args.front().refs.front())), + 1, + 1}, + kLinearWorkGrid); + gwg.wrap_linear_dispatch(adapter->max_compute_workgroup_count(), lwg); + } + gwg.validate(lwg, adapter->max_compute_workgroup_count()); + return {gwg, lwg}; +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/WorkgroupUtils.h b/backends/vulkan/test/ops/benchmark_v2/framework/WorkgroupUtils.h new file mode 100644 index 00000000000..ef76c607641 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/WorkgroupUtils.h @@ -0,0 +1,28 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +#include +#include + +namespace vkcompute::shader_profiling { + +struct WorkgroupSizes { + GlobalWorkGrid gwg; + LocalWorkGroup lwg; +}; + +WorkgroupSizes pick_workgroup_sizes( + const Json& spec, + ComputeGraph& graph, + const vkapi::ShaderInfo& shader, + const std::map& tensors); + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/test/case_check_registry_test.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/test/case_check_registry_test.cpp new file mode 100644 index 00000000000..f54c2ec12f3 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/test/case_check_registry_test.cpp @@ -0,0 +1,92 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +#include +namespace vkcompute::shader_profiling { +using GpuCaseCheckRegistry = VulkanTest; + +namespace { +Json pass_check(const Json&, ComputeGraph&) { + return {{"status", "PASS"}}; +} +Json fail_check(const Json&, ComputeGraph&) { + return {{"status", "FAIL"}, {"error", "intentional"}}; +} +Json unsupported_shader_check(const Json&, ComputeGraph&) { + throw vkapi::ShaderNotSupportedError( + "unsupported_test_shader", vkapi::VulkanExtension::INTEGER_DOT_PRODUCT); +} +Json unsupported_route_check(const Json&, ComputeGraph&) { + throw UnsupportedRoute("Unsupported test route"); +} +Json error_check(const Json&, ComputeGraph&) { + throw std::runtime_error("Ordinary test error"); +} +REGISTER_CASE_CHECKS { + REGISTER_CASE_CHECK(test_case_check_pass, pass_check); + REGISTER_CASE_CHECK(test_case_check_fail, fail_check); + REGISTER_CASE_CHECK( + test_case_check_unsupported_shader, unsupported_shader_check); + REGISTER_CASE_CHECK( + test_case_check_unsupported_route, unsupported_route_check); + REGISTER_CASE_CHECK(test_case_check_error, error_check); +} +TEST(CaseCheckRegistry, RejectsNullAndDuplicateChecks) { + CaseCheckRegistry registry; + registry.register_check("check", pass_check); + EXPECT_EQ(registry.get_check("check"), pass_check); + EXPECT_THROW( + registry.register_check("check", fail_check), std::invalid_argument); + EXPECT_EQ(registry.get_check("check"), pass_check); + EXPECT_THROW(registry.register_check("", pass_check), std::invalid_argument); + EXPECT_THROW(registry.register_check("null", nullptr), std::invalid_argument); + EXPECT_THROW(registry.get_check("missing"), std::invalid_argument); +} +TEST_F(GpuCaseCheckRegistry, ExecutesRegisteredChecksAndSurfacesFailure) { + const auto pass = run_case_check( + {{"name", "pass"}, + {"check", "test_case_check_pass"}, + {"legacy_case", {{"custom", true}}}}); + EXPECT_EQ(pass.at("validation").at("status"), "PASS"); + EXPECT_FALSE(pass.contains("legacy_case")); + const auto fail = run_case_check({{"check", "test_case_check_fail"}}); + EXPECT_EQ(fail.at("validation").at("status"), "FAIL"); + EXPECT_EQ(fail.at("validation").at("error"), "intentional"); +} +TEST(CaseCheckRegistry, ParsesOnlyUnambiguousStandaloneChecks) { + const auto parsed = parse_configs( + {{"check", "test_case_check_pass"}, {"parameters", {{"value", 3}}}}); + ASSERT_EQ(parsed.size(), 1); + EXPECT_EQ(parsed[0].at("parameters").at("value"), 3); + EXPECT_THROW(parse_configs({{"check", ""}}), std::invalid_argument); + EXPECT_THROW( + parse_configs({{"check", "x"}, {"operator", "op"}}), + std::invalid_argument); + EXPECT_THROW( + parse_configs({{"check", "x"}, {"args_sweep", Json::array()}}), + std::invalid_argument); +} +TEST_F(GpuCaseCheckRegistry, SkipsNotSupportedExceptionsWithoutRequirements) { + for (const char* name : + {"test_case_check_unsupported_shader", + "test_case_check_unsupported_route"}) { + const auto result = run_case_check({{"check", name}}); + EXPECT_EQ(result.at("status"), "SKIP"); + EXPECT_EQ(result.at("validation").at("status"), "SKIP"); + EXPECT_FALSE(result.at("skip_reason").get().empty()); + } +} +TEST_F(GpuCaseCheckRegistry, OrdinaryExceptionsRemainFailures) { + const auto result = run_case_check({{"check", "test_case_check_error"}}); + EXPECT_EQ(result.at("validation").at("status"), "FAIL"); + EXPECT_EQ(result.at("validation").at("error"), "Ordinary test error"); + EXPECT_FALSE(result.contains("skip_reason")); +} +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/test/config_test.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/test/config_test.cpp new file mode 100644 index 00000000000..0568a50abeb --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/test/config_test.cpp @@ -0,0 +1,937 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +#include +#include +#include + +namespace vkcompute::shader_profiling { +namespace { + +Json config() { + return { + {"shader", "buffer_to_buffer_float"}, + {"gwg_size", {8, 1, 1}}, + {"lwg_size", {8, 1, 1}}, + {"bindings", + {{{"buffer", {{"name", "out"}, {"sizes", {8}}, {"access", "write"}}}}}}}; +} + +Json operator_config() { + const Json args = Json::array( + {{{"tensor", {{"name", "x"}, {"sizes", {2, 8}}}}}, + {{"tensor", {{"name", "y"}, {"sizes", {2, 4}}}}}}); + return { + {"operator", "aten.linear.default"}, {"args_sweep", Json::array({args})}}; +} + +TEST(ShaderProfilingConfig, AcceptsSingleAndMultipleConfigs) { + EXPECT_EQ(parse_configs(config()).size(), 1); + EXPECT_EQ(parse_configs(Json::array({config(), config()})).size(), 2); + EXPECT_THROW(parse_configs(Json::array()), std::invalid_argument); +} + +TEST(ShaderProfilingConfig, ThroughputFlagsMustBeBooleans) { + for (const char* key : + {"show_memory_throughput", "show_compute_throughput"}) { + Json spec = config(); + spec[key] = true; + EXPECT_EQ(parse_configs(spec)[0].at(key), true); + spec[key] = false; + EXPECT_EQ(parse_configs(spec)[0].at(key), false); + for (const Json& invalid : {Json("true"), Json(1), Json(), Json::array()}) { + spec[key] = invalid; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + } + } +} + +TEST(ShaderProfilingConfig, ExpandsShaderNamesWithSharedIndependentInputs) { + Json spec = config(); + spec.erase("shader"); + spec["shader_sweep"] = {"shader_b", "shader_a", "shader_b"}; + spec["name"] = "shared inputs"; + spec["target_ms"] = 7; + spec["reference"] = "copy"; + auto parsed = parse_configs(spec); + ASSERT_EQ(parsed.size(), 3); + for (size_t i = 0; i < parsed.size(); ++i) { + EXPECT_EQ(parsed[i].at("shader"), spec.at("shader_sweep")[i]); + EXPECT_EQ(parsed[i].at("sweep_index"), i); + EXPECT_EQ(parsed[i].at("name"), "shared inputs"); + EXPECT_EQ(parsed[i].at("target_ms"), 7); + EXPECT_EQ(parsed[i].at("reference"), "copy"); + EXPECT_FALSE(parsed[i].contains("shader_sweep")); + EXPECT_EQ(parsed[i].at("bindings"), parsed[0].at("bindings")); + EXPECT_EQ(parsed[i].at("tensors"), parsed[0].at("tensors")); + } + parsed[0]["tensors"]["out"]["sizes"] = {16}; + EXPECT_EQ(parsed[1].at("tensors").at("out").at("sizes"), Json::array({8})); +} + +TEST(ShaderProfilingConfig, CombinesShaderAndLocalWorkgroupListsInOrder) { + Json spec = config(); + spec.erase("shader"); + spec["shader_sweep"] = {"shader_a", "shader_b"}; + spec["lwg_size"] = {{8, 1, 1}, {10, 2, 1}}; + const auto parsed = parse_configs(spec); + ASSERT_EQ(parsed.size(), 4); + for (size_t i = 0; i < parsed.size(); ++i) { + EXPECT_EQ(parsed[i].at("shader"), spec.at("shader_sweep")[i / 2]); + EXPECT_EQ(parsed[i].at("lwg_size"), spec.at("lwg_size")[i % 2]); + EXPECT_EQ(parsed[i].at("sweep_index"), i); + } +} + +TEST(ShaderProfilingConfig, CombinesShaderListAndCartesianWorkgroupRange) { + Json spec = config(); + spec.erase("shader"); + spec["shader_sweep"] = {"shader_a", "shader_b"}; + spec.erase("lwg_size"); + spec["lwg_size_sweep"] = {{"x", {8, 12, 4}}, {"y", {1, 2, 1}}}; + const auto parsed = parse_configs(spec); + ASSERT_EQ(parsed.size(), 8); + const Json sizes = {{8, 1, 1}, {8, 2, 1}, {12, 1, 1}, {12, 2, 1}}; + for (size_t i = 0; i < parsed.size(); ++i) { + EXPECT_EQ(parsed[i].at("shader"), spec.at("shader_sweep")[i / 4]); + EXPECT_EQ(parsed[i].at("lwg_size"), sizes[i % 4]); + EXPECT_EQ(parsed[i].at("sweep_index"), i); + EXPECT_FALSE(parsed[i].contains("lwg_size_sweep")); + } +} + +TEST(ShaderProfilingConfig, RejectsMalformedOrAmbiguousShaderSweeps) { + Json spec = config(); + spec["shader_sweep"] = {"shader_a"}; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + Json op = operator_config(); + op["shader_sweep"] = {"shader_a"}; + EXPECT_THROW(parse_configs(op), std::invalid_argument); + spec.erase("shader"); + for (const Json& invalid : + {Json::array(), + Json("shader_a"), + Json::object(), + Json::array({""}), + Json::array({"shader_a", 1}), + Json::array({nullptr})}) { + spec["shader_sweep"] = invalid; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + } +} + +TEST(ShaderProfilingConfig, CapsCombinedShaderAndWorkgroupSweep) { + Json spec = config(); + spec.erase("shader"); + spec["shader_sweep"] = {"shader_a", "shader_b"}; + spec.erase("lwg_size"); + spec["lwg_size_sweep"] = {{"x", {1, 32769, 1}}}; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + spec["shader_sweep"] = std::vector(65537, "shader_a"); + spec.erase("lwg_size_sweep"); + spec["lwg_size"] = {8, 1, 1}; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); +} + +TEST(ShaderProfilingConfig, PreservesMixedDefinitionOrderWithShaderSweeps) { + Json sweep = config(); + sweep.erase("shader"); + sweep["shader_sweep"] = {"shader_a", "shader_b"}; + const auto parsed = parse_configs( + Json::array({config(), sweep, operator_config(), config()})); + ASSERT_EQ(parsed.size(), 5); + EXPECT_EQ(parsed[0].at("shader"), "buffer_to_buffer_float"); + EXPECT_EQ(parsed[1].at("shader"), "shader_a"); + EXPECT_EQ(parsed[2].at("shader"), "shader_b"); + EXPECT_TRUE(parsed[3].contains("operator")); + EXPECT_EQ(parsed[4].at("shader"), "buffer_to_buffer_float"); + EXPECT_FALSE(parsed[0].contains("sweep_index")); + EXPECT_FALSE(parsed[4].contains("sweep_index")); +} + +TEST(ShaderProfilingConfig, RejectsInvalidGridAndTensorSizes) { + Json spec = config(); + for (const Json& invalid : + {Json{0, 1, 1}, Json{-1, 1, 1}, Json{1.5, 1, 1}, Json{1, 1}}) { + spec["gwg_size"] = invalid; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + } + EXPECT_EQ(tensor_numel({2, 3, 4}), 24); + EXPECT_THROW(tensor_numel({2, 0}), std::invalid_argument); + EXPECT_THROW(tensor_numel({2147483647, 2}), std::invalid_argument); +} + +TEST(ShaderProfilingConfig, RejectsUnknownTensorAndInvalidOptions) { + Json spec = config(); + spec["bindings"][0] = {{"buffer", {{"ref", "missing"}}}}; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + spec = config(); + spec["benchmark_runs"] = 0; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); +} + +TEST(ShaderProfilingConfig, PacksTypedInputsAndRejectsTruncation) { + auto bytes = make_data({{"dtype", "int32"}, {"data", {-7, 42}}}, 2); + int32_t values[2]; + std::memcpy(values, bytes.data(), sizeof(values)); + EXPECT_EQ(values[0], -7); + EXPECT_EQ(values[1], 42); + EXPECT_THROW( + make_data({{"dtype", "int8"}, {"data", {128}}}, 1), + std::invalid_argument); + EXPECT_THROW(make_data({{"data", {1, 2}}}, 3), std::invalid_argument); +} + +TEST(ShaderProfilingConfig, StructuredDataInitializersAreTypedAndReproducible) { + const auto constants = + make_data({{"dtype", "int8"}, {"data", {{"fill", -7}}}}, 4); + EXPECT_EQ(constants, std::vector(4, 249)); + const Json spec = { + {"dtype", "float32"}, {"data", {{"rand_uniform_float", {0, 1}}}}}; + const auto random = make_data(spec, 64, 7); + EXPECT_EQ(random, make_data(spec, 64, 7)); + EXPECT_NE(random, make_data(spec, 64, 8)); + Json half = spec; + half["dtype"] = "float16"; + EXPECT_EQ(random, make_data(half, 64, 7)); + float values[64]; + std::memcpy(values, random.data(), sizeof(values)); + for (float value : values) { + EXPECT_GE(value, 0); + EXPECT_LE(value, 1); + } + Json seeded = spec; + seeded["data"]["seed"] = 42; + EXPECT_EQ(make_data(seeded, 64, 1), make_data(seeded, 64, 2)); +} + +TEST(ShaderProfilingConfig, UniformIntegerDataIsInclusiveAndReproducible) { + const Json spec = { + {"dtype", "int8"}, {"data", {{"rand_uniform_int", {-4, 4}}}}}; + const auto data = make_data(spec, 512, 7); + EXPECT_EQ(data, make_data(spec, 512, 7)); + EXPECT_NE(data, make_data(spec, 512, 8)); + int8_t values[512]; + std::memcpy(values, data.data(), sizeof(values)); + bool saw_min = false; + bool saw_max = false; + for (int8_t value : values) { + EXPECT_GE(value, -4); + EXPECT_LE(value, 4); + saw_min |= value == -4; + saw_max |= value == 4; + } + EXPECT_TRUE(saw_min); + EXPECT_TRUE(saw_max); + const auto large = make_data( + {{"dtype", "uint32"}, + {"data", {{"rand_uniform_int", {0, 2147483648LL}}}}}, + 256); + uint32_t wide[256]; + std::memcpy(wide, large.data(), sizeof(wide)); + for (uint32_t value : wide) { + EXPECT_LE(value, 2147483648u); + } + EXPECT_THROW( + make_data({{"data", {{"rand_uniform_int", {0, 1.5}}}}}, 4), + std::invalid_argument); + EXPECT_THROW( + make_data( + {{"dtype", "int8"}, {"data", {{"rand_uniform_int", {-129, 127}}}}}, + 4), + std::invalid_argument); + EXPECT_THROW( + make_data( + {{"dtype", "int32"}, {"data", {{"rand_uniform_int", {0, 1.5}}}}}, 4), + std::invalid_argument); +} + +TEST( + ShaderProfilingConfig, + RowSumsTrackRandomWeightsAndResolveForwardReferences) { + const Json tensors = { + {"a_sums", + {{"sizes", {2}}, + {"dtype", "int32"}, + {"data", {{"row_sum", "z_weights"}}}}}, + {"z_weights", + {{"sizes", {2, 3}}, + {"dtype", "int8"}, + {"data", {{"rand_uniform_int", {-4, 4}}}}}}}; + const auto data = make_tensor_data(tensors); + int8_t weights[6]; + int32_t sums[2]; + std::memcpy(weights, data.at("z_weights").data(), sizeof(weights)); + std::memcpy(sums, data.at("a_sums").data(), sizeof(sums)); + EXPECT_EQ(sums[0], weights[0] + weights[1] + weights[2]); + EXPECT_EQ(sums[1], weights[3] + weights[4] + weights[5]); + EXPECT_EQ(data, make_tensor_data(tensors)); + Json invalid = tensors; + invalid["a_sums"]["data"]["row_sum"] = "missing"; + EXPECT_THROW(make_tensor_data(invalid), std::invalid_argument); + invalid["a_sums"]["data"]["row_sum"] = "a_sums"; + EXPECT_THROW(make_tensor_data(invalid), std::invalid_argument); + invalid = tensors; + invalid["a_sums"]["sizes"] = {1}; + EXPECT_THROW(make_tensor_data(invalid), std::invalid_argument); +} + +TEST(ShaderProfilingConfig, RejectsMalformedDataInitializers) { + for (const Json& data : + {Json::object(), + Json{{"unknown", 0}}, + Json{{"constant", 0}}, + Json{{"fill", "bad"}}, + Json{{"rand_uniform_float", {1, 0}}}, + Json{{"rand_uniform_float", {0}}}, + Json{{"rand_uniform_float", {0, 1}}, {"seed", -1}}, + Json{{"fill", 0}, {"rand_uniform_float", {0, 1}}}}) { + EXPECT_THROW(make_data({{"data", data}}, 4), std::invalid_argument); + } + EXPECT_THROW( + make_data( + {{"dtype", "int8"}, {"data", {{"rand_uniform_float", {0, 1}}}}}, 4), + std::invalid_argument); + EXPECT_THROW( + make_data({{"dtype", "int8"}, {"data", {{"fill", 128}}}}, 4), + std::invalid_argument); + EXPECT_THROW( + make_data({{"fill", 1}, {"data", {{"fill", 0}}}}, 4), + std::invalid_argument); +} + +TEST(ShaderProfilingConfig, BoundsAdaptiveDispatchCount) { + EXPECT_EQ(choose_dispatch_count(10, 1000, 4096), 100); + EXPECT_EQ(choose_dispatch_count(2000, 1000, 4096), 1); + EXPECT_EQ(choose_dispatch_count(0, 1000, 128), 128); +} + +TEST(ShaderProfilingConfig, AcceptsOperatorAndMixedCases) { + Json op = operator_config(); + op["args_sweep"][0].push_back(nullptr); + op["args_sweep"][0].push_back({{"int_list", {1, 2}}}); + EXPECT_EQ(parse_configs(op).size(), 1); + EXPECT_EQ(parse_configs(Json::array({config(), op})).size(), 2); +} + +TEST(ShaderProfilingConfig, ConstantStorageIsOperatorOnlyAndReplacesTheFlag) { + Json op = operator_config(); + op["args_sweep"][0][0]["tensor"]["storage"] = "constant"; + op["args_sweep"][0][0]["tensor"]["sizes"] = {1, 2, 3, 4, 5}; + const Json parsed = parse_configs(op).at(0); + EXPECT_EQ(parsed.at("tensors").at("x").at("storage"), "constant"); + EXPECT_FALSE(parsed.at("tensors").at("x").contains("constant")); + op = operator_config(); + op["args_sweep"][0][0]["tensor"]["constant"] = true; + EXPECT_THROW(parse_configs(op), std::invalid_argument); + Json shader = config(); + shader["bindings"][0]["buffer"]["storage"] = "constant"; + EXPECT_THROW(parse_configs(shader), std::invalid_argument); +} + +TEST(ShaderProfilingConfig, CartesianVariablesPreserveTypesAndTemplateOrder) { + Json op = operator_config(); + op["sweep_variables"] = { + {"v_DTYPE", {"float32", "float16"}}, + {"v_STORAGE", {"buffer", "texture3d"}}, + {"v_M", {4}}}; + for (auto& arg : op["args_sweep"][0]) { + arg["tensor"]["dtype"] = "v_DTYPE"; + arg["tensor"]["storage"] = "v_STORAGE"; + arg["tensor"]["sizes"][0] = "v_M"; + arg["tensor"]["data"] = {{"rand_uniform_float", {0, 1}}}; + } + op["args_sweep"].push_back(op["args_sweep"][0]); + const auto parsed = parse_configs(op); + ASSERT_EQ(parsed.size(), 8); + for (size_t i = 0; i < parsed.size(); ++i) { + EXPECT_EQ(parsed[i].at("sweep_index"), i); + EXPECT_EQ(parsed[i].at("args_sweep_index"), i / 4); + EXPECT_EQ(parsed[i].at("tensors").at("x").at("sizes")[0], 4); + EXPECT_EQ( + parsed[i].at("tensors").at("x").at("dtype"), + i % 4 < 2 ? "float32" : "float16"); + EXPECT_EQ( + parsed[i].at("tensors").at("x").at("storage"), + i % 2 == 0 ? "buffer" : "texture3d"); + EXPECT_EQ(parsed[i].at("sweep_variables").at("v_M"), 4); + EXPECT_FALSE(parsed[i].at("args")[1].contains("access")); + } +} + +TEST(ShaderProfilingConfig, RejectsInvalidOrUnboundSweepVariables) { + for (const Json& variables : + {Json::array(), + Json{{"v_DTYPE", Json::array()}}, + Json{{"v_DTYPE", "float32"}}, + Json{{"DTYPE", {"float32"}}}}) { + Json op = operator_config(); + op["sweep_variables"] = variables; + EXPECT_THROW(parse_configs(op), std::invalid_argument); + } + Json op = operator_config(); + op["args_sweep"][0][0]["tensor"]["dtype"] = "v_MISSING"; + EXPECT_THROW(parse_configs(op), std::invalid_argument); +} + +TEST(ShaderProfilingConfig, NormalizesExplicitOperatorRoutes) { + Json op = operator_config(); + EXPECT_FALSE(parse_configs(op).at(0).contains("route")); + op["route"] = "direct"; + EXPECT_EQ( + parse_configs(op).at(0).at("route"), + (Json{{"name", "direct"}, {"options", Json::object()}})); + op["route"] = {{"name", "tiled"}}; + EXPECT_EQ( + parse_configs(op).at(0).at("route"), + (Json{{"name", "tiled"}, {"options", Json::object()}})); + op["route"]["options"] = {{"tile", {2, 4}}, {"custom_option", true}}; + EXPECT_EQ(parse_configs(op).at(0).at("route"), op.at("route")); +} + +TEST(ShaderProfilingConfig, ExpandsRouteSweepInsideArgumentVariables) { + Json op = operator_config(); + op["sweep_variables"] = {{"v_DTYPE", {"float32", "float16"}}}; + op["args_sweep"][0][0]["tensor"]["dtype"] = "v_DTYPE"; + op["args_sweep"][0][0]["tensor"]["data"] = {{"rand_uniform_float", {-1, 1}}}; + op["args_sweep"].push_back(op["args_sweep"][0]); + op["args_sweep"][1][0]["tensor"]["sizes"] = {4, 8}; + op["route_sweep"] = Json::array( + {"auto", {{"name", "tiled"}, {"options", {{"tile_rows", 2}}}}}); + auto cases = parse_configs(op); + ASSERT_EQ(cases.size(), 8); + for (size_t i = 0; i < cases.size(); ++i) { + EXPECT_EQ(cases[i].at("sweep_index"), i); + EXPECT_EQ(cases[i].at("args_sweep_index"), i / 4); + EXPECT_EQ( + cases[i].at("sweep_variables").at("v_DTYPE"), + i % 4 < 2 ? "float32" : "float16"); + EXPECT_EQ(cases[i].at("route").at("name"), i % 2 == 0 ? "auto" : "tiled"); + EXPECT_FALSE(cases[i].contains("route_sweep")); + EXPECT_EQ(cases[i].at("tensors").at("x").at("sizes")[0], i < 4 ? 2 : 4); + } + EXPECT_EQ( + make_tensor_data(cases[0].at("tensors")), + make_tensor_data(cases[1].at("tensors"))); + cases[0]["tensors"]["x"]["sizes"] = {8, 8}; + EXPECT_EQ(cases[1].at("tensors").at("x").at("sizes"), (Json{2, 8})); +} + +TEST(ShaderProfilingConfig, SubstitutesRouteNamesAndTypedOptions) { + Json op = operator_config(); + op["sweep_variables"] = { + {"v_ROUTE", {"direct", "tiled"}}, {"v_TILE", {{2, 4}, {4, 8}}}}; + op["route"] = {{"name", "v_ROUTE"}, {"options", {{"tile", "v_TILE"}}}}; + const auto cases = parse_configs(op); + ASSERT_EQ(cases.size(), 4); + EXPECT_EQ(cases[0].at("route").at("name"), "direct"); + EXPECT_EQ(cases[2].at("route").at("name"), "tiled"); + EXPECT_EQ(cases[0].at("route").at("options").at("tile"), (Json{2, 4})); + EXPECT_EQ(cases[1].at("route").at("options").at("tile"), (Json{4, 8})); + op.erase("route"); + op["route_sweep"] = + Json::array({"v_ROUTE", {{"name", "auto"}, {"options", "v_OPTIONS"}}}); + op["sweep_variables"] = { + {"v_ROUTE", {"direct", "tiled"}}, + {"v_OPTIONS", Json::array({Json{{"tile_rows", 2}}})}}; + const auto swept = parse_configs(op); + ASSERT_EQ(swept.size(), 4); + EXPECT_EQ(swept[0].at("route").at("name"), "direct"); + EXPECT_EQ(swept[2].at("route").at("name"), "tiled"); + EXPECT_EQ(swept[1].at("route").at("options").at("tile_rows"), 2); +} + +TEST(ShaderProfilingConfig, RejectsMalformedOrShaderRouteSelections) { + for (const Json& invalid : + {Json(nullptr), + Json(1), + Json(false), + Json(""), + Json::array({"direct"}), + Json::object(), + Json{{"options", Json::object()}}, + Json{{"name", 1}}, + Json{{"name", ""}}, + Json{{"name", "direct"}, {"unknown", 1}}, + Json{{"name", "direct"}, {"options", nullptr}}, + Json{{"name", "direct"}, {"options", Json::array()}}, + Json{{"name", "direct"}, {"options", 2}}}) { + Json op = operator_config(); + op["route"] = invalid; + EXPECT_THROW(parse_configs(op), std::invalid_argument); + op.erase("route"); + op["route_sweep"] = Json::array({"auto", invalid}); + EXPECT_THROW(parse_configs(op), std::invalid_argument); + } + for (const Json& invalid : + {Json(nullptr), Json("direct"), Json::object(), Json::array()}) { + Json op = operator_config(); + op["route_sweep"] = invalid; + EXPECT_THROW(parse_configs(op), std::invalid_argument); + } + Json op = operator_config(); + op["route"] = "direct"; + op["route_sweep"] = {"auto"}; + EXPECT_THROW(parse_configs(op), std::invalid_argument); + for (const char* key : {"route", "route_sweep"}) { + Json shader = config(); + shader[key] = + key == std::string("route") ? Json("direct") : Json::array({"direct"}); + EXPECT_THROW(parse_configs(shader), std::invalid_argument); + } + op = operator_config(); + op["route"] = {{"name", "v_MISSING"}}; + EXPECT_THROW(parse_configs(op), std::invalid_argument); + op["sweep_variables"] = {{"v_MISSING", {1}}}; + EXPECT_THROW(parse_configs(op), std::invalid_argument); +} + +TEST(ShaderProfilingConfig, RejectsLegacyRouteSelectionKeys) { + for (const char* key : {"implementation", "implementation_sweep"}) { + for (Json spec : {operator_config(), config()}) { + spec[key] = key == std::string("implementation") + ? Json("direct") + : Json::array({"direct"}); + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + } + } +} + +TEST(ShaderProfilingConfig, CapsCombinedRouteCartesianExpansion) { + Json op = operator_config(); + op["args_sweep"] = Json::array_t(32, Json::array({{{"int", 1}}})); + op["sweep_variables"] = {{"v_VALUE", Json::array_t(128, 1)}}; + op["route_sweep"] = Json::array_t(32, "auto"); + EXPECT_THROW(parse_configs(op), std::invalid_argument); + op["args_sweep"] = Json::array_t(17, Json::array({{{"int", 1}}})); + EXPECT_THROW(parse_configs(op), std::invalid_argument); +} + +TEST( + ShaderProfilingConfig, + ExpandsSweepsInOrderWithIndependentTensorDefinitions) { + Json op = operator_config(); + op["name"] = "linear"; + op["target_ms"] = 7; + Json half_args = op.at("args_sweep").at(0); + half_args[0]["tensor"]["dtype"] = "float16"; + half_args[0]["tensor"]["sizes"] = {4, 8}; + half_args[1]["tensor"]["dtype"] = "float16"; + half_args[1]["tensor"]["sizes"] = {4, 4}; + op["args_sweep"].push_back(half_args); + const auto parsed = parse_configs(Json::array({config(), op, config()})); + ASSERT_EQ(parsed.size(), 4); + EXPECT_TRUE(parsed[0].contains("shader")); + EXPECT_TRUE(parsed[3].contains("shader")); + for (size_t i = 0; i < 2; ++i) { + EXPECT_EQ(parsed[i + 1].at("name"), "linear"); + EXPECT_EQ(parsed[i + 1].at("target_ms"), 7); + EXPECT_EQ(parsed[i + 1].at("sweep_index"), i); + EXPECT_FALSE(parsed[i + 1].contains("args_sweep")); + EXPECT_EQ(parsed[i + 1].at("tensors").size(), 2); + } + EXPECT_EQ(parsed[1].at("tensors").at("x").at("sizes"), Json::array({2, 8})); + EXPECT_EQ(parsed[2].at("tensors").at("x").at("sizes"), Json::array({4, 8})); + EXPECT_EQ( + parsed[1].at("tensors").at("x").value("dtype", "float32"), "float32"); + EXPECT_EQ(parsed[2].at("tensors").at("x").at("dtype"), "float16"); + op["args_sweep"][1][0]["tensor"] = {{"ref", "x"}}; + EXPECT_THROW(parse_configs(op), std::invalid_argument); +} + +TEST(ShaderProfilingConfig, RejectsEmptyAndMalformedSweeps) { + for (const Json& invalid : + {Json::array(), + Json::object(), + Json(1), + Json::array({Json::object()}), + Json::array({Json::array()})}) { + Json op = operator_config(); + op["args_sweep"] = invalid; + EXPECT_THROW(parse_configs(op), std::invalid_argument); + } + Json op = operator_config(); + op.erase("args_sweep"); + EXPECT_THROW(parse_configs(op), std::invalid_argument); + Json shader = config(); + shader["args_sweep"] = Json::array({Json::array()}); + EXPECT_THROW(parse_configs(shader), std::invalid_argument); +} + +TEST(ShaderProfilingConfig, RejectsAmbiguousCasesAndOperatorArguments) { + Json op = operator_config(); + op["shader"] = "buffer_to_buffer_float"; + EXPECT_THROW(parse_configs(op), std::invalid_argument); + op.erase("shader"); + op["args_sweep"][0][0]["tensor"] = {{"ref", "missing"}}; + EXPECT_THROW(parse_configs(op), std::invalid_argument); + op["args_sweep"][0] = Json::array({{{"int", 1.5}}}); + EXPECT_THROW(parse_configs(op), std::invalid_argument); +} + +TEST(ShaderProfilingConfig, SupportsPackedInt8Activations) { + Json op = operator_config(); + op["operator"] = "et_vk.q8ta_conv2d.default"; + op["args_sweep"][0] = Json::array( + {{{"tensor", + {{"sizes", {1, 4, 8, 8}}, {"dtype", "int8x4"}, {"layout", "4w4c"}}}}, + {{"tensor", + {{"sizes", {1, 8, 8, 8}}, + {"dtype", "int8x4"}, + {"layout", "4w4c"}, + {"access", "write"}}}}}); + EXPECT_EQ(parse_configs(op).size(), 1); + EXPECT_EQ( + make_data({{"dtype", "int8x4"}, {"fill", -7}}, 4), + std::vector(4, 249)); +} + +TEST(ShaderProfilingConfig, InlineOperatorTensorsPreserveReferencesAndData) { + Json op = operator_config(); + op["args_sweep"][0] = Json::array( + {{{"tensor", {{"ref", "state"}}}}, + {{"tensor", + {{"name", "weight"}, + {"sizes", {2}}, + {"dtype", "int32"}, + {"storage", "constant"}, + {"data", {-7, 42}}}}}, + nullptr, + {{"tensor", {{"sizes", {2}}, {"fill", 0.5}}}}, + {{"tensor", + {{"name", "state"}, + {"sizes", {2}}, + {"storage", "texture3d"}, + {"access", "write"}}}}, + {{"tensor", {{"ref", "arg_3"}, {"access", "read_write"}}}}}); + const Json parsed = parse_configs(op).at(0); + EXPECT_EQ(parsed.at("tensors").size(), 3); + EXPECT_FALSE(parsed.contains("inputs")); + EXPECT_FALSE(parsed.contains("outputs")); + EXPECT_EQ(parsed.at("args").size(), 6); + EXPECT_EQ(parsed.at("args").at(0).at("tensor"), "state"); + EXPECT_TRUE(parsed.at("args").at(2).is_null()); + EXPECT_EQ(parsed.at("args").at(4).at("tensor"), "state"); + EXPECT_EQ(parsed.at("args").at(4).at("access"), "write"); + EXPECT_EQ(parsed.at("args").at(5).at("tensor"), "arg_3"); + EXPECT_EQ(parsed.at("args").at(5).at("access"), "read_write"); + EXPECT_EQ( + parsed.at("tensors").at("weight").at("data"), Json::array({-7, 42})); + EXPECT_EQ(parsed.at("tensors").at("weight").at("storage"), "constant"); + EXPECT_EQ(parsed.at("tensors").at("arg_3").at("fill"), 0.5); +} + +TEST( + ShaderProfilingConfig, + RejectsExternalOperatorTensorsAndInvalidDefinitions) { + Json op = operator_config(); + op["tensors"] = {{"old", {{"sizes", {8}}}}}; + EXPECT_THROW(parse_configs(op), std::invalid_argument); + for (const Json& invalid : + {Json("x"), + Json{{"name", "x"}}, + Json{{"sizes", {0}}}, + Json{{"name", ""}, {"sizes", {2}}}, + Json{{"name", "y"}, {"sizes", {2}}}, + Json{{"ref", "missing"}}, + Json{{"ref", "y"}, {"sizes", {2}}}, + Json{{"sizes", {2}}, {"dtype", "unsupported"}}}) { + op = operator_config(); + op["args_sweep"][0][0]["tensor"] = invalid; + EXPECT_THROW(parse_configs(op), std::invalid_argument); + } + op = operator_config(); + op["args_sweep"][0][1]["tensor"]["storage"] = "constant"; + op["args_sweep"][0][1]["tensor"]["access"] = "write"; + EXPECT_THROW(parse_configs(op), std::invalid_argument); + op["args_sweep"][0] = Json::array({{{"float", "bad"}}}); + EXPECT_THROW(parse_configs(op), std::invalid_argument); + op["args_sweep"][0] = Json::array(); + EXPECT_THROW(parse_configs(op), std::invalid_argument); +} + +TEST(ShaderProfilingConfig, RejectsOldOperatorListsAndInvalidAccess) { + for (const char* key : {"args", "inputs", "outputs"}) { + Json op = operator_config(); + op[key] = Json::array(); + EXPECT_THROW(parse_configs(op), std::invalid_argument); + } + for (const Json& access : {Json("unknown"), Json(1), Json(false)}) { + Json op = operator_config(); + op["args_sweep"][0][1]["tensor"]["access"] = access; + EXPECT_THROW(parse_configs(op), std::invalid_argument); + } + Json op = operator_config(); + op["args_sweep"][0][0]["tensor"]["storage"] = "constant"; + op["args_sweep"][0][1]["tensor"] = {{"ref", "x"}, {"access", "write"}}; + EXPECT_THROW(parse_configs(op), std::invalid_argument); +} + +TEST(ShaderProfilingConfig, ReadWriteTensorKeepsOneArgument) { + Json op = operator_config(); + op["operator"] = "test.inplace.default"; + op["args_sweep"][0] = + Json::array({{{"tensor", {{"sizes", {8}}, {"access", "read_write"}}}}}); + const Json parsed = parse_configs(op).at(0); + EXPECT_EQ(parsed.at("args").size(), 1); + EXPECT_EQ(parsed.at("args").at(0).at("tensor"), "arg_0"); + EXPECT_EQ(parsed.at("args").at(0).at("access"), "read_write"); +} + +TEST(ShaderProfilingConfig, InlineBindingsPreserveDataAndSharedReferences) { + Json spec = config(); + spec["bindings"] = Json::array( + {{{"meta_ubo", "shared"}, {"field", "numel"}}, + {{"buffer", + {{"name", "shared"}, + {"sizes", {2}}, + {"dtype", "int32"}, + {"data", {-7, 42}}}}}, + {{"buffer", {{"ref", "shared"}, {"access", "read_write"}}}}, + {{"buffer", {{"sizes", {2}}, {"fill", 0.5}}}}}); + const Json parsed = parse_configs(spec).at(0); + EXPECT_EQ(parsed.at("tensors").size(), 2); + EXPECT_EQ( + parsed.at("tensors").at("shared").at("data"), Json::array({-7, 42})); + EXPECT_EQ(parsed.at("tensors").at("shared").at("dtype"), "int32"); + EXPECT_EQ(parsed.at("bindings").at(1).at("tensor"), "shared"); + EXPECT_EQ(parsed.at("bindings").at(2).at("tensor"), "shared"); +} + +TEST( + ShaderProfilingConfig, + RejectsExternalTensorListsAndDuplicateBindingNames) { + Json spec = config(); + spec["tensors"] = {{"old", {{"sizes", {8}}}}}; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + spec.erase("tensors"); + spec["bindings"].push_back({{"buffer", {{"name", "out"}, {"sizes", {8}}}}}); + EXPECT_THROW(parse_configs(spec), std::invalid_argument); +} + +TEST(ShaderProfilingConfig, RejectsInvalidInlineTensorAndLayoutReferences) { + Json spec = config(); + spec["bindings"][0]["buffer"]["sizes"] = {0}; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + spec = config(); + spec["specialization_constants"] = Json::array({{{"layout", "missing"}}}); + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + spec = config(); + spec["outputs"] = {"missing"}; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); +} + +TEST(ShaderProfilingConfig, BindingTypeDeterminesTensorStorage) { + for (const char* type : {"buffer", "texture2d", "texture3d"}) { + Json spec = config(); + spec["bindings"] = + Json::array({{{type, {{"name", "input"}, {"sizes", {8}}}}}}); + EXPECT_EQ( + parse_configs(spec).at(0).at("tensors").at("input").at("storage"), + type); + } +} + +TEST(ShaderProfilingConfig, RejectsUntypedAndConflictingBindings) { + Json spec = config(); + spec["bindings"] = Json::array({{{"name", "input"}, {"sizes", {8}}}}); + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + spec = config(); + spec["bindings"][0]["texture3d"] = {{"sizes", {8}}}; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + spec = config(); + spec["bindings"][0]["buffer"]["storage"] = "texture3d"; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + spec = config(); + spec["bindings"].push_back({{"texture3d", {{"ref", "out"}}}}); + EXPECT_THROW(parse_configs(spec), std::invalid_argument); +} + +TEST(ShaderProfilingConfig, AcceptsNamedWorkgroupPickers) { + Json spec = config(); + spec["gwg_size"] = "default_pick_gwg"; + spec["lwg_size"] = "default_pick_lwg"; + EXPECT_EQ(parse_configs(spec).at(0).at("gwg_size"), "default_pick_gwg"); + spec["lwg_size"] = {8, 1, 1}; + EXPECT_EQ(parse_configs(spec).size(), 1); + spec["gwg_size"] = {8, 1, 1}; + spec["lwg_size"] = "pick_xy_square_lwg"; + EXPECT_EQ(parse_configs(spec).size(), 1); +} + +TEST(ShaderProfilingConfig, ExpandsLocalWorkgroupListInSpecifiedOrder) { + Json spec = config(); + spec["lwg_size"] = {{8, 8, 1}, {10, 10, 1}}; + const auto cases = parse_configs(spec); + ASSERT_EQ(cases.size(), 2); + EXPECT_EQ(cases[0].at("lwg_size"), (Json{8, 8, 1})); + EXPECT_EQ(cases[1].at("lwg_size"), (Json{10, 10, 1})); + EXPECT_EQ(cases[0].at("sweep_index"), 0); + EXPECT_EQ(cases[1].at("sweep_index"), 1); + EXPECT_EQ(cases[0].at("tensors"), cases[1].at("tensors")); + spec["lwg_size"] = {{8, 1, 1}, {0, 1, 1}}; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + spec["lwg_size"] = Json::array({Json{8, 1, 1}, "default_pick_lwg"}); + EXPECT_THROW(parse_configs(spec), std::invalid_argument); +} + +TEST( + ShaderProfilingConfig, + ExpandsInclusiveLocalWorkgroupRangeCartesianProduct) { + Json spec = config(); + spec.erase("lwg_size"); + spec["lwg_size_sweep"] = {{"x", {8, 10, 2}}, {"y", {1, 3, 2}}}; + const auto cases = parse_configs(spec); + ASSERT_EQ(cases.size(), 4); + const Json expected = {{8, 1, 1}, {8, 3, 1}, {10, 1, 1}, {10, 3, 1}}; + for (size_t i = 0; i < cases.size(); ++i) { + EXPECT_EQ(cases[i].at("lwg_size"), expected[i]); + EXPECT_EQ(cases[i].at("sweep_index"), i); + EXPECT_FALSE(cases[i].contains("lwg_size_sweep")); + } + spec["lwg_size_sweep"] = {{"x", {8, 11, 2}}}; + EXPECT_EQ(parse_configs(spec).size(), 2); + spec["lwg_size_sweep"] = {{"x", {4294967294ULL, 4294967295ULL, 2}}}; + EXPECT_EQ(parse_configs(spec).size(), 1); +} + +TEST(ShaderProfilingConfig, RejectsInvalidLocalWorkgroupSweeps) { + Json spec = config(); + spec["lwg_size_sweep"] = {{"x", {1, 2, 1}}}; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + spec.erase("lwg_size"); + for (const Json& invalid : + {Json::array(), + Json{{"w", {1, 2, 1}}}, + Json{{"x", {2, 1, 1}}}, + Json{{"x", {1, 2, 0}}}, + Json{{"x", {1, 2, -1}}}, + Json{{"x", {1, 2}}}, + Json{{"x", {1, 2, 0.5}}}, + Json{{"x", {0, 2, 1}}}, + Json{{"x", {1, 4294967296ULL, 1}}}, + Json{{"x", {1, 65537, 1}}}, + Json{{"x", {1, 257, 1}}, {"y", {1, 256, 1}}}}) { + spec["lwg_size_sweep"] = invalid; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + } + Json op = operator_config(); + op["lwg_size_sweep"] = {{"x", {1, 2, 1}}}; + EXPECT_THROW(parse_configs(op), std::invalid_argument); +} + +TEST(ShaderProfilingConfig, ValidatesReferenceComparisonOptions) { + Json spec = config(); + spec["reference"] = "copy"; + spec["validation_method"] = { + {"function", "is_close"}, {"atol", 0.01}, {"rtol", 0.02}}; + EXPECT_EQ(parse_configs(spec)[0].at("reference"), "copy"); + spec["reference"] = "custom_reference"; + EXPECT_EQ(parse_configs(spec).size(), 1); + for (const Json& invalid : {Json(0), Json("")}) { + spec["reference"] = invalid; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + } + spec["reference"] = "copy"; + for (const Json& invalid : + {Json("is_close"), + Json{{"function", "unknown"}}, + Json{{"atol", 0.01}}, + Json{{"function", "is_close"}, {"atol", -1}}, + Json{{"function", "is_close"}, {"rtol", "0.1"}}, + Json{ + {"function", "is_close"}, + {"atol", std::numeric_limits::infinity()}}, + Json{{"function", "is_close"}, {"extra", 1}}}) { + spec["validation_method"] = invalid; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + } +} + +TEST(ShaderProfilingConfig, SelectsNamedValidationOutputs) { + Json spec = config(); + spec["validation_method"] = {{"function", "is_close"}, {"outputs", {"out"}}}; + EXPECT_EQ( + parse_configs(spec)[0].at("validation_method").at("outputs"), + Json::array({"out"})); + for (const Json& invalid : + {Json("out"), + Json::array(), + Json::array({""}), + Json::array({0}), + Json::array({"missing"}), + Json::array({"out", "out"})}) { + spec["validation_method"]["outputs"] = invalid; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + } + spec = operator_config(); + spec["validation_method"] = {{"function", "is_close"}, {"outputs", {"y"}}}; + EXPECT_EQ(parse_configs(spec).size(), 1); + spec["args_sweep"][0][1]["tensor"]["storage"] = "constant"; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); +} + +TEST(ShaderProfilingConfig, RejectsInvalidWorkgroupPickerValues) { + for (const char* key : {"gwg_size", "lwg_size"}) { + for (const Json& invalid : + {Json(""), Json(1), Json(false), Json::object()}) { + Json spec = config(); + spec[key] = invalid; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + } + } +} + +TEST(ShaderProfilingConfig, RejectsOldWorkgroupAndMetadataKeys) { + Json spec = config(); + spec["global_size"] = {8, 1, 1}; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + spec.erase("global_size"); + spec["local_size"] = {8, 1, 1}; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + spec = config(); + spec["bindings"].push_back({{"metadata", "out"}}); + EXPECT_THROW(parse_configs(spec), std::invalid_argument); +} + +} // namespace +TEST(ShaderProfilingConfig, RejectsRemovedNarrowInt4Setting) { + for (Json spec : + {config(), + operator_config(), + Json{{"check", "unsigned_dot_selection"}}}) { + for (bool value : {false, true}) { + spec["force_narrow_int4_tile"] = value; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + } + } +} + +TEST(ShaderProfilingConfig, RejectsRemovedBenchmarkControls) { + const Json removed = { + {"force_resize", true}, + {"invocations_per_batch", 17}, + {"gpu_timing", "batch"}, + {"shader_filter", Json::array()}, + {"requirements", {{"features", {"int8_dot_product"}}}}, + {"device_variables", {{"v_SIZE", {{"property", "subgroup_size"}}}}}}; + for (const Json& base : + {config(), + operator_config(), + Json{{"check", "unsigned_dot_selection"}}}) { + for (const auto& option : removed.items()) { + Json spec = base; + spec[option.key()] = option.value(); + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + } + } +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/test/data_initializer_test.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/test/data_initializer_test.cpp new file mode 100644 index 00000000000..ee48ca27e19 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/test/data_initializer_test.cpp @@ -0,0 +1,255 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +#include +#include +#include + +namespace vkcompute::shader_profiling { +namespace { + +template +std::vector unpack(const std::vector& data) { + std::vector result(data.size() / sizeof(T)); + std::memcpy(result.data(), data.data(), data.size()); + return result; +} +Json tensor(const char* dtype, const Json& sizes, const Json& data) { + return {{"dtype", dtype}, {"sizes", sizes}, {"data", data}}; +} + +TEST(DataInitializer, GeneratesReproducibleIntegerValuedFloatAndHalfInputs) { + const Json fp = + tensor("float32", {512}, {{"rand_uniform_int", {-10, 10}}, {"seed", 23}}); + Json half = fp; + half["dtype"] = "float16"; + const auto data = make_data(fp, 512, 1); + EXPECT_EQ(data, make_data(fp, 512, 99)); + EXPECT_EQ(data, make_data(half, 512, 2)); + bool minimum = false, maximum = false; + for (float value : unpack(data)) { + EXPECT_EQ(std::trunc(value), value); + EXPECT_GE(value, -10); + EXPECT_LE(value, 10); + minimum |= value == -10; + maximum |= value == 10; + } + EXPECT_TRUE(minimum); + EXPECT_TRUE(maximum); +} +TEST(DataInitializer, RejectsFractionalOrOversizedIntegerRandomBounds) { + for (const Json& bounds : + {Json{0, 1.5}, + Json{2, 1}, + Json{0, 4294967296ULL}, + Json{-2147483648LL, 4294967295ULL}}) { + EXPECT_THROW( + validate_data_initializer( + tensor("float32", {8}, {{"rand_uniform_int", bounds}}), 8), + std::invalid_argument); + } +} +TEST(DataInitializer, RepeatsPatternsAndValidatesEveryValue) { + EXPECT_EQ( + unpack( + make_data(tensor("int8", {8}, {{"repeat", {-128, -1, 127}}}), 8)), + (std::vector{-128, -1, 127, -128, -1, 127, -128, -1})); + EXPECT_EQ( + unpack( + make_data(tensor("float16", {5}, {{"repeat", {.5, -1}}}), 5)), + (std::vector{.5f, -1, .5f, -1, .5f})); + for (const Json& pattern : + {Json::array(), + Json("bad"), + Json{1, 128}, + Json{1, 1.5}, + Json{1, nullptr}}) { + EXPECT_THROW( + validate_data_initializer( + tensor("int8", {1}, {{"repeat", pattern}}), 1), + std::invalid_argument); + } +} +TEST(DataInitializer, AbsoluteModuloMatchesRawSignedDraws) { + const Json source = + tensor("int32", {512}, {{"rand_uniform_int", {-100, 100}}, {"seed", 3}}); + const Json derived = tensor( + "int32", + {512}, + {{"abs_mod", {{"bounds", {-100, 100}}, {"mod", 7}, {"seed", 3}}}}); + const auto raw = unpack(make_data(source, 512)); + const auto values = unpack(make_data(derived, 512)); + for (size_t i = 0; i < values.size(); ++i) + EXPECT_EQ(values[i], std::abs(raw[i]) % 7); + EXPECT_EQ( + unpack(make_data( + tensor( + "int32", + {2}, + {{"abs_mod", + {{"bounds", {-2147483648LL, -2147483648LL}}, {"mod", 3}}}}), + 2)), + (std::vector{2, 2})); + EXPECT_EQ( + unpack(make_data( + tensor( + "uint8", + {2}, + {{"abs_mod", {{"bounds", {-1000, -1000}}, {"mod", 7}}}}), + 2)), + (std::vector{6, 6})); +} +TEST(DataInitializer, RejectsInvalidAbsoluteModuloDefinitions) { + for (const Json& definition : + {Json{{"bounds", {0, 1}}, {"mod", 0}}, + Json{{"bounds", {0, 1}}, {"mod", 1.5}}, + Json{{"bounds", {0, 1.5}}, {"mod", 2}}, + Json{{"bounds", {0, 1}}, {"mod", 2}, {"extra", 1}}, + Json{{"bounds", {0, 1}}, {"mod", 2}, {"seed", -1}}}) { + EXPECT_THROW( + validate_data_initializer( + tensor("int32", {4}, {{"abs_mod", definition}}), 4), + std::invalid_argument); + } +} +TEST(DataInitializer, GeneratesEntirePaddedRowsBeforeZeroingColumns) { + const Json initializer = {{"rand_uniform_int", {-128, 127}}, {"seed", 11}}; + const auto raw = + unpack(make_data(tensor("int8", {3, 8}, initializer), 24)); + const Json padded = tensor( + "int8", + {3, 8}, + {{"pad_columns", {{"source", initializer}, {"valid_columns", 5}}}}); + const auto values = unpack(make_data(padded, 24)); + for (size_t i = 0; i < 24; ++i) + EXPECT_EQ(values[i], i % 8 < 5 ? raw[i] : 0); + const auto repeat = unpack(make_data( + tensor( + "float32", + {2, 4}, + {{"pad_columns", + {{"source", {{"repeat", {1, 2, 3}}}}, {"valid_columns", 2}}}}), + 8)); + EXPECT_EQ(repeat, (std::vector{1, 2, 0, 0, 2, 3, 0, 0})); +} +TEST(DataInitializer, RejectsInvalidPaddingShapesAndSources) { + for (const Json& definition : + {Json{{"source", {{"repeat", {1}}}}, {"valid_columns", -1}}, + Json{{"source", {{"repeat", {1}}}}, {"valid_columns", 5}}, + Json{{"source", {{"repeat", {1}}}}, {"valid_columns", 1.5}}, + Json{{"source", {{"row_sum", "other"}}}, {"valid_columns", 1}}, + Json{ + {"source", {{"repeat", {1}}}}, + {"valid_columns", 1}, + {"extra", 1}}}) { + EXPECT_THROW( + validate_data_initializer( + tensor("float32", {2, 4}, {{"pad_columns", definition}}), 8), + std::invalid_argument); + } + EXPECT_THROW( + make_data( + tensor( + "float32", + {2, 4}, + {{"pad_columns", + {{"source", {{"repeat", {1}}}}, {"valid_columns", 1}}}}), + 7), + std::invalid_argument); +} +TEST(DataInitializer, PadsRowSumsAndReducesAllLeadingColumnDimensions) { + const Json specs = { + {"weight", + tensor("int8", {2, 2, 3}, {1, 2, 3, 4, 5, 6, -1, -2, -3, -4, -5, -6})}, + {"rows", tensor("int32", {4}, {{"row_sum", "weight"}})}, + {"columns", tensor("int32", {4}, {{"column_sum", "weight"}})}}; + const auto values = make_tensor_data(specs); + EXPECT_EQ( + unpack(values.at("rows")), + (std::vector{21, -21, 0, 0})); + EXPECT_EQ( + unpack(values.at("columns")), + (std::vector{0, 0, 0, 0})); + Json nonzero = specs; + nonzero["weight"]["data"] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}; + EXPECT_EQ( + unpack(make_tensor_data(nonzero).at("columns")), + (std::vector{22, 26, 30, 0})); +} +TEST(DataInitializer, RejectsShortSumsUnknownSourcesCyclesAndOverflow) { + Json specs = { + {"weight", tensor("int32", {2, 2}, {1, 2, 3, 4})}, + {"sum", tensor("int32", {1}, {{"row_sum", "weight"}})}}; + EXPECT_THROW(make_tensor_data(specs), std::invalid_argument); + specs["sum"]["data"] = {{"column_sum", "weight"}}; + EXPECT_THROW(make_tensor_data(specs), std::invalid_argument); + specs["sum"]["data"] = {{"row_sum", "missing"}}; + EXPECT_THROW(make_tensor_data(specs), std::invalid_argument); + specs = { + {"a", tensor("int32", {1}, {{"column_sum", "b"}})}, + {"b", tensor("int32", {1}, {{"row_sum", "a"}})}}; + EXPECT_THROW(make_tensor_data(specs), std::invalid_argument); + specs = { + {"weight", tensor("int32", {1, 2}, {2147483647, 1})}, + {"sum", tensor("int32", {1}, {{"row_sum", "weight"}})}}; + EXPECT_THROW(make_tensor_data(specs), std::invalid_argument); + specs["weight"]["dtype"] = "float32"; + EXPECT_THROW(make_tensor_data(specs), std::invalid_argument); +} +TEST(DataInitializer, PackedQ4GroupSumsUseGroupMajorOrderAndPadOutputChannels) { + const Json packed = {0x98, 0x76, 0xAF, 0x80}; + const Json base = { + {"weight", tensor("uint8", {2, 2}, packed)}, + {"sum", + tensor( + "int32", + {2, 4}, + {{"q4_group_sum", {{"tensor", "weight"}, {"group_size", 2}}}})}}; + const auto values = unpack(make_tensor_data(base).at("sum")); + EXPECT_EQ(values, (std::vector{1, 9, 0, 0, -3, -8, 0, 0})); + Json signed_source = base; + signed_source["weight"]["dtype"] = "int8"; + signed_source["weight"]["data"] = {-104, 118, -81, -128}; + EXPECT_EQ(unpack(make_tensor_data(signed_source).at("sum")), values); + for (const char* dtype : {"float32", "float16"}) { + Json fp = base; + fp["sum"]["dtype"] = dtype; + EXPECT_EQ( + unpack(make_tensor_data(fp).at("sum")), + (std::vector{1, 9, 0, 0, -3, -8, 0, 0})); + } +} +TEST(DataInitializer, RejectsInvalidQ4GroupShapesSourcesAndCycles) { + Json specs = { + {"weight", tensor("uint8", {2, 2}, {0, 0, 0, 0})}, + {"sum", + tensor( + "int32", + {2, 2}, + {{"q4_group_sum", {{"tensor", "weight"}, {"group_size", 2}}}})}}; + for (int size : {0, 3, 8}) { + Json invalid = specs; + invalid["sum"]["data"]["q4_group_sum"]["group_size"] = size; + EXPECT_THROW(make_tensor_data(invalid), std::invalid_argument); + } + Json invalid = specs; + invalid["sum"]["sizes"] = {1, 2}; + EXPECT_THROW(make_tensor_data(invalid), std::invalid_argument); + invalid = specs; + invalid["sum"]["sizes"] = {2, 1}; + EXPECT_THROW(make_tensor_data(invalid), std::invalid_argument); + invalid = specs; + invalid["weight"]["dtype"] = "int32"; + EXPECT_THROW(make_tensor_data(invalid), std::invalid_argument); + invalid = specs; + invalid["sum"]["data"]["q4_group_sum"]["tensor"] = "sum"; + EXPECT_THROW(make_tensor_data(invalid), std::invalid_argument); +} +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/test/gpu_clock_test.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/test/gpu_clock_test.cpp new file mode 100644 index 00000000000..c4bf9877877 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/test/gpu_clock_test.cpp @@ -0,0 +1,112 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +#include +#include +#include + +namespace vkcompute::shader_profiling { +namespace { + +class GpuClockTest : public ::testing::Test { + protected: + std::filesystem::path root_; + + void SetUp() override { + do { + root_ = std::filesystem::temp_directory_path() / + ("benchmark_gpu_clock_" + std::to_string(std::random_device{}())); + } while (!std::filesystem::create_directory(root_)); + } + + void TearDown() override { + std::filesystem::remove_all(root_); + } + + void write(const std::string& path, const std::string& value) { + const auto file = root_ / path; + std::filesystem::create_directories(file.parent_path()); + std::ofstream output(file); + ASSERT_TRUE(output.good()); + output << value; + } +}; + +TEST_F(GpuClockTest, ReadsChangingAdrenoFrequencyInHz) { + const std::string path = "class/devfreq/3d00000.qcom,kgsl-3d0/cur_freq"; + write(path, "903000000\n"); + GpuClock clock(root_.string()); + EXPECT_EQ(clock.read_mhz(), 903.0); + write(path, "366000000\n"); + EXPECT_EQ(clock.read_mhz(), 366.0); +} + +TEST_F(GpuClockTest, ReadsClassicKgslFrequencyInHz) { + write("class/kgsl/kgsl-3d0/gpuclk", "578000000\n"); + EXPECT_EQ(GpuClock(root_.string()).read_mhz(), 578.0); +} + +TEST_F(GpuClockTest, ReadsPixelMaliFrequencyInKHz) { + write("devices/platform/1f000000.mali/cur_freq", "150000\n"); + EXPECT_EQ(GpuClock(root_.string()).read_mhz(), 150.0); +} + +TEST_F(GpuClockTest, ReadsLegacyMaliClockInMHz) { + write("devices/platform/18500000.mali/clock", "702\n"); + EXPECT_EQ(GpuClock(root_.string()).read_mhz(), 702.0); +} + +TEST_F(GpuClockTest, ReadsExynosSgpuFrequencyInKHz) { + write("class/devfreq/22200000.sgpu/cur_freq", "1000000\n"); + EXPECT_EQ(GpuClock(root_.string()).read_mhz(), 1000.0); +} + +TEST_F(GpuClockTest, KernelFallbackUsesDriverSpecificUnits) { + write("class/devfreq/22200000.sgpu/name", "sgpu\n"); + write("kernel/gpu/gpu_clock", "800000\n"); + EXPECT_EQ(GpuClock(root_.string()).read_mhz(), 800.0); + std::filesystem::remove_all(root_ / "class/devfreq/22200000.sgpu"); + write("class/devfreq/3d00000.qcom,kgsl-3d0/name", "kgsl\n"); + write("kernel/gpu/gpu_clock", "903\n"); + EXPECT_EQ(GpuClock(root_.string()).read_mhz(), 903.0); +} + +TEST_F(GpuClockTest, IgnoresBusAndNonGpuDevfreqDevices) { + write("class/devfreq/kgsl-busmon/cur_freq", "999000000\n"); + write("class/devfreq/gpu-bw/cur_freq", "999000000\n"); + write("class/devfreq/17000010.devfreq_mif/cur_freq", "999000000\n"); + write("class/devfreq/1c000000.gpu/cur_freq", "450000000\n"); + EXPECT_EQ(GpuClock(root_.string()).read_mhz(), 450.0); +} + +TEST_F(GpuClockTest, MissingAndInvalidSourcesAreUnavailable) { + EXPECT_EQ(GpuClock(root_.string()).read_mhz(), std::nullopt); + write("class/kgsl/kgsl-3d0/gpuclk", "invalid"); + EXPECT_EQ(GpuClock(root_.string()).read_mhz(), std::nullopt); + write("class/devfreq/3d00000.qcom,kgsl-3d0/cur_freq", "903000000\n"); + EXPECT_EQ(GpuClock(root_.string()).read_mhz(), 903.0); +} + +TEST_F(GpuClockTest, FailedSamplesAreUnavailableAndZeroIsValid) { + const std::string path = "class/kgsl/kgsl-3d0/gpuclk"; + write(path, "903000000\n"); + GpuClock clock(root_.string()); + for (const char* value : {"", "bad", "-1", "nan", "inf", "100 200"}) { + write(path, value); + EXPECT_EQ(clock.read_mhz(), std::nullopt); + } + write(path, "0\n"); + EXPECT_EQ(clock.read_mhz(), 0.0); + std::filesystem::remove(root_ / path); + EXPECT_EQ(clock.read_mhz(), std::nullopt); +} + +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/test/operator_route_test.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/test/operator_route_test.cpp new file mode 100644 index 00000000000..29dbb35d8e5 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/test/operator_route_test.cpp @@ -0,0 +1,426 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +#include + +#include + +#include + +#include + +namespace vkcompute::shader_profiling { +using BenchmarkOperatorRoute = VulkanTest; + +namespace { + +struct FpConvCase { + std::vector weights; + std::vector bias; + std::vector half_weights; + std::vector half_bias; + ComputeGraph graph{GraphConfig{}}; + std::vector args; + + explicit FpConvCase( + vkapi::ScalarType dtype = vkapi::kFloat, + utils::StorageType storage = utils::kTexture3D, + utils::GPUMemoryLayout layout = utils::kChannelsPacked, + int64_t batch = 1, + int64_t channels = 4, + int64_t groups = 1) + : weights(8 * (channels / groups) * 9, 0.25f), + bias(8, 0.1f), + half_weights(weights.size(), executorch::runtime::etensor::Half(0.25f)), + half_bias(bias.size(), executorch::runtime::etensor::Half(0.1f)) { + const void* weight_data = dtype == vkapi::kHalf + ? static_cast(half_weights.data()) + : weights.data(); + const void* bias_data = dtype == vkapi::kHalf + ? static_cast(half_bias.data()) + : bias.data(); + args = { + graph.add_tensor({batch, channels, 7, 7}, dtype, storage, layout), + graph.add_tensorref({8, channels / groups, 3, 3}, dtype, weight_data), + graph.add_tensorref({8}, dtype, bias_data), + graph.add_scalar_list(std::vector{1, 1}), + graph.add_scalar_list(std::vector{1, 1}), + graph.add_scalar_list(std::vector{1, 1}), + graph.add_scalar(false), + graph.add_scalar_list(std::vector{0, 0}), + graph.add_scalar(groups), + graph.add_tensor({batch, 8, 7, 7}, dtype, storage, layout)}; + } +}; + +struct Q8ConvCase { + std::vector weights; + std::vector sums; + std::vector scales; + std::vector bias; + ComputeGraph graph{GraphConfig{}}; + std::vector args; + + explicit Q8ConvCase( + int64_t batch = 1, + int64_t channels = 4, + int64_t groups = 1) + : weights(8 * (channels / groups) * 9, 1), + sums(8, int32_t((channels / groups) * 9)), + scales(8, 0.1f), + bias(8, 0.0f) { + args = { + graph.add_tensor( + {batch, channels, 7, 7}, + vkapi::kInt8x4, + utils::kBuffer, + utils::kPackedInt8_4W4C), + graph.add_scalar(0.25), + graph.add_scalar(int64_t(0)), + graph.add_tensorref( + {8, (channels / groups) * 9}, vkapi::kChar, weights.data()), + graph.add_tensorref({8}, vkapi::kInt, sums.data()), + graph.add_tensorref({8}, vkapi::kFloat, scales.data()), + graph.add_scalar(0.25), + graph.add_scalar(int64_t(0)), + graph.add_tensorref({8}, vkapi::kFloat, bias.data()), + graph.add_scalar_list(std::vector{3, 3}), + graph.add_scalar_list(std::vector{1, 1}), + graph.add_scalar_list(std::vector{1, 1}), + graph.add_scalar_list(std::vector{1, 1}), + graph.add_scalar(groups), + graph.add_string("none"), + graph.add_tensor( + {batch, 8, 7, 7}, + vkapi::kInt8x4, + utils::kBuffer, + utils::kPackedInt8_4W4C)}; + } +}; + +TEST_F( + BenchmarkOperatorRoute, + DirectConvolutionBuildsProductionShaderForBothDtypes) { + for (const auto dtype : {vkapi::kFloat, vkapi::kHalf}) { + FpConvCase test(dtype); + const Json selected = + build_conv2d_direct(test.graph, test.args, Json::object()); + EXPECT_EQ(selected.at("name"), "direct"); + EXPECT_TRUE(selected.at("options").empty()); + ASSERT_EQ(test.graph.execute_nodes().size(), 1); + EXPECT_EQ(test.graph.execute_nodes()[0]->name().find("conv2d_"), 0); + EXPECT_EQ( + inspect_conv2d_route(test.graph, test.args, 0, 1).at("name"), "direct"); + } +} + +TEST_F( + BenchmarkOperatorRoute, + Im2colConvolutionHonorsAllIntermediateStorageOverrides) { + for (const char* storage : {"buffer", "texture2d", "texture3d"}) { + FpConvCase test; + const Json selected = build_conv2d_im2col( + test.graph, test.args, {{"intermediate_storage", storage}}); + EXPECT_EQ(selected.at("name"), "im2col"); + EXPECT_EQ(selected.at("options").at("intermediate_storage"), storage); + ASSERT_EQ(test.graph.execute_nodes().size(), 2); + EXPECT_EQ( + test.graph.execute_nodes()[0]->name().find( + std::string("conv2d_im2col_") + storage + "_"), + 0); + EXPECT_EQ( + test.graph.execute_nodes()[1]->name().find( + std::string("conv2d_gemm_") + storage + "_"), + 0); + EXPECT_EQ(inspect_conv2d_route(test.graph, test.args, 0, 2), selected); + } +} + +TEST_F( + BenchmarkOperatorRoute, + ForcedIm2colAllowsDilatedCasesExcludedByAutoPolicy) { + FpConvCase test; + test.args[4] = test.graph.add_scalar_list(std::vector{2, 2}); + test.args[5] = test.graph.add_scalar_list(std::vector{2, 2}); + EXPECT_EQ( + build_conv2d_im2col( + test.graph, test.args, {{"intermediate_storage", "buffer"}}) + .at("name"), + "im2col"); + EXPECT_EQ(test.graph.execute_nodes().size(), 2); +} + +TEST_F( + BenchmarkOperatorRoute, + UnsupportedConvolutionRoutesRejectBeforeAddingNodes) { + FpConvCase batched( + vkapi::kFloat, utils::kTexture3D, utils::kChannelsPacked, 2); + EXPECT_THROW( + build_conv2d_direct(batched.graph, batched.args, Json::object()), + UnsupportedRoute); + EXPECT_THROW( + build_conv2d_im2col(batched.graph, batched.args, Json::object()), + UnsupportedRoute); + EXPECT_TRUE(batched.graph.execute_nodes().empty()); + EXPECT_TRUE(batched.graph.prepack_nodes().empty()); + FpConvCase buffer(vkapi::kFloat, utils::kBuffer); + EXPECT_THROW( + build_conv2d_direct(buffer.graph, buffer.args, Json::object()), + UnsupportedRoute); + EXPECT_THROW( + build_conv2d_im2col(buffer.graph, buffer.args, Json::object()), + UnsupportedRoute); + FpConvCase width(vkapi::kFloat, utils::kTexture3D, utils::kWidthPacked); + EXPECT_THROW( + build_conv2d_direct(width.graph, width.args, Json::object()), + UnsupportedRoute); + FpConvCase grouped( + vkapi::kFloat, utils::kTexture3D, utils::kChannelsPacked, 1, 8, 2); + EXPECT_THROW( + build_conv2d_im2col(grouped.graph, grouped.args, Json::object()), + UnsupportedRoute); + FpConvCase transposed; + transposed.args[6] = transposed.graph.add_scalar(true); + EXPECT_THROW( + build_conv2d_im2col(transposed.graph, transposed.args, Json::object()), + UnsupportedRoute); +} + +TEST_F(BenchmarkOperatorRoute, ForcedDirectSupportsAlignedGroupedConvolution) { + FpConvCase test( + vkapi::kFloat, utils::kTexture3D, utils::kChannelsPacked, 1, 8, 2); + EXPECT_EQ( + build_conv2d_direct(test.graph, test.args, Json::object()).at("name"), + "direct"); + EXPECT_EQ(test.graph.execute_nodes().size(), 1); +} + +TEST_F(BenchmarkOperatorRoute, UnknownOptionsAndGpuWeightsAreRejected) { + FpConvCase test; + EXPECT_THROW( + build_conv2d_direct( + test.graph, test.args, {{"intermediate_storage", "buffer"}}), + std::invalid_argument); + EXPECT_THROW( + build_conv2d_im2col(test.graph, test.args, {{"unknown", true}}), + std::invalid_argument); + EXPECT_THROW( + build_conv2d_im2col( + test.graph, test.args, {{"intermediate_storage", "unknown"}}), + std::invalid_argument); + test.args[1] = test.graph.add_tensor( + {8, 4, 3, 3}, vkapi::kFloat, utils::kTexture3D, utils::kChannelsPacked); + EXPECT_THROW( + build_conv2d_direct(test.graph, test.args, Json::object()), + UnsupportedRoute); + EXPECT_THROW( + build_conv2d_im2col(test.graph, test.args, Json::object()), + UnsupportedRoute); + EXPECT_TRUE(test.graph.execute_nodes().empty()); +} + +TEST_F( + BenchmarkOperatorRoute, + Q8RoutesBuildNativeGeneralAndIm2colWithoutPerfPolicy) { + for (int64_t batch : {int64_t(1), int64_t(2)}) { + Q8ConvCase direct(batch); + EXPECT_EQ( + build_q8ta_conv2d_direct(direct.graph, direct.args, Json::object()) + .at("name"), + "direct"); + ASSERT_EQ(direct.graph.execute_nodes().size(), 1); + EXPECT_EQ(direct.graph.execute_nodes()[0]->name().find("q8ta_conv2d"), 0); + Q8ConvCase im2col(batch); + EXPECT_EQ( + build_q8ta_conv2d_im2col(im2col.graph, im2col.args, Json::object()) + .at("name"), + "im2col"); + ASSERT_EQ(im2col.graph.execute_nodes().size(), 2); + EXPECT_EQ(im2col.graph.execute_nodes()[0]->name(), "q8ta_im2col"); + EXPECT_EQ( + inspect_q8ta_conv2d_route(im2col.graph, im2col.args, 0, 2).at("name"), + "im2col"); + } +} + +TEST_F(BenchmarkOperatorRoute, Q8Im2colRejectsPackingAndUnsupportedOptions) { + Q8ConvCase test; + test.args[0] = test.graph.add_tensor( + {1, 4, 7, 7}, vkapi::kInt8x4, utils::kBuffer, utils::kPackedInt8_4W); + EXPECT_THROW( + build_q8ta_conv2d_im2col(test.graph, test.args, Json::object()), + UnsupportedRoute); + EXPECT_THROW( + build_q8ta_conv2d_direct(test.graph, test.args, {{"unknown", true}}), + std::invalid_argument); + EXPECT_TRUE(test.graph.execute_nodes().empty()); +} + +TEST_F( + BenchmarkOperatorRoute, + RegistryPreservesArgumentsAndReportsRequestedAndResolvedRoutes) { + FpConvCase explicit_case; + const auto original_args = explicit_case.args; + const Json requested = { + {"name", "im2col"}, {"options", {{"intermediate_storage", "texture2d"}}}}; + const Json result = build_operator_route( + {{"operator", "aten.convolution.default"}, {"route", requested}}, + explicit_case.graph, + explicit_case.args); + EXPECT_EQ(result.at("requested"), requested); + EXPECT_EQ(result.at("resolved").at("name"), "im2col"); + EXPECT_EQ( + result.at("resolved").at("options").at("intermediate_storage"), + "texture2d"); + EXPECT_EQ(explicit_case.args, original_args); + EXPECT_EQ( + explicit_case.graph.sizes_of(original_args[0]), + (std::vector{1, 4, 7, 7})); + FpConvCase automatic; + const Json auto_result = build_operator_route( + {{"operator", "aten.convolution.default"}}, + automatic.graph, + automatic.args); + EXPECT_EQ(auto_result.at("requested").at("name"), "auto"); + EXPECT_TRUE(auto_result.at("requested").at("options").empty()); + EXPECT_EQ( + auto_result.at("resolved"), + inspect_conv2d_route( + automatic.graph, + automatic.args, + 0, + automatic.graph.execute_nodes().size())); + EXPECT_NE(auto_result.at("resolved").at("name"), "unknown"); +} + +TEST_F( + BenchmarkOperatorRoute, + RegistryRejectsUnknownNamesAndAutoOptionsWithoutDispatches) { + FpConvCase test; + EXPECT_THROW( + build_operator_route( + {{"operator", "aten.convolution.default"}, + {"route", + {{"name", "not_registered"}, {"options", Json::object()}}}}, + test.graph, + test.args), + std::invalid_argument); + EXPECT_THROW( + build_operator_route( + {{"operator", "aten.convolution.default"}, + {"route", + {{"name", "auto"}, + {"options", {{"intermediate_storage", "buffer"}}}}}}, + test.graph, + test.args), + std::invalid_argument); + EXPECT_TRUE(test.graph.execute_nodes().empty()); + EXPECT_TRUE(test.graph.prepack_nodes().empty()); +} + +TEST_F(BenchmarkOperatorRoute, GroupedRoutesRejectOutputBlocksCrossingGroups) { + FpConvCase fp( + vkapi::kFloat, utils::kTexture3D, utils::kChannelsPacked, 1, 16, 4); + EXPECT_THROW( + build_conv2d_direct(fp.graph, fp.args, Json::object()), UnsupportedRoute); + EXPECT_TRUE(fp.graph.prepack_nodes().empty()); + Q8ConvCase q8(1, 16, 4); + EXPECT_THROW( + build_q8ta_conv2d_direct(q8.graph, q8.args, Json::object()), + UnsupportedRoute); + EXPECT_THROW( + build_q8ta_conv2d_im2col(q8.graph, q8.args, Json::object()), + UnsupportedRoute); + EXPECT_TRUE(q8.graph.execute_nodes().empty()); + EXPECT_TRUE(q8.graph.prepack_nodes().empty()); +} + +TEST_F( + BenchmarkOperatorRoute, + ForcedQ8Im2colDoesNotFallBackWhenScratchPlanIsInfeasible) { + std::vector large_weights(8 * 65536, 1); + Q8ConvCase test; + test.args[0] = test.graph.add_tensor( + {1, 65536, 1, 1024}, + vkapi::kInt8x4, + utils::kBuffer, + utils::kPackedInt8_4W4C); + test.args[3] = + test.graph.add_tensorref({8, 65536}, vkapi::kChar, large_weights.data()); + test.args[9] = test.graph.add_scalar_list(std::vector{1, 1}); + test.args[11] = test.graph.add_scalar_list(std::vector{0, 0}); + test.args[15] = test.graph.add_tensor( + {1, 8, 1, 1024}, vkapi::kInt8x4, utils::kBuffer, utils::kPackedInt8_4W4C); + EXPECT_THROW( + build_q8ta_conv2d_im2col(test.graph, test.args, Json::object()), + UnsupportedRoute); + EXPECT_TRUE(test.graph.execute_nodes().empty()); + EXPECT_TRUE(test.graph.prepack_nodes().empty()); +} + +TEST_F( + BenchmarkOperatorRoute, + FloatingConstantDtypesMustMatchTheirPrepackShader) { + FpConvCase mismatched_bias; + mismatched_bias.args[2] = mismatched_bias.graph.add_tensorref( + {8}, vkapi::kHalf, mismatched_bias.half_bias.data()); + EXPECT_THROW( + build_conv2d_direct( + mismatched_bias.graph, mismatched_bias.args, Json::object()), + UnsupportedRoute); + EXPECT_THROW( + build_conv2d_im2col( + mismatched_bias.graph, mismatched_bias.args, Json::object()), + UnsupportedRoute); + EXPECT_TRUE(mismatched_bias.graph.prepack_nodes().empty()); + FpConvCase half_io(vkapi::kHalf); + if (half_io.graph.dtype_of(half_io.args[0]) == vkapi::kHalf) { + half_io.args[1] = half_io.graph.add_tensorref( + {8, 4, 3, 3}, vkapi::kFloat, half_io.weights.data()); + half_io.args[2] = + half_io.graph.add_tensorref({8}, vkapi::kFloat, half_io.bias.data()); + EXPECT_THROW( + build_conv2d_direct(half_io.graph, half_io.args, Json::object()), + UnsupportedRoute); + } +} + +} // namespace +TEST_F( + BenchmarkOperatorRoute, + LegacyQdqRejectsUnsupportedFloatingBufferLayout) { + ComputeGraph graph{GraphConfig{}}; + const auto fp = graph.add_tensor( + {1, 3, 15, 15}, vkapi::kFloat, utils::kBuffer, utils::kChannelsPacked); + const auto quant = graph.add_tensor( + {1, 3, 15, 15}, vkapi::kInt8x4, utils::kBuffer, utils::kPackedInt8_4W4C); + const auto scale = graph.add_scalar(0.1); + const auto zero = graph.add_scalar(int64_t(0)); + const auto qmin = graph.add_scalar(int64_t(-128)); + const auto qmax = graph.add_scalar(int64_t(127)); + const auto dtype = graph.add_scalar(int64_t(2)); + const auto out_dtype = graph.add_scalar(int64_t(10)); + const auto quantize = route_registry().operator_route( + "quantized_decomposed.quantize_per_tensor.default", "legacy_4w4c"); + const auto dequantize = route_registry().operator_route( + "quantized_decomposed.dequantize_per_tensor.default", "legacy_4w4c"); + EXPECT_THROW( + quantize( + graph, {fp, scale, zero, qmin, qmax, dtype, quant}, Json::object()), + UnsupportedRoute); + EXPECT_THROW( + dequantize( + graph, + {quant, scale, zero, qmin, qmax, dtype, out_dtype, fp}, + Json::object()), + UnsupportedRoute); + EXPECT_TRUE(graph.execute_nodes().empty()); +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/test/reference_registry_test.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/test/reference_registry_test.cpp new file mode 100644 index 00000000000..637caaf4343 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/test/reference_registry_test.cpp @@ -0,0 +1,251 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +#include +#include +#include + +namespace vkcompute::shader_profiling { +namespace { + +reference::Tensor +identity_reference(const Json&, const reference::TensorMap& tensors, size_t) { + return tensors.at("x"); +} + +reference::Tensor offset_reference( + const Json&, + const reference::TensorMap& tensors, + size_t threads) { + reference::Tensor result = tensors.at("x"); + for (double& value : result.values) { + value += threads; + } + return result; +} + +ReferenceOutputs +multiple_reference(const Json&, const reference::TensorMap&, size_t) { + return {{"scale", {0.12345}}, {"zero_point", {-3}}}; +} + +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_FN(test_identity, identity_reference); + REGISTER_REFERENCE_OUTPUTS_FN(test_multiple, multiple_reference); + REGISTER_REFERENCE_FN(test_offset, offset_reference); + REGISTER_OPERATOR_REFERENCE(aten.test_reference.default, test_identity); + REGISTER_SHADER_REFERENCE(test_identity_shader, test_identity); +} + +TEST(BenchmarkReferenceRegistry, RegisteredNamesSelectCallables) { + ReferenceRegistry registry; + registry.register_reference("identity", identity_reference); + registry.register_reference("offset", offset_reference); + EXPECT_EQ(registry.get_reference("identity"), identity_reference); + EXPECT_EQ(registry.get_reference("offset"), offset_reference); +} + +TEST(BenchmarkReferenceRegistry, MultipleNamesCanShareAReference) { + ReferenceRegistry registry; + registry.register_reference("identity", identity_reference); + registry.register_operator_reference("aten.first.default", "identity"); + registry.register_operator_reference("aten.second.default", "identity"); + registry.register_shader_reference("shader_a", "identity"); + registry.register_shader_reference("shader_b", "identity"); + EXPECT_EQ(registry.operator_reference("aten.first.default"), "identity"); + EXPECT_EQ(registry.operator_reference("aten.second.default"), "identity"); + EXPECT_EQ(registry.shader_reference("shader_a"), "identity"); + EXPECT_EQ(registry.shader_reference("shader_b"), "identity"); +} + +TEST(BenchmarkReferenceRegistry, AssociationsCanPrecedeFunctionRegistration) { + ReferenceRegistry registry; + registry.register_operator_reference("aten.later.default", "later"); + registry.register_shader_reference("later_shader", "later"); + EXPECT_THROW( + registry.operator_reference("aten.later.default"), std::invalid_argument); + EXPECT_THROW( + registry.shader_reference("later_shader"), std::invalid_argument); + registry.register_reference("later", identity_reference); + EXPECT_EQ(registry.operator_reference("aten.later.default"), "later"); + EXPECT_EQ(registry.shader_reference("later_shader"), "later"); +} + +TEST(BenchmarkReferenceRegistry, OperatorAndShaderNamespacesAreSeparate) { + ReferenceRegistry registry; + registry.register_reference("identity", identity_reference); + registry.register_reference("offset", offset_reference); + registry.register_operator_reference("same_name", "identity"); + registry.register_shader_reference("same_name", "offset"); + EXPECT_EQ(registry.operator_reference("same_name"), "identity"); + EXPECT_EQ(registry.shader_reference("same_name"), "offset"); +} + +TEST(BenchmarkReferenceRegistry, DuplicateFunctionNamesKeepOriginalFunction) { + ReferenceRegistry registry; + registry.register_reference("identity", identity_reference); + EXPECT_THROW( + registry.register_reference("identity", offset_reference), + std::invalid_argument); + EXPECT_EQ(registry.get_reference("identity"), identity_reference); +} + +TEST(BenchmarkReferenceRegistry, DuplicateAssociationsKeepOriginalMapping) { + ReferenceRegistry registry; + registry.register_reference("identity", identity_reference); + registry.register_reference("offset", offset_reference); + registry.register_operator_reference("aten.same.default", "identity"); + registry.register_shader_reference("same_shader", "identity"); + EXPECT_THROW( + registry.register_operator_reference("aten.same.default", "offset"), + std::invalid_argument); + EXPECT_THROW( + registry.register_shader_reference("same_shader", "offset"), + std::invalid_argument); + EXPECT_EQ(registry.operator_reference("aten.same.default"), "identity"); + EXPECT_EQ(registry.shader_reference("same_shader"), "identity"); +} + +TEST(BenchmarkReferenceRegistry, MissingFunctionsAndMappingsFailLookup) { + ReferenceRegistry registry; + EXPECT_THROW(registry.get_reference("unknown"), std::invalid_argument); + EXPECT_THROW(registry.operator_reference("unknown"), std::invalid_argument); + EXPECT_THROW(registry.shader_reference("unknown"), std::invalid_argument); +} + +TEST(BenchmarkReferenceRegistry, RejectsEmptyNamesAndNullFunctions) { + ReferenceRegistry registry; + EXPECT_THROW( + registry.register_reference("", identity_reference), + std::invalid_argument); + EXPECT_THROW( + registry.register_reference("null", nullptr), std::invalid_argument); + EXPECT_THROW( + registry.register_operator_reference("", "identity"), + std::invalid_argument); + EXPECT_THROW( + registry.register_operator_reference("op", ""), std::invalid_argument); + EXPECT_THROW( + registry.register_shader_reference("", "identity"), + std::invalid_argument); + EXPECT_THROW( + registry.register_shader_reference("shader", ""), std::invalid_argument); +} + +Json registered_case(bool shader) { + Json spec = { + {"tensors", {{"x", {{"sizes", {2}}}}, {"out", {{"sizes", {2}}}}}}}; + if (shader) { + spec["shader"] = "test_identity_shader"; + spec["bindings"] = {{{"tensor", "out"}}, {{"tensor", "x"}}}; + } else { + spec["operator"] = "aten.test_reference.default"; + spec["args"] = {{{"tensor", "x"}}, {{"tensor", "out"}}}; + } + return spec; +} + +TEST( + BenchmarkReferenceRegistry, + StaticAssociationsDispatchRegisteredFunctions) { + const std::vector values = {1.5f, -2.0f}; + std::vector bytes(values.size() * sizeof(float)); + std::memcpy(bytes.data(), values.data(), bytes.size()); + for (bool shader : {false, true}) { + const Json spec = registered_case(shader); + EXPECT_EQ(reference_name(spec), "test_identity"); + EXPECT_EQ( + reference_outputs(spec, {}, {{"x", bytes}}).at("out"), + (std::vector{1.5, -2.0})); + } +} + +TEST( + BenchmarkReferenceRegistry, + JsonOverrideSelectsCallableAndForwardsThreads) { + Json spec = registered_case(false); + spec["reference"] = "test_offset"; + const std::vector values = {1, 2}; + std::vector bytes(values.size() * sizeof(float)); + std::memcpy(bytes.data(), values.data(), bytes.size()); + EXPECT_EQ(reference_name(spec), "test_offset"); + EXPECT_EQ( + reference_outputs(spec, {}, {{"x", bytes}}, 3).at("out"), + (std::vector{4, 5})); + spec["reference"] = "not_registered"; + EXPECT_THROW(reference_name(spec), std::invalid_argument); +} + +TEST(BenchmarkReferenceRegistry, BuiltinRegistrationsAreLinkedFromFamilyFiles) { + const auto& registry = reference_registry(); + EXPECT_EQ(registry.operator_reference("aten.linear.default"), "linear"); + EXPECT_EQ(registry.operator_reference("aten.convolution.default"), "conv2d"); + EXPECT_EQ( + registry.operator_reference("et_vk.q8ta_conv2d.default"), "q8ta_conv2d"); + EXPECT_EQ( + registry.operator_reference("et_vk.q4gsw_linear.default"), + "q4gsw_linear"); + EXPECT_EQ(registry.shader_reference("buffer_to_buffer_float"), "copy"); + EXPECT_EQ(registry.shader_reference("buffer_to_buffer_half"), "copy"); + EXPECT_EQ(registry.shader_reference("binary_add_texture3d_float"), "add"); + EXPECT_EQ(registry.shader_reference("binary_add_texture3d_half"), "add"); +} + +} // namespace +TEST(BenchmarkReferenceRegistry, MultiOutputReferencesShareTheNameNamespace) { + ReferenceRegistry registry; + registry.register_reference_outputs("multiple", multiple_reference); + registry.register_operator_reference("multi_op", "multiple"); + EXPECT_TRUE(registry.has_reference("multiple")); + EXPECT_EQ(registry.operator_reference("multi_op"), "multiple"); + EXPECT_EQ(registry.get_reference_outputs("multiple"), multiple_reference); + EXPECT_THROW( + registry.register_reference("multiple", identity_reference), + std::invalid_argument); + EXPECT_THROW( + registry.register_reference_outputs("multiple", multiple_reference), + std::invalid_argument); + registry.register_reference("single", identity_reference); + EXPECT_EQ(registry.get_reference_outputs("single"), nullptr); + EXPECT_THROW( + registry.register_reference_outputs("single", multiple_reference), + std::invalid_argument); + EXPECT_THROW( + registry.register_reference_outputs("", multiple_reference), + std::invalid_argument); + EXPECT_THROW( + registry.register_reference_outputs("null", nullptr), + std::invalid_argument); +} + +TEST( + BenchmarkReferenceRegistry, + MultiOutputReferencesValidateCountsAndRoundPerDtype) { + Json spec{ + {"operator", "test_multi"}, + {"reference", "test_multiple"}, + {"tensors", + {{"scale", {{"sizes", {1}}, {"dtype", "float16"}}}, + {"zero_point", {{"sizes", {1}}, {"dtype", "int8"}}}}}}; + const auto results = reference_outputs(spec, Json::object(), {}); + ASSERT_EQ(results.size(), 2); + EXPECT_EQ( + results.at("scale")[0], + float(executorch::runtime::etensor::Half(0.12345f))); + EXPECT_EQ(results.at("zero_point")[0], -3); + spec["tensors"]["scale"]["sizes"] = {2}; + EXPECT_THROW( + reference_outputs(spec, Json::object(), {}), std::invalid_argument); + spec["tensors"].erase("scale"); + EXPECT_THROW( + reference_outputs(spec, Json::object(), {}), std::invalid_argument); +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/test/reference_test.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/test/reference_test.cpp new file mode 100644 index 00000000000..40eb28e1e02 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/test/reference_test.cpp @@ -0,0 +1,362 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +#include + +#include +#include +#include + +namespace vkcompute::shader_profiling { +namespace { + +Json tensor(const Json& sizes, const std::string& dtype = "float32") { + return {{"sizes", sizes}, {"dtype", dtype}, {"storage", "buffer"}}; +} + +template +std::vector bytes(const std::vector& values) { + std::vector result(values.size() * sizeof(T)); + std::memcpy(result.data(), values.data(), result.size()); + return result; +} + +Json linear_spec() { + return { + {"operator", "aten.linear.default"}, + {"args", + {{{"tensor", "x"}}, + {{"tensor", "w"}}, + {{"tensor", "b"}}, + {{"tensor", "y"}}}}, + {"tensors", + {{"x", tensor({2, 3})}, + {"w", tensor({2, 3})}, + {"b", tensor({2})}, + {"y", tensor({2, 2})}}}}; +} + +TEST(BenchmarkReference, LinearComputesBatchedProjectionWithBias) { + const auto result = reference_outputs( + linear_spec(), + {}, + {{"x", bytes({1, 2, 3, -1, 0, 2})}, + {"w", bytes({2, -1, 1, 0, 3, -2})}, + {"b", bytes({0.5, -0.5})}}, + 2); + EXPECT_EQ(result.at("y"), (std::vector{3.5, -0.5, 0.5, -4.5})); +} + +TEST(BenchmarkReference, LinearSupportsLeadingDimensionsAndNoBias) { + Json spec = linear_spec(); + spec["tensors"]["x"]["sizes"] = {1, 2, 3}; + spec["tensors"]["y"]["sizes"] = {1, 2, 2}; + spec["args"][2] = nullptr; + const auto result = reference_outputs( + spec, + {}, + {{"x", bytes({1, 2, 3, -1, 0, 2})}, + {"w", bytes({2, -1, 1, 0, 3, -2})}}); + EXPECT_EQ(result.at("y"), (std::vector{3, 0, 0, -4})); +} + +TEST( + BenchmarkReference, + LinearTilesAndTailsMatchScalarOracleAcrossThreadCounts) { + constexpr size_t kM = 7; + constexpr size_t kK = 17; + constexpr size_t kN = 129; + Json spec = linear_spec(); + spec["tensors"]["x"]["sizes"] = {kM, kK}; + spec["tensors"]["w"]["sizes"] = {kN, kK}; + spec["tensors"]["b"]["sizes"] = {kN}; + spec["tensors"]["y"]["sizes"] = {kM, kN}; + std::vector input(kM * kK); + std::vector weight(kN * kK); + std::vector bias(kN); + for (size_t i = 0; i < input.size(); ++i) { + input[i] = int(i % 7) - 3; + } + for (size_t i = 0; i < weight.size(); ++i) { + weight[i] = int(i % 11) - 5; + } + for (size_t i = 0; i < bias.size(); ++i) { + bias[i] = int(i % 3) - 1; + } + std::vector expected(kM * kN); + for (size_t row = 0; row < kM; ++row) { + for (size_t column = 0; column < kN; ++column) { + double value = bias[column]; + for (size_t inner = 0; inner < kK; ++inner) { + value += double(input[row * kK + inner]) * + double(weight[column * kK + inner]); + } + expected[row * kN + column] = value; + } + } + const std::map> data = { + {"x", bytes(input)}, {"w", bytes(weight)}, {"b", bytes(bias)}}; + const auto single_thread = reference_outputs(spec, {}, data, 1); + const auto eight_threads = reference_outputs(spec, {}, data, 8); + EXPECT_EQ(single_thread.at("y"), expected); + EXPECT_EQ(eight_threads, single_thread); +} + +TEST(BenchmarkReference, HalfInputsAreRoundedAndPromotedInputsStayFloat) { + Json spec = linear_spec(); + spec["args"][2] = nullptr; + spec["tensors"]["x"] = tensor({1, 1}, "float16"); + spec["tensors"]["w"] = tensor({1, 1}, "float16"); + spec["tensors"]["w"]["storage"] = "constant"; + spec["tensors"]["y"] = tensor({1, 1}, "float16"); + using Half = executorch::runtime::etensor::Half; + const std::map> data = { + {"x", bytes({1.0004f})}, {"w", bytes({Half(1.0f)})}}; + EXPECT_EQ(reference_outputs(spec, {}, data).at("y")[0], 1.0); + EXPECT_EQ( + reference_outputs(spec, {{"x", "float32"}, {"y", "float32"}}, data) + .at("y")[0], + double(1.0004f)); +} + +TEST(BenchmarkReference, LinearRejectsMismatchedOutputShape) { + Json spec = linear_spec(); + spec["tensors"]["y"]["sizes"] = {4, 1}; + EXPECT_THROW( + reference_outputs( + spec, + {}, + {{"x", bytes({1, 2, 3, -1, 0, 2})}, + {"w", bytes({2, -1, 1, 0, 3, -2})}, + {"b", bytes({0, 0})}}), + std::invalid_argument); +} + +TEST(BenchmarkReference, GroupedConvolutionHonorsStridePaddingAndDilation) { + const Json spec = { + {"operator", "aten.convolution.default"}, + {"args", + {{{"tensor", "x"}}, + {{"tensor", "w"}}, + {{"tensor", "b"}}, + {{"int_list", {1, 2}}}, + {{"int_list", {0, 1}}}, + {{"int_list", {1, 2}}}, + {{"bool", false}}, + {{"int_list", {0, 0}}}, + {{"int", 2}}, + {{"tensor", "y"}}}}, + {"tensors", + {{"x", tensor({1, 2, 1, 5})}, + {"w", tensor({2, 1, 1, 2})}, + {"b", tensor({2})}, + {"y", tensor({1, 2, 1, 3})}}}}; + const auto result = reference_outputs( + spec, + {}, + {{"x", bytes({1, 2, 3, 4, 5, 10, 20, 30, 40, 50})}, + {"w", bytes({1, 2, -1, 1})}, + {"b", bytes({1, -1})}}, + 2); + EXPECT_EQ(result.at("y"), (std::vector{5, 11, 5, 19, 19, -41})); +} + +TEST(BenchmarkReference, Q4LinearUnpacksNibblesAndUsesPerGroupScales) { + const Json spec = { + {"operator", "et_vk.q4gsw_linear.default"}, + {"args", + {{{"tensor", "x"}}, + {{"tensor", "w"}}, + {{"tensor", "scales"}}, + {{"int", 2}}, + {{"tensor", "b"}}, + {{"tensor", "y"}}}}, + {"tensors", + {{"x", tensor({1, 4})}, + {"w", tensor({2, 2}, "uint8")}, + {"scales", tensor({2, 2})}, + {"b", tensor({2})}, + {"y", tensor({1, 2})}}}}; + const auto result = reference_outputs( + spec, + {}, + {{"x", bytes({1, 2, 3, 4})}, + {"w", bytes({0x98, 0x76, 0xAF, 0x80})}, + {"scales", bytes({1, 2, 3, 4})}, + {"b", bytes({0.5, -0.5})}}); + EXPECT_EQ(result.at("y"), (std::vector{-27.5, -74.5})); +} + +TEST(BenchmarkReference, Q8ConvolutionAppliesZeroPointsScalesBiasAndClamping) { + const Json spec = { + {"operator", "et_vk.q8ta_conv2d.default"}, + {"args", + {{{"tensor", "x"}}, + {{"float", 0.5}}, + {{"int", 2}}, + {{"tensor", "w"}}, + {{"tensor", "sums"}}, + {{"tensor", "scales"}}, + {{"float", 0.25}}, + {{"int", -3}}, + {{"tensor", "b"}}, + {{"int_list", {1, 1}}}, + {{"int_list", {1, 1}}}, + {{"int_list", {0, 0}}}, + {{"int_list", {1, 1}}}, + {{"int", 1}}, + {{"string", "none"}}, + {{"tensor", "y"}}}}, + {"tensors", + {{"x", tensor({1, 2, 1, 2}, "int8x4")}, + {"w", tensor({2, 2}, "int8")}, + {"sums", tensor({2}, "int32")}, + {"scales", tensor({2})}, + {"b", tensor({2})}, + {"y", tensor({1, 2, 1, 2}, "int8x4")}}}}; + const auto result = reference_outputs( + spec, + {}, + {{"x", bytes({4, -2, 8, 3})}, + {"w", bytes({1, 2, -2, 1})}, + {"sums", bytes({3, -1})}, + {"scales", bytes({0.25, 20})}, + {"b", bytes({0.5, -1})}}); + EXPECT_EQ(result.at("y"), (std::vector{6, -2, 73, 127})); + Json relu = spec; + relu["args"][14]["string"] = "relu"; + EXPECT_EQ( + reference_outputs( + relu, + {}, + {{"x", bytes({0, 0, 0, 0})}, + {"w", bytes({1, 2, -2, 1})}, + {"sums", bytes({3, -1})}, + {"scales", bytes({0.25, 20})}, + {"b", bytes({0, 0})}}) + .at("y"), + (std::vector{-3, -3, 77, 77})); +} +TEST(BenchmarkReference, RawAdditionBroadcastsAndAppliesAlpha) { + const Json spec = { + {"shader", "binary_add_texture3d_float"}, + {"bindings", + {{{"tensor", "out"}, {"access", "write"}}, + {{"tensor", "a"}}, + {{"tensor", "b"}}}}, + {"push_constants", {{{"dtype", "float32"}, {"data", {2}}}}}, + {"tensors", + {{"out", tensor({2, 3})}, {"a", tensor({2, 1})}, {"b", tensor({3})}}}}; + EXPECT_EQ( + reference_outputs( + spec, + {}, + {{"a", bytes({1, 10})}, {"b", bytes({2, 3, 4})}}) + .at("out"), + (std::vector{5, 7, 9, 14, 16, 18})); +} + +TEST(BenchmarkReference, CopyPreservesLogicalValues) { + const Json spec = { + {"shader", "buffer_to_buffer_float"}, + {"bindings", + {{{"tensor", "out"}, {"access", "write"}}, {{"tensor", "in"}}}}, + {"tensors", {{"out", tensor({4})}, {"in", tensor({4})}}}}; + EXPECT_EQ( + reference_outputs(spec, {}, {{"in", bytes({-1, 0, 2, 4})}}) + .at("out"), + (std::vector{-1, 0, 2, 4})); +} + +TEST(BenchmarkReference, ExplicitReferenceOverridesUnknownShader) { + const Json spec = {{"shader", "custom"}, {"reference", "copy"}}; + EXPECT_EQ(reference_name(spec), "copy"); + EXPECT_THROW(reference_name({{"shader", "custom"}}), std::invalid_argument); +} + +TEST(BenchmarkReference, ComparisonReportsBoundedMismatchSummary) { + Json spec = linear_spec(); + spec["validation_method"] = { + {"function", "is_close"}, {"atol", 0.1}, {"rtol", 0.01}}; + const Report result = compare_outputs( + spec, {{"y", {1, 2, 3, 4}}}, {{"y", {1.05, 2.5, 3, 4.5}}}, {}); + EXPECT_EQ(result.at("status"), "FAIL"); + EXPECT_EQ(result.at("outputs").at("y").at("mismatch_count"), 2); + EXPECT_DOUBLE_EQ(result.at("outputs").at("y").at("max_abs_error"), 0.5); + EXPECT_FALSE(result.at("outputs").at("y").contains("values")); + EXPECT_EQ( + compare_outputs(spec, {{"y", {1, 2, 3, 4}}}, {{"y", {1.05, 2, 3, 4}}}, {}) + .at("status"), + "PASS"); +} + +TEST(BenchmarkReference, SelectedOutputsDoNotHideMissingReferenceResults) { + Json spec = linear_spec(); + const ReferenceOutputs expected{{"y", {1, 2, 3, 4}}, {"b", {1, 2}}}; + const ReferenceOutputs actual{{"y", {1, 2, 3, 4}}, {"b", {10, 20}}}; + EXPECT_EQ(compare_outputs(spec, expected, actual, {}).at("status"), "FAIL"); + spec["validation_method"] = {{"function", "is_close"}, {"outputs", {"y"}}}; + const Report selected = compare_outputs(spec, expected, actual, {}); + EXPECT_EQ(selected.at("status"), "PASS"); + EXPECT_EQ(selected.at("outputs").size(), 1); + EXPECT_EQ( + compare_outputs(spec, expected, {{"b", {1, 2}}}, {}).at("status"), + "FAIL"); + EXPECT_EQ( + compare_outputs(spec, {{"b", {1, 2}}}, actual, {}).at("status"), "FAIL"); +} + +TEST(BenchmarkReference, ComparisonRejectsMissingShortAndUnexpectedOutputs) { + const Json spec = linear_spec(); + const ReferenceOutputs expected = {{"y", {1, 2, 3, 4}}}; + EXPECT_EQ(compare_outputs(spec, expected, {}, {}).at("status"), "FAIL"); + EXPECT_EQ( + compare_outputs(spec, expected, {{"y", {1}}}, {}).at("status"), "FAIL"); + EXPECT_EQ( + compare_outputs(spec, expected, {{"y", {1, 2, 3, 4}}, {"extra", {1}}}, {}) + .at("status"), + "FAIL"); +} + +TEST(BenchmarkReference, ComparisonRejectsNonfiniteValues) { + const Json spec = linear_spec(); + const double nan = std::numeric_limits::quiet_NaN(); + const double inf = std::numeric_limits::infinity(); + EXPECT_EQ( + compare_outputs( + spec, {{"y", {1, 2, 3, 4}}}, {{"y", {1, nan, inf, 4}}}, {}) + .at("status"), + "FAIL"); + EXPECT_EQ( + compare_outputs( + spec, {{"y", {nan, 2, 3, 4}}}, {{"y", {nan, 2, 3, 4}}}, {}) + .at("status"), + "FAIL"); +} + +TEST(BenchmarkReference, ComparisonDefaultsDependOnActualDtype) { + Json spec = linear_spec(); + spec["tensors"]["y"]["dtype"] = "float16"; + const ReferenceOutputs expected = {{"y", {1, 2, 3, 4}}}; + const ReferenceOutputs close = {{"y", {1.005, 2, 3, 4}}}; + EXPECT_EQ(compare_outputs(spec, expected, close, {}).at("status"), "PASS"); + EXPECT_EQ( + compare_outputs(spec, expected, close, {{"y", "float32"}}).at("status"), + "FAIL"); + EXPECT_EQ( + compare_outputs(spec, expected, close, {{"y", "int8x4"}}).at("status"), + "FAIL"); + spec["validation_method"] = {{"function", "is_close"}, {"atol", -1}}; + EXPECT_THROW( + compare_outputs(spec, expected, close, {}), std::invalid_argument); +} + +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/test/report_test.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/test/report_test.cpp new file mode 100644 index 00000000000..7550d0f650f --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/test/report_test.cpp @@ -0,0 +1,980 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +#include +#include + +namespace vkcompute::shader_profiling { +namespace { + +TEST(BenchmarkReport, ShaderBreakdownKeepsDispatchOrderAndDuplicateShaders) { + const Report dispatches = Report::array( + {{{"dispatch_index", 0}, + {"shader_name", "copy"}, + {"gwg_size", {{"method", "operator"}, {"value", {8, 1, 1}}}}, + {"lwg_size", {{"method", "operator"}, {"value", {4, 1, 1}}}}}, + {{"dispatch_index", 1}, + {"shader_name", "copy"}, + {"gwg_size", {{"method", "operator"}, {"value", {16, 1, 1}}}}, + {"lwg_size", {{"method", "operator"}, {"value", {8, 1, 1}}}}}}); + const Report breakdown = + shader_breakdown_report(dispatches, {{1, 10}, {2, 20}, {3, 30}}); + ASSERT_EQ(breakdown.size(), 2); + EXPECT_EQ(breakdown.at(0).at("shader_name"), "copy"); + EXPECT_EQ(breakdown.at(1).at("dispatch_index"), 1); + EXPECT_EQ( + breakdown.at(1).at("gwg_size").at("value"), Report::array({16, 1, 1})); + EXPECT_EQ(breakdown.at(0).at("gpu_timestamp_shader_us").at("mean"), 2); + EXPECT_NEAR( + breakdown.at(1).at("gpu_timestamp_shader_us").at("p90").get(), + 28, + 1e-12); + EXPECT_TRUE(shader_breakdown_report(Report::array(), {{}, {}}).empty()); + const Report averaged = shader_breakdown_report( + dispatches, {{1, 10, 3, 30}, {2, 20, 4, 40}, {3, 30, 5, 50}}, 2); + EXPECT_EQ(averaged.at(0).at("gpu_timestamp_shader_us").at("mean"), 3); + EXPECT_EQ(averaged.at(1).at("gpu_timestamp_shader_us").at("p90"), 38); + EXPECT_THROW( + shader_breakdown_report(dispatches, {{1}}), std::invalid_argument); + EXPECT_THROW( + shader_breakdown_report(dispatches, {}, 0), std::invalid_argument); +} + +TEST(BenchmarkReport, TableShowsShaderBreakdownUnderOperator) { + Report report = { + {"operator_name", "aten.linear.default"}, + {"args", Report::array()}, + {"shader_breakdown", + Report::array( + {{{"shader_name", "linear_shader"}, + {"gwg_size", {{"method", "operator"}, {"value", {8, 1, 1}}}}, + {"lwg_size", {{"method", "operator"}, {"value", {4, 1, 1}}}}, + {"gpu_timestamp_shader_us", {{"mean", 1.25}, {"p90", 2.5}}}}})}}; + report.update(measurement_report(1, true, {5, 10}, {})); + const std::string table = format_table(Report::array({report})); + EXPECT_NE(table.find(" linear_shader"), std::string::npos); + EXPECT_LT(table.find("aten.linear.default"), table.find("linear_shader")); + EXPECT_NE(table.find("[8,1,1]"), std::string::npos); + EXPECT_NE(table.find("[4,1,1]"), std::string::npos); + EXPECT_NE(table.find("1.250"), std::string::npos); + EXPECT_NE(table.find("2.500"), std::string::npos); +} + +TEST(BenchmarkReport, TableShowsResolvedSizesInputsAndPerShaderStatistics) { + Report report = { + {"shader_name", "add"}, + {"gwg_size", {{"method", "default_pick_gwg"}, {"value", {8, 2, 1}}}}, + {"lwg_size", {{"method", "explicit"}, {"value", {4, 2, 1}}}}, + {"bindings", + {{"out", + {{"type", "buffer"}, + {"access", "write"}, + {"sizes", {99}}, + {"dtype", "int32"}}}, + {"a", + {{"type", "texture3d"}, + {"access", "read"}, + {"sizes", {1, 1, 1, 8}}, + {"dtype", "float32"}}}, + {"b", + {{"type", "buffer"}, + {"access", "read_write"}, + {"sizes", {8}}, + {"dtype", "float16"}}}, + {"meta", {{"type", "meta_ubo"}}}}}}; + report.update(measurement_report(10, false, {10, 20, 30}, {})); + const std::string table = format_table(Report::array({report})); + EXPECT_EQ( + table, + "Shader / operator GWG LWG Inputs Mean GPU us P90 GPU us\n" + "-------------------------------------------------------------------------------------------------\n" + "add [8,2,1] [4,2,1] t3[1,1,1,8]f32.width, b[8]f16.width 2.000 2.800"); +} + +TEST(BenchmarkReport, TableSupportsOperatorsAndUnavailableTimestamps) { + Report report = { + {"operator_name", "aten.linear.default"}, + {"args", + Report::array( + {{{"tensor", + {{"sizes", {16, 32}}, {"dtype", "int8"}, {"access", "read"}}}}, + nullptr, + {{"int", 32}}, + {{"tensor", + {{"sizes", {4, 32}}, + {"dtype", "float32"}, + {"requested_dtype", "float16"}, + {"access", "read_write"}}}}, + {{"tensor", + {{"sizes", {4, 16}}, + {"dtype", "float32"}, + {"access", "write"}}}}})}}; + report.update(measurement_report(2, true, {}, {4})); + const std::string table = format_table(Report::array({report})); + EXPECT_NE( + table.find("b[16,32]i8.width, null, 32, b[4,32]f32.width"), + std::string::npos); + EXPECT_EQ(table.find("[4,16]"), std::string::npos); + EXPECT_EQ(table.find("f16"), std::string::npos); + EXPECT_NE(table.find("N/A"), std::string::npos); +} + +TEST(BenchmarkReport, TableIncludesShaderUniformsPushConstantsAndSweepIndexes) { + const Json spec = { + {"shader", "add"}, + {"sweep_index", 3}, + {"gwg_size", {8, 1, 1}}, + {"lwg_size", {4, 1, 1}}, + {"tensors", Json::object()}, + {"bindings", + Json::array( + {{{"uniform", + Json::array({{{"dtype", "int32"}, {"data", {4, 5}}}})}}})}, + {"push_constants", Json::array({{{"dtype", "float32"}, {"data", {2}}}})}}; + Report report = shader_report( + spec, {8, 1, 1}, {4, 1, 1}, Json::object(), Report::object()); + report.update(measurement_report(1, false, {2}, {})); + const Report error = error_report(spec, "bad shader"); + EXPECT_EQ(report.at("sweep_index"), 3); + EXPECT_EQ(error.at("sweep_index"), 3); + const std::string table = format_table(Report::array({report, error})); + EXPECT_NE(table.find("add [3]"), std::string::npos); + EXPECT_NE(table.find("[4,5], [2]"), std::string::npos); + EXPECT_NE(table.find("ERROR add [3]: bad shader"), std::string::npos); +} + +TEST(BenchmarkReport, TableIncludesScalarInputsAndAllStorageTypes) { + Report report = { + {"operator_name", "aten.convolution.default"}, + {"args", + Report::array( + {{{"tensor", + {{"type", "buffer"}, + {"sizes", {1, 3, 32, 32}}, + {"dtype", "float32"}}}}, + {{"tensor", + {{"type", "constant"}, + {"sizes", {16, 3, 3, 3}}, + {"layout", "height"}, + {"dtype", "float16"}}}}, + {{"tensor", + {{"type", "texture2d"}, + {"sizes", {16}}, + {"layout", "height"}, + {"dtype", "float32"}}}}, + {{"tensor", + {{"type", "buffer"}, + {"sizes", {60, 64, 40, 51}}, + {"layout", "4w4c"}, + {"dtype", "int8x4"}}}}, + {{"int_list", {2, 2}}}, + {{"int_list", {1, 1}}}, + {{"int_list", {1, 1}}}, + {{"bool", false}}, + {{"float", 0.5}}, + {{"string", "same"}}, + {{"int", 1}}, + {{"tensor", + {{"type", "texture3d"}, + {"sizes", {1, 16, 16, 16}}, + {"dtype", "float32"}, + {"access", "write"}}}}})}}; + report.update(measurement_report(1, true, {1}, {})); + TableOptions options; + options.inputs_max_chars = 0; + const std::string table = format_table(Report::array({report}), options); + EXPECT_NE( + table.find( + "b[1,3,32,32]f32.width, c[16,3,3,3]f16, t2[16]f32.height, b[60,64,40,51]i8x4.4w4c, [2,2], [1,1], [1,1], false, 0.5, \"same\", 1"), + std::string::npos); + EXPECT_EQ(table.find("c[16,3,3,3]f16."), std::string::npos); + EXPECT_EQ(table.find("t3[1,16,16,16]"), std::string::npos); +} + +TEST(BenchmarkReport, InputFloatsUseThreeSignificantFigures) { + Report report = { + {"operator_name", "op"}, + {"args", + Report::array( + {{{"float", 1.234567}}, + {{"float", -0.01234567}}, + {{"float", 9.999}}, + {{"float_list", {12345.67, 0.0000001234567, 0.0, 0.5}}}, + {{"int", 1234567}}, + {{"int_list", {12345, 67890}}}, + {{"string", "1.234567"}}, + false, + nullptr})}}; + report.update(measurement_report(1, true, {1}, {})); + const Report original = report; + const std::string expected = + "1.23, -0.0123, 10, [1.23e+04,1.23e-07,0,0.5], 1234567, [12345,67890], \"1.234567\", false, null"; + TableOptions options; + options.inputs_max_chars = 0; + EXPECT_NE( + format_table(Report::array({report}), options).find(expected), + std::string::npos); + const Report compact = Report::parse(format_condensed_report(report)); + EXPECT_EQ(compact.at("args_str"), expected); + EXPECT_EQ( + Report::parse(format_report(report)).at("args"), original.at("args")); + EXPECT_EQ(report, original); +} + +TEST(BenchmarkReport, ShaderInputFloatsUseThreeSignificantFigures) { + Report report = { + {"shader_name", "shader"}, + {"gwg_size", {{"value", {1, 1, 1}}}}, + {"lwg_size", {{"value", {1, 1, 1}}}}, + {"bindings", + {{"params", + {{"type", "uniform"}, + {"values", Report::array({{{"data", {1.234567, 1234567}}}})}}}}}, + {"push_constants", Report::array({{{"data", {-0.01234567, 9.999}}}})}}; + report.update(measurement_report(1, false, {1}, {})); + EXPECT_NE( + format_table(Report::array({report})) + .find("[1.23,1234567], [-0.0123,10]"), + std::string::npos); + const Report compact = Report::parse(format_condensed_report(report)); + EXPECT_EQ(compact.at("bindings"), report.at("bindings")); + EXPECT_EQ(compact.at("push_constants"), report.at("push_constants")); +} + +TEST(BenchmarkReport, TableSeparatesCasesButKeepsTheirShadersTogether) { + Report report = { + {"operator_name", "first"}, + {"args", Report::array()}, + {"shader_breakdown", + Report::array( + {{{"shader_name", "child"}, + {"gwg_size", {{"value", {1, 1, 1}}}}, + {"lwg_size", {{"value", {1, 1, 1}}}}, + {"gpu_timestamp_shader_us", {{"mean", 1}, {"p90", 1}}}}})}}; + report.update(measurement_report(1, true, {2}, {})); + Report second = report; + second["operator_name"] = "second"; + second.erase("shader_breakdown"); + std::istringstream stream(format_table(Report::array({report, second}))); + std::vector lines; + for (std::string line; std::getline(stream, line);) { + lines.push_back(line); + } + ASSERT_EQ(lines.size(), 6); + EXPECT_EQ(lines[1], std::string(lines[0].size(), '-')); + EXPECT_EQ(lines[4], lines[1]); + EXPECT_EQ(lines[2].find("first"), 0); + EXPECT_EQ(lines[3].find(" child"), 0); + EXPECT_EQ(lines[5].find("second"), 0); +} + +TEST(BenchmarkReport, TableLimitsTextWithoutChangingReports) { + Report report = { + {"operator_name", "operator_name_is_long"}, + {"args", Report::array({{{"string", "abcdefghijklmnopqrstuvwxyz"}}})}, + {"shader_breakdown", + Report::array( + {{{"shader_name", "shader_name_is_long"}, + {"gwg_size", {{"value", {1, 1, 1}}}}, + {"lwg_size", {{"value", {1, 1, 1}}}}, + {"gpu_timestamp_shader_us", {{"mean", 1}, {"p90", 1}}}}})}}; + report.update(measurement_report(1, true, {2}, {})); + const Report original = report; + TableOptions options; + options.name_max_chars = 12; + options.inputs_max_chars = 10; + const std::string limited = format_table(Report::array({report}), options); + EXPECT_NE(limited.find("operator_..."), std::string::npos); + EXPECT_NE(limited.find(" shader_..."), std::string::npos); + EXPECT_NE(limited.find("\"abcdef..."), std::string::npos); + EXPECT_EQ(limited.find("shader_name_is_long"), std::string::npos); + options.name_max_chars = 0; + options.inputs_max_chars = 0; + const std::string full = format_table(Report::array({report}), options); + EXPECT_NE(full.find("operator_name_is_long"), std::string::npos); + EXPECT_NE(full.find(" shader_name_is_long"), std::string::npos); + EXPECT_NE(full.find("\"abcdefghijklmnopqrstuvwxyz\""), std::string::npos); + options.name_max_chars = 3; + options.inputs_max_chars = 2; + const std::string tiny = format_table(Report::array({report}), options); + EXPECT_NE(tiny.find(" ."), std::string::npos); + EXPECT_NE(tiny.find(".."), std::string::npos); + EXPECT_EQ(report, original); +} + +TEST(BenchmarkReport, TableDisplaysOptionalValidationStatus) { + Report report = {{"operator_name", "op"}, {"args", Report::array()}}; + report.update(measurement_report(1, true, {2}, {})); + EXPECT_EQ( + format_table(Report::array({report})).find("Validation"), + std::string::npos); + Report passed = report; + passed["validation"] = {{"status", "PASS"}}; + Report failed = report; + failed["validation"] = {{"status", "FAIL"}}; + const std::string table = + format_table(Report::array({passed, failed, report})); + EXPECT_NE(table.find("Validation"), std::string::npos); + EXPECT_NE(table.find("PASS"), std::string::npos); + EXPECT_NE(table.find("FAIL"), std::string::npos); + EXPECT_NE(table.find("SKIP"), std::string::npos); +} + +TEST(BenchmarkReport, ReportsOmitLegacyCaseMetadata) { + const Json spec = { + {"operator", "aten.linear.default"}, + {"args", Json::array()}, + {"tensors", Json::object()}, + {"legacy_case", {{"op", "linear"}, {"set", "debug"}, {"index", 0}}}}; + EXPECT_FALSE(operator_report(spec, {}).contains("legacy_case")); + EXPECT_FALSE(error_report(spec, "test error").contains("legacy_case")); +} + +TEST(BenchmarkReport, ExplicitRouteMetadataSurvivesResultsAndErrors) { + Json spec = { + {"operator", "aten.convolution.default"}, + {"args", Json::array()}, + {"tensors", Json::object()}, + {"route", + {{"name", "im2col"}, + {"options", {{"intermediate_storage", "buffer"}}}}}}; + const Report expected = { + {"requested", spec.at("route")}, {"resolved", nullptr}}; + EXPECT_EQ(operator_report(spec, Json::object()).at("route"), expected); + EXPECT_EQ(error_report(spec, "Unsupported shape").at("route"), expected); + spec.erase("route"); + EXPECT_FALSE(operator_report(spec, Json::object()).contains("route")); + EXPECT_FALSE(error_report(spec, "Unsupported shape").contains("route")); +} + +TEST(BenchmarkReport, TableShowsRequestedAndResolvedRoutes) { + Report forced = { + {"operator_name", "forced"}, + {"args", Report::array()}, + {"route", + {{"requested", + {{"name", "im2col"}, + {"options", {{"intermediate_storage", "buffer"}}}}}, + {"resolved", + {{"name", "im2col"}, + {"options", {{"intermediate_storage", "buffer"}}}}}}}, + {"shader_breakdown", + Report::array( + {{{"shader_name", "child"}, + {"gwg_size", {{"value", {1, 1, 1}}}}, + {"lwg_size", {{"value", {1, 1, 1}}}}, + {"gpu_timestamp_shader_us", {{"mean", 1}, {"p90", 1}}}}})}}; + forced.update(measurement_report(1, true, {2}, {})); + Report automatic = forced; + automatic["operator_name"] = "automatic"; + automatic["route"] = { + {"requested", {{"name", "auto"}, {"options", Report::object()}}}, + {"resolved", {{"name", "depthwise"}, {"options", Report::object()}}}}; + automatic.erase("shader_breakdown"); + Report unknown = automatic; + unknown["operator_name"] = "unknown"; + unknown["route"]["resolved"] = nullptr; + Report omitted = unknown; + omitted["operator_name"] = "omitted"; + omitted.erase("route"); + const std::string table = + format_table(Report::array({forced, automatic, unknown, omitted})); + EXPECT_LT(table.find("Shader / operator"), table.find("Route")); + EXPECT_LT(table.find("Route"), table.find("GWG")); + EXPECT_NE(table.find("im2col[buffer]"), std::string::npos); + EXPECT_NE(table.find("auto->depthwise"), std::string::npos); + std::istringstream stream(table); + std::vector lines; + for (std::string line; std::getline(stream, line);) { + lines.push_back(line); + } + ASSERT_EQ(lines.size(), 10); + EXPECT_EQ(lines[1], std::string(lines[0].size(), '-')); + std::istringstream child(lines[3]); + std::string shader_name, route, gwg; + child >> shader_name >> route >> gwg; + EXPECT_EQ(shader_name, "child"); + EXPECT_EQ(route, "-"); + EXPECT_EQ(gwg, "[1,1,1]"); + EXPECT_NE(lines[7].find("auto"), std::string::npos); + EXPECT_EQ(lines[7].find("auto->"), std::string::npos); +} + +TEST(BenchmarkReport, TableKeepsRouteOnUnsupportedCaseErrors) { + const Report error = error_report( + {{"operator", "aten.convolution.default"}, + {"route", + {{"name", "im2col"}, + {"options", {{"intermediate_storage", "texture3d"}}}}}}, + "Unsupported shape"); + const std::string table = format_table(Report::array({error})); + EXPECT_NE(table.find("im2col[texture3d]"), std::string::npos); + EXPECT_NE(table.find("ERROR"), std::string::npos); + EXPECT_NE(table.find("Unsupported shape"), std::string::npos); +} + +TEST(BenchmarkReport, TableRendersSkippedCasesWithoutTimingMeasurements) { + const Report skipped = { + {"case_name", "forced convolution"}, + {"operator_name", "aten.convolution.default"}, + {"sweep_index", 2}, + {"status", "SKIP"}, + {"skip_reason", "Unsupported\nshape"}, + {"route", + {{"requested", + {{"name", "im2col"}, + {"options", {{"intermediate_storage", "buffer"}}}}}, + {"resolved", nullptr}}}, + {"args", + Report::array( + {{{"tensor", + {{"sizes", {1, 3, 8, 8}}, + {"dtype", "float32"}, + {"type", "buffer"}}}}, + {{"int_list", {2, 2}}}, + {{"tensor", + {{"sizes", {1, 16, 4, 4}}, + {"dtype", "float32"}, + {"type", "buffer"}, + {"access", "write"}}}}})}, + {"throughput", {{"memory_bytes", nullptr}}}}; + TableOptions options; + options.name_max_chars = 0; + const std::string table = format_table(Report::array({skipped}), options); + EXPECT_NE( + table.find("aten.convolution.default [2] (SKIP)"), std::string::npos); + EXPECT_NE(table.find("im2col[buffer]"), std::string::npos); + EXPECT_NE(table.find("b[1,3,8,8]f32.width, [2,2]"), std::string::npos); + EXPECT_EQ(table.find("b[1,16,4,4]"), std::string::npos); + EXPECT_NE(table.find("N/A"), std::string::npos); + EXPECT_NE( + table.find("SKIP forced convolution [2]: Unsupported shape"), + std::string::npos); + Report without_args = skipped; + without_args.erase("args"); + without_args["validation"] = {{"status", "SKIP"}}; + EXPECT_NE( + format_table(Report::array({without_args})).find("Validation"), + std::string::npos); +} + +TEST(BenchmarkReport, TableDisplaysRequestedGpuThroughputColumns) { + Report report = {{"operator_name", "op"}, {"args", Report::array()}}; + report.update(measurement_report(1, true, {2}, {})); + const std::string disabled = format_table(Report::array({report})); + EXPECT_EQ(disabled.find("GPU GiB/s"), std::string::npos); + EXPECT_EQ(disabled.find("GPU GFLOPS"), std::string::npos); + + report["throughput"] = { + {"memory_bytes", 4096}, + {"gpu_timestamp", {{"memory_gib_per_s", {{"mean", 4.125}}}}}}; + const std::string memory_only = format_table(Report::array({report})); + EXPECT_NE(memory_only.find("GPU GiB/s"), std::string::npos); + EXPECT_NE(memory_only.find("4.125"), std::string::npos); + EXPECT_EQ(memory_only.find("GPU GFLOPS"), std::string::npos); + + report["throughput"] = { + {"compute_flops", 8192}, + {"gpu_timestamp", {{"compute_gflops", {{"mean", 8.625}}}}}}; + const std::string compute_only = format_table(Report::array({report})); + EXPECT_EQ(compute_only.find("GPU GiB/s"), std::string::npos); + EXPECT_NE(compute_only.find("GPU GFLOPS"), std::string::npos); + EXPECT_NE(compute_only.find("8.625"), std::string::npos); + + report["validation"] = {{"status", "PASS"}}; + report["throughput"]["memory_bytes"] = 0; + report["throughput"]["gpu_timestamp"]["memory_gib_per_s"] = {{"mean", 0}}; + const std::string both = format_table(Report::array({report})); + EXPECT_LT(both.find("Validation"), both.find("GPU GiB/s")); + EXPECT_LT(both.find("GPU GiB/s"), both.find("GPU GFLOPS")); + EXPECT_NE(both.find("0.000"), std::string::npos); + EXPECT_NE(both.find("8.625"), std::string::npos); +} + +TEST(BenchmarkReport, TableIncludesShaderThroughputBelowOperator) { + Report report = { + {"operator_name", "op"}, + {"args", Report::array()}, + {"throughput", + {{"memory_bytes", 4096}, + {"compute_flops", 8192}, + {"gpu_timestamp", + {{"memory_gib_per_s", {{"mean", 4.125}}}, + {"compute_gflops", {{"mean", 8.625}}}}}}}, + {"shader_breakdown", + Report::array( + {{{"shader_name", "child"}, + {"gwg_size", {{"value", {1, 1, 1}}}}, + {"lwg_size", {{"value", {1, 1, 1}}}}, + {"gpu_timestamp_shader_us", {{"mean", 1}, {"p90", 1}}}, + {"throughput", + {{"memory_bytes", 1024}, + {"compute_flops", 2048}, + {"gpu_timestamp", + {{"memory_gib_per_s", {{"mean", 1.125}}}, + {"compute_gflops", {{"mean", 2.625}}}}}}}}})}}; + report.update(measurement_report(1, true, {2}, {})); + std::istringstream stream(format_table(Report::array({report}))); + std::vector lines; + for (std::string line; std::getline(stream, line);) { + lines.push_back(line); + } + ASSERT_EQ(lines.size(), 4); + EXPECT_EQ(lines[1], std::string(lines[0].size(), '-')); + EXPECT_NE(lines[2].find("4.125"), std::string::npos); + EXPECT_NE(lines[2].find("8.625"), std::string::npos); + EXPECT_EQ(lines[3].find(" child"), 0); + EXPECT_NE(lines[3].find("1.125"), std::string::npos); + EXPECT_NE(lines[3].find("2.625"), std::string::npos); +} + +TEST( + BenchmarkReport, + TableThroughputUsesNaForMissingEstimatesAndGpuTimestamps) { + Report report = { + {"operator_name", "no_timestamps"}, {"args", Report::array()}}; + report.update(measurement_report(1, true, {}, {2})); + report["throughput"] = { + {"memory_bytes", 4096}, + {"compute_flops", nullptr}, + {"gpu_timestamp", + {{"memory_gib_per_s", nullptr}, {"compute_gflops", nullptr}}}, + {"wall_time", {{"memory_gib_per_s", {111.625}}}}, + {"errors", {{"compute", "No estimate"}}}}; + Report without_estimates = report; + without_estimates["operator_name"] = "without_estimates"; + without_estimates.erase("throughput"); + const Report error = error_report({{"operator", "error_case"}}, "Bad input"); + std::istringstream stream( + format_table(Report::array({report, without_estimates, error}))); + std::vector lines; + for (std::string line; std::getline(stream, line);) { + lines.push_back(line); + } + ASSERT_EQ(lines.size(), 8); + EXPECT_NE(lines[0].find("GPU GiB/s"), std::string::npos); + EXPECT_NE(lines[0].find("GPU GFLOPS"), std::string::npos); + for (size_t i : {2u, 4u, 6u}) { + EXPECT_EQ(lines[i].substr(lines[i].size() - 3), "N/A"); + EXPECT_EQ(lines[i].find("111.625"), std::string::npos); + } + EXPECT_NE(lines[7].find("ERROR error_case: Bad input"), std::string::npos); +} + +TEST(BenchmarkReport, OperatorTensorMetadataIsInlineWithoutValues) { + const Json arg_set = Json::array( + {{{"tensor", + {{"name", "x"}, + {"sizes", {2}}, + {"dtype", "float16"}, + {"data", {1, 2}}}}}, + {{"tensor", + {{"name", "w"}, + {"sizes", {2}}, + {"storage", "constant"}, + {"fill", 0.5}}}}, + nullptr, + {{"int", 32}}, + {{"tensor", {{"ref", "x"}}}}, + {{"tensor", {{"ref", "x"}, {"access", "write"}}}}}); + const Json spec = parse_configs({{"operator", "aten.linear.default"}, + {"args_sweep", Json::array({arg_set})}}) + .at(0); + const Report report = + operator_report(spec, {{"x", "float32"}, {"w", "float32"}}); + EXPECT_FALSE(report.contains("tensors")); + EXPECT_FALSE(report.contains("inputs")); + EXPECT_FALSE(report.contains("outputs")); + EXPECT_EQ(report.at("sweep_index"), 0); + const auto& args = report.at("args"); + ASSERT_EQ(args.size(), 6); + EXPECT_EQ(args.at(0).at("tensor").at("name"), "x"); + EXPECT_EQ(args.at(0).at("tensor").at("dtype"), "float32"); + EXPECT_EQ(args.at(0).at("tensor").at("requested_dtype"), "float16"); + EXPECT_EQ(args.at(0).at("tensor").at("access"), "read"); + EXPECT_EQ(args.at(1).at("tensor").at("type"), "constant"); + EXPECT_FALSE(args.at(1).at("tensor").contains("constant")); + EXPECT_TRUE(args.at(2).is_null()); + EXPECT_EQ(args.at(3), (Report{{"int", 32}})); + EXPECT_EQ(args.at(4).at("tensor"), args.at(0).at("tensor")); + Report output = args.at(0).at("tensor"); + output["access"] = "write"; + EXPECT_EQ(args.at(5).at("tensor"), output); + EXPECT_EQ(report.dump().find("\"data\""), std::string::npos); + EXPECT_EQ(report.dump().find("\"fill\""), std::string::npos); +} + +TEST(BenchmarkReport, InferredTensorRolesAndResolvedVariablesAreVisible) { + const Json args = Json::array( + {{{"tensor", {{"name", "x"}, {"sizes", {2}}, {"dtype", "v_DTYPE"}}}}, + {{"tensor", {{"name", "y"}, {"sizes", {2}}, {"dtype", "v_DTYPE"}}}}}); + const auto cases = parse_configs( + {{"operator", "aten.relu.default"}, + {"sweep_variables", {{"v_DTYPE", {"float32", "float16"}}}}, + {"args_sweep", Json::array({args})}}); + const Report report = operator_report( + cases[1], + {{"x", "float16"}, {"y", "float16"}}, + {{"x", "read"}, {"y", "write"}}); + EXPECT_EQ(report.at("sweep_index"), 1); + EXPECT_EQ(report.at("args_sweep_index"), 0); + EXPECT_EQ(report.at("sweep_variables").at("v_DTYPE"), "float16"); + EXPECT_EQ(report.at("args")[0].at("tensor").at("access"), "read"); + EXPECT_EQ(report.at("args")[1].at("tensor").at("access"), "write"); +} + +TEST(BenchmarkReport, SweepIndexesIdentifyResultsAndErrors) { + Report result = { + {"case_name", "linear"}, + {"operator_name", "aten.linear.default"}, + {"sweep_index", 1}, + {"args", Report::array()}}; + result.update(measurement_report(1, true, {5}, {6})); + const Report error = error_report( + {{"name", "linear"}, + {"operator", "aten.linear.default"}, + {"sweep_index", 0}}, + "missing argument"); + EXPECT_EQ(error.at("sweep_index"), 0); + const std::string table = format_table(Report::array({error, result})); + EXPECT_NE(table.find("aten.linear.default [0]"), std::string::npos); + EXPECT_NE(table.find("aten.linear.default [1]"), std::string::npos); + EXPECT_NE( + table.find("ERROR linear [0]: missing argument"), std::string::npos); +} + +TEST(BenchmarkReport, TablePreservesCaseErrors) { + const Report report = error_report( + {{"shader", "missing"}, {"name", "bad\ncase"}}, "Unknown\tshader"); + const std::string table = format_table(Report::array({report})); + EXPECT_NE(table.find("ERROR bad case: Unknown shader"), std::string::npos); + EXPECT_NE(table.find("missing"), std::string::npos); +} + +TEST(BenchmarkReport, TableShowsObservedClockRangeAcrossAllPasses) { + Report report = { + {"operator_name", "op"}, + {"args", Report::array()}, + {"gpu_batch_observed_gpu_clock_freq_MHz", {{300, 400}, {nullptr, 350}}}, + {"wall_time_batch_observed_gpu_clock_freq_MHz", {{200, 250}, {0, 100}}}, + {"shader_profile_batch_observed_gpu_clock_freq_MHz", + {{600, 700}, {500, nullptr}}}, + {"shader_breakdown", + Report::array( + {{{"shader_name", "child"}, + {"gwg_size", {{"value", {1, 1, 1}}}}, + {"lwg_size", {{"value", {1, 1, 1}}}}, + {"gpu_timestamp_shader_us", {{"mean", 1}, {"p90", 2}}}}})}}; + report.update(measurement_report(1, true, {2, 3, 4}, {})); + Report missing = {{"operator_name", "missing"}, {"args", Report::array()}}; + missing.update(measurement_report(1, true, {2}, {})); + const std::string table = format_table(Report::array({report, missing})); + EXPECT_NE(table.find("Min GPU MHz"), std::string::npos); + EXPECT_NE(table.find("Max GPU MHz"), std::string::npos); + std::istringstream lines(table); + for (std::string line; std::getline(lines, line);) { + std::istringstream words(line); + std::vector fields; + for (std::string word; words >> word;) + fields.push_back(word); + if (fields.empty()) + continue; + if (fields[0] == "op") { + EXPECT_EQ(fields[fields.size() - 2], "0"); + EXPECT_EQ(fields.back(), "700"); + } else if (fields[0] == "child") { + EXPECT_EQ(fields[fields.size() - 2], "500"); + EXPECT_EQ(fields.back(), "700"); + } else if (fields[0] == "missing") { + EXPECT_EQ(fields[fields.size() - 2], "N/A"); + EXPECT_EQ(fields.back(), "N/A"); + } + } +} + +TEST(BenchmarkReport, TableHandlesUnavailableAndFractionalClockReadings) { + Report report = { + {"shader_name", "copy"}, + {"gwg_size", {{"value", {1, 1, 1}}}}, + {"lwg_size", {{"value", {1, 1, 1}}}}, + {"bindings", Report::object()}, + {"gpu_batch_observed_gpu_clock_freq_MHz", {{nullptr, nullptr}}}}; + report.update(measurement_report(1, false, {}, {10, 20})); + std::string table = format_table(Report::array({report})); + EXPECT_NE(table.find("Min GPU MHz"), std::string::npos); + EXPECT_EQ(table.substr(table.size() - 3), "N/A"); + report["wall_time_batch_observed_gpu_clock_freq_MHz"] = {{333.125, 444.75}}; + table = format_table(Report::array({report})); + EXPECT_NE(table.find("333.125"), std::string::npos); + EXPECT_NE(table.find("444.75"), std::string::npos); +} + +TEST(BenchmarkReport, GpuClockPairsStayOnOneLine) { + Report report = { + {"shader_name", "copy"}, + {"gwg_size", {{"value", {1, 1, 1}}}}, + {"lwg_size", {{"value", {1, 1, 1}}}}, + {"bindings", Report::object()}}; + report.update(measurement_report(1, false, {2}, {3})); + for (const char* field : + {"gpu_batch_observed_gpu_clock_freq_MHz", + "wall_time_batch_observed_gpu_clock_freq_MHz", + "shader_profile_batch_observed_gpu_clock_freq_MHz"}) { + Report clocked = report; + clocked[field] = {{903, 950}, {nullptr, 800}}; + const std::string text = format_report(clocked); + EXPECT_NE(text.find("[903, 950]"), std::string::npos); + EXPECT_NE(text.find("[null, 800]"), std::string::npos); + EXPECT_EQ(Report::parse(text), clocked); + } +} + +TEST(BenchmarkReport, DimensionArraysStayOnOneLine) { + const Report report = Report::array( + {{{"gwg_size", {{"method", "explicit"}, {"value", {8, 8, 1}}}}, + {"lwg_size", {{"method", "default_pick_lwg"}, {"value", {4, 4, 1}}}}, + {"bindings", + {{"input", {{"type", "texture3d"}, {"sizes", {1, 3, 224, 224}}}}}}, + {"wall_time_batch_us", {12.5, 13.5}}}}); + const std::string text = format_report(report); + EXPECT_NE(text.find("\"value\": [8, 8, 1]"), std::string::npos); + EXPECT_NE(text.find("\"value\": [4, 4, 1]"), std::string::npos); + EXPECT_NE(text.find("\"sizes\": [1, 3, 224, 224]"), std::string::npos); + EXPECT_NE(text.find("\"wall_time_batch_us\": [\n"), std::string::npos); + EXPECT_EQ(Report::parse(text), report); +} + +TEST(BenchmarkReport, FormattingPreservesEscapedStringsAndFieldOrder) { + const Report report = { + {"case_name", "quoted \"name\"\nline"}, + {"labels", {"[", "\"", "\\", "\n"}}, + {"tensors", {{"weight", {{"sizes", {8, 32}}}}}}, + {"empty_object", Report::object()}, + {"empty_array", Report::array()}}; + const std::string text = format_report(report); + EXPECT_EQ(Report::parse(text), report); + EXPECT_NE(text.find("\"sizes\": [8, 32]"), std::string::npos); +} + +TEST(BenchmarkReport, CondensedJsonUsesInputsAndSummarizesBatchTimings) { + Report report = { + {"operator_name", "aten.linear.default"}, + {"args", + Report::array( + {{{"tensor", + {{"type", "buffer"}, + {"sizes", {4, 32}}, + {"dtype", "float32"}, + {"layout", "width"}, + {"access", "read"}}}}, + {{"tensor", + {{"type", "constant"}, + {"sizes", {16, 32}}, + {"dtype", "float32"}, + {"layout", "width"}}}}, + {{"int_list", {1, 2}}}, + {{"tensor", + {{"type", "buffer"}, + {"sizes", {4, 16}}, + {"dtype", "float32"}, + {"access", "write"}}}}})}, + {"benchmark_runs", 4}, + {"shader_breakdown", + Report::array( + {{{"shader_name", "linear"}, + {"gpu_timestamp_shader_us", latency_statistics({1, 2, 3})}}})}, + {"validation", {{"status", "PASS"}}}}; + report.update( + measurement_report(10, true, {60, 20, 90, 30}, {80, 40, 20, 60})); + const Report original = report; + const Report condensed = Report::parse(format_condensed_report(report)); + EXPECT_FALSE(condensed.contains("args")); + EXPECT_EQ(condensed.at("args_str"), "b[4,32]f32.width, c[16,32]f32, [1,2]"); + EXPECT_EQ( + condensed.at("gpu_timestamp_operator_us"), + (Report{{"min", 2}, {"median", 4.5}, {"max", 9}})); + EXPECT_EQ( + condensed.at("gpu_timestamp_batch_us"), + (Report{{"min", 20}, {"median", 45}, {"max", 90}})); + EXPECT_EQ( + condensed.at("wall_time_operator_us"), + (Report{{"min", 2}, {"median", 5}, {"max", 8}})); + EXPECT_EQ(condensed.at("shader_breakdown"), report.at("shader_breakdown")); + EXPECT_EQ(condensed.at("validation"), report.at("validation")); + EXPECT_EQ(report, original); +} + +TEST(BenchmarkReport, CondensedJsonSummarizesPassClockRanges) { + const Report report = { + {"gpu_batch_observed_gpu_clock_freq_MHz", {{903, 800}, {nullptr, 950}}}, + {"wall_time_batch_observed_gpu_clock_freq_MHz", {{0, 700}, {600, 650}}}, + {"shader_profile_batch_observed_gpu_clock_freq_MHz", + {{nullptr, nullptr}}}, + {"gpu_timestamp_operator_us", Report::array()}, + {"gpu_timestamp_batch_us", Report::array()}, + {"wall_time_batch_us", {3, 1, 2}}}; + const Report condensed = Report::parse(format_condensed_report(report)); + EXPECT_EQ( + condensed.at("gpu_batch_observed_gpu_clock_freq_MHz"), + (Report{{"min", 800}, {"max", 950}})); + EXPECT_EQ( + condensed.at("wall_time_batch_observed_gpu_clock_freq_MHz"), + (Report{{"min", 0}, {"max", 700}})); + EXPECT_TRUE(condensed.at("shader_profile_batch_observed_gpu_clock_freq_MHz") + .is_null()); + EXPECT_TRUE(condensed.at("gpu_timestamp_operator_us").is_null()); + EXPECT_TRUE(condensed.at("gpu_timestamp_batch_us").is_null()); + EXPECT_EQ( + condensed.at("wall_time_batch_us"), + (Report{{"min", 1}, {"median", 2}, {"max", 3}})); +} + +TEST(BenchmarkReport, CondensedJsonPreservesShaderStatisticsAndCaseGroups) { + Report report = { + {"shader_name", "copy"}, + {"bindings", {{"input", {{"sizes", {8}}, {"dtype", "float32"}}}}}}; + report.update(measurement_report(10, false, {10, 20, 30}, {30, 10, 20})); + const Report error = error_report({{"operator", "bad"}}, "test error"); + const Report condensed = + Report::parse(format_condensed_report(Report::array({report, error}))); + ASSERT_EQ(condensed.size(), 2); + EXPECT_EQ(condensed[0].at("bindings"), report.at("bindings")); + EXPECT_EQ( + condensed[0].at("gpu_timestamp_shader_us"), + report.at("gpu_timestamp_shader_us")); + EXPECT_EQ( + condensed[0].at("wall_time_shader_us"), + (Report{{"min", 1}, {"median", 2}, {"max", 3}})); + EXPECT_EQ(condensed[1].at("error"), "test error"); +} + +TEST(BenchmarkReport, StatisticsUseInterpolatedPercentiles) { + const Report stats = latency_statistics({5, 1, 4, 2, 3}); + ASSERT_TRUE(stats.is_object()); + EXPECT_EQ(stats.at("min"), 1); + EXPECT_EQ(stats.at("max"), 5); + EXPECT_EQ(stats.at("mean"), 3); + EXPECT_EQ(stats.at("p50"), 3); + EXPECT_NEAR(stats.at("p90").get(), 4.6, 1e-12); + EXPECT_NEAR(stats.at("p95").get(), 4.8, 1e-12); + EXPECT_TRUE(latency_statistics({}).is_null()); +} + +TEST(BenchmarkReport, PerInvocationSamplesAreDividedByRepetitions) { + const Report report = + measurement_report(10, false, {10, 20, 30}, {20, 40, 60}); + ASSERT_TRUE(report.contains("gpu_timestamp_shader_us")); + EXPECT_EQ(report.at("gpu_timestamp_shader_us").at("mean"), 2); + EXPECT_EQ(report.at("gpu_timestamp_batch_us"), Report::array({10, 20, 30})); + EXPECT_EQ(report.at("wall_time_shader_us"), Report::array({2, 4, 6})); + EXPECT_EQ(report.at("wall_time_batch_us"), Report::array({20, 40, 60})); + EXPECT_THROW(measurement_report(0, false, {}, {1}), std::invalid_argument); +} + +TEST(BenchmarkReport, OperatorGpuSamplesPreserveRunOrderAndBatchAverages) { + Report report = {{"operator_name", "op"}, {"args", Report::array()}}; + report.update(measurement_report(10, true, {30, 10, 20}, {60, 20, 40})); + EXPECT_EQ(report.at("gpu_timestamp_batch_us"), Report::array({30, 10, 20})); + EXPECT_EQ(report.at("gpu_timestamp_operator_us"), Report::array({3, 1, 2})); + EXPECT_EQ(report.at("wall_time_operator_us"), Report::array({6, 2, 4})); + const std::string table = format_table(Report::array({report})); + EXPECT_NE(table.find("2.000"), std::string::npos); + EXPECT_NE(table.find("2.800"), std::string::npos); +} + +TEST(BenchmarkReport, UnavailableTimestampsAndOperatorsUseExplicitLabels) { + const Report report = measurement_report(2, true, {}, {4, 8}); + ASSERT_TRUE(report.contains("gpu_timestamp_operator_us")); + EXPECT_EQ(report.at("gpu_timestamp_operator_us"), Report::array()); + EXPECT_EQ(report.at("gpu_timestamp_batch_us"), Report::array()); + EXPECT_EQ(report.at("wall_time_operator_us"), Report::array({2, 4})); + EXPECT_FALSE(report.contains("gpu_timestamp_shader_us")); +} + +TEST(BenchmarkReport, ShaderDetailsHaveStableOrderAndNoTensorValues) { + const Json spec = + parse_configs( + {{"name", "copy"}, + {"shader", "buffer_to_buffer_float"}, + {"gwg_size", {8, 1, 1}}, + {"lwg_size", "default_pick_lwg"}, + {"bindings", + Json::array( + {{{"buffer", + {{"name", "output"}, {"sizes", {8}}, {"access", "write"}}}}, + {{"buffer", + {{"name", "input"}, + {"sizes", {8}}, + {"data", {1, 2, 3, 4, 5, 6, 7, 8}}}}}, + {{"meta_ubo", "output"}, {"field", "numel"}}})}, + {"outputs", {"output"}}, + {"dump_outputs", true}}) + .at(0); + const Report report = shader_report( + spec, + {8, 1, 1}, + {8, 1, 1}, + {{"input", "float32"}, {"output", "float32"}}, + Report::object()); + std::vector keys; + for (const auto& item : report.items()) { + keys.push_back(item.key()); + } + EXPECT_EQ( + keys, + (std::vector{ + "case_name", + "shader_name", + "gwg_size", + "lwg_size", + "bindings", + "specialization_constants"})); + ASSERT_TRUE(report.contains("bindings")); + EXPECT_EQ(report.at("gwg_size").at("method"), "explicit"); + EXPECT_EQ(report.at("lwg_size").at("method"), "default_pick_lwg"); + EXPECT_EQ(report.at("bindings").at("input").at("type"), "buffer"); + EXPECT_EQ(report.at("bindings").at("input").at("binding_index"), 1); + EXPECT_FALSE(report.at("bindings").at("input").contains("data")); + EXPECT_FALSE(report.contains("outputs")); +} + +TEST(BenchmarkReport, SharedBindingsAndPrecisionPromotionRemainVisible) { + const Json spec = + parse_configs({{"shader", "buffer_to_buffer_float"}, + {"gwg_size", {8, 1, 1}}, + {"lwg_size", {8, 1, 1}}, + {"bindings", + Json::array( + {{{"buffer", + {{"name", "shared"}, + {"sizes", {8}}, + {"dtype", "float16"}, + {"access", "write"}}}}, + {{"buffer", {{"ref", "shared"}}}}, + {{"meta_ubo", "shared"}, {"field", "numel"}}})}}) + .at(0); + const Report report = shader_report( + spec, {8, 1, 1}, {8, 1, 1}, {{"shared", "float32"}}, Report::object()); + const auto& bindings = report.at("bindings"); + ASSERT_EQ(bindings.size(), 3); + EXPECT_EQ(bindings.at("shared").at("dtype"), "float32"); + EXPECT_EQ(bindings.at("shared").at("requested_dtype"), "float16"); + std::vector indices; + for (const auto& info : bindings) { + indices.push_back(info.at("binding_index")); + if (info.at("binding_index") == 1) { + EXPECT_EQ(info.at("ref"), "shared"); + } + } + EXPECT_EQ(indices, (std::vector{0, 1, 2})); +} + +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/test/route_registry_test.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/test/route_registry_test.cpp new file mode 100644 index 00000000000..ae9878aa76f --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/test/route_registry_test.cpp @@ -0,0 +1,111 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +namespace vkcompute::shader_profiling { +namespace { + +Json first_builder( + ComputeGraph&, + const std::vector&, + const Json& options) { + return {{"name", "first"}, {"options", options}}; +} +Json second_builder( + ComputeGraph&, + const std::vector&, + const Json& options) { + return {{"name", "second"}, {"options", options}}; +} +Json inspect_builder( + ComputeGraph&, + const std::vector&, + size_t, + size_t) { + return {{"name", "first"}, {"options", Json::object()}}; +} + +REGISTER_OPERATOR_ROUTES { + REGISTER_OPERATOR_ROUTE(test_operator, candidate, first_builder); + REGISTER_ROUTE_INSPECTOR(test_operator, inspect_builder); +} + +TEST(BenchmarkRouteRegistry, RoutesAreScopedToOperators) { + RouteRegistry registry; + registry.register_operator_route("op_a", "direct", first_builder); + registry.register_operator_route("op_b", "direct", second_builder); + registry.register_operator_route("op_a", "other", second_builder); + EXPECT_EQ(registry.operator_route("op_a", "direct"), first_builder); + EXPECT_EQ(registry.operator_route("op_b", "direct"), second_builder); + EXPECT_EQ(registry.operator_route("op_a", "other"), second_builder); +} + +TEST(BenchmarkRouteRegistry, BuildersCanBeSharedAcrossRoutes) { + RouteRegistry registry; + registry.register_operator_route("op_a", "direct", first_builder); + registry.register_operator_route("op_a", "other", first_builder); + registry.register_operator_route("op_b", "direct", first_builder); + EXPECT_EQ(registry.operator_route("op_a", "direct"), first_builder); + EXPECT_EQ(registry.operator_route("op_a", "other"), first_builder); + EXPECT_EQ(registry.operator_route("op_b", "direct"), first_builder); +} + +TEST(BenchmarkRouteRegistry, DuplicateRoutesPreserveOriginals) { + RouteRegistry registry; + registry.register_operator_route("op", "direct", first_builder); + EXPECT_THROW( + registry.register_operator_route("op", "direct", second_builder), + std::invalid_argument); + EXPECT_EQ(registry.operator_route("op", "direct"), first_builder); +} + +TEST(BenchmarkRouteRegistry, InspectorsAreOptionalAndUnique) { + RouteRegistry registry; + EXPECT_EQ(registry.operator_inspector("op"), nullptr); + registry.register_operator_inspector("op", inspect_builder); + EXPECT_EQ(registry.operator_inspector("op"), inspect_builder); + EXPECT_THROW( + registry.register_operator_inspector("op", inspect_builder), + std::invalid_argument); +} + +TEST(BenchmarkRouteRegistry, RejectsMissingAndInvalidRegistrations) { + RouteRegistry registry; + EXPECT_THROW(registry.operator_route("op", "missing"), std::invalid_argument); + registry.register_operator_route("op", "direct", first_builder); + EXPECT_THROW(registry.operator_route("op", "missing"), std::invalid_argument); + EXPECT_THROW( + registry.register_operator_route("", "direct", first_builder), + std::invalid_argument); + EXPECT_THROW( + registry.register_operator_route("op", "", first_builder), + std::invalid_argument); + EXPECT_THROW( + registry.register_operator_route("op", "direct", nullptr), + std::invalid_argument); + EXPECT_THROW( + registry.register_operator_route("op", "auto", first_builder), + std::invalid_argument); + EXPECT_THROW( + registry.register_operator_inspector("", inspect_builder), + std::invalid_argument); + EXPECT_THROW( + registry.register_operator_inspector("op", nullptr), + std::invalid_argument); +} + +TEST(BenchmarkRouteRegistry, StaticRegistrationsAreAvailable) { + const auto& registry = route_registry(); + EXPECT_EQ( + registry.operator_route("test_operator", "candidate"), first_builder); + EXPECT_EQ(registry.operator_inspector("test_operator"), inspect_builder); +} + +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/test/throughput_registry_test.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/test/throughput_registry_test.cpp new file mode 100644 index 00000000000..de8e1336943 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/test/throughput_registry_test.cpp @@ -0,0 +1,510 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +#include +#include + +namespace vkcompute::shader_profiling { +namespace { + +double memory_count(const Json&) { + return 40; +} +double compute_count(const Json&) { + return 20; +} +double negative_count(const Json&) { + return -1; +} +double infinite_count(const Json&) { + return std::numeric_limits::infinity(); +} +double nan_count(const Json&) { + return std::numeric_limits::quiet_NaN(); +} +double throwing_count(const Json&) { + throw std::invalid_argument("bad work shape"); +} + +REGISTER_THROUGHPUT_FUNCTIONS { + REGISTER_OPERATOR_MEMORY_TRAFFIC(test_counter_case, memory_count); + REGISTER_OPERATOR_COMPUTE_FLOPS(test_counter_case, compute_count); + REGISTER_SHADER_MEMORY(test_counter_case, compute_count); + REGISTER_SHADER_COMPUTE(test_counter_case, memory_count); + REGISTER_SHADER_MEMORY(test_negative, negative_count); + REGISTER_SHADER_MEMORY(test_infinite, infinite_count); + REGISTER_SHADER_MEMORY(test_nan, nan_count); + REGISTER_SHADER_MEMORY(test_throwing, throwing_count); + REGISTER_SHADER_COMPUTE(test_throwing, compute_count); + REGISTER_SHADER_MEMORY_TRAFFIC(test_prefix, memory_count); + REGISTER_SHADER_COMPUTE_FLOPS(test_prefix, compute_count); +} + +Json tensor( + const Json& sizes, + const std::string& dtype = "float32", + const std::string& access = "read", + const std::string& type = "buffer") { + return { + {"sizes", sizes}, {"dtype", dtype}, {"access", access}, {"type", type}}; +} + +Json linear_case() { + return { + {"operator_name", "aten.linear.default"}, + {"args", + {{{"tensor", tensor({2, 3})}}, + {{"tensor", tensor({4, 3}, "float32", "read", "constant")}}, + {{"tensor", tensor({4}, "float32", "read", "constant")}}, + {{"tensor", tensor({2, 4}, "float32", "write")}}}}}; +} + +TEST(BenchmarkThroughputRegistry, OperatorAndShaderCountersAreIndependent) { + ThroughputRegistry registry; + registry.register_operator_memory("same", memory_count); + registry.register_operator_compute("same", compute_count); + registry.register_shader_memory("same", compute_count); + registry.register_shader_compute("same", memory_count); + EXPECT_EQ(registry.operator_memory("same"), memory_count); + EXPECT_EQ(registry.operator_compute("same"), compute_count); + EXPECT_EQ(registry.shader_memory("same"), compute_count); + EXPECT_EQ(registry.shader_compute("same"), memory_count); +} + +TEST(BenchmarkThroughputRegistry, ShaderPrefixesMatchOnlyStartsOfNames) { + ThroughputRegistry registry; + registry.register_shader_memory_prefix("conv2d", memory_count); + registry.register_shader_compute_prefix("conv2d", compute_count); + for (const char* name : + {"conv2d", "conv2d_float", "conv2d_gemm_texture3d_half"}) { + EXPECT_EQ(registry.shader_memory(name), memory_count); + EXPECT_EQ(registry.shader_compute(name), compute_count); + } + for (const char* name : {"q8ta_conv2d_float", "xconv2d", "Conv2d_float"}) { + EXPECT_THROW(registry.shader_memory(name), std::invalid_argument); + EXPECT_THROW(registry.shader_compute(name), std::invalid_argument); + } + EXPECT_THROW(registry.operator_memory("conv2d_float"), std::invalid_argument); +} + +TEST( + BenchmarkThroughputRegistry, + ExactShaderNamesOverrideLongestMatchingPrefix) { + for (bool broad_first : {false, true}) { + ThroughputRegistry registry; + for (const char* prefix : broad_first + ? std::vector{"conv2d", "conv2d_im2col"} + : std::vector{"conv2d_im2col", "conv2d"}) { + const auto function = + std::string(prefix) == "conv2d" ? memory_count : compute_count; + registry.register_shader_memory_prefix(prefix, function); + registry.register_shader_compute_prefix(prefix, function); + } + registry.register_shader_memory( + "conv2d_im2col_buffer_float", negative_count); + registry.register_shader_compute( + "conv2d_im2col_buffer_float", negative_count); + EXPECT_EQ(registry.shader_memory("conv2d_float"), memory_count); + EXPECT_EQ(registry.shader_compute("conv2d_float"), memory_count); + EXPECT_EQ( + registry.shader_memory("conv2d_im2col_texture3d_half"), compute_count); + EXPECT_EQ( + registry.shader_compute("conv2d_im2col_texture3d_half"), compute_count); + EXPECT_EQ( + registry.shader_memory("conv2d_im2col_buffer_float"), negative_count); + EXPECT_EQ( + registry.shader_compute("conv2d_im2col_buffer_float"), negative_count); + } +} + +TEST( + BenchmarkThroughputRegistry, + PrefixRegistrationsRejectInvalidAndDuplicateNames) { + ThroughputRegistry registry; + registry.register_shader_memory_prefix("conv2d", memory_count); + registry.register_shader_compute_prefix("conv2d", memory_count); + EXPECT_THROW( + registry.register_shader_memory_prefix("conv2d", compute_count), + std::invalid_argument); + EXPECT_THROW( + registry.register_shader_compute_prefix("conv2d", compute_count), + std::invalid_argument); + EXPECT_EQ(registry.shader_memory("conv2d_half"), memory_count); + EXPECT_EQ(registry.shader_compute("conv2d_half"), memory_count); + EXPECT_THROW( + registry.register_shader_memory_prefix("", memory_count), + std::invalid_argument); + EXPECT_THROW( + registry.register_shader_memory_prefix("conv2d", nullptr), + std::invalid_argument); + EXPECT_THROW( + registry.register_shader_compute_prefix("", memory_count), + std::invalid_argument); + EXPECT_THROW( + registry.register_shader_compute_prefix("conv2d", nullptr), + std::invalid_argument); +} + +TEST(BenchmarkThroughputRegistry, NullShaderCountersAreRejectedBeforeLookup) { + ThroughputRegistry registry; + registry.register_shader_memory_prefix("conv2d", memory_count); + registry.register_shader_compute_prefix("conv2d", compute_count); + EXPECT_THROW( + registry.register_shader_memory_prefix("conv2d_gemm", nullptr), + std::invalid_argument); + EXPECT_THROW( + registry.register_shader_compute_prefix("conv2d_gemm", nullptr), + std::invalid_argument); + EXPECT_THROW( + registry.register_shader_memory("conv2d_gemm_half", nullptr), + std::invalid_argument); + EXPECT_THROW( + registry.register_shader_compute("conv2d_gemm_half", nullptr), + std::invalid_argument); + EXPECT_EQ(registry.shader_memory("conv2d_gemm_half"), memory_count); + EXPECT_EQ(registry.shader_compute("conv2d_gemm_half"), compute_count); +} + +TEST( + BenchmarkThroughputRegistry, + StaticPrefixCountersReceiveResolvedShaderMetadata) { + const auto work = + estimate_work({{"shader_name", "test_prefix_float"}}, true, true); + EXPECT_EQ(work.memory_bytes, 40); + EXPECT_EQ(work.compute_flops, 20); + EXPECT_TRUE(work.diagnostics.empty()); +} + +TEST( + BenchmarkThroughputRegistry, + FamilyPrefixesCoverUnenumeratedShaderVariants) { + const auto& registry = throughput_registry(); + for (const char* name : + {"conv2d_gemm_texture2d_half", + "conv2d_new_variant_float", + "q8ta_conv2d_float", + "q8ta_im2col_new_variant", + "linear_vec_bias_tile_row_4_buffer_texture2d_half", + "buffer_to_buffer_new_dtype"}) { + EXPECT_EQ(registry.shader_memory(name), throughput::tensor_traffic); + } + EXPECT_EQ( + registry.shader_compute( + "linear_vec_bias_tile_row_4_buffer_texture2d_half"), + registry.shader_compute( + "linear_vec_bias_tile_row_1_buffer_texture2d_half")); + EXPECT_EQ( + registry.shader_compute( + "linear_vec_tile_row_4_texture3d_texture2d_float"), + registry.shader_compute( + "linear_vec_tile_row_1_texture3d_texture2d_float")); + EXPECT_NE( + registry.shader_compute("linear_vec_bias_float"), + registry.shader_compute("linear_vec_float")); + EXPECT_EQ( + registry.shader_memory("q4gsw_linear_gemm__w_8x8_nc_buffer_half"), + registry.shader_memory("q4gsw_linear_gemm__w_4x8_nc_buffer_float")); + EXPECT_EQ( + registry.shader_memory("q4gsw_linear_gemm__tin__w_8x8_nc_buffer_half"), + registry.shader_memory("q4gsw_linear_gemm__tin__w_4x8_nc_buffer_half")); + EXPECT_NE( + registry.shader_memory("q4gsw_linear_gemm__tin__float"), + registry.shader_memory("q4gsw_linear_gemm__float")); + EXPECT_EQ( + registry.shader_compute("binary_add_buffer_float"), + registry.shader_compute("binary_add_texture3d_float")); +} + +TEST( + BenchmarkThroughputRegistry, + CounterFunctionsCanBeSharedAcrossRegistrations) { + ThroughputRegistry registry; + registry.register_operator_memory("first", memory_count); + registry.register_operator_memory("second", memory_count); + registry.register_shader_memory("first", memory_count); + registry.register_shader_compute_prefix("second", memory_count); + EXPECT_EQ(registry.operator_memory("first"), memory_count); + EXPECT_EQ(registry.operator_memory("second"), memory_count); + EXPECT_EQ(registry.shader_memory("first"), memory_count); + EXPECT_EQ(registry.shader_compute("second_half"), memory_count); +} + +TEST(BenchmarkThroughputRegistry, DuplicateRegistrationsKeepOriginalMapping) { + ThroughputRegistry registry; + registry.register_operator_memory("same", memory_count); + registry.register_operator_compute("same", memory_count); + registry.register_shader_memory("same", memory_count); + registry.register_shader_compute("same", memory_count); + EXPECT_THROW( + registry.register_operator_memory("same", compute_count), + std::invalid_argument); + EXPECT_THROW( + registry.register_operator_compute("same", compute_count), + std::invalid_argument); + EXPECT_THROW( + registry.register_shader_memory("same", compute_count), + std::invalid_argument); + EXPECT_THROW( + registry.register_shader_compute("same", compute_count), + std::invalid_argument); + EXPECT_EQ(registry.operator_memory("same"), memory_count); + EXPECT_EQ(registry.operator_compute("same"), memory_count); + EXPECT_EQ(registry.shader_memory("same"), memory_count); + EXPECT_EQ(registry.shader_compute("same"), memory_count); +} + +TEST(BenchmarkThroughputRegistry, InvalidNamesAndFunctionsAreRejected) { + ThroughputRegistry registry; + EXPECT_THROW( + registry.register_operator_memory("", memory_count), + std::invalid_argument); + EXPECT_THROW( + registry.register_operator_compute("", compute_count), + std::invalid_argument); + EXPECT_THROW( + registry.register_shader_memory("", memory_count), std::invalid_argument); + EXPECT_THROW( + registry.register_shader_compute("", compute_count), + std::invalid_argument); + EXPECT_THROW( + registry.register_operator_memory("same", nullptr), + std::invalid_argument); + EXPECT_THROW( + registry.register_operator_compute("same", nullptr), + std::invalid_argument); + EXPECT_THROW(registry.shader_compute("missing"), std::invalid_argument); + EXPECT_THROW(registry.operator_memory("missing"), std::invalid_argument); +} + +TEST( + BenchmarkThroughputRegistry, + StaticRegistrationsSelectOperatorAndShaderCounters) { + const auto op = + estimate_work({{"operator_name", "test_counter_case"}}, true, true); + const auto shader = + estimate_work({{"shader_name", "test_counter_case"}}, true, true); + EXPECT_EQ(op.memory_bytes, 40); + EXPECT_EQ(op.compute_flops, 20); + EXPECT_EQ(shader.memory_bytes, 20); + EXPECT_EQ(shader.compute_flops, 40); + EXPECT_TRUE(op.diagnostics.empty()); +} + +TEST(BenchmarkThroughputRegistry, MissingCountersAreOptionalWithDiagnostics) { + const auto absent = estimate_work({{"shader_name", "missing"}}, true, true); + EXPECT_FALSE(absent.memory_bytes); + EXPECT_FALSE(absent.compute_flops); + EXPECT_EQ(absent.diagnostics.size(), 2); + const auto disabled = estimate_work(Json(), false, false); + EXPECT_FALSE(disabled.memory_bytes); + EXPECT_TRUE(disabled.diagnostics.empty()); +} + +TEST(BenchmarkThroughputRegistry, InvalidWorkCountsDoNotHideValidOtherCounter) { + for (const char* name : + {"test_negative", "test_infinite", "test_nan", "test_throwing"}) { + const auto result = estimate_work( + {{"shader_name", name}}, true, name == std::string("test_throwing")); + EXPECT_FALSE(result.memory_bytes) << name; + ASSERT_FALSE(result.diagnostics.empty()) << name; + if (name == std::string("test_throwing")) { + EXPECT_EQ(result.compute_flops, 20); + EXPECT_NE( + result.diagnostics[0].find("bad work shape"), std::string::npos); + } + } +} + +TEST( + BenchmarkThroughputRegistry, + LinearCountsLogicalTrafficAndMacsIncludingBias) { + const auto work = estimate_work(linear_case(), true, true); + EXPECT_EQ(work.memory_bytes, 120); + EXPECT_EQ(work.compute_flops, 56); + EXPECT_TRUE(work.diagnostics.empty()); + Json half = linear_case(); + for (auto& arg : half["args"]) { + arg["tensor"]["dtype"] = "float16"; + } + EXPECT_EQ(estimate_work(half, true, false).memory_bytes, 60); + Json no_bias = linear_case(); + no_bias["args"][2] = nullptr; + const auto without_bias = estimate_work(no_bias, true, true); + EXPECT_EQ(without_bias.memory_bytes, 104); + EXPECT_EQ(without_bias.compute_flops, 48); +} + +TEST( + BenchmarkThroughputRegistry, + Q4CountsPackedBytesAndArithmeticEquivalentMacs) { + const Json spec = { + {"operator_name", "et_vk.q4gsw_linear.default"}, + {"args", + {{{"tensor", tensor({2, 32})}}, + {{"tensor", tensor({4, 16}, "uint8", "read", "constant")}}, + {{"tensor", tensor({1, 4}, "float32", "read", "constant")}}, + {{"int", 32}}, + {{"tensor", tensor({4}, "float32", "read", "constant")}}, + {{"tensor", tensor({2, 4}, "float32", "write")}}}}}; + const auto work = estimate_work(spec, true, true); + EXPECT_EQ(work.memory_bytes, 384); + EXPECT_EQ(work.compute_flops, 520); +} + +TEST( + BenchmarkThroughputRegistry, + ConvolutionCountsGroupedStrideAndPaddingCase) { + const Json spec = { + {"operator_name", "aten.convolution.default"}, + {"args", + {{{"tensor", tensor({1, 4, 5, 5})}}, + {{"tensor", tensor({8, 2, 3, 3}, "float32", "read", "constant")}}, + {{"tensor", tensor({8}, "float32", "read", "constant")}}, + {{"int_list", {2, 2}}}, + {{"int_list", {1, 1}}}, + {{"int_list", {1, 1}}}, + {{"bool", false}}, + {{"int_list", {0, 0}}}, + {{"int", 2}}, + {{"tensor", tensor({1, 8, 3, 3}, "float32", "write")}}}}}; + const auto work = estimate_work(spec, true, true); + EXPECT_EQ(work.memory_bytes, 1296); + EXPECT_EQ(work.compute_flops, 2664); + Json transposed = spec; + transposed["args"][6]["bool"] = true; + EXPECT_FALSE(estimate_work(transposed, false, true).compute_flops); +} + +TEST( + BenchmarkThroughputRegistry, + Q8CountsPackedInt8ElementsAndCorrectionScales) { + const Json spec = { + {"operator_name", "et_vk.q8ta_conv2d.default"}, + {"args", + {{{"tensor", tensor({1, 4, 5, 5}, "int8x4")}}, + {{"float", 0.25}}, + {{"int", 0}}, + {{"tensor", tensor({8, 36}, "int8", "read", "constant")}}, + {{"tensor", tensor({8}, "int32", "read", "constant")}}, + {{"tensor", tensor({8}, "float32", "read", "constant")}}, + {{"float", 0.25}}, + {{"int", 0}}, + {{"tensor", tensor({8}, "float32", "read", "constant")}}, + {{"int_list", {3, 3}}}, + {{"int_list", {1, 1}}}, + {{"int_list", {1, 1}}}, + {{"int_list", {1, 1}}}, + {{"int", 1}}, + {{"string", "none"}}, + {{"tensor", tensor({1, 8, 5, 5}, "int8x4", "write")}}}}}; + const auto work = estimate_work(spec, true, true); + EXPECT_EQ(work.memory_bytes, 684); + EXPECT_EQ(work.compute_flops, 14600); +} + +TEST( + BenchmarkThroughputRegistry, + CopyAndBroadcastAdditionHaveLogicalWorkCounts) { + const Json copy = { + {"shader_name", "buffer_to_buffer_float"}, + {"bindings", + {{"input", tensor({8})}, + {"output", tensor({8}, "float32", "write")}, + {"meta", {{"type", "meta_ubo"}}}}}}; + const auto copied = estimate_work(copy, true, true); + EXPECT_EQ(copied.memory_bytes, 64); + EXPECT_EQ(copied.compute_flops, 0); + const Json add = { + {"shader_name", "binary_add_texture3d_float"}, + {"bindings", + {{"a", tensor({2, 1}, "float32", "read", "texture3d")}, + {"b", tensor({3}, "float32", "read", "texture3d")}, + {"out", tensor({2, 3}, "float32", "write", "texture3d")}}}}; + const auto added = estimate_work(add, true, true); + EXPECT_EQ(added.memory_bytes, 44); + EXPECT_EQ(added.compute_flops, 12); +} + +TEST( + BenchmarkThroughputRegistry, + ShaderLinearCountsActualBindingsWithoutParentWork) { + const Json spec = { + {"shader_name", "linear_vec_bias_tile_row_1_buffer_texture2d_float"}, + {"bindings", + {{"binding_0", tensor({1, 8}, "float32", "write")}, + {"binding_1", tensor({1, 4})}, + {"binding_2", tensor({8, 4}, "float32", "read", "texture2d")}, + {"binding_3", tensor({8})}}}}; + const auto work = estimate_work(spec, true, true); + EXPECT_EQ(work.memory_bytes, 208); + EXPECT_EQ(work.compute_flops, 72); + EXPECT_TRUE(work.diagnostics.empty()); + Json missing = spec; + missing["shader_name"] = "unregistered_child_shader"; + EXPECT_FALSE(estimate_work(missing, true, true).memory_bytes); +} + +TEST( + BenchmarkThroughputRegistry, + LogicalTrafficCountsReadWriteTwiceAndDeduplicatesRefs) { + const Json spec = { + {"shader_name", "buffer_to_buffer_float"}, + {"bindings", + {{"x", + {{"name", "x"}, + {"type", "buffer"}, + {"sizes", {8}}, + {"dtype", "float32"}, + {"access", "read_write"}}}, + {"x_alias", + {{"ref", "x"}, + {"type", "buffer"}, + {"sizes", {8}}, + {"dtype", "float32"}, + {"access", "read"}}}}}}; + EXPECT_EQ(estimate_work(spec, true, false).memory_bytes, 64); + Json malformed = spec; + malformed["bindings"]["x"]["dtype"] = "unknown"; + const auto result = estimate_work(malformed, true, false); + EXPECT_FALSE(result.memory_bytes); + EXPECT_FALSE(result.diagnostics.empty()); +} + +TEST(BenchmarkThroughputRegistry, Q4ShaderMemoryExcludesInactiveBindingSlots) { + Json fp32 = { + {"shader_name", "q4gsw_linear_gemm__w_4x8_nc_buffer_float"}, + {"bindings", + {{"binding_0", tensor({4, 8}, "float32", "write")}, + {"binding_1", tensor({4, 32})}, + {"binding_2", tensor(Json::array())}, + {"binding_3", tensor({8, 4}, "int32")}, + {"binding_4", tensor({1, 8})}, + {"binding_5", tensor({8})}}}}; + for (size_t i = 0; i < 6; ++i) { + fp32["bindings"]["binding_" + std::to_string(i)]["binding_index"] = i; + } + const auto fp32_work = estimate_work(fp32, true, true); + EXPECT_EQ(fp32_work.memory_bytes, 832); + EXPECT_FALSE(fp32_work.compute_flops); + Json fp16 = fp32; + fp16["shader_name"] = "q4gsw_linear_gemm__tin__w_4x8_nc_buffer_half"; + fp16["bindings"]["binding_2"]["sizes"] = {32, 4}; + for (size_t i : {0, 1, 2, 4, 5}) { + fp16["bindings"]["binding_" + std::to_string(i)]["dtype"] = "float16"; + } + const auto fp16_work = estimate_work(fp16, true, false); + EXPECT_EQ(fp16_work.memory_bytes, 480); + EXPECT_TRUE(fp16_work.diagnostics.empty()); + fp16["bindings"]["binding_5"]["sizes"] = Json::array(); + EXPECT_EQ(estimate_work(fp16, true, false).memory_bytes, 464); +} + +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/test/throughput_report_test.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/test/throughput_report_test.cpp new file mode 100644 index 00000000000..b5493e54514 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/test/throughput_report_test.cpp @@ -0,0 +1,135 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +#include +#include + +namespace vkcompute::shader_profiling { +namespace { + +const Json kBoth = { + {"show_memory_throughput", true}, + {"show_compute_throughput", true}}; + +TEST( + BenchmarkThroughputReport, + CalculatesRatesPerSampleWithCorrectBinaryAndFlopUnits) { + const WorkAmounts work{1073741824.0, 2000000000.0, {}}; + const auto report = + throughput_report(kBoth, work, 2, {2000000, 4000000}, {4000000, 8000000}); + const auto& gpu = report.at("gpu_timestamp"); + const auto& memory = gpu.at("memory_gib_per_s"); + EXPECT_DOUBLE_EQ(memory.at("min"), 0.5); + EXPECT_DOUBLE_EQ(memory.at("max"), 1.0); + EXPECT_DOUBLE_EQ(memory.at("mean"), 0.75); + EXPECT_NEAR(memory.at("p90").get(), 0.95, 1e-12); + EXPECT_DOUBLE_EQ(gpu.at("memory_mib_per_s").at("mean"), 768); + EXPECT_DOUBLE_EQ(gpu.at("compute_gflops").at("mean"), 1.5); + EXPECT_EQ(report.at("gpu_timestamp_batch"), gpu); + EXPECT_EQ( + report.at("wall_time").at("memory_gib_per_s"), + Report::array({0.5, 0.25})); + EXPECT_EQ( + report.at("wall_time").at("compute_gflops"), Report::array({1.0, 0.5})); + EXPECT_EQ(report.at("wall_time_batch"), report.at("wall_time")); +} + +TEST(BenchmarkThroughputReport, FlagsControlOnlyRequestedMetrics) { + const WorkAmounts work{1024.0, 4.0, {}}; + EXPECT_TRUE(throughput_report({}, work, 1, {1}, {2}).empty()); + const auto memory = + throughput_report({{"show_memory_throughput", true}}, work, 1, {1}, {2}); + EXPECT_TRUE(memory.contains("memory_bytes")); + EXPECT_FALSE(memory.contains("compute_flops")); + EXPECT_FALSE(memory.at("gpu_timestamp").contains("compute_gflops")); + const auto compute = + throughput_report({{"show_compute_throughput", true}}, work, 1, {1}, {2}); + EXPECT_FALSE(compute.contains("memory_bytes")); + EXPECT_TRUE(compute.contains("compute_flops")); + EXPECT_FALSE(compute.at("gpu_timestamp").contains("memory_gib_per_s")); +} + +TEST(BenchmarkThroughputReport, UnknownCountsAreNullAndDoNotEraseValidMetrics) { + const WorkAmounts work{std::nullopt, 4.0, {"No memory counter"}}; + const auto report = throughput_report(kBoth, work, 1, {2}, {4}); + EXPECT_TRUE(report.at("memory_bytes").is_null()); + EXPECT_TRUE(report.at("gpu_timestamp").at("memory_gib_per_s").is_null()); + EXPECT_DOUBLE_EQ( + report.at("gpu_timestamp").at("compute_gflops").at("mean"), 0.002); + EXPECT_EQ(report.at("diagnostics"), Report::array({"No memory counter"})); +} + +TEST( + BenchmarkThroughputReport, + ZeroWorkAndUnavailableTimestampsRemainDistinct) { + const WorkAmounts work{0.0, 0.0, {}}; + const auto report = throughput_report(kBoth, work, 1, {}, {1, 2}); + EXPECT_TRUE(report.at("gpu_timestamp").at("compute_gflops").is_null()); + EXPECT_EQ(report.at("wall_time").at("compute_gflops"), Report::array({0, 0})); + EXPECT_DOUBLE_EQ( + throughput_report(kBoth, work, 1, {1}, {}) + .at("gpu_timestamp") + .at("compute_gflops") + .at("mean"), + 0); +} + +TEST(BenchmarkThroughputReport, InvalidIntervalsCannotGenerateInfiniteRates) { + const WorkAmounts work{4.0, 4.0, {}}; + const auto report = throughput_report(kBoth, work, 1, {0, 1}, {0, 2}); + EXPECT_TRUE(report.at("gpu_timestamp").at("compute_gflops").is_null()); + EXPECT_TRUE(report.at("wall_time").at("compute_gflops")[0].is_null()); + EXPECT_DOUBLE_EQ(report.at("wall_time").at("compute_gflops")[1], 0.002); + EXPECT_THROW( + throughput_report(kBoth, work, 0, {1}, {2}), std::invalid_argument); +} + +TEST( + BenchmarkThroughputReport, + ShaderBreakdownUsesWorkAndSamplesForEachDispatch) { + const Report dispatches = + Report::array({{{"shader_name", "first"}}, {{"shader_name", "second"}}}); + const std::vector work = { + {1048576.0, 1000000000.0, {}}, {2097152.0, 3000000000.0, {}}}; + const auto result = shader_breakdown_report( + dispatches, {{1000000, 2000000, 1000000, 2000000}}, 2, kBoth, work); + EXPECT_DOUBLE_EQ( + result[0] + .at("throughput") + .at("gpu_timestamp") + .at("memory_mib_per_s") + .at("mean"), + 1); + EXPECT_DOUBLE_EQ( + result[1] + .at("throughput") + .at("gpu_timestamp") + .at("memory_mib_per_s") + .at("mean"), + 1); + EXPECT_DOUBLE_EQ( + result[0] + .at("throughput") + .at("gpu_timestamp") + .at("compute_gflops") + .at("mean"), + 1); + EXPECT_DOUBLE_EQ( + result[1] + .at("throughput") + .at("gpu_timestamp") + .at("compute_gflops") + .at("mean"), + 1.5); + EXPECT_FALSE(result[0].at("throughput").contains("wall_time")); +} + +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/test/validation_test.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/test/validation_test.cpp new file mode 100644 index 00000000000..43f5d064ec5 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/test/validation_test.cpp @@ -0,0 +1,254 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +#include + +#include + +#include +#include + +namespace vkcompute::shader_profiling { +using BenchmarkValidation = VulkanTest; + +namespace { + +Json copy_case(const Json& gwg) { + return parse_configs({{"shader", "buffer_to_buffer_float"}, + {"gwg_size", gwg}, + {"lwg_size", {1, 1, 1}}, + {"bindings", + Json::array( + {{{"buffer", + {{"name", "out"}, + {"sizes", {8}}, + {"access", "write"}, + {"data", {{"fill", 0}}}}}}, + {{"buffer", + {{"name", "in"}, + {"sizes", {8}}, + {"data", {1, 2, 3, 4, 5, 6, 7, 8}}}}}, + {{"meta_ubo", "out"}, {"field", "numel"}}})}}) + .at(0); +} + +TEST_F(BenchmarkValidation, StagedShaderCopyPassesWithoutBenchmarking) { + const Json spec = copy_case({8, 1, 1}); + const Report result = benchmark_shader(spec, prepare_tensor_data(spec), true); + EXPECT_EQ(result.at("validation").at("status"), "PASS"); + EXPECT_FALSE(result.contains("gpu_timestamp_shader_us")); + EXPECT_FALSE(result.contains("wall_time_shader_us")); +} + +TEST_F(BenchmarkValidation, GraphCreationReusesPreparedHostData) { + const Json spec = copy_case({8, 1, 1}); + auto host_data = prepare_tensor_data(spec); + host_data.at("in") = + make_data({{"data", {-1, -2, -3, -4, -5, -6, -7, -8}}}, 8); + const auto expected = host_data; + const auto* input_data = host_data.at("in").data(); + { + ComputeGraph graph{GraphConfig{}}; + create_tensors(spec, graph, host_data, true); + EXPECT_EQ(host_data, expected); + EXPECT_EQ(host_data.at("in").data(), input_data); + } + { + ComputeGraph graph{GraphConfig{}}; + create_tensors(spec, graph, host_data, false); + EXPECT_EQ(host_data, expected); + EXPECT_EQ(host_data.at("in").data(), input_data); + EXPECT_TRUE(graph.inputs().empty()); + EXPECT_TRUE(graph.execute_nodes().empty()); + } +} + +TEST_F(BenchmarkValidation, ValidationSelectionRequiresWritableTensors) { + Json spec = copy_case({8, 1, 1}); + spec["validation_method"] = {{"function", "is_close"}, {"outputs", {"out"}}}; + ComputeGraph graph{GraphConfig{}}; + const auto host_data = prepare_tensor_data(spec); + const auto tensors = create_tensors(spec, graph, host_data); + const Json accesses{{"out", "write"}, {"in", "read"}}; + const auto outputs = add_validation_outputs(spec, graph, tensors, accesses); + ASSERT_EQ(outputs.size(), 1); + EXPECT_EQ(outputs.begin()->first, "out"); + spec["validation_method"]["outputs"] = {"in"}; + EXPECT_THROW( + add_validation_outputs(spec, graph, tensors, accesses), + std::invalid_argument); +} + +TEST_F(BenchmarkValidation, RejectsWrongOutputShapeWithMatchingElementCount) { + Json spec = copy_case({8, 1, 1}); + spec["tensors"]["in"]["sizes"] = {1, 8}; + spec["tensors"]["out"]["sizes"] = {1, 8}; + ComputeGraph graph{GraphConfig{}}; + const auto host_data = prepare_tensor_data(spec); + const auto tensors = create_tensors(spec, graph, host_data); + graph.virtual_resize(tensors.at("out"), {2, 4}); + const ValueRef staging = + graph.add_staging(vkapi::kFloat, 8, vkapi::CopyDirection::DEVICE_TO_HOST); + graph.prepare(); + const std::array values = {1, 2, 3, 4, 5, 6, 7, 8}; + graph.maybe_cast_and_copy_into_staging( + staging, values.data(), values.size(), vkapi::kFloat); + EXPECT_THROW( + validate_outputs(spec, graph, tensors, {{"out", staging}}, host_data), + std::invalid_argument); +} + +TEST_F(BenchmarkValidation, IncompleteShaderGridReportsOutputMismatches) { + const Json spec = copy_case({1, 1, 1}); + const Report result = benchmark_shader(spec, prepare_tensor_data(spec), true); + EXPECT_EQ(result.at("validation").at("status"), "FAIL"); + EXPECT_EQ( + result.at("validation").at("outputs").at("out").at("mismatch_count"), 7); +} + +TEST_F(BenchmarkValidation, StagedTextureAdditionUsesOneTargetDescriptorSet) { + const Json spec = + parse_configs( + {{"shader", "binary_add_texture3d_float"}, + {"gwg_size", "default_pick_gwg"}, + {"lwg_size", {10, 10, 1}}, + {"bindings", + Json::array( + {{{"texture3d", + {{"name", "out"}, + {"sizes", {1, 1, 1, 8}}, + {"access", "write"}, + {"data", {{"fill", 0}}}}}}, + {{"texture3d", + {{"name", "a"}, + {"sizes", {1, 1, 1, 8}}, + {"data", {1, 2, 3, 4, 5, 6, 7, 8}}}}}, + {{"texture3d", + {{"name", "b"}, + {"sizes", {1, 1, 1, 8}}, + {"data", {-1, -2, -3, -4, -5, -6, -7, -8}}}}}, + {{"meta_ubo", "out"}}, + {{"meta_ubo", "a"}}, + {{"meta_ubo", "b"}}})}, + {"specialization_constants", + {{{"layout", "out"}}, {{"layout", "a"}}, {{"layout", "b"}}}}, + {"push_constants", {{{"dtype", "float32"}, {"data", {2}}}}}}) + .at(0); + EXPECT_EQ( + benchmark_shader(spec, prepare_tensor_data(spec), true) + .at("validation") + .at("status"), + "PASS"); +} + +TEST_F(BenchmarkValidation, StagedLinearUsesConstantWeightsAndOptionalBias) { + const Json spec = + parse_configs( + {{"operator", "aten.linear.default"}, + {"args_sweep", + Json::array({Json::array( + {{{"tensor", + {{"name", "x"}, {"sizes", {1, 4}}, {"data", {1, 2, 3, 4}}}}}, + {{"tensor", + {{"name", "w"}, + {"sizes", {4, 4}}, + {"storage", "constant"}, + {"data", + {1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1}}}}}, + nullptr, + {{"tensor", + {{"name", "y"}, + {"sizes", {1, 4}}, + {"data", {{"fill", 0}}}}}}})})}}) + .at(0); + for (const char* dtype : {"float32", "float16"}) { + Json typed = spec; + for (auto& tensor : typed["tensors"].items()) { + tensor.value()["dtype"] = dtype; + } + typed.update( + {{"target_ms", 0.01}, + {"benchmark_runs", 2}, + {"warmup_runs", 0}, + {"max_dispatches", 2}}); + const auto host_data = prepare_tensor_data(typed); + const auto expected = host_data; + EXPECT_EQ( + benchmark_operator(typed, host_data, true) + .at("validation") + .at("status"), + "PASS"); + EXPECT_TRUE( + benchmark_operator(typed, host_data).contains("wall_time_operator_us")); + EXPECT_EQ( + benchmark_operator(typed, host_data, true) + .at("validation") + .at("status"), + "PASS"); + EXPECT_EQ(host_data, expected); + EXPECT_EQ( + host_data.at("w").size(), std::string(dtype) == "float16" ? 32 : 64); + } +} + +TEST_F(BenchmarkValidation, BenchmarkAddsRegisteredWorkAndWallThroughput) { + Json spec = copy_case({8, 1, 1}); + spec.update( + {{"show_memory_throughput", true}, + {"show_compute_throughput", true}, + {"target_ms", 0.01}, + {"benchmark_runs", 2}, + {"warmup_runs", 0}, + {"max_dispatches", 2}}); + const Report result = benchmark_shader(spec, prepare_tensor_data(spec)); + const auto& throughput = result.at("throughput"); + EXPECT_DOUBLE_EQ(throughput.at("memory_bytes"), 64); + EXPECT_DOUBLE_EQ(throughput.at("compute_flops"), 0); + EXPECT_EQ(throughput.at("wall_time").at("memory_gib_per_s").size(), 2); + EXPECT_EQ( + throughput.at("wall_time").at("compute_gflops"), Report::array({0, 0})); + const double count = result.at("dispatches_per_batch"); + for (size_t i = 0; i < 2; ++i) { + const double expected = 64.0 * count * 1e6 / + result.at("wall_time_batch_us")[i].get() / 1073741824.0; + EXPECT_NEAR( + throughput.at("wall_time").at("memory_gib_per_s")[i].get(), + expected, + 1e-12); + } +} + +TEST_F(BenchmarkValidation, UnsupportedShaderIsRejectedBeforePipelineCreation) { + ComputeGraph graph{GraphConfig{}}; + if (graph.context()->adapter_ptr()->supports_int8_dot_product()) { + GTEST_SKIP() << "Requires a device without integer dot-product support"; + } + const auto shader = VK_KERNEL_FROM_STR("q8ta_conv2d_pw_float"); + ASSERT_TRUE(shader.requires_integer_dot_product); + EXPECT_THROW( + graph.register_pipeline_to_create( + shader, LocalWorkGroup(1, 1, 1), {}, {}), + vkapi::ShaderNotSupportedError); +} + +TEST_F( + BenchmarkValidation, + UnrecognizedSpirvCapabilityIsRejectedBeforePipelineCreation) { + ComputeGraph graph{GraphConfig{}}; + vkapi::ShaderInfo shader; + shader.kernel_name = "unsupported_capability"; + shader.required_capabilities = {1, 65535}; + EXPECT_THROW( + graph.register_pipeline_to_create( + shader, LocalWorkGroup(1, 1, 1), {}, {}), + vkapi::ShaderNotSupportedError); +} + +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/MigrationReferenceTest.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/MigrationReferenceTest.cpp new file mode 100644 index 00000000000..e20a5a00081 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/MigrationReferenceTest.cpp @@ -0,0 +1,221 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace vkcompute::shader_profiling { +namespace { +Json tensor(const char* name) { + return {{"tensor", name}}; +} +TEST(MigratedReferences, Conv1dDepthwiseBatchStridePadding) { + const Json spec = { + {"args", + {tensor("input"), + tensor("weight"), + nullptr, + {{"int_list", {2}}}, + {{"int_list", {1}}}, + {{"int_list", {1}}}, + {{"bool", false}}, + {{"int_list", {0}}}, + {{"int", 2}}, + tensor("output")}}}; + const reference::TensorMap tensors = { + {"input", + {{2, 2, 4}, {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}}}, + {"weight", {{2, 1, 3}, {1, 1, 1, 2, 2, 2}}}}; + const auto out = reference::conv1d_reference(spec, tensors, 2); + EXPECT_EQ(out.sizes, (std::vector{2, 2, 2})); + EXPECT_EQ(out.values, (std::vector{3, 9, 22, 42, 19, 33, 54, 90})); +} +TEST(MigratedReferences, BatchedRectangularMatmul) { + const Json spec = { + {"operator", "aten.bmm.default"}, + {"args", {tensor("a"), tensor("b"), tensor("output")}}}; + const reference::TensorMap tensors = { + {"a", {{2, 1, 2}, {1, 2, 3, 4}}}, + {"b", {{2, 2, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}}}}; + const auto out = reference::matmul_reference(spec, tensors, 2); + EXPECT_EQ(out.sizes, (std::vector{2, 1, 3})); + EXPECT_EQ(out.values, (std::vector{9, 12, 15, 61, 68, 75})); +} +TEST(MigratedReferences, AddmmAlphaBeta) { + const Json spec = { + {"operator", "aten.addmm.default"}, + {"args", + {tensor("bias"), + tensor("a"), + tensor("b"), + {{"float", 3}}, + {{"float", 2}}, + tensor("output")}}}; + const reference::TensorMap tensors = { + {"a", {{1, 2}, {1, 2}}}, + {"b", {{2, 2}, {3, 4, 5, 6}}}, + {"bias", {{2}, {10, 20}}}}; + EXPECT_EQ( + reference::matmul_reference(spec, tensors, 1).values, + (std::vector{56, 92})); +} +TEST(MigratedReferences, EmbeddingNibbleOrderAndGroupedScales) { + Json spec = { + {"args", + {tensor("w"), + tensor("s"), + {{"int", 2}}, + tensor("i"), + {{"bool", false}}, + tensor("output")}}}; + const reference::TensorMap tensors = { + {"w", {{2, 2}, {0x9a, 0xbc, 0xde, 0xf8}}}, + {"s", {{2, 2}, {1, 2, 3, 4}}}, + {"i", {{2}, {1, 0}}}}; + EXPECT_EQ( + reference::embedding_q4gsw_reference(spec, tensors, 1).values, + (std::vector{15, 18, 28, 0, 1, 2, 6, 8})); + spec["args"][4]["bool"] = true; + EXPECT_EQ( + reference::embedding_q4gsw_reference(spec, tensors, 1).values, + (std::vector{18, 15, 0, 28, 2, 1, 8, 6})); +} +TEST(MigratedReferences, DynamicQ4Clipping) { + Json spec = { + {"args", + {tensor("x"), + tensor("is"), + tensor("zp"), + tensor("w"), + tensor("sum"), + tensor("s"), + {{"int", 2}}, + nullptr, + tensor("output")}}, + {"route", {{"name", "auto"}}}}; + const reference::TensorMap tensors = { + {"x", {{1, 2}, {100, -100}}}, + {"is", {{1, 1}, {0.5}}}, + {"zp", {{1, 1}, {0}}}, + {"w", {{1, 1}, {0x99}}}, + {"s", {{1, 1}, {1}}}}; + EXPECT_EQ( + reference::dq8ca_q4gsw_reference(spec, tensors, 1).values, + (std::vector{-0.5})); +} + +TEST(MigratedReferences, DynamicQ4UsesWeightOnlyFallbackWithoutIntegerDot) { + Json spec = { + {"args", + {tensor("x"), + tensor("is"), + tensor("zp"), + tensor("w"), + tensor("sum"), + tensor("s"), + {{"int", 2}}, + nullptr, + tensor("output")}}, + {"reference_context", {{"int8_dot_product", false}}}}; + const reference::TensorMap tensors = { + {"x", {{1, 2}, {100, -100}}}, + {"w", {{1, 1}, {0x99}}}, + {"s", {{1, 1}, {1}}}}; + EXPECT_EQ( + reference::dq8ca_q4gsw_reference(spec, tensors, 1).values, + (std::vector{0})); +} + +TEST(MigratedReferences, DynamicQ4RejectsMalformedReferenceCapabilities) { + Json spec = { + {"args", + {tensor("x"), + tensor("is"), + tensor("zp"), + tensor("w"), + tensor("sum"), + tensor("s"), + {{"int", 2}}, + nullptr, + tensor("output")}}, + {"reference_context", {{"int8_dot_product", "false"}}}}; + EXPECT_THROW( + reference::dq8ca_q4gsw_reference(spec, reference::TensorMap{}, 1), + std::invalid_argument); +} +TEST(MigratedReferences, SdpaFusedCacheUpdateAndCausalPrefix) { + Json spec = { + {"operator", "sdpa_with_kv_cache.default"}, + {"args", + {tensor("q"), + tensor("pk"), + tensor("pv"), + tensor("k"), + tensor("v"), + {{"int", 1}}, + {{"int", 2}}, + nullptr, + {{"float", 0}}, + {{"bool", true}}, + {{"float", 1}}, + tensor("output")}}}; + const reference::TensorMap tensors = { + {"q", {{1, 2, 2, 1}, {0, 0, 0, 0}}}, + {"k", {{1, 4, 1, 1}, {0, 0, 0, 0}}}, + {"v", {{1, 4, 1, 1}, {2, 100, 200, 1000}}}, + {"pk", {{1, 2, 1, 1}, {0, 0}}}, + {"pv", {{1, 2, 1, 1}, {4, 6}}}}; + const auto out = reference::sdpa_reference(spec, tensors, 2); + EXPECT_EQ(out.values, (std::vector{3, 3, 4, 4})); + spec["operator"] = "llama.custom_sdpa.default"; + spec["args"] = { + tensor("q"), + tensor("k"), + tensor("v"), + {{"int", 1}}, + nullptr, + {{"float", 0}}, + {{"bool", true}}, + {{"float", 1}}, + tensor("output")}; + const auto unchanged = reference::sdpa_reference(spec, tensors, 1); + EXPECT_FLOAT_EQ(unchanged.values[0], 51); + EXPECT_FLOAT_EQ(unchanged.values[2], 302.0f / 3); +} +TEST(MigratedReferences, NativeAddBroadcastAndAlpha) { + const Json spec = { + {"operator", "aten.add.Tensor"}, + {"args", {tensor("a"), tensor("b"), {{"float", 0.5}}, tensor("out")}}, + {"tensors", {{"out", {{"sizes", {2, 3}}}}}}}; + const reference::TensorMap tensors = { + {"a", {{2, 1}, {1, 2}}}, {"b", {{1, 3}, {2, 4, 6}}}}; + const auto out = reference::add_tensor_reference(spec, tensors, 2); + EXPECT_EQ(out.sizes, (std::vector{2, 3})); + EXPECT_EQ(out.values, (std::vector{2, 3, 4, 3, 4, 5})); +} +TEST(MigratedReferences, ChooseQparamsZeroAndAsymmetricRange) { + const Json spec = { + {"args", + {tensor("x"), + {{"int", -128}}, + {{"int", 127}}, + tensor("scale"), + tensor("zero_point")}}}; + const reference::TensorMap tensors = {{"x", {{2, 3}, {0, 0, 0, 0, 1, 2}}}}; + const auto out = reference::choose_qparams_reference(spec, tensors, 2); + EXPECT_FLOAT_EQ(out.at("scale")[0], 0.1f); + EXPECT_FLOAT_EQ(out.at("scale")[1], 2.0f / 255); + EXPECT_EQ(out.at("zero_point"), (std::vector{127, -128})); +} +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/SdpaReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/SdpaReference.cpp new file mode 100644 index 00000000000..2efbc15e818 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/SdpaReference.cpp @@ -0,0 +1,104 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +#include +#include +namespace vkcompute::shader_profiling::reference { +Tensor +sdpa_reference(const Json& spec, const TensorMap& tensors, size_t threads) { + const Tensor& q = tensor_arg(spec, 0, tensors); + const bool fused = spec.at("operator") == "sdpa_with_kv_cache.default"; + const Tensor& k = tensor_arg(spec, fused ? 3 : 1, tensors); + const Tensor& v = tensor_arg(spec, fused ? 4 : 2, tensors); + const int64_t pos = spec.at("args").at(fused ? 5 : 3).at("int"); + require( + q.sizes.size() == 4 && k.sizes.size() == 4 && v.sizes == k.sizes && + q.sizes[0] == 1, + "SDPA expects batch1 rank4 tensors"); + const size_t s = q.sizes[1], h = q.sizes[2], d = q.sizes[3], hk = k.sizes[2], + capacity = k.sizes[1]; + require( + h % hk == 0 && k.sizes[3] == int64_t(d) && pos >= 0 && + size_t(pos) + s <= capacity, + "Invalid SDPA head dimensions or cache position"); + const Json& scale_arg = spec.at("args").at(fused ? 10 : 7); + const float scale = scale_arg.is_null() ? 1 / std::sqrt(float(d)) + : scale_arg.at("float").get(); + const Tensor* pk = fused ? &tensor_arg(spec, 1, tensors) : nullptr; + const Tensor* pv = fused ? &tensor_arg(spec, 2, tensors) : nullptr; + Tensor out{q.sizes, std::vector(q.values.size())}; + parallel_for(s * h, threads, [&](size_t job) { + const size_t token = job / h, head = job % h, kvhead = head / (h / hk), + context = pos + token + 1; + std::vector probs(context); + float max = -std::numeric_limits::infinity(); + for (size_t c = 0; c < context; ++c) { + float dot = 0; + const Tensor& cache = pk && c >= size_t(pos) ? *pk : k; + const size_t ci = pk && c >= size_t(pos) ? c - pos : c; + for (size_t j = 0; j < d; ++j) + dot = std::fma( + float(q.values[job * d + j]), + float(cache.values[(ci * hk + kvhead) * d + j]), + dot); + probs[c] = dot * scale; + max = std::max(max, probs[c]); + } + float denom = 0; + for (float& p : probs) { + p = std::exp(p - max); + denom += p; + } + for (size_t j = 0; j < d; ++j) { + float value = 0; + for (size_t c = 0; c < context; ++c) { + const Tensor& cache = pv && c >= size_t(pos) ? *pv : v; + const size_t ci = pv && c >= size_t(pos) ? c - pos : c; + value += + probs[c] / denom * float(cache.values[(ci * hk + kvhead) * d + j]); + } + out.values[job * d + j] = value; + } + }); + return out; +} +static ReferenceOutputs sdpa_reference_outputs( + const Json& spec, + const TensorMap& tensors, + size_t threads) { + Tensor out = sdpa_reference(spec, tensors, threads); + ReferenceOutputs result{ + {spec.at("args").back().at("tensor").get(), + std::move(out.values)}}; + if (spec.at("operator") == "sdpa_with_kv_cache.default") { + const size_t pos = spec.at("args").at(5).at("int"); + for (const auto& pair : {std::pair{3, 1}, {4, 2}}) { + const Tensor& cache = tensor_arg(spec, pair.first, tensors); + const Tensor& projected = tensor_arg(spec, pair.second, tensors); + auto values = cache.values; + const size_t row_size = cache.sizes[2] * cache.sizes[3]; + require( + pos * row_size + projected.values.size() <= values.size(), + "SDPA projected cache update exceeds capacity"); + std::copy( + projected.values.begin(), + projected.values.end(), + values.begin() + pos * row_size); + result[spec.at("args").at(pair.first).at("tensor").get()] = + std::move(values); + } + } + return result; +} +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_OUTPUTS_FN(sdpa, sdpa_reference_outputs); + REGISTER_OPERATOR_REFERENCE(llama.custom_sdpa.default, sdpa); + REGISTER_OPERATOR_REFERENCE(sdpa_with_kv_cache.default, sdpa); +} +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/SdpaReference.h b/backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/SdpaReference.h new file mode 100644 index 00000000000..cf8962baf44 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/SdpaReference.h @@ -0,0 +1,16 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +namespace vkcompute::shader_profiling::reference { + +Tensor +sdpa_reference(const Json& spec, const TensorMap& tensors, size_t threads); + +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/SdpaRoutes.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/SdpaRoutes.cpp new file mode 100644 index 00000000000..8df993ff582 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/SdpaRoutes.cpp @@ -0,0 +1,51 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include + +namespace vkcompute::shader_profiling { +namespace { + +template +Json build_sdpa_route( + ComputeGraph& graph, + const std::vector& args, + const Json& options) { + if (!options.empty()) + throw std::invalid_argument("SDPA routes accept no options"); + if (args.size() != 9) + throw std::invalid_argument("SDPA expects nine production arguments"); + sdpa_impl_with_shader_override(graph, args, graph.add_scalar(Override)); + const char* name = Override == kShaderOverrideForceGqa ? "gqa" + : Override == kShaderOverrideForceTile2 ? "gqa_tile2" + : Override == kShaderOverrideForceBase ? "gqa_base" + : "non_gqa"; + return {{"name", name}, {"options", options}}; +} + +REGISTER_OPERATOR_ROUTES { + REGISTER_OPERATOR_ROUTE( + llama.custom_sdpa.default, + gqa, + (build_sdpa_route)); + REGISTER_OPERATOR_ROUTE( + llama.custom_sdpa.default, + non_gqa, + (build_sdpa_route)); + REGISTER_OPERATOR_ROUTE( + llama.custom_sdpa.default, + gqa_tile2, + (build_sdpa_route)); + REGISTER_OPERATOR_ROUTE( + llama.custom_sdpa.default, + gqa_base, + (build_sdpa_route)); +} + +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/SdpaThroughput.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/SdpaThroughput.cpp new file mode 100644 index 00000000000..018f2c441fe --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/SdpaThroughput.cpp @@ -0,0 +1,37 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +namespace vkcompute::shader_profiling { +namespace { +double sdpa_flops(const Json& spec) { + const auto q = throughput::tensor_argument(spec, 0) + .at("sizes") + .get>(); + const bool fused = spec.value("operator_name", spec.value("operator", "")) == + "sdpa_with_kv_cache.default"; + const auto k = throughput::tensor_argument(spec, fused ? 3 : 1) + .at("sizes") + .get>(); + const int64_t pos = spec.at("args").at(fused ? 5 : 3).at("int"); + const int64_t context = pos + q.at(1); + if (pos < 0 || context > k.at(1)) + throw std::invalid_argument( + "SDPA work count context exceeds cache capacity"); + return 4.0 * q.at(1) * q.at(2) * q.at(3) * context; +} +REGISTER_THROUGHPUT_FUNCTIONS { + REGISTER_OPERATOR_MEMORY_TRAFFIC( + llama.custom_sdpa.default, throughput::tensor_traffic); + REGISTER_OPERATOR_COMPUTE_FLOPS(llama.custom_sdpa.default, sdpa_flops); + REGISTER_OPERATOR_MEMORY_TRAFFIC( + sdpa_with_kv_cache.default, throughput::tensor_traffic); + REGISTER_OPERATOR_COMPUTE_FLOPS(sdpa_with_kv_cache.default, sdpa_flops); +} + +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/cases/correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/cases/correctness.json new file mode 100644 index 00000000000..8b2a50f06ec --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/cases/correctness.json @@ -0,0 +1,1522 @@ +[ + { + "name": "sdpa Llama-3.2-1B prefill D64 H32 Hkv8 S128 C128 P0 float16 texture3d default", + "operator": "llama.custom_sdpa.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.01, + "rtol": 0.01 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": [1, 128, 32, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 128, 8, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 128, 8, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": 0 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 32, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa float16 texture3d width", + "operator": "llama.custom_sdpa.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.01, + "rtol": 0.01 + }, + "sweep_variables": { + "v_SIZES": [ + [1, 128, 16, 128], + [1, 128, 24, 128] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": "v_SIZES", + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 128, 8, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 128, 8, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": 0 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + null, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa default_scale decode D64 H8 Hkv2 S1 C32 P31 float32 texture3d production", + "operator": "llama.custom_sdpa.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": [1, 1, 8, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 32, 2, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 32, 2, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": 31 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 8, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa accu prefill D64 H8 Hkv2 S16 C16 P0 float32 texture3d default", + "operator": "llama.custom_sdpa.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": [1, 16, 8, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 16, 2, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 16, 2, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": 0 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 8, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa Gemma-4-E2B sliding_decode D256 H8 Hkv1 S1 C32 P31 float32 texture3d production", + "operator": "llama.custom_sdpa.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": [1, 1, 8, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 32, 1, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 32, 1, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": 31 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 8, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa Gemma-4-E2B full_decode D512 H8 Hkv1 S1 C16 P15 float32 texture3d production", + "operator": "llama.custom_sdpa.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": [1, 1, 8, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 16, 1, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 16, 1, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": 15 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 8, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa Gemma-4-E2B sliding_decode D256 H8 Hkv1 S1 C32 P31 float32 texture3d", + "operator": "llama.custom_sdpa.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["gqa_tile2", "gqa_base", "non_gqa"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": [1, 1, 8, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 32, 1, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 32, 1, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": 31 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 8, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa Gemma-4-E2B full_decode D512 H8 Hkv1 S1 C16 P15 float32 texture3d", + "operator": "llama.custom_sdpa.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["gqa_tile2", "gqa_base", "non_gqa"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": [1, 1, 8, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 16, 1, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 16, 1, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": 15 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 8, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa Gemma-4-E2B sliding_prefill D256 H8 Hkv1 S4 C4 P0 float32 texture3d fused", + "operator": "sdpa_with_kv_cache.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001, + "outputs": ["output"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": [1, 4, 8, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_projected", + "sizes": [1, 4, 1, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_projected", + "sizes": [1, 4, 1, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 4, 1, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + }, + "access": "read_write" + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 4, 1, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + }, + "access": "read_write" + } + }, + { + "int": 0 + }, + { + "int": 4 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 4, 8, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa Gemma-4-E2B full_prefill D512 H8 Hkv1 S4 C4 P0 float32 texture3d fused", + "operator": "sdpa_with_kv_cache.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001, + "outputs": ["output"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": [1, 4, 8, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_projected", + "sizes": [1, 4, 1, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_projected", + "sizes": [1, 4, 1, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 4, 1, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + }, + "access": "read_write" + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 4, 1, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + }, + "access": "read_write" + } + }, + { + "int": 0 + }, + { + "int": 4 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 4, 8, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa Gemma-4-E2B sliding_prefill_real D256 H8 Hkv1 S18 C512 P0 float32 texture3d fused", + "operator": "sdpa_with_kv_cache.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001, + "outputs": ["output"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": [1, 18, 8, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_projected", + "sizes": [1, 18, 1, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_projected", + "sizes": [1, 18, 1, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 512, 1, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + }, + "access": "read_write" + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 512, 1, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + }, + "access": "read_write" + } + }, + { + "int": 0 + }, + { + "int": 18 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 18, 8, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa Gemma-4-E2B full_prefill_real D512 H8 Hkv1 S18 C512 P0 float32 texture3d fused", + "operator": "sdpa_with_kv_cache.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001, + "outputs": ["output"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": [1, 18, 8, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_projected", + "sizes": [1, 18, 1, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_projected", + "sizes": [1, 18, 1, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 512, 1, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + }, + "access": "read_write" + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 512, 1, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + }, + "access": "read_write" + } + }, + { + "int": 0 + }, + { + "int": 18 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 18, 8, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa Gemma-4-E2B sliding_decode_prefix D256 H8 Hkv1 S1 C512 P17 float32 texture3d fused", + "operator": "sdpa_with_kv_cache.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001, + "outputs": ["output"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": [1, 1, 8, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_projected", + "sizes": [1, 1, 1, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_projected", + "sizes": [1, 1, 1, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 512, 1, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + }, + "access": "read_write" + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 512, 1, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + }, + "access": "read_write" + } + }, + { + "int": 17 + }, + { + "int": 1 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 8, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa Gemma-4-E2B full_decode_prefix D512 H8 Hkv1 S1 C512 P17 float32 texture3d fused", + "operator": "sdpa_with_kv_cache.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001, + "outputs": ["output"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": [1, 1, 8, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_projected", + "sizes": [1, 1, 1, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_projected", + "sizes": [1, 1, 1, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 512, 1, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + }, + "access": "read_write" + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 512, 1, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + }, + "access": "read_write" + } + }, + { + "int": 17 + }, + { + "int": 1 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 8, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa float32 width", + "operator": "llama.custom_sdpa.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["gqa", "non_gqa", "gqa_tile2"], + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": [1, 1, 8, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 32, 2, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 32, 2, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": 31 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 8, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa float32 width (15)", + "operator": "llama.custom_sdpa.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["gqa", "non_gqa", "gqa_tile2"], + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": [1, 1, 8, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 32, 2, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 32, 2, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": 31 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 8, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa accu_tile2_d4 decode D4 H8 Hkv2 S1 C32 P31 float32", + "operator": "llama.custom_sdpa.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "gqa_tile2", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": [1, 1, 8, 4], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 32, 2, 4], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 32, 2, 4], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": 31 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 8, 4], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/cases/decode.json b/backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/cases/decode.json new file mode 100644 index 00000000000..b3b5d355a0a --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/cases/decode.json @@ -0,0 +1,494 @@ +[ + { + "name": "sdpa Llama-3.2-1B decode D64 H32 Hkv8 S1 C512 P511 float16 texture3d", + "operator": "llama.custom_sdpa.default", + "target_ms": 100, + "warmup_runs": 10, + "benchmark_runs": 30, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.01, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["gqa", "non_gqa"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": [1, 1, 32, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 512, 8, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 512, 8, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": 511 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 32, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa", + "operator": "llama.custom_sdpa.default", + "target_ms": 100, + "warmup_runs": 10, + "benchmark_runs": 30, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.01, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["gqa", "non_gqa"], + "v_SIZES": [ + [1, 1, 16, 128], + [1, 1, 24, 128] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": "v_SIZES", + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 512, 8, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 512, 8, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": 511 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + null, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa Llama-3.2-1B decode D64 H32 Hkv8 S1 C1024 P1023 float16 texture3d", + "operator": "llama.custom_sdpa.default", + "target_ms": 100, + "warmup_runs": 10, + "benchmark_runs": 30, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.01, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["gqa", "non_gqa"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": [1, 1, 32, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 1024, 8, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 1024, 8, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": 1023 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 32, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa (4)", + "operator": "llama.custom_sdpa.default", + "target_ms": 100, + "warmup_runs": 10, + "benchmark_runs": 30, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.01, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["gqa", "non_gqa"], + "v_SIZES": [ + [1, 1, 16, 128], + [1, 1, 24, 128] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": "v_SIZES", + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 1024, 8, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 1024, 8, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": 1023 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + null, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa Llama-3.2-1B decode D64 H32 Hkv8 S1 C4096 P4095 float16 texture3d", + "operator": "llama.custom_sdpa.default", + "target_ms": 100, + "warmup_runs": 10, + "benchmark_runs": 30, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.01, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["gqa", "non_gqa"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": [1, 1, 32, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 4096, 8, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 4096, 8, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": 4095 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 32, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa (6)", + "operator": "llama.custom_sdpa.default", + "target_ms": 100, + "warmup_runs": 10, + "benchmark_runs": 30, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.01, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["gqa", "non_gqa"], + "v_SIZES": [ + [1, 1, 16, 128], + [1, 1, 24, 128] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": "v_SIZES", + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 4096, 8, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 4096, 8, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": 4095 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + null, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/cases/performance.json b/backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/cases/performance.json new file mode 100644 index 00000000000..04392ffce6e --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/cases/performance.json @@ -0,0 +1,160 @@ +[ + { + "name": "sdpa Llama-3.2-1B prefill D64 H32 Hkv8 S512 C512 P0 float16 texture3d default", + "operator": "llama.custom_sdpa.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.01, + "rtol": 0.01 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": [1, 512, 32, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 512, 8, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 512, 8, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": 0 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 512, 32, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa", + "operator": "llama.custom_sdpa.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.01, + "rtol": 0.01 + }, + "sweep_variables": { + "v_SIZES": [ + [1, 512, 16, 128], + [1, 512, 24, 128] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": "v_SIZES", + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 512, 8, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 512, 8, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": 0 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + null, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/binary/BinaryReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/binary/BinaryReference.cpp new file mode 100644 index 00000000000..11c6c870c92 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/binary/BinaryReference.cpp @@ -0,0 +1,111 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +#include + +namespace vkcompute::shader_profiling::reference { + +Tensor binary_reference( + const Json& spec, + const TensorMap& tensors, + const std::string& function, + size_t threads) { + std::vector bindings; + for (const Json& binding : spec.at("bindings")) { + if (binding.contains("tensor")) { + bindings.push_back(binding.at("tensor")); + } + } + require( + bindings.size() == (function == "copy" ? 2u : 3u), + "Copy/add reference requires output followed by tensor input bindings"); + const Tensor& a = tensors.at(bindings[1]); + const auto sizes = spec.at("tensors") + .at(bindings[0]) + .at("sizes") + .get>(); + if (function == "copy") { + require(a.sizes == sizes, "Copy input/output shapes must match"); + return a; + } + const Tensor& b = tensors.at(bindings[2]); + const size_t rank = std::max(a.sizes.size(), b.sizes.size()); + std::vector output_sizes(rank, 1); + std::vector a_strides(rank), b_strides(rank); + size_t a_stride = 1; + size_t b_stride = 1; + for (size_t dim = rank; dim > 0; --dim) { + const size_t i = dim - 1; + const int64_t a_size = dim > rank - a.sizes.size() + ? a.sizes[dim - (rank - a.sizes.size()) - 1] + : 1; + const int64_t b_size = dim > rank - b.sizes.size() + ? b.sizes[dim - (rank - b.sizes.size()) - 1] + : 1; + require( + a_size == b_size || a_size == 1 || b_size == 1, + "Addition input shapes do not broadcast"); + output_sizes[i] = std::max(a_size, b_size); + a_strides[i] = a_size == 1 ? 0 : a_stride; + b_strides[i] = b_size == 1 ? 0 : b_stride; + a_stride *= a_size; + b_stride *= b_size; + } + require( + output_sizes == sizes, + "Addition output shape does not match broadcast shape"); + double alpha = 1; + if (spec.contains("push_constants") && !spec.at("push_constants").empty()) { + alpha = spec.at("push_constants").at(0).at("data").at(0); + } + Tensor output{sizes, std::vector(tensor_numel(sizes))}; + constexpr size_t kChunk = 4096; + parallel_for( + (output.values.size() + kChunk - 1) / kChunk, threads, [&](size_t chunk) { + const size_t end = std::min(output.values.size(), (chunk + 1) * kChunk); + for (size_t index = chunk * kChunk; index < end; ++index) { + size_t remainder = index; + size_t ai = 0; + size_t bi = 0; + for (size_t dim = rank; dim > 0; --dim) { + const size_t coordinate = remainder % output_sizes[dim - 1]; + remainder /= output_sizes[dim - 1]; + ai += coordinate * a_strides[dim - 1]; + bi += coordinate * b_strides[dim - 1]; + } + output.values[index] = a.values[ai] + alpha * b.values[bi]; + } + }); + return output; +} + +namespace { + +Tensor +copy_reference(const Json& spec, const TensorMap& tensors, size_t threads) { + return binary_reference(spec, tensors, "copy", threads); +} + +Tensor +add_reference(const Json& spec, const TensorMap& tensors, size_t threads) { + return binary_reference(spec, tensors, "add", threads); +} + +} // namespace + +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_FN(copy, copy_reference); + REGISTER_REFERENCE_FN(add, add_reference); + REGISTER_SHADER_REFERENCE(buffer_to_buffer_float, copy); + REGISTER_SHADER_REFERENCE(buffer_to_buffer_half, copy); + REGISTER_SHADER_REFERENCE(binary_add_texture3d_float, add); + REGISTER_SHADER_REFERENCE(binary_add_texture3d_half, add); +} + +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/binary/BinaryReference.h b/backends/vulkan/test/ops/benchmark_v2/operators/binary/BinaryReference.h new file mode 100644 index 00000000000..ebc2a53042e --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/binary/BinaryReference.h @@ -0,0 +1,19 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +namespace vkcompute::shader_profiling::reference { + +Tensor binary_reference( + const Json& spec, + const TensorMap& tensors, + const std::string& function, + size_t threads); + +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/binary/BinaryThroughput.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/binary/BinaryThroughput.cpp new file mode 100644 index 00000000000..15295080587 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/binary/BinaryThroughput.cpp @@ -0,0 +1,29 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +namespace vkcompute::shader_profiling { +namespace { + +double copy_flops(const Json&) { + return 0; +} + +double add_flops(const Json& spec) { + return 2 * throughput::tensor_elements(throughput::output_tensor(spec)); +} + +} // namespace + +REGISTER_THROUGHPUT_FUNCTIONS { + REGISTER_SHADER_MEMORY_TRAFFIC(buffer_to_buffer, throughput::tensor_traffic); + REGISTER_SHADER_COMPUTE_FLOPS(buffer_to_buffer, copy_flops); + REGISTER_SHADER_MEMORY_TRAFFIC(binary_add, throughput::tensor_traffic); + REGISTER_SHADER_COMPUTE_FLOPS(binary_add, add_flops); +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/binary/add/AddReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/binary/add/AddReference.cpp new file mode 100644 index 00000000000..c9d0fc165df --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/binary/add/AddReference.cpp @@ -0,0 +1,30 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include + +namespace vkcompute::shader_profiling::reference { +Tensor add_tensor_reference( + const Json& spec, + const TensorMap& tensors, + size_t threads) { + Json shader_spec = spec; + shader_spec["bindings"] = { + spec.at("args").back(), spec.at("args").at(0), spec.at("args").at(1)}; + const Json& alpha = spec.at("args").at(2); + const double alpha_value = alpha.is_null() ? 1.0 + : alpha.contains("float") ? alpha.at("float").get() + : alpha.at("int").get(); + shader_spec["push_constants"] = {{{"data", {alpha_value}}}}; + return binary_reference(shader_spec, tensors, "add", threads); +} +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_FN(add_tensor, add_tensor_reference); + REGISTER_OPERATOR_REFERENCE(aten.add.Tensor, add_tensor); +} +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/binary/add/AddReference.h b/backends/vulkan/test/ops/benchmark_v2/operators/binary/add/AddReference.h new file mode 100644 index 00000000000..212dfd621fd --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/binary/add/AddReference.h @@ -0,0 +1,18 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +namespace vkcompute::shader_profiling::reference { + +Tensor add_tensor_reference( + const Json& spec, + const TensorMap& tensors, + size_t threads); + +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/binary/add/AddThroughput.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/binary/add/AddThroughput.cpp new file mode 100644 index 00000000000..8cc7d7de64a --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/binary/add/AddThroughput.cpp @@ -0,0 +1,19 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +namespace vkcompute::shader_profiling { +namespace { +double add_tensor_flops(const Json& spec) { + return 2 * throughput::tensor_elements(throughput::output_tensor(spec)); +} +REGISTER_THROUGHPUT_FUNCTIONS { + REGISTER_OPERATOR_MEMORY_TRAFFIC(aten.add.Tensor, throughput::tensor_traffic); + REGISTER_OPERATOR_COMPUTE_FLOPS(aten.add.Tensor, add_tensor_flops); +} +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/binary/add/cases/correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/binary/add/cases/correctness.json new file mode 100644 index 00000000000..175fe082aa9 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/binary/add/cases/correctness.json @@ -0,0 +1,72 @@ +[ + { + "name": "add correctness", + "operator": "aten.add.Tensor", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_DTYPE": ["float32", "float16"], + "v_SIZES": [ + [1, 64, 64], + [1, 128, 128], + [1, 256, 256], + [1, 512, 512], + [1, 1, 1024], + [1, 1024, 1], + [32, 32, 32], + [16, 128, 64] + ], + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "a", + "sizes": "v_SIZES", + "dtype": "v_DTYPE", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [1, 1] + } + } + }, + { + "tensor": { + "name": "b", + "sizes": "v_SIZES", + "dtype": "v_DTYPE", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "v_DTYPE", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/binary/q8ta/Q8taBinaryReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/binary/q8ta/Q8taBinaryReference.cpp new file mode 100644 index 00000000000..bdd24d4aaf3 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/binary/q8ta/Q8taBinaryReference.cpp @@ -0,0 +1,35 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +namespace vkcompute::shader_profiling::reference { +static Tensor +q8ta_add_reference(const Json& spec, const TensorMap& tensors, size_t threads) { + const Tensor& a = tensor_arg(spec, 0, tensors); + const Tensor& b = tensor_arg(spec, 1, tensors); + Tensor out = q8ta::output_tensor(spec, spec.at("args").size() - 1); + const float as = q8ta::scale(spec, 2), bs = q8ta::scale(spec, 4), + os = q8ta::scale(spec, 6), alpha = q8ta::scale(spec, 8); + const int64_t az = q8ta::integer(spec, 3), bz = q8ta::integer(spec, 5), + oz = q8ta::integer(spec, 7); + parallel_for(out.values.size(), threads, [&](size_t i) { + const int64_t qa = + int64_t(a.values[q8ta::broadcast_index(i, out.sizes, a.sizes)]); + const int64_t qb = + int64_t(b.values[q8ta::broadcast_index(i, out.sizes, b.sizes)]); + const float value = + q8ta::dequantize(qa, as, az) + alpha * q8ta::dequantize(qb, bs, bz); + out.values[i] = double(q8ta::quantize(value, os, oz)); + }); + return out; +} +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_FN(q8ta_add, q8ta_add_reference); + REGISTER_OPERATOR_REFERENCE(et_vk.q8ta_add.default, q8ta_add); +} +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/binary/q8ta/Q8taBinaryThroughput.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/binary/q8ta/Q8taBinaryThroughput.cpp new file mode 100644 index 00000000000..462011239ee --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/binary/q8ta/Q8taBinaryThroughput.cpp @@ -0,0 +1,20 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +namespace vkcompute::shader_profiling { +namespace { +double q8ta_add_flops(const Json& spec) { + return 6 * throughput::tensor_elements(throughput::output_tensor(spec)); +} +REGISTER_THROUGHPUT_FUNCTIONS { + REGISTER_OPERATOR_MEMORY_TRAFFIC( + et_vk.q8ta_add.default, throughput::tensor_traffic); + REGISTER_OPERATOR_COMPUTE_FLOPS(et_vk.q8ta_add.default, q8ta_add_flops); +} +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/binary/q8ta/cases/correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/binary/q8ta/cases/correctness.json new file mode 100644 index 00000000000..eefa059c49b --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/binary/q8ta/cases/correctness.json @@ -0,0 +1,4087 @@ +[ + { + "name": "q8ta/binary/correctness", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [144], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [144], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [144], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [144], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [144], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [144], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (2)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [90], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [90], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [90], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [90], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [90], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [90], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (3)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"], + "v_OTHER_SIZES": [[1, 16, 32], [1, 16, 1], [32]] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": "v_OTHER_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": "v_OTHER_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (4)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 3, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 3, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 3, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 3, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (5)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 144], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 144], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 144], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 144], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 144], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 144], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (6)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 90], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 90], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 90], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 90], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 90], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 90], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (7)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 4], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 4], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 4], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 4], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (8)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [3, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [3, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [3, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [3, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [3, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [3, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (9)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 3, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 3, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 3, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 3, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (10)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 8, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 8, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (11)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 24, 24], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 16, 24, 24], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 24, 24], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 24, 24], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 16, 24, 24], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 24, 24], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (12)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 12, 12], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 32, 12, 12], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 12, 12], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 12, 12], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 32, 12, 12], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 12, 12], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (13)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 1, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 1, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 1, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 1, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (14)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 3, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 3, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 3, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 3, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (15)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 4, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 4, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 4, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 4, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (16)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 20, 20], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 8, 20, 20], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 20, 20], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 20, 20], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 8, 20, 20], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 20, 20], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (17)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 14, 14], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 16, 14, 14], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 14, 14], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 14, 14], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 16, 14, 14], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 14, 14], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (18)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 28, 28], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 8, 28, 28], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 28, 28], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 28, 28], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 8, 28, 28], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 28, 28], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (19)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 15, 15], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 3, 15, 15], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 3, 15, 15], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 15, 15], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 3, 15, 15], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 3, 15, 15], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (20)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 13, 31, 31], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 13, 31, 31], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 13, 31, 31], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 13, 31, 31], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 13, 31, 31], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 13, 31, 31], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (21)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 17, 23, 23], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 17, 23, 23], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 17, 23, 23], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 17, 23, 23], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 17, 23, 23], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 17, 23, 23], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (22)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (23)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (24)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 56, 56], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 128, 56, 56], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 56, 56], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 56, 56], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 128, 56, 56], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 56, 56], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (25)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (26)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 60, 512], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 1, 512], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 60, 512], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 60, 512], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 1, 512], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 60, 512], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (27)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 1, 512], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 60, 512], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 60, 512], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 1, 512], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 60, 512], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 60, 512], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (28)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"], + "v_OTHER_SIZES": [ + [1, 8, 1, 1], + [1, 1, 16, 16] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": "v_OTHER_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": "v_OTHER_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (29)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [2, 8, 6, 6], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 8, 6, 6], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [2, 8, 6, 6], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [2, 8, 6, 6], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 8, 6, 6], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [2, 8, 6, 6], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (30)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 13, 7, 9], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 1, 7, 1], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 13, 7, 9], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 13, 7, 9], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 1, 7, 1], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 13, 7, 9], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/binary/q8ta/cases/debug.json b/backends/vulkan/test/ops/benchmark_v2/operators/binary/q8ta/cases/debug.json new file mode 100644 index 00000000000..9b4fa9d73fa --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/binary/q8ta/cases/debug.json @@ -0,0 +1,483 @@ +[ + { + "name": "q8ta/binary/debug", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 16, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 16, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/debug (2)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 144], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 144], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 144], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 144], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 144], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 144], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/debug (3)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 90], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 90], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 90], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 90], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 90], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 90], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta add alpha", + "operator": "et_vk.q8ta_add.default", + "target_ms": 10, + "benchmark_runs": 5, + "warmup_runs": 1, + "max_dispatches": 512, + "validation_method": { + "function": "is_close", + "atol": 0, + "rtol": 0 + }, + "sweep_variables": { + "v_ARG8_FLOAT": [0, 0.5, -1] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [2, 1], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w", + "data": [120, -120] + } + }, + { + "tensor": { + "name": "other", + "sizes": [2], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w", + "data": [-20, 20] + } + }, + { + "float": 1 + }, + { + "int": 0 + }, + { + "float": 1 + }, + { + "int": 0 + }, + { + "float": 1 + }, + { + "int": 0 + }, + { + "float": "v_ARG8_FLOAT" + }, + { + "tensor": { + "name": "output", + "sizes": [2, 2], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/clone/q8ta/Q8taCloneReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/clone/q8ta/Q8taCloneReference.cpp new file mode 100644 index 00000000000..6bb270cc4fe --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/clone/q8ta/Q8taCloneReference.cpp @@ -0,0 +1,18 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +namespace vkcompute::shader_profiling::reference { +static Tensor +clone_reference(const Json& spec, const TensorMap& tensors, size_t) { + return tensor_arg(spec, 0, tensors); +} +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_FN(clone, clone_reference); + REGISTER_OPERATOR_REFERENCE(aten.clone.default, clone); +} +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/clone/q8ta/Q8taCloneThroughput.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/clone/q8ta/Q8taCloneThroughput.cpp new file mode 100644 index 00000000000..2befab916a5 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/clone/q8ta/Q8taCloneThroughput.cpp @@ -0,0 +1,20 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +namespace vkcompute::shader_profiling { +namespace { +double clone_flops(const Json&) { + return 0; +} +REGISTER_THROUGHPUT_FUNCTIONS { + REGISTER_OPERATOR_MEMORY_TRAFFIC( + aten.clone.default, throughput::tensor_traffic); + REGISTER_OPERATOR_COMPUTE_FLOPS(aten.clone.default, clone_flops); +} +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/clone/q8ta/cases/correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/clone/q8ta/cases/correctness.json new file mode 100644 index 00000000000..bbec31650b8 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/clone/q8ta/cases/correctness.json @@ -0,0 +1,94 @@ +[ + { + "name": "q8ta/clone/correctness", + "operator": "aten.clone.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.014060718249784, + "rtol": 0 + }, + "sweep_variables": { + "v_INPUT_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"], + "v_SIZES": [ + [1, 3, 16, 16], + [1, 8, 32, 32], + [1, 16, 24, 24], + [1, 32, 12, 12], + [1, 1, 64, 64], + [1, 3, 64, 64], + [1, 4, 16, 16], + [1, 8, 20, 20], + [1, 16, 14, 14], + [1, 8, 28, 28], + [1, 3, 15, 15], + [1, 13, 31, 31], + [1, 17, 23, 23], + [1, 64, 128, 128], + [1, 32, 64, 64], + [1, 128, 56, 56], + [1, 128, 128, 128] + ], + "v_OUTPUT_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_INPUT_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_OUTPUT_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_INPUT_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_OUTPUT_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/clone/q8ta/cases/debug.json b/backends/vulkan/test/ops/benchmark_v2/operators/clone/q8ta/cases/debug.json new file mode 100644 index 00000000000..7b9ff06c23e --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/clone/q8ta/cases/debug.json @@ -0,0 +1,74 @@ +[ + { + "name": "q8ta/clone/debug", + "operator": "aten.clone.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.014060718249784, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dReference.cpp new file mode 100644 index 00000000000..df865939107 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dReference.cpp @@ -0,0 +1,68 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +#include + +#include + +namespace vkcompute::shader_profiling::reference { + +Tensor +conv2d_reference(const Json& spec, const TensorMap& tensors, size_t threads) { + if (tensor_arg(spec, 0, tensors).sizes.size() == 3) { + return conv1d_reference(spec, tensors, threads); + } + require( + !spec.at("args").at(6).at("bool").get(), + "Conv2d reference does not support transposed convolution"); + const Tensor& input = tensor_arg(spec, 0, tensors); + const Tensor& weight = tensor_arg(spec, 1, tensors); + require(weight.sizes.size() == 4, "Conv2d reference requires rank-4 weight"); + const Conv2dParameters p{ + {weight.sizes[2], weight.sizes[3]}, + conv2d_pair_arg(spec, 3), + conv2d_pair_arg(spec, 4), + conv2d_pair_arg(spec, 5), + spec.at("args").at(8).at("int")}; + const Tensor* bias = + spec.at("args").at(2).is_null() ? nullptr : &tensor_arg(spec, 2, tensors); + const auto g = conv2d_geometry(input, weight, bias, p); + require( + weight.sizes == + std::vector( + {g.out_channels, + g.in_channels_per_group, + p.kernel[0], + p.kernel[1]}), + "Conv2d weight shape must be [Cout, Cin/groups, Kh, Kw]"); + return convolve( + input, + weight, + p, + g, + threads, + [&](int64_t oc, int64_t ci, int64_t ky, int64_t kx) { + return ((oc * g.in_channels_per_group + ci) * p.kernel[0] + ky) * + p.kernel[1] + + kx; + }, + [](float sum, double value, double weight_value) { + return std::fma(float(value), float(weight_value), sum); + }, + [&](float sum, int64_t oc) { + return sum + (bias ? float(bias->values[oc]) : 0.0f); + }); +} + +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_FN(conv2d, conv2d_reference); + REGISTER_OPERATOR_REFERENCE(aten.convolution.default, conv2d); +} + +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dReference.h b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dReference.h new file mode 100644 index 00000000000..15e8d9cd418 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dReference.h @@ -0,0 +1,15 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +namespace vkcompute::shader_profiling::reference { + +Tensor +conv2d_reference(const Json& spec, const TensorMap& tensors, size_t threads); +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dReferenceCommon.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dReferenceCommon.cpp new file mode 100644 index 00000000000..6e48de9cfa9 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dReferenceCommon.cpp @@ -0,0 +1,58 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +namespace vkcompute::shader_profiling::reference { + +std::array conv2d_pair_arg(const Json& spec, size_t index) { + const auto values = + spec.at("args").at(index).at("int_list").get>(); + require(values.size() == 2, "Conv2d spatial parameters require two values"); + return {values[0], values[1]}; +} + +Conv2dGeometry conv2d_geometry( + const Tensor& input, + const Tensor& weight, + const Tensor* bias, + const Conv2dParameters& p) { + require( + input.sizes.size() == 4 && !weight.sizes.empty(), + "Conv2d requires NCHW input"); + const int64_t channels = input.sizes[1]; + const int64_t out_channels = weight.sizes[0]; + require( + p.groups > 0 && channels % p.groups == 0 && out_channels % p.groups == 0, + "Conv2d channels must be divisible by groups"); + for (size_t dim = 0; dim < 2; ++dim) { + require( + p.kernel[dim] > 0 && p.stride[dim] > 0 && p.dilation[dim] > 0 && + p.padding[dim] >= 0, + "Conv2d requires positive kernel/stride/dilation and nonnegative padding"); + } + require( + !bias || bias->values.size() == size_t(out_channels), + "Conv2d bias size must equal output channels"); + const int64_t hn = + input.sizes[2] + 2 * p.padding[0] - p.dilation[0] * (p.kernel[0] - 1) - 1; + const int64_t wn = + input.sizes[3] + 2 * p.padding[1] - p.dilation[1] * (p.kernel[1] - 1) - 1; + require(hn >= 0 && wn >= 0, "Conv2d kernel exceeds padded input"); + return { + input.sizes[0], + channels, + input.sizes[2], + input.sizes[3], + out_channels, + channels / p.groups, + out_channels / p.groups, + channels / p.groups * p.kernel[0] * p.kernel[1], + hn / p.stride[0] + 1, + wn / p.stride[1] + 1}; +} + +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dReferenceCommon.h b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dReferenceCommon.h new file mode 100644 index 00000000000..6dbae353c5d --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dReferenceCommon.h @@ -0,0 +1,97 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +#include + +namespace vkcompute::shader_profiling::reference { + +struct Conv2dParameters { + std::array kernel; + std::array stride; + std::array padding; + std::array dilation; + int64_t groups; +}; + +struct Conv2dGeometry { + int64_t batches; + int64_t channels; + int64_t height; + int64_t width; + int64_t out_channels; + int64_t in_channels_per_group; + int64_t out_channels_per_group; + int64_t row_size; + int64_t out_height; + int64_t out_width; +}; + +std::array conv2d_pair_arg(const Json& spec, size_t index); +Conv2dGeometry conv2d_geometry( + const Tensor& input, + const Tensor& weight, + const Tensor* bias, + const Conv2dParameters& p); + +template < + typename Accumulator, + typename WeightIndex, + typename Accumulate, + typename Finish> +Tensor convolve( + const Tensor& input, + const Tensor& weight, + const Conv2dParameters& p, + const Conv2dGeometry& g, + size_t threads, + WeightIndex weight_index, + Accumulate accumulate, + Finish finish) { + Tensor output{ + {g.batches, g.out_channels, g.out_height, g.out_width}, + std::vector( + size_t(g.batches * g.out_channels * g.out_height * g.out_width))}; + const size_t rows = size_t(g.batches * g.out_channels * g.out_height); + parallel_for(rows, threads, [&](size_t row) { + const int64_t oy = row % g.out_height; + const int64_t oc = (row / g.out_height) % g.out_channels; + const int64_t batch = row / (g.out_height * g.out_channels); + const int64_t input_channel_start = + (oc / g.out_channels_per_group) * g.in_channels_per_group; + for (int64_t ox = 0; ox < g.out_width; ++ox) { + Accumulator sum = 0; + for (int64_t ci = 0; ci < g.in_channels_per_group; ++ci) { + const int64_t input_channel_offset = + (batch * g.channels + input_channel_start + ci) * g.height * + g.width; + for (int64_t ky = 0; ky < p.kernel[0]; ++ky) { + const int64_t iy = + oy * p.stride[0] - p.padding[0] + ky * p.dilation[0]; + if (iy < 0 || iy >= g.height) + continue; + for (int64_t kx = 0; kx < p.kernel[1]; ++kx) { + const int64_t ix = + ox * p.stride[1] - p.padding[1] + kx * p.dilation[1]; + if (ix < 0 || ix >= g.width) + continue; + sum = accumulate( + sum, + input.values[input_channel_offset + iy * g.width + ix], + weight.values[weight_index(oc, ci, ky, kx)]); + } + } + } + output.values[row * g.out_width + ox] = finish(sum, oc); + } + }); + return output; +} + +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dRouteCommon.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dRouteCommon.cpp new file mode 100644 index 00000000000..ab9cb74b1c5 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dRouteCommon.cpp @@ -0,0 +1,135 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +namespace vkcompute::shader_profiling::convolution { + +void supported(bool condition, const std::string& message) { + if (!condition) { + throw UnsupportedRoute(message); + } +} + +void check_options(const Json& options, bool intermediate_storage) { + if (!options.is_object()) { + throw std::invalid_argument("Route options must be an object"); + } + for (const auto& item : options.items()) { + if (!intermediate_storage || item.key() != "intermediate_storage") { + throw std::invalid_argument( + "Unknown convolution route option: " + item.key()); + } + } +} + +std::array +spatial_parameter(ComputeGraph& graph, ValueRef value, bool positive) { + const std::vector values = *graph.get_int_list(value); + supported(values.size() == 2, "Conv2d requires two spatial parameter values"); + for (int64_t item : values) { + supported( + item >= (positive ? 1 : 0) && item <= INT32_MAX, + "Conv2d spatial parameters are out of range"); + } + return {values[0], values[1]}; +} + +void constant_tensor( + ComputeGraph& graph, + ValueRef value, + const std::string& name) { + supported(graph.val_is_tref(value), name + " requires storage: constant"); +} + +int64_t product(int64_t a, int64_t b) { + supported( + a >= 0 && b >= 0 && + (b == 0 || a <= std::numeric_limits::max() / b), + "Convolution route dimensions overflow int64"); + return a * b; +} + +int64_t padded4(int64_t value) { + supported( + value > 0 && value <= INT32_MAX, + "Convolution channel dimension is out of range"); + return (value + 3) / 4 * 4; +} + +void grouped_packing(const ConvShape& shape) { + if (shape.groups > 1) { + supported( + shape.input[1] / shape.groups % 4 == 0 && + shape.output[1] / shape.groups % 4 == 0, + "Grouped convolution requires input/output channels per group divisible by 4"); + } +} + +Json selected(const std::string& name, const Json& options) { + return {{"name", name}, {"options", options}}; +} + +ConvShape conv_shape_base( + ComputeGraph& graph, + ValueRef input, + ValueRef weight, + ValueRef output, + int64_t groups, + size_t weight_rank) { + supported( + graph.val_is_tensor(input) && graph.val_is_tensor(output), + "Convolution input/output must be GPU tensors"); + constant_tensor(graph, weight, "Convolution weights"); + ConvShape shape{ + graph.sizes_of(input), + graph.sizes_of(output), + graph.sizes_of(weight), + {}, + groups}; + supported( + shape.input.size() == 4 && shape.output.size() == 4 && + shape.weight.size() == weight_rank, + "Convolution route supports rank-4 NCHW input/output and canonical weights"); + for (const auto* sizes : {&shape.input, &shape.output, &shape.weight}) { + for (int64_t size : *sizes) { + supported( + size > 0 && size <= INT32_MAX, + "Convolution tensor dimensions are out of range"); + } + } + supported( + shape.groups > 0 && shape.groups <= INT32_MAX && + shape.input[1] % shape.groups == 0 && + shape.output[1] % shape.groups == 0, + "Convolution channels must be divisible by positive groups"); + supported( + shape.input[0] == shape.output[0] && shape.weight[0] == shape.output[1], + "Convolution input/output/weight dimensions disagree"); + return shape; +} + +void check_conv_output( + ComputeGraph& graph, + const ConvShape& shape, + ValueRef stride_value, + ValueRef padding_value, + ValueRef dilation_value) { + const auto stride = spatial_parameter(graph, stride_value, true); + const auto padding = spatial_parameter(graph, padding_value, false); + const auto dilation = spatial_parameter(graph, dilation_value, true); + for (size_t i = 0; i < 2; ++i) { + const int64_t numerator = shape.input[i + 2] + 2 * padding[i] - + product(dilation[i], shape.kernel[i] - 1) - 1; + supported( + numerator >= 0 && numerator / stride[i] + 1 == shape.output[i + 2], + "Convolution declared output shape disagrees with spatial parameters"); + } +} + +} // namespace vkcompute::shader_profiling::convolution diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dRouteCommon.h b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dRouteCommon.h new file mode 100644 index 00000000000..947372b14db --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dRouteCommon.h @@ -0,0 +1,47 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once +#include + +#include + +namespace vkcompute::shader_profiling::convolution { + +struct ConvShape { + std::vector input; + std::vector output; + std::vector weight; + std::array kernel; + int64_t groups; +}; + +void supported(bool condition, const std::string& message); +void check_options(const Json& options, bool intermediate_storage = false); +std::array +spatial_parameter(ComputeGraph& graph, ValueRef value, bool positive); +void constant_tensor( + ComputeGraph& graph, + ValueRef value, + const std::string& name); +int64_t product(int64_t a, int64_t b); +int64_t padded4(int64_t value); +void grouped_packing(const ConvShape& shape); +Json selected(const std::string& name, const Json& options = Json::object()); +ConvShape conv_shape_base( + ComputeGraph& graph, + ValueRef input, + ValueRef weight, + ValueRef output, + int64_t groups, + size_t weight_rank); +void check_conv_output( + ComputeGraph& graph, + const ConvShape& shape, + ValueRef stride, + ValueRef padding, + ValueRef dilation); +} // namespace vkcompute::shader_profiling::convolution diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dRoutes.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dRoutes.cpp new file mode 100644 index 00000000000..48da7d5d6ce --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dRoutes.cpp @@ -0,0 +1,235 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +#include +#include + +#include +#include + +namespace vkcompute::shader_profiling { +using namespace convolution; +namespace { + +std::optional storage_option(const Json& options) { + check_options(options, true); + if (!options.contains("intermediate_storage")) { + return std::nullopt; + } + if (!options.at("intermediate_storage").is_string()) { + throw std::invalid_argument( + "intermediate_storage must be a storage type name"); + } + const std::string storage = options.at("intermediate_storage"); + if (storage == "buffer") { + return utils::kBuffer; + } + if (storage == "texture2d") { + return utils::kTexture2D; + } + if (storage == "texture3d") { + return utils::kTexture3D; + } + throw std::invalid_argument("Unknown intermediate_storage: " + storage); +} + +std::string storage_name(utils::StorageType storage) { + switch (storage) { + case utils::kBuffer: + return "buffer"; + case utils::kTexture2D: + return "texture2d"; + case utils::kTexture3D: + return "texture3d"; + } + throw std::invalid_argument("Unknown intermediate tensor storage"); +} + +bool floating(vkapi::ScalarType dtype) { + return dtype == vkapi::kFloat || dtype == vkapi::kHalf; +} + +ConvShape conv_shape(ComputeGraph& graph, const std::vector& args) { + if (args.size() != 10u) + throw std::invalid_argument( + "Convolution route received wrong argument count"); + const ValueRef input = args[0]; + const ValueRef output = args.back(); + const ValueRef weight = args[1]; + ConvShape shape = + conv_shape_base(graph, input, weight, output, graph.get_int(args[8]), 4); + + supported( + !graph.get_bool(args[6]), + "Explicit FP convolution routes support forward convolution only"); + supported( + shape.input[0] == 1, "Explicit FP convolution routes require batch 1"); + supported( + shape.weight[1] == shape.input[1] / shape.groups, + "FP convolution weight channels disagree"); + spatial_parameter(graph, args[7], false); + shape.kernel = {shape.weight[2], shape.weight[3]}; + check_conv_output(graph, shape, args[3], args[4], args[5]); + + const auto weight_dtype = graph.dtype_of(weight); + const auto effective_weight_dtype = weight_dtype == vkapi::kHalf && + !graph.context()->adapter_ptr()->has_full_float16_buffers_support() + ? vkapi::kFloat + : weight_dtype; + supported( + floating(graph.dtype_of(input)) && + graph.dtype_of(input) == graph.dtype_of(output) && + floating(weight_dtype) && + effective_weight_dtype == graph.dtype_of(input), + "FP convolution requires matching effective input/output/weight dtypes"); + supported( + graph.storage_type_of(input) == utils::kTexture3D && + graph.storage_type_of(output) == utils::kTexture3D && + graph.packed_dim_of(input) == WHCN::kChannelsDim && + graph.packed_dim_of(output) == WHCN::kChannelsDim, + "Explicit FP convolution routes require channels-packed texture3d IO"); + if (graph.val_is_not_none(args[2])) { + constant_tensor(graph, args[2], "FP convolution bias"); + supported( + graph.dtype_of(args[2]) == weight_dtype && + graph.sizes_of(args[2]) == std::vector{shape.output[1]}, + "FP convolution bias must match weight dtype and output channels"); + } + return shape; +} + +void check_fp_im2col_buffer_resources( + ComputeGraph& graph, + ValueRef input, + const ConvShape& shape, + std::optional override) { + const auto* adapter = graph.context()->adapter_ptr(); + const int64_t k = product( + product(padded4(shape.input[1]), shape.kernel[0]), shape.kernel[1]); + supported(k <= INT32_MAX, "FP im2col reduction size must fit int32"); + const int64_t k4 = k / 4; + const int64_t max2d = adapter->max_texture2d_dim(); + const int64_t elem_size = vkapi::element_size(graph.dtype_of(input)); + const int64_t bytes_per_row = product(product(shape.output[3], k), elem_size); + constexpr int64_t kScratchBudgetBytes = 16 * 1024 * 1024; + const int64_t rows = std::min( + shape.output[2], + std::max(int64_t(1), kScratchBudgetBytes / bytes_per_row)); + const int64_t matrix_rows = product(rows, shape.output[3]); + const int64_t max3d = adapter->max_texture3d_dim(); + const bool fits2d = k4 <= max2d && matrix_rows <= max2d; + const bool fits3d = shape.output[3] <= max3d && rows <= max3d && k4 <= max3d; + const utils::StorageType storage = override.value_or( + graph.device_is_mali() ? utils::kBuffer + : fits2d ? utils::kTexture2D + : fits3d ? utils::kTexture3D + : utils::kBuffer); + if (storage == utils::kBuffer) { + supported( + product(rows, bytes_per_row) <= graph.max_buffer_numel(), + "FP im2col scratch buffer exceeds device range"); + } +} + +} // namespace + +Json build_conv2d_direct( + ComputeGraph& graph, + const std::vector& args, + const Json& options) { + check_options(options); + const ConvShape shape = conv_shape(graph, args); + grouped_packing(shape); + conv2d_direct_impl( + graph, + args[0], + args[1], + args[2], + args[3], + args[4], + args[5], + args[6], + args[7], + args[8], + args[9], + false, + 0, + 0); + return selected("direct"); +} + +Json build_conv2d_im2col( + ComputeGraph& graph, + const std::vector& args, + const Json& options) { + const auto storage = storage_option(options); + const ConvShape shape = conv_shape(graph, args); + supported(shape.groups == 1, "FP im2col convolution requires groups 1"); + check_fp_im2col_buffer_resources(graph, args[0], shape, storage); + const size_t first = graph.execute_nodes().size(); + conv2d_gemm_impl( + graph, + args[0], + args[1], + args[2], + args[3], + args[4], + args[5], + args[9], + false, + 0, + 0, + storage); + const ValueRef scratch = + graph.execute_nodes().at(first)->args().at(0).refs.at(0); + return selected( + "im2col", + {{"intermediate_storage", storage_name(graph.storage_type_of(scratch))}}); +} + +Json inspect_conv2d_route( + ComputeGraph& graph, + const std::vector&, + size_t first, + size_t end) { + for (size_t i = first; i < end; ++i) { + const auto& node = graph.execute_nodes().at(i); + const std::string name = node->name(); + if (name.rfind("conv2d_im2col_", 0) == 0) { + const ValueRef scratch = node->args().at(0).refs.at(0); + return selected( + "im2col", + {{"intermediate_storage", + storage_name(graph.storage_type_of(scratch))}}); + } + if (name.rfind("conv2d_pw", 0) == 0) { + return selected("pointwise"); + } + if (name.rfind("conv2d_dw", 0) == 0) { + return selected("depthwise"); + } + if (name.rfind("conv_transpose2d", 0) == 0) { + return selected("transposed"); + } + if (name == "conv2d_float" || name == "conv2d_half") { + return selected("direct"); + } + } + return Json(); +} + +REGISTER_OPERATOR_ROUTES { + REGISTER_OPERATOR_ROUTE( + aten.convolution.default, direct, build_conv2d_direct); + REGISTER_OPERATOR_ROUTE( + aten.convolution.default, im2col, build_conv2d_im2col); + REGISTER_ROUTE_INSPECTOR(aten.convolution.default, inspect_conv2d_route); +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dRoutes.h b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dRoutes.h new file mode 100644 index 00000000000..0af21aa769d --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dRoutes.h @@ -0,0 +1,26 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +namespace vkcompute::shader_profiling { +Json build_conv2d_direct( + ComputeGraph& graph, + const std::vector& args, + const Json& options); +Json build_conv2d_im2col( + ComputeGraph& graph, + const std::vector& args, + const Json& options); +Json inspect_conv2d_route( + ComputeGraph& graph, + const std::vector& args, + size_t first, + size_t end); +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dThroughput.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dThroughput.cpp new file mode 100644 index 00000000000..b52a03eb307 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dThroughput.cpp @@ -0,0 +1,43 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +namespace vkcompute::shader_profiling { +namespace { + +double conv2d_flops(const Json& spec) { + const auto weight_sizes = throughput::tensor_argument(spec, 1) + .at("sizes") + .get>(); + if (weight_sizes.size() != 4) + throw std::invalid_argument( + "Conv2d work count requires rank-4 input/output and supported weight shape"); + if (spec.at("args").at(6).at("bool").get()) + throw std::invalid_argument( + "Conv2d work count does not support transposed convolution"); + return throughput::convolution_flops( + spec, + 1, + 2, + weight_sizes, + spec.at("args").at(3).at("int_list").get>(), + spec.at("args").at(4).at("int_list").get>(), + spec.at("args").at(5).at("int_list").get>(), + spec.at("args").at(8).at("int")); +} +} // namespace + +REGISTER_THROUGHPUT_FUNCTIONS { + REGISTER_OPERATOR_MEMORY_TRAFFIC( + aten.convolution.default, throughput::tensor_traffic); + REGISTER_OPERATOR_COMPUTE_FLOPS(aten.convolution.default, conv2d_flops); + REGISTER_SHADER_MEMORY_TRAFFIC(conv2d, throughput::tensor_traffic); +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dThroughputCommon.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dThroughputCommon.cpp new file mode 100644 index 00000000000..21dfdc9c837 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dThroughputCommon.cpp @@ -0,0 +1,76 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include + +namespace vkcompute::shader_profiling::throughput { + +double convolution_flops( + const Json& spec, + size_t weight_index, + size_t bias_index, + const std::vector& weight_sizes, + const std::vector& stride, + const std::vector& padding, + const std::vector& dilation, + int64_t groups) { + const Json& input = tensor_argument(spec, 0); + const Json& weight = tensor_argument(spec, weight_index); + const Json& output = output_tensor(spec); + const auto input_sizes = input.at("sizes").get>(); + const auto output_sizes = output.at("sizes").get>(); + tensor_elements(input); + tensor_elements(weight); + const double output_elements = tensor_elements(output); + if (input_sizes.size() != 4 || output_sizes.size() != 4 || + weight_sizes.size() != 4u) { + throw std::invalid_argument( + "Conv2d work count requires rank-4 input/output and supported weight shape"); + } + const std::vector kernel{weight_sizes[2], weight_sizes[3]}; + if (groups <= 0 || input_sizes[1] % groups != 0 || + output_sizes[1] % groups != 0 || stride.size() != 2 || + padding.size() != 2 || dilation.size() != 2 || + input_sizes[0] != output_sizes[0] || weight_sizes[0] != output_sizes[1]) { + throw std::invalid_argument( + "Conv2d work count dimensions/groups/parameters disagree"); + } + for (size_t i = 0; i < 2; ++i) { + if (kernel[i] <= 0 || stride[i] <= 0 || padding[i] < 0 || + dilation[i] <= 0) { + throw std::invalid_argument( + "Conv2d work count requires positive spatial parameters"); + } + const double expected = + std::floor( + (double(input_sizes[i + 2]) + 2 * double(padding[i]) - + double(dilation[i]) * double(kernel[i] - 1) - 1) / + double(stride[i])) + + 1; + if (expected != double(output_sizes[i + 2])) { + throw std::invalid_argument( + "Conv2d work count output shape does not match stride/padding/dilation"); + } + } + const double inner = + double(input_sizes[1] / groups) * double(kernel[0]) * double(kernel[1]); + if (weight_sizes[1] != input_sizes[1] / groups) { + throw std::invalid_argument( + "Conv2d work count weight channels/kernel dimensions disagree"); + } + const bool bias = !spec.at("args").at(bias_index).is_null(); + if (bias && + tensor_elements(tensor_argument(spec, bias_index)) != output_sizes[1]) { + throw std::invalid_argument( + "Conv2d work count bias must match output channels"); + } + return output_elements * (2 * inner + (bias ? 1 : 0)); +} + +} // namespace vkcompute::shader_profiling::throughput diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dThroughputCommon.h b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dThroughputCommon.h new file mode 100644 index 00000000000..382b31e173a --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dThroughputCommon.h @@ -0,0 +1,21 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once +#include + +namespace vkcompute::shader_profiling::throughput { + +double convolution_flops( + const Json& spec, + size_t weight_index, + size_t bias_index, + const std::vector& weight_sizes, + const std::vector& stride, + const std::vector& padding, + const std::vector& dilation, + int64_t groups); +} // namespace vkcompute::shader_profiling::throughput diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv1d/Conv1dReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv1d/Conv1dReference.cpp new file mode 100644 index 00000000000..d8f47912a56 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv1d/Conv1dReference.cpp @@ -0,0 +1,56 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +namespace vkcompute::shader_profiling::reference { +Tensor +conv1d_reference(const Json& spec, const TensorMap& tensors, size_t threads) { + const Tensor& input = tensor_arg(spec, 0, tensors); + const Tensor& weight = tensor_arg(spec, 1, tensors); + require( + input.sizes.size() == 3 && weight.sizes.size() == 3, + "Conv1d requires rank3 input and weight"); + const int64_t n = input.sizes[0], ci = input.sizes[1], l = input.sizes[2]; + const int64_t co = weight.sizes[0], cg = weight.sizes[1], k = weight.sizes[2]; + const int64_t groups = spec.at("args").at(8).at("int"); + const int64_t stride = spec.at("args").at(3).at("int_list").at(0); + const int64_t padding = spec.at("args").at(4).at("int_list").at(0); + const int64_t dilation = spec.at("args").at(5).at("int_list").at(0); + require( + groups > 0 && ci == cg * groups && co % groups == 0 && stride > 0 && + dilation > 0, + "Invalid Conv1d dimensions or groups"); + require( + !spec.at("args").at(6).at("bool").get(), + "Transposed Conv1d reference is unsupported"); + const int64_t lo = (l + 2 * padding - dilation * (k - 1) - 1) / stride + 1; + require(lo > 0, "Conv1d output length must be positive"); + const Tensor* bias = + spec.at("args").at(2).is_null() ? nullptr : &tensor_arg(spec, 2, tensors); + Tensor out{{n, co, lo}, std::vector(n * co * lo)}; + parallel_for(n * co, threads, [&](size_t slice) { + const int64_t batch = slice / co, oc = slice % co, + group = oc / (co / groups); + for (int64_t x = 0; x < lo; ++x) { + float sum = 0; + for (int64_t c = 0; c < cg; ++c) + for (int64_t j = 0; j < k; ++j) { + const int64_t ix = x * stride - padding + j * dilation; + if (ix >= 0 && ix < l) + sum = std::fma( + float(input.values[(batch * ci + group * cg + c) * l + ix]), + float(weight.values[(oc * cg + c) * k + j]), + sum); + } + out.values[slice * lo + x] = + sum + (bias ? float(bias->values[oc]) : 0.0f); + } + }); + return out; +} +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv1d/Conv1dReference.h b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv1d/Conv1dReference.h new file mode 100644 index 00000000000..f0432816534 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv1d/Conv1dReference.h @@ -0,0 +1,11 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once +#include +namespace vkcompute::shader_profiling::reference { +Tensor conv1d_reference(const Json&, const TensorMap&, size_t); +} diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv1d/cases/dw_correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv1d/cases/dw_correctness.json new file mode 100644 index 00000000000..08eb7a17655 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv1d/cases/dw_correctness.json @@ -0,0 +1,796 @@ +[ + { + "name": "conv1d dw_correctness [1, 16, 64, 3, 1, 1, 1, False]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 1, 3], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1] + }, + { + "int_list": [1] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 16 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d dw_correctness [1, 5, 64, 3, 1, 1, 1, False]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 5, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [5, 1, 3], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1] + }, + { + "int_list": [1] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 5 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 5, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d dw_correctness [2, 7, 64, 3, 1, 1, 1, False]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [2, 7, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [7, 1, 3], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1] + }, + { + "int_list": [1] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 7 + }, + { + "tensor": { + "name": "output", + "sizes": [2, 7, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d dw_correctness [1, 16, 64, 7, 1, 3, 2, False]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 1, 7], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1] + }, + { + "int_list": [3] + }, + { + "int_list": [2] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 16 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 58], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d dw_correctness [1, 7, 32, 5, 1, 2, 1, False]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 7, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [7, 1, 5], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1] + }, + { + "int_list": [2] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 7 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 7, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d dw_correctness [1, 64, 32, 3, 2, 1, 1, False]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 1, 3], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2] + }, + { + "int_list": [1] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 64 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 16], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d dw_correctness [1, 32, 128, 5, 1, 2, 1, True]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 1, 5], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1] + }, + { + "int_list": [2] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 32 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d dw_correctness [2, 16, 64, 3, 1, 1, 1, True]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [2, 16, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 1, 3], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1] + }, + { + "int_list": [1] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 16 + }, + { + "tensor": { + "name": "output", + "sizes": [2, 16, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d dw_correctness [1, 5, 64, 3, 1, 1, 1, True]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 5, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [5, 1, 3], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [5], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1] + }, + { + "int_list": [1] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 5 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 5, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d dw_correctness [1, 13, 48, 3, 2, 1, 1, True]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 13, 48], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [13, 1, 3], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [13], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [2] + }, + { + "int_list": [1] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 13 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 13, 24], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv1d/cases/dw_performance.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv1d/cases/dw_performance.json new file mode 100644 index 00000000000..bd68751ee35 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv1d/cases/dw_performance.json @@ -0,0 +1,474 @@ +[ + { + "name": "conv1d dw_performance [1, 256, 1024, 3, 1, 1, 1, False] float32 height", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256, 1024], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 1, 3], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1] + }, + { + "int_list": [1] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 256 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 256, 1024], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d dw_performance [1, 128, 4096, 31, 1, 15, 1, False] float32 height", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 4096], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 1, 31], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1] + }, + { + "int_list": [15] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 128 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 4096], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d dw_performance [1, 256, 1024, 3, 1, 1, 1, False] float16 height", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256, 1024], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 1, 3], + "dtype": "float16", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1] + }, + { + "int_list": [1] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 256 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 256, 1024], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d dw_performance [1, 128, 4096, 31, 1, 15, 1, False] float16 height", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 1, 31], + "dtype": "float16", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1] + }, + { + "int_list": [15] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 128 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d dw_performance [1, 512, 2048, 5, 1, 2, 1, True] float32 height", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 512, 2048], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [512, 1, 5], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [512], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1] + }, + { + "int_list": [2] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 512 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 512, 2048], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d dw_performance [1, 512, 2048, 5, 1, 2, 1, True] float16 height", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 512, 2048], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [512, 1, 5], + "dtype": "float16", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [512], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1] + }, + { + "int_list": [2] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 512 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 512, 2048], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv1d/cases/pw_correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv1d/cases/pw_correctness.json new file mode 100644 index 00000000000..c1fee9a524d --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv1d/cases/pw_correctness.json @@ -0,0 +1,1129 @@ +[ + { + "name": "conv1d pw_correctness [1, 16, 32, 64, False]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 16, 1], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d pw_correctness [1, 32, 16, 128, False]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 32, 1], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d pw_correctness [1, 64, 64, 32, False]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64, 1], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d pw_correctness [2, 16, 32, 64, False]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [2, 16, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 16, 1], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [2, 32, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d pw_correctness [1, 5, 7, 64, False]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 5, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [7, 5, 1], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 7, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d pw_correctness [1, 13, 17, 48, False]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 13, 48], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [17, 13, 1], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 17, 48], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d pw_correctness [1, 7, 5, 32, False]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 7, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [5, 7, 1], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 5, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d pw_correctness [1, 16, 32, 64, True]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 16, 1], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d pw_correctness [1, 32, 16, 128, True]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 32, 1], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d pw_correctness [1, 128, 256, 16, True]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 16], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128, 1], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 256, 16], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d pw_correctness [2, 16, 32, 64, True]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [2, 16, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 16, 1], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [2, 32, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d pw_correctness [1, 5, 7, 64, True]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 5, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [7, 5, 1], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [7], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 7, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d pw_correctness [1, 13, 17, 48, True]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 13, 48], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [17, 13, 1], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [17], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 17, 48], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d pw_correctness [2, 5, 13, 64, True]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [2, 5, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [13, 5, 1], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [13], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [2, 13, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv1d/cases/pw_performance.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv1d/cases/pw_performance.json new file mode 100644 index 00000000000..1ad860ca7bc --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv1d/cases/pw_performance.json @@ -0,0 +1,646 @@ +[ + { + "name": "conv1d pw_performance [1, 256, 512, 1024, False] float32 height", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256, 1024], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [512, 256, 1], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 512, 1024], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d pw_performance [1, 512, 256, 2048, False] float32 height", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 512, 2048], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 512, 1], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 256, 2048], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d pw_performance [1, 256, 512, 1024, False] float16 height", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256, 1024], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [512, 256, 1], + "dtype": "float16", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 512, 1024], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d pw_performance [1, 512, 256, 2048, False] float16 height", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 512, 2048], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 512, 1], + "dtype": "float16", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 256, 2048], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d pw_performance [1, 256, 512, 1024, True] float32 height", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256, 1024], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [512, 256, 1], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [512], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 512, 1024], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d pw_performance [1, 128, 128, 4096, True] float32 height", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 4096], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128, 1], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 4096], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d pw_performance [1, 256, 512, 1024, True] float16 height", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256, 1024], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [512, 256, 1], + "dtype": "float16", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [512], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 512, 1024], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d pw_performance [1, 128, 128, 4096, True] float16 height", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128, 1], + "dtype": "float16", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/DepthwiseRoutes.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/DepthwiseRoutes.cpp new file mode 100644 index 00000000000..bcf6b02e352 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/DepthwiseRoutes.cpp @@ -0,0 +1,300 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace vkcompute { + +struct Conv2dDWParams final { + utils::ivec2 overlay_region; + int in_group_size; +}; + +struct OutputParams final { + float out_min; + float out_max; +}; + +static std::string pick_conv2d_dw_shader_with_selector( + ComputeGraph& graph, + const ValueRef weight_data, + const ValueRef out, + const bool stride_equals_dilation, + const bool clamp_out, + const std::string& impl_selector) { + std::string kernel_name = "conv2d_dw"; + kernel_name.reserve(40); + + const auto& weight_sizes = graph.get_tref(weight_data)->sizes; + const bool is_3x3 = weight_sizes.at(2) == 3 && weight_sizes.at(3) == 3; + const bool is_5x5 = weight_sizes.at(2) == 5 && weight_sizes.at(3) == 5; + + if (!stride_equals_dilation && (is_3x3 || is_5x5)) { + kernel_name += "_sned"; + } + + if (is_3x3) { + kernel_name += "_output_tile_3x3"; + if (impl_selector == "b1x1") { + if (stride_equals_dilation) { + kernel_name += "_b1x1"; + } + } else if (impl_selector == "b4x2") { + } else { + if (stride_equals_dilation && graph.device_is_mali()) { + kernel_name += "_b1x1"; + } + } + } else if (is_5x5) { + kernel_name += "_output_tile_5x5"; + } + + if (clamp_out) { + kernel_name += "_clamp"; + } + add_dtype_suffix(kernel_name, graph.dtype_of(out)); + + return kernel_name; +} + +static GlobalWorkGrid conv2d_dw_gwg_fn( + ComputeGraph* graph, + const vkapi::ShaderInfo& shader, + const std::vector& args, + const std::vector& resize_args) { + (void)resize_args; + const ValueRef out = args.at(0).refs.at(0); + + const bool uses_output_tile = + shader.kernel_name.find("_output_tile") != std::string::npos; + + if (uses_output_tile) { + const bool is_sned = shader.kernel_name.find("_sned") != std::string::npos; + const utils::uvec3 image_extents = graph->create_gwg(out).extents(); + + if (is_sned) { + return GlobalWorkGrid( + {image_extents[0] * image_extents[1], image_extents[2], 1u}, + kTiledWorkGrid); + } + + uint32_t batch_x = 4u; + uint32_t batch_y = 2u; + if (shader.kernel_name.find("_b1x1") != std::string::npos) { + batch_x = 1u; + batch_y = 1u; + } + + uint32_t scaled_x = utils::div_up(image_extents[0], batch_x); + uint32_t scaled_y = utils::div_up(image_extents[1], batch_y); + return GlobalWorkGrid( + {scaled_x * scaled_y, image_extents[2], 1u}, kTiledWorkGrid); + } + + const utils::uvec3 base_extents = graph->create_gwg(out).extents(); + return graph->create_linear_gwg( + base_extents[0] * base_extents[1] * base_extents[2]); +} + +static LocalWorkGroup conv2d_dw_lwg_fn( + ComputeGraph* graph, + const vkapi::ShaderInfo& shader, + const GlobalWorkGrid& gwg, + const std::vector& args, + const std::vector& resize_args) { + (void)graph; + (void)shader; + (void)gwg; + (void)args; + (void)resize_args; + return LocalWorkGroup(64u, 1u, 1u); +} + +static ValueRef prepack_dw_weights(ComputeGraph& graph, const ValueRef vref) { + const auto original_sizes = graph.sizes_of(vref); + + int64_t out_channels_padded = + utils::align_up_4(utils::val_at(-4, original_sizes)); + int64_t height = utils::val_at(-2, original_sizes); + int64_t width = utils::val_at(-1, original_sizes); + + const std::vector final_sizes = { + 4, out_channels_padded / 4, height * width}; + + ValueRef v = graph.add_tensor( + final_sizes, + graph.dtype_of(vref), + utils::kTexture2D, + utils::kChannelsPacked); + + std::string kernel_name = "conv2d_dw_prepack_weights"; + add_dtype_suffix(kernel_name, graph.dtype_of(v)); + add_dtype_suffix(kernel_name, graph.get_staging_dtype_for(vref)); + + const auto original_sizes_pc = utils::make_ivec4(original_sizes, true); + graph.prepack_nodes().emplace_back(new PrepackNode( + graph, + VK_KERNEL_FROM_STR(kernel_name), + graph.create_gwg(v), + graph.create_lwg(v), + vref, + v, + {}, + {graph.packed_dim_of(v)}, + {graph.sizes_pc_of(v), + PushConstantDataInfo(&original_sizes_pc, sizeof(original_sizes_pc))})); + + return v; +} + +static void conv2d_dw_with_selector( + ComputeGraph& graph, + const ValueRef in, + const ValueRef weight_data, + const ValueRef bias, + const ValueRef stride, + const ValueRef padding, + const ValueRef dilation, + const ValueRef out, + const std::string& impl_selector) { + ValueRef arg_weight = prepack_dw_weights(graph, weight_data); + ValueRef arg_bias = prepack_biases( + graph, bias, weight_data, false, utils::kTexture2D, utils::kWidthPacked); + + check_conv_args(graph, in, out); + + Kernel2dParams kernel_params = create_kernel2d_params( + graph, weight_data, false, stride, padding, dilation); + + const bool stride_equals_dilation = + (kernel_params.stride[0] == kernel_params.dilation[0] && + kernel_params.stride[1] == kernel_params.dilation[1]); + + const auto& overlay_region = utils::make_ivec2({ + kernel_params.kernel_size[0] + + (kernel_params.kernel_size[0] - 1) * (kernel_params.dilation[0] - 1), + kernel_params.kernel_size[1] + + (kernel_params.kernel_size[1] - 1) * (kernel_params.dilation[1] - 1), + }); + const auto weight_sizes = graph.sizes_of(weight_data); + const int32_t in_group_size = + utils::safe_downcast(utils::align_up_4(weight_sizes.at(1))); + Conv2dDWParams extra_params = {overlay_region, in_group_size}; + + OutputParams out_params = { + std::numeric_limits::lowest(), std::numeric_limits::max()}; + + std::string kernel_name = pick_conv2d_dw_shader_with_selector( + graph, weight_data, out, stride_equals_dilation, false, impl_selector); + + vkapi::ShaderInfo shader = VK_KERNEL_FROM_STR(kernel_name); + + vkapi::ParamsBindList param_buffers; + std::vector push_constants; + + const bool uses_output_tile = + kernel_name.find("_output_tile") != std::string::npos; + + if (uses_output_tile) { + const utils::ivec4 kernel_param_size_stride = { + kernel_params.kernel_size[0], + kernel_params.kernel_size[1], + kernel_params.stride[0], + kernel_params.stride[1]}; + + const utils::ivec4 kernel_param_pad_dial = { + kernel_params.padding[0], + kernel_params.padding[1], + kernel_params.dilation[0], + kernel_params.dilation[1]}; + + push_constants = { + graph.logical_limits_pc_of(out), + graph.sizes_pc_of(in), + PushConstantDataInfo( + &kernel_param_size_stride, sizeof(kernel_param_size_stride)), + PushConstantDataInfo( + &kernel_param_pad_dial, sizeof(kernel_param_pad_dial)), + PushConstantDataInfo( + &extra_params, sizeof(extra_params), sizeof(utils::ivec4)), + PushConstantDataInfo(&out_params, sizeof(out_params)), + }; + } else { + param_buffers = { + graph.logical_limits_ubo(out), + graph.sizes_ubo(in), + graph.create_params_buffer(kernel_params), + graph.create_params_buffer(extra_params), + graph.create_params_buffer(out_params), + }; + } + + ValueRef transposed_ref = graph.add_scalar(false); + ValueRef output_padding = graph.add_none(); + + graph.execute_nodes().emplace_back(new DynamicDispatchNode( + graph, + shader, + conv2d_dw_gwg_fn, + conv2d_dw_lwg_fn, + {{out, vkapi::kWrite}, {{in, arg_weight, arg_bias}, vkapi::kRead}}, + param_buffers, + push_constants, + {}, + {weight_data, stride, padding, dilation, transposed_ref, output_padding}, + resize_conv2d_node)); +} + +namespace shader_profiling { +namespace { +template +Json build_dw_route( + ComputeGraph& graph, + const std::vector& args, + const Json& options) { + if (!options.empty()) + throw std::invalid_argument("Depthwise tile routes accept no options"); + if (args.size() != 10 || graph.extract_scalar(args[6]) || + graph.extract_scalar(args[8]) != graph.sizes_of(args[0]).at(1)) + throw UnsupportedRoute( + "Depthwise tile routes require forward depthwise convolution"); + if (!graph.is_texture_storage(args[0]) || + !graph.is_texture_storage(args.back())) + throw UnsupportedRoute("Depthwise tile routes require texture IO"); + conv2d_dw_with_selector( + graph, + args[0], + args[1], + args[2], + args[3], + args[4], + args[5], + args[9], + B1 ? "b1x1" : "b4x2"); + const auto& sizes = graph.sizes_of(args[1]); + const auto stride = *graph.get_int_list(args[3]); + const auto dilation = *graph.get_int_list(args[5]); + const bool selected = + B1 && sizes[2] == 3 && sizes[3] == 3 && stride == dilation; + return {{"name", selected ? "b1x1" : "b4x2"}, {"options", options}}; +} +REGISTER_OPERATOR_ROUTES { + REGISTER_OPERATOR_ROUTE( + aten.convolution.default, b1x1, (build_dw_route)); + REGISTER_OPERATOR_ROUTE( + aten.convolution.default, b4x2, (build_dw_route)); +} +} // namespace +} // namespace shader_profiling +} // namespace vkcompute diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/cases/correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/cases/correctness.json new file mode 100644 index 00000000000..d131e8f7a51 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/cases/correctness.json @@ -0,0 +1,1464 @@ +[ + { + "name": "conv2d correctness [[1, 8, 8, 8], 8, [3, 3], [1, 1], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [8, 8, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d correctness [[1, 16, 8, 8], 8, [3, 3], [1, 1], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [8, 16, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d correctness [[1, 11, 8, 8], 13, [3, 3], [1, 1], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 11, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [13, 11, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 13, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d correctness [[1, 8, 16, 16], 16, [3, 3], [2, 2], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 8, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d correctness [[1, 3, 16, 16], 8, [3, 3], [2, 2], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [8, 3, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d correctness [[1, 8, 8, 8], 8, [3, 3], [1, 1], [1, 1], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 8, 8], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [8, 8, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 8, 8], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d correctness [[1, 16, 8, 8], 8, [3, 3], [1, 1], [1, 1], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 8, 8], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [8, 16, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 8, 8], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d correctness [[1, 11, 8, 8], 13, [3, 3], [1, 1], [1, 1], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 11, 8, 8], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [13, 11, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 13, 8, 8], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d correctness [[1, 8, 16, 16], 16, [3, 3], [2, 2], [1, 1], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 16, 16], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 8, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 8, 8], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d correctness [[1, 3, 16, 16], 8, [3, 3], [2, 2], [1, 1], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 16, 16], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [8, 3, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 8, 8], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d correctness [[1, 8, 8, 8], 8, [3, 3], [1, 1], [1, 1], [1, 1], True] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [8, 8, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [8], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d correctness [[1, 16, 16, 16], 16, [3, 3], [1, 1], [1, 1], [1, 1], True] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 16, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d correctness [[1, 8, 8, 8], 8, [3, 3], [1, 1], [1, 1], [1, 1], True] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 8, 8], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [8, 8, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [8], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 8, 8], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d correctness [[1, 16, 16, 16], 16, [3, 3], [1, 1], [1, 1], [1, 1], True] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 16, 16], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 16, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 16, 16], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d correctness [[1, 16, float32 texture3d channels", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": { + "name": "im2col", + "options": { + "intermediate_storage": "v_OPTIONS_INTERMEDIATE_STORAGE" + } + }, + "sweep_variables": { + "v_OPTIONS_INTERMEDIATE_STORAGE": ["buffer", "texture2d", "texture3d"], + "v_SIZES": [ + [1, 16, 16, 16], + [1, 16, 48, 48] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 16, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d correctness [[1, 16, float16 texture3d channels", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": { + "name": "im2col", + "options": { + "intermediate_storage": "v_OPTIONS_INTERMEDIATE_STORAGE" + } + }, + "sweep_variables": { + "v_OPTIONS_INTERMEDIATE_STORAGE": ["buffer", "texture2d", "texture3d"], + "v_SIZES": [ + [1, 16, 16, 16], + [1, 16, 48, 48] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 16, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d correctness [[1, 11, 12, 12], 13, [3, 3], [1, 1], [1, 1], [1, 1], False] float32 {'name': 'im2col', 'options': {'intermediate_storage':", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": { + "name": "im2col", + "options": { + "intermediate_storage": "v_OPTIONS_INTERMEDIATE_STORAGE" + } + }, + "sweep_variables": { + "v_OPTIONS_INTERMEDIATE_STORAGE": ["buffer", "texture2d", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 11, 12, 12], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [13, 11, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 13, 12, 12], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d correctness [[1, 11, 12, 12], 13, [3, 3], [1, 1], [1, 1], [1, 1], False] float16 {'name': 'im2col', 'options': {'intermediate_storage':", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": { + "name": "im2col", + "options": { + "intermediate_storage": "v_OPTIONS_INTERMEDIATE_STORAGE" + } + }, + "sweep_variables": { + "v_OPTIONS_INTERMEDIATE_STORAGE": ["buffer", "texture2d", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 11, 12, 12], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [13, 11, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 13, 12, 12], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/cases/dw_correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/cases/dw_correctness.json new file mode 100644 index 00000000000..52abbb085d9 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/cases/dw_correctness.json @@ -0,0 +1,698 @@ +[ + { + "name": "conv2d dw_correctness [[1, 8, 16, 16], [3, 3], [1, 1], [1, 1], [1, 1], False] float32 auto", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [8, 1, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 8 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_correctness [[1, 11, 16, 16], [3, 3], [1, 1], [1, 1], [1, 1], False] float32 auto", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 11, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [11, 1, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 11 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 11, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_correctness [[1, 3, 16, 16], [3, 3], [1, 1], [1, 1], [1, 1], False] float32 auto", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 1, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 3 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 3, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_correctness [[1, 8, 16, 16], [5, 5], [1, 1], [2, 2], [1, 1], False] float32 auto", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [8, 1, 5, 5], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 8 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_correctness [[1, 8, 16, 16],", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_SIZES": [ + [8, 1, 3, 3], + [8, 1, 4, 4] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": "v_WEIGHT_SIZES", + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 8 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_correctness [[1, 8, 32, 32], [6, 6], [3, 3], [2, 2], [3, 3], False] float32 auto", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [8, 1, 6, 6], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [3, 3] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 8 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 7, 7], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_correctness [[1, 8, 16, 16], [3, 3], [1, 1], [1, 1], [1, 1], True] float32 auto", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [8, 1, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [8], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 8 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_correctness [[1, 8, 32, 32], [7, 7], [1, 1], [6, 6], [2, 2], True] float32 auto", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [8, 1, 7, 7], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [8], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [6, 6] + }, + { + "int_list": [2, 2] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 8 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_correctness [[1, 8, 31, 37], [3, 5], [2, 1], [1, 2], [1, 1], True] float32 auto", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 31, 37], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [8, 1, 3, 5], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [8], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [2, 1] + }, + { + "int_list": [1, 2] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 8 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 16, 37], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/cases/dw_performance.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/cases/dw_performance.json new file mode 100644 index 00000000000..eaf0dcea897 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/cases/dw_performance.json @@ -0,0 +1,1826 @@ +[ + { + "name": "conv2d dw_performance [[1, 24, 512, 512], [3, 3], [2, 2], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 24, 512, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [24, 1, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 24 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 24, 256, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 48, 256, 256], [3, 3], [2, 2], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 48, 256, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [48, 1, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 48 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 48, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 96, 128, 128], [3, 3], [2, 2], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 1, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 96 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 64, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 192, 64, 64], [3, 3], [2, 2], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 192, 64, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [192, 1, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 192 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 192, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 48, 256, 256], [3, 3], [1, 1], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 48, 256, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [48, 1, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 48 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 48, 256, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 96, 128, 128], [3, 3], [1, 1], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 1, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 96 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 192, 64, 64], [3, 3], [1, 1], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 192, 64, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [192, 1, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 192 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 192, 64, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 384, 32, 32], [3, 3], [1, 1], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 384, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [384, 1, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 384 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 384, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 256, 256, 256], [3, 3], [1, 1], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256, 256, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 1, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 256 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 256, 256, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 256, 128, 128], [3, 3], [1, 1], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 1, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 256 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 256, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 48, 256, 256], [5, 5], [1, 1], [2, 2], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 48, 256, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [48, 1, 5, 5], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 48 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 48, 256, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 96, 128, 128], [5, 5], [1, 1], [2, 2], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 1, 5, 5], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 96 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 24, 512, 512], [3, 3], [2, 2], [1, 1], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 24, 512, 512], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [24, 1, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 24 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 24, 256, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 48, 256, 256], [3, 3], [2, 2], [1, 1], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 48, 256, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [48, 1, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 48 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 48, 128, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 96, 128, 128], [3, 3], [2, 2], [1, 1], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 128, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 1, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 96 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 64, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 192, 64, 64], [3, 3], [2, 2], [1, 1], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 192, 64, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [192, 1, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 192 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 192, 32, 32], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 48, 256, 256], [3, 3], [1, 1], [1, 1], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 48, 256, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [48, 1, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 48 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 48, 256, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 96, 128, 128], [3, 3], [1, 1], [1, 1], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 128, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 1, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 96 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 128, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 192, 64, 64], [3, 3], [1, 1], [1, 1], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 192, 64, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [192, 1, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 192 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 192, 64, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 384, 32, 32], [3, 3], [1, 1], [1, 1], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 384, 32, 32], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [384, 1, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 384 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 384, 32, 32], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 256, 256, 256], [3, 3], [1, 1], [1, 1], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256, 256, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 1, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 256 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 256, 256, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 256, 128, 128], [3, 3], [1, 1], [1, 1], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256, 128, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 1, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 256 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 256, 128, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 48, 256, 256], [5, 5], [1, 1], [2, 2], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 48, 256, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [48, 1, 5, 5], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 48 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 48, 256, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 96, 128, 128], [5, 5], [1, 1], [2, 2], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 128, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 1, 5, 5], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 96 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 128, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/cases/performance.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/cases/performance.json new file mode 100644 index 00000000000..98accd0c5d9 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/cases/performance.json @@ -0,0 +1,1786 @@ +[ + { + "name": "conv2d performance [[1, 16, 8, 8], 64, [3, 3], [1, 1], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 16, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d performance [[1, 16, 8, 8], 128, [3, 3], [1, 1], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 16, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d performance [[1, 128, 18, 24], 64, [3, 3], [1, 1], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 18, 24], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 18, 24], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d performance [[1, 64, 36, 48], 32, [3, 3], [1, 1], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 36, 48], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 64, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 36, 48], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d performance [[1, 32, 72, 96], 128, [3, 3], [2, 2], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 72, 96], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 32, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 36, 48], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d performance [[1, 32, 72, 96], 32, [3, 3], [2, 2], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 72, 96], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 36, 48], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d performance [[1, 64, 36, 48], 64, [3, 3], [2, 2], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 36, 48], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 18, 24], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d performance [[1, 3, 144, 192], 32, [3, 3], [2, 2], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 144, 192], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 3, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 72, 96], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d performance [[1, 32, 72, 96], 128, [3, 3], [2, 2], [1, 1], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 72, 96], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 32, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 36, 48], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d performance [[1, 32, 72, 96], 32, [3, 3], [2, 2], [1, 1], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 72, 96], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 36, 48], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d performance [[1, 64, 36, 48], 64, [3, 3], [2, 2], [1, 1], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 36, 48], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 18, 24], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d performance [[1, 3, 144, 192], 32, [3, 3], [2, 2], [1, 1], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 144, 192], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 3, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 72, 96], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d performance [[1, 128, 18, 24], 64, [3, 3], [1, 1], [1, 1], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 18, 24], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 18, 24], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d performance [[1, 64, 36, 48], 32, [3, 3], [1, 1], [1, 1], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 36, 48], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 64, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 36, 48], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d performance [[1, 128, float32 texture3d channels", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"], + "v_SIZES": [ + [1, 128, 36, 48], + [1, 128, 18, 24] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d performance [[1, 256, 18, 24], 256, [3, 3], [1, 1], [1, 1], [1, 1], True] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256, 18, 24], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 256, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 256, 18, 24], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d performance [[1, 64, float32 texture3d channels", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"], + "v_SIZES": [ + [1, 64, 72, 96], + [1, 64, 36, 48] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d performance [[1, 32, float32 texture3d channels", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"], + "v_SIZES": [ + [1, 32, 144, 192], + [1, 32, 72, 96] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d performance [[1, 128, float16 texture3d channels", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"], + "v_SIZES": [ + [1, 128, 36, 48], + [1, 128, 18, 24] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d performance [[1, 256, 18, 24], 256, [3, 3], [1, 1], [1, 1], [1, 1], True] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256, 18, 24], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 256, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 256, 18, 24], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d performance [[1, 64, float16 texture3d channels", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"], + "v_SIZES": [ + [1, 64, 72, 96], + [1, 64, 36, 48] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d performance [[1, 32, float16 texture3d channels", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"], + "v_SIZES": [ + [1, 32, 144, 192], + [1, 32, 72, 96] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/cases/pw_correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/cases/pw_correctness.json new file mode 100644 index 00000000000..a6c2ea90a98 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/cases/pw_correctness.json @@ -0,0 +1,378 @@ +[ + { + "name": "conv2d pw_correctness [1, 16, 32, 8, 8, False] float32 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 16, 1, 1], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_correctness [1, 32, 16, 8, 8, False] float32 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 32, 1, 1], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_correctness [1, 16, 32, 8, 8, True] float32 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 16, 1, 1], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_correctness [1, 13, 27, 8, 8, False] float32 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 13, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [27, 13, 1, 1], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 27, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_correctness [1, 33, 17, 8, 8, False] float32 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 33, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [17, 33, 1, 1], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 17, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/cases/pw_performance.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/cases/pw_performance.json new file mode 100644 index 00000000000..0ab8e008cc4 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/cases/pw_performance.json @@ -0,0 +1,1900 @@ +[ + { + "name": "conv2d pw_performance [1, 48, 96, 16, 16, False] float32 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 48, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 48, 1, 1], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 96, 48, 16, 16, False] float32 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [48, 96, 1, 1], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 48, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 48, 96, 256, 256, False] float32 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 48, 256, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 48, 1, 1], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 256, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 48, 96, 256, 256, False] float16 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 48, 256, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 48, 1, 1], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 256, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 96, 48, 256, 256, False] float32 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 256, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [48, 96, 1, 1], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 48, 256, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 96, 48, 256, 256, False] float16 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 256, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [48, 96, 1, 1], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 48, 256, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 96, 192, 128, 128, False] float32 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [192, 96, 1, 1], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 192, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 96, 192, 128, 128, False] float16 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 128, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [192, 96, 1, 1], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 192, 128, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 192, 96, 128, 128, False] float32 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 192, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 192, 1, 1], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 192, 96, 128, 128, False] float16 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 192, 128, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 192, 1, 1], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 128, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 192, 384, 64, 64, False] float32 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 192, 64, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [384, 192, 1, 1], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 384, 64, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 192, 384, 64, 64, False] float16 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 192, 64, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [384, 192, 1, 1], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 384, 64, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 384, 192, 64, 64, False] float32 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 384, 64, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [192, 384, 1, 1], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 192, 64, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 384, 192, 64, 64, False] float16 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 384, 64, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [192, 384, 1, 1], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 192, 64, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 384, 768, 32, 32, False] float32 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 384, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [768, 384, 1, 1], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 768, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 384, 768, 32, 32, False] float16 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 384, 32, 32], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [768, 384, 1, 1], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 768, 32, 32], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 768, 384, 32, 32, False] float32 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 768, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [384, 768, 1, 1], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 384, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 768, 384, 32, 32, False] float16 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 768, 32, 32], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [384, 768, 1, 1], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 384, 32, 32], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 48, 256, 256, 256, False] float32 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 48, 256, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 48, 1, 1], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 256, 256, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 48, 256, 256, 256, False] float16 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 48, 256, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 48, 1, 1], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 256, 256, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 256, 32, 256, 256, False] float32 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256, 256, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 256, 1, 1], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 256, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 256, 32, 256, 256, False] float16 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256, 256, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 256, 1, 1], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 256, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 96, 256, 128, 128, False] float32 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 96, 1, 1], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 256, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 96, 256, 128, 128, False] float16 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 128, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 96, 1, 1], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 256, 128, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 256, 64, 128, 128, False] float32 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 256, 1, 1], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 256, 64, 128, 128, False] float16 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256, 128, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 256, 1, 1], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 128, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taConv2dReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taConv2dReference.cpp new file mode 100644 index 00000000000..dae94c71558 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taConv2dReference.cpp @@ -0,0 +1,146 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +#include + +#include +#include + +namespace vkcompute::shader_profiling::reference { +namespace { +bool fused_bias(const Json& spec) { + const Json context = spec.value("reference_context", Json::object()); + if (context.contains("shaders")) { + for (const Json& node : context.at("shaders")) { + if (node.get().rfind("q8ta_conv2d_pw", 0) == 0) + return true; + } + return false; + } + const Json route = spec.value("route", Json("auto")); + const std::string name = route.is_string() ? route.get() + : route.value("name", "auto"); + return spec.at("operator") == "et_vk.q8ta_conv2d_pw.default" || + name.rfind("im2col", 0) == 0; +} +} // namespace + +Tensor q8ta_conv2d_reference( + const Json& spec, + const TensorMap& tensors, + size_t threads) { + const Conv2dParameters p{ + conv2d_pair_arg(spec, 9), + conv2d_pair_arg(spec, 10), + conv2d_pair_arg(spec, 11), + conv2d_pair_arg(spec, 12), + q8ta::integer(spec, 13)}; + const float input_scale = q8ta::scale(spec, 1); + const float output_scale = q8ta::scale(spec, 6); + const int64_t input_zp = q8ta::integer(spec, 2); + const int64_t output_zp = q8ta::integer(spec, 7); + require( + std::isfinite(input_scale) && input_scale > 0 && + std::isfinite(output_scale) && output_scale > 0, + "Q8 convolution scales must be positive and finite"); + require( + input_zp >= -128 && input_zp <= 127 && output_zp >= -128 && + output_zp <= 127, + "Q8 convolution zero points must fit int8"); + const bool legacy = spec.at("args").size() == 15; + const std::string activation = + legacy ? "none" : spec.at("args").at(14).at("string").get(); + require( + activation == "none" || activation == "relu", + "Q8 convolution reference supports none/relu activation"); + const Tensor& input = tensor_arg(spec, 0, tensors); + const Tensor& weight = tensor_arg(spec, 3, tensors); + const Tensor& sums = tensor_arg(spec, 4, tensors); + const Tensor& scales = tensor_arg(spec, 5, tensors); + const Tensor* bias = + spec.at("args").at(8).is_null() ? nullptr : &tensor_arg(spec, 8, tensors); + const std::string output_name = spec.at("args").back().at("tensor"); + const auto output_sizes = spec.at("tensors") + .at(output_name) + .at("sizes") + .get>(); + require(output_sizes.size() == 4, "Q8 convolution output must be NCHW"); + const Tensor geometry_weight{{output_sizes[1]}, {}}; + const auto g = conv2d_geometry(input, geometry_weight, nullptr, p); + require( + output_sizes == + std::vector{ + g.batches, g.out_channels, g.out_height, g.out_width}, + "Q8 convolution output shape disagrees with parameters"); + const bool depthwise = weight.sizes.size() == 3; + require( + depthwise + ? (weight.sizes == + std::vector{ + p.kernel[0], p.kernel[1], g.out_channels} && + g.channels == g.out_channels && p.groups == g.channels) + : (weight.sizes.size() == 2 && weight.sizes[0] == g.out_channels && + weight.sizes[1] >= g.row_size), + "Q8 convolution weights require OHWI rows or depthwise HWC"); + require( + sums.values.size() >= size_t(g.out_channels) && + scales.values.size() >= size_t(g.out_channels) && + (!bias || bias->values.size() >= size_t(g.out_channels)), + "Q8 convolution requires per-output-channel sums/scales/bias"); + const auto weight_index = + [&](int64_t oc, int64_t ci, int64_t ky, int64_t kx) { + return depthwise ? (ky * p.kernel[1] + kx) * g.out_channels + oc + : oc * weight.sizes[1] + + (ky * p.kernel[1] + kx) * g.in_channels_per_group + ci; + }; + std::vector logical_sums(size_t(g.out_channels), 0); + for (int64_t oc = 0; oc < g.out_channels; ++oc) + for (int64_t ci = 0; ci < g.in_channels_per_group; ++ci) + for (int64_t ky = 0; ky < p.kernel[0]; ++ky) + for (int64_t kx = 0; kx < p.kernel[1]; ++kx) + logical_sums[size_t(oc)] += + int64_t(weight.values[size_t(weight_index(oc, ci, ky, kx))]); + const bool fused = !legacy && fused_bias(spec); + return convolve( + input, + weight, + p, + g, + threads, + weight_index, + [&](int64_t sum, double value, double weight_value) { + return sum + (int64_t(value) - input_zp) * int64_t(weight_value); + }, + [&](int64_t sum, int64_t oc) { + const int64_t raw = sum + input_zp * logical_sums[size_t(oc)]; + const float corrected = fused + ? float(raw - input_zp * int64_t(sums.values[size_t(oc)])) + : float(raw) - float(sums.values[size_t(oc)]) * float(input_zp); + const float multiplier = float(scales.values[size_t(oc)]) * input_scale; + const float channel_bias = bias ? float(bias->values[size_t(oc)]) : 0; + const float scaled = corrected * multiplier; + float value = fused ? std::fma(corrected, multiplier, channel_bias) + : scaled + channel_bias; + if (activation == "relu") + value = std::max(value, 0.0f); + return double(q8ta::requantize(value, 1.0f / output_scale, output_zp)); + }); +} + +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_FN(q8ta_conv2d, q8ta_conv2d_reference); + REGISTER_OPERATOR_REFERENCE(et_vk.q8ta_conv2d.default, q8ta_conv2d); + REGISTER_OPERATOR_REFERENCE(et_vk.q8ta_conv2d_general.default, q8ta_conv2d); + REGISTER_OPERATOR_REFERENCE(et_vk.q8ta_conv2d_im2col.default, q8ta_conv2d); + REGISTER_OPERATOR_REFERENCE(et_vk.q8ta_conv2d_dw.default, q8ta_conv2d); + REGISTER_OPERATOR_REFERENCE(et_vk.q8ta_conv2d_pw.default, q8ta_conv2d); + REGISTER_OPERATOR_REFERENCE( + et_vk.conv2d_q8ta_q8csw_q8to.default, q8ta_conv2d); +} +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taConv2dReference.h b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taConv2dReference.h new file mode 100644 index 00000000000..251f382d8d0 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taConv2dReference.h @@ -0,0 +1,21 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +namespace vkcompute::shader_profiling::reference { + +Tensor q8ta_conv2d_reference( + const Json& spec, + const TensorMap& tensors, + size_t threads); +Tensor q8ta_transposed_reference( + const Json& spec, + const TensorMap& tensors, + size_t threads); +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taConv2dRoutes.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taConv2dRoutes.cpp new file mode 100644 index 00000000000..8c017cf3279 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taConv2dRoutes.cpp @@ -0,0 +1,298 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace vkcompute::shader_profiling { +using namespace convolution; +namespace { + +ConvShape conv_shape(ComputeGraph& graph, const std::vector& args) { + if (args.size() != 16u) + throw std::invalid_argument( + "Convolution route received wrong argument count"); + const ValueRef input = args[0]; + const ValueRef output = args.back(); + const ValueRef weight = args[3]; + ConvShape shape = + conv_shape_base(graph, input, weight, output, graph.get_int(args[13]), 2); + + shape.kernel = spatial_parameter(graph, args[9], true); + const int64_t kernel_columns = product( + product(shape.input[1] / shape.groups, shape.kernel[0]), shape.kernel[1]); + supported( + shape.weight[1] == padded4(kernel_columns), + "Q8 convolution weights require aligned [Cout, Kh*Kw*Cin/groups] rows"); + check_conv_output(graph, shape, args[10], args[11], args[12]); + + supported( + graph.dtype_of(input) == vkapi::kInt8x4 && + graph.dtype_of(output) == vkapi::kInt8x4 && + graph.is_buffer_storage(input) && graph.is_buffer_storage(output), + "Q8 convolution requires Int8x4 buffer input/output"); + supported( + q8ta_conv2d_check_packed_dim_info(graph.packed_dim_info_of(input)) && + q8ta_conv2d_check_packed_dim_info(graph.packed_dim_info_of(output)), + "Q8 convolution requires channels-packed4 with outer width block 1 or 4"); + supported( + graph.dtype_of(weight) == vkapi::kChar, + "Q8 convolution weights require int8 dtype"); + for (size_t i : {4, 5, 8}) { + if (i == 8 && graph.val_is_none(args[i])) { + continue; + } + constant_tensor(graph, args[i], "Q8 convolution channel parameters"); + supported( + graph.dtype_of(args[i]) == (i == 4 ? vkapi::kInt : vkapi::kFloat) && + graph.dim_of(args[i]) == 1 && + graph.size_at(0, args[i]) >= shape.output[1], + "Q8 convolution sums/scales/bias require canonical int32/float32 channel vectors"); + } + for (size_t i : {1, 6}) { + const double scale = graph.extract_scalar(args[i]); + supported( + std::isfinite(scale) && scale > 0, + "Q8 convolution scales must be positive and finite"); + } + for (size_t i : {2, 7}) { + const int64_t zero_point = graph.extract_scalar(args[i]); + supported( + zero_point >= -128 && zero_point <= 127, + "Q8 convolution zero points must fit int8"); + } + const std::string activation = graph.extract_string(args[14]); + supported( + activation == "none" || activation == "relu", + "Q8 convolution supports none/relu activation"); + return shape; +} + +} // namespace + +Json build_q8ta_conv2d_direct( + ComputeGraph& graph, + const std::vector& args, + const Json& options) { + check_options(options); + const ConvShape shape = conv_shape(graph, args); + grouped_packing(shape); + q8ta_conv2d_general_impl(graph, args); + return selected("direct"); +} + +namespace { +enum class DotRoute { Auto, Signed, Unsigned, Fallback }; + +Json build_im2col( + ComputeGraph& graph, + const std::vector& args, + const Json& options, + DotRoute dot, + const char* route_name) { + check_options(options); + const ConvShape shape = conv_shape(graph, args); + grouped_packing(shape); + supported( + shape.input[1] / shape.groups % 4 == 0, + "Q8 im2col convolution requires input channels per group divisible by4"); + const auto sizes = + calculate_q8ta_im2col_sizes(&graph, args[0], args[15], args[9], args[13]); + const auto plan = make_q8ta_conv2d_stream_plan_for_device( + sizes[0], sizes[1], sizes[2], sizes[3], graph.max_buffer_numel()); + supported(plan.feasible, "Q8 im2col has no feasible device scratch plan"); + if (dot == DotRoute::Unsigned) { + supported( + graph.can_use_int8_dot_product() && + shape.weight[1] <= kMaxUnsignedDotAccumulatorBytes, + "Unsigned Q8 im2col requires packed integer dot and bounded K"); + q8ta_conv2d_im2col_impl(graph, true, args); + } else if (dot == DotRoute::Signed) { + q8ta_conv2d_im2col_impl(graph, false, args); + } else if (dot == DotRoute::Fallback) { + vkapi::ScopedAdapterCapabilityOverride no_dot( + graph.context()->adapter_ptr(), + vkapi::AdapterCapabilityOverrides::without_dot_product_support()); + q8ta_conv2d_im2col_impl(graph, false, args); + } else { + q8ta_conv2d_im2col(graph, args); + } + return selected(route_name); +} + +Json build_im2col_signed( + ComputeGraph& graph, + const std::vector& args, + const Json& options) { + return build_im2col(graph, args, options, DotRoute::Signed, "im2col"); +} +Json build_im2col_unsigned( + ComputeGraph& graph, + const std::vector& args, + const Json& options) { + return build_im2col( + graph, args, options, DotRoute::Unsigned, "im2col_unsigned"); +} +Json build_im2col_fallback( + ComputeGraph& graph, + const std::vector& args, + const Json& options) { + return build_im2col( + graph, args, options, DotRoute::Fallback, "im2col_fallback"); +} + +Json build_pointwise( + ComputeGraph& graph, + const std::vector& args, + const Json& options, + DotRoute dot, + const char* route_name) { + check_options(options); + const ConvShape shape = conv_shape(graph, args); + supported( + shape.kernel == std::array{1, 1} && shape.groups == 1 && + spatial_parameter(graph, args[10], true) == + std::array{1, 1} && + spatial_parameter(graph, args[11], false) == + std::array{0, 0}, + "Q8 pointwise route requires ungrouped1x1, unit stride, zero padding"); + if (dot == DotRoute::Unsigned) { + supported( + graph.can_use_int8_dot_product() && + shape.weight[1] <= kMaxUnsignedDotAccumulatorBytes, + "Unsigned Q8 pointwise requires packed integer dot and bounded K"); + q8ta_conv2d_pw_impl(graph, true, args); + } else if (dot == DotRoute::Signed) { + q8ta_conv2d_pw_impl(graph, false, args); + } else { + q8ta_conv2d_pw(graph, args); + } + return selected(route_name); +} +Json build_pw_signed( + ComputeGraph& graph, + const std::vector& args, + const Json& options) { + return build_pointwise(graph, args, options, DotRoute::Signed, "pw_signed"); +} +Json build_pw_unsigned( + ComputeGraph& graph, + const std::vector& args, + const Json& options) { + return build_pointwise( + graph, args, options, DotRoute::Unsigned, "pw_unsigned"); +} +Json build_pw_auto( + ComputeGraph& graph, + const std::vector& args, + const Json& options) { + return build_pointwise(graph, args, options, DotRoute::Auto, "pw_auto"); +} + +Json build_legacy_4w4c( + ComputeGraph& graph, + const std::vector& args, + const Json& options) { + check_options(options); + supported(args.size() == 16, "Q8 legacy route requires16 native arguments"); + for (ValueRef tensor : {args[0], args[15]}) { + const auto& packing = graph.packed_dim_info_of(tensor); + supported( + graph.dtype_of(tensor) == vkapi::kInt8x4 && + graph.is_buffer_storage(tensor) && + q8ta_conv2d_check_packed_dim_info(packing) && + packing.outer_packed_dim_block_size == 4, + "Q8 legacy route requires4w4c input/output buffers"); + } + supported( + graph.extract_string(args[14]) == "none", + "Q8 legacy route has no fused activation argument"); + std::vector native(args.begin(), args.begin() + 14); + native.push_back(args[15]); + VK_GET_OP_FN("et_vk.conv2d_q8ta_q8csw_q8to.default")(graph, native); + return selected("legacy_4w4c"); +} +} // namespace + +Json build_q8ta_conv2d_im2col( + ComputeGraph& graph, + const std::vector& args, + const Json& options) { + return build_im2col(graph, args, options, DotRoute::Auto, "im2col"); +} + +Json inspect_q8ta_conv2d_route( + ComputeGraph& graph, + const std::vector&, + size_t first, + size_t end) { + for (size_t i = first; i < end; ++i) { + const std::string name = graph.execute_nodes().at(i)->name(); + if (name == "q8ta_im2col") { + return selected("im2col"); + } + if (name.rfind("q8ta_conv2d_pw", 0) == 0) { + return selected("pointwise"); + } + if (name.rfind("q8ta_conv2d_dw", 0) == 0) { + return selected("depthwise"); + } + if (name.rfind("q8ta_conv2d_transposed", 0) == 0) { + return selected("transposed"); + } + if (name == "q8ta_conv2d_float" || name == "q8ta_conv2d_fallback_float") { + return selected("direct"); + } + } + return Json(); +} + +REGISTER_OPERATOR_ROUTES { + REGISTER_OPERATOR_ROUTE( + et_vk.q8ta_conv2d.default, direct, build_q8ta_conv2d_direct); + REGISTER_OPERATOR_ROUTE( + et_vk.q8ta_conv2d.default, general, build_q8ta_conv2d_direct); + REGISTER_OPERATOR_ROUTE( + et_vk.q8ta_conv2d.default, im2col, build_im2col_signed); + REGISTER_OPERATOR_ROUTE( + et_vk.q8ta_conv2d.default, im2col_auto, build_q8ta_conv2d_im2col); + REGISTER_OPERATOR_ROUTE( + et_vk.q8ta_conv2d.default, im2col_unsigned, build_im2col_unsigned); + REGISTER_OPERATOR_ROUTE( + et_vk.q8ta_conv2d.default, im2col_fallback, build_im2col_fallback); + REGISTER_OPERATOR_ROUTE( + et_vk.q8ta_conv2d.default, legacy_4w4c, build_legacy_4w4c); + REGISTER_OPERATOR_ROUTE( + et_vk.q8ta_conv2d_dw.default, legacy_4w4c, build_legacy_4w4c); + REGISTER_OPERATOR_ROUTE( + et_vk.q8ta_conv2d_pw.default, legacy_4w4c, build_legacy_4w4c); + REGISTER_OPERATOR_ROUTE( + et_vk.q8ta_conv2d_pw.default, pw_signed, build_pw_signed); + REGISTER_OPERATOR_ROUTE( + et_vk.q8ta_conv2d_pw.default, pw_unsigned, build_pw_unsigned); + REGISTER_OPERATOR_ROUTE(et_vk.q8ta_conv2d_pw.default, pw_auto, build_pw_auto); + REGISTER_ROUTE_INSPECTOR( + et_vk.q8ta_conv2d.default, inspect_q8ta_conv2d_route); + REGISTER_ROUTE_INSPECTOR( + et_vk.q8ta_conv2d_general.default, inspect_q8ta_conv2d_route); + REGISTER_ROUTE_INSPECTOR( + et_vk.q8ta_conv2d_im2col.default, inspect_q8ta_conv2d_route); + REGISTER_ROUTE_INSPECTOR( + et_vk.q8ta_conv2d_dw.default, inspect_q8ta_conv2d_route); + REGISTER_ROUTE_INSPECTOR( + et_vk.q8ta_conv2d_pw.default, inspect_q8ta_conv2d_route); + REGISTER_ROUTE_INSPECTOR( + et_vk.q8ta_conv2d_transposed.default, inspect_q8ta_conv2d_route); +} +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taConv2dRoutes.h b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taConv2dRoutes.h new file mode 100644 index 00000000000..ca9f8163d5a --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taConv2dRoutes.h @@ -0,0 +1,26 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +namespace vkcompute::shader_profiling { +Json build_q8ta_conv2d_direct( + ComputeGraph& graph, + const std::vector& args, + const Json& options); +Json build_q8ta_conv2d_im2col( + ComputeGraph& graph, + const std::vector& args, + const Json& options); +Json inspect_q8ta_conv2d_route( + ComputeGraph& graph, + const std::vector& args, + size_t first, + size_t end); +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taConv2dThroughput.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taConv2dThroughput.cpp new file mode 100644 index 00000000000..a18f440541f --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taConv2dThroughput.cpp @@ -0,0 +1,122 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +namespace vkcompute::shader_profiling { +namespace { +double q8ta_conv2d_flops(const Json& spec) { + const auto input = throughput::tensor_argument(spec, 0) + .at("sizes") + .get>(); + const auto weight = throughput::tensor_argument(spec, 3) + .at("sizes") + .get>(); + const Json& output_tensor = throughput::output_tensor(spec); + const auto output = output_tensor.at("sizes").get>(); + const auto kernel = + spec.at("args").at(9).at("int_list").get>(); + const bool transposed = + spec.value("operator_name", spec.value("operator", "")) == + "et_vk.q8ta_conv2d_transposed.default"; + const int64_t groups = spec.at("args").at(transposed ? 14 : 13).at("int"); + if (input.size() != 4 || output.size() != 4 || kernel.size() != 2 || + groups <= 0 || input[1] % groups || output[1] % groups || + input[0] != output[0] || kernel[0] <= 0 || kernel[1] <= 0) + throw std::invalid_argument( + "Q8 convolution work count dimensions disagree"); + const int64_t logical_columns = input[1] / groups * kernel[0] * kernel[1]; + const int64_t padded_columns = (logical_columns + 3) / 4 * 4; + if (weight.size() == 3) { + if (weight != std::vector{kernel[0], kernel[1], output[1]} || + groups != input[1] || input[1] != output[1]) + throw std::invalid_argument( + "Q8 depthwise work count requires HWC weights"); + } else if ( + weight.size() != 2 || weight[0] != output[1] || + weight[1] != padded_columns) { + throw std::invalid_argument("Q8 work count requires aligned OHWI rows"); + } + const auto stride = + spec.at("args").at(10).at("int_list").get>(); + const auto pad = + spec.at("args").at(11).at("int_list").get>(); + const auto dilation = spec.at("args") + .at(transposed ? 13 : 12) + .at("int_list") + .get>(); + const auto output_pad = transposed + ? spec.at("args").at(12).at("int_list").get>() + : std::vector{0, 0}; + if (stride.size() != 2 || pad.size() != 2 || dilation.size() != 2 || + output_pad.size() != 2) + throw std::invalid_argument("Q8 work count requires spatial pairs"); + for (size_t d = 0; d < 2; ++d) { + if (stride[d] <= 0 || pad[d] < 0 || dilation[d] <= 0) + throw std::invalid_argument( + "Q8 work count spatial parameters are invalid"); + const int64_t expected = transposed + ? (input[d + 2] - 1) * stride[d] - 2 * pad[d] + + dilation[d] * (kernel[d] - 1) + output_pad[d] + 1 + : (input[d + 2] + 2 * pad[d] - dilation[d] * (kernel[d] - 1) - 1) / + stride[d] + + 1; + if (expected != output[d + 2]) + throw std::invalid_argument("Q8 work count output shape disagrees"); + } + const double elements = throughput::tensor_elements(output_tensor); + throughput::tensor_elements(throughput::tensor_argument(spec, 0)); + throughput::tensor_elements(throughput::tensor_argument(spec, 3)); + const bool bias = !spec.at("args").at(8).is_null(); + if (bias && + throughput::tensor_elements(throughput::tensor_argument(spec, 8)) < + double(output[1])) + throw std::invalid_argument( + "Q8 work count bias must cover output channels"); + const double multiply_adds = transposed + ? 2.0 * double(input[0]) * double(input[1]) * double(input[2]) * + double(input[3]) * double(output[1] / groups) * double(kernel[0]) * + double(kernel[1]) + : elements * 2.0 * double(logical_columns); + return multiply_adds + (bias ? elements : 0); +} +} // namespace + +REGISTER_THROUGHPUT_FUNCTIONS { + REGISTER_OPERATOR_MEMORY_TRAFFIC( + et_vk.q8ta_conv2d.default, throughput::tensor_traffic); + REGISTER_OPERATOR_MEMORY_TRAFFIC( + et_vk.q8ta_conv2d_general.default, throughput::tensor_traffic); + REGISTER_OPERATOR_MEMORY_TRAFFIC( + et_vk.q8ta_conv2d_im2col.default, throughput::tensor_traffic); + REGISTER_OPERATOR_MEMORY_TRAFFIC( + et_vk.q8ta_conv2d_dw.default, throughput::tensor_traffic); + REGISTER_OPERATOR_MEMORY_TRAFFIC( + et_vk.q8ta_conv2d_pw.default, throughput::tensor_traffic); + REGISTER_OPERATOR_MEMORY_TRAFFIC( + et_vk.q8ta_conv2d_transposed.default, throughput::tensor_traffic); + REGISTER_OPERATOR_MEMORY_TRAFFIC( + et_vk.conv2d_q8ta_q8csw_q8to.default, throughput::tensor_traffic); + REGISTER_OPERATOR_COMPUTE_FLOPS(et_vk.q8ta_conv2d.default, q8ta_conv2d_flops); + REGISTER_OPERATOR_COMPUTE_FLOPS( + et_vk.q8ta_conv2d_general.default, q8ta_conv2d_flops); + REGISTER_OPERATOR_COMPUTE_FLOPS( + et_vk.q8ta_conv2d_im2col.default, q8ta_conv2d_flops); + REGISTER_OPERATOR_COMPUTE_FLOPS( + et_vk.q8ta_conv2d_dw.default, q8ta_conv2d_flops); + REGISTER_OPERATOR_COMPUTE_FLOPS( + et_vk.q8ta_conv2d_pw.default, q8ta_conv2d_flops); + REGISTER_OPERATOR_COMPUTE_FLOPS( + et_vk.q8ta_conv2d_transposed.default, q8ta_conv2d_flops); + REGISTER_OPERATOR_COMPUTE_FLOPS( + et_vk.conv2d_q8ta_q8csw_q8to.default, q8ta_conv2d_flops); + REGISTER_SHADER_MEMORY_TRAFFIC(q8ta_conv2d, throughput::tensor_traffic); + REGISTER_SHADER_MEMORY_TRAFFIC(q8ta_im2col, throughput::tensor_traffic); + REGISTER_SHADER_MEMORY_TRAFFIC(conv2d_dw_q8ta, throughput::tensor_traffic); +} +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taValidationChecks.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taValidationChecks.cpp new file mode 100644 index 00000000000..d7f667a9fe6 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taValidationChecks.cpp @@ -0,0 +1,158 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +#include +#include +#include +#include +#include +namespace vkcompute::shader_profiling { +Json check_q8ta_unsigned_dot_selection(const Json& spec, ComputeGraph& graph) { + const auto& parameters = spec.at("parameters"); + const int64_t expected_limit = parameters.at("accumulator_limit_bytes"); + reference::require( + expected_limit == kMaxUnsignedDotAccumulatorBytes, + "Unsigned dot accumulator boundary changed"); + const auto& adapter = *graph.context()->adapter_ptr(); + const bool prefers_unsigned = adapter.accelerates_unsigned_packed4x8_dot() && + !adapter.accelerates_signed_packed4x8_dot(); + Json decisions = Json::array(); + for (const Json& size : parameters.at("test_kernel_sizes")) { + const int64_t k = size.get(); + const bool selected = can_use_unsigned_pw_dot(adapter, k); + reference::require( + selected == (prefers_unsigned && k <= expected_limit), + "Unsigned dot selection violated capability or accumulator limit"); + decisions.push_back({{"kernel_bytes", k}, {"unsigned_selected", selected}}); + } + return { + {"status", "PASS"}, + {"prefers_unsigned", prefers_unsigned}, + {"decisions", decisions}}; +} +Json check_q8ta_streaming_dynamic_shrink( + const Json& spec, + ComputeGraph& graph) { + const Json& parameters = spec.at("parameters"); + const Json& plan_args = parameters.at("stream_plan"); + const auto plan = make_q8ta_conv2d_stream_plan( + plan_args.at("batch").get(), + plan_args.at("flattened_kernel_size").get(), + plan_args.at("out_height").get(), + plan_args.at("out_width").get(), + plan_args.at("scratch_budget_bytes").get()); + const Json& expected = parameters.at("expected_plan"); + reference::require( + plan.feasible && + plan.num_tiles == expected.at("num_tiles").get() && + plan.rows_per_tile >= expected.at("min_rows_per_tile").get(), + "Dynamic shrink case did not create dead stream tiles"); + const int64_t resized_bytes = + plan_args.at("flattened_kernel_size").get() * + plan.rows_per_tile * + utils::align_up_4(parameters.at("resized_width").get()); + reference::require( + resized_bytes <= plan_args.at("scratch_budget_bytes").get(), + "Resized im2col scratch exceeds stream budget"); + Json operator_spec = parse_configs(parameters.at("operator_case")).front(); + const std::string output_name = parameters.at("output_name"); + Json input_spec = operator_spec; + input_spec["tensors"].erase(output_name); + Json output_spec = operator_spec; + output_spec["tensors"] = { + {output_name, operator_spec.at("tensors").at(output_name)}}; + auto host_data = prepare_tensor_data(input_spec); + const auto input_tensors = create_tensors(input_spec, graph, host_data, true); + auto output_data = prepare_tensor_data(output_spec); + const auto output_tensors = + create_tensors(output_spec, graph, output_data, false); + host_data.merge(output_data); + auto tensors = input_tensors; + tensors.insert(output_tensors.begin(), output_tensors.end()); + for (const auto& input : graph.inputs()) + reference::require( + input.value != tensors.at(output_name), + "Dynamic validation output must not be staged as an input"); + reference::require( + graph.dtype_of(tensors.at( + parameters.at("input_name").get())) == vkapi::kInt8x4 && + graph.dtype_of(tensors.at(output_name)) == vkapi::kInt8x4, + "Dynamic Q8 convolution requires native int8x4 input/output"); + const auto args = build_operator_args(operator_spec, graph, tensors); + build_operator_route(operator_spec, graph, args); + Json accesses = Json::object(); + for (const auto& tensor : tensors) + accesses[tensor.first] = tensor.first == output_name ? "write" : "read"; + const auto staging = + add_validation_outputs(operator_spec, graph, tensors, accesses); + graph.prepare(); + graph.prepack(); + Json validations = Json::array(); + for (const Json& step : spec.at("validation_steps")) { + Json resized = operator_spec; + auto data = host_data; + const Json& sizes = step.at("tensor_sizes"); + reference::require( + !sizes.contains(output_name), + "Dynamic validation steps must not resize the output manually"); + for (const auto& item : sizes.items()) { + const auto shape = item.value().get>(); + const ValueRef ref = input_tensors.at(item.key()); + bool found = false; + for (size_t i = 0; i < graph.inputs().size(); ++i) + if (graph.inputs().at(i).value == ref) { + graph.resize_input(i, shape); + found = true; + break; + } + reference::require( + found, "Dynamic validation size target is not a staged input"); + resized["tensors"][item.key()]["sizes"] = shape; + const size_t bytes = tensor_numel(item.value()); + reference::require( + bytes <= data.at(item.key()).size(), + "Dynamic validation shape exceeds upper input allocation"); + data.at(item.key()).resize(bytes); + } + resized["tensors"][output_name]["sizes"] = step.at("expected_output_sizes"); + graph.propagate_resize(); + reference::require( + graph.sizes_of(tensors.at(output_name)) == + step.at("expected_output_sizes").get>(), + "Streaming im2col output did not resize"); + const size_t output_bytes = tensor_numel(step.at("expected_output_sizes")); + reference::require( + output_bytes <= data.at(output_name).size(), + "Dynamic validation output exceeds upper allocation"); + data.at(output_name).resize(output_bytes); + copy_validation_inputs(resized, graph, input_tensors, data); + graph.execute(); + if (step.value("validate_output", true)) { + const Json validation = + validate_outputs(resized, graph, tensors, staging, data); + reference::require( + validation.at("status") == "PASS", + "Streaming im2col resize validation failed: " + validation.dump()); + validations.push_back(validation); + } + } + return { + {"status", "PASS"}, + {"validation_steps", spec.at("validation_steps").size()}, + {"validated_outputs", validations.size()}, + {"stream_tiles", plan.num_tiles}, + {"resized_scratch_bytes", resized_bytes}}; +} +REGISTER_CASE_CHECKS { + REGISTER_CASE_CHECK( + q8ta_unsigned_dot_selection, check_q8ta_unsigned_dot_selection); + REGISTER_CASE_CHECK( + q8ta_streaming_dynamic_shrink, check_q8ta_streaming_dynamic_shrink); +} +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taValidationChecks.h b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taValidationChecks.h new file mode 100644 index 00000000000..845ef6fb368 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taValidationChecks.h @@ -0,0 +1,13 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once +#include +#include +namespace vkcompute::shader_profiling { +Json check_q8ta_unsigned_dot_selection(const Json& spec, ComputeGraph& graph); +Json check_q8ta_streaming_dynamic_shrink(const Json& spec, ComputeGraph& graph); +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/correctness.json new file mode 100644 index 00000000000..19f3961636d --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/correctness.json @@ -0,0 +1,3474 @@ +[ + { + "name": "q8ta/conv2d/correctness int8x4 buffer general", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "general", + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 28], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 27 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer auto", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 28], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 27 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness [3]", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "legacy_4w4c", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 28], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 27 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4c1w", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4w4c", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "legacy_4w4c", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4c", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4c1w (6)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4w4c (7)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "legacy_4w4c", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4c (8)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4c1w (9)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 77, 77], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [16], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 77, 77], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4w4c (10)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "legacy_4w4c", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 77, 77], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [16], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 77, 77], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4c (11)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 77, 77], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [16], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 77, 77], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col"], + "v_LAYOUT": ["4c1w", "4w4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [2, 32, 7, 7], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [16], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [2, 16, 7, 7], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4c1w (13)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 144], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 2 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4w4c (14)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "legacy_4w4c", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 144], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 2 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4c (15)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 144], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 2 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4c1w (16)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 28], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 27 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4c1w (17)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4w4c (18)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "legacy_4w4c", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 28], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 27 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4w4c (19)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "legacy_4w4c", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4c (20)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 28], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 27 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4c (21)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4c1w (22)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 81, 81], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [96], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 3 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 41, 41], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4w4c (23)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "legacy_4w4c", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 81, 81], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [96], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 3 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 41, 41], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4c (24)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 81, 81], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [96], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 3 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 41, 41], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4c1w (25)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 600], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [96], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [5, 5] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 4 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4w4c (26)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "legacy_4w4c", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 600], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [96], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [5, 5] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 4 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4c (27)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 600], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [96], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [5, 5] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 4 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/debug.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/debug.json new file mode 100644 index 00000000000..dbf8316980a --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/debug.json @@ -0,0 +1,368 @@ +[ + { + "name": "q8ta/conv2d/debug int8x4 buffer auto", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 5, 5], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 72], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [16], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 5, 5], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/debug int8x4 buffer im2col", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "im2col", + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 5, 5], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 72], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [16], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 5, 5], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/debug [4]", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "legacy_4w4c", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 5, 5], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 72], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [16], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 5, 5], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/dw_correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/dw_correctness.json new file mode 100644 index 00000000000..65e007ae390 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/dw_correctness.json @@ -0,0 +1,1826 @@ +[ + { + "name": "q8ta/conv2d/dw_correctness", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 32], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 32 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_correctness (2)", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 80, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 80], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [80], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [80], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [80], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 80 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 80, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_correctness (3)", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 57, 33], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 16], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [16], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 16 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 57, 33], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_correctness (4)", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [2, 8, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 8], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [8], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [8], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [8], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 8 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [2, 8, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_correctness (5)", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [5, 5, 32], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [5, 5] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 32 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_correctness (6)", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 64 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 17, 17], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_correctness (7)", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [5, 5, 96], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [96], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [5, 5] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 96 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_correctness (8)", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 64 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_correctness [2]", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "legacy_4w4c", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 32], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 32 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_correctness [14]", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "legacy_4w4c", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 80, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 80], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [80], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [80], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [80], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 80 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 80, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_correctness [18]", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "legacy_4w4c", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 57, 33], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 16], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [16], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 16 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 57, 33], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_correctness [22]", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "legacy_4w4c", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [5, 5, 32], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [5, 5] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 32 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_correctness [6]", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "legacy_4w4c", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 64 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 17, 17], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_correctness [26]", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "legacy_4w4c", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [5, 5, 96], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [96], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [5, 5] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 96 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_correctness [10]", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "legacy_4w4c", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 64 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/dw_debug.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/dw_debug.json new file mode 100644 index 00000000000..0dad77a1b68 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/dw_debug.json @@ -0,0 +1,245 @@ +[ + { + "name": "q8ta/conv2d/dw_debug", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 8], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [8], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [8], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [8], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 8 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_debug [2]", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "legacy_4w4c", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 8], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [8], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [8], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [8], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 8 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/dw_narrow_workgroup.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/dw_narrow_workgroup.json new file mode 100644 index 00000000000..d2d994c03da --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/dw_narrow_workgroup.json @@ -0,0 +1,248 @@ +[ + { + "name": "q8ta/conv2d/dw_narrow_workgroup [0]", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 7, 7], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 128 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 7, 7], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_narrow_workgroup", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "sweep_variables": { + "v_SIZES": [ + [1, 64, 9, 9], + [1, 64, 13, 13] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 64 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/dw_performance.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/dw_performance.json new file mode 100644 index 00000000000..b0c7e7917f3 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/dw_performance.json @@ -0,0 +1,1214 @@ +[ + { + "name": "q8ta/conv2d/dw_performance", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 128 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_performance (2)", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 64 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_performance (3)", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 288, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [288], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [288], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [288], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 288 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 288, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_performance (4)", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 32], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 32 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 130, 130], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_performance [16]", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [5, 5, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [5, 5] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 128 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_performance [17]", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [60, 128, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [5, 5, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [5, 5] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 128 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [60, 128, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_performance [2]", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "legacy_4w4c", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 128 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_performance [6]", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "legacy_4w4c", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 64 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_performance [10]", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "legacy_4w4c", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 288, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [288], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [288], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [288], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 288 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 288, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_performance [14]", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "legacy_4w4c", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 32], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 32 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 130, 130], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/im2col.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/im2col.json new file mode 100644 index 00000000000..888dbfd8844 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/im2col.json @@ -0,0 +1,961 @@ +[ + { + "name": "q8ta/conv2d/im2col", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.25, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["im2col", "im2col_unsigned", "im2col_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4, 5, 5], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "repeat": [-128, -1, 0, 1, 127] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [5, 36], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [-1, 127, 0, -128, 1] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [8], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [8], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0.00390625] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [8], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 5, 3, 3], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/im2col (2)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.25, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["im2col_unsigned", "im2col_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4, 1, 1], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [262145, 4], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [262148], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [262148], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0.00390625] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [262148], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 262145, 1, 1], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/im2col [19]", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.25, + "rtol": 0 + }, + "route": "im2col_auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4, 91, 91], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 33124], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0.00390625] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [91, 91] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 1, 1], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/im2col (4)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.25, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["im2col", "im2col_unsigned", "im2col_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4, 90, 91], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "repeat": [127] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 0 + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 32760], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [9.999999974752427e-7] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [90, 91] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 1, 1], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/im2col (5)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.25, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["im2col", "im2col_unsigned", "im2col_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 6, 7], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "repeat": [-128, -1, 0, 1, 127] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 11 + }, + { + "tensor": { + "name": "weight", + "sizes": [8, 36], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [-1, 127, 0, -128, 1] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [8], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [8], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0.00390625] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -3 + }, + { + "tensor": { + "name": "bias", + "sizes": [8], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 2 + }, + { + "string": "relu" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 6, 7], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/im2col (6)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.25, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["im2col", "im2col_unsigned", "im2col_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 7, 7], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "repeat": [-128, -1, 0, 1, 127] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 127 + }, + { + "tensor": { + "name": "weight", + "sizes": [12, 72], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [-1, 127, 0, -128, 1] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [12], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [12], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0.00390625] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -5 + }, + { + "tensor": { + "name": "bias", + "sizes": [12], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "relu" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 12, 4, 4], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/im2col (7)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.25, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["im2col", "im2col_unsigned", "im2col_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 5, 5], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "repeat": [-128, -1, 0, 1, 127] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": -7 + }, + { + "tensor": { + "name": "weight", + "sizes": [8, 72], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [-1, 127, 0, -128, 1] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [8], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [8], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0.00390625] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": 3 + }, + null, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 5, 5], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/im2col (8)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.25, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["im2col", "im2col_unsigned", "im2col_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 9, 9], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "repeat": [-128, -1, 0, 1, 127] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": -128 + }, + { + "tensor": { + "name": "weight", + "sizes": [12, 72], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [-1, 127, 0, -128, 1] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [12], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [12], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0.00390625] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": 5 + }, + null, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [2, 2] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 12, 9, 9], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/narrow_workgroup.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/narrow_workgroup.json new file mode 100644 index 00000000000..2b75cb265ee --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/narrow_workgroup.json @@ -0,0 +1,242 @@ +[ + { + "name": "q8ta/conv2d/narrow_workgroup [0]", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "general", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 9, 9], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 9, 9], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/narrow_workgroup [1]", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "general", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 7, 7], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 7, 7], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/performance.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/performance.json new file mode 100644 index 00000000000..4ac51f6ee99 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/performance.json @@ -0,0 +1,4074 @@ +[ + { + "name": "q8ta/conv2d/performance int8x4 buffer general", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "general", + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 28], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 27 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 254, 254], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer auto", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 28], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 27 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 254, 254], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance [3]", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "legacy_4w4c", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 28], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 27 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 254, 254], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4c1w", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4c1w (4)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 576], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4c1w (5)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 512, 7, 7], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [512, 4608], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [512], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [512], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [512], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 512, 7, 7], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4w4c", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "legacy_4w4c", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4w4c (7)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "legacy_4w4c", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 576], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4w4c (8)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "legacy_4w4c", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 512, 7, 7], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [512, 4608], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [512], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [512], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [512], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 512, 7, 7], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4c", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4c (10)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 576], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4c (11)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 512, 7, 7], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [512, 4608], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [512], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [512], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [512], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 512, 7, 7], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4c1w (12)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 2 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4c1w (13)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 2 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4w4c (14)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "legacy_4w4c", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 2 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4w4c (15)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "legacy_4w4c", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 2 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4c (16)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 2 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4c (17)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 2 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4c1w (18)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [96], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 3 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4w4c (19)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "legacy_4w4c", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [96], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 3 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4c (20)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [96], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 3 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance [97]", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 28], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 27 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance [98]", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [60, 3, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 28], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 27 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [60, 32, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance [99]", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [60, 256, 10, 13], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [512, 2304], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [512], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [512], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [512], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [60, 512, 5, 7], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4c1w (24)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 800], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [5, 5] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 4 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4c1w (25)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 800], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [5, 5] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 4 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4w4c (26)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "legacy_4w4c", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 800], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [5, 5] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 4 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4w4c (27)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "legacy_4w4c", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 800], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [5, 5] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 4 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4c (28)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 800], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [5, 5] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 4 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4c (29)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 800], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [5, 5] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 4 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4c1w (30)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 1024, 14, 14], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 1024], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2048], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 2048, 7, 7], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4w4c (31)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "legacy_4w4c", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 1024, 14, 14], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 1024], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2048], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 2048, 7, 7], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4c (32)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 1024, 14, 14], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 1024], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2048], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 2048, 7, 7], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/pw_correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/pw_correctness.json new file mode 100644 index 00000000000..19b3062de7b --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/pw_correctness.json @@ -0,0 +1,3355 @@ +[ + { + "name": "q8ta/conv2d/pw_correctness", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 16], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 16 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (2)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [2, 16], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 16 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 2, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (3)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 16], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 16 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 3, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (4)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto", "legacy_4w4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 16], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 16 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (5)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto", "legacy_4w4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [2, 16], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 16 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 2, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (6)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto", "legacy_4w4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 16], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 16 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 3, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (7)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 16], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 16 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (8)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [2, 16], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 16 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 2, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (9)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 16], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 16 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 3, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (10)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 4], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 3 + } + }, + "valid_columns": 3 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (11)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto", "legacy_4w4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 4], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 3 + } + }, + "valid_columns": 3 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (12)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 4], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 3 + } + }, + "valid_columns": 3 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (13)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 32], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 32 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (14)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto", "legacy_4w4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 32], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 32 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (15)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 32], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 32 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (16)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 64 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [96], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (17)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto", "legacy_4w4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 64 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [96], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (18)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 64 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [96], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (19)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 7, 57, 33], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [13, 8], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 7 + } + }, + "valid_columns": 7 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [16], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 13, 57, 33], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (20)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto", "legacy_4w4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 7, 57, 33], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [13, 8], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 7 + } + }, + "valid_columns": 7 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [16], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 13, 57, 33], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (21)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 7, 57, 33], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [13, 8], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 7 + } + }, + "valid_columns": 7 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [16], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 13, 57, 33], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (22)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 40, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [80, 40], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 40 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [80], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [80], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [80], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 80, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (23)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto", "legacy_4w4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 40, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [80, 40], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 40 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [80], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [80], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [80], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 80, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (24)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 40, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [80, 40], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 40 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [80], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [80], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [80], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 80, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (25)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"], + "v_OUTPUT_LAYOUT": ["4c1w", "4w4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [2, 8, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [8, 8], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 8 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [8], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [8], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [8], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [2, 8, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_OUTPUT_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (26)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 7, 7, 5], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-128, -5] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": -128 + }, + { + "tensor": { + "name": "weight", + "sizes": [13, 8], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 7 + } + }, + "valid_columns": 7 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [16], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 13, 7, 5], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/pw_performance.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/pw_performance.json new file mode 100644 index 00000000000..62e7ec82731 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/pw_performance.json @@ -0,0 +1,2178 @@ +[ + { + "name": "q8ta/conv2d/pw_performance", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 480, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [160, 480], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 480 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [160], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [160], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [160], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 160, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_performance (2)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto", "legacy_4w4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 480, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [160, 480], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 480 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [160], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [160], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [160], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 160, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_performance (3)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 480, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [160, 480], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 480 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [160], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [160], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [160], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 160, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_performance (4)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 48, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [22, 48], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 48 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [24], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [24], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [24], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 22, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_performance (5)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto", "legacy_4w4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 48, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [22, 48], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 48 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [24], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [24], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [24], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 22, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_performance (6)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 48, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [22, 48], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 48 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [24], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [24], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [24], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 22, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_performance (7)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 48, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [48, 48], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 48 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [48], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [48], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [48], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 48, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_performance (8)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto", "legacy_4w4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 48, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [48, 48], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 48 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [48], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [48], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [48], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 48, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_performance (9)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 48, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [48, 48], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 48 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [48], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [48], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [48], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 48, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_performance (10)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 128 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_performance (11)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto", "legacy_4w4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 128 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_performance (12)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 128 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_performance (13)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 576, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 576], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 576 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_performance (14)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto", "legacy_4w4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 576, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 576], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 576 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_performance (15)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 576, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 576], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 576 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_performance (16)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 32], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 32 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_performance (17)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [60, 32, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 32], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 32 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [60, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/scenex_grouped.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/scenex_grouped.json new file mode 100644 index 00000000000..2b8bb602687 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/scenex_grouped.json @@ -0,0 +1,371 @@ +[ + { + "name": "q8ta/conv2d/scenex_grouped", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 10, + "warmup_runs": 3, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "general", "im2col"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [60, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-8, 12] + } + } + }, + { + "float": 1.0 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 2 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [60, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/scenex_grouped (2)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 10, + "warmup_runs": 3, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "general", "im2col"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [60, 64, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-8, 12] + } + } + }, + { + "float": 1.0 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 2 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [60, 64, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/scenex_grouped (3)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 10, + "warmup_runs": 3, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "general", "im2col"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [60, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-8, 12] + } + } + }, + { + "float": 1.0 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 800], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [5, 5] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 4 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [60, 128, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/scenex_regular.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/scenex_regular.json new file mode 100644 index 00000000000..a05e32bac5b --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/scenex_regular.json @@ -0,0 +1,248 @@ +[ + { + "name": "q8ta/conv2d/scenex_regular", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 10, + "warmup_runs": 3, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "general", "im2col"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [60, 64, 40, 51], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-8, 12] + } + } + }, + { + "float": 1.0 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 576], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [60, 128, 20, 26], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/scenex_regular (2)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 10, + "warmup_runs": 3, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "general", "im2col"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [60, 128, 20, 26], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-8, 12] + } + } + }, + { + "float": 1.0 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 1152], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [256], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [60, 256, 10, 13], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/streaming_dynamic_shrink.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/streaming_dynamic_shrink.json new file mode 100644 index 00000000000..4b8103fa770 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/streaming_dynamic_shrink.json @@ -0,0 +1,200 @@ +{ + "name": "q8ta/conv2d/streaming_dynamic_shrink", + "check": "q8ta_streaming_dynamic_shrink", + "parameters": { + "operator_case": { + "name": "q8ta streaming dynamic shrink operator", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "im2col_auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [10, 64, 30, 99], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-8, 12] + } + } + }, + { + "float": 1.0 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [4, 576], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [ + 0.004999999888241291, 0.014999999664723873 + ] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [10, 4, 30, 99], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + "input_name": "input", + "output_name": "output", + "stream_plan": { + "batch": 10, + "flattened_kernel_size": 576, + "out_height": 30, + "out_width": 99, + "scratch_budget_bytes": 16777216 + }, + "expected_plan": { + "num_tiles": 2, + "min_rows_per_tile": 21 + }, + "resized_width": 51 + }, + "validation_steps": [ + { + "tensor_sizes": { + "input": [10, 64, 30, 99] + }, + "expected_output_sizes": [10, 4, 30, 99], + "validate_output": false + }, + { + "tensor_sizes": { + "input": [1, 64, 20, 51] + }, + "expected_output_sizes": [1, 4, 20, 51], + "validate_output": true + }, + { + "tensor_sizes": { + "input": [10, 64, 30, 99] + }, + "expected_output_sizes": [10, 4, 30, 99], + "validate_output": false + }, + { + "tensor_sizes": { + "input": [1, 64, 20, 51] + }, + "expected_output_sizes": [1, 4, 20, 51], + "validate_output": true + }, + { + "tensor_sizes": { + "input": [10, 64, 30, 99] + }, + "expected_output_sizes": [10, 4, 30, 99], + "validate_output": false + }, + { + "tensor_sizes": { + "input": [1, 64, 20, 51] + }, + "expected_output_sizes": [1, 4, 20, 51], + "validate_output": true + }, + { + "tensor_sizes": { + "input": [10, 64, 30, 99] + }, + "expected_output_sizes": [10, 4, 30, 99], + "validate_output": false + }, + { + "tensor_sizes": { + "input": [1, 64, 20, 51] + }, + "expected_output_sizes": [1, 4, 20, 51], + "validate_output": true + } + ] +} diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/streaming_im2col.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/streaming_im2col.json new file mode 100644 index 00000000000..1097d2d7015 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/streaming_im2col.json @@ -0,0 +1,725 @@ +[ + { + "name": "q8ta/conv2d/streaming_im2col [0]", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "im2col_fallback", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [10, 32, 30, 99], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-8, 12] + } + } + }, + { + "float": 1.0 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [4, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [10, 4, 30, 99], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/streaming_im2col [1]", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "im2col_fallback", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [10, 64, 30, 99], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-8, 12] + } + } + }, + { + "float": 1.0 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [4, 576], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [10, 4, 30, 99], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/streaming_im2col", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "im2col_auto", + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [10, 32, 30, 99], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-8, 12] + } + } + }, + { + "float": 1.0 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [4, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [10, 4, 30, 99], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/streaming_im2col [4]", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "im2col_auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [10, 64, 30, 99], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-8, 12] + } + } + }, + { + "float": 1.0 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [4, 576], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [10, 4, 30, 99], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/streaming_im2col [6]", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "im2col_auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [20, 32, 30, 99], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-8, 12] + } + } + }, + { + "float": 1.0 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [10, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [12], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [12], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [12], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [20, 10, 30, 99], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/streaming_im2col [5]", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "im2col_auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [20, 32, 30, 99], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-8, 12] + } + } + }, + { + "float": 1.0 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 144], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [16], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 2 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [20, 16, 30, 99], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/unsigned_dot_selection.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/unsigned_dot_selection.json new file mode 100644 index 00000000000..9ea92c4d2a1 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/unsigned_dot_selection.json @@ -0,0 +1,8 @@ +{ + "name": "q8ta/conv2d/unsigned_dot_selection", + "check": "q8ta_unsigned_dot_selection", + "parameters": { + "accumulator_limit_bytes": 33025, + "test_kernel_sizes": [33025, 33026] + } +} diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/test/q8ta_native_reference_test.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/test/q8ta_native_reference_test.cpp new file mode 100644 index 00000000000..cc60107bba8 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/test/q8ta_native_reference_test.cpp @@ -0,0 +1,159 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +#include + +#include + +namespace vkcompute::shader_profiling { +namespace { +using reference::Tensor; +using reference::TensorMap; + +Json native_spec() { + return { + {"operator", "et_vk.q8ta_conv2d.default"}, + {"args", + {{{"tensor", "x"}}, + {{"float", 1.0}}, + {{"int", 0}}, + {{"tensor", "w"}}, + {{"tensor", "sums"}}, + {{"tensor", "scales"}}, + {{"float", 1.0}}, + {{"int", 0}}, + {{"tensor", "bias"}}, + {{"int_list", {1, 1}}}, + {{"int_list", {1, 1}}}, + {{"int_list", {0, 0}}}, + {{"int_list", {1, 1}}}, + {{"int", 1}}, + {{"string", "none"}}, + {{"tensor", "y"}}}}, + {"tensors", + {{"y", + {{"sizes", {1, 1, 1, 1}}, + {"dtype", "int8x4"}, + {"storage", "buffer"}, + {"layout", "4c1w"}}}}}}; +} + +TEST(Q8taNativeReference, RegistersNativeOperatorsOnly) { + for (const char* name : + {"et_vk.q8ta_conv2d.default", + "et_vk.q8ta_conv2d_general.default", + "et_vk.q8ta_conv2d_im2col.default", + "et_vk.q8ta_conv2d_dw.default", + "et_vk.q8ta_conv2d_pw.default"}) + EXPECT_EQ(reference_registry().operator_reference(name), "q8ta_conv2d"); + EXPECT_EQ( + reference_registry().operator_reference( + "et_vk.q8ta_conv2d_transposed.default"), + "q8ta_transposed"); + EXPECT_THROW( + reference_registry().operator_reference( + "benchmark_v2.q8ta_conv2d.default"), + std::invalid_argument); +} + +TEST( + Q8taNativeReference, + QuantizedInputOutputHonorZeroPointsBiasAndPaddedRows) { + Json spec = native_spec(); + spec["args"][1]["float"] = .5; + spec["args"][2]["int"] = 2; + spec["args"][6]["float"] = .5; + spec["args"][7]["int"] = -3; + const TensorMap data{ + {"x", {{1, 1, 1, 1}, {5}}}, + {"w", {{1, 4}, {2, 0, 0, 0}}}, + {"sums", {{1}, {2}}}, + {"scales", {{1}, {.25}}}, + {"bias", {{1}, {.25}}}}; + EXPECT_EQ( + reference::q8ta_conv2d_reference(spec, data, 1).values, + (std::vector{-1})); +} + +TEST(Q8taNativeReference, DepthwiseHwcWeightsPreserveChannelAndKernelOrder) { + Json spec = native_spec(); + spec["operator"] = "et_vk.q8ta_conv2d_dw.default"; + spec["args"][8] = nullptr; + spec["args"][9]["int_list"] = {1, 2}; + spec["args"][13]["int"] = 2; + spec["tensors"]["y"]["sizes"] = {1, 2, 1, 2}; + const TensorMap data{ + {"x", {{1, 2, 1, 3}, {1, 2, 3, 10, 20, 30}}}, + {"w", {{1, 2, 2}, {1, 4, 2, -1}}}, + {"sums", {{2}, {3, 3}}}, + {"scales", {{2}, {1, 1}}}}; + const auto one = reference::q8ta_conv2d_reference(spec, data, 1); + EXPECT_EQ(one.values, (std::vector{5, 8, 20, 50})); + EXPECT_EQ(reference::q8ta_conv2d_reference(spec, data, 4).values, one.values); +} + +TEST(Q8taNativeReference, TransposedOutputPaddingUsesNativeArgumentPositions) { + Json spec = native_spec(); + spec["operator"] = "et_vk.q8ta_conv2d_transposed.default"; + spec["args"][8] = nullptr; + spec["args"][9]["int_list"] = {1, 2}; + spec["args"][10]["int_list"] = {1, 2}; + spec["args"].insert(spec["args"].begin() + 12, Json{{"int_list", {0, 1}}}); + spec["tensors"]["y"]["sizes"] = {1, 1, 1, 5}; + const TensorMap data{ + {"x", {{1, 1, 1, 2}, {1, 2}}}, + {"w", {{1, 4}, {1, 2, 0, 0}}}, + {"sums", {{1}, {3}}}, + {"scales", {{1}, {1}}}}; + EXPECT_EQ( + reference::q8ta_transposed_reference(spec, data, 2).values, + (std::vector{1, 2, 2, 4, 0})); +} + +TEST(Q8taNativeReference, RequantizesNearestEvenAndSaturatesWithRelu) { + Json spec = native_spec(); + spec["args"][1]["float"] = .5; + spec["args"][8] = nullptr; + spec["tensors"]["y"]["sizes"] = {1, 1, 1, 6}; + const TensorMap data{ + {"x", {{1, 1, 1, 6}, {1, 3, -1, -3, 127, -128}}}, + {"w", {{1, 4}, {1, 0, 0, 0}}}, + {"sums", {{1}, {1}}}, + {"scales", {{1}, {1}}}}; + EXPECT_EQ( + reference::q8ta_conv2d_reference(spec, data, 1).values, + (std::vector{0, 2, 0, -2, 64, -64})); + spec["args"][1]["float"] = 4; + spec["args"][14]["string"] = "relu"; + EXPECT_EQ( + reference::q8ta_conv2d_reference(spec, data, 1).values, + (std::vector{4, 12, 0, 0, 127, 0})); +} + +TEST(Q8taNativeReference, ActualShaderSelectsIntegerCorrectionAndFusedBias) { + Json spec = native_spec(); + Tensor input{{1, 1042, 1, 1}, std::vector(1042, 127)}; + Tensor weight{{1, 1044}, std::vector(1044, 127)}; + input.values[1041] = 1; + weight.values[1040] = 24; + weight.values[1041] = 7; + weight.values[1042] = weight.values[1043] = 0; + const TensorMap data{ + {"x", input}, + {"w", weight}, + {"sums", {{1}, {132111}}}, + {"scales", {{1}, {std::nextafter(1.0f, 2.0f)}}}, + {"bias", {{1}, {-16777216}}}}; + spec["reference_context"]["shaders"] = {"q8ta_conv2d_float"}; + EXPECT_EQ(reference::q8ta_conv2d_reference(spec, data, 1).values[0], 0); + spec["reference_context"]["shaders"] = {"q8ta_conv2d_pw_float"}; + EXPECT_EQ(reference::q8ta_conv2d_reference(spec, data, 1).values[0], 1); +} +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/transposed/Q8taTransposedReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/transposed/Q8taTransposedReference.cpp new file mode 100644 index 00000000000..22dd82fc7e7 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/transposed/Q8taTransposedReference.cpp @@ -0,0 +1,115 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +#include + +namespace vkcompute::shader_profiling::reference { +Tensor q8ta_transposed_reference( + const Json& spec, + const TensorMap& tensors, + size_t threads) { + const Tensor& input = tensor_arg(spec, 0, tensors); + const Tensor& weight = tensor_arg(spec, 3, tensors); + const Tensor& sums = tensor_arg(spec, 4, tensors); + const Tensor& scales = tensor_arg(spec, 5, tensors); + const Tensor* bias = + spec.at("args").at(8).is_null() ? nullptr : &tensor_arg(spec, 8, tensors); + Tensor out = q8ta::output_tensor(spec, 16); + const auto kernel = conv2d_pair_arg(spec, 9); + const auto stride = conv2d_pair_arg(spec, 10); + const auto pad = conv2d_pair_arg(spec, 11); + const auto output_padding = conv2d_pair_arg(spec, 12); + const auto dilation = conv2d_pair_arg(spec, 13); + const int64_t groups = q8ta::integer(spec, 14); + require( + input.sizes.size() == 4 && out.sizes.size() == 4 && groups > 0, + "Q8 transposed convolution requires NCHW tensors and positive groups"); + const int64_t cin = input.sizes[1], cout = out.sizes[1]; + const int64_t height = out.sizes[2], width = out.sizes[3]; + require( + cin % groups == 0 && cout % groups == 0, + "Q8 transposed convolution channels must be divisible by groups"); + const int64_t columns = cin / groups * kernel[0] * kernel[1]; + require( + weight.sizes.size() == 2 && weight.sizes[0] == cout && + weight.sizes[1] >= columns && sums.values.size() >= size_t(cout) && + scales.values.size() >= size_t(cout) && + (!bias || bias->values.size() >= size_t(cout)), + "Q8 transposed convolution requires OHWI weights and channel parameters"); + for (size_t d = 0; d < 2; ++d) { + require( + kernel[d] > 0 && stride[d] > 0 && dilation[d] > 0 && pad[d] >= 0 && + output_padding[d] >= 0 && + (output_padding[d] < stride[d] || output_padding[d] < dilation[d]), + "Q8 transposed convolution spatial parameters are invalid"); + require( + out.sizes[d + 2] == + (input.sizes[d + 2] - 1) * stride[d] - 2 * pad[d] + + dilation[d] * (kernel[d] - 1) + output_padding[d] + 1, + "Q8 transposed convolution output shape disagrees with parameters"); + } + require(out.sizes[0] == input.sizes[0], "Q8 transposed batch sizes disagree"); + const float is = q8ta::scale(spec, 1), os = q8ta::scale(spec, 6); + const int64_t iz = q8ta::integer(spec, 2), oz = q8ta::integer(spec, 7); + require( + std::isfinite(is) && is > 0 && std::isfinite(os) && os > 0 && + iz >= -128 && iz <= 127 && oz >= -128 && oz <= 127, + "Q8 transposed convolution quantization parameters are invalid"); + const std::string activation = spec.at("args").at(15).at("string"); + require( + activation == "none" || activation == "relu", + "Q8 transposed convolution reference supports none/relu"); + std::vector logical_sums(size_t(cout), 0); + for (int64_t oc = 0; oc < cout; ++oc) + for (int64_t k = 0; k < columns; ++k) + logical_sums[size_t(oc)] += + int64_t(weight.values[size_t(oc * weight.sizes[1] + k)]); + parallel_for(out.values.size(), threads, [&](size_t index) { + const int64_t x = int64_t(index) % width; + const int64_t y = int64_t(index) / width % height; + const int64_t oc = int64_t(index) / (width * height) % cout; + const int64_t n = int64_t(index) / (width * height * cout); + const int64_t group = oc / (cout / groups); + int64_t sum = 0; + for (int64_t c = 0; c < cin / groups; ++c) + for (int64_t ky = 0; ky < kernel[0]; ++ky) + for (int64_t kx = 0; kx < kernel[1]; ++kx) { + const int64_t iy = y + pad[0] - ky * dilation[0]; + const int64_t ix = x + pad[1] - kx * dilation[1]; + if (iy < 0 || ix < 0 || iy % stride[0] || ix % stride[1] || + iy / stride[0] >= input.sizes[2] || + ix / stride[1] >= input.sizes[3]) + continue; + const int64_t source = + ((n * cin + group * (cin / groups) + c) * input.sizes[2] + + iy / stride[0]) * + input.sizes[3] + + ix / stride[1]; + const int64_t wi = + oc * weight.sizes[1] + (ky * kernel[1] + kx) * (cin / groups) + c; + sum += (int64_t(input.values[size_t(source)]) - iz) * + int64_t(weight.values[size_t(wi)]); + } + const int64_t raw = sum + iz * logical_sums[size_t(oc)]; + const float corrected = + float(raw) - float(sums.values[size_t(oc)]) * float(iz); + float value = corrected * (float(scales.values[size_t(oc)]) * is); + value += bias ? float(bias->values[size_t(oc)]) : 0; + if (activation == "relu") + value = std::max(value, 0.0f); + out.values[index] = double(q8ta::requantize(value, 1.0f / os, oz)); + }); + return out; +} +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_FN(q8ta_transposed, q8ta_transposed_reference); + REGISTER_OPERATOR_REFERENCE( + et_vk.q8ta_conv2d_transposed.default, q8ta_transposed); +} +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/transposed/cases/correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/transposed/cases/correctness.json new file mode 100644 index 00000000000..d683c61ba61 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/transposed/cases/correctness.json @@ -0,0 +1,627 @@ +[ + { + "name": "q8ta/conv2d_transposed/correctness", + "operator": "et_vk.q8ta_conv2d_transposed.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [16], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d_transposed/correctness (2)", + "operator": "et_vk.q8ta_conv2d_transposed.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 4, 4], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 576], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d_transposed/correctness (3)", + "operator": "et_vk.q8ta_conv2d_transposed.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 2 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d_transposed/correctness (4)", + "operator": "et_vk.q8ta_conv2d_transposed.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 512], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [16], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [4, 4] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d_transposed/correctness (5)", + "operator": "et_vk.q8ta_conv2d_transposed.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 144], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [16], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/transposed/cases/debug.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/transposed/cases/debug.json new file mode 100644 index 00000000000..011df2c097c --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/transposed/cases/debug.json @@ -0,0 +1,127 @@ +[ + { + "name": "q8ta/conv2d_transposed/debug", + "operator": "et_vk.q8ta_conv2d_transposed.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [16], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/transposed/cases/performance.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/transposed/cases/performance.json new file mode 100644 index 00000000000..42421212493 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/transposed/cases/performance.json @@ -0,0 +1,377 @@ +[ + { + "name": "q8ta/conv2d_transposed/performance", + "operator": "et_vk.q8ta_conv2d_transposed.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 2048], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [4, 4] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d_transposed/performance (2)", + "operator": "et_vk.q8ta_conv2d_transposed.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 4096], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [4, 4] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d_transposed/performance (3)", + "operator": "et_vk.q8ta_conv2d_transposed.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 1152], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/qw/q8csw/Q8cswConv2dReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/qw/q8csw/Q8cswConv2dReference.cpp new file mode 100644 index 00000000000..25321801aba --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/qw/q8csw/Q8cswConv2dReference.cpp @@ -0,0 +1,78 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +#include +namespace vkcompute::shader_profiling::reference { +static Tensor q8csw_conv2d_reference( + const Json& spec, + const TensorMap& tensors, + size_t threads) { + const bool quantized = + spec.at("operator") == "et_vk.conv2d_q8ta_q8csw.default"; + const size_t wi = quantized ? 3 : 1, si = quantized ? 5 : 2, + bi = quantized ? 6 : 3, ki = quantized ? 7 : 4; + Tensor input = tensor_arg(spec, 0, tensors); + const Tensor& weight = tensor_arg(spec, wi, tensors); + const Tensor& scales = tensor_arg(spec, si, tensors); + const Tensor* bias = spec.at("args").at(bi).is_null() + ? nullptr + : &tensor_arg(spec, bi, tensors); + const Conv2dParameters p{ + conv2d_pair_arg(spec, ki), + conv2d_pair_arg(spec, ki + 1), + conv2d_pair_arg(spec, ki + 2), + conv2d_pair_arg(spec, ki + 3), + q8ta::integer(spec, ki + 4)}; + const auto g = conv2d_geometry(input, weight, bias, p); + const auto weight_index = + [&](int64_t oc, int64_t ci, int64_t ky, int64_t kx) { + return oc * weight.sizes[1] + + (ky * p.kernel[1] + kx) * g.in_channels_per_group + ci; + }; + if (quantized) { + const float scale = q8ta::scale(spec, 1); + const int64_t zp = q8ta::integer(spec, 2); + input = q8ta::quantized_input(input, scale, zp); + return convolve( + input, + weight, + p, + g, + threads, + weight_index, + [&](int64_t sum, double value, double w) { + return sum + (int64_t(value) - zp) * int64_t(w); + }, + [&](int64_t sum, int64_t oc) { + const float combined_scale = scale * float(scales.values[size_t(oc)]); + return std::fma(float(sum), combined_scale, 0.0f) + + (bias ? float(bias->values[size_t(oc)]) : 0.0f); + }); + } + return convolve( + input, + weight, + p, + g, + threads, + weight_index, + [](float sum, double value, double w) { + return sum + float(value) * float(w); + }, + [&](float sum, int64_t oc) { + return sum * float(scales.values[size_t(oc)]) + + (bias ? float(bias->values[size_t(oc)]) : 0.0f); + }); +} +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_FN(q8csw_conv2d, q8csw_conv2d_reference); + REGISTER_OPERATOR_REFERENCE(et_vk.conv2d_q8csw.default, q8csw_conv2d); + REGISTER_OPERATOR_REFERENCE(et_vk.conv2d_q8ta_q8csw.default, q8csw_conv2d); +} +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/qw/q8csw/cases/correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/qw/q8csw/cases/correctness.json new file mode 100644 index 00000000000..85198f03ba4 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/qw/q8csw/cases/correctness.json @@ -0,0 +1,2396 @@ +[ + { + "name": "q8csw_conv2d/correctness", + "operator": "et_vk.conv2d_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 64, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.07000000029802322 + }, + { + "int": -3 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 28], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 64, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.07000000029802322 + }, + { + "int": -3 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 28], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/correctness [10]", + "operator": "et_vk.conv2d_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.07000000029802322 + }, + { + "int": -3 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/correctness [2]", + "operator": "et_vk.conv2d_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.07000000029802322 + }, + { + "int": -3 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 144], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/correctness [4]", + "operator": "et_vk.conv2d_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.07000000029802322 + }, + { + "int": -3 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/correctness [6]", + "operator": "et_vk.conv2d_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 55, 55], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.07000000029802322 + }, + { + "int": -3 + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [1], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [1], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [1], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 55, 55], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/correctness [12]", + "operator": "et_vk.conv2d_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 28, 28], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.07000000029802322 + }, + { + "int": -3 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 400], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [5, 5] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 28, 28], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/correctness [14]", + "operator": "et_vk.conv2d_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 14, 14], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.07000000029802322 + }, + { + "int": -3 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 1568], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [7, 7] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 14, 14], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/correctness [20]", + "operator": "et_vk.conv2d_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.07000000029802322 + }, + { + "int": -3 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 72], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 4 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/correctness [22]", + "operator": "et_vk.conv2d_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.07000000029802322 + }, + { + "int": -3 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 72], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 8 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/correctness [16]", + "operator": "et_vk.conv2d_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.07000000029802322 + }, + { + "int": -3 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 144], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [2, 2] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/correctness [18]", + "operator": "et_vk.conv2d_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.07000000029802322 + }, + { + "int": -3 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [3, 3] + }, + { + "int_list": [3, 3] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/correctness (12)", + "operator": "et_vk.conv2d_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 64, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 28], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 64, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 28], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/correctness [11]", + "operator": "et_vk.conv2d_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/correctness [3]", + "operator": "et_vk.conv2d_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 144], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/correctness [5]", + "operator": "et_vk.conv2d_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/correctness [7]", + "operator": "et_vk.conv2d_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 55, 55], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [1], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [1], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 55, 55], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/correctness [21]", + "operator": "et_vk.conv2d_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 72], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 4 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/correctness [23]", + "operator": "et_vk.conv2d_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 72], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 8 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/correctness [17]", + "operator": "et_vk.conv2d_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 144], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [2, 2] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/correctness [19]", + "operator": "et_vk.conv2d_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [3, 3] + }, + { + "int_list": [3, 3] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/correctness [13]", + "operator": "et_vk.conv2d_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 28, 28], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 400], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [5, 5] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 28, 28], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/correctness [15]", + "operator": "et_vk.conv2d_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 14, 14], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 1568], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [7, 7] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 14, 14], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/qw/q8csw/cases/performance.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/qw/q8csw/cases/performance.json new file mode 100644 index 00000000000..0b17381a26e --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/qw/q8csw/cases/performance.json @@ -0,0 +1,608 @@ +[ + { + "name": "q8csw_conv2d/performance [0]", + "operator": "et_vk.conv2d_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.07000000029802322 + }, + { + "int": -3 + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [256], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 256, 130, 130], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/performance [1]", + "operator": "et_vk.conv2d_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 256, 130, 130], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/performance [2]", + "operator": "et_vk.conv2d_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.07000000029802322 + }, + { + "int": -3 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 576], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/performance [3]", + "operator": "et_vk.conv2d_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 576], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/performance [4]", + "operator": "et_vk.conv2d_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 1024, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.07000000029802322 + }, + { + "int": -3 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 1024], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/performance [5]", + "operator": "et_vk.conv2d_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 1024, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 1024], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/embedding/qw/q4gsw/EmbeddingReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/embedding/qw/q4gsw/EmbeddingReference.cpp new file mode 100644 index 00000000000..21c1dc4f51a --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/embedding/qw/q4gsw/EmbeddingReference.cpp @@ -0,0 +1,45 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +namespace vkcompute::shader_profiling::reference { +Tensor embedding_q4gsw_reference( + const Json& spec, + const TensorMap& tensors, + size_t threads) { + const Tensor& weight = tensor_arg(spec, 0, tensors); + const Tensor& scales = tensor_arg(spec, 1, tensors); + const Tensor& indices = tensor_arg(spec, 3, tensors); + const int64_t group = spec.at("args").at(2).at("int"); + const bool linear = spec.at("args").at(4).at("bool"); + require( + weight.sizes.size() == 2 && group > 0, + "Embedding needs rank2 weight and positive group size"); + const size_t dim = weight.sizes[1] * 2, vocab = weight.sizes[0], + groups = dim / group; + require( + dim % group == 0 && scales.values.size() == vocab * groups, + "Embedding scale dimensions differ"); + Tensor out{indices.sizes, std::vector(indices.values.size() * dim)}; + out.sizes.push_back(dim); + parallel_for(indices.values.size(), threads, [&](size_t i) { + const int64_t row = indices.values[i]; + require(row >= 0 && row < int64_t(vocab), "Embedding index out of bounds"); + for (size_t d = 0; d < dim; ++d) { + const uint8_t byte = weight.values[row * (dim / 2) + d / 2]; + const size_t shift = linear ? 4 * (d % 2) : 4 * (1 - d % 2); + out.values[i * dim + d] = float(int((byte >> shift) & 15) - 8) * + float(scales.values[row * groups + d / group]); + } + }); + return out; +} +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_FN(embedding_q4gsw, embedding_q4gsw_reference); + REGISTER_OPERATOR_REFERENCE(et_vk.embedding_q4gsw.default, embedding_q4gsw); +} +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/embedding/qw/q4gsw/EmbeddingReference.h b/backends/vulkan/test/ops/benchmark_v2/operators/embedding/qw/q4gsw/EmbeddingReference.h new file mode 100644 index 00000000000..0af55a1f32e --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/embedding/qw/q4gsw/EmbeddingReference.h @@ -0,0 +1,18 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +namespace vkcompute::shader_profiling::reference { + +Tensor embedding_q4gsw_reference( + const Json& spec, + const TensorMap& tensors, + size_t threads); + +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/embedding/qw/q4gsw/cases/correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/embedding/qw/q4gsw/cases/correctness.json new file mode 100644 index 00000000000..2e90ae24b73 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/embedding/qw/q4gsw/cases/correctness.json @@ -0,0 +1,922 @@ +[ + { + "name": "embedding_q4gsw correctness", + "operator": "et_vk.embedding_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_OUTPUT_DTYPE": ["float16", "float32"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "weight", + "sizes": [16, 16], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [16, 1], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "indices", + "sizes": [4], + "dtype": "int32", + "storage": "buffer", + "layout": "width", + "data": { + "abs_mod": { + "bounds": [-10, 10], + "mod": 16 + } + } + } + }, + { + "bool": true + }, + { + "tensor": { + "name": "output", + "sizes": [4, 32], + "dtype": "v_OUTPUT_DTYPE", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "embedding_q4gsw correctness small_1d_texture_linear_weight texture3d", + "operator": "et_vk.embedding_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "weight", + "sizes": [16, 16], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [16, 1], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "indices", + "sizes": [4], + "dtype": "int32", + "storage": "texture3d", + "layout": "width", + "data": { + "abs_mod": { + "bounds": [-10, 10], + "mod": 16 + } + } + } + }, + { + "bool": true + }, + { + "tensor": { + "name": "output", + "sizes": [4, 32], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "embedding_q4gsw correctness small_2d_linear_weight buffer", + "operator": "et_vk.embedding_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 2], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "indices", + "sizes": [2, 3], + "dtype": "int32", + "storage": "buffer", + "layout": "width", + "data": { + "abs_mod": { + "bounds": [-10, 10], + "mod": 32 + } + } + } + }, + { + "bool": true + }, + { + "tensor": { + "name": "output", + "sizes": [2, 3, 64], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "embedding_q4gsw correctness medium_multigroup_linear_weight buffer", + "operator": "et_vk.embedding_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "weight", + "sizes": [100, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [100, 4], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "indices", + "sizes": [4, 8], + "dtype": "int32", + "storage": "buffer", + "layout": "width", + "data": { + "abs_mod": { + "bounds": [-10, 10], + "mod": 100 + } + } + } + }, + { + "bool": true + }, + { + "tensor": { + "name": "output", + "sizes": [4, 8, 128], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "embedding_q4gsw correctness (5)", + "operator": "et_vk.embedding_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_SCALES_DTYPE": ["float16", "float32"], + "v_STORAGE": ["buffer", "texture3d"], + "v_OUTPUT_DTYPE": ["float16", "float32"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "weight", + "sizes": [16, 16], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [16, 1], + "dtype": "v_SCALES_DTYPE", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "indices", + "sizes": [4], + "dtype": "int32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "abs_mod": { + "bounds": [-10, 10], + "mod": 16 + } + } + } + }, + { + "bool": false + }, + { + "tensor": { + "name": "output", + "sizes": [4, 32], + "dtype": "v_OUTPUT_DTYPE", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "embedding_q4gsw correctness (6)", + "operator": "et_vk.embedding_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["buffer", "texture3d"], + "v_OUTPUT_DTYPE": ["float16", "float32"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 2], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "indices", + "sizes": [2, 3], + "dtype": "int32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "abs_mod": { + "bounds": [-10, 10], + "mod": 32 + } + } + } + }, + { + "bool": false + }, + { + "tensor": { + "name": "output", + "sizes": [2, 3, 64], + "dtype": "v_OUTPUT_DTYPE", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "embedding_q4gsw correctness (7)", + "operator": "et_vk.embedding_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_OUTPUT_DTYPE": ["float16", "float32"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 2], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "indices", + "sizes": [2, 3], + "dtype": "int32", + "storage": "buffer", + "layout": "width", + "data": { + "abs_mod": { + "bounds": [-10, 10], + "mod": 32 + } + } + } + }, + { + "bool": false + }, + { + "tensor": { + "name": "output", + "sizes": [2, 3, 64], + "dtype": "v_OUTPUT_DTYPE", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "embedding_q4gsw correctness (8)", + "operator": "et_vk.embedding_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["buffer", "texture3d"], + "v_OUTPUT_DTYPE": ["float16", "float32"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "weight", + "sizes": [100, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [100, 4], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "indices", + "sizes": [4, 8], + "dtype": "int32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "abs_mod": { + "bounds": [-10, 10], + "mod": 100 + } + } + } + }, + { + "bool": false + }, + { + "tensor": { + "name": "output", + "sizes": [4, 8, 128], + "dtype": "v_OUTPUT_DTYPE", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "embedding_q4gsw correctness (9)", + "operator": "et_vk.embedding_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_OUTPUT_DTYPE": ["float16", "float32"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "weight", + "sizes": [100, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [100, 4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "indices", + "sizes": [4, 8], + "dtype": "int32", + "storage": "buffer", + "layout": "width", + "data": { + "abs_mod": { + "bounds": [-10, 10], + "mod": 100 + } + } + } + }, + { + "bool": false + }, + { + "tensor": { + "name": "output", + "sizes": [4, 8, 128], + "dtype": "v_OUTPUT_DTYPE", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "embedding_q4gsw correctness (10)", + "operator": "et_vk.embedding_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["buffer", "texture3d"], + "v_OUTPUT_DTYPE": ["float16", "float32"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "weight", + "sizes": [50, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [50, 1], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + { + "tensor": { + "name": "indices", + "sizes": [2, 4], + "dtype": "int32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "abs_mod": { + "bounds": [-10, 10], + "mod": 50 + } + } + } + }, + { + "bool": false + }, + { + "tensor": { + "name": "output", + "sizes": [2, 4, 64], + "dtype": "v_OUTPUT_DTYPE", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "embedding_q4gsw correctness gemma4_token_embedding_per_row", + "operator": "et_vk.embedding_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "weight", + "sizes": [16, 768], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 1536 + }, + { + "tensor": { + "name": "indices", + "sizes": [1, 4], + "dtype": "int32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "abs_mod": { + "bounds": [-10, 10], + "mod": 16 + } + } + } + }, + { + "bool": true + }, + { + "tensor": { + "name": "output", + "sizes": [1, 4, 1536], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "embedding_q4gsw correctness gemma4_ple_embedding_per_row", + "operator": "et_vk.embedding_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "weight", + "sizes": [16, 4480], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 8960 + }, + { + "tensor": { + "name": "indices", + "sizes": [1, 4], + "dtype": "int32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "abs_mod": { + "bounds": [-10, 10], + "mod": 16 + } + } + } + }, + { + "bool": true + }, + { + "tensor": { + "name": "output", + "sizes": [1, 4, 8960], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/embedding/qw/q4gsw/cases/performance.json b/backends/vulkan/test/ops/benchmark_v2/operators/embedding/qw/q4gsw/cases/performance.json new file mode 100644 index 00000000000..8e93c4808d8 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/embedding/qw/q4gsw/cases/performance.json @@ -0,0 +1,154 @@ +[ + { + "name": "embedding_q4gsw performance llama_3_2_1b_prefill_fp32_linear_weight buffer", + "operator": "et_vk.embedding_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "weight", + "sizes": [128256, 1024], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [128256, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "indices", + "sizes": [1, 2047], + "dtype": "int32", + "storage": "buffer", + "layout": "width", + "data": { + "abs_mod": { + "bounds": [-10, 10], + "mod": 128256 + } + } + } + }, + { + "bool": true + }, + { + "tensor": { + "name": "output", + "sizes": [1, 2047, 2048], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "embedding_q4gsw performance", + "operator": "et_vk.embedding_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_SCALES_DTYPE": ["float16", "float32"], + "v_OUTPUT_DTYPE": ["float32", "float16"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "weight", + "sizes": [128256, 1024], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [128256, 64], + "dtype": "v_SCALES_DTYPE", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "indices", + "sizes": [1, 2047], + "dtype": "int32", + "storage": "buffer", + "layout": "width", + "data": { + "abs_mod": { + "bounds": [-10, 10], + "mod": 128256 + } + } + } + }, + { + "bool": false + }, + { + "tensor": { + "name": "output", + "sizes": [1, 2047, 2048], + "dtype": "v_OUTPUT_DTYPE", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/GemmReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/GemmReference.cpp new file mode 100644 index 00000000000..e6f22f684c0 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/GemmReference.cpp @@ -0,0 +1,68 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include + +namespace vkcompute::shader_profiling::reference { + +Tensor linear_projection( + const Tensor& input, + const Tensor& weight, + const Tensor* bias, + size_t threads) { + require( + !input.sizes.empty() && weight.sizes.size() == 2, + "Linear requires input rank >= 1 and weight rank 2"); + const size_t k = input.sizes.back(); + const size_t n = weight.sizes[0]; + require( + k == size_t(weight.sizes[1]), "Linear input/weight dimensions differ"); + require( + !bias || bias->values.size() == n, + "Linear bias size must equal output features"); + const size_t m = input.values.size() / k; + Tensor output{input.sizes, std::vector(m * n)}; + output.sizes.back() = n; + std::vector packed(k * n); + parallel_for(n, threads, [&](size_t column) { + for (size_t inner = 0; inner < k; ++inner) { + packed[inner * n + column] = weight.values[column * k + inner]; + } + }); + std::vector activations(input.values.begin(), input.values.end()); + constexpr size_t kRows = 4; + constexpr size_t kColumns = 64; + const size_t column_tiles = (n + kColumns - 1) / kColumns; + const size_t row_tiles = (m + kRows - 1) / kRows; + parallel_for(row_tiles * column_tiles, threads, [&](size_t tile) { + const size_t row = (tile / column_tiles) * kRows; + const size_t column = (tile % column_tiles) * kColumns; + const size_t rows = std::min(kRows, m - row); + const size_t columns = std::min(kColumns, n - column); + std::array, kRows> accumulators{}; + for (size_t inner = 0; inner < k; ++inner) { + const float* weights = packed.data() + inner * n + column; + for (size_t i = 0; i < rows; ++i) { + const float activation = activations[(row + i) * k + inner]; + for (size_t j = 0; j < columns; ++j) { + accumulators[i][j] += activation * weights[j]; + } + } + } + for (size_t i = 0; i < rows; ++i) { + for (size_t j = 0; j < columns; ++j) { + output.values[(row + i) * n + column + j] = accumulators[i][j] + + (bias ? float(bias->values[column + j]) : 0.0f); + } + } + }); + return output; +} + +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/GemmReference.h b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/GemmReference.h new file mode 100644 index 00000000000..8b8caf57f47 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/GemmReference.h @@ -0,0 +1,18 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +namespace vkcompute::shader_profiling::reference { + +Tensor linear_projection( + const Tensor& input, + const Tensor& weight, + const Tensor* bias, + size_t threads); +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/GemmThroughputCommon.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/GemmThroughputCommon.cpp new file mode 100644 index 00000000000..b784b607d34 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/GemmThroughputCommon.cpp @@ -0,0 +1,44 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +namespace vkcompute::shader_profiling::throughput { + +double gemm_flops(const Json& spec, size_t weight_packing, size_t bias_index) { + if (weight_packing == 0) + throw std::invalid_argument("GEMM weight packing must be positive"); + const Json& input = throughput::tensor_argument(spec, 0); + const Json& weight = throughput::tensor_argument(spec, 1); + const Json& output = throughput::output_tensor(spec); + const auto input_sizes = input.at("sizes").get>(); + const auto weight_sizes = weight.at("sizes").get>(); + const auto output_sizes = output.at("sizes").get>(); + const double input_elements = throughput::tensor_elements(input); + const double output_elements = throughput::tensor_elements(output); + throughput::tensor_elements(weight); + if (weight_sizes.size() != 2 || input_sizes.empty() || output_sizes.empty()) { + throw std::invalid_argument( + "GEMM work count requires input/output ranks >= 1 and weight rank 2"); + } + const int64_t k = input_sizes.back(); + const int64_t n = output_sizes.back(); + if (weight_sizes[0] != n || weight_sizes[1] != k / weight_packing || + input_elements / k != output_elements / n || k % weight_packing != 0) { + throw std::invalid_argument("GEMM work count tensor dimensions disagree"); + } + const bool bias = !spec.at("args").at(bias_index).is_null(); + if (bias && + throughput::tensor_elements( + throughput::tensor_argument(spec, bias_index)) != n) { + throw std::invalid_argument("GEMM work count bias must have N elements"); + } + return output_elements * (2 * double(k) + (bias ? 1 : 0)); +} + +} // namespace vkcompute::shader_profiling::throughput diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/GemmThroughputCommon.h b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/GemmThroughputCommon.h new file mode 100644 index 00000000000..cb0a157ba82 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/GemmThroughputCommon.h @@ -0,0 +1,13 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once +#include + +namespace vkcompute::shader_profiling::throughput { + +double gemm_flops(const Json& spec, size_t weight_packing, size_t bias_index); +} // namespace vkcompute::shader_profiling::throughput diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/LinearReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/LinearReference.cpp new file mode 100644 index 00000000000..90a2f15f5c2 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/LinearReference.cpp @@ -0,0 +1,29 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include + +namespace vkcompute::shader_profiling::reference { + +Tensor +linear_reference(const Json& spec, const TensorMap& tensors, size_t threads) { + const Tensor* bias = + spec.at("args").at(2).is_null() ? nullptr : &tensor_arg(spec, 2, tensors); + return linear_projection( + tensor_arg(spec, 0, tensors), + tensor_arg(spec, 1, tensors), + bias, + threads); +} + +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_FN(linear, linear_reference); + REGISTER_OPERATOR_REFERENCE(aten.linear.default, linear); +} + +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/LinearReference.h b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/LinearReference.h new file mode 100644 index 00000000000..958eed7ccea --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/LinearReference.h @@ -0,0 +1,15 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +namespace vkcompute::shader_profiling::reference { + +Tensor +linear_reference(const Json& spec, const TensorMap& tensors, size_t threads); +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/LinearThroughput.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/LinearThroughput.cpp new file mode 100644 index 00000000000..d030700c0e6 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/LinearThroughput.cpp @@ -0,0 +1,62 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +namespace vkcompute::shader_profiling { +namespace { + +double linear_flops(const Json& spec) { + return throughput::gemm_flops(spec, 1, 2); +} + +double linear_shader_flops(const Json& spec) { + const Json* input = nullptr; + for (const auto& binding : spec.at("bindings").items()) { + const Json& tensor = binding.value(); + if (tensor.contains("sizes") && tensor.value("access", "") == "read" && + (!input || + tensor.value("binding_index", SIZE_MAX) < + input->value("binding_index", SIZE_MAX))) { + input = &tensor; + } + } + if (!input) { + throw std::invalid_argument( + "Linear shader work count requires first input binding"); + } + const Json& output = throughput::output_tensor(spec); + const double output_elements = throughput::tensor_elements(output); + const double input_elements = throughput::tensor_elements(*input); + const auto input_sizes = input->at("sizes").get>(); + const auto output_sizes = output.at("sizes").get>(); + if (input_elements / input_sizes.back() != + output_elements / output_sizes.back()) { + throw std::invalid_argument( + "Linear shader input/output row counts disagree"); + } + return output_elements * 2 * double(input_sizes.back()); +} + +double linear_bias_shader_flops(const Json& spec) { + return linear_shader_flops(spec) + + throughput::tensor_elements(throughput::output_tensor(spec)); +} + +} // namespace + +REGISTER_THROUGHPUT_FUNCTIONS { + REGISTER_OPERATOR_MEMORY_TRAFFIC( + aten.linear.default, throughput::tensor_traffic); + REGISTER_OPERATOR_COMPUTE_FLOPS(aten.linear.default, linear_flops); + REGISTER_SHADER_MEMORY_TRAFFIC(linear_vec, throughput::tensor_traffic); + REGISTER_SHADER_COMPUTE_FLOPS(linear_vec, linear_shader_flops); + REGISTER_SHADER_COMPUTE_FLOPS(linear_vec_bias, linear_bias_shader_flops); +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/q8ta/Q8taLinearReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/q8ta/Q8taLinearReference.cpp new file mode 100644 index 00000000000..1573db58e0d --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/q8ta/Q8taLinearReference.cpp @@ -0,0 +1,75 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +namespace vkcompute::shader_profiling::reference { +static Tensor q8ta_linear_reference( + const Json& spec, + const TensorMap& tensors, + size_t threads) { + const Tensor& input = tensor_arg(spec, 0, tensors); + const Tensor& weight = tensor_arg(spec, 3, tensors); + const Tensor& weight_sums = tensor_arg(spec, 4, tensors); + const Tensor& scales = tensor_arg(spec, 5, tensors); + const Tensor* bias = + spec.at("args").at(8).is_null() ? nullptr : &tensor_arg(spec, 8, tensors); + require( + !input.sizes.empty() && weight.sizes.size() == 2, + "Q8 linear requires input rank >= 1 and matrix weights"); + const int64_t k = input.sizes.back(), n = weight.sizes[0]; + require(k > 0 && n > 0, "Q8 linear requires positive feature dimensions"); + const int64_t m = int64_t(input.values.size()) / k; + require(weight.sizes[1] == k, "Q8 linear weight/input features disagree"); + auto out_sizes = input.sizes; + out_sizes.back() = n; + Tensor out{std::move(out_sizes), std::vector(size_t(m * n))}; + const float is = q8ta::scale(spec, 1), os = q8ta::scale(spec, 6); + const int64_t iz = q8ta::integer(spec, 2), oz = q8ta::integer(spec, 7); + std::vector quantized(input.values.size()); + for (size_t i = 0; i < quantized.size(); ++i) + quantized[i] = int16_t(input.values[i]); + std::vector packed_weight(weight.values.size()); + for (int64_t col = 0; col < n; ++col) + for (int64_t j = 0; j < k; ++j) + packed_weight[size_t(j * n + col)] = + int8_t(weight.values[size_t(col * k + j)]); + constexpr int64_t tile = 32; + const int64_t blocks = (n + tile - 1) / tile; + parallel_for(size_t(m * blocks), threads, [&](size_t index) { + const int64_t row = int64_t(index) / blocks, + start = int64_t(index) % blocks * tile, + count = std::min(tile, n - start); + std::array sums{}; + for (int64_t j = 0; j < k; ++j) { + const int64_t value = quantized[size_t(row * k + j)]; + for (int64_t col = 0; col < count; ++col) + sums[size_t(col)] += value * packed_weight[size_t(j * n + start + col)]; + } + for (int64_t col = 0; col < count; ++col) { + const size_t oc = size_t(start + col); + const int64_t corrected = + sums[size_t(col)] - iz * int64_t(weight_sums.values[oc]); + out.values[size_t(row * n) + oc] = double(q8ta::requantize_accumulator( + corrected, + is, + float(scales.values[oc]), + os, + oz, + bias ? float(bias->values[oc]) : 0.0f, + spec.at("args").at(9).at("string") == "relu", + false)); + } + }); + return out; +} +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_FN(q8ta_linear, q8ta_linear_reference); + REGISTER_OPERATOR_REFERENCE(et_vk.q8ta_linear.default, q8ta_linear); + REGISTER_OPERATOR_REFERENCE(et_vk.q8ta_linear_gemv.default, q8ta_linear); +} +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/q8ta/Q8taLinearRoutes.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/q8ta/Q8taLinearRoutes.cpp new file mode 100644 index 00000000000..1c61a982115 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/q8ta/Q8taLinearRoutes.cpp @@ -0,0 +1,24 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +namespace vkcompute::shader_profiling { +static Json q8ta_linear_gemv_route( + ComputeGraph& graph, + const std::vector& args, + const Json& options) { + if (!options.empty()) + throw std::invalid_argument("Q8 linear gemv route accepts no options"); + VK_GET_OP_FN("et_vk.q8ta_linear_gemv.default")(graph, args); + return {{"name", "gemv"}, {"options", Json::object()}}; +} +REGISTER_OPERATOR_ROUTES { + REGISTER_OPERATOR_ROUTE( + et_vk.q8ta_linear.default, gemv, q8ta_linear_gemv_route); +} +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/q8ta/Q8taLinearThroughput.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/q8ta/Q8taLinearThroughput.cpp new file mode 100644 index 00000000000..b6c28cc2a79 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/q8ta/Q8taLinearThroughput.cpp @@ -0,0 +1,26 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +namespace vkcompute::shader_profiling { +namespace { +double q8ta_linear_flops(const Json& spec) { + const auto& input = throughput::tensor_argument(spec, 0); + const double k = input.at("sizes").back().get(); + return 2 * k * throughput::tensor_elements(throughput::output_tensor(spec)); +} +REGISTER_THROUGHPUT_FUNCTIONS { + REGISTER_OPERATOR_MEMORY_TRAFFIC( + et_vk.q8ta_linear.default, throughput::tensor_traffic); + REGISTER_OPERATOR_MEMORY_TRAFFIC( + et_vk.q8ta_linear_gemv.default, throughput::tensor_traffic); + REGISTER_OPERATOR_COMPUTE_FLOPS(et_vk.q8ta_linear.default, q8ta_linear_flops); + REGISTER_OPERATOR_COMPUTE_FLOPS( + et_vk.q8ta_linear_gemv.default, q8ta_linear_flops); +} +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/q8ta/cases/correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/q8ta/cases/correctness.json new file mode 100644 index 00000000000..f9c5f4ebde9 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/q8ta/cases/correctness.json @@ -0,0 +1,1638 @@ +[ + { + "name": "q8ta/linear/correctness [0]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/correctness [1]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "gemv", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/correctness [2]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/correctness [3]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "gemv", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/correctness [4]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/correctness [5]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "gemv", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/correctness [6]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + null, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/correctness [7]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "gemv", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + null, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/correctness [8]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [4, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/correctness [9]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [4, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/correctness [10]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [4, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/correctness [11]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/correctness [12]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [32, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/correctness [13]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [32, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/correctness [14]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + null, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [32, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/correctness [15]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + null, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [32, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/q8ta/cases/performance.json b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/q8ta/cases/performance.json new file mode 100644 index 00000000000..6379d8c2073 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/q8ta/cases/performance.json @@ -0,0 +1,947 @@ +[ + { + "name": "q8ta/linear/performance [0]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 512], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [512, 512], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [512], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [512], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [512], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 512], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/performance [1]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "gemv", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 512], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [512, 512], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [512], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [512], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [512], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 512], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/performance [2]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 2048], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 2048], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2048], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 2048], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/performance [3]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "gemv", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 2048], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 2048], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2048], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 2048], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/performance [4]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 512], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [9059, 512], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [9059], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [9059], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [9059], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 9059], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/performance [5]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "gemv", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 512], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [9059, 512], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [9059], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [9059], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [9059], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 9059], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/performance [6]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [256, 2048], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 2048], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2048], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [256, 2048], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/performance [7]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [512, 2048], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 2048], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2048], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [512, 2048], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/performance [8]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1024, 2048], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 2048], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2048], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1024, 2048], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/dq8ca_q4gsw/Dq8caQ4gswReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/dq8ca_q4gsw/Dq8caQ4gswReference.cpp new file mode 100644 index 00000000000..168db9c689c --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/dq8ca_q4gsw/Dq8caQ4gswReference.cpp @@ -0,0 +1,59 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +#include +#include +#include +namespace vkcompute::shader_profiling::reference { +Tensor dq8ca_q4gsw_reference( + const Json& spec, + const TensorMap& tensors, + size_t threads) { + Json canonical = spec; + canonical["args"] = { + spec.at("args").at(0), + spec.at("args").at(3), + spec.at("args").at(5), + spec.at("args").at(6), + spec.at("args").at(7), + spec.at("args").at(8)}; + const Json context = spec.value("reference_context", Json::object()); + require(context.is_object(), "Reference context must be an object"); + const Json dot = context.value("int8_dot_product", Json(true)); + require(dot.is_boolean(), "DQ8CA reference capability must be boolean"); + if (!dot.get()) { + return q4gsw_linear_reference(canonical, tensors, threads); + } + TensorMap converted = tensors; + const std::string name = spec.at("args").at(0).at("tensor"); + Tensor& input = converted.at(name); + const Tensor& scales = tensor_arg(spec, 1, tensors); + const Tensor& zeros = tensor_arg(spec, 2, tensors); + const size_t k = input.sizes.back(), m = input.values.size() / k; + require( + scales.values.size() == m && zeros.values.size() == m, + "Dynamic Q4 activation qparams must be per row"); + parallel_for(m, threads, [&](size_t row) { + const float scale = scales.values[row], zp = zeros.values[row]; + require(scale > 0, "Activation scale must be positive"); + for (size_t i = 0; i < k; ++i) { + const float q = std::clamp( + std::round(float(input.values[row * k + i]) / scale) + zp, + -128.0f, + 127.0f); + input.values[row * k + i] = (q - zp) * scale; + } + }); + return q4gsw_linear_reference(canonical, converted, threads); +} +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_FN(dq8ca_q4gsw, dq8ca_q4gsw_reference); + REGISTER_OPERATOR_REFERENCE(et_vk.linear_dq8ca_q4gsw.default, dq8ca_q4gsw); +} +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/dq8ca_q4gsw/Dq8caQ4gswReference.h b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/dq8ca_q4gsw/Dq8caQ4gswReference.h new file mode 100644 index 00000000000..2aeb21eb299 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/dq8ca_q4gsw/Dq8caQ4gswReference.h @@ -0,0 +1,18 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +namespace vkcompute::shader_profiling::reference { + +Tensor dq8ca_q4gsw_reference( + const Json& spec, + const TensorMap& tensors, + size_t threads); + +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/dq8ca_q4gsw/Dq8caQ4gswThroughput.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/dq8ca_q4gsw/Dq8caQ4gswThroughput.cpp new file mode 100644 index 00000000000..27fc7ab8b4d --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/dq8ca_q4gsw/Dq8caQ4gswThroughput.cpp @@ -0,0 +1,26 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +namespace vkcompute::shader_profiling { +namespace { +double quantized_linear_legacy_flops(const Json& spec) { + const auto sizes = throughput::tensor_argument(spec, 0) + .at("sizes") + .get>(); + return throughput::tensor_elements(throughput::output_tensor(spec)) * + (3.0 * sizes.back() + 1); +} +REGISTER_THROUGHPUT_FUNCTIONS { + REGISTER_OPERATOR_MEMORY_TRAFFIC( + et_vk.linear_dq8ca_q4gsw.default, throughput::tensor_traffic); + REGISTER_OPERATOR_COMPUTE_FLOPS( + et_vk.linear_dq8ca_q4gsw.default, quantized_linear_legacy_flops); +} + +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/Q4gswLinearReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/Q4gswLinearReference.cpp new file mode 100644 index 00000000000..12394c6f58d --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/Q4gswLinearReference.cpp @@ -0,0 +1,56 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include + +namespace vkcompute::shader_profiling::reference { + +Tensor q4gsw_linear_reference( + const Json& spec, + const TensorMap& tensors, + size_t threads) { + const Tensor& input = tensor_arg(spec, 0, tensors); + const Tensor& packed = tensor_arg(spec, 1, tensors); + const Tensor& scales = tensor_arg(spec, 2, tensors); + require( + !input.sizes.empty() && packed.sizes.size() == 2, + "Q4 linear requires input rank >= 1 and packed weight rank 2"); + const size_t k = input.sizes.back(); + const size_t n = packed.sizes[0]; + const int64_t group_size = spec.at("args").at(3).at("int"); + require( + group_size > 0 && k % group_size == 0 && k % 2 == 0, + "Q4 linear requires even K divisible by group_size"); + require( + size_t(packed.sizes[1]) == k / 2, + "Q4 packed weight shape must be [N, K/2]"); + require( + scales.sizes == + std::vector({int64_t(k / group_size), int64_t(n)}), + "Q4 scale shape must be [K/group_size, N]"); + Tensor weight{{int64_t(n), int64_t(k)}, std::vector(n * k)}; + parallel_for(n, threads, [&](size_t column) { + for (size_t inner = 0; inner < k; ++inner) { + const uint8_t byte = packed.values[column * (k / 2) + inner / 2]; + const int value = ((byte >> (4 * (inner % 2))) & 15) - 8; + weight.values[column * k + inner] = float(value) * + float(scales.values[(inner / group_size) * n + column]); + } + }); + const Tensor* bias = + spec.at("args").at(4).is_null() ? nullptr : &tensor_arg(spec, 4, tensors); + return linear_projection(input, weight, bias, threads); +} + +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_FN(q4gsw_linear, q4gsw_linear_reference); + REGISTER_OPERATOR_REFERENCE(et_vk.q4gsw_linear.default, q4gsw_linear); + REGISTER_OPERATOR_REFERENCE(et_vk.linear_q4gsw.default, q4gsw_linear); +} + +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/Q4gswLinearReference.h b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/Q4gswLinearReference.h new file mode 100644 index 00000000000..75b4846d686 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/Q4gswLinearReference.h @@ -0,0 +1,17 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +namespace vkcompute::shader_profiling::reference { + +Tensor q4gsw_linear_reference( + const Json& spec, + const TensorMap& tensors, + size_t threads); +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/Q4gswLinearThroughput.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/Q4gswLinearThroughput.cpp new file mode 100644 index 00000000000..66b388943f5 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/Q4gswLinearThroughput.cpp @@ -0,0 +1,91 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +namespace vkcompute::shader_profiling { +namespace { + +double q4gsw_linear_flops(const Json& spec) { + const double flops = throughput::gemm_flops(spec, 2, 4); + const auto input_sizes = throughput::tensor_argument(spec, 0) + .at("sizes") + .get>(); + const auto output_sizes = + throughput::output_tensor(spec).at("sizes").get>(); + const int64_t k = input_sizes.back(); + const int64_t n = output_sizes.back(); + + const int64_t group_size = spec.at("args").at(3).at("int"); + const Json& scales = throughput::tensor_argument(spec, 2); + const auto scale_sizes = scales.at("sizes").get>(); + if (group_size <= 0 || k % group_size != 0 || + scale_sizes != std::vector({k / group_size, n})) { + throw std::invalid_argument( + "Q4 work count requires scales [K/group_size, N]"); + } + return flops; +} + +double q4_shader_traffic(const Json& spec, size_t inactive_input) { + Json filtered = spec; + Json& bindings = filtered.at("bindings"); + unsigned seen_indices = 0; + for (auto binding = bindings.begin(); binding != bindings.end();) { + const Json& tensor = binding.value(); + if (!tensor.contains("sizes")) { + ++binding; + continue; + } + const size_t index = tensor.at("binding_index").get(); + if (index >= 6 || (seen_indices & (1u << index))) { + throw std::invalid_argument( + "Q4 shader memory count requires unique tensor binding indices 0..5"); + } + seen_indices |= 1u << index; + if (index == inactive_input || (index == 5 && tensor.at("sizes").empty())) { + binding = bindings.erase(binding); + } else { + ++binding; + } + } + if (seen_indices != 63) { + throw std::invalid_argument( + "Q4 shader memory count requires all six tensor binding slots"); + } + return throughput::tensor_traffic(filtered); +} + +double q4_gemm_shader_traffic(const Json& spec) { + return q4_shader_traffic(spec, 2); +} + +double q4_tin_shader_traffic(const Json& spec) { + return q4_shader_traffic(spec, 1); +} + +} // namespace + +REGISTER_THROUGHPUT_FUNCTIONS { + REGISTER_OPERATOR_MEMORY_TRAFFIC( + et_vk.q4gsw_linear.default, throughput::tensor_traffic); + REGISTER_OPERATOR_COMPUTE_FLOPS( + et_vk.q4gsw_linear.default, q4gsw_linear_flops); + REGISTER_OPERATOR_MEMORY_TRAFFIC( + et_vk.linear_q4gsw.default, throughput::tensor_traffic); + REGISTER_OPERATOR_COMPUTE_FLOPS( + et_vk.linear_q4gsw.default, q4gsw_linear_flops); + REGISTER_SHADER_MEMORY( + transpose_cast_contig_to_vectorized_half_buffer_half_buffer, + throughput::tensor_traffic); + REGISTER_SHADER_MEMORY_TRAFFIC(q4gsw_linear_gemm__, q4_gemm_shader_traffic); + REGISTER_SHADER_MEMORY_TRAFFIC( + q4gsw_linear_gemm__tin__, q4_tin_shader_traffic); +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/Q4gswRoutes.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/Q4gswRoutes.cpp new file mode 100644 index 00000000000..47df7a3d394 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/Q4gswRoutes.cpp @@ -0,0 +1,766 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace vkcompute { + +namespace { + +enum class TestKernelKind { + PROD, + GEMM_W_4X8, + GEMM_TIN_W_4X8, + GEMV_W_4X8, + GEMV_W_4X8_NOSG, + LEGACY, + GEMV_COOP_W_4X8_NC_BUFFER, + + GEMV_COOP_W_4X8_NC_BUFFER_G1W64, + GEMV_COOP_W_4X8_NC_BUFFER_G4W16, + GEMV_COOP_W_4X8_NC_BUFFER_G8W8, +}; + +TestKernelKind selector_to_kind(int32_t selector, bool is_gemv) { + if (is_gemv) { + switch (selector) { + case 0: + return TestKernelKind::PROD; + case 1: + return TestKernelKind::GEMV_W_4X8; + case 2: + return TestKernelKind::GEMV_W_4X8_NOSG; + case 3: + return TestKernelKind::LEGACY; + case 13: + return TestKernelKind::GEMV_COOP_W_4X8_NC_BUFFER; + case 14: + return TestKernelKind::GEMV_COOP_W_4X8_NC_BUFFER_G1W64; + case 15: + return TestKernelKind::GEMV_COOP_W_4X8_NC_BUFFER_G4W16; + case 16: + return TestKernelKind::GEMV_COOP_W_4X8_NC_BUFFER_G8W8; + default: + return TestKernelKind::PROD; + } + } + switch (selector) { + case 0: + return TestKernelKind::PROD; + case 1: + return TestKernelKind::GEMM_W_4X8; + case 2: + return TestKernelKind::GEMM_TIN_W_4X8; + case 3: + return TestKernelKind::LEGACY; + default: + return TestKernelKind::PROD; + } +} + +const char* forced_kind_base_name(TestKernelKind kind) { + switch (kind) { + case TestKernelKind::GEMM_W_4X8: + return "q4gsw_linear_gemm__w_4x8_nc"; + case TestKernelKind::GEMM_TIN_W_4X8: + return "q4gsw_linear_gemm__tin__w_4x8_nc"; + case TestKernelKind::GEMV_W_4X8: + return "q4gsw_linear_gemv__w_4x8_nc"; + case TestKernelKind::GEMV_W_4X8_NOSG: + return "q4gsw_linear_gemv__w_4x8_nc_nosg"; + case TestKernelKind::GEMV_COOP_W_4X8_NC_BUFFER: + case TestKernelKind::GEMV_COOP_W_4X8_NC_BUFFER_G1W64: + return "q4gsw_linear_gemv_coop__w_4x8_nc_buffer_g1w64"; + case TestKernelKind::GEMV_COOP_W_4X8_NC_BUFFER_G4W16: + return "q4gsw_linear_gemv_coop__w_4x8_nc_buffer_g4w16"; + case TestKernelKind::GEMV_COOP_W_4X8_NC_BUFFER_G8W8: + return "q4gsw_linear_gemv_coop__w_4x8_nc_buffer_g8w8"; + case TestKernelKind::PROD: + case TestKernelKind::LEGACY: + default: + return ""; + } +} + +template +vkapi::ShaderInfo pick_forced_shader( + ComputeGraph* graph, + const std::vector& args, + const std::vector& resize_args) { + (void)resize_args; + const ValueRef out = args.at(0).refs.at(0); + std::string kernel_name = forced_kind_base_name(KIND); + add_storage_type_suffix(kernel_name, graph->storage_type_of(out)); + add_dtype_suffix(kernel_name, graph->dtype_of(out)); + return VK_KERNEL_FROM_STR(kernel_name); +} + +template +vkapi::ShaderInfo pick_forced_shader_coop_kc( + ComputeGraph* graph, + const std::vector& args, + const std::vector& resize_args) { + (void)resize_args; + const ValueRef out = args.at(0).refs.at(0); + std::string kernel_name = forced_kind_base_name(KIND); + add_storage_type_suffix(kernel_name, graph->storage_type_of(out)); + add_dtype_suffix(kernel_name, graph->dtype_of(out)); + return VK_KERNEL_FROM_STR(kernel_name); +} + +template +GlobalWorkGrid pick_q4gsw_coop_gwg( + ComputeGraph* graph, + const vkapi::ShaderInfo& shader, + const std::vector& args, + const std::vector& resize_args) { + (void)shader; + (void)resize_args; + const ValueRef out = args.at(0).refs.at(0); + const std::vector out_sizes = graph->sizes_of(out); + const uint32_t N = + utils::safe_downcast(utils::val_at(-1, out_sizes)); + const uint32_t N8 = (N + 7u) / 8u; + const uint32_t wgs_along_x = utils::div_up(N8, NUM_GROUPS); + const LocalWorkGroup lwg(1u, NUM_GROUPS, WORKERS_PER_GROUP); + GlobalWorkGrid gwg({wgs_along_x * NUM_GROUPS, 1u, 1u}, kLinearWorkGrid); + gwg.wrap_linear_dispatch( + graph->context()->adapter_ptr()->max_compute_workgroup_count(), lwg); + return gwg; +} + +template +LocalWorkGroup pick_q4gsw_coop_lwg( + ComputeGraph* graph, + const vkapi::ShaderInfo& shader, + const GlobalWorkGrid& gwg, + const std::vector& args, + const std::vector& resize_args) { + (void)graph; + (void)shader; + (void)gwg; + (void)args; + (void)resize_args; + return LocalWorkGroup(1u, NUM_GROUPS, WORKERS_PER_GROUP); +} + +constexpr uint32_t kGemvSubgroupSize = 64u; +constexpr uint32_t kGemvNumSubgroups = 4u; + +GlobalWorkGrid pick_q4gsw_legacy_gemv_gwg( + ComputeGraph* graph, + const vkapi::ShaderInfo& shader, + const std::vector& args, + const std::vector& resize_args) { + (void)shader; + (void)resize_args; + const ValueRef out = args.at(0).refs.at(0); + const uint32_t N = + utils::safe_downcast(utils::val_at(-1, graph->sizes_of(out))); + + return GlobalWorkGrid({N / 2u, kGemvNumSubgroups, 1u}, kTiledWorkGrid); +} + +LocalWorkGroup pick_q4gsw_legacy_gemv_lwg( + ComputeGraph* graph, + const vkapi::ShaderInfo& shader, + const GlobalWorkGrid& gwg, + const std::vector& args, + const std::vector& resize_args) { + (void)graph; + (void)shader; + (void)gwg; + (void)args; + (void)resize_args; + return LocalWorkGroup(kGemvSubgroupSize, kGemvNumSubgroups, 1u); +} + +void legacy_q4gsw_resize_linear_node( + ComputeGraph* graph, + const std::vector& args, + const std::vector& extra_args) { + (void)extra_args; + + ValueRef output = args.at(0).refs.at(0); + ValueRef fp_input = args.at(1).refs.at(0); + ValueRef weight_data = extra_args.at(1); + + std::vector mat1_sizes = graph->sizes_of(fp_input); + std::vector mat2_sizes = graph->sizes_of(weight_data); + + const int64_t out_cols = utils::val_at(-2, mat1_sizes); + const int64_t out_rows = utils::val_at(-2, mat2_sizes); + + std::vector new_out_sizes(3); + if (mat1_sizes.size() == 2) { + new_out_sizes.resize(2); + new_out_sizes.at(0) = out_cols; + new_out_sizes.at(1) = out_rows; + } else { + new_out_sizes.at(0) = mat1_sizes.at(0); + new_out_sizes.at(1) = out_cols; + new_out_sizes.at(2) = out_rows; + } + + graph->virtual_resize(output, new_out_sizes); +} + +GlobalWorkGrid legacy_q4gsw_gwg( + ComputeGraph* graph, + const vkapi::ShaderInfo& shader, + const std::vector& args, + const std::vector& resize_args) { + (void)resize_args; + const ValueRef out = args.at(0).refs.at(0); + + std::vector out_sizes = graph->sizes_of(out); + + const uint32_t N = + utils::safe_downcast(utils::val_at(-1, out_sizes)); + + const uint32_t M = + utils::safe_downcast(utils::val_at(-2, out_sizes)); + + uint32_t N_per_tile = 8; + uint32_t M_per_tile = 4; + if (shader.kernel_name.find("coop") != std::string::npos) { + M_per_tile = 1; + } + + const uint32_t num_N_tiles = utils::div_up(N, N_per_tile); + const uint32_t num_M_tiles = utils::div_up(M, M_per_tile); + + return GlobalWorkGrid({num_N_tiles, num_M_tiles, 1u}, kTiledWorkGrid); +} + +LocalWorkGroup legacy_q4gsw_lwg( + ComputeGraph* graph, + const vkapi::ShaderInfo& shader, + const GlobalWorkGrid& gwg, + const std::vector& args, + const std::vector& resize_args) { + const bool use_coop_algorithm = + shader.kernel_name.find("_coop") != std::string::npos; + + if (use_coop_algorithm) { + return LocalWorkGroup(1u, 1u, 64u); + } + return pick_xy_square_lwg(graph, shader, gwg, args, resize_args); +} + +vkapi::ShaderInfo legacy_q4gsw_pick_shader( + ComputeGraph* graph, + const std::vector& args, + const std::vector& resize_args) { + (void)resize_args; + const ValueRef output = args.at(0).refs.at(0); + const ValueRef fp_input = args.at(1).refs.at(0); + const ValueRef packed_int_weight = args.at(1).refs.at(1); + + const bool is_gemv_case = is_gemv(graph, fp_input); + + std::string kernel_name = "linear_q4gsw"; + kernel_name += is_gemv_case ? "_coop" : "_tiled"; + + add_storage_type_suffix(kernel_name, graph->storage_type_of(output)); + add_storage_type_suffix( + kernel_name, graph->storage_type_of(packed_int_weight)); + add_dtype_suffix(kernel_name, graph->dtype_of(output)); + + return VK_KERNEL_FROM_STR(kernel_name); +} + +ValueRef legacy_prepack_q4gsw_weight( + ComputeGraph& graph, + const ValueRef qmat2_data) { + std::vector qmat2_orig_sizes = graph.sizes_of(qmat2_data); + const int64_t ndim = graph.dim_of(qmat2_data); + + const int64_t qmat2_width = qmat2_orig_sizes.at(ndim - 1); + const int64_t qmat2_height = qmat2_orig_sizes.at(ndim - 2); + + const int64_t K = qmat2_width * 2; + const int64_t N = qmat2_height; + + VK_CHECK_COND(K % 8 == 0); + + const int64_t N_per_block = 8; + const int64_t K_per_block = 4; + + const int64_t num_blocks_K = utils::div_up(K, K_per_block); + const int64_t num_blocks_N = utils::div_up(N, N_per_block); + + const int64_t output_height = num_blocks_N; + const int64_t output_width = num_blocks_K * 4; + + utils::ivec2 orig_sizes = { + utils::safe_downcast(K), utils::safe_downcast(N)}; + + std::vector qmat2_sizes{output_height, output_width}; + + utils::StorageType storage_type = utils::kTexture2D; + uint32_t max_extent = graph.context()->adapter_ptr()->max_texture2d_dim(); + if (output_width > max_extent * 4 || output_height > max_extent) { + storage_type = utils::kBuffer; + } + + std::string kernel_name = "pack_q4_linear_weight"; + add_storage_type_suffix(kernel_name, storage_type); + + ValueRef cached = graph.get_cached_prepack(qmat2_data, kernel_name); + if (is_valid(cached)) { + return cached; + } + + ValueRef qmat2 = graph.add_tensor( + qmat2_sizes, vkapi::kInt, storage_type, utils::kWidthPacked); + + const GlobalWorkGrid gwg( + {utils::safe_downcast(utils::div_up(num_blocks_K, int64_t(2))), + utils::safe_downcast(num_blocks_N), + 1u}, + kTiledWorkGrid); + + graph.prepack_nodes().emplace_back(new PrepackNode( + graph, + VK_KERNEL_FROM_STR(kernel_name), + gwg, + graph.create_lwg(gwg), + qmat2_data, + qmat2, + + {}, + + {}, + + {graph.sizes_pc_of(qmat2), + PushConstantDataInfo(&orig_sizes, sizeof(utils::ivec2))})); + + graph.cache_prepack(qmat2_data, kernel_name, qmat2); + return qmat2; +} + +void add_legacy_q4gsw_linear_node( + ComputeGraph& graph, + const ValueRef fp_input, + const ValueRef weight_data, + const ValueRef weight_scales_data, + const ValueRef group_size_ref, + const ValueRef bias_data, + const ValueRef output) { + std::vector input_sizes = graph.sizes_of(fp_input); + const int64_t K = utils::val_at(-1, input_sizes); + + VK_CHECK_COND(K % 4 == 0); + + const ValueRef packed_weight = + legacy_prepack_q4gsw_weight(graph, weight_data); + const ValueRef packed_weight_scales = prepack_standard( + graph, weight_scales_data, utils::kBuffer, utils::kWidthPacked); + + TmpTensor dummy_bias( + &graph, {}, graph.dtype_of(output), utils::kBuffer, utils::kWidthPacked); + ValueRef packed_bias = dummy_bias.vref; + uint32_t apply_bias = 0; + if (graph.val_is_not_none(bias_data)) { + packed_bias = + prepack_standard(graph, bias_data, utils::kBuffer, utils::kWidthPacked); + apply_bias = 1; + } + + const int32_t group_size_val = graph.extract_scalar(group_size_ref); + const int32_t K4_per_group = utils::div_up(group_size_val, int32_t(4)); + + vkapi::ParamsBindList param_buffers = { + graph.sizes_ubo(output), graph.sizes_ubo(fp_input)}; + + graph.execute_nodes().emplace_back(new DynamicDispatchNode( + graph, + legacy_q4gsw_pick_shader, + legacy_q4gsw_gwg, + legacy_q4gsw_lwg, + + {{output, vkapi::kWrite}, + {{fp_input, packed_weight, packed_weight_scales, packed_bias}, + vkapi::kRead}}, + + param_buffers, + + {}, + + {apply_bias, K4_per_group}, + + {kDummyValueRef, weight_data}, + legacy_q4gsw_resize_linear_node)); +} + +void add_q4gsw_linear_coop_kc_forced_node( + ComputeGraph& graph, + const ValueRef fp_input, + const ValueRef weight_data, + const ValueRef weight_scales_data, + const ValueRef group_size_ref, + const ValueRef bias_data, + const ValueRef output, + TestKernelKind kind) { + const vkapi::ScalarType in_dtype = graph.dtype_of(fp_input); + + const int64_t group_size_val = graph.extract_scalar(group_size_ref); + + std::vector weight_sizes = graph.sizes_of(weight_data); + const int64_t K = weight_sizes.at(1) * 2; + const uint32_t K_val = static_cast(K); + + const ValueRef packed_weight_kc = + prepack_q4_w_4x8_nc_buffer(graph, weight_data); + const ValueRef packed_scales = + prepack_q4_scales(graph, weight_scales_data, in_dtype); + + TmpTensor dummy_bias( + &graph, {}, graph.dtype_of(output), utils::kBuffer, utils::kWidthPacked); + ValueRef packed_bias = dummy_bias.vref; + uint32_t apply_bias = 0; + if (graph.val_is_not_none(bias_data)) { + packed_bias = + prepack_standard(graph, bias_data, utils::kBuffer, utils::kWidthPacked); + apply_bias = 1; + } + + TmpTensor dummy_transposed_input( + &graph, {}, in_dtype, utils::kBuffer, utils::kWidthPacked); + + using PickShaderFn = vkapi::ShaderInfo (*)( + ComputeGraph*, + const std::vector&, + const std::vector&); + using PickGwgFn = GlobalWorkGrid (*)( + ComputeGraph*, + const vkapi::ShaderInfo&, + const std::vector&, + const std::vector&); + using PickLwgFn = LocalWorkGroup (*)( + ComputeGraph*, + const vkapi::ShaderInfo&, + const GlobalWorkGrid&, + const std::vector&, + const std::vector&); + + PickShaderFn pick_shader = nullptr; + PickGwgFn pick_gwg = nullptr; + PickLwgFn pick_lwg = nullptr; + + switch (kind) { + case TestKernelKind::GEMV_COOP_W_4X8_NC_BUFFER: + case TestKernelKind::GEMV_COOP_W_4X8_NC_BUFFER_G1W64: + pick_shader = + pick_forced_shader_coop_kc; + pick_gwg = pick_q4gsw_coop_gwg<1u, 64u>; + pick_lwg = pick_q4gsw_coop_lwg<1u, 64u>; + break; + case TestKernelKind::GEMV_COOP_W_4X8_NC_BUFFER_G4W16: + pick_shader = pick_forced_shader_coop_kc< + TestKernelKind::GEMV_COOP_W_4X8_NC_BUFFER_G4W16>; + pick_gwg = pick_q4gsw_coop_gwg<4u, 16u>; + pick_lwg = pick_q4gsw_coop_lwg<4u, 16u>; + break; + case TestKernelKind::GEMV_COOP_W_4X8_NC_BUFFER_G8W8: + pick_shader = pick_forced_shader_coop_kc< + TestKernelKind::GEMV_COOP_W_4X8_NC_BUFFER_G8W8>; + pick_gwg = pick_q4gsw_coop_gwg<8u, 8u>; + pick_lwg = pick_q4gsw_coop_lwg<8u, 8u>; + break; + default: + VK_THROW("add_q4gsw_linear_coop_kc_forced_node: non-coop kind"); + } + + graph.execute_nodes().emplace_back(new DynamicDispatchNode( + graph, + pick_shader, + pick_gwg, + pick_lwg, + {{output, vkapi::kWrite}, + {{fp_input, + dummy_transposed_input.vref, + packed_weight_kc, + packed_scales, + packed_bias}, + vkapi::kRead}}, + {graph.sizes_ubo(output), graph.sizes_ubo(fp_input)}, + {}, + {apply_bias, K_val, static_cast(group_size_val)}, + {weight_data, fp_input}, + resize_q4gsw_linear_node)); +} + +void add_q4gsw_linear_forced_node( + ComputeGraph& graph, + const ValueRef fp_input, + const ValueRef weight_data, + const ValueRef weight_scales_data, + const ValueRef group_size_ref, + const ValueRef bias_data, + const ValueRef output, + TestKernelKind kind) { + const vkapi::ScalarType in_dtype = graph.dtype_of(fp_input); + + const int64_t group_size_val = graph.extract_scalar(group_size_ref); + + std::vector weight_sizes = graph.sizes_of(weight_data); + const int64_t K = weight_sizes.at(1) * 2; + const uint32_t K_val = static_cast(K); + + const ValueRef packed_weight = prepack_q4_w_4x8_nc_buffer(graph, weight_data); + const ValueRef packed_scales = + prepack_q4_scales(graph, weight_scales_data, in_dtype); + + TmpTensor dummy_bias( + &graph, {}, graph.dtype_of(output), utils::kBuffer, utils::kWidthPacked); + ValueRef packed_bias = dummy_bias.vref; + uint32_t apply_bias = 0; + if (graph.val_is_not_none(bias_data)) { + packed_bias = + prepack_standard(graph, bias_data, utils::kBuffer, utils::kWidthPacked); + apply_bias = 1; + } + + const bool need_transpose = (kind == TestKernelKind::GEMM_TIN_W_4X8); + + std::vector in_sizes = graph.sizes_of(fp_input); + const uint32_t M_val = + utils::safe_downcast(utils::val_at(-2, in_sizes)); + const int64_t M4 = (static_cast(M_val) + 3) / 4; + + TmpTensor dummy_transposed_input( + &graph, {}, in_dtype, utils::kBuffer, utils::kWidthPacked); + TmpTensor real_transposed_input( + &graph, + {static_cast(K_val) * M4 * 4}, + in_dtype, + utils::kBuffer, + utils::kWidthPacked); + + ValueRef transposed_input_ref; + if (need_transpose) { + transposed_input_ref = real_transposed_input.vref; + add_transpose_cast_contig_to_vectorized_node( + graph, fp_input, transposed_input_ref); + } else { + transposed_input_ref = dummy_transposed_input.vref; + } + + using PickShaderFn = vkapi::ShaderInfo (*)( + ComputeGraph*, + const std::vector&, + const std::vector&); + using PickGwgFn = GlobalWorkGrid (*)( + ComputeGraph*, + const vkapi::ShaderInfo&, + const std::vector&, + const std::vector&); + using PickLwgFn = LocalWorkGroup (*)( + ComputeGraph*, + const vkapi::ShaderInfo&, + const GlobalWorkGrid&, + const std::vector&, + const std::vector&); + + PickShaderFn pick_shader = nullptr; + PickGwgFn pick_gwg = nullptr; + PickLwgFn pick_lwg = nullptr; + + switch (kind) { + case TestKernelKind::GEMM_W_4X8: + pick_shader = pick_forced_shader; + pick_gwg = pick_q4gsw_linear_gemm_gwg; + pick_lwg = pick_q4gsw_linear_gemm_lwg; + break; + case TestKernelKind::GEMV_W_4X8: + pick_shader = pick_forced_shader; + pick_gwg = pick_q4gsw_legacy_gemv_gwg; + pick_lwg = pick_q4gsw_legacy_gemv_lwg; + break; + case TestKernelKind::GEMM_TIN_W_4X8: + pick_shader = pick_forced_shader; + pick_gwg = pick_q4gsw_linear_tin_gemm_gwg; + pick_lwg = pick_q4gsw_linear_tin_gemm_lwg; + break; + case TestKernelKind::GEMV_W_4X8_NOSG: + pick_shader = pick_forced_shader; + pick_gwg = pick_q4gsw_legacy_gemv_gwg; + pick_lwg = pick_q4gsw_legacy_gemv_lwg; + break; + case TestKernelKind::PROD: + default: + VK_THROW("PROD kind must be dispatched via production entry points"); + } + + graph.execute_nodes().emplace_back(new DynamicDispatchNode( + graph, + pick_shader, + pick_gwg, + pick_lwg, + {{output, vkapi::kWrite}, + {{fp_input, + transposed_input_ref, + packed_weight, + packed_scales, + packed_bias}, + vkapi::kRead}}, + {graph.sizes_ubo(output), graph.sizes_ubo(fp_input)}, + {}, + {apply_bias, K_val, static_cast(group_size_val)}, + {weight_data, fp_input}, + resize_q4gsw_linear_node)); +} + +void add_fpa_q4gsw_linear_node( + ComputeGraph& graph, + const ValueRef fp_input, + const ValueRef weight_data, + const ValueRef weight_scales_data, + const ValueRef group_size_ref, + const ValueRef bias_data, + int32_t impl_selector_int, + bool is_gemv, + const ValueRef output) { + TestKernelKind kind = selector_to_kind(impl_selector_int, is_gemv); + + if (kind == TestKernelKind::PROD) { + std::vector q4gsw_linear_args = { + fp_input, + weight_data, + weight_scales_data, + group_size_ref, + bias_data, + output}; + VK_GET_OP_FN("et_vk.q4gsw_linear.default")(graph, q4gsw_linear_args); + return; + } + + if (kind == TestKernelKind::LEGACY) { + add_legacy_q4gsw_linear_node( + graph, + fp_input, + weight_data, + weight_scales_data, + group_size_ref, + bias_data, + output); + return; + } + + if (kind == TestKernelKind::GEMV_COOP_W_4X8_NC_BUFFER || + kind == TestKernelKind::GEMV_COOP_W_4X8_NC_BUFFER_G1W64 || + kind == TestKernelKind::GEMV_COOP_W_4X8_NC_BUFFER_G4W16 || + kind == TestKernelKind::GEMV_COOP_W_4X8_NC_BUFFER_G8W8) { + add_q4gsw_linear_coop_kc_forced_node( + graph, + fp_input, + weight_data, + weight_scales_data, + group_size_ref, + bias_data, + output, + kind); + return; + } + + add_q4gsw_linear_forced_node( + graph, + fp_input, + weight_data, + weight_scales_data, + group_size_ref, + bias_data, + output, + kind); +} + +} // namespace + +namespace shader_profiling { +namespace { +template +Json build_q4_route( + ComputeGraph& graph, + const std::vector& args, + const Json& options) { + if (!options.empty()) + throw std::invalid_argument("Q4 routes accept no options"); + if (args.size() != 6) + throw std::invalid_argument("Q4 route requires six canonical arguments"); + const auto* adapter = graph.context()->adapter_ptr(); + if constexpr (Selector == 1 && Gemv) { + if (!adapter->supports_required_subgroup_size_for_compute() || + adapter->min_subgroup_size() > 64 || + adapter->max_subgroup_size() < 64 || + !adapter->supports_subgroup_compute_basic() || + !adapter->supports_subgroup_compute_ballot() || + !adapter->supports_subgroup_compute_shuffle()) + throw UnsupportedRoute( + "gemv_sg requires supported subgroup64 basic/ballot/shuffle"); + } + if constexpr ((Selector == 1 || Selector == 2) && Gemv) { + if (adapter->max_compute_workgroup_invocations() < 256 || + adapter->max_compute_workgroup_size()[0] < 64 || + adapter->max_compute_workgroup_size()[1] < 4) + throw UnsupportedRoute("Legacy GEMV requires a 64x4 workgroup"); + } + if constexpr (Gemv) { + if (graph.sizes_of(args[0]).at(0) != 1) + throw UnsupportedRoute("GEMV route requires M=1"); + } + add_fpa_q4gsw_linear_node( + graph, + args[0], + args[1], + args[2], + args[3], + args[4], + Selector, + Gemv, + args[5]); + const char* name = Selector == 3 ? "legacy" + : Gemv ? (Selector == 1 ? "gemv_sg" + : Selector == 2 ? "gemv_nosg" + : Selector == 13 ? "gemv_coop" + : Selector == 14 ? "gemv_g1w64" + : Selector == 15 ? "gemv_g4w16" + : "gemv_g8w8") + : (Selector == 1 ? "gemm" : "gemm_tin"); + return {{"name", name}, {"options", options}}; +} +REGISTER_OPERATOR_ROUTES { + REGISTER_OPERATOR_ROUTE( + et_vk.q4gsw_linear.default, gemm, (build_q4_route<1, false>)); + REGISTER_OPERATOR_ROUTE( + et_vk.q4gsw_linear.default, gemm_tin, (build_q4_route<2, false>)); + REGISTER_OPERATOR_ROUTE( + et_vk.q4gsw_linear.default, legacy, (build_q4_route<3, false>)); + REGISTER_OPERATOR_ROUTE( + et_vk.q4gsw_linear.default, gemv_sg, (build_q4_route<1, true>)); + REGISTER_OPERATOR_ROUTE( + et_vk.q4gsw_linear.default, gemv_nosg, (build_q4_route<2, true>)); + REGISTER_OPERATOR_ROUTE( + et_vk.q4gsw_linear.default, gemv_coop, (build_q4_route<13, true>)); + REGISTER_OPERATOR_ROUTE( + et_vk.q4gsw_linear.default, gemv_g1w64, (build_q4_route<14, true>)); + REGISTER_OPERATOR_ROUTE( + et_vk.q4gsw_linear.default, gemv_g4w16, (build_q4_route<15, true>)); + REGISTER_OPERATOR_ROUTE( + et_vk.q4gsw_linear.default, gemv_g8w8, (build_q4_route<16, true>)); +} +} // namespace +} // namespace shader_profiling +} // namespace vkcompute diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/coopmat_correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/coopmat_correctness.json new file mode 100644 index 00000000000..0934e94faf1 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/coopmat_correctness.json @@ -0,0 +1,1466 @@ +[ + { + "name": "q4gsw_linear coopmat_correctness [64, 128, 64, 64]", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.5, + "rtol": 0.05 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "repeat": [0.5, 0.625, 0.75, 0.875, 1.0, 1.125, 1.25, 1.375] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [153, 170, 187, 204, 221, 238] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [64, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_correctness [64, 128, 256, 64]", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.5, + "rtol": 0.05 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "repeat": [0.5, 0.625, 0.75, 0.875, 1.0, 1.125, 1.25, 1.375] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [153, 170, 187, 204, 221, 238] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 256], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [64, 256], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_correctness [128, 256, 128, 64]", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.5, + "rtol": 0.05 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 256], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "repeat": [0.5, 0.625, 0.75, 0.875, 1.0, 1.125, 1.25, 1.375] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [153, 170, 187, 204, 221, 238] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [128, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_correctness [128, 128, 128, 64]", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.5, + "rtol": 0.05 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "repeat": [0.5, 0.625, 0.75, 0.875, 1.0, 1.125, 1.25, 1.375] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [153, 170, 187, 204, 221, 238] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [128, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_correctness [128, 128, 256, 64]", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.5, + "rtol": 0.05 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "repeat": [0.5, 0.625, 0.75, 0.875, 1.0, 1.125, 1.25, 1.375] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [153, 170, 187, 204, 221, 238] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 256], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [128, 256], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_correctness [256, 256, 256, 64]", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.5, + "rtol": 0.05 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [256, 256], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "repeat": [0.5, 0.625, 0.75, 0.875, 1.0, 1.125, 1.25, 1.375] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [153, 170, 187, 204, 221, 238] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 256], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [256, 256], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_correctness [256, 128, 128, 64]", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.5, + "rtol": 0.05 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [256, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "repeat": [0.5, 0.625, 0.75, 0.875, 1.0, 1.125, 1.25, 1.375] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [153, 170, 187, 204, 221, 238] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [256, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_correctness [256, 128, 64, 64]", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.5, + "rtol": 0.05 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [256, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "repeat": [0.5, 0.625, 0.75, 0.875, 1.0, 1.125, 1.25, 1.375] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [153, 170, 187, 204, 221, 238] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [256, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_correctness [64, 128, 64, 64] (2)", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.5, + "rtol": 0.05 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "repeat": [0.5, 0.625, 0.75, 0.875, 1.0, 1.125, 1.25, 1.375] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0.0625] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 0] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [153, 170, 187, 204, 221, 238] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 64 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [64, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_correctness [64, 128, 256, 64] (2)", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.5, + "rtol": 0.05 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "repeat": [0.5, 0.625, 0.75, 0.875, 1.0, 1.125, 1.25, 1.375] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0.0625] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 0] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [153, 170, 187, 204, 221, 238] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 256], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 64 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 256], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [64, 256], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_correctness [128, 256, 128, 64] (2)", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.5, + "rtol": 0.05 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 256], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "repeat": [0.5, 0.625, 0.75, 0.875, 1.0, 1.125, 1.25, 1.375] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0.0625] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 0] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [153, 170, 187, 204, 221, 238] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 64 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [128, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_correctness [128, 128, 128, 64] (2)", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.5, + "rtol": 0.05 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "repeat": [0.5, 0.625, 0.75, 0.875, 1.0, 1.125, 1.25, 1.375] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0.0625] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 0] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [153, 170, 187, 204, 221, 238] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 64 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [128, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_correctness [128, 128, 256, 64] (2)", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.5, + "rtol": 0.05 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "repeat": [0.5, 0.625, 0.75, 0.875, 1.0, 1.125, 1.25, 1.375] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0.0625] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 0] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [153, 170, 187, 204, 221, 238] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 256], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 64 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 256], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [128, 256], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_correctness [256, 256, 256, 64] (2)", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.5, + "rtol": 0.05 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [256, 256], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "repeat": [0.5, 0.625, 0.75, 0.875, 1.0, 1.125, 1.25, 1.375] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 256], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0.0625] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 256], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 0] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [153, 170, 187, 204, 221, 238] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 256], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 64 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 256], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [256, 256], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_correctness [256, 128, 128, 64] (2)", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.5, + "rtol": 0.05 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [256, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "repeat": [0.5, 0.625, 0.75, 0.875, 1.0, 1.125, 1.25, 1.375] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 256], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0.0625] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 256], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 0] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [153, 170, 187, 204, 221, 238] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 64 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [256, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_correctness [256, 128, 64, 64] (2)", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.5, + "rtol": 0.05 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [256, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "repeat": [0.5, 0.625, 0.75, 0.875, 1.0, 1.125, 1.25, 1.375] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 256], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0.0625] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 256], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 0] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [153, 170, 187, 204, 221, 238] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 64 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [256, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/coopmat_performance.json b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/coopmat_performance.json new file mode 100644 index 00000000000..31f8f4feb24 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/coopmat_performance.json @@ -0,0 +1,734 @@ +[ + { + "name": "q4gsw_linear coopmat_performance [1024, 4096, 4096, 128]", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1024, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [4096, 2048], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 4096], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 128 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1024, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_performance [1024, 4096, 1024, 128]", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1024, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1024, 2048], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 1024], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 128 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1024, 1024], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_performance [1024, 4096, 14336, 128]", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1024, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [14336, 2048], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 14336], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 128 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1024, 14336], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_performance [1024, 14336, 4096, 128]", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1024, 14336], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [4096, 7168], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [112, 4096], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 128 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1024, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_performance [1024, 4096, 4096, 128] (2)", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1024, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 1024], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 1024], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [4096, 2048], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32, 4096], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 128 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 4096], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 128 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1024, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_performance [1024, 4096, 1024, 128] (2)", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1024, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 1024], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 1024], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1024, 2048], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32, 1024], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 128 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 1024], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 128 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1024, 1024], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_performance [1024, 4096, 14336, 128] (2)", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1024, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 1024], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 1024], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [14336, 2048], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32, 14336], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 128 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 14336], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 128 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1024, 14336], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_performance [1024, 14336, 4096, 128] (2)", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1024, 14336], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 1024], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 1024], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [4096, 7168], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [112, 4096], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 128 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [112, 4096], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 128 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1024, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/correctness.json new file mode 100644 index 00000000000..5d452a04a64 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/correctness.json @@ -0,0 +1,6290 @@ +[ + { + "name": "q4gsw_linear correctness [4, 64, 32, 16] float32 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 4], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 16 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 16 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [32, 64, 32, 16] float32 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 32], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 16 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 16 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [4, 128, 64, 32] float32 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 4], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [32, 128, 64, 32] float32 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 32], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [256, 128, 128, 32] float32 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [256, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 256], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [256, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness default M2 biasFalse float32 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [2, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 2], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 2], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [2, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness default M3 biasFalse float32 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [3, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 3], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 3], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [3, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness default M5 biasFalse float32 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [5, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 5], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 5], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [5, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [4, 256, 128, 64] float32 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 4], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 64 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [32, 256, 128, 64] float32 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 32], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 64 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [64, 64, 64, 64] float32 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [1, 64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 64 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [1, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [64, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [64, 128, 64, 64] float32 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 64 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [64, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [64, 256, 128, 128] float32 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 128 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 128 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [64, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [4, 64, 32, 16] float16 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 4], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 4], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 16 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 32], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 16 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4, 32], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [32, 64, 32, 16] float16 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 32], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 32], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 16 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 32], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 16 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 32], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [64, 64, 64, 64] float16 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [1, 64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 64 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [1, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [64, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness default M2 biasFalse float16 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [2, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 2], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 2], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 32], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [2, 32], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness default M3 biasFalse float16 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [3, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 3], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 3], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 32], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [3, 32], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness default M5 biasFalse float16 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [5, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 5], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 5], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 32], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [5, 32], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [4, 128, 64, 32] float16 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.1313708498984762, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 4], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 4], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [32, 128, 64, 32] float16 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.1313708498984762, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 32], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 32], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [256, 128, 128, 32] float16 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.1313708498984762, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [256, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 256], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 256], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [256, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [64, 128, 64, 64] float16 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.1313708498984762, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 64 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [64, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [4, 256, 128, 64] float16 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.6, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 256], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 4], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 4], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 64 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [32, 256, 128, 64] float16 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.6, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 256], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 32], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 32], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 64 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [64, 256, 128, 128] float16 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.6, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 256], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 128 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 128 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [64, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [4, 64, 32, 16] (3)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 16 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [32, 64, 32, 16] (3)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 16 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [4, 128, 64, 32] (3)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [32, 128, 64, 32] (3)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [256, 128, 128, 32] (3)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [256, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [256, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [4, 256, 128, 64] (3)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [32, 256, 128, 64] (3)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [64, 64, 64, 64] (3)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [1, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [64, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [64, 128, 64, 64] (3)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [64, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [64, 256, 128, 128] (3)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 128 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [64, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [4, 64, 32, 16] float16 width (36)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 32], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 16 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4, 32], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [32, 64, 32, 16] float16 width (37)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 32], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 16 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 32], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [64, 64, 64, 64] float16 width (38)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [1, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [64, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [4, 128, 64, 32] float16 width (39)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.1313708498984762, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [32, 128, 64, 32] float16 width (40)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.1313708498984762, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [256, 128, 128, 32] float16 width (41)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.1313708498984762, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [256, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [256, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [64, 128, 64, 64] float16 width (42)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.1313708498984762, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [64, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [4, 256, 128, 64] float16 width (43)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.6, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 256], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [32, 256, 128, 64] float16 width (44)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.6, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 256], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [64, 256, 128, 128] float16 width (45)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.6, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 256], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 128 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [64, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [4, 64, 32, 16, True] float32 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 4], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 16 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 16 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [4, 128, 64, 32, True] float32 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 4], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [32, 128, 64, 32, True] float32 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 32], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [32, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness default M2 biasTrue float32 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [2, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 2], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 2], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [2, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness default M3 biasTrue float32 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [3, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 3], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 3], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [3, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness default M5 biasTrue float32 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [5, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 5], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 5], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [5, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [4, 64, 32, 16, True] float16 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 4], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 4], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 16 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 32], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 16 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4, 32], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness default M2 biasTrue float16 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [2, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 2], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 2], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 32], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [2, 32], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness default M3 biasTrue float16 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [3, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 3], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 3], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 32], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [3, 32], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness default M5 biasTrue float16 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [5, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 5], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 5], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 32], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [5, 32], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [4, 128, 64, 32, True] float16 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.1313708498984762, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 4], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 4], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [32, 128, 64, 32, True] float16 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.1313708498984762, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 32], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 32], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [32, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [4, 64, 32, 16, True] (3)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 16 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [4, 128, 64, 32, True] (3)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [32, 128, 64, 32, True] (3)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [32, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [4, 64, 32, 16, True] float16 width (61)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 32], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 16 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4, 32], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [4, 128, 64, 32, True] float16 width (62)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.1313708498984762, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [32, 128, 64, 32, True] float16 width (63)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.1313708498984762, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [32, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/debug.json b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/debug.json new file mode 100644 index 00000000000..93ea2bda3f2 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/debug.json @@ -0,0 +1,124 @@ +[ + { + "name": "q4gsw_linear debug", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [8, 16], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 8], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 8], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 8], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 16], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 8 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 8 + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [8, 16], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/fpa_correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/fpa_correctness.json new file mode 100644 index 00000000000..db32654242e --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/fpa_correctness.json @@ -0,0 +1,1458 @@ +[ + { + "name": "q4gsw_linear fpa_correctness M1 K128 N128 float32", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemv_nosg", "gemv_coop"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_correctness M1 K128 N256 float32", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["gemv_g1w64", "gemv_g4w16", "gemv_g8w8"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_correctness M32 K128 N12 float32", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemm", "gemm_tin"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [12, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 12], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 12], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_correctness M32 K128 N20 float32", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemm", "gemm_tin"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [20, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 20], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 20], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_correctness M32 K128 N128 float32 texture3d selector0", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.001 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_correctness M31 K128 N128 float32 texture3d selector0", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.001 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [31, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [31, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_correctness M33 K128 N128 float32 texture3d selector0", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.001 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [33, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [33, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_correctness M1 K128 N128 float16", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.4, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemv_nosg", "gemv_coop"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_correctness M1 K128 N256 float16", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.4, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["gemv_g1w64", "gemv_g4w16", "gemv_g8w8"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 256], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 256], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_correctness M32 K128 N12 float16", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.4, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemm", "gemm_tin"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [12, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 12], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 12], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_correctness M32 K128 N20 float16", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.4, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemm", "gemm_tin"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [20, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 20], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 20], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_correctness M32 K128 N128 float16 texture3d selector0", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.4, + "rtol": 0.001 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_correctness M31 K128 N128 float16 texture3d selector0", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.4, + "rtol": 0.001 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [31, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [31, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_correctness M33 K128 N128 float16 texture3d selector0", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.4, + "rtol": 0.001 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [33, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [33, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_correctness M1 K256 N256 float32", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.2, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemv_nosg", "gemv_coop"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [8, 256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_correctness M1 K256 N128 float32", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.2, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["gemv_g1w64", "gemv_g4w16", "gemv_g8w8"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [8, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_correctness M1 K64 N128 float16", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.2, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["gemv_g1w64", "gemv_g4w16", "gemv_g8w8"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_correctness M1 K256 N256 float16", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.8, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemv_nosg", "gemv_coop"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [8, 256], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 256], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_correctness M1 K256 N128 float16", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.8, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["gemv_g1w64", "gemv_g4w16", "gemv_g8w8"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [8, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_correctness M1 K64 N128 float32", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.05, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["gemv_g1w64", "gemv_g4w16", "gemv_g8w8"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/fpa_performance.json b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/fpa_performance.json new file mode 100644 index 00000000000..3607514c87a --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/fpa_performance.json @@ -0,0 +1,1897 @@ +[ + { + "name": "q4gsw_linear fpa_performance M1 K1024 N2048 float32", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.8, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemv_sg", "gemv_nosg", "legacy", "gemv_coop"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 1024], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 512], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 2048], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M32 K1024 N2048 float32", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.8, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemm", "gemm_tin", "legacy"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 1024], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 512], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 2048], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M128 K1024 N2048 float32", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.8, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemm", "gemm_tin", "legacy"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 1024], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 512], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [128, 2048], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M256 K1024 N2048 float32", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.8, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemm", "gemm_tin", "legacy"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [256, 1024], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 512], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [256, 2048], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M1 K1024 N2048 float16 buffer", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 3.2, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemv_sg", "gemv_nosg", "legacy", "gemv_coop"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 1024], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 512], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 2048], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 2048], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M1 K1024 N2048 float16 texture3d", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 3.2, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemv_sg", "gemv_nosg", "legacy", "gemv_coop"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 1024], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 512], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 2048], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 2048], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 1024], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 512], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 2048], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 2048], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M1 K1024 N1024 float16 texture3d", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 3.2, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemv_sg", "gemv_nosg", "legacy", "gemv_coop"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 1024], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1024, 512], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 1024], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 1024], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M1 K1024 N3072 float16 texture3d", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 3.2, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemv_sg", "gemv_nosg", "legacy", "gemv_coop"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 1024], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [3072, 512], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 3072], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 3072], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M32 K1024 N2048 float16", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 3.2, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemm", "gemm_tin", "legacy"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 1024], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 512], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 2048], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 2048], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M128 K1024 N2048 float16", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 3.2, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemm", "gemm_tin", "legacy"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 1024], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 512], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 2048], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [128, 2048], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M256 K1024 N2048 float16", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 3.2, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemm", "gemm_tin", "legacy"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [256, 1024], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 512], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 2048], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [256, 2048], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M1 K4096 N4096 float32", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 3.2, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemv_sg", "gemv_nosg", "legacy", "gemv_coop"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4096], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [4096, 2048], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [128, 4096], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 4096], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M32 K4096 N4096 float32", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 3.2, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemm", "gemm_tin", "legacy"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 4096], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [4096, 2048], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [128, 4096], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 4096], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M128 K4096 N4096 float32", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 3.2, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemm", "gemm_tin", "legacy"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 4096], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [4096, 2048], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [128, 4096], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [128, 4096], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M256 K4096 N4096 float32", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 3.2, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemm", "gemm_tin", "legacy"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [256, 4096], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [4096, 2048], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [128, 4096], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [256, 4096], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M1 K4096 N4096 float16", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 12.8, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemv_sg", "gemv_nosg", "legacy", "gemv_coop"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [4096, 2048], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [128, 4096], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M32 K4096 N4096 float16", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 12.8, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemm", "gemm_tin", "legacy"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [4096, 2048], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [128, 4096], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M128 K4096 N4096 float16", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 12.8, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemm", "gemm_tin", "legacy"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [4096, 2048], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [128, 4096], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [128, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M256 K4096 N4096 float16", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 12.8, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemm", "gemm_tin", "legacy"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [256, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [4096, 2048], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [128, 4096], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [256, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M1 K2048 N512 float16 texture3d", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 6.4, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemv_sg", "gemv_nosg", "legacy", "gemv_coop"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 2048], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [512, 1024], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [64, 512], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 512], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M1 K2048 N2048 float16 texture3d", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 6.4, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemv_sg", "gemv_nosg", "legacy", "gemv_coop"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 2048], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 1024], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [64, 2048], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 2048], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M1 K2048 N8192 float16 texture3d", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 6.4, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemv_sg", "gemv_nosg", "legacy", "gemv_coop"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 2048], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [8192, 1024], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [64, 8192], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 8192], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M1 K8192 N2048 float16 texture3d", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 25.6, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemv_sg", "gemv_nosg", "legacy", "gemv_coop"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8192], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 4096], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [256, 2048], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 2048], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M1 K3072 N1024 float16 texture3d", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 9.600000000000001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemv_sg", "gemv_nosg", "legacy", "gemv_coop"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3072], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1024, 1536], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [96, 1024], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 1024], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M1 K2048 N2048 float32", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.6, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemv_sg", "gemv_nosg", "legacy", "gemv_coop"], + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 2048], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 1024], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [64, 2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 2048], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/performance.json b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/performance.json new file mode 100644 index 00000000000..3a23488efde --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/performance.json @@ -0,0 +1,734 @@ +[ + { + "name": "q4gsw_linear performance [128, 2048, 2048, 128] float32 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 2048], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 1024], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [16, 2048], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 128 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [16, 2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 128 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [128, 2048], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear performance [1024, 4096, 4096, 128] float32 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1024, 4096], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 1024], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 1024], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [4096, 2048], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32, 4096], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 128 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 4096], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 128 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1024, 4096], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear performance [128, 2048, 2048, 128] float16 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 4.525483399593905, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 2048], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 1024], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [16, 2048], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 128 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [16, 2048], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 128 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [128, 2048], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear performance [1024, 4096, 4096, 128] float16 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 6.4, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1024, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 1024], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 1024], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [4096, 2048], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32, 4096], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 128 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 4096], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 128 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1024, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear performance [128, 2048, 2048, 128] (3)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 2048], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 1024], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [16, 2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 128 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [128, 2048], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear performance [1024, 4096, 4096, 128] (3)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1024, 4096], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [4096, 2048], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 4096], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 128 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1024, 4096], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear performance [128, 2048, 2048, 128] float16 width (6)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 4.525483399593905, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 2048], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 1024], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [16, 2048], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 128 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [128, 2048], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear performance [1024, 4096, 4096, 128] float16 width (7)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 6.4, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1024, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [4096, 2048], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 4096], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 128 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1024, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/glsl/q4gsw_linear_gemv__w_4x8.glsl b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/glsl/q4gsw_linear_gemv__w_4x8.glsl new file mode 100644 index 00000000000..e3c6ccdfba7 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/glsl/q4gsw_linear_gemv__w_4x8.glsl @@ -0,0 +1,371 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +// q4gsw linear GEMV kernel — row-pair broadcast dequant-accumulate over the +// shared w_4x8 weight prepack. +// +// Shader naming convention: +// q4gsw_linear_gemv__w_4x8_[_nosg] +// ^^^^^^^^^^^^^^^^^ ^^^^^ ^^^^^^^ +// op base (gemv) tile tile arrangement (nc or kc) +// +// Weight binding: +// The shared pack_q4_linear_weight__w_4x8 shader writes a W_4X8 block-packed uvec2 buffer +// where the uvec2 at logical tile index [k4, n4] lives at the 2 consecutive +// uint slots: +// t_q4_weights[2 * tile_idx + 0] = .x (row pair {N0, N1}) +// t_q4_weights[2 * tile_idx + 1] = .y (row pair {N2, N3}) +// Read as a scalar uint buffer, the uint at +// word_idx = 2 * tile_idx + half (half in {0, 1}) +// is one N-row-pair's 4 K-step payload. With n2 = 2 * n4 + half and +// k_slot = k4, under WEIGHT_TILE_CONTIG_DIM=0: +// word_idx = k_slot * (N/2) + n2 +// which is the index formula used by the per-row-pair loop here. +// +// Interleaved (dp4a-style) byte-pair layout: each uvec2 lane's 4 bytes hold +// 4 K-consecutive positions for a pair of N rows. As a scalar uint, byte b +// of w_pack = (N_even, K=b) | (N_odd, K=b) << 4 — the low nibble per byte +// is the even-N row, the high nibble is odd-N. This byte packing is the +// natural memory split for the paired-row dequant below and lets the same +// shader body be repurposed later for int8/int4 integer matmul that +// operates directly on byte-interleaved nibble pairs. +// +// Scale binding: +// Uses the production dtype-matched scale bytes but reinterprets them as a +// gvec2 (vec2 / f16vec2) array. The scale prepack emits vec4 bytes indexed +// as t_scales[group_i * N4 + n4] where each vec4 holds 4 N-row scales. The +// same byte layout is addressable as vec2 with index +// vec2_idx = 2 * (group_i * N4 + n4) + half = group_i * N2 + n2 +// since each vec4 = 2 consecutive vec2 slots (low half = rows {2*n4, 2*n4+1}, +// high half = rows {2*n4+2, 2*n4+3}). Binding as vec2 halves the scale load +// byte volume and eliminates the 2 wasted components per load. +// +// WG layout: SUBGROUP_SIZE=64, NUM_SUBGROUPS=4. y-dim splits K-blocks across waves. +// Each thread owns one row-pair (n2) and writes two output floats. + +#version 450 core + +${define_required_extensions(IO_STORAGE, DTYPE)} +${define_required_extensions("buffer", DTYPE)} +#extension GL_EXT_shader_explicit_arithmetic_types_float16 : require +#extension GL_EXT_control_flow_attributes : require +$if USE_SUBGROUP_BROADCAST: + #extension GL_KHR_shader_subgroup_basic : require + #extension GL_KHR_shader_subgroup_ballot : require + #extension GL_KHR_shader_subgroup_shuffle : require + +#define PRECISION ${PRECISION} + +#define T ${texel_load_component_type(DTYPE, "buffer")} + +$if IO_STORAGE == "buffer": + #define IO_BUFFER + +#define NUM_SUBGROUPS ${NUM_SUBGROUPS} +#define SUBGROUP_SIZE ${SUBGROUP_SIZE} +// Workgroup x-dim size — used for shared-mem indexing in the inter-wave +// reduction. Chosen to match LWG.x set by the host (kGemvSubgroupSize=64). +// In the sg variant, lanes happen to align 1:1 with x-threads when +// subgroupSize >= LWG_X_SIZE; in the nosg variant, x-thread index alone +// addresses shared-mem slots so any subgroup width is safe. +#define LWG_X_SIZE ${LWG_X_SIZE} +// Number of K elements processed per outer k-loop iteration. Format-level +// constant — each iteration loads 8 vec4 of activations (= 32 K-vals) and 8 +// uint32 weight packs (4 hi + 4 lo, each holding 4 K-vals × 2 N-rows). Distinct +// from `group_size` (the quantization group): blocks_per_group = group_size / +// K_PER_STEP tells how many consecutive K-blocks share one scale pair. +#define K_PER_STEP 32 + +#define WEIGHT_TILE_CONTIG_DIM ${WEIGHT_TILE_CONTIG_DIM} + +layout(std430) buffer; + +// Unified 6-binding layout shared across q4gsw_linear shaders so a single +// DynamicDispatchNode with pick_shader_fn can switch between GEMM and GEMV +// kernels. This shader reads t_fp_input (the raw activation). The +// t_transposed_input binding is declared to preserve slot order but is never +// referenced here — the driver compiles it out to zero runtime cost; only +// the descriptor slot is allocated. +// +// Output: [1, N] scalar DTYPE buffer OR 1x1xN/4 texture3d. +// is_scalar_array is only meaningful for buffer storage; ignored for texture. +${layout_declare_tensor(B, "w", "t_output", DTYPE, IO_STORAGE, is_scalar_array=True)} +// Activations: [1, K] vec4-packed. +${layout_declare_tensor(B, "r", "t_fp_input", DTYPE, IO_STORAGE, is_scalar_array=False)} +// Unused transposed input — declared only so this shader shares the +// descriptor set layout with the tin GEMM shader. +${layout_declare_tensor(B, "r", "t_transposed_input", DTYPE, "buffer", is_scalar_array=False)} +// Weight: same uvec2 W_4X8 block-packed buffer produced by pack_q4_linear_weight__w_4x8, +// bound here as a scalar uint array so the per-row-pair index math can address +// individual uint slots directly. See header comment for byte-layout proof. +${layout_declare_tensor(B, "r", "t_q4_weights", "int", "buffer")} +// Scales: dtype-matched gvec2 reinterpret of the GEMM vec4 scale prepack. +// Indexed as t_scales[group_idx * N2 + n2]. +${layout_declare_tensor(B, "r", "t_scales", DTYPE, "buffer", is_scalar_array=False, vec_size=2)} +// Bias: [N] DTYPE buffer. +${layout_declare_tensor(B, "r", "t_bias", DTYPE, "buffer", is_scalar_array=True)} + +${layout_declare_ubo(B, "ivec4", "output_sizes")} +${layout_declare_ubo(B, "ivec4", "input_sizes")} + +layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; + +${layout_declare_spec_const(C, "int", "apply_bias", "0")} +// `K` is declared only to keep the spec-constant layout aligned with the GEMM +// shaders so both variants can share a single DynamicDispatchNode with a +// runtime shader picker. It is not referenced in the GEMV body — the local +// `K` (derived from `input_sizes.x`) shadows it inside main(). +${layout_declare_spec_const(C, "int", "K", "1024")} +// Quantization group size in elements. `blocks_per_group` (the original GEMV +// spec constant) is recomputed from `group_size` below since K_PER_STEP = 32. +${layout_declare_spec_const(C, "int", "group_size", "32")} + +// Inter-wave reduction buffer (NUM_SUBGROUPS - 1 slabs of LWG_X_SIZE vec2). +// Slots are addressed by x-thread index, not subgroup lane index — sized to +// LWG.x so the shader is portable across subgroup widths. +shared vec2 partial_sums[LWG_X_SIZE * (NUM_SUBGROUPS - 1)]; + +$if not USE_SUBGROUP_BROADCAST: + // Used by the texture-storage write path to swap acc with the n2-XOR-1 + // partner thread. Replaces subgroupShuffleXor in the nosg variant. + shared vec2 nosg_n2_partner[LWG_X_SIZE]; + +// Load a vec4 of activations from input at (vec4 index) idx. +vec4 load_input_vec4(const int idx) { +#ifdef IO_BUFFER + return vec4(t_fp_input[idx]); +#else + return vec4(texelFetch(t_fp_input, ivec3(idx, 0, 0), 0)); +#endif +} + +// Load 2 scales for (n2, group) directly as a gvec2. +// The scale prepack bytes are reinterpreted as gvec2[group_idx * N2 + n2] +// where gvec2 is vec2 (fp32) or f16vec2 (fp16). The vec2(...) cast is a no-op +// for fp32 and an f16 -> f32 widening for fp16. +vec2 load_scale_pair(const int n2, const int group_idx, const int N2) { + return vec2(t_scales[group_idx * N2 + n2]); +} + +void main() { + $if USE_SUBGROUP_BROADCAST: + // sg path: lane_id == subgroup invocation; relies on subgroupSize == LWG.x + // (=64) so subgroup lane and x-thread coincide for shared-mem indexing. + const uint lane_id = gl_SubgroupInvocationID; + $else: + // nosg path: lane_id == x-thread within workgroup; portable across any + // subgroup width since shared-mem slots are addressed purely by LWG.x. + const uint lane_id = gl_LocalInvocationID.x; + const int k_wave_id = int(gl_LocalInvocationID.y); + const int n2 = int(gl_GlobalInvocationID.x); + + const int N = output_sizes.x; + const int K = input_sizes.x; + const int N2 = N / 2; + const int num_steps = K / K_PER_STEP; + // Derived from the shared `group_size` spec constant. Each K_PER_STEP (=32) + // K-block is one "block"; blocks_per_group tells how many consecutive blocks + // share a single scale pair along K. + const int blocks_per_group = group_size / K_PER_STEP; + + // Words per K-block in the weight buffer. One K-block covers K_PER_STEP K-vals + // = K_PER_STEP/4 k4 slices, and each k4 slice is N2 word-pairs. So + // words_per_k_block = (K_PER_STEP / 4) * N2. + // `k * K_BLOCK_STRIDE_W` gives the absolute word offset to the start of + // K-block `k`. + const int K_BLOCK_STRIDE_W = (K_PER_STEP / 4) * N2; + + if (n2 >= N2) { + return; + } + + vec2 acc = vec2(0.0); + + // Loop over k-blocks, waves split k-blocks (k_wave_id, k_wave_id+NUM_SUBGROUPS, ...). + for (int k = k_wave_id; k < num_steps; k += NUM_SUBGROUPS) { + // --- Load scale pair for this (n2, group) --- + const int group_idx = k / blocks_per_group; + vec2 scale_pair = load_scale_pair(n2, group_idx, N2); + float scale0 = scale_pair.x; + float scale1 = scale_pair.y; + + $if USE_SUBGROUP_BROADCAST: + // --- Load 8 activations per participating lane (only lanes 0..3) --- + // Lanes 0..3 each load 2 vec4s; other lanes receive via subgroupBroadcast + // in the dequant loops below. + vec4 in_vecs[2] = vec4[2](vec4(0.0), vec4(0.0)); + if (lane_id < 4u) { + const int vec4_base = k * 8 + int(lane_id) * 2; + in_vecs[0] = load_input_vec4(vec4_base); + in_vecs[1] = load_input_vec4(vec4_base + 1); + } + $else: + // --- Load all 8 activation vec4s per thread (no subgroup broadcast) --- + // Each thread independently reads the 32 activations (8 vec4) for this + // k-block. All lanes hit the same addresses, so L1 serves ~1 load from + // DRAM per unique address across the wave. + vec4 in_vecs[8]; + const int vec4_base = k * 8; + [[unroll]] for (int i = 0; i < 8; ++i) { + in_vecs[i] = load_input_vec4(vec4_base + i); + } + + // --- Load 4 int32s for the "hi" half (K positions 0..15) --- + int w_pack0 = t_q4_weights[n2 + k * K_BLOCK_STRIDE_W + N2 * 0]; + int w_pack1 = t_q4_weights[n2 + k * K_BLOCK_STRIDE_W + N2 * 1]; + int w_pack2 = t_q4_weights[n2 + k * K_BLOCK_STRIDE_W + N2 * 2]; + int w_pack3 = t_q4_weights[n2 + k * K_BLOCK_STRIDE_W + N2 * 3]; + + // --- Dequant + accumulate for the "hi" block (K = 0..15 of this K_PER_STEP). --- + // Each regA word contains interleaved byte pairs: byte b = (N_even, K=b) + // in the low nibble, (N_odd, K=b) in the high nibble. + float in_val; + + [[unroll]] for (int k4i = 0; k4i < 4; ++k4i) { + $if USE_SUBGROUP_BROADCAST: + in_val = subgroupBroadcast(in_vecs[0][k4i], 0u); + $else: + in_val = in_vecs[0][k4i]; + acc.x += (float(int((uint(w_pack0) >> (8 * k4i)) & 0xFu)) - 8.0) * scale0 * in_val; + acc.y += (float(int((uint(w_pack0) >> (8 * k4i + 4)) & 0xFu)) - 8.0) * scale1 * in_val; + } + + [[unroll]] for (int k4i = 0; k4i < 4; ++k4i) { + $if USE_SUBGROUP_BROADCAST: + in_val = subgroupBroadcast(in_vecs[1][k4i], 0u); + $else: + in_val = in_vecs[1][k4i]; + acc.x += (float(int((uint(w_pack1) >> (8 * k4i)) & 0xFu)) - 8.0) * scale0 * in_val; + acc.y += (float(int((uint(w_pack1) >> (8 * k4i + 4)) & 0xFu)) - 8.0) * scale1 * in_val; + } + + [[unroll]] for (int k4i = 0; k4i < 4; ++k4i) { + $if USE_SUBGROUP_BROADCAST: + in_val = subgroupBroadcast(in_vecs[0][k4i], 1u); + $else: + in_val = in_vecs[2][k4i]; + acc.x += (float(int((uint(w_pack2) >> (8 * k4i)) & 0xFu)) - 8.0) * scale0 * in_val; + acc.y += (float(int((uint(w_pack2) >> (8 * k4i + 4)) & 0xFu)) - 8.0) * scale1 * in_val; + } + + [[unroll]] for (int k4i = 0; k4i < 4; ++k4i) { + $if USE_SUBGROUP_BROADCAST: + in_val = subgroupBroadcast(in_vecs[1][k4i], 1u); + $else: + in_val = in_vecs[3][k4i]; + acc.x += (float(int((uint(w_pack3) >> (8 * k4i)) & 0xFu)) - 8.0) * scale0 * in_val; + acc.y += (float(int((uint(w_pack3) >> (8 * k4i + 4)) & 0xFu)) - 8.0) * scale1 * in_val; + } + + // --- Load 4 int32s for the "lo" half (K positions 16..31). --- + w_pack0 = t_q4_weights[n2 + k * K_BLOCK_STRIDE_W + N2 * 4]; + w_pack1 = t_q4_weights[n2 + k * K_BLOCK_STRIDE_W + N2 * 5]; + w_pack2 = t_q4_weights[n2 + k * K_BLOCK_STRIDE_W + N2 * 6]; + w_pack3 = t_q4_weights[n2 + k * K_BLOCK_STRIDE_W + N2 * 7]; + + // --- Dequant + accumulate for the "lo" block (K = 16..31). --- + [[unroll]] for (int k4i = 0; k4i < 4; ++k4i) { + $if USE_SUBGROUP_BROADCAST: + in_val = subgroupBroadcast(in_vecs[0][k4i], 2u); + $else: + in_val = in_vecs[4][k4i]; + acc.x += (float(int((uint(w_pack0) >> (8 * k4i)) & 0xFu)) - 8.0) * scale0 * in_val; + acc.y += (float(int((uint(w_pack0) >> (8 * k4i + 4)) & 0xFu)) - 8.0) * scale1 * in_val; + } + + [[unroll]] for (int k4i = 0; k4i < 4; ++k4i) { + $if USE_SUBGROUP_BROADCAST: + in_val = subgroupBroadcast(in_vecs[1][k4i], 2u); + $else: + in_val = in_vecs[5][k4i]; + acc.x += (float(int((uint(w_pack1) >> (8 * k4i)) & 0xFu)) - 8.0) * scale0 * in_val; + acc.y += (float(int((uint(w_pack1) >> (8 * k4i + 4)) & 0xFu)) - 8.0) * scale1 * in_val; + } + + [[unroll]] for (int k4i = 0; k4i < 4; ++k4i) { + $if USE_SUBGROUP_BROADCAST: + in_val = subgroupBroadcast(in_vecs[0][k4i], 3u); + $else: + in_val = in_vecs[6][k4i]; + acc.x += (float(int((uint(w_pack2) >> (8 * k4i)) & 0xFu)) - 8.0) * scale0 * in_val; + acc.y += (float(int((uint(w_pack2) >> (8 * k4i + 4)) & 0xFu)) - 8.0) * scale1 * in_val; + } + + [[unroll]] for (int k4i = 0; k4i < 4; ++k4i) { + $if USE_SUBGROUP_BROADCAST: + in_val = subgroupBroadcast(in_vecs[1][k4i], 3u); + $else: + in_val = in_vecs[7][k4i]; + acc.x += (float(int((uint(w_pack3) >> (8 * k4i)) & 0xFu)) - 8.0) * scale0 * in_val; + acc.y += (float(int((uint(w_pack3) >> (8 * k4i + 4)) & 0xFu)) - 8.0) * scale1 * in_val; + } + } + + // --- Inter-wave reduction via flat shared memory (matches OpenCL) --- + if (k_wave_id >= 1) { + partial_sums[(k_wave_id - 1) * LWG_X_SIZE + int(lane_id)] = acc; + } + barrier(); + if (k_wave_id == 0) { + [[unroll]] for (int w = 0; w < NUM_SUBGROUPS - 1; ++w) { + acc += partial_sums[w * LWG_X_SIZE + int(lane_id)]; + } + + // Apply bias if present + if (apply_bias > 0) { + acc.x += float(t_bias[n2 * 2]); + acc.y += float(t_bias[n2 * 2 + 1]); + } + } + + // --- Write 2 outputs --- +#ifdef IO_BUFFER + if (k_wave_id == 0) { + t_output[n2 * 2] = T(acc.x); + t_output[n2 * 2 + 1] = T(acc.y); + } +#else + // texture3d: output stored as width-packed vec4 at (n4, 0, 0). + // Each thread owns 2 outputs (n2*2, n2*2+1). Two consecutive n2s share + // one vec4; only the even-n2 thread assembles and writes the full vec4. + $if USE_SUBGROUP_BROADCAST: + vec2 partner = vec2( + subgroupShuffleXor(acc.x, 1u), + subgroupShuffleXor(acc.y, 1u)); + if (k_wave_id == 0 && (n2 & 1) == 0) { + vec4 out_vec; + out_vec.xy = acc; + out_vec.zw = partner; + const int n4 = n2 / 2; + imageStore(t_output, ivec3(n4, 0, 0), out_vec); + } + $else: + // Subgroup-free partner exchange via shared memory. Only k_wave_id==0 + // threads have a valid reduced `acc`, so only those threads write the + // partner slot; all threads must reach the barrier (uniform control + // flow). Then the even-n2 k_wave_id==0 threads read the n2-XOR-1 + // partner slot and assemble the output vec4. A barrier before the + // write resynchronizes after the inter-wave reduction read of + // partial_sums (which conflicts with the partner-exchange shared + // memory only conceptually — they are separate arrays — but the + // pre-barrier matches the OpenCL reference style and is cheap). + barrier(); + if (k_wave_id == 0) { + nosg_n2_partner[lane_id] = acc; + } + barrier(); + if (k_wave_id == 0 && (n2 & 1) == 0) { + vec2 partner = nosg_n2_partner[lane_id ^ 1u]; + vec4 out_vec; + out_vec.xy = acc; + out_vec.zw = partner; + const int n4 = n2 / 2; + imageStore(t_output, ivec3(n4, 0, 0), out_vec); + } +#endif +} diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/glsl/q4gsw_linear_gemv__w_4x8.yaml b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/glsl/q4gsw_linear_gemv__w_4x8.yaml new file mode 100644 index 00000000000..d5e853d2aed --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/glsl/q4gsw_linear_gemv__w_4x8.yaml @@ -0,0 +1,40 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +q4gsw_linear_gemv__w_4x8: + parameter_names_with_default_values: + DTYPE: float + IO_STORAGE: buffer + NUM_SUBGROUPS: 4 + SUBGROUP_SIZE: 64 + LWG_X_SIZE: 64 + USE_SUBGROUP_BROADCAST: true + WEIGHT_TILE_CONTIG_DIM: 0 + generate_variant_forall: + IO_STORAGE: + - VALUE: buffer + - VALUE: texture3d + DTYPE: + - VALUE: float + - VALUE: half + shader_variants: + # interleaved nibble, N dim contiguous, subgroup broadcast. + # SUBGROUP_SIZE=64 (inherited from top-level) auto-pins the pipeline's + # required subgroup size to 64 so subgroupBroadcast indexing matches the + # host-side LWG.x assumption (currently 64). Without this the driver + # may pick any size in [minSubgroupSize, maxSubgroupSize] (e.g. 128 on + # Adreno 750) and break the reduction. + - NAME: q4gsw_linear_gemv__w_4x8_nc + WEIGHT_TILE_CONTIG_DIM: 0 + # interleaved nibble, N dim contiguous, no subgroup ops — portable + # across any subgroup width (works on Mali / 16-wide as well as Adreno). + # SUBGROUP_SIZE=0 opts out of the pipeline subgroup-size pin (the + # SUBGROUP_SIZE macro is unused on the nosg path; shared-mem layout uses + # LWG_X_SIZE). + - NAME: q4gsw_linear_gemv__w_4x8_nc_nosg + WEIGHT_TILE_CONTIG_DIM: 0 + USE_SUBGROUP_BROADCAST: false + SUBGROUP_SIZE: 0 diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q8csw/Q8cswLinearReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q8csw/Q8cswLinearReference.cpp new file mode 100644 index 00000000000..e0a9243a883 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q8csw/Q8cswLinearReference.cpp @@ -0,0 +1,83 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +#include +#include +namespace vkcompute::shader_profiling::reference { +static Tensor q8csw_linear_reference( + const Json& spec, + const TensorMap& tensors, + size_t threads) { + const bool quantized = + spec.at("operator") == "et_vk.linear_q8ta_q8csw.default"; + const size_t wi = quantized ? 3 : 1, si = quantized ? 5 : 2, + bi = quantized ? 6 : 3; + Tensor input = tensor_arg(spec, 0, tensors), + weight = tensor_arg(spec, wi, tensors); + const Tensor& scales = tensor_arg(spec, si, tensors); + const Tensor* bias = spec.at("args").at(bi).is_null() + ? nullptr + : &tensor_arg(spec, bi, tensors); + if (quantized) { + const float scale = q8ta::scale(spec, 1); + const int64_t zp = q8ta::integer(spec, 2); + require( + weight.sizes.size() == 2 && input.sizes.back() == weight.sizes[1], + "Quantized linear weight dimensions must match input features"); + const int64_t k = input.sizes.back(), n = weight.sizes[0]; + const int64_t m = int64_t(input.values.size()) / k; + require( + scales.values.size() == size_t(n) && + (!bias || bias->values.size() == size_t(n)), + "Quantized linear scales and bias must match output features"); + Tensor output{input.sizes, std::vector(size_t(m * n))}; + output.sizes.back() = n; + std::vector quantized_input(input.values.size()); + for (size_t i = 0; i < quantized_input.size(); ++i) + quantized_input[i] = + int16_t(q8ta::quantize(float(input.values[i]), scale, zp) - zp); + std::vector packed_weight(weight.values.size()); + for (int64_t col = 0; col < n; ++col) + for (int64_t j = 0; j < k; ++j) + packed_weight[size_t(j * n + col)] = + int8_t(weight.values[size_t(col * k + j)]); + constexpr int64_t tile = 32; + const int64_t blocks = (n + tile - 1) / tile; + parallel_for(size_t(m * blocks), threads, [&](size_t index) { + const int64_t row = int64_t(index) / blocks; + const int64_t start = int64_t(index) % blocks * tile; + const int64_t count = std::min(tile, n - start); + std::array sums{}; + for (int64_t j = 0; j < k; ++j) { + const int64_t value = quantized_input[size_t(row * k + j)]; + for (int64_t col = 0; col < count; ++col) + sums[size_t(col)] += + value * packed_weight[size_t(j * n + start + col)]; + } + for (int64_t col = 0; col < count; ++col) { + const size_t oc = size_t(start + col); + const float combined_scale = scale * float(scales.values[oc]); + output.values[size_t(row * n) + oc] = + std::fma(float(sums[size_t(col)]), combined_scale, 0.0f) + + (bias ? float(bias->values[oc]) : 0.0f); + } + }); + return output; + } + for (size_t i = 0; i < weight.values.size(); ++i) + weight.values[i] = float(weight.values[i]) * + float(scales.values[i / size_t(weight.sizes[1])]); + return linear_projection(input, weight, bias, threads); +} +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_FN(q8csw_linear, q8csw_linear_reference); + REGISTER_OPERATOR_REFERENCE(et_vk.linear_q8csw.default, q8csw_linear); + REGISTER_OPERATOR_REFERENCE(et_vk.linear_q8ta_q8csw.default, q8csw_linear); +} +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q8csw/cases/correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q8csw/cases/correctness.json new file mode 100644 index 00000000000..49641442f3f --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q8csw/cases/correctness.json @@ -0,0 +1,1382 @@ +[ + { + "name": "q8csw_linear/correctness", + "operator": "et_vk.linear_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_linear/correctness (2)", + "operator": "et_vk.linear_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_linear/correctness (3)", + "operator": "et_vk.linear_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_linear/correctness (4)", + "operator": "et_vk.linear_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [32, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_linear/correctness (5)", + "operator": "et_vk.linear_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [32, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_linear/correctness (6)", + "operator": "et_vk.linear_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [32, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_linear/correctness (7)", + "operator": "et_vk.linear_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_linear/correctness (8)", + "operator": "et_vk.linear_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_linear/correctness (9)", + "operator": "et_vk.linear_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_linear/correctness (10)", + "operator": "et_vk.linear_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [32, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_linear/correctness (11)", + "operator": "et_vk.linear_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [32, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_linear/correctness (12)", + "operator": "et_vk.linear_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [32, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_linear/correctness (13)", + "operator": "et_vk.linear_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_linear/correctness (14)", + "operator": "et_vk.linear_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_linear/correctness (15)", + "operator": "et_vk.linear_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_linear/correctness (16)", + "operator": "et_vk.linear_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q8csw/cases/debug.json b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q8csw/cases/debug.json new file mode 100644 index 00000000000..d6f0f530173 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q8csw/cases/debug.json @@ -0,0 +1,100 @@ +[ + { + "name": "q8csw_linear/debug", + "operator": "et_vk.linear_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 4], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [4, 4], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4, 4], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q8csw/cases/performance.json b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q8csw/cases/performance.json new file mode 100644 index 00000000000..14afdb25816 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q8csw/cases/performance.json @@ -0,0 +1,190 @@ +[ + { + "name": "q8csw_linear/performance", + "operator": "et_vk.linear_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "sweep_variables": { + "v_SIZES": [ + [256, 2048], + [512, 2048], + [1024, 2048] + ], + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 2048], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2048], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_linear/performance (2)", + "operator": "et_vk.linear_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "sweep_variables": { + "v_SIZES": [ + [256, 2048], + [512, 2048], + [1024, 2048] + ], + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 2048], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/matmul/MatmulReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/matmul/MatmulReference.cpp new file mode 100644 index 00000000000..bc3efa771dd --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/matmul/MatmulReference.cpp @@ -0,0 +1,61 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +namespace vkcompute::shader_profiling::reference { +Tensor +matmul_reference(const Json& spec, const TensorMap& tensors, size_t threads) { + const bool add = spec.at("operator") == "aten.addmm.default"; + const Tensor& a = tensor_arg(spec, add ? 1 : 0, tensors); + const Tensor& b = tensor_arg(spec, add ? 2 : 1, tensors); + require( + (a.sizes.size() == 2 || a.sizes.size() == 3) && + a.sizes.size() == b.sizes.size(), + "Matmul expects two rank2 or rank3 inputs"); + const size_t k = a.sizes.back(), m = a.sizes[a.sizes.size() - 2], + n = b.sizes.back(); + const size_t batches = a.sizes.size() == 3 ? a.sizes[0] : 1; + require( + b.sizes[b.sizes.size() - 2] == int64_t(k) && + (a.sizes.size() == 2 || a.sizes[0] == b.sizes[0]), + "Matmul dimensions differ"); + Tensor out{a.sizes, std::vector(batches * m * n)}; + out.sizes.back() = n; + const Tensor* bias = add ? &tensor_arg(spec, 0, tensors) : nullptr; + const float beta = + add ? spec.at("args").at(3).at("float").get() : 0.0f; + const float alpha = + add ? spec.at("args").at(4).at("float").get() : 1.0f; + require( + !bias || bias->values.size() == n, + "Addmm reference requires vector bias"); + for (size_t batch = 0; batch < batches; ++batch) { + Tensor input{ + {int64_t(m), int64_t(k)}, + std::vector( + a.values.begin() + batch * m * k, + a.values.begin() + (batch + 1) * m * k)}; + Tensor weight{{int64_t(n), int64_t(k)}, std::vector(n * k)}; + parallel_for(n, threads, [&](size_t j) { + for (size_t i = 0; i < k; ++i) + weight.values[j * k + i] = b.values[(batch * k + i) * n + j]; + }); + Tensor result = linear_projection(input, weight, nullptr, threads); + for (size_t i = 0; i < m * n; ++i) + out.values[batch * m * n + i] = alpha * float(result.values[i]) + + (bias ? beta * float(bias->values[i % n]) : 0.0f); + } + return out; +} +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_FN(matmul, matmul_reference); + REGISTER_OPERATOR_REFERENCE(aten.mm.default, matmul); + REGISTER_OPERATOR_REFERENCE(aten.bmm.default, matmul); + REGISTER_OPERATOR_REFERENCE(aten.addmm.default, matmul); +} +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/matmul/MatmulReference.h b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/matmul/MatmulReference.h new file mode 100644 index 00000000000..4181a0547b9 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/matmul/MatmulReference.h @@ -0,0 +1,16 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +namespace vkcompute::shader_profiling::reference { + +Tensor +matmul_reference(const Json& spec, const TensorMap& tensors, size_t threads); + +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/matmul/MatmulRoutes.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/matmul/MatmulRoutes.cpp new file mode 100644 index 00000000000..97cf9d87b2b --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/matmul/MatmulRoutes.cpp @@ -0,0 +1,84 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +#include +#include +#include +#include +namespace vkcompute::shader_profiling { +namespace { +template +Json build_gemm_route( + ComputeGraph& graph, + const std::vector& args, + const Json& options) { + if (!options.empty()) + throw std::invalid_argument("GEMM routes accept no options"); + const ValueRef input = args.at(0), weight = args.at(1), out = args.back(); + if constexpr (Coop) { + const auto* adapter = graph.context()->adapter_ptr(); + const auto sizes = graph.sizes_of(input), outs = graph.sizes_of(out); + if (!adapter->supports_cooperative_matrix() || + adapter->subgroup_size() != 64 || !graph.is_buffer_storage(out) || + sizes.size() != 2 || sizes[0] % 64 || sizes[1] % 32 || outs.back() % 64) + throw UnsupportedRoute( + "coopmat requires cooperative matrices, subgroup64, buffer output, and M/N/K alignment64/64/32"); + } + if (Linear || graph.val_is_tref(weight)) { + const int64_t batch = + graph.sizes_of(weight).size() == 3 ? graph.sizes_of(weight).at(0) : 1; + const ValueRef packed = + prepack_fp_linear_weight(graph, weight, Linear, batch, Coop); + const bool has_bias = Linear && graph.val_is_not_none(args.at(2)); + ValueRef bias = kDummyValueRef; + if (has_bias) + bias = prepack_standard( + graph, + args.at(2), + graph.storage_type_of(out), + utils::kWidthPacked, + false); + if constexpr (Coop) + add_linear_coopmat_node( + graph, + input, + packed, + bias, + has_bias, + out, + utils::safe_downcast(batch)); + else + add_linear_tiled_node( + graph, + input, + packed, + bias, + has_bias, + out, + utils::safe_downcast(batch)); + } else { + if constexpr (Coop) + add_matmul_coopmat_node(graph, input, weight, out); + else + add_matmul_tiled_node(graph, input, weight, out); + } + return {{"name", Coop ? "coopmat" : "tiled"}, {"options", options}}; +} +REGISTER_OPERATOR_ROUTES { + REGISTER_OPERATOR_ROUTE( + aten.mm.default, tiled, (build_gemm_route)); + REGISTER_OPERATOR_ROUTE( + aten.mm.default, coopmat, (build_gemm_route)); + REGISTER_OPERATOR_ROUTE( + aten.linear.default, tiled, (build_gemm_route)); + REGISTER_OPERATOR_ROUTE( + aten.linear.default, coopmat, (build_gemm_route)); +} +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/matmul/MatmulThroughput.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/matmul/MatmulThroughput.cpp new file mode 100644 index 00000000000..2f69c7c7db1 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/matmul/MatmulThroughput.cpp @@ -0,0 +1,32 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +namespace vkcompute::shader_profiling { +namespace { +double matmul_flops(const Json& spec) { + const size_t input_index = + spec.at("operator") == "aten.addmm.default" ? 1 : 0; + const auto sizes = throughput::tensor_argument(spec, input_index) + .at("sizes") + .get>(); + return 2.0 * sizes.back() * + throughput::tensor_elements(throughput::output_tensor(spec)); +} +REGISTER_THROUGHPUT_FUNCTIONS { + REGISTER_OPERATOR_MEMORY_TRAFFIC(aten.mm.default, throughput::tensor_traffic); + REGISTER_OPERATOR_COMPUTE_FLOPS(aten.mm.default, matmul_flops); + REGISTER_OPERATOR_MEMORY_TRAFFIC( + aten.bmm.default, throughput::tensor_traffic); + REGISTER_OPERATOR_COMPUTE_FLOPS(aten.bmm.default, matmul_flops); + REGISTER_OPERATOR_MEMORY_TRAFFIC( + aten.addmm.default, throughput::tensor_traffic); + REGISTER_OPERATOR_COMPUTE_FLOPS(aten.addmm.default, matmul_flops); +} + +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/matmul/cases/correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/matmul/cases/correctness.json new file mode 100644 index 00000000000..f5286476345 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/matmul/cases/correctness.json @@ -0,0 +1,3681 @@ +[ + { + "name": "mm correctness [0,64,128,64] aten.mm.default texture3d float32", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [64, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,128,47] aten.mm.default texture3d float32", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 47], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [64, 47], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,128,64] aten.mm.default buffer float32 float32 buffer width", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "tiled"], + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [64, 64], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,128,47] aten.mm.default buffer float32", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 47], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [64, 47], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,128,256,128] aten.mm.default texture3d float32", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,128,256,128] aten.mm.default buffer float32 float32 buffer width", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "tiled"], + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [128, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,32,64,256] aten.mm.default texture3d float32", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 256], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [32, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,32,64,256] aten.mm.default buffer float32", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 64], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 256], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [32, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,57,131,43] aten.mm.default texture3d float32", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [57, 131], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [131, 43], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [57, 43], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,57,131,43] aten.mm.default buffer float32", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [57, 131], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [131, 43], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [57, 43], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,33,67,91] aten.mm.default texture3d float32", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [33, 67], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [67, 91], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [33, 91], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,33,67,91] aten.mm.default buffer float32", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [33, 67], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [67, 91], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [33, 91], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,47,128] aten.mm.default texture3d float32", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 47], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [47, 128], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [64, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,47,128] aten.mm.default buffer float32", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 47], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [47, 128], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [64, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,128,64] aten.mm.default buffer float32 float32 buffer width coopmat", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "coopmat", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [64, 64], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,128,256,128] aten.mm.default buffer float32 float32 buffer width coopmat", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "coopmat", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [128, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [1,64,128,64] aten.bmm.default texture3d float32", + "operator": "aten.bmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 128, 64], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [1,64,128,64] aten.bmm.default buffer float32", + "operator": "aten.bmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 128, 64], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 64], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [1,4,32,16] aten.bmm.default texture3d float32", + "operator": "aten.bmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 32, 16], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [1, 4, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [1,4,32,16] aten.bmm.default buffer float32", + "operator": "aten.bmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4, 32], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 32, 16], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [1, 4, 16], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [1,19,53,37] aten.bmm.default texture3d float32", + "operator": "aten.bmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 19, 53], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 53, 37], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [1, 19, 37], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [1,19,53,37] aten.bmm.default buffer float32", + "operator": "aten.bmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 19, 53], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 53, 37], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [1, 19, 37], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,128,64] aten.addmm.default texture3d float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [64, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,128,64] aten.addmm.default buffer float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [64, 64], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,128,256,128] aten.addmm.default texture3d float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [128, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,47,128] aten.addmm.default texture3d float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [64, 47], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [47, 128], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [64, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,128,256,128] aten.addmm.default buffer float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [128, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [128, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,47,128] aten.addmm.default buffer float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [64, 47], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [47, 128], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [64, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,32,64,256] aten.addmm.default texture3d float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [32, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 256], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [32, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,32,64,256] aten.addmm.default buffer float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [32, 64], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 256], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [32, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,57,131,43] aten.addmm.default texture3d float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [43], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [57, 131], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [131, 43], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [57, 43], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,57,131,43] aten.addmm.default buffer float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [43], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [57, 131], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [131, 43], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [57, 43], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,33,67,91] aten.addmm.default texture3d float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [91], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [33, 67], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [67, 91], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [33, 91], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,33,67,91] aten.addmm.default buffer float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [91], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [33, 67], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [67, 91], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [33, 91], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,128,47] aten.addmm.default texture3d float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [47], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 47], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [64, 47], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,128,47] aten.addmm.default buffer float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [47], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 47], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [64, 47], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,128,64] aten.linear.default", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [64, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,128,47] aten.linear.default", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [47, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [64, 47], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,128,256,128] aten.linear.default", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [128, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,32,64,256] aten.linear.default", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,57,131,43] aten.linear.default", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [57, 131], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [43, 131], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [57, 43], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,33,67,91] aten.linear.default", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [33, 67], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [91, 67], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [33, 91], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,47,128] aten.linear.default", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 47], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 47], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [64, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,128,64] aten.linear.default buffer float32 constant=False tiled", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "tiled", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [64, 64], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,128,256,128] aten.linear.default buffer float32 constant=False tiled", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "tiled", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [128, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,128,64] aten.linear.default buffer float32 constant=False coopmat", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "coopmat", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [64, 64], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,128,256,128] aten.linear.default buffer float32 constant=False coopmat", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "coopmat", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [128, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,128,64] aten.linear.default (4)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [64, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,128,47] aten.linear.default (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [47, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [47], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [64, 47], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,128,256,128] aten.linear.default (4)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [128, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,32,64,256] aten.linear.default (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [32, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,57,131,43] aten.linear.default (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [57, 131], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [43, 131], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [43], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [57, 43], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,33,67,91] aten.linear.default (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [33, 67], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [91, 67], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [91], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [33, 91], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,47,128] aten.linear.default (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 47], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 47], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [64, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,128,64] aten.linear.default buffer float32 constant=False tiled (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "tiled", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [64, 64], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,128,256,128] aten.linear.default buffer float32 constant=False tiled (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "tiled", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [128, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,128,64] aten.linear.default buffer float32 constant=False coopmat (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "coopmat", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [64, 64], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,128,256,128] aten.linear.default buffer float32 constant=False coopmat (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "coopmat", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [128, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/matmul/cases/performance.json b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/matmul/cases/performance.json new file mode 100644 index 00000000000..70cf7953aa0 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/matmul/cases/performance.json @@ -0,0 +1,5502 @@ +[ + { + "name": "mm performance [0,4096,1024,256] aten.mm.default texture3d float32", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 1024], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1024, 256], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,1024,256] aten.mm.default buffer float32 float32 buffer width", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "tiled"], + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 1024], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1024, 256], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,256,128] aten.mm.default texture3d float32", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,256,128] aten.mm.default buffer float32 float32 buffer width", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "tiled"], + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,128,256] aten.mm.default texture3d float32", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,128,256] aten.mm.default buffer float32 float32 buffer width", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "tiled"], + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,64,128] aten.mm.default texture3d float32", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,64,128] aten.mm.default buffer float32 float32 buffer width", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "tiled"], + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 64], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,1024,256] aten.mm.default buffer float32 float32 buffer width coopmat", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "coopmat", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 1024], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1024, 256], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,256,128] aten.mm.default buffer float32 float32 buffer width coopmat", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "coopmat", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,128,256] aten.mm.default buffer float32 float32 buffer width coopmat", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "coopmat", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,64,128] aten.mm.default buffer float32 float32 buffer width coopmat", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "coopmat", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 64], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,1024,256] aten.mm.default texture3d float16", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 1024], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1024, 256], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,1024,256] aten.mm.default buffer float16", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "tiled", "coopmat"], + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 1024], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1024, 256], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,256,128] aten.mm.default texture3d float16", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,256,128] aten.mm.default buffer float16", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "tiled", "coopmat"], + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,128,256] aten.mm.default texture3d float16", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,128,256] aten.mm.default buffer float16", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "tiled", "coopmat"], + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,64,128] aten.mm.default texture3d float16", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,64,128] aten.mm.default buffer float16", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "tiled", "coopmat"], + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 64], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [1,4096,256,1024] aten.bmm.default texture3d float32", + "operator": "aten.bmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4096, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 256, 1024], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [1, 4096, 1024], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [1,4096,256,128] aten.bmm.default texture3d float32", + "operator": "aten.bmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4096, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 256, 128], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [1, 4096, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [1,4096,256,1024] aten.bmm.default buffer float32", + "operator": "aten.bmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 256, 1024], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [1, 4096, 1024], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [1,4096,256,128] aten.bmm.default buffer float32", + "operator": "aten.bmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 256, 128], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [1, 4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [1,256,4096,64] aten.bmm.default texture3d float32", + "operator": "aten.bmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256, 4096], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 4096, 64], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [1, 256, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [1,256,4096,64] aten.bmm.default buffer float32", + "operator": "aten.bmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256, 4096], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 4096, 64], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [1, 256, 64], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [1,4096,256,1024] aten.bmm.default texture3d float16", + "operator": "aten.bmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4096, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 256, 1024], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [1, 4096, 1024], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [1,4096,256,128] aten.bmm.default texture3d float16", + "operator": "aten.bmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4096, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 256, 128], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [1, 4096, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [1,4096,256,1024] aten.bmm.default buffer float16", + "operator": "aten.bmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4096, 256], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 256, 1024], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [1, 4096, 1024], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [1,4096,256,128] aten.bmm.default buffer float16", + "operator": "aten.bmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4096, 256], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 256, 128], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [1, 4096, 128], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [1,256,4096,64] aten.bmm.default texture3d float16", + "operator": "aten.bmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256, 4096], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 4096, 64], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [1, 256, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [1,256,4096,64] aten.bmm.default buffer float16", + "operator": "aten.bmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256, 4096], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 4096, 64], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [1, 256, 64], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,1024,256] aten.addmm.default texture3d float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [4096, 1024], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1024, 256], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,128,256] aten.addmm.default texture3d float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,1024,256] aten.addmm.default buffer float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [4096, 1024], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1024, 256], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,128,256] aten.addmm.default buffer float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,256,128] aten.addmm.default texture3d float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,64,128] aten.addmm.default texture3d float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [4096, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,256,128] aten.addmm.default buffer float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,64,128] aten.addmm.default buffer float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [4096, 64], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,1024,256] aten.addmm.default texture3d float16", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [4096, 1024], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1024, 256], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,128,256] aten.addmm.default texture3d float16", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,1024,256] aten.addmm.default buffer float16", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [4096, 1024], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1024, 256], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,128,256] aten.addmm.default buffer float16", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,256,128] aten.addmm.default texture3d float16", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,64,128] aten.addmm.default texture3d float16", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [4096, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,256,128] aten.addmm.default buffer float16", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,64,128] aten.addmm.default buffer float16", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [4096, 64], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,1024,256] aten.linear.default", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 1024], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 1024], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,256,128] aten.linear.default", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,128,256] aten.linear.default", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,64,128] aten.linear.default", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,1024,256] aten.linear.default buffer float32 constant=False tiled", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "tiled", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 1024], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 1024], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,256,128] aten.linear.default buffer float32 constant=False tiled", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "tiled", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,128,256] aten.linear.default buffer float32 constant=False tiled", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "tiled", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,64,128] aten.linear.default buffer float32 constant=False tiled", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "tiled", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 64], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,1024,256] aten.linear.default buffer float32 constant=False coopmat", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "coopmat", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 1024], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 1024], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,256,128] aten.linear.default buffer float32 constant=False coopmat", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "coopmat", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,128,256] aten.linear.default buffer float32 constant=False coopmat", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "coopmat", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,64,128] aten.linear.default buffer float32 constant=False coopmat", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "coopmat", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 64], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,1024,256] aten.linear.default texture3d float16 constant=False auto", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 1024], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 1024], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,1024,256] aten.linear.default buffer float16 constant=False", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "tiled", "coopmat"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 1024], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 1024], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,256,128] aten.linear.default texture3d float16 constant=False auto", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,256,128] aten.linear.default buffer float16 constant=False", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "tiled", "coopmat"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,128,256] aten.linear.default texture3d float16 constant=False auto", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,128,256] aten.linear.default buffer float16 constant=False", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "tiled", "coopmat"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,64,128] aten.linear.default texture3d float16 constant=False auto", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,64,128] aten.linear.default buffer float16 constant=False", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "tiled", "coopmat"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 64], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,1024,256] aten.linear.default (6)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 1024], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 1024], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,256,128] aten.linear.default (6)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,128,256] aten.linear.default (6)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,64,128] aten.linear.default (6)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,1024,256] aten.linear.default buffer float32 constant=False tiled (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "tiled", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 1024], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 1024], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,256,128] aten.linear.default buffer float32 constant=False tiled (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "tiled", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,128,256] aten.linear.default buffer float32 constant=False tiled (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "tiled", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,64,128] aten.linear.default buffer float32 constant=False tiled (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "tiled", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 64], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,1024,256] aten.linear.default buffer float32 constant=False coopmat (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "coopmat", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 1024], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 1024], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,256,128] aten.linear.default buffer float32 constant=False coopmat (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "coopmat", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,128,256] aten.linear.default buffer float32 constant=False coopmat (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "coopmat", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,64,128] aten.linear.default buffer float32 constant=False coopmat (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "coopmat", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 64], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,1024,256] aten.linear.default texture3d float16 constant=False auto (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 1024], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 1024], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,1024,256] aten.linear.default buffer float16 constant=False (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "tiled", "coopmat"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 1024], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 1024], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,256,128] aten.linear.default texture3d float16 constant=False auto (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,256,128] aten.linear.default buffer float16 constant=False (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "tiled", "coopmat"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,128,256] aten.linear.default texture3d float16 constant=False auto (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,128,256] aten.linear.default buffer float16 constant=False (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "tiled", "coopmat"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,64,128] aten.linear.default texture3d float16 constant=False auto (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,64,128] aten.linear.default buffer float16 constant=False (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "tiled", "coopmat"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 64], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/pixel_shuffle/q8ta/Q8taPixelShuffleReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/pixel_shuffle/q8ta/Q8taPixelShuffleReference.cpp new file mode 100644 index 00000000000..3c74741ca83 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/pixel_shuffle/q8ta/Q8taPixelShuffleReference.cpp @@ -0,0 +1,49 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +namespace vkcompute::shader_profiling::reference { +static Tensor q8ta_pixel_shuffle_reference( + const Json& spec, + const TensorMap& tensors, + size_t threads) { + const Tensor& input = tensor_arg(spec, 0, tensors); + Tensor out = q8ta::output_tensor(spec, spec.at("args").size() - 1); + const float is = q8ta::scale(spec, 1), + inverse_out_scale = q8ta::scale(spec, 3); + const int64_t iz = q8ta::integer(spec, 2), oz = q8ta::integer(spec, 4), + r = q8ta::integer(spec, 5); + const float actual_out_scale = 1.0f / inverse_out_scale; + const bool passthrough = std::abs(is - actual_out_scale) <= + 1e-7f * std::max(std::abs(is), 1e-7f) && + iz == oz; + const int64_t channels = out.sizes[1], height = out.sizes[2], + width = out.sizes[3]; + parallel_for(out.values.size(), threads, [&](size_t i) { + const int64_t x = int64_t(i) % width, y = int64_t(i) / width % height; + const int64_t c = int64_t(i) / (width * height) % channels, + n = int64_t(i) / (width * height * channels); + const int64_t source_c = c * r * r + y % r * r + x % r; + const int64_t source = + ((n * input.sizes[1] + source_c) * input.sizes[2] + y / r) * + input.sizes[3] + + x / r; + const int64_t value = int64_t(input.values[source]); + out.values[i] = passthrough + ? double(value) + : double(q8ta::requantize( + q8ta::dequantize(value, is, iz), inverse_out_scale, oz)); + }); + return out; +} +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_FN(q8ta_pixel_shuffle, q8ta_pixel_shuffle_reference); + REGISTER_OPERATOR_REFERENCE( + et_vk.q8ta_pixel_shuffle.default, q8ta_pixel_shuffle); +} +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/pixel_shuffle/q8ta/Q8taPixelShuffleThroughput.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/pixel_shuffle/q8ta/Q8taPixelShuffleThroughput.cpp new file mode 100644 index 00000000000..4334d243439 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/pixel_shuffle/q8ta/Q8taPixelShuffleThroughput.cpp @@ -0,0 +1,30 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include + +namespace vkcompute::shader_profiling { +namespace { +double pixel_shuffle_flops(const Json& spec) { + const float in_scale = spec.at("args").at(1).at("float").get(); + const float out_scale = 1.0f / spec.at("args").at(3).at("float").get(); + const bool passthrough = std::abs(in_scale - out_scale) <= + 1e-7f * std::max(std::abs(in_scale), 1e-7f) && + spec.at("args").at(2).at("int") == spec.at("args").at(4).at("int"); + return (passthrough ? 0 : 3) * + throughput::tensor_elements(throughput::output_tensor(spec)); +} +REGISTER_THROUGHPUT_FUNCTIONS { + REGISTER_OPERATOR_MEMORY_TRAFFIC( + et_vk.q8ta_pixel_shuffle.default, throughput::tensor_traffic); + REGISTER_OPERATOR_COMPUTE_FLOPS( + et_vk.q8ta_pixel_shuffle.default, pixel_shuffle_flops); +} +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/pixel_shuffle/q8ta/cases/correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/pixel_shuffle/q8ta/cases/correctness.json new file mode 100644 index 00000000000..124af8ff463 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/pixel_shuffle/q8ta/cases/correctness.json @@ -0,0 +1,482 @@ +[ + { + "name": "q8ta/pixel_shuffle/correctness", + "operator": "et_vk.q8ta_pixel_shuffle.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.014060718249784, + "rtol": 0 + }, + "sweep_variables": { + "v_INPUT_LAYOUT": ["4w4c", "4c1w"], + "v_OUTPUT_LAYOUT": ["4w4c", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 4, 4], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_INPUT_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "float": 140.607421875 + }, + { + "int": 0 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 4, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_OUTPUT_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/pixel_shuffle/correctness (2)", + "operator": "et_vk.q8ta_pixel_shuffle.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.014060718249784, + "rtol": 0 + }, + "sweep_variables": { + "v_INPUT_LAYOUT": ["4w4c", "4c1w"], + "v_OUTPUT_LAYOUT": ["4w4c", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 24, 8, 4], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_INPUT_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "float": 140.607421875 + }, + { + "int": 0 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 6, 16, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_OUTPUT_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/pixel_shuffle/correctness (3)", + "operator": "et_vk.q8ta_pixel_shuffle.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.014060718249784, + "rtol": 0 + }, + "sweep_variables": { + "v_INPUT_LAYOUT": ["4w4c", "4c1w"], + "v_OUTPUT_LAYOUT": ["4w4c", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 12, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_INPUT_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "float": 140.607421875 + }, + { + "int": 0 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 24, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_OUTPUT_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/pixel_shuffle/correctness (4)", + "operator": "et_vk.q8ta_pixel_shuffle.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.014060718249784, + "rtol": 0 + }, + "sweep_variables": { + "v_INPUT_LAYOUT": ["4w4c", "4c1w"], + "v_OUTPUT_LAYOUT": ["4w4c", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 16, 9], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_INPUT_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "float": 140.607421875 + }, + { + "int": 0 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 24, 32, 18], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_OUTPUT_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/pixel_shuffle/correctness (5)", + "operator": "et_vk.q8ta_pixel_shuffle.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0076922944664253, + "rtol": 0 + }, + "sweep_variables": { + "v_INPUT_LAYOUT": ["4w4c", "4c1w"], + "v_OUTPUT_LAYOUT": ["4w4c", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 4, 4], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_INPUT_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "float": 76.9230728149414 + }, + { + "int": 5 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 4, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_OUTPUT_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/pixel_shuffle/correctness (6)", + "operator": "et_vk.q8ta_pixel_shuffle.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0076922944664253, + "rtol": 0 + }, + "sweep_variables": { + "v_INPUT_LAYOUT": ["4w4c", "4c1w"], + "v_OUTPUT_LAYOUT": ["4w4c", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 24, 8, 4], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_INPUT_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "float": 76.9230728149414 + }, + { + "int": 5 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 6, 16, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_OUTPUT_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/pixel_shuffle/correctness (7)", + "operator": "et_vk.q8ta_pixel_shuffle.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0076922944664253, + "rtol": 0 + }, + "sweep_variables": { + "v_INPUT_LAYOUT": ["4w4c", "4c1w"], + "v_OUTPUT_LAYOUT": ["4w4c", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 12, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_INPUT_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "float": 76.9230728149414 + }, + { + "int": 5 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 24, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_OUTPUT_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/pixel_shuffle/correctness (8)", + "operator": "et_vk.q8ta_pixel_shuffle.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0076922944664253, + "rtol": 0 + }, + "sweep_variables": { + "v_INPUT_LAYOUT": ["4w4c", "4c1w"], + "v_OUTPUT_LAYOUT": ["4w4c", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 16, 9], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_INPUT_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "float": 76.9230728149414 + }, + { + "int": 5 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 24, 32, 18], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_OUTPUT_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/pixel_shuffle/q8ta/cases/performance.json b/backends/vulkan/test/ops/benchmark_v2/operators/pixel_shuffle/q8ta/cases/performance.json new file mode 100644 index 00000000000..e33e0a9327b --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/pixel_shuffle/q8ta/cases/performance.json @@ -0,0 +1,242 @@ +[ + { + "name": "q8ta/pixel_shuffle/performance", + "operator": "et_vk.q8ta_pixel_shuffle.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.014060718249784, + "rtol": 0 + }, + "sweep_variables": { + "v_INPUT_LAYOUT": ["4w4c", "4c1w"], + "v_OUTPUT_LAYOUT": ["4w4c", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 16, 9], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_INPUT_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "float": 140.607421875 + }, + { + "int": 0 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 24, 32, 18], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_OUTPUT_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/pixel_shuffle/performance (2)", + "operator": "et_vk.q8ta_pixel_shuffle.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.014060718249784, + "rtol": 0 + }, + "sweep_variables": { + "v_INPUT_LAYOUT": ["4w4c", "4c1w"], + "v_OUTPUT_LAYOUT": ["4w4c", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 32, 18], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_INPUT_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "float": 140.607421875 + }, + { + "int": 0 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 24, 64, 36], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_OUTPUT_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/pixel_shuffle/performance (3)", + "operator": "et_vk.q8ta_pixel_shuffle.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.014060718249784, + "rtol": 0 + }, + "sweep_variables": { + "v_INPUT_LAYOUT": ["4w4c", "4c1w"], + "v_OUTPUT_LAYOUT": ["4w4c", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 64, 36], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_INPUT_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "float": 140.607421875 + }, + { + "int": 0 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 24, 128, 72], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_OUTPUT_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/pixel_shuffle/performance (4)", + "operator": "et_vk.q8ta_pixel_shuffle.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.014060718249784, + "rtol": 0 + }, + "sweep_variables": { + "v_INPUT_LAYOUT": ["4w4c", "4c1w"], + "v_OUTPUT_LAYOUT": ["4w4c", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 128, 72], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_INPUT_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "float": 140.607421875 + }, + { + "int": 0 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 24, 256, 144], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_OUTPUT_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/quantization/choose_qparams_per_row/ChooseQparamsReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/quantization/choose_qparams_per_row/ChooseQparamsReference.cpp new file mode 100644 index 00000000000..5ebb89e22ca --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/quantization/choose_qparams_per_row/ChooseQparamsReference.cpp @@ -0,0 +1,67 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +#include +#include +namespace vkcompute::shader_profiling::reference { +ReferenceOutputs choose_qparams_reference( + const Json& spec, + const TensorMap& tensors, + size_t threads) { + const Tensor& input = tensor_arg(spec, 0, tensors); + require(input.sizes.size() == 2, "Choose qparams expects [rows,columns]"); + const int32_t qmin = spec.at("args").at(1).at("int"), + qmax = spec.at("args").at(2).at("int"); + require(qmin < qmax, "Quantization bounds must be increasing"); + const size_t rows = input.sizes[0], columns = input.sizes[1]; + std::vector scales(rows), zeros(rows); + parallel_for(rows, threads, [&](size_t row) { + float lo = 0, hi = 0; + for (size_t j = 0; j < columns; ++j) { + const float value = input.values[row * columns + j]; + lo = std::min(lo, value); + hi = std::max(hi, value); + } + double scale = (double(hi) - lo) / (qmax - qmin); + if (float(scale) == 0 || std::isinf(1.0f / float(scale))) + scale = 0.1; + constexpr float threshold = 6.1e-5f; + if (scale < threshold) { + const float original = scale; + scale = threshold; + if (lo == 0) + hi = threshold * (qmax - qmin); + else if (hi == 0) + lo = -threshold * (qmax - qmin); + else { + lo *= threshold / original; + hi *= threshold / original; + } + } + const double from_lo = qmin - lo / scale, from_hi = qmax - hi / scale; + const double error_lo = std::abs(qmin) - std::abs(lo / scale), + error_hi = std::abs(qmax) - std::abs(hi / scale); + const double initial = error_lo < error_hi ? from_lo : from_hi; + scales[row] = float(scale); + zeros[row] = std::clamp( + double(std::nearbyint(float(initial))), double(qmin), double(qmax)); + }); + return { + {spec.at("args").at(3).at("tensor").get(), + std::move(scales)}, + {spec.at("args").at(4).at("tensor").get(), + std::move(zeros)}}; +} +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_OUTPUTS_FN( + choose_qparams_per_row, choose_qparams_reference); + REGISTER_OPERATOR_REFERENCE( + etvk.choose_qparams_per_row.default, choose_qparams_per_row); +} +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/quantization/choose_qparams_per_row/ChooseQparamsReference.h b/backends/vulkan/test/ops/benchmark_v2/operators/quantization/choose_qparams_per_row/ChooseQparamsReference.h new file mode 100644 index 00000000000..d2395522788 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/quantization/choose_qparams_per_row/ChooseQparamsReference.h @@ -0,0 +1,18 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +namespace vkcompute::shader_profiling::reference { + +ReferenceOutputs choose_qparams_reference( + const Json& spec, + const TensorMap& tensors, + size_t threads); + +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/quantization/choose_qparams_per_row/ChooseQparamsThroughput.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/quantization/choose_qparams_per_row/ChooseQparamsThroughput.cpp new file mode 100644 index 00000000000..44122a260fe --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/quantization/choose_qparams_per_row/ChooseQparamsThroughput.cpp @@ -0,0 +1,25 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +namespace vkcompute::shader_profiling { +namespace { +double choose_qparams_flops(const Json& spec) { + const auto sizes = throughput::tensor_argument(spec, 0) + .at("sizes") + .get>(); + return double(sizes.at(0)) * (2.0 * sizes.at(1) + 15); +} +REGISTER_THROUGHPUT_FUNCTIONS { + REGISTER_OPERATOR_MEMORY_TRAFFIC( + etvk.choose_qparams_per_row.default, throughput::tensor_traffic); + REGISTER_OPERATOR_COMPUTE_FLOPS( + etvk.choose_qparams_per_row.default, choose_qparams_flops); +} + +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/quantization/choose_qparams_per_row/cases/correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/quantization/choose_qparams_per_row/cases/correctness.json new file mode 100644 index 00000000000..5e9fca4c228 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/quantization/choose_qparams_per_row/cases/correctness.json @@ -0,0 +1,564 @@ +[ + { + "name": "choose_qparams_per_row correctness [4, 16]", + "operator": "etvk.choose_qparams_per_row.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_INPUT_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 16], + "dtype": "float32", + "storage": "v_INPUT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "tensor": { + "name": "scale", + "sizes": [4], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + }, + { + "tensor": { + "name": "zero_point", + "sizes": [4], + "dtype": "int8", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "choose_qparams_per_row correctness [8, 32]", + "operator": "etvk.choose_qparams_per_row.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_INPUT_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [8, 32], + "dtype": "float32", + "storage": "v_INPUT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "tensor": { + "name": "scale", + "sizes": [8], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + }, + { + "tensor": { + "name": "zero_point", + "sizes": [8], + "dtype": "int8", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "choose_qparams_per_row correctness [16, 64]", + "operator": "etvk.choose_qparams_per_row.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_INPUT_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [16, 64], + "dtype": "float32", + "storage": "v_INPUT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "tensor": { + "name": "scale", + "sizes": [16], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + }, + { + "tensor": { + "name": "zero_point", + "sizes": [16], + "dtype": "int8", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "choose_qparams_per_row correctness [32, 128]", + "operator": "etvk.choose_qparams_per_row.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_INPUT_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 128], + "dtype": "float32", + "storage": "v_INPUT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "tensor": { + "name": "scale", + "sizes": [32], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + }, + { + "tensor": { + "name": "zero_point", + "sizes": [32], + "dtype": "int8", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "choose_qparams_per_row correctness [64, 256]", + "operator": "etvk.choose_qparams_per_row.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_INPUT_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 256], + "dtype": "float32", + "storage": "v_INPUT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "tensor": { + "name": "scale", + "sizes": [64], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + }, + { + "tensor": { + "name": "zero_point", + "sizes": [64], + "dtype": "int8", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "choose_qparams_per_row correctness [128, 512]", + "operator": "etvk.choose_qparams_per_row.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_INPUT_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 512], + "dtype": "float32", + "storage": "v_INPUT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "tensor": { + "name": "scale", + "sizes": [128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + }, + { + "tensor": { + "name": "zero_point", + "sizes": [128], + "dtype": "int8", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "choose_qparams_per_row correctness [1,", + "operator": "etvk.choose_qparams_per_row.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_INPUT_SIZES": [ + [1, 512], + [1, 2048] + ], + "v_INPUT_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_INPUT_SIZES", + "dtype": "float32", + "storage": "v_INPUT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "tensor": { + "name": "scale", + "sizes": [1], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + }, + { + "tensor": { + "name": "zero_point", + "sizes": [1], + "dtype": "int8", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "choose_qparams_per_row correctness [256, 1024]", + "operator": "etvk.choose_qparams_per_row.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_INPUT_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [256, 1024], + "dtype": "float32", + "storage": "v_INPUT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "tensor": { + "name": "scale", + "sizes": [256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + }, + { + "tensor": { + "name": "zero_point", + "sizes": [256], + "dtype": "int8", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "choose_qparams_per_row correctness [512, 2048]", + "operator": "etvk.choose_qparams_per_row.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_INPUT_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [512, 2048], + "dtype": "float32", + "storage": "v_INPUT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "tensor": { + "name": "scale", + "sizes": [512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + }, + { + "tensor": { + "name": "zero_point", + "sizes": [512], + "dtype": "int8", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/quantization/choose_qparams_per_row/cases/debug.json b/backends/vulkan/test/ops/benchmark_v2/operators/quantization/choose_qparams_per_row/cases/debug.json new file mode 100644 index 00000000000..b5e7d19a524 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/quantization/choose_qparams_per_row/cases/debug.json @@ -0,0 +1,64 @@ +[ + { + "name": "choose_qparams_per_row debug [4, 8]", + "operator": "etvk.choose_qparams_per_row.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_INPUT_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 8], + "dtype": "float32", + "storage": "v_INPUT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "tensor": { + "name": "scale", + "sizes": [4], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + }, + { + "tensor": { + "name": "zero_point", + "sizes": [4], + "dtype": "int8", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/quantization/choose_qparams_per_row/cases/performance.json b/backends/vulkan/test/ops/benchmark_v2/operators/quantization/choose_qparams_per_row/cases/performance.json new file mode 100644 index 00000000000..fbeac1b3a1a --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/quantization/choose_qparams_per_row/cases/performance.json @@ -0,0 +1,64 @@ +[ + { + "name": "choose_qparams_per_row performance [1, 8096]", + "operator": "etvk.choose_qparams_per_row.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_INPUT_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8096], + "dtype": "float32", + "storage": "v_INPUT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "tensor": { + "name": "scale", + "sizes": [1], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + }, + { + "tensor": { + "name": "zero_point", + "sizes": [1], + "dtype": "int8", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/Q8taQdqReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/Q8taQdqReference.cpp new file mode 100644 index 00000000000..1b072e4d3e1 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/Q8taQdqReference.cpp @@ -0,0 +1,49 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +namespace vkcompute::shader_profiling::reference { +static Tensor q8ta_quantize_reference( + const Json& spec, + const TensorMap& tensors, + size_t threads) { + Tensor out = tensor_arg(spec, 0, tensors); + const float s = q8ta::scale(spec, 1); + const int64_t zp = q8ta::integer(spec, 2); + const int64_t minimum = q8ta::integer(spec, 3), + maximum = q8ta::integer(spec, 4); + require( + -128 <= minimum && minimum <= maximum && maximum <= 127, + "Quantize reference requires signed int8 quantization bounds"); + parallel_for(out.values.size(), threads, [&](size_t i) { + out.values[i] = double(std::clamp( + q8ta::quantize(float(out.values[i]), s, zp), minimum, maximum)); + }); + return out; +} +static Tensor q8ta_dequantize_reference( + const Json& spec, + const TensorMap& tensors, + size_t threads) { + Tensor out = tensor_arg(spec, 0, tensors); + const float s = q8ta::scale(spec, 1); + const int64_t zp = q8ta::integer(spec, 2); + parallel_for(out.values.size(), threads, [&](size_t i) { + out.values[i] = q8ta::dequantize(int64_t(out.values[i]), s, zp); + }); + return out; +} +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_FN(q8ta_quantize, q8ta_quantize_reference); + REGISTER_REFERENCE_FN(q8ta_dequantize, q8ta_dequantize_reference); + REGISTER_OPERATOR_REFERENCE( + quantized_decomposed.quantize_per_tensor.default, q8ta_quantize); + REGISTER_OPERATOR_REFERENCE( + quantized_decomposed.dequantize_per_tensor.default, q8ta_dequantize); +} +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/Q8taQdqRoutes.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/Q8taQdqRoutes.cpp new file mode 100644 index 00000000000..b2b070b8242 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/Q8taQdqRoutes.cpp @@ -0,0 +1,67 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include + +namespace vkcompute::shader_profiling { +namespace { +void check_legacy_fp_layout(ComputeGraph& graph, ValueRef tensor) { + const auto storage = graph.storage_type_of(tensor); + const auto layout = graph.estimate_memory_layout_of(tensor); + if (!((storage == utils::kBuffer && layout == utils::kWidthPacked) || + (storage == utils::kTexture3D && layout == utils::kChannelsPacked))) { + throw UnsupportedRoute( + "Legacy 4w4c Q/DQ requires width-packed floating buffers or " + "channels-packed texture3d floating tensors"); + } +} +} // namespace + +static Json quantize_legacy_route( + ComputeGraph& graph, + const std::vector& args, + const Json& options) { + if (!options.empty()) + throw std::invalid_argument("Legacy quantize route accepts no options"); + const ValueRef output = args.back(); + if (graph.estimate_memory_layout_of(output) != utils::kPackedInt8_4W4C) { + VK_GET_OP_FN("quantized_decomposed.quantize_per_tensor.default") + (graph, args); + return {{"name", "auto"}, {"options", Json::object()}}; + } + check_legacy_fp_layout(graph, args[0]); + add_quantize_and_pack_4w4c_node(graph, args[0], args[1], args[2], output); + return {{"name", "legacy_4w4c"}, {"options", Json::object()}}; +} +static Json dequantize_legacy_route( + ComputeGraph& graph, + const std::vector& args, + const Json& options) { + if (!options.empty()) + throw std::invalid_argument("Legacy dequantize route accepts no options"); + if (graph.estimate_memory_layout_of(args[0]) != utils::kPackedInt8_4W4C) { + VK_GET_OP_FN("quantized_decomposed.dequantize_per_tensor.default") + (graph, args); + return {{"name", "auto"}, {"options", Json::object()}}; + } + check_legacy_fp_layout(graph, args.back()); + add_unpack_4w4c_and_dequantize_node( + graph, args[0], args[1], args[2], args.back()); + return {{"name", "legacy_4w4c"}, {"options", Json::object()}}; +} +REGISTER_OPERATOR_ROUTES { + REGISTER_OPERATOR_ROUTE( + quantized_decomposed.quantize_per_tensor.default, + legacy_4w4c, + quantize_legacy_route); + REGISTER_OPERATOR_ROUTE( + quantized_decomposed.dequantize_per_tensor.default, + legacy_4w4c, + dequantize_legacy_route); +} +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/Q8taQdqThroughput.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/Q8taQdqThroughput.cpp new file mode 100644 index 00000000000..5ef4126554c --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/Q8taQdqThroughput.cpp @@ -0,0 +1,30 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +namespace vkcompute::shader_profiling { +namespace { +double quantize_flops(const Json& spec) { + return 2 * throughput::tensor_elements(throughput::output_tensor(spec)); +} +double dequantize_flops(const Json& spec) { + return throughput::tensor_elements(throughput::output_tensor(spec)); +} +REGISTER_THROUGHPUT_FUNCTIONS { + REGISTER_OPERATOR_MEMORY_TRAFFIC( + quantized_decomposed.quantize_per_tensor.default, + throughput::tensor_traffic); + REGISTER_OPERATOR_MEMORY_TRAFFIC( + quantized_decomposed.dequantize_per_tensor.default, + throughput::tensor_traffic); + REGISTER_OPERATOR_COMPUTE_FLOPS( + quantized_decomposed.quantize_per_tensor.default, quantize_flops); + REGISTER_OPERATOR_COMPUTE_FLOPS( + quantized_decomposed.dequantize_per_tensor.default, dequantize_flops); +} +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/Q8taReferenceCommon.h b/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/Q8taReferenceCommon.h new file mode 100644 index 00000000000..319b6dc5dcd --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/Q8taReferenceCommon.h @@ -0,0 +1,90 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once +#include +#include +#include +#include + +namespace vkcompute::shader_profiling::reference::q8ta { +inline float scale(const Json& spec, size_t i) { + return spec.at("args").at(i).at("float").get(); +} +inline int64_t integer(const Json& spec, size_t i) { + return spec.at("args").at(i).at("int").get(); +} +inline int64_t requantize(float value, float inverse_scale, int64_t zp) { + const float scaled = value * inverse_scale; + if (scaled < float(-128 - zp) - 1.0f) + return -128; + if (scaled > float(127 - zp) + 1.0f) + return 127; + const float base = std::floor(scaled); + const float fraction = scaled - base; + const bool round_up = + fraction > .5f || (fraction == .5f && std::fmod(base, 2.0f) != 0.0f); + const int64_t rounded = int64_t(base) + int64_t(round_up); + return std::clamp(rounded + zp, int64_t(-128), int64_t(127)); +} +inline int64_t quantize(float value, float s, int64_t zp) { + return requantize(value, 1.0f / s, zp); +} +inline int64_t requantize_accumulator( + int64_t corrected_accumulator, + float input_scale, + float weight_scale, + float output_scale, + int64_t output_zero_point, + float bias = 0.0f, + bool relu = false, + bool fused_bias = true) { + const float combined_scale = weight_scale * input_scale; + float value = std::fma( + float(corrected_accumulator), combined_scale, fused_bias ? bias : 0.0f); + if (!fused_bias) + value += bias; + if (relu) + value = std::max(value, 0.0f); + return requantize(value, 1.0f / output_scale, output_zero_point); +} +inline float dequantize(int64_t value, float s, int64_t zp) { + return float(value - zp) * s; +} +inline Tensor quantized_input(const Tensor& input, float s, int64_t zp) { + Tensor result = input; + for (double& value : result.values) + value = double(quantize(float(value), s, zp)); + return result; +} +inline Tensor output_tensor(const Json& spec, size_t index) { + const std::string name = spec.at("args").at(index).at("tensor"); + auto sizes = + spec.at("tensors").at(name).at("sizes").get>(); + size_t count = 1; + for (int64_t size : sizes) + count *= size_t(size); + return {std::move(sizes), std::vector(count)}; +} +inline size_t broadcast_index( + size_t index, + const std::vector& out, + const std::vector& sizes) { + size_t source = 0, stride = 1; + for (size_t i = 0; i < out.size(); ++i) { + const size_t dim = out.size() - 1 - i; + const size_t coordinate = index % size_t(out[dim]); + index /= size_t(out[dim]); + if (i < sizes.size()) { + const size_t source_dim = sizes.size() - 1 - i; + if (sizes[source_dim] != 1) + source += coordinate * stride; + stride *= size_t(sizes[source_dim]); + } + } + return source; +} +} // namespace vkcompute::shader_profiling::reference::q8ta diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/cases/correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/cases/correctness.json new file mode 100644 index 00000000000..5fc97e12388 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/cases/correctness.json @@ -0,0 +1,710 @@ +[ + { + "name": "q8ta/qdq/correctness quantize buffer", + "operator": "quantized_decomposed.quantize_per_tensor.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.014060718249784, + "rtol": 0 + }, + "route": "auto", + "sweep_variables": { + "v_SIZES": [ + [144], + [90], + [1, 144], + [1, 90], + [1, 4], + [3, 32], + [1, 16, 32], + [1, 3, 64], + [1, 3, 16, 16], + [1, 8, 32, 32], + [1, 16, 24, 24], + [1, 32, 12, 12], + [1, 1, 64, 64], + [1, 3, 64, 64], + [1, 4, 16, 16], + [1, 8, 20, 20], + [1, 16, 14, 14], + [1, 8, 28, 28], + [1, 3, 15, 15], + [1, 13, 31, 31], + [1, 17, 23, 23], + [1, 64, 128, 128], + [1, 32, 64, 64], + [1, 128, 56, 56], + [2, 3, 16, 16], + [2, 8, 8, 8], + [2, 4, 8, 16], + [2, 3, 8, 8] + ], + "v_FLOAT_LAYOUT": ["width", "channels"], + "v_PACKED_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "buffer", + "layout": "v_FLOAT_LAYOUT", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_PACKED_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/qdq/correctness quantize texture3d", + "operator": "quantized_decomposed.quantize_per_tensor.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.014060718249784, + "rtol": 0 + }, + "route": "auto", + "sweep_variables": { + "v_SIZES": [ + [144], + [90], + [1, 144], + [1, 90], + [1, 4], + [3, 32], + [1, 16, 32], + [1, 3, 64], + [1, 3, 16, 16], + [1, 8, 32, 32], + [1, 16, 24, 24], + [1, 32, 12, 12], + [1, 1, 64, 64], + [1, 3, 64, 64], + [1, 4, 16, 16], + [1, 8, 20, 20], + [1, 16, 14, 14], + [1, 8, 28, 28], + [1, 3, 15, 15], + [1, 13, 31, 31], + [1, 17, 23, 23], + [1, 64, 128, 128], + [1, 32, 64, 64], + [1, 128, 56, 56] + ], + "v_FLOAT_LAYOUT": ["width", "channels"], + "v_PACKED_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "texture3d", + "layout": "v_FLOAT_LAYOUT", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_PACKED_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/qdq/correctness quantize buffer legacy_4w4c", + "operator": "quantized_decomposed.quantize_per_tensor.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.014060718249784, + "rtol": 0 + }, + "route": "legacy_4w4c", + "sweep_variables": { + "v_SIZES": [ + [144], + [90], + [1, 144], + [1, 90], + [1, 4], + [3, 32], + [1, 16, 32], + [1, 3, 64], + [1, 3, 16, 16], + [1, 8, 32, 32], + [1, 16, 24, 24], + [1, 32, 12, 12], + [1, 1, 64, 64], + [1, 3, 64, 64], + [1, 4, 16, 16], + [1, 8, 20, 20], + [1, 16, 14, 14], + [1, 8, 28, 28], + [1, 3, 15, 15], + [1, 13, 31, 31], + [1, 17, 23, 23], + [1, 64, 128, 128], + [1, 32, 64, 64], + [1, 128, 56, 56] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "buffer", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/qdq/correctness quantize texture3d legacy_4w4c", + "operator": "quantized_decomposed.quantize_per_tensor.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.014060718249784, + "rtol": 0 + }, + "route": "legacy_4w4c", + "sweep_variables": { + "v_SIZES": [ + [144], + [90], + [1, 144], + [1, 90], + [1, 4], + [3, 32], + [1, 16, 32], + [1, 3, 64], + [1, 3, 16, 16], + [1, 8, 32, 32], + [1, 16, 24, 24], + [1, 32, 12, 12], + [1, 1, 64, 64], + [1, 3, 64, 64], + [1, 4, 16, 16], + [1, 8, 20, 20], + [1, 16, 14, 14], + [1, 8, 28, 28], + [1, 3, 15, 15], + [1, 13, 31, 31], + [1, 17, 23, 23], + [1, 64, 128, 128], + [1, 32, 64, 64], + [1, 128, 56, 56] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/qdq/correctness dequantize buffer", + "operator": "quantized_decomposed.dequantize_per_tensor.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0072119999676942825, + "rtol": 0.0010000000474974513 + }, + "route": "auto", + "sweep_variables": { + "v_SIZES": [ + [144], + [90], + [1, 144], + [1, 90], + [1, 4], + [3, 32], + [1, 16, 32], + [1, 3, 64], + [1, 3, 16, 16], + [1, 8, 32, 32], + [1, 16, 24, 24], + [1, 32, 12, 12], + [1, 1, 64, 64], + [1, 3, 64, 64], + [1, 4, 16, 16], + [1, 8, 20, 20], + [1, 16, 14, 14], + [1, 8, 28, 28], + [1, 3, 15, 15], + [1, 13, 31, 31], + [1, 17, 23, 23], + [1, 64, 128, 128], + [1, 32, 64, 64], + [1, 128, 56, 56], + [2, 3, 16, 16], + [2, 8, 8, 8], + [2, 4, 8, 16], + [2, 3, 8, 8] + ], + "v_FLOAT_LAYOUT": ["width", "channels"], + "v_PACKED_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_PACKED_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "int": 2 + }, + { + "int": 10 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "buffer", + "layout": "v_FLOAT_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/qdq/correctness dequantize texture3d", + "operator": "quantized_decomposed.dequantize_per_tensor.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0072119999676942825, + "rtol": 0.0010000000474974513 + }, + "route": "auto", + "sweep_variables": { + "v_SIZES": [ + [144], + [90], + [1, 144], + [1, 90], + [1, 4], + [3, 32], + [1, 16, 32], + [1, 3, 64], + [1, 3, 16, 16], + [1, 8, 32, 32], + [1, 16, 24, 24], + [1, 32, 12, 12], + [1, 1, 64, 64], + [1, 3, 64, 64], + [1, 4, 16, 16], + [1, 8, 20, 20], + [1, 16, 14, 14], + [1, 8, 28, 28], + [1, 3, 15, 15], + [1, 13, 31, 31], + [1, 17, 23, 23], + [1, 64, 128, 128], + [1, 32, 64, 64], + [1, 128, 56, 56] + ], + "v_FLOAT_LAYOUT": ["width", "channels"], + "v_PACKED_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_PACKED_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "int": 2 + }, + { + "int": 10 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "texture3d", + "layout": "v_FLOAT_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/qdq/correctness dequantize buffer legacy_4w4c", + "operator": "quantized_decomposed.dequantize_per_tensor.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0072119999676942825, + "rtol": 0.0010000000474974513 + }, + "route": "legacy_4w4c", + "sweep_variables": { + "v_SIZES": [ + [144], + [90], + [1, 144], + [1, 90], + [1, 4], + [3, 32], + [1, 16, 32], + [1, 3, 64], + [1, 3, 16, 16], + [1, 8, 32, 32], + [1, 16, 24, 24], + [1, 32, 12, 12], + [1, 1, 64, 64], + [1, 3, 64, 64], + [1, 4, 16, 16], + [1, 8, 20, 20], + [1, 16, 14, 14], + [1, 8, 28, 28], + [1, 3, 15, 15], + [1, 13, 31, 31], + [1, 17, 23, 23], + [1, 64, 128, 128], + [1, 32, 64, 64], + [1, 128, 56, 56] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "int": 2 + }, + { + "int": 10 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "buffer", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/qdq/correctness dequantize texture3d legacy_4w4c", + "operator": "quantized_decomposed.dequantize_per_tensor.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0072119999676942825, + "rtol": 0.0010000000474974513 + }, + "route": "legacy_4w4c", + "sweep_variables": { + "v_SIZES": [ + [144], + [90], + [1, 144], + [1, 90], + [1, 4], + [3, 32], + [1, 16, 32], + [1, 3, 64], + [1, 3, 16, 16], + [1, 8, 32, 32], + [1, 16, 24, 24], + [1, 32, 12, 12], + [1, 1, 64, 64], + [1, 3, 64, 64], + [1, 4, 16, 16], + [1, 8, 20, 20], + [1, 16, 14, 14], + [1, 8, 28, 28], + [1, 3, 15, 15], + [1, 13, 31, 31], + [1, 17, 23, 23], + [1, 64, 128, 128], + [1, 32, 64, 64], + [1, 128, 56, 56] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "int": 2 + }, + { + "int": 10 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/cases/debug.json b/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/cases/debug.json new file mode 100644 index 00000000000..20f670d615f --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/cases/debug.json @@ -0,0 +1,274 @@ +[ + { + "name": "q8ta/qdq/debug float32 buffer auto", + "operator": "quantized_decomposed.quantize_per_tensor.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.014060718249784, + "rtol": 0 + }, + "route": "auto", + "sweep_variables": { + "v_INPUT_LAYOUT": ["width", "channels"], + "v_SIZES": [ + [1, 16, 16, 16], + [1, 144], + [1, 90], + [2, 3, 16, 16], + [2, 4, 8, 16] + ], + "v_OUTPUT_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "buffer", + "layout": "v_INPUT_LAYOUT", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_OUTPUT_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/qdq/debug float32 buffer legacy_4w4c", + "operator": "quantized_decomposed.quantize_per_tensor.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.014060718249784, + "rtol": 0 + }, + "route": "legacy_4w4c", + "sweep_variables": { + "v_INPUT_LAYOUT": ["width", "channels"], + "v_SIZES": [ + [1, 16, 16, 16], + [1, 144], + [1, 90] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "buffer", + "layout": "v_INPUT_LAYOUT", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/qdq/debug int8x4 buffer auto", + "operator": "quantized_decomposed.dequantize_per_tensor.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0072119999676942825, + "rtol": 0.0010000000474974513 + }, + "route": "auto", + "sweep_variables": { + "v_INPUT_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"], + "v_SIZES": [ + [1, 16, 16, 16], + [1, 144], + [1, 90], + [2, 3, 16, 16], + [2, 4, 8, 16] + ], + "v_OUTPUT_LAYOUT": ["width", "channels"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_INPUT_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "int": 2 + }, + { + "int": 10 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "buffer", + "layout": "v_OUTPUT_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/qdq/debug int8x4 buffer 4w4c legacy_4w4c", + "operator": "quantized_decomposed.dequantize_per_tensor.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0072119999676942825, + "rtol": 0.0010000000474974513 + }, + "route": "legacy_4w4c", + "sweep_variables": { + "v_SIZES": [ + [1, 16, 16, 16], + [1, 144], + [1, 90] + ], + "v_OUTPUT_LAYOUT": ["width", "channels"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "int": 2 + }, + { + "int": 10 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "buffer", + "layout": "v_OUTPUT_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/test/q8_benchmark_reference_test.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/test/q8_benchmark_reference_test.cpp new file mode 100644 index 00000000000..bdab0110e8b --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/test/q8_benchmark_reference_test.cpp @@ -0,0 +1,333 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +#include +#include +namespace vkcompute::shader_profiling { +namespace { +Json tensor_arg( + const char* name, + const Json& sizes, + const Json& data, + const char* dtype = "float32", + const char* layout = "width") { + return { + {"tensor", + {{"name", name}, + {"sizes", sizes}, + {"dtype", dtype}, + {"layout", layout}, + {"storage", + std::string(name) == "input" || std::string(name) == "output" || + std::string(name) == "other" + ? "buffer" + : "constant"}, + {"data", data}}}}; +} +std::vector compute(const char* op, const Json& args) { + const Json spec = + parse_configs({{"operator", op}, {"args_sweep", Json::array({args})}}) + .front(); + auto data = make_tensor_data(spec.at("tensors")); + data.erase("output"); + return reference_outputs(spec, {}, data, 2).at("output"); +} +TEST(Q8BenchmarkReference, AddBroadcastsQuantizedInputsAndHonorsAlpha) { + const Json args = Json::array( + {tensor_arg("input", {2, 1}, {2, 4}, "int8x4", "4w"), + tensor_arg("other", {2}, {-2, 4}, "int8x4", "4w"), + {{"float", .5}}, + {{"int", 0}}, + {{"float", .25}}, + {{"int", 0}}, + {{"float", .5}}, + {{"int", 0}}, + {{"float", 2}}, + tensor_arg("output", {2, 2}, {{"fill", 0}}, "int8x4", "4w")}); + EXPECT_EQ( + compute("et_vk.q8ta_add.default", args), + (std::vector{0, 6, 2, 8})); +} +TEST(Q8BenchmarkReference, AddAlphaAndSaturationWithBroadcasting) { + Json args = Json::array( + {tensor_arg("input", {2, 1}, {120, -120}, "int8x4", "4w"), + tensor_arg("other", {2}, {-20, 20}, "int8x4", "4w"), + {{"float", 1}}, + {{"int", 0}}, + {{"float", 1}}, + {{"int", 0}}, + {{"float", 1}}, + {{"int", 0}}, + {{"float", 0}}, + tensor_arg("output", {2, 2}, {{"fill", 0}}, "int8x4", "4w")}); + EXPECT_EQ( + compute("et_vk.q8ta_add.default", args), + (std::vector{120, 120, -120, -120})); + args[8]["float"] = .5; + EXPECT_EQ( + compute("et_vk.q8ta_add.default", args), + (std::vector{110, 127, -128, -110})); + args[8]["float"] = -1; + EXPECT_EQ( + compute("et_vk.q8ta_add.default", args), + (std::vector{127, 100, -100, -128})); +} +TEST(Q8BenchmarkReference, ClonePreservesSignedBytesAcrossLayouts) { + const Json args = Json::array( + {tensor_arg("input", {5}, {-128, -1, 0, 1, 127}, "int8x4", "4w"), + nullptr, + tensor_arg("output", {5}, {{"fill", 0}}, "int8x4", "4c1w")}); + EXPECT_EQ( + compute("aten.clone.default", args), + (std::vector{-128, -1, 0, 1, 127})); +} +TEST(Q8BenchmarkReference, QuantizeRoundsAndClampsIntegerOutput) { + const Json args = Json::array( + {tensor_arg("input", {5}, {-100, -.26, 0, .26, 100}), + {{"float", .5}}, + {{"int", 0}}, + {{"int", -128}}, + {{"int", 127}}, + {{"int", 2}}, + tensor_arg("output", {5}, {{"fill", 0}}, "int8x4", "4w")}); + EXPECT_EQ( + compute("quantized_decomposed.quantize_per_tensor.default", args), + (std::vector{-128, -1, 0, 1, 127})); +} +TEST(Q8BenchmarkReference, DequantizeAppliesScaleAndZeroPoint) { + const Json args = Json::array( + {tensor_arg("input", {5}, {-128, -1, 0, 1, 127}, "int8x4", "4w"), + {{"float", .5}}, + {{"int", -2}}, + {{"int", -128}}, + {{"int", 127}}, + {{"int", 2}}, + {{"int", 10}}, + tensor_arg("output", {5}, {{"fill", 0}})}); + EXPECT_EQ( + compute("quantized_decomposed.dequantize_per_tensor.default", args), + (std::vector{-63, .5, 1, 1.5, 64.5})); +} +TEST(Q8BenchmarkReference, ReluUsesQuantizedZeroPoints) { + const Json args = Json::array( + {tensor_arg("input", {4}, {-4, -2, 0, 2}, "int8x4", "4w"), + {{"float", 1}}, + {{"int", -2}}, + {{"float", .5}}, + {{"int", 3}}, + tensor_arg("output", {4}, {{"fill", 0}}, "int8x4", "4w")}); + EXPECT_EQ( + compute("et_vk.q8ta_relu.default", args), + (std::vector{3, 3, 7, 11})); +} +TEST(Q8BenchmarkReference, PixelShufflePermutesQuantizedBytes) { + const Json args = Json::array( + {tensor_arg("input", {1, 4, 1, 1}, {1, 2, 3, 4}, "int8x4", "4w4c"), + {{"float", 1}}, + {{"int", 0}}, + {{"float", 1}}, + {{"int", 0}}, + {{"int", 2}}, + tensor_arg("output", {1, 1, 2, 2}, {{"fill", 0}}, "int8x4", "4c1w")}); + EXPECT_EQ( + compute("et_vk.q8ta_pixel_shuffle.default", args), + (std::vector{1, 2, 3, 4})); +} +TEST(Q8BenchmarkReference, PixelShuffleRequantizesUsingInverseOutputScale) { + const Json args = Json::array( + {tensor_arg("input", {1, 4, 1, 1}, {-128, -1, 1, 127}, "int8x4", "4w4c"), + {{"float", .5}}, + {{"int", -1}}, + {{"float", 2}}, + {{"int", 3}}, + {{"int", 2}}, + tensor_arg("output", {1, 1, 2, 2}, {{"fill", 0}}, "int8x4", "4c1w")}); + EXPECT_EQ( + compute("et_vk.q8ta_pixel_shuffle.default", args), + (std::vector{-124, 3, 5, 127})); +} +Json convolution_args( + const Json& input, + const Json& weight, + const Json& scales, + const Json& bias, + const Json& output) { + return Json::array( + {input, + {{"float", 1}}, + {{"int", 0}}, + weight, + tensor_arg("sums", {4}, {0, 0, 0, 0}, "int32"), + scales, + {{"float", 1}}, + {{"int", 0}}, + bias, + {{"int_list", {1, 1}}}, + {{"int_list", {1, 1}}}, + {{"int_list", {0, 0}}}, + {{"int_list", {1, 1}}}, + {{"int", 1}}, + {{"string", "none"}}, + output}); +} +TEST(Q8BenchmarkReference, ConvolutionIgnoresPaddedWeightAndChannelParameters) { + const Json args = convolution_args( + tensor_arg("input", {1, 3, 1, 1}, {1, 2, 3}, "int8x4", "4c"), + tensor_arg("weight", {2, 4}, {1, 2, 3, 99, 4, 5, 6, -99}, "int8"), + tensor_arg("scales", {4}, {1, 1, 99, 99}), + tensor_arg("bias", {4}, {0, 0, 99, 99}), + tensor_arg("output", {1, 2, 1, 1}, {{"fill", 0}}, "int8x4", "4c")); + EXPECT_EQ( + compute("et_vk.q8ta_conv2d.default", args), + (std::vector{14, 32})); +} +TEST(Q8BenchmarkReference, DepthwiseReadsHeightWidthChannelWeightOrder) { + Json args = convolution_args( + tensor_arg("input", {1, 2, 1, 2}, {1, 2, 3, 4}, "int8x4", "4c"), + tensor_arg("weight", {1, 1, 2}, {2, -1}, "int8"), + tensor_arg("scales", {2}, {1, 1}), + tensor_arg("bias", {2}, {0, 0}), + tensor_arg("output", {1, 2, 1, 2}, {{"fill", 0}}, "int8x4", "4c")); + args[13]["int"] = 2; + EXPECT_EQ( + compute("et_vk.q8ta_conv2d_dw.default", args), + (std::vector{2, 4, -3, -4})); +} +TEST(Q8BenchmarkReference, LinearUsesIntegerInputsAndWeightSumCorrection) { + const Json args = Json::array( + {tensor_arg("input", {1, 4}, {4, 2, 3, 1}, "int8x4", "4h4w"), + {{"float", 1}}, + {{"int", 1}}, + tensor_arg("weight", {2, 4}, {1, -1, 0, 0, 2, 0, -1, 0}, "int8"), + tensor_arg("sums", {2}, {0, 1}, "int32"), + tensor_arg("scales", {2}, {1, 1}), + {{"float", 1}}, + {{"int", 0}}, + tensor_arg("bias", {2}, {1, -1}), + {{"string", "none"}}, + tensor_arg("output", {1, 2}, {{"fill", 0}}, "int8x4", "4h4w")}); + EXPECT_EQ( + compute("et_vk.q8ta_linear.default", args), (std::vector{3, 3})); +} +TEST(Q8BenchmarkReference, TransposedConvolutionExpandsStridedKernel) { + Json args = convolution_args( + tensor_arg("input", {1, 1, 1, 1}, {2}, "int8x4", "4c1w"), + tensor_arg("weight", {1, 4}, {1, 2, 3, 4}, "int8"), + tensor_arg("scales", {1}, {1}), + nullptr, + tensor_arg("output", {1, 1, 2, 2}, {{"fill", 0}}, "int8x4", "4c1w")); + args[9]["int_list"] = {2, 2}; + args[10]["int_list"] = {2, 2}; + args.insert(args.begin() + 12, Json{{"int_list", {0, 0}}}); + EXPECT_EQ( + compute("et_vk.q8ta_conv2d_transposed.default", args), + (std::vector{2, 4, 6, 8})); +} +TEST( + Q8BenchmarkReference, + AccumulatorRequantizationHandlesBiasReluAndClamping) { + EXPECT_EQ( + reference::q8ta::requantize_accumulator(7, .5f, 1.0f, 1.0f, -3, .5f), 1); + EXPECT_EQ( + reference::q8ta::requantize_accumulator( + -8, .5f, 1.0f, 1.0f, 4, 0.0f, true), + 4); + EXPECT_EQ( + reference::q8ta::requantize_accumulator(10000, 1.0f, 1.0f, 1.0f, 0), 127); + EXPECT_EQ( + reference::q8ta::requantize_accumulator(-10000, 1.0f, 1.0f, 1.0f, 0), + -128); +} +TEST( + Q8BenchmarkReference, + NativeLinearThroughputCountsIntegerTensorTrafficAndMacs) { + const Json args = Json::array( + {tensor_arg("input", {1, 4}, {1, 2, 3, 4}, "int8x4", "4h4w"), + {{"float", 1}}, + {{"int", 0}}, + tensor_arg("weight", {2, 4}, {1, 1, 1, 1, 1, 1, 1, 1}, "int8"), + tensor_arg("sums", {2}, {4, 4}, "int32"), + tensor_arg("scales", {2}, {1, 1}), + {{"float", 1}}, + {{"int", 0}}, + nullptr, + {{"string", "none"}}, + tensor_arg("output", {1, 2}, {{"fill", 0}}, "int8x4", "4h4w")}); + const Json spec = parse_configs({{"operator", "et_vk.q8ta_linear.default"}, + {"args_sweep", Json::array({args})}}) + .front(); + const auto work = estimate_work( + Json(operator_report(spec, Json::object(), {{"output", "write"}})), + true, + true); + ASSERT_TRUE(work.memory_bytes.has_value()); + ASSERT_TRUE(work.compute_flops.has_value()); + EXPECT_EQ(*work.memory_bytes, 30); + EXPECT_EQ(*work.compute_flops, 16); +} +TEST( + Q8BenchmarkReference, + QuantizationUsesReciprocalAndRoundsTiesBeforeZeroPoint) { + EXPECT_EQ( + reference::q8ta::quantize(-0.17499999702f, 0.070000000298f, -3), -5); + EXPECT_EQ(reference::q8ta::quantize(0.17499999702f, 0.070000000298f, 3), 5); + EXPECT_EQ( + reference::q8ta::quantize(-0.324000000954f, 0.00800000037998f, -2), -42); + const std::vector values{ + .5f, 1.5f, 2.5f, 3.5f, -.5f, -1.5f, -2.5f, -3.5f}; + const std::vector expected{3, 5, 5, 7, 3, 1, 1, -1}; + for (size_t i = 0; i < values.size(); ++i) + EXPECT_EQ(reference::q8ta::quantize(values[i], 1.0f, 3), expected[i]); +} +TEST( + Q8BenchmarkReference, + ActivationQuantizedConvolutionAppliesScaleAfterIntegerDot) { + const float scale = .07f, weight_scale = .013f; + const Json args = Json::array( + {tensor_arg("input", {1, 1, 1, 1}, {-0.17499999702f}), + {{"float", scale}}, + {{"int", -3}}, + tensor_arg("weight", {1, 4}, {127, 0, 0, 0}, "int8"), + tensor_arg("sums", {1}, {127}, "int32"), + tensor_arg("scales", {1}, {weight_scale}), + tensor_arg("bias", {1}, {.5}), + {{"int_list", {1, 1}}}, + {{"int_list", {1, 1}}}, + {{"int_list", {0, 0}}}, + {{"int_list", {1, 1}}}, + {{"int", 1}}, + tensor_arg("output", {1, 1, 1, 1}, {{"fill", 0}})}); + const float expected = float(-2 * 127) * (scale * weight_scale) + .5f; + EXPECT_EQ( + compute("et_vk.conv2d_q8ta_q8csw.default", args), + (std::vector{expected})); +} +TEST( + Q8BenchmarkReference, + ActivationQuantizedLinearCancelsIntegersBeforeScaling) { + constexpr int64_t k = 2048; + Json values = Json::array_t(size_t(k), Json(.07f)); + Json weights = Json::array_t(size_t(k), Json(-127)); + for (size_t i = 0; i < size_t(k) / 2; ++i) + weights[i] = 127; + const Json args = Json::array( + {tensor_arg("input", {1, k}, values), + {{"float", .07f}}, + {{"int", 0}}, + tensor_arg("weight", {1, k}, weights, "int8"), + tensor_arg("sums", {1}, {0}, "int32"), + tensor_arg("scales", {1}, {.013f}), + tensor_arg("bias", {1}, {.5}), + tensor_arg("output", {1, 1}, {{"fill", 0}})}); + EXPECT_EQ( + compute("et_vk.linear_q8ta_q8csw.default", args), + (std::vector{.5})); +} +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/unary/q8ta/Q8taUnaryReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/unary/q8ta/Q8taUnaryReference.cpp new file mode 100644 index 00000000000..7d16feea0c8 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/unary/q8ta/Q8taUnaryReference.cpp @@ -0,0 +1,30 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +namespace vkcompute::shader_profiling::reference { +static Tensor q8ta_relu_reference( + const Json& spec, + const TensorMap& tensors, + size_t threads) { + Tensor out = tensor_arg(spec, 0, tensors); + const float in_scale = q8ta::scale(spec, 1), out_scale = q8ta::scale(spec, 3); + const int64_t in_zp = q8ta::integer(spec, 2), out_zp = q8ta::integer(spec, 4); + parallel_for(out.values.size(), threads, [&](size_t i) { + const float value = + q8ta::dequantize(int64_t(out.values[i]), in_scale, in_zp); + out.values[i] = + double(q8ta::quantize(std::max(value, 0.0f), out_scale, out_zp)); + }); + return out; +} +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_FN(q8ta_relu, q8ta_relu_reference); + REGISTER_OPERATOR_REFERENCE(et_vk.q8ta_relu.default, q8ta_relu); +} +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/unary/q8ta/Q8taUnaryThroughput.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/unary/q8ta/Q8taUnaryThroughput.cpp new file mode 100644 index 00000000000..97a52bfa5a8 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/unary/q8ta/Q8taUnaryThroughput.cpp @@ -0,0 +1,20 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +namespace vkcompute::shader_profiling { +namespace { +double relu_flops(const Json& spec) { + return 3 * throughput::tensor_elements(throughput::output_tensor(spec)); +} +REGISTER_THROUGHPUT_FUNCTIONS { + REGISTER_OPERATOR_MEMORY_TRAFFIC( + et_vk.q8ta_relu.default, throughput::tensor_traffic); + REGISTER_OPERATOR_COMPUTE_FLOPS(et_vk.q8ta_relu.default, relu_flops); +} +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/unary/q8ta/cases/correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/unary/q8ta/cases/correctness.json new file mode 100644 index 00000000000..1a21215d212 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/unary/q8ta/cases/correctness.json @@ -0,0 +1,115 @@ +[ + { + "name": "q8ta/unary/correctness", + "operator": "et_vk.q8ta_relu.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.014060718249784, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"], + "v_SIZES": [ + [1, 3, 16, 16], + [1, 8, 32, 32], + [1, 16, 24, 24], + [1, 32, 12, 12], + [1, 1, 64, 64], + [1, 3, 64, 64], + [1, 4, 16, 16], + [1, 8, 20, 20], + [1, 16, 14, 14], + [1, 8, 28, 28], + [1, 3, 15, 15], + [1, 13, 31, 31], + [1, 17, 23, 23], + [1, 64, 128, 128], + [1, 32, 64, 64], + [1, 128, 56, 56], + [1, 128, 128, 128] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/unary/q8ta/cases/debug.json b/backends/vulkan/test/ops/benchmark_v2/operators/unary/q8ta/cases/debug.json new file mode 100644 index 00000000000..dd2b1ad1d1a --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/unary/q8ta/cases/debug.json @@ -0,0 +1,96 @@ +[ + { + "name": "q8ta/unary/debug", + "operator": "et_vk.q8ta_relu.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.014060718249784, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/targets.bzl b/backends/vulkan/test/ops/benchmark_v2/targets.bzl new file mode 100644 index 00000000000..278873c9807 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/targets.bzl @@ -0,0 +1,358 @@ +load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") +load("@fbsource//xplat/executorch/backends/vulkan:targets.bzl", "get_platforms", "vulkan_spv_shader_lib") + +def define_common_targets(is_fbcode = False): + if is_fbcode: + return + + runtime.filegroup( + name = "migration_shaders", + srcs = native.glob(["operators/gemm/linear/qw/q4gsw/glsl/**/*"]), + visibility = ["PUBLIC"], + ) + + vulkan_spv_shader_lib( + name = "migration_shaderlib", + spv_filegroups = {":migration_shaders": "operators"}, + is_fbcode = is_fbcode, + ) + + runtime.cxx_library( + name = "config", + srcs = [ + "framework/Config.cpp", + "framework/DataInitializer.cpp", + "framework/Sweep.cpp", + ], + exported_headers = [ + "framework/Config.h", + "framework/DataInitializer.h", + "framework/Sweep.h", + ], + platforms = get_platforms(), + exported_external_deps = ["nlohmann_json"], + visibility = ["PUBLIC"], + ) + + runtime.cxx_library( + name = "throughput", + srcs = [ + "framework/ThroughputRegistry.cpp", + "operators/binary/add/AddThroughput.cpp", + "operators/binary/q8ta/Q8taBinaryThroughput.cpp", + "operators/clone/q8ta/Q8taCloneThroughput.cpp", + "operators/gemm/linear/q8ta/Q8taLinearThroughput.cpp", + "operators/pixel_shuffle/q8ta/Q8taPixelShuffleThroughput.cpp", + "operators/quantize/q8ta/Q8taQdqThroughput.cpp", + "operators/unary/q8ta/Q8taUnaryThroughput.cpp", + "operators/attention/sdpa/SdpaThroughput.cpp", + "operators/gemm/matmul/MatmulThroughput.cpp", + "operators/quantization/choose_qparams_per_row/ChooseQparamsThroughput.cpp", + "operators/gemm/linear/qw/dq8ca_q4gsw/Dq8caQ4gswThroughput.cpp", + "operators/binary/BinaryThroughput.cpp", + "operators/gemm/GemmThroughputCommon.cpp", + "operators/gemm/linear/LinearThroughput.cpp", + "operators/gemm/linear/qw/q4gsw/Q4gswLinearThroughput.cpp", + "operators/convolution/Conv2dThroughputCommon.cpp", + "operators/convolution/Conv2dThroughput.cpp", + "operators/convolution/q8ta/Q8taConv2dThroughput.cpp", + ], + exported_headers = [ + "framework/ThroughputRegistry.h", + "operators/convolution/Conv2dThroughputCommon.h", + "operators/gemm/GemmThroughputCommon.h", + ], + platforms = get_platforms(), + exported_deps = [":config"], + # @lint-ignore BUCKLINT: Avoid `link_whole=True` + link_whole = True, + visibility = ["PUBLIC"], + ) + + runtime.cxx_test( + name = "throughput_registry_test", + srcs = ["framework/test/throughput_registry_test.cpp"], + platforms = get_platforms(), + deps = [":throughput", "//third-party/googletest:gtest_main"], + ) + + runtime.cxx_test( + name = "throughput_report_test", + srcs = ["framework/test/throughput_report_test.cpp"], + platforms = get_platforms(), + deps = [":report", "//third-party/googletest:gtest_main"], + ) + + runtime.cxx_library( + name = "report", + srcs = ["framework/Report.cpp"], + exported_headers = ["framework/Report.h"], + platforms = get_platforms(), + exported_deps = [":config", ":throughput"], + visibility = ["PUBLIC"], + ) + + runtime.cxx_test( + name = "report_test", + srcs = ["framework/test/report_test.cpp"], + platforms = get_platforms(), + deps = [ + ":report", + "//third-party/googletest:gtest_main", + ], + ) + + runtime.cxx_library( + name = "reference", + srcs = [ + "framework/Reference.cpp", + "framework/ReferenceRegistry.cpp", + "operators/binary/add/AddReference.cpp", + "operators/convolution/conv1d/Conv1dReference.cpp", + "operators/gemm/matmul/MatmulReference.cpp", + "operators/embedding/qw/q4gsw/EmbeddingReference.cpp", + "operators/gemm/linear/qw/dq8ca_q4gsw/Dq8caQ4gswReference.cpp", + "operators/attention/sdpa/SdpaReference.cpp", + "operators/quantization/choose_qparams_per_row/ChooseQparamsReference.cpp", + "operators/binary/q8ta/Q8taBinaryReference.cpp", + "operators/clone/q8ta/Q8taCloneReference.cpp", + "operators/convolution/q8ta/transposed/Q8taTransposedReference.cpp", + "operators/convolution/qw/q8csw/Q8cswConv2dReference.cpp", + "operators/gemm/linear/q8ta/Q8taLinearReference.cpp", + "operators/gemm/linear/qw/q8csw/Q8cswLinearReference.cpp", + "operators/pixel_shuffle/q8ta/Q8taPixelShuffleReference.cpp", + "operators/quantize/q8ta/Q8taQdqReference.cpp", + "operators/unary/q8ta/Q8taUnaryReference.cpp", + "operators/binary/BinaryReference.cpp", + "operators/convolution/Conv2dReferenceCommon.cpp", + "operators/convolution/Conv2dReference.cpp", + "operators/convolution/q8ta/Q8taConv2dReference.cpp", + "operators/gemm/GemmReference.cpp", + "operators/gemm/linear/LinearReference.cpp", + "operators/gemm/linear/qw/q4gsw/Q4gswLinearReference.cpp", + ], + exported_headers = [ + "framework/Reference.h", + "framework/ReferenceRegistry.h", + "operators/binary/add/AddReference.h", + "operators/gemm/matmul/MatmulReference.h", + "operators/embedding/qw/q4gsw/EmbeddingReference.h", + "operators/gemm/linear/qw/dq8ca_q4gsw/Dq8caQ4gswReference.h", + "operators/attention/sdpa/SdpaReference.h", + "operators/convolution/conv1d/Conv1dReference.h", + "operators/quantize/q8ta/Q8taReferenceCommon.h", + "operators/quantization/choose_qparams_per_row/ChooseQparamsReference.h", + "operators/binary/BinaryReference.h", + "operators/convolution/Conv2dReferenceCommon.h", + "operators/convolution/Conv2dReference.h", + "operators/convolution/q8ta/Q8taConv2dReference.h", + "operators/gemm/GemmReference.h", + "operators/gemm/linear/LinearReference.h", + "operators/gemm/linear/qw/q4gsw/Q4gswLinearReference.h", + ], + compiler_flags = ["-O3"], + # Static reference registrations must survive archive dead stripping. + # @lint-ignore BUCKLINT: Avoid `link_whole=True` + link_whole = True, + platforms = get_platforms(), + exported_deps = [ + ":report", + "//executorch/runtime/core/exec_aten:lib", + ], + visibility = ["PUBLIC"], + ) + + runtime.cxx_test( + name = "reference_test", + srcs = ["framework/test/reference_test.cpp"], + platforms = get_platforms(), + deps = [ + ":reference", + "//third-party/googletest:gtest_main", + ], + ) + + runtime.cxx_test( + name = "reference_registry_test", + srcs = ["framework/test/reference_registry_test.cpp"], + platforms = get_platforms(), + deps = [ + ":reference", + "//third-party/googletest:gtest_main", + ], + ) + + runtime.cxx_library( + name = "routes", + srcs = [ + "framework/RouteRegistry.cpp", + "operators/convolution/Conv2dRouteCommon.cpp", + "operators/convolution/Conv2dRoutes.cpp", + "operators/convolution/q8ta/Q8taConv2dRoutes.cpp", + "operators/gemm/matmul/MatmulRoutes.cpp", + "operators/convolution/conv2d/DepthwiseRoutes.cpp", + "operators/gemm/linear/qw/q4gsw/Q4gswRoutes.cpp", + "operators/attention/sdpa/SdpaRoutes.cpp", + "operators/gemm/linear/q8ta/Q8taLinearRoutes.cpp", + "operators/quantize/q8ta/Q8taQdqRoutes.cpp", + ], + exported_headers = [ + "framework/RouteRegistry.h", + "operators/convolution/Conv2dRouteCommon.h", + "operators/convolution/Conv2dRoutes.h", + "operators/convolution/q8ta/Q8taConv2dRoutes.h", + ], + platforms = get_platforms(), + exported_deps = [":config", ":migration_shaderlib", "//executorch/backends/vulkan:vulkan_graph_runtime"], + # @lint-ignore BUCKLINT: Avoid `link_whole=True` + link_whole = True, + visibility = ["PUBLIC"], + ) + + runtime.cxx_test( + name = "route_registry_test", + srcs = ["framework/test/route_registry_test.cpp"], + platforms = get_platforms(), + deps = [":routes", "//third-party/googletest:gtest_main"], + ) + + runtime.cxx_test( + name = "operator_route_test", + srcs = ["framework/test/operator_route_test.cpp"], + platforms = get_platforms(), + deps = [ + "//executorch/backends/vulkan/test/api_abstractions:test_utils", + ":routes", + "//executorch/runtime/core/exec_aten:lib", + "//third-party/googletest:gtest_main", + ], + ) + + runtime.cxx_library( + name = "gpu_clock", + srcs = ["framework/GpuClock.cpp"], + exported_headers = ["framework/GpuClock.h"], + platforms = get_platforms(), + visibility = ["PUBLIC"], + ) + + runtime.cxx_test( + name = "gpu_clock_test", + srcs = ["framework/test/gpu_clock_test.cpp"], + platforms = get_platforms(), + deps = [":gpu_clock", "//third-party/googletest:gtest_main"], + ) + + runtime.cxx_library( + name = "framework", + srcs = [ + "framework/CaseCheckRegistry.cpp", + "framework/OperatorBenchmark.cpp", + "framework/ShaderBenchmark.cpp", + "framework/TensorUtils.cpp", + "framework/Timestamps.cpp", + "framework/Timing.cpp", + "framework/Validation.cpp", + "framework/WorkgroupUtils.cpp", + ], + exported_headers = [ + "framework/CaseCheckRegistry.h", + "framework/OperatorBenchmark.h", + "framework/ShaderBenchmark.h", + "framework/TensorUtils.h", + "framework/Timestamps.h", + "framework/Timing.h", + "framework/Validation.h", + "framework/WorkgroupUtils.h", + ], + platforms = get_platforms(), + exported_deps = [ + ":config", + ":gpu_clock", + ":report", + ":reference", + ":routes", + "//executorch/backends/vulkan:vulkan_graph_runtime", + ], + visibility = ["PUBLIC"], + ) + + runtime.cxx_test( + name = "data_initializer_test", + srcs = ["framework/test/data_initializer_test.cpp"], + platforms = get_platforms(), + deps = [":config", "//third-party/googletest:gtest_main"], + ) + + runtime.cxx_test( + name = "config_test", + srcs = ["framework/test/config_test.cpp"], + platforms = get_platforms(), + deps = [ + ":config", + "//third-party/googletest:gtest_main", + ], + ) + + runtime.cxx_test( + name = "validation_test", + srcs = ["framework/test/validation_test.cpp"], + platforms = get_platforms(), + deps = [ + "//executorch/backends/vulkan/test/api_abstractions:test_utils", + ":framework", + "//third-party/googletest:gtest_main", + ], + ) + + runtime.cxx_library( + name = "case_checks", + srcs = ["operators/convolution/q8ta/Q8taValidationChecks.cpp"], + exported_headers = ["operators/convolution/q8ta/Q8taValidationChecks.h"], + platforms = get_platforms(), + exported_deps = [":framework"], + # @lint-ignore BUCKLINT: Avoid `link_whole=True` + link_whole = True, + visibility = ["PUBLIC"], + ) + + runtime.cxx_test( + name = "case_check_registry_test", + srcs = ["framework/test/case_check_registry_test.cpp"], + platforms = get_platforms(), + deps = [ + "//executorch/backends/vulkan/test/api_abstractions:test_utils",":framework", "//third-party/googletest:gtest_main"], + ) + + runtime.cxx_test( + name = "migration_reference_test", + srcs = ["operators/MigrationReferenceTest.cpp"], + platforms = get_platforms(), + deps = [":reference", "//third-party/googletest:gtest_main"], + ) + + runtime.cxx_test( + name = "q8_benchmark_reference_test", + srcs = ["operators/quantize/q8ta/test/q8_benchmark_reference_test.cpp"], + platforms = get_platforms(), + deps = [":reference", "//third-party/googletest:gtest_main"], + ) + + runtime.cxx_test( + name = "q8ta_native_reference_test", + srcs = ["operators/convolution/q8ta/test/q8ta_native_reference_test.cpp"], + platforms = get_platforms(), + deps = [":reference", "//third-party/googletest:gtest_main"], + ) + + runtime.cxx_binary( + name = "etvk_ops_bench", + srcs = ["etvk_ops_bench.cpp"], + platforms = get_platforms(), + define_static_target = False, + deps = [ + ":framework", + ":case_checks", + ], + external_deps = ["gflags"], + )