diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0c7e5d92..e54f77b5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,6 +19,7 @@ variables: CI_CONFIG_DUAL: 'cachepool_dual_fpu_4g' CI_CONFIG_16G: 'cachepool_fpu_16g' CI_CONFIG_DUAL_16G: 'cachepool_dual_fpu_16g' + # Binary prefix of the tests whose KERNEL has no group prefix (rlc-, ctrl-) SW_PREFIX: 'test-' default: @@ -169,16 +170,16 @@ test: - fmatmul-32b_M64_N64_K64 - fft-32b_M1024_N16 - byte-enable - - multi_producer_single_consumer_double_linked_list_M1_N1350_K10 - - multi_producer_single_consumer_double_linked_list_M1_N1350_K4_P2_C2_am_G1024_tbchk - - multi_producer_single_consumer_double_linked_list_P1_C3_ul_ci + - rlc-multi_producer_single_consumer_double_linked_list_M1_N1350_K10 + - rlc-multi_producer_single_consumer_double_linked_list_M1_N1350_K4_P2_C2_am_G1024_tbchk + - rlc-multi_producer_single_consumer_double_linked_list_P1_C3_ul_ci script: # The vsim script writes a .rtlbinary marker here; monitor logs need the # core/noc/others subfolders. Build artifacts exclude sim/bin/logs/, so # these must be recreated in the test job too. - mkdir -p sim/bin/logs sim/bin/logs/core sim/bin/logs/noc sim/bin/logs/others - chmod +x sim/bin/cachepool_cluster.vsim - - BIN="${SW_PREFIX}${KERNEL}" + - case "$KERNEL" in rlc-*|ctrl-*) BIN="$KERNEL" ;; *) BIN="${SW_PREFIX}${KERNEL}" ;; esac - sim/bin/cachepool_cluster.vsim software/build/CachePoolTests/$BIN 2>&1 | tee test_${KERNEL}.log - python3 util/auto-benchmark/check-ci.py test_${KERNEL}.log artifacts: @@ -209,7 +210,7 @@ test-dual: script: - mkdir -p sim/bin/logs sim/bin/logs/core sim/bin/logs/noc sim/bin/logs/others - chmod +x sim/bin/cachepool_cluster.vsim - - BIN="${SW_PREFIX}${KERNEL}" + - case "$KERNEL" in rlc-*|ctrl-*) BIN="$KERNEL" ;; *) BIN="${SW_PREFIX}${KERNEL}" ;; esac - sim/bin/cachepool_cluster.vsim software/build/CachePoolTests/$BIN 2>&1 | tee test_dual_${KERNEL}.log - python3 util/auto-benchmark/check-ci.py test_dual_${KERNEL}.log artifacts: @@ -235,11 +236,11 @@ test-16g: - fmatmul-32b_M128_N128_K128 - fft-32b_M1024_N16 - byte-enable - - multi_producer_single_consumer_double_linked_list_M1_N1350_K10 + - rlc-multi_producer_single_consumer_double_linked_list_M1_N1350_K10 script: - mkdir -p sim/bin/logs sim/bin/logs/core sim/bin/logs/noc sim/bin/logs/others - chmod +x sim/bin/cachepool_cluster.vsim - - BIN="${SW_PREFIX}${KERNEL}" + - case "$KERNEL" in rlc-*|ctrl-*) BIN="$KERNEL" ;; *) BIN="${SW_PREFIX}${KERNEL}" ;; esac - sim/bin/cachepool_cluster.vsim software/build/CachePoolTests/$BIN 2>&1 | tee test_16g_${KERNEL}.log - python3 util/auto-benchmark/check-ci.py test_16g_${KERNEL}.log artifacts: @@ -268,7 +269,7 @@ test-dual-16g: script: - mkdir -p sim/bin/logs sim/bin/logs/core sim/bin/logs/noc sim/bin/logs/others - chmod +x sim/bin/cachepool_cluster.vsim - - BIN="${SW_PREFIX}${KERNEL}" + - case "$KERNEL" in rlc-*|ctrl-*) BIN="$KERNEL" ;; *) BIN="${SW_PREFIX}${KERNEL}" ;; esac - sim/bin/cachepool_cluster.vsim software/build/CachePoolTests/$BIN 2>&1 | tee test_dual_16g_${KERNEL}.log - python3 util/auto-benchmark/check-ci.py test_dual_16g_${KERNEL}.log artifacts: diff --git a/Bender.lock b/Bender.lock index 61477304..982bb86e 100644 --- a/Bender.lock +++ b/Bender.lock @@ -63,7 +63,7 @@ packages: dependencies: - axi floo_noc: - revision: 8413d9b044a51f958a7d4ef261e2b01e73db4544 + revision: 7e5104cd0875c3ded16d53366caaef2dc465c49d version: null source: Git: https://github.com/pulp-platform/FlooNoC.git @@ -150,7 +150,7 @@ packages: dependencies: - tech_cells_generic spatz: - revision: 6acbd2e0cb9131f870209d03ffb85c46fbd2ea9c + revision: f574402d29ed54f2dbede81099419889575c4584 version: null source: Git: https://github.com/pulp-platform/spatz.git diff --git a/Bender.yml b/Bender.yml index 59ec724d..5cc78d46 100644 --- a/Bender.yml +++ b/Bender.yml @@ -27,7 +27,6 @@ export_include_dirs: sources: # Level 0 - hardware/src/reqrsp_to_reg.sv - - hardware/src/reqrsp_w2n_converter.sv - hardware/src/reqrsp_xbar.sv - hardware/src/tcdm_cache_interco.sv - hardware/src/tcdm_id_remapper.sv @@ -35,12 +34,15 @@ sources: # Memory-mapped register - hardware/cachepool_peripheral/cachepool_peripheral_reg_pkg.sv - hardware/cachepool_peripheral/cachepool_peripheral_reg_top.sv - - hardware/cachepool_peripheral/cachepool_peripheral.sv # Bootrom - hardware/bootrom/bootrom.sv # Level 1 - hardware/generated/floo_cachepool_noc_pkg.sv - hardware/src/cachepool_pkg.sv + - hardware/cachepool_peripheral/cachepool_peripheral.sv + # Group configuration registers (cluster sender + group-level register) + - hardware/src/cachepool_group_cfg_sender.sv + - hardware/src/cachepool_group_cfg.sv - hardware/src/cachepool_acc_cut.sv - hardware/src/cachepool_cc.sv - hardware/src/cachepool_spatz_lock.sv diff --git a/Makefile b/Makefile index def098f5..0440c572 100644 --- a/Makefile +++ b/Makefile @@ -309,6 +309,15 @@ CACHEPOOL_DEFS += -DDATA_WIDTH=$(data_width) CACHEPOOL_DEFS += -DADDR_WIDTH=$(addr_width) CACHEPOOL_DEFS += -DNUM_BARRIER_SLOTS=$(num_barrier_slots) +# Group-level configuration registers and the reset values shared with the cluster peripheral +CACHEPOOL_DEFS += -DGROUP_CFG_BASE=$(group_cfg_base) +CACHEPOOL_DEFS += -DGROUP_CFG_STRIDE=$(group_cfg_stride) +CACHEPOOL_DEFS += -DL1D_XBAR_OFFSET_RST=$(l1d_xbar_offset_rst) +CACHEPOOL_DEFS += -DL1D_PRIVATE_RST=$(l1d_private_rst) +CACHEPOOL_DEFS += -DL1D_PRIVATE_ADDR_RST=$(l1d_private_addr_rst) +CACHEPOOL_DEFS += -DICACHE_PREFETCH_RST=$(icache_prefetch_rst) +CACHEPOOL_DEFS += -DBOOTROM_DATA_WIDTH=$(bootrom_data_width) + # Tile configuration CACHEPOOL_DEFS += -DREFILL_DATA_WIDTH=$(refill_data_width) @@ -395,7 +404,7 @@ include sim/sim.mk TESTS_DIR := $(SOFTWARE_DIR)/tests # Test-group roots (relative to SOFTWARE_DIR) scanned for data-generated tests. -TEST_ROOTS := cache sync kernels/fp kernels/int rlc tests +TEST_ROOTS := cache sync kernels/fp kernels/int rlc control tests # Auto-discover every test that has a script/gen_data.py. # Convention: script/data_.json → data/data_.h @@ -468,7 +477,11 @@ SW_CMAKE_FLAGS = \ -DGCC_PATH=${GCC_INSTALL_DIR} \ -DPYTHON=${PYTHON} \ -DBUILD_TESTS=ON \ - -DNUM_SCALAR_PER_CORE=$(num_scalar_per_core) + -DNUM_SCALAR_PER_CORE=$(num_scalar_per_core) \ + -DL2_CHANNEL=$(l2_channel) \ + -DL2_INTERLEAVE=$(l2_interleave) \ + -DL2_BANK_WIDTH=$(l2_bank_width) \ + -DL1D_CACHELINE_WIDTH=$(l1d_cacheline_width) .PHONY: sw sw: generate bootrom gen-data diff --git a/README.md b/README.md index 9e884f9e..2569ac69 100644 --- a/README.md +++ b/README.md @@ -302,7 +302,8 @@ The default boundary is `0xA000_0000`. This means data in `.pdcp_src` (at `0xA00 Flush instructions are issued through the cluster peripheral and dispatched to cache controllers via a two-level controller: -- **Cluster level** (`cachepool_peripheral`): holds the instruction and a per-tile one-hot tile-select mask. Issues the instruction to all tiles simultaneously and tracks completion at tile granularity. +- **Cluster level** (`cachepool_peripheral`): holds the instruction and a per-tile one-hot tile-select mask. On `L1D_INSN_COMMIT` it sends the instruction, with each group's slice of the mask, to the group-level configuration register of every group over the L2 mesh (`cachepool_group_cfg_sender`), and stays busy until every group has answered. +- **Group level** (`cachepool_group_cfg`): starts the instruction in its selected tiles and answers only once all of them have finished. - **Tile level** (`cachepool_tile`): receives the instruction, determines which of its local cache controllers to activate based on the partition field, and returns a single ready pulse to the cluster controller when all targeted controllers finish. ### Flush instruction encoding @@ -342,7 +343,7 @@ l1d_xbar_config(uint32_t offset); Lower-level, single-shot primitives (`l1d_flush()`, `l1d_private_flush()`, `l1d_shared_flush()`, `l1d_commit()`, `l1d_wait()`) exist for building custom sequences but issue only the register write — the caller is responsible for cluster-wide fence/barrier synchronization around them. -`l1d_addr()` has no hardware completion status bit (unlike flush, which polls `L1D_FLUSH_STATUS` via `l1d_wait()`); its peripheral register write is posted over the interconnect, so it bridges the gap with a fixed-cycle delay on core 0 before the final barrier. +Only flush/invalidate instructions are committed. The configuration registers (`L1D_PRIVATE`, `L1D_ADDR`, `XBAR_OFFSET`, `ICACHE_PREFETCH_ENABLE`) need no commit: when a value changes, the peripheral sends it to every group on its own. `L1D_FLUSH_STATUS` stays busy until every group has applied it, so `l1d_part()`, `l1d_addr()` and `l1d_xbar_config()` end with `l1d_wait()`. Writing the value a register already holds sends nothing. > Changing the partition mode, boundary address, or crossbar offset while the cache contains valid data requires a flush first — `l1d_xbar_config()` and `l1d_part()` flush internally; `l1d_addr()` does not (call a cluster flush before it if needed). diff --git a/config/cachepool.hjson.tmpl b/config/cachepool.hjson.tmpl index 8ced3ab5..a4bcb214 100644 --- a/config/cachepool.hjson.tmpl +++ b/config/cachepool.hjson.tmpl @@ -5,6 +5,7 @@ cluster: { mempool: 0, boot_addr: ${boot_addr}, // e.g., 4096 (0x1000) + bootrom_data_width: ${bootrom_data_width}, cluster_base_addr: ${stack_addr}, // 0xBFFF_F800 typically cluster_base_offset: 0, cluster_base_hartid: 0, diff --git a/config/config.mk b/config/config.mk index 9a3c732e..26c0c259 100644 --- a/config/config.mk +++ b/config/config.mk @@ -74,8 +74,9 @@ data_width ?= 32 # Core addrwidth addr_width ?= 32 -# Bootrom datawidth -bootrom_data_width ?= 32 +# Bootrom datawidth: must equal the L2 refill width, the bootrom is only read by full-line +# iCache refills on the wide L2 path +bootrom_data_width ?= $(refill_data_width) ###################### @@ -227,3 +228,21 @@ boot_addr ?= 4096 # UART address (32'hC001_0000) uart_addr ?= 3221291008 + +# Group-level configuration registers, written by the cluster peripheral over the +# L2 mesh. Group g owns one word at group_cfg_base + g * group_cfg_stride. +# Base (32'hD000_0000) +group_cfg_base ?= 3489660928 +# Stride per group (32'h100) +group_cfg_stride ?= 256 + +# Reset values of the L1D/xbar configuration, shared by the cluster peripheral +# registers and the group-level copies so both always come out of reset equal. +# Cache xbar offset (bit position selecting the cache bank) +l1d_xbar_offset_rst ?= 14 +# Private partition size and fold (L1D_PRIVATE) +l1d_private_rst ?= 0 +# Start address of the private L1D partition (32'hA000_0000) +l1d_private_addr_rst ?= 2684354560 +# Instruction cache prefetching enable +icache_prefetch_rst ?= 1 diff --git a/hardware/bootrom/bootrom.S b/hardware/bootrom/bootrom.S index 20749fb2..c3712be8 100644 --- a/hardware/bootrom/bootrom.S +++ b/hardware/bootrom/bootrom.S @@ -9,10 +9,13 @@ .global BOOTDATA _start: - la t1, exception + // lla, not la: la loads the address from the GOT, a 32-bit data read of the bootrom. + // The bootrom is only served on the wide L2 path (instruction refills), so it must + // never see data loads. + lla t1, exception csrw mtvec, t1 csrr a0, mhartid - la a1, BOOTDATA + lla a1, BOOTDATA // Enable MSI interrupt csrw mie, 0xF diff --git a/hardware/cachepool_peripheral/cachepool_peripheral.sv b/hardware/cachepool_peripheral/cachepool_peripheral.sv index 26e60506..46e079af 100644 --- a/hardware/cachepool_peripheral/cachepool_peripheral.sv +++ b/hardware/cachepool_peripheral/cachepool_peripheral.sv @@ -2,13 +2,19 @@ // Solderpad Hardware License, Version 0.51, see LICENSE for details. // SPDX-License-Identifier: SHL-0.51 -/// Exposes cluster confugration and information as memory mapped information +/// style-waive: COMMENT_BLOCK: function description +/// Exposes cluster confugration and information as memory mapped information. +/// The L1D/xbar configuration is not wired to the groups: every changed setting and every +/// L1D instruction commit is handed to cachepool_group_cfg_sender as one batch, which writes +/// that register into each group's cachepool_group_cfg over the L2 mesh. L1D_FLUSH_STATUS +/// stays busy until no batch is pending or in flight. `include "common_cells/assertions.svh" `include "common_cells/registers.svh" module cachepool_peripheral import snitch_pkg::*; + import cachepool_pkg::*; import cachepool_peripheral_reg_pkg::*; #( parameter int unsigned AddrWidth = 0, @@ -24,11 +30,9 @@ module cachepool_peripheral parameter int unsigned PrivateWidth = 4, parameter type reg_req_t = logic, parameter type reg_rsp_t = logic, - parameter type cache_insn_t = logic, /// Derived parameter *Do not override* parameter type addr_t = logic [AddrWidth-1:0], - parameter type spm_size_t = logic [SPMWidth-1:0], - parameter type private_sel_t = logic [PrivateWidth-1:0] + parameter type spm_size_t = logic [SPMWidth-1:0] ) ( input logic clk_i, input logic rst_ni, @@ -39,18 +43,17 @@ module cachepool_peripheral output logic [3:0] eoc_o, input addr_t tcdm_start_address_i, input addr_t tcdm_end_address_i, - output addr_t private_start_addr_o, - output logic icache_prefetch_enable_o, output logic cluster_probe_o, input logic [9:0] cluster_hart_base_id_i, - /// For cache xbar dynamic configuration - output logic [4:0] dynamic_offset_o, output spm_size_t l1d_spm_size_o, - output private_sel_t l1d_private_o, - output cache_insn_t l1d_insn_o, - output logic l1d_insn_valid_o, - input logic [NumTiles-1:0] l1d_insn_ready_i, + // Per-tile flush interlock, high from an L1D instruction commit until its batch is done output logic [NumTiles-1:0] l1d_busy_o, + // Configuration batch towards cachepool_group_cfg_sender, stable until cfg_done_i + output logic cfg_start_o, + output group_cfg_reg_e cfg_reg_o, + output logic [31:0] cfg_data_o, + output logic [NumTiles-1:0] cfg_tile_sel_o, + input logic cfg_done_i, // One group-participation mask per barrier slot (see HW_BARRIER_PARTICIPATION_MASK multireg) output logic [NumBarrierSlots-1:0][NumGroups-1:0] barrier_part_mask_o ); @@ -59,6 +62,8 @@ module cachepool_peripheral `ASSERT_INIT(CheckBarrierSlotsMatchRegPkg, NumBarrierSlots == cachepool_peripheral_reg_pkg::NumBarrierSlots, "NumBarrierSlots does not match cachepool_peripheral_reg_pkg::NumBarrierSlots -- regenerate the peripheral register file for this config") + `ASSERT_INIT(CheckPrivateWidth, PrivateWidth == L1PrivWidth, + "PrivateWidth does not match cachepool_pkg::L1PrivWidth") cachepool_peripheral_reg2hw_t reg2hw; cachepool_peripheral_hw2reg_t hw2reg; @@ -79,45 +84,19 @@ module cachepool_peripheral //////////// EOC ///////////// assign eoc_o = reg2hw.cluster_eoc_exit.q; - //////////// Cache XBar //////////// - logic [4:0] xbar_offset_d, xbar_offset_q; - assign dynamic_offset_o = xbar_offset_q; - logic xbar_offset_commit; - assign xbar_offset_commit = reg2hw.xbar_offset_commit.q; - always_comb begin : xbar_offset_cfg - xbar_offset_d = xbar_offset_q; - hw2reg.xbar_offset_commit.d = 1'b0; - hw2reg.xbar_offset_commit.de = 1'b0; - - if (xbar_offset_commit) begin - xbar_offset_d = reg2hw.xbar_offset.q; - hw2reg.xbar_offset_commit.d = 1'b0; - hw2reg.xbar_offset_commit.de = 1'b1; - end - end - // Default value is 13 - `FF(xbar_offset_q, xbar_offset_d, 5'd14, clk_i, rst_ni) - - - //////////// L1 DCache //////////// + //////////// L1 DCache SPM //////////// logic [9:0] l1d_spm_size_d, l1d_spm_size_q; - logic [PrivateWidth-1:0] l1d_private_d, l1d_private_q; - addr_t private_start_addr_d, private_start_addr_q; - // L1 is running flush/invalidation - logic [NumTiles-1:0] l1d_lock_d, l1d_lock_q; - logic l1d_spm_commit, l1d_insn_commit; + logic l1d_spm_commit; - // L1D Cache // For committing the cfg, if the cfg is taken, it will be pulled to 0; // Otherwise, it will be kept at 1 until taken. assign l1d_spm_commit = reg2hw.l1d_spm_commit.q; - assign l1d_insn_commit = reg2hw.l1d_insn_commit.q; // TODO: Change it to power of 2 to save space // SPM Size always_comb begin : l1d_spm_cfg l1d_spm_size_d = l1d_spm_size_q; - + hw2reg.l1d_spm_commit.d = 1'b0; hw2reg.l1d_spm_commit.de = 1'b0; @@ -133,9 +112,7 @@ module cachepool_peripheral // 10b is enough for 1024 cache lines, we should not need all of them assign l1d_spm_size_o = l1d_spm_size_q[SPMWidth-1:0]; - assign l1d_private_o = l1d_private_q[PrivateWidth-1:0]; - assign private_start_addr_o = private_start_addr_q; - + //////////// L1D / Cache XBar configuration //////////// // Concatenate all tile-select register words into one wide vector and slice // to NumTiles. Unused upper bits (for small configs) are optimised away. localparam int unsigned NumTileSelWords = cachepool_peripheral_reg_pkg::NumTileSelRegs; @@ -146,66 +123,121 @@ module cachepool_peripheral end end - // One 32-bit register per barrier slot; single register is enough at - // group granularity (max 32 groups). - always_comb begin : barrier_mask_concat - for (int s = 0; s < NumBarrierSlots; s++) begin - barrier_part_mask_o[s] = reg2hw.hw_barrier_participation_mask[s].q[NumGroups-1:0]; - end - end - - // Cache Flush Controller, per tile: l1d_lock_q[t] is set on issue and cleared on ready. - // Busy while any selected tile has not completed. - always_comb begin : l1d_insn_cfg - // Flush takes time, we cannot take next insn while flushing - l1d_insn_o = '0; - l1d_insn_valid_o = '0; - l1d_lock_d = l1d_lock_q; - l1d_private_d = l1d_private_q; - private_start_addr_d = private_start_addr_q; + // Settings as currently written by software + l1d_cfg_t reg_cfg; + assign reg_cfg.l1d_private = {reg2hw.l1d_private.group_fold.q, reg2hw.l1d_private.partition.q}; + assign reg_cfg.private_addr = reg2hw.l1d_addr.q; + assign reg_cfg.xbar_offset = reg2hw.xbar_offset.q; + assign reg_cfg.prefetch_en = reg2hw.icache_prefetch_enable.q; + + // Committed settings, the copy every group-level register holds once a batch is done + l1d_cfg_t cfg_d, cfg_q; + // A batch is in flight from its start until all groups have answered + logic busy_d, busy_q; + logic start_d, start_q; + group_cfg_reg_e reg_d, reg_q; + logic [31:0] data_d, data_q; + logic [NumTiles-1:0] tile_sel_d, tile_sel_q; + // L1 is running flush/invalidation + logic [NumTiles-1:0] l1d_lock_d, l1d_lock_q; + + // A setting differing from its committed copy is a pending update; only the L1D instruction + // waits for an explicit commit + logic l1d_insn_commit, xbar_update, private_update, addr_update, prefetch_update; + + assign l1d_insn_commit = reg2hw.l1d_insn_commit.q; + assign xbar_update = (reg_cfg.xbar_offset != cfg_q.xbar_offset); + assign private_update = (reg_cfg.l1d_private != cfg_q.l1d_private); + assign addr_update = (reg_cfg.private_addr != cfg_q.private_addr); + assign prefetch_update = (reg_cfg.prefetch_en != cfg_q.prefetch_en); + + // A batch carries one register and is only started while no batch is in flight; anything + // else stays pending until the previous batch is done. INSN first. + always_comb begin : cfg_ctrl + cfg_d = cfg_q; + busy_d = busy_q; + start_d = 1'b0; + reg_d = reg_q; + data_d = data_q; + tile_sel_d = tile_sel_q; + l1d_lock_d = l1d_lock_q; hw2reg.l1d_insn_commit.d = 1'b0; hw2reg.l1d_insn_commit.de = 1'b0; - if (l1d_insn_commit) begin - l1d_private_d = {reg2hw.l1d_private.group_fold.q, - reg2hw.l1d_private.partition.q}; - private_start_addr_d = reg2hw.l1d_addr.q; - // User issues a flush/invalidation - if (|l1d_lock_q == '0) begin - // We are ready to accept a new instruction. - // Build the cache_insn_t: pack insn + tile_sel. - // For non-private modes (shared/all/init), tile_sel is forced to '1. - l1d_insn_o.insn = reg2hw.cfg_l1d_insn.q; - l1d_insn_o.tile_sel = (reg2hw.cfg_l1d_insn.q == 2'b00) - ? tile_sel_raw[NumTiles-1:0] - : {NumTiles{1'b1}}; - l1d_insn_valid_o = 1'b1; - // Lock only the tiles that will receive the instruction. - l1d_lock_d = l1d_insn_o.tile_sel; - // Clear the commit - hw2reg.l1d_insn_commit.d = 1'b0; - hw2reg.l1d_insn_commit.de = 1'b1; + if (busy_q) begin + if (cfg_done_i) begin + busy_d = 1'b0; + l1d_lock_d = '0; end - end - - for (int t = 0; t < NumTiles; t++) begin - // Unlock tile t when it signals completion (one-cycle ready pulse). - if (l1d_insn_ready_i[t]) begin - l1d_lock_d[t] = 1'b0; - end - l1d_busy_o[t] = l1d_lock_q[t]; + end else if (l1d_insn_commit) begin + reg_d = GroupCfgL1dInsn; + data_d = '0; + data_d[1:0] = reg2hw.cfg_l1d_insn.q; + // For non-private modes (shared/all/init), tile_sel is forced to '1. + tile_sel_d = (reg2hw.cfg_l1d_insn.q == 2'b00) ? tile_sel_raw[NumTiles-1:0] : '1; + // Lock only the tiles that will receive the instruction. + l1d_lock_d = tile_sel_d; + busy_d = 1'b1; + start_d = 1'b1; + hw2reg.l1d_insn_commit.de = 1'b1; + end else if (xbar_update) begin + cfg_d.xbar_offset = reg_cfg.xbar_offset; + reg_d = GroupCfgXbarOffset; + data_d = '0; + data_d[$bits(xbar_offset_t)-1:0] = reg_cfg.xbar_offset; + busy_d = 1'b1; + start_d = 1'b1; + end else if (private_update) begin + cfg_d.l1d_private = reg_cfg.l1d_private; + reg_d = GroupCfgL1dPrivate; + data_d = '0; + data_d[L1PrivWidth-1:0] = reg_cfg.l1d_private; + busy_d = 1'b1; + start_d = 1'b1; + end else if (addr_update) begin + cfg_d.private_addr = reg_cfg.private_addr; + reg_d = GroupCfgL1dAddr; + data_d = reg_cfg.private_addr; + busy_d = 1'b1; + start_d = 1'b1; + end else if (prefetch_update) begin + cfg_d.prefetch_en = reg_cfg.prefetch_en; + reg_d = GroupCfgPrefetch; + data_d = '0; + data_d[0] = reg_cfg.prefetch_en; + busy_d = 1'b1; + start_d = 1'b1; end end - `FF(private_start_addr_q, private_start_addr_d, 32'hA000_0000, clk_i, rst_ni) - `FF(l1d_private_q, l1d_private_d, 0, clk_i, rst_ni) - `FF(l1d_lock_q, l1d_lock_d, '0, clk_i, rst_ni) - // To show if the current flush/invalidation is complete - assign hw2reg.l1d_flush_status.d = (l1d_lock_q != '0); + `FF(cfg_q, cfg_d, L1dCfgRst, clk_i, rst_ni) + `FF(busy_q, busy_d, 1'b0, clk_i, rst_ni) + `FF(start_q, start_d, 1'b0, clk_i, rst_ni) + `FF(reg_q, reg_d, GroupCfgXbarOffset, clk_i, rst_ni) + `FF(data_q, data_d, '0, clk_i, rst_ni) + `FF(tile_sel_q, tile_sel_d, '0, clk_i, rst_ni) + `FF(l1d_lock_q, l1d_lock_d, '0, clk_i, rst_ni) + + // The batch starts one cycle after it is taken, so the sender sees the updated values + assign cfg_start_o = start_q; + assign cfg_reg_o = reg_q; + assign cfg_data_o = data_q; + assign cfg_tile_sel_o = tile_sel_q; + + assign l1d_busy_o = l1d_lock_q; + // Busy from the cycle a commit or a changed setting is visible, so a status read right behind + // the write never sees idle before the batch has started. + assign hw2reg.l1d_flush_status.d = busy_q | l1d_insn_commit | xbar_update | private_update | + addr_update | prefetch_update; - // Enable icache prefetch - assign icache_prefetch_enable_o = reg2hw.icache_prefetch_enable.q; + // One 32-bit register per barrier slot; single register is enough at + // group granularity (max 32 groups). + always_comb begin : barrier_mask_concat + for (int s = 0; s < NumBarrierSlots; s++) begin + barrier_part_mask_o[s] = reg2hw.hw_barrier_participation_mask[s].q[NumGroups-1:0]; + end + end // Probe assign cluster_probe_o = reg2hw.spatz_status.q; diff --git a/hardware/cachepool_peripheral/cachepool_peripheral_reg.hjson.tmpl b/hardware/cachepool_peripheral/cachepool_peripheral_reg.hjson.tmpl index 0ba3a382..df17724a 100644 --- a/hardware/cachepool_peripheral/cachepool_peripheral_reg.hjson.tmpl +++ b/hardware/cachepool_peripheral/cachepool_peripheral_reg.hjson.tmpl @@ -69,11 +69,11 @@ desc: '''Controls prefetching of the instruction cache.''' swaccess: "wo", hwaccess: "hro", - resval: "1", + resval: "${icache_prefetch_rst}", fields: [{ bits: "0:0", name: "ICACHE_PREFETCH_ENABLE", - desc: "Hardware barrier register." + desc: "Enable instruction cache prefetching." }] }, { @@ -208,7 +208,9 @@ }, { name: "L1D_INSN_COMMIT", - desc: '''Controls the L1 DCache flushing and invalidation.''' + desc: '''Starts the L1D instruction in CFG_L1D_INSN on the tiles selected by + CFG_L1D_TILE_SEL. L1D_PRIVATE, L1D_ADDR, XBAR_OFFSET and ICACHE_PREFETCH_ENABLE need + no commit: a changed value is applied on its own.''' // hwext: "true", // hwqe: "true", swaccess: "rw", @@ -217,12 +219,13 @@ fields: [{ bits: "0", name: "COMMIT", - desc: "Commit the configurations for instructions." + desc: "Commit the L1D instruction." }] }, { name: "L1D_FLUSH_STATUS", - desc: '''Indicate the status of flushing ''' + desc: '''Busy while a committed L1D instruction or a changed setting is still being + applied by the group-level configuration registers.''' hwext: "true", swaccess: "ro", hwaccess: "hwo", @@ -235,10 +238,11 @@ }, { name: "L1D_PRIVATE", - desc: '''L1D partitioning: private bank count and how it is folded ''' + desc: '''L1D partitioning: private bank count and how it is folded. Applied when the + value changes, no commit needed.''' swaccess: "rw", hwaccess: "hro", - resval: "0", + resval: "${l1d_private_rst}", fields: [{ bits: "2:0", name: "PARTITION", @@ -251,10 +255,11 @@ }, { name: "L1D_ADDR", - desc: '''Starting address of private L1D partition ''' + desc: '''Starting address of private L1D partition. Applied when the value changes, no + commit needed.''' swaccess: "rw", hwaccess: "hro", - resval: "2684354560", + resval: "${l1d_private_addr_rst}", fields: [{ bits: "31:0", name: "ADDR", @@ -263,27 +268,16 @@ }, { name: "XBAR_OFFSET", - desc: '''Cache xbar offset setting ''' + desc: '''Cache xbar offset setting. Applied when the value changes, no commit + needed.''' swaccess: "rw", hwaccess: "hro", - resval: "0", + resval: "${l1d_xbar_offset_rst}", fields: [{ bits: "4:0", name: "OFFSET", desc: "range: 0-31" }] - }, - { - name: "XBAR_OFFSET_COMMIT", - desc: '''Cache xbar offset setting ''' - swaccess: "rw", - hwaccess: "hrw", - resval: "0", - fields: [{ - bits: "0", - name: "COMMIT", - desc: "Commit the xbar offset configurations." - }] } ] } diff --git a/hardware/cachepool_peripheral/cachepool_peripheral_reg_pkg.sv b/hardware/cachepool_peripheral/cachepool_peripheral_reg_pkg.sv index 3ace3df2..31804b1c 100644 --- a/hardware/cachepool_peripheral/cachepool_peripheral_reg_pkg.sv +++ b/hardware/cachepool_peripheral/cachepool_peripheral_reg_pkg.sv @@ -90,10 +90,6 @@ package cachepool_peripheral_reg_pkg; logic [4:0] q; } cachepool_peripheral_reg2hw_xbar_offset_reg_t; - typedef struct packed { - logic q; - } cachepool_peripheral_reg2hw_xbar_offset_commit_reg_t; - typedef struct packed { logic [31:0] d; } cachepool_peripheral_hw2reg_hw_barrier_reg_t; @@ -120,42 +116,35 @@ package cachepool_peripheral_reg_pkg; logic d; } cachepool_peripheral_hw2reg_l1d_flush_status_reg_t; - typedef struct packed { - logic d; - logic de; - } cachepool_peripheral_hw2reg_xbar_offset_commit_reg_t; - // Register -> HW type typedef struct packed { - cachepool_peripheral_reg2hw_hw_barrier_reg_t hw_barrier; // [349:318] - cachepool_peripheral_reg2hw_spatz_lock_acquire_reg_t spatz_lock_acquire; // [317:286] - cachepool_peripheral_reg2hw_spatz_lock_release_reg_t spatz_lock_release; // [285:254] - cachepool_peripheral_reg2hw_icache_prefetch_enable_reg_t icache_prefetch_enable; // [253:253] - cachepool_peripheral_reg2hw_spatz_status_reg_t spatz_status; // [252:252] - cachepool_peripheral_reg2hw_spatz_cycle_reg_t spatz_cycle; // [251:220] - cachepool_peripheral_reg2hw_cluster_boot_control_reg_t cluster_boot_control; // [219:188] - cachepool_peripheral_reg2hw_cluster_eoc_exit_reg_t cluster_eoc_exit; // [187:184] - cachepool_peripheral_reg2hw_cfg_l1d_spm_reg_t cfg_l1d_spm; // [183:174] - cachepool_peripheral_reg2hw_cfg_l1d_insn_reg_t cfg_l1d_insn; // [173:172] - cachepool_peripheral_reg2hw_cfg_l1d_tile_sel_mreg_t [1:0] cfg_l1d_tile_sel; // [171:108] - cachepool_peripheral_reg2hw_hw_barrier_participation_mask_mreg_t [1:0] hw_barrier_participation_mask; // [107:44] - cachepool_peripheral_reg2hw_l1d_spm_commit_reg_t l1d_spm_commit; // [43:43] - cachepool_peripheral_reg2hw_l1d_insn_commit_reg_t l1d_insn_commit; // [42:42] - cachepool_peripheral_reg2hw_l1d_private_reg_t l1d_private; // [41:38] - cachepool_peripheral_reg2hw_l1d_addr_reg_t l1d_addr; // [37:6] - cachepool_peripheral_reg2hw_xbar_offset_reg_t xbar_offset; // [5:1] - cachepool_peripheral_reg2hw_xbar_offset_commit_reg_t xbar_offset_commit; // [0:0] + cachepool_peripheral_reg2hw_hw_barrier_reg_t hw_barrier; // [348:317] + cachepool_peripheral_reg2hw_spatz_lock_acquire_reg_t spatz_lock_acquire; // [316:285] + cachepool_peripheral_reg2hw_spatz_lock_release_reg_t spatz_lock_release; // [284:253] + cachepool_peripheral_reg2hw_icache_prefetch_enable_reg_t icache_prefetch_enable; // [252:252] + cachepool_peripheral_reg2hw_spatz_status_reg_t spatz_status; // [251:251] + cachepool_peripheral_reg2hw_spatz_cycle_reg_t spatz_cycle; // [250:219] + cachepool_peripheral_reg2hw_cluster_boot_control_reg_t cluster_boot_control; // [218:187] + cachepool_peripheral_reg2hw_cluster_eoc_exit_reg_t cluster_eoc_exit; // [186:183] + cachepool_peripheral_reg2hw_cfg_l1d_spm_reg_t cfg_l1d_spm; // [182:173] + cachepool_peripheral_reg2hw_cfg_l1d_insn_reg_t cfg_l1d_insn; // [172:171] + cachepool_peripheral_reg2hw_cfg_l1d_tile_sel_mreg_t [1:0] cfg_l1d_tile_sel; // [170:107] + cachepool_peripheral_reg2hw_hw_barrier_participation_mask_mreg_t [1:0] hw_barrier_participation_mask; // [106:43] + cachepool_peripheral_reg2hw_l1d_spm_commit_reg_t l1d_spm_commit; // [42:42] + cachepool_peripheral_reg2hw_l1d_insn_commit_reg_t l1d_insn_commit; // [41:41] + cachepool_peripheral_reg2hw_l1d_private_reg_t l1d_private; // [40:37] + cachepool_peripheral_reg2hw_l1d_addr_reg_t l1d_addr; // [36:5] + cachepool_peripheral_reg2hw_xbar_offset_reg_t xbar_offset; // [4:0] } cachepool_peripheral_reg2hw_t; // HW -> register type typedef struct packed { - cachepool_peripheral_hw2reg_hw_barrier_reg_t hw_barrier; // [102:71] - cachepool_peripheral_hw2reg_spatz_lock_acquire_reg_t spatz_lock_acquire; // [70:39] - cachepool_peripheral_hw2reg_spatz_lock_release_reg_t spatz_lock_release; // [38:7] - cachepool_peripheral_hw2reg_l1d_spm_commit_reg_t l1d_spm_commit; // [6:5] - cachepool_peripheral_hw2reg_l1d_insn_commit_reg_t l1d_insn_commit; // [4:3] - cachepool_peripheral_hw2reg_l1d_flush_status_reg_t l1d_flush_status; // [2:2] - cachepool_peripheral_hw2reg_xbar_offset_commit_reg_t xbar_offset_commit; // [1:0] + cachepool_peripheral_hw2reg_hw_barrier_reg_t hw_barrier; // [100:69] + cachepool_peripheral_hw2reg_spatz_lock_acquire_reg_t spatz_lock_acquire; // [68:37] + cachepool_peripheral_hw2reg_spatz_lock_release_reg_t spatz_lock_release; // [36:5] + cachepool_peripheral_hw2reg_l1d_spm_commit_reg_t l1d_spm_commit; // [4:3] + cachepool_peripheral_hw2reg_l1d_insn_commit_reg_t l1d_insn_commit; // [2:1] + cachepool_peripheral_hw2reg_l1d_flush_status_reg_t l1d_flush_status; // [0:0] } cachepool_peripheral_hw2reg_t; // Register offsets @@ -179,7 +168,6 @@ package cachepool_peripheral_reg_pkg; parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_L1D_PRIVATE_OFFSET = 7'h 44; parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_L1D_ADDR_OFFSET = 7'h 48; parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_XBAR_OFFSET_OFFSET = 7'h 4c; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_XBAR_OFFSET_COMMIT_OFFSET = 7'h 50; // Reset values for hwext registers and their fields parameter logic [31:0] CACHEPOOL_PERIPHERAL_HW_BARRIER_RESVAL = 32'h 0; @@ -209,12 +197,11 @@ package cachepool_peripheral_reg_pkg; CACHEPOOL_PERIPHERAL_L1D_FLUSH_STATUS, CACHEPOOL_PERIPHERAL_L1D_PRIVATE, CACHEPOOL_PERIPHERAL_L1D_ADDR, - CACHEPOOL_PERIPHERAL_XBAR_OFFSET, - CACHEPOOL_PERIPHERAL_XBAR_OFFSET_COMMIT + CACHEPOOL_PERIPHERAL_XBAR_OFFSET } cachepool_peripheral_id_e; // Register width information to check illegal writes - parameter logic [3:0] CACHEPOOL_PERIPHERAL_PERMIT [21] = '{ + parameter logic [3:0] CACHEPOOL_PERIPHERAL_PERMIT [20] = '{ 4'b 1111, // index[ 0] CACHEPOOL_PERIPHERAL_HW_BARRIER 4'b 1111, // index[ 1] CACHEPOOL_PERIPHERAL_SPATZ_LOCK_ACQUIRE 4'b 1111, // index[ 2] CACHEPOOL_PERIPHERAL_SPATZ_LOCK_RELEASE @@ -234,8 +221,7 @@ package cachepool_peripheral_reg_pkg; 4'b 0001, // index[16] CACHEPOOL_PERIPHERAL_L1D_FLUSH_STATUS 4'b 0001, // index[17] CACHEPOOL_PERIPHERAL_L1D_PRIVATE 4'b 1111, // index[18] CACHEPOOL_PERIPHERAL_L1D_ADDR - 4'b 0001, // index[19] CACHEPOOL_PERIPHERAL_XBAR_OFFSET - 4'b 0001 // index[20] CACHEPOOL_PERIPHERAL_XBAR_OFFSET_COMMIT + 4'b 0001 // index[19] CACHEPOOL_PERIPHERAL_XBAR_OFFSET }; endpackage diff --git a/hardware/cachepool_peripheral/cachepool_peripheral_reg_top.sv b/hardware/cachepool_peripheral/cachepool_peripheral_reg_top.sv index b86669b5..94ba3f7e 100644 --- a/hardware/cachepool_peripheral/cachepool_peripheral_reg_top.sv +++ b/hardware/cachepool_peripheral/cachepool_peripheral_reg_top.sv @@ -125,9 +125,6 @@ module cachepool_peripheral_reg_top #( logic [4:0] xbar_offset_qs; logic [4:0] xbar_offset_wd; logic xbar_offset_we; - logic xbar_offset_commit_qs; - logic xbar_offset_commit_wd; - logic xbar_offset_commit_we; // Register instances // R[hw_barrier]: V(True) @@ -633,7 +630,7 @@ module cachepool_peripheral_reg_top #( prim_subreg #( .DW (5), .SWACCESS("RW"), - .RESVAL (5'h0) + .RESVAL (5'he) ) u_xbar_offset ( .clk_i (clk_i ), .rst_ni (rst_ni ), @@ -655,36 +652,9 @@ module cachepool_peripheral_reg_top #( ); - // R[xbar_offset_commit]: V(False) - - prim_subreg #( - .DW (1), - .SWACCESS("RW"), - .RESVAL (1'h0) - ) u_xbar_offset_commit ( - .clk_i (clk_i ), - .rst_ni (rst_ni ), - - // from register interface - .we (xbar_offset_commit_we), - .wd (xbar_offset_commit_wd), - - // from internal hardware - .de (hw2reg.xbar_offset_commit.de), - .d (hw2reg.xbar_offset_commit.d ), - - // to internal hardware - .qe (), - .q (reg2hw.xbar_offset_commit.q ), - - // to register interface (read) - .qs (xbar_offset_commit_qs) - ); - - - logic [20:0] addr_hit; + logic [19:0] addr_hit; always_comb begin addr_hit = '0; addr_hit[ 0] = (reg_addr == CACHEPOOL_PERIPHERAL_HW_BARRIER_OFFSET); @@ -707,7 +677,6 @@ module cachepool_peripheral_reg_top #( addr_hit[17] = (reg_addr == CACHEPOOL_PERIPHERAL_L1D_PRIVATE_OFFSET); addr_hit[18] = (reg_addr == CACHEPOOL_PERIPHERAL_L1D_ADDR_OFFSET); addr_hit[19] = (reg_addr == CACHEPOOL_PERIPHERAL_XBAR_OFFSET_OFFSET); - addr_hit[20] = (reg_addr == CACHEPOOL_PERIPHERAL_XBAR_OFFSET_COMMIT_OFFSET); end assign addrmiss = (reg_re || reg_we) ? ~|addr_hit : 1'b0 ; @@ -734,8 +703,7 @@ module cachepool_peripheral_reg_top #( (addr_hit[16] & (|(CACHEPOOL_PERIPHERAL_PERMIT[16] & ~reg_be))) | (addr_hit[17] & (|(CACHEPOOL_PERIPHERAL_PERMIT[17] & ~reg_be))) | (addr_hit[18] & (|(CACHEPOOL_PERIPHERAL_PERMIT[18] & ~reg_be))) | - (addr_hit[19] & (|(CACHEPOOL_PERIPHERAL_PERMIT[19] & ~reg_be))) | - (addr_hit[20] & (|(CACHEPOOL_PERIPHERAL_PERMIT[20] & ~reg_be))))); + (addr_hit[19] & (|(CACHEPOOL_PERIPHERAL_PERMIT[19] & ~reg_be))))); end assign hw_barrier_re = addr_hit[0] & reg_re & !reg_error; @@ -797,9 +765,6 @@ module cachepool_peripheral_reg_top #( assign xbar_offset_we = addr_hit[19] & reg_we & !reg_error; assign xbar_offset_wd = reg_wdata[4:0]; - assign xbar_offset_commit_we = addr_hit[20] & reg_we & !reg_error; - assign xbar_offset_commit_wd = reg_wdata[0]; - // Read data return always_comb begin reg_rdata_next = '0; @@ -885,10 +850,6 @@ module cachepool_peripheral_reg_top #( reg_rdata_next[4:0] = xbar_offset_qs; end - addr_hit[20]: begin - reg_rdata_next[0] = xbar_offset_commit_qs; - end - default: begin reg_rdata_next = '1; end diff --git a/hardware/src/cachepool_cluster.sv b/hardware/src/cachepool_cluster.sv index 6afebcfd..32b0fd4d 100644 --- a/hardware/src/cachepool_cluster.sv +++ b/hardware/src/cachepool_cluster.sv @@ -172,19 +172,24 @@ module cachepool_cluster axi_slv_cache_req_t [ClusterWideOutAxiPorts-1:0] wide_axi_slv_req; axi_slv_cache_resp_t [ClusterWideOutAxiPorts-1:0] wide_axi_slv_rsp; - // HBM0 peripheral demux port: narrowed 32b REQRSP with refill_user_t (same user as wide side). - peri_narrow_req_t hbm0_peri_narrow_req; - peri_narrow_rsp_t hbm0_peri_narrow_rsp; + // Narrow L2 mesh endpoint of the cluster peripheral, subordinate side: core accesses to the + // CSRs and the UART. + peri_noc_req_chan_t peri_sbr_req; + logic peri_sbr_req_valid; + logic peri_sbr_req_ready; + peri_narrow_rsp_chan_t peri_sbr_rsp; + logic peri_sbr_rsp_valid; + logic peri_sbr_rsp_ready; // 2. Peripherals - axi_addr_t private_start_addr; - logic icache_prefetch_enable; - logic [$clog2(L1AddrWidth)-1:0] dynamic_offset; - cache_insn_t l1d_insn; - logic l1d_insn_valid; - logic [NumTiles-1:0] l1d_insn_ready; logic [NumTiles-1:0] l1d_busy; - logic [L1PrivWidth-1:0] l1d_private; + + // Configuration batch from the peripheral to the group configuration sender + logic cfg_start; + group_cfg_reg_e cfg_reg; + logic [31:0] cfg_data; + logic [NumTiles-1:0] cfg_tile_sel; + logic cfg_done; // Per-group error signals. logic [NumGroups-1:0] group_error; @@ -250,6 +255,20 @@ module cachepool_cluster logic [NumGroupsX-1:0][NumGroupsY-1:0][3:0] l2_rsp_in_valid; logic [NumGroupsX-1:0][NumGroupsY-1:0][3:0] l2_rsp_in_ready; + // Per-group narrow L2 mesh signals [NumGroupsX][NumGroupsY][direction 0..3] + peri_noc_req_t [NumGroupsX-1:0][NumGroupsY-1:0][3:0] peri_req_out; + logic [NumGroupsX-1:0][NumGroupsY-1:0][3:0] peri_req_out_valid; + logic [NumGroupsX-1:0][NumGroupsY-1:0][3:0] peri_req_out_ready; + peri_noc_req_t [NumGroupsX-1:0][NumGroupsY-1:0][3:0] peri_req_in; + logic [NumGroupsX-1:0][NumGroupsY-1:0][3:0] peri_req_in_valid; + logic [NumGroupsX-1:0][NumGroupsY-1:0][3:0] peri_req_in_ready; + peri_noc_rsp_t [NumGroupsX-1:0][NumGroupsY-1:0][3:0] peri_rsp_out; + logic [NumGroupsX-1:0][NumGroupsY-1:0][3:0] peri_rsp_out_valid; + logic [NumGroupsX-1:0][NumGroupsY-1:0][3:0] peri_rsp_out_ready; + peri_noc_rsp_t [NumGroupsX-1:0][NumGroupsY-1:0][3:0] peri_rsp_in; + logic [NumGroupsX-1:0][NumGroupsY-1:0][3:0] peri_rsp_in_valid; + logic [NumGroupsX-1:0][NumGroupsY-1:0][3:0] peri_rsp_in_ready; + assign error_o = |group_error; for (genvar gy = 0; gy < NumGroupsY; gy++) begin : gen_group_y @@ -306,7 +325,6 @@ module cachepool_cluster .hart_base_id_i ( HartBaseId + 10'(g * NumCoreGroup * NumScalarPerCC) ), .tile_base_id_i ( TileIDWidth'(g * NumTilesPerGroup) ), .cluster_base_addr_i ( cluster_base_addr_i ), - .private_start_addr_i ( private_start_addr ), // L2 refill TCDM mesh (4 directions) .l2_req_o ( l2_req_out [gx][gy] ), .l2_req_valid_o ( l2_req_out_valid [gx][gy] ), @@ -320,16 +338,23 @@ module cachepool_cluster .l2_rsp_i ( l2_rsp_in [gx][gy] ), .l2_rsp_valid_i ( l2_rsp_in_valid [gx][gy] ), .l2_rsp_ready_o ( l2_rsp_in_ready [gx][gy] ), + // Narrow L2 mesh (4 directions) + .peri_req_o ( peri_req_out [gx][gy] ), + .peri_req_valid_o ( peri_req_out_valid [gx][gy] ), + .peri_req_ready_i ( peri_req_out_ready [gx][gy] ), + .peri_req_i ( peri_req_in [gx][gy] ), + .peri_req_valid_i ( peri_req_in_valid [gx][gy] ), + .peri_req_ready_o ( peri_req_in_ready [gx][gy] ), + .peri_rsp_o ( peri_rsp_out [gx][gy] ), + .peri_rsp_valid_o ( peri_rsp_out_valid [gx][gy] ), + .peri_rsp_ready_i ( peri_rsp_out_ready [gx][gy] ), + .peri_rsp_i ( peri_rsp_in [gx][gy] ), + .peri_rsp_valid_i ( peri_rsp_in_valid [gx][gy] ), + .peri_rsp_ready_o ( peri_rsp_in_ready [gx][gy] ), .l2_id_i ( floo_cachepool_noc_pkg::id_t'(gx * NumGroupsY + gy) ), .l2_route_table_i ( floo_cachepool_noc_pkg::RoutingTables[gx * NumGroupsY + gy] ), // Peripherals .icache_events_o ( /* unused */ ), - .icache_prefetch_enable_i ( icache_prefetch_enable ), - .dynamic_offset_i ( dynamic_offset ), - .l1d_private_i ( l1d_private ), - .l1d_insn_i ( l1d_insn ), - .l1d_insn_valid_i ( l1d_insn_valid ), - .l1d_insn_ready_o ( l1d_insn_ready[g*NumTilesPerGroup +: NumTilesPerGroup]), .l1d_busy_i ( l1d_busy [g*NumTilesPerGroup +: NumTilesPerGroup]), .group_xy_id_i ( group_xy_id_t'{x: gx, y: gy, @@ -545,6 +570,9 @@ module cachepool_cluster // L2 Refill TCDM Mesh + HBM Edge Adapters // ------------------------------------------------------- + // The group configuration registers are only reachable over the L2 mesh. + `ASSERT_INIT(CheckMultiGroup, NumGroups > 1, "cachepool_cluster: NumGroups must be > 1") + if (NumGroups > 1) begin : gen_l2_refill_mesh // -------------------------------------------------- @@ -591,14 +619,80 @@ module cachepool_cluster end end + // -------------------------------------------------- + // Narrow L2 mesh: same interior links as the wide mesh + // -------------------------------------------------- + + for (genvar gx = 0; gx < NumGroupsX - 1; gx++) begin : gen_peri_ew + for (genvar gy = 0; gy < NumGroupsY; gy++) begin : gen_peri_ew_y + // East(1) output of (gx,gy) <-> West(3) input of (gx+1,gy), both ways + assign peri_req_in [gx+1][gy][3] = peri_req_out [gx] [gy][1]; + assign peri_req_in_valid [gx+1][gy][3] = peri_req_out_valid [gx] [gy][1]; + assign peri_req_out_ready[gx] [gy][1] = peri_req_in_ready [gx+1][gy][3]; + assign peri_rsp_in [gx+1][gy][3] = peri_rsp_out [gx] [gy][1]; + assign peri_rsp_in_valid [gx+1][gy][3] = peri_rsp_out_valid [gx] [gy][1]; + assign peri_rsp_out_ready[gx] [gy][1] = peri_rsp_in_ready [gx+1][gy][3]; + assign peri_req_in [gx] [gy][1] = peri_req_out [gx+1][gy][3]; + assign peri_req_in_valid [gx] [gy][1] = peri_req_out_valid [gx+1][gy][3]; + assign peri_req_out_ready[gx+1][gy][3] = peri_req_in_ready [gx] [gy][1]; + assign peri_rsp_in [gx] [gy][1] = peri_rsp_out [gx+1][gy][3]; + assign peri_rsp_in_valid [gx] [gy][1] = peri_rsp_out_valid [gx+1][gy][3]; + assign peri_rsp_out_ready[gx+1][gy][3] = peri_rsp_in_ready [gx] [gy][1]; + end + end + + for (genvar gx = 0; gx < NumGroupsX; gx++) begin : gen_peri_ns + for (genvar gy = 0; gy < NumGroupsY - 1; gy++) begin : gen_peri_ns_y + // North(0) output of (gx,gy) <-> South(2) input of (gx,gy+1), both ways + assign peri_req_in [gx][gy+1][2] = peri_req_out [gx][gy] [0]; + assign peri_req_in_valid [gx][gy+1][2] = peri_req_out_valid [gx][gy] [0]; + assign peri_req_out_ready[gx][gy] [0] = peri_req_in_ready [gx][gy+1][2]; + assign peri_rsp_in [gx][gy+1][2] = peri_rsp_out [gx][gy] [0]; + assign peri_rsp_in_valid [gx][gy+1][2] = peri_rsp_out_valid [gx][gy] [0]; + assign peri_rsp_out_ready[gx][gy] [0] = peri_rsp_in_ready [gx][gy+1][2]; + assign peri_req_in [gx][gy] [0] = peri_req_out [gx][gy+1][2]; + assign peri_req_in_valid [gx][gy] [0] = peri_req_out_valid [gx][gy+1][2]; + assign peri_req_out_ready[gx][gy+1][2] = peri_req_in_ready [gx][gy] [0]; + assign peri_rsp_in [gx][gy] [0] = peri_rsp_out [gx][gy+1][2]; + assign peri_rsp_in_valid [gx][gy] [0] = peri_rsp_out_valid [gx][gy+1][2]; + assign peri_rsp_out_ready[gx][gy+1][2] = peri_rsp_in_ready [gx][gy] [0]; + end + end + + // Narrow mesh boundary: only the West port of group (0,0) has an endpoint (i_peri_chimney + // below). Every other edge port is tied off and never sees a flit. + for (genvar gx = 0; gx < NumGroupsX; gx++) begin : gen_peri_bnd_x + for (genvar gy = 0; gy < NumGroupsY; gy++) begin : gen_peri_bnd_y + for (genvar d = 0; d < 4; d++) begin : gen_peri_bnd_d + localparam bit IsEdge = ((d == 0) && (gy == NumGroupsY - 1)) || + ((d == 1) && (gx == NumGroupsX - 1)) || + ((d == 2) && (gy == 0)) || + ((d == 3) && (gx == 0)); + localparam bit IsPeriPort = (d == 3) && (gx == 0) && (gy == 0); + if (IsEdge && !IsPeriPort) begin : gen_tieoff + assign peri_req_in [gx][gy][d] = '0; + assign peri_req_in_valid [gx][gy][d] = 1'b0; + assign peri_req_out_ready[gx][gy][d] = 1'b1; + assign peri_rsp_in [gx][gy][d] = '0; + assign peri_rsp_in_valid [gx][gy][d] = 1'b0; + assign peri_rsp_out_ready[gx][gy][d] = 1'b1; + `ASSERT(PeriEdgeNoReq, !peri_req_out_valid[gx][gy][d], clk_i, !rst_ni, + "cachepool_cluster: narrow request lost at a mesh edge") + `ASSERT(PeriEdgeNoRsp, !peri_rsp_out_valid[gx][gy][d], clk_i, !rst_ni, + "cachepool_cluster: narrow response lost at a mesh edge") + end + end + end + end + // -------------------------------------------------- // North/South boundary: tied off (8ch) or hosts extra HBM chimneys for // more refill bandwidth (16ch). Assertion catches a mismatched mesh // shape/channel count. // -------------------------------------------------- + // NumHbmAllSides, HbmPerSide and L2PeriphChannel come from cachepool_pkg. localparam int unsigned NumHbmWestEast = 2 * NumGroupsY; - localparam int unsigned NumHbmAllSides = 2 * NumGroupsY + 2 * NumGroupsX; `ifndef TARGET_SYNTHESIS initial assert (NumL2Channel == NumHbmWestEast || NumL2Channel == NumHbmAllSides) else $fatal(1, "NumL2Channel must equal the West/East or all-sides chimney count."); @@ -608,17 +702,9 @@ module cachepool_cluster // group reaches its own-ID channel in 1 hop. All-sides uses the side-contiguous scheme // below instead. SAM and address scrambling are unaffected either way. localparam bit L2SameIdPlacement = (NumL2Channel == NumGroups); - // All-sides placement packs one aligned block of channels per mesh edge: - // four adjacent channels share one controller IP and cannot be split - // across edges. Order within a side follows the floorplan. - localparam int unsigned HbmPerSide = NumHbmAllSides / 4; - // The peripheral demux stays on the West chimney at gy=0 -- the corner - // next to group 0 -- so its channel ID follows the placement. - localparam int unsigned L2PeriphChannel = - (NumL2Channel == NumHbmAllSides) ? HbmPerSide : 0; - // The side-contiguous IDs below assume four channels per edge on a 4x4 - // mesh, so reject any other mesh shape asking for all-sides chimneys - // rather than mis-wiring it. + // All-sides placement: one aligned block of HbmPerSide channels per edge (one controller IP, + // in floorplan order). The side-contiguous IDs below assume a 4x4 mesh, so reject any other + // mesh shape asking for all-sides chimneys rather than mis-wiring it. `ifndef TARGET_SYNTHESIS initial assert (NumL2Channel != NumHbmAllSides || (NumGroupsX == 4 && NumGroupsY == 4)) else $fatal(1, "All-sides HBM placement is only implemented for a 4x4 group mesh."); @@ -656,7 +742,7 @@ module cachepool_cluster // -------------------------------------------------- // West HBM ejection points, one per mesh row: floo_tcdm_chimney (SbrPort, no router) // unpacks req flits -> reqrsp_to_axi -> DRAM and source-routes responses back. - // HBM0 also splits off peripheral traffic (reqrsp_xbar, see gen_hbm0_demux). + // HBM0 also serves the BootROM (reqrsp_xbar, see gen_hbm0_demux). // -------------------------------------------------- for (genvar gy = 0; gy < NumGroupsY; gy++) begin : gen_hbm_west @@ -713,27 +799,23 @@ module cachepool_cluster // Routing .id_i ( floo_cachepool_noc_pkg::id_t'(HbmEndpointId) ), .route_table_i ( floo_cachepool_noc_pkg::RoutingTables[HbmEndpointId]), - // Request flit from mesh West(3) edge (no router) - .floo_req_o ( ), - .floo_req_valid_o ( ), - .floo_req_ready_i ( 1'b1 ), + // Request flits on the mesh West(3) edge (no router). Without a manager port the + // chimney injects nothing and drains stray responses. + .floo_req_o ( l2_req_in[0][gy][3] ), + .floo_req_valid_o ( l2_req_in_valid[0][gy][3] ), + .floo_req_ready_i ( l2_req_in_ready[0][gy][3] ), .floo_req_i ( l2_req_out[0][gy][3] ), .floo_req_valid_i ( l2_req_out_valid[0][gy][3] ), .floo_req_ready_o ( l2_req_out_ready[0][gy][3] ), - // Response flit into mesh West(3) edge + // Response flits on the mesh West(3) edge .floo_rsp_o ( l2_rsp_in[0][gy][3] ), .floo_rsp_valid_o ( l2_rsp_in_valid[0][gy][3] ), .floo_rsp_ready_i ( l2_rsp_in_ready[0][gy][3] ), - .floo_rsp_i ( '0 ), - .floo_rsp_valid_i ( 1'b0 ), - .floo_rsp_ready_o ( ) + .floo_rsp_i ( l2_rsp_out[0][gy][3] ), + .floo_rsp_valid_i ( l2_rsp_out_valid[0][gy][3] ), + .floo_rsp_ready_o ( l2_rsp_out_ready[0][gy][3] ) ); - // Tie off: no requests injected from HBM, drain stray outgoing responses - assign l2_req_in [0][gy][3] = '0; - assign l2_req_in_valid [0][gy][3] = 1'b0; - assign l2_rsp_out_ready[0][gy][3] = 1'b1; - // Pack chimney sbr signals into REQRSP bundle cache_trans_req_t hbm_reqrsp_req; cache_trans_rsp_t hbm_reqrsp_rsp; @@ -747,18 +829,25 @@ module cachepool_cluster assign hbm_sbr_rsp = hbm_reqrsp_rsp.p; assign hbm_sbr_rsp_valid = hbm_reqrsp_rsp.p_valid; - // HBM0: split DRAM vs peripheral with reqrsp_xbar, not reqrsp_demux, whose shared in-order + // HBM0: split DRAM vs BootROM with reqrsp_xbar, not reqrsp_demux, whose shared in-order // ID FIFO capped outstanding refills. Cross-path reordering is safe (miss info is in - // refill_user_t); a reqrsp_to_axi per path keeps each user-field FIFO in order. + // refill_user_t). if (HbmIdx == L2PeriphChannel) begin : gen_hbm0_demux - // Address-based select: DRAM = port 0, Peripheral = port 1. The - // window is this chimney's own channel slice: under all-sides - // placement that is channel L2PeriphChannel, not channel 0. + // Select DRAM = port 0, BootROM = port 1. The DRAM window is this chimney's channel slice + // (channel L2PeriphChannel under all-sides placement); the interconnect carries scrambled + // addresses, revertAddr recovers the physical one. localparam int unsigned HbmChBase = DramAddr + HbmIdx * DramPerChSize; - logic hbm0_reqrsp_sel; + addr_t hbm0_pa; + logic hbm0_reqrsp_sel; + assign hbm0_pa = revertAddr(hbm_reqrsp_req.q.addr); assign hbm0_reqrsp_sel = !((hbm_reqrsp_req.q.addr >= HbmChBase) && - (hbm_reqrsp_req.q.addr < HbmChBase + DramPerChSize)); + (hbm_reqrsp_req.q.addr < HbmChBase + DramPerChSize)); + + // Everything outside the DRAM slice must be an iCache refill of the BootROM + `ASSERT(Hbm0BootromOnly, (hbm_reqrsp_req.q_valid && hbm0_reqrsp_sel) |-> + ((hbm0_pa >= BootAddr) && (hbm0_pa < BootAddr + BootromSize)), clk_i, !rst_ni, + $sformatf("cachepool_cluster: wide L2 access to unmapped address 0x%08x", hbm0_pa)) cache_trans_req_t [1:0] hbm0_demux_req; cache_trans_rsp_t [1:0] hbm0_demux_rsp; @@ -850,46 +939,54 @@ module cachepool_cluster assign wide_axi_slv_req[HbmIdx] = hbm0_dram_axi_req; assign hbm0_dram_axi_rsp = wide_axi_slv_rsp[HbmIdx]; + // Port 1: BootROM, one full-width word per iCache line refill + `REG_BUS_TYPEDEF_ALL(reg_bootrom, addr_t, data_cache_t, strb_cache_t) + + cache_trans_req_t hbm0_bootrom_req; + reg_bootrom_req_t bootrom_reg_req; + reg_bootrom_rsp_t bootrom_reg_rsp; - // Port 1: Non-DRAM — revert VA back to PA, then DW convert to 32b - // All paths through the interconnect use scrambled (VA) addresses. - // revertAddr recovers the physical address for the peripheral endpoint. - cache_trans_req_t hbm0_peri_reqrsp_req; always_comb begin - hbm0_peri_reqrsp_req = hbm0_demux_req[1]; - hbm0_peri_reqrsp_req.q.addr = revertAddr(hbm0_demux_req[1].q.addr); + hbm0_bootrom_req = hbm0_demux_req[1]; + hbm0_bootrom_req.q.addr = revertAddr(hbm0_demux_req[1].q.addr); end - cache_trans_rsp_t hbm0_peri_reqrsp_rsp; - assign hbm0_demux_rsp[1] = hbm0_peri_reqrsp_rsp; - - // W2N converter: 512b REQRSP → 32b REQRSP - // Same user type (refill_user_t) on both sides. - // We add a small 1-entry buffer with MSHR inside to reduce boot time - reqrsp_w2n_converter #( - .SlvDataWidth ( AxiDataWidth ), - .MstDataWidth ( 32 ), - .AddrWidth ( AxiAddrWidth ), - .SlvUserWidth ( $bits(refill_user_t) ), - .MstUserWidth ( $bits(refill_user_t) ), - .MaxTrans ( 4 ), - .EnRdBuf ( 1'b1 ), - .CacheableBase ( BootAddr ), - .CacheableSize ( 32'h1_0000 ), - .slv_req_t ( cache_trans_req_t ), - .slv_rsp_t ( cache_trans_rsp_t ), - .mst_req_t ( peri_narrow_req_t ), - .mst_rsp_t ( peri_narrow_rsp_t ), - .slv_user_t ( refill_user_t ) - ) i_hbm0_peri_dw ( - .clk_i ( clk_i ), - .rst_ni ( rst_ni ), - .slv_req_i ( hbm0_peri_reqrsp_req ), - .slv_rsp_o ( hbm0_peri_reqrsp_rsp ), - .mst_req_o ( hbm0_peri_narrow_req ), - .mst_rsp_i ( hbm0_peri_narrow_rsp ) + reqrsp_to_reg #( + .AddrWidth ( AxiAddrWidth ), + .DataWidth ( AxiDataWidth ), + .UserWidth ( $bits(refill_user_t) ), + .ShiftResponse ( 1'b0 ), + .user_t ( refill_user_t ), + .reqrsp_req_t ( cache_trans_req_t ), + .reqrsp_rsp_t ( cache_trans_rsp_t ), + .reg_req_t ( reg_bootrom_req_t ), + .reg_rsp_t ( reg_bootrom_rsp_t ) + ) i_reqrsp_to_reg_bootrom ( + .clk_i ( clk_i ), + .rst_ni ( rst_ni ), + .reqrsp_req_i ( hbm0_bootrom_req ), + .reqrsp_rsp_o ( hbm0_demux_rsp[1] ), + .reg_req_o ( bootrom_reg_req ), + .reg_rsp_i ( bootrom_reg_rsp ) ); + bootrom #( + .DataWidth ( AxiDataWidth ), + .AddrWidth ( AxiAddrWidth ) + ) i_bootrom ( + .clk_i ( clk_i ), + .req_i ( bootrom_reg_req.valid ), + .addr_i ( addr_t'(bootrom_reg_req.addr) ), + .rdata_o ( bootrom_reg_rsp.rdata ) + ); + // The ROM registers its address: ready one cycle after each new address, so + // back-to-back requests get a setup cycle each. + `FF(bootrom_reg_rsp.ready, bootrom_reg_req.valid & ~bootrom_reg_rsp.ready, 1'b0) + assign bootrom_reg_rsp.error = 1'b0; + + `ASSERT_INIT(CheckBootromWidth, BootromDataWidth == AxiDataWidth, + "cachepool_cluster: bootrom_data_width must equal the L2 refill width") + end else begin : gen_hbm_direct // Non-HBM0: single reqrsp_to_axi → DRAM axi_slv_cache_req_t hbm_axi_req; @@ -923,6 +1020,94 @@ module cachepool_cluster end end + // -------------------------------------------------- + // Narrow L2 mesh endpoint of the cluster peripheral, West of group (0,0) with HBM0's ID and + // routes. Subordinate port: core CSR/UART accesses; manager port: configuration batches, + // one register write per group. + // -------------------------------------------------- + + peri_noc_req_chan_t peri_mgr_req; + logic peri_mgr_req_valid; + logic peri_mgr_req_ready; + peri_narrow_rsp_chan_t peri_mgr_rsp; + logic peri_mgr_rsp_valid; + logic peri_mgr_rsp_ready; + + // Requests carry this endpoint's ID so the groups route their responses back here. + cachepool_group_cfg_sender #( + .req_chan_t ( peri_noc_req_chan_t ), + .rsp_chan_t ( peri_narrow_rsp_chan_t ), + .user_t ( peri_narrow_user_t ), + .ReqUser ( '{l2_src_id: floo_cachepool_noc_pkg::id_t'(PeriEndpointId), default: '0} ) + ) i_group_cfg_sender ( + .clk_i ( clk_i ), + .rst_ni ( rst_ni ), + .start_i ( cfg_start ), + .reg_i ( cfg_reg ), + .data_i ( cfg_data ), + .tile_sel_i ( cfg_tile_sel ), + .done_o ( cfg_done ), + .req_o ( peri_mgr_req ), + .req_valid_o ( peri_mgr_req_valid ), + .req_ready_i ( peri_mgr_req_ready ), + .rsp_i ( peri_mgr_rsp ), + .rsp_valid_i ( peri_mgr_rsp_valid ), + .rsp_ready_o ( peri_mgr_rsp_ready ) + ); + + floo_tcdm_chimney #( + .RouteCfg ( PeriRouteCfg ), + .EnMgrPort ( 1'b1 ), + .EnSbrPort ( 1'b1 ), + .id_t ( floo_cachepool_noc_pkg::id_t ), + .route_t ( floo_cachepool_noc_pkg::route_t ), + .dst_t ( floo_cachepool_noc_pkg::route_t ), + .hdr_t ( l2_noc_hdr_t ), + .req_chan_t ( peri_noc_req_chan_t ), + .rsp_chan_t ( peri_narrow_rsp_chan_t ), + .floo_req_t ( peri_noc_req_t ), + .floo_rsp_t ( peri_noc_rsp_t ), + .addr_t ( axi_addr_t ), + .sam_rule_t ( floo_cachepool_noc_pkg::sam_rule_t ), + .Sam ( PeriSam ) + ) i_peri_chimney ( + .clk_i ( clk_i ), + .rst_ni ( rst_ni ), + .test_enable_i ( 1'b0 ), + // Manager port: group configuration writes + .mgr_req_i ( peri_mgr_req ), + .mgr_req_valid_i ( peri_mgr_req_valid ), + .mgr_req_ready_o ( peri_mgr_req_ready ), + .mgr_rsp_o ( peri_mgr_rsp ), + .mgr_rsp_valid_o ( peri_mgr_rsp_valid ), + .mgr_rsp_ready_i ( peri_mgr_rsp_ready ), + // Subordinate port: core accesses to the peripheral xbar + .sbr_req_o ( peri_sbr_req ), + .sbr_req_valid_o ( peri_sbr_req_valid ), + .sbr_req_ready_i ( peri_sbr_req_ready ), + .sbr_rsp_i ( peri_sbr_rsp ), + .sbr_rsp_valid_i ( peri_sbr_rsp_valid ), + .sbr_rsp_ready_o ( peri_sbr_rsp_ready ), + .sbr_txn_id_i ( peri_sbr_rsp.user.l2_src_id ), + // Routing + .id_i ( floo_cachepool_noc_pkg::id_t'(PeriEndpointId) ), + .route_table_i ( floo_cachepool_noc_pkg::RoutingTables[PeriEndpointId] ), + // Request flits on the narrow mesh West(3) edge of group (0,0) + .floo_req_o ( peri_req_in[0][0][3] ), + .floo_req_valid_o ( peri_req_in_valid[0][0][3] ), + .floo_req_ready_i ( peri_req_in_ready[0][0][3] ), + .floo_req_i ( peri_req_out[0][0][3] ), + .floo_req_valid_i ( peri_req_out_valid[0][0][3] ), + .floo_req_ready_o ( peri_req_out_ready[0][0][3] ), + // Response flits on the narrow mesh West(3) edge of group (0,0) + .floo_rsp_o ( peri_rsp_in[0][0][3] ), + .floo_rsp_valid_o ( peri_rsp_in_valid[0][0][3] ), + .floo_rsp_ready_i ( peri_rsp_in_ready[0][0][3] ), + .floo_rsp_i ( peri_rsp_out[0][0][3] ), + .floo_rsp_valid_i ( peri_rsp_out_valid[0][0][3] ), + .floo_rsp_ready_o ( peri_rsp_out_ready[0][0][3] ) + ); + // -------------------------------------------------- // East HBM ejection points (one per mesh row, channel ID per HbmIdx below) // floo_tcdm_chimney (SbrPort, no router) at mesh edge. @@ -1318,26 +1503,26 @@ module cachepool_cluster // style-waive: COMMENT_BLOCK: peripheral fabric data path diagram // -------------------------------------------------- - // Peripheral Fabric (REQRSP-based 2×3 xbar) + // Peripheral Fabric (REQRSP-based 2×2 xbar) // -------------------------------------------------- // // Data path (all 32b REQRSP): - // Source [0]: HBM0 demux port 1 -> reqrsp_w2n_converter (512b-32b) -> pad src_id=0 - // Source [1]: peri_ext_req_i (TB/SoC, already 32b REQRSP) -> pad src_id=1 + // Source [0]: narrow L2 mesh (i_peri_chimney subordinate port) -> pad src_id=0 + // Source [1]: peri_ext_req_i (TB/SoC, already 32b REQRSP) -> pad src_id=1 // - // Target [0]: BootROM -> reqrsp_to_reg -> bootrom - // Target [1]: CSR -> reqrsp_to_reg -> cachepool_peripheral - // Target [2]: UART -> reqrsp_to_axi -> axi_narrow_req_o + // Target [0]: CSR -> reqrsp_to_reg -> cachepool_peripheral + // Target [1]: UART -> reqrsp_to_axi -> axi_narrow_req_o + // + // The BootROM is not reachable here: it sits on the wide mesh (HBM0 demux port 1). // // Response routing: mst_sel_i = mst_rsp.user.src_id (1-bit, 0=NoC, 1=TB) localparam int unsigned PeriXbarNumSrc = 2; - localparam int unsigned PeriXbarNumTgt = 3; + localparam int unsigned PeriXbarNumTgt = 2; typedef enum int unsigned { - PeriTgtBootROM = 0, - PeriTgtCSR = 1, - PeriTgtUART = 2 + PeriTgtCSR = 0, + PeriTgtUART = 1 } peri_tgt_e; // Calculate the peripheral base address (needed for xbar rules). @@ -1351,15 +1536,13 @@ module cachepool_cluster // style-waive: AUTOMATIC: boundary checks function automatic peri_tgt_sel_t peri_addr_decode(input addr_t addr); - if (addr >= BootAddr && addr < BootAddr + 32'h1_0000) - return peri_tgt_sel_t'(PeriTgtBootROM); - else if (addr >= UartAddr && addr < UartAddr + 32'h1_0000) + if (addr >= UartAddr && addr < UartAddr + 32'h1_0000) return peri_tgt_sel_t'(PeriTgtUART); else return peri_tgt_sel_t'(PeriTgtCSR); endfunction - // ---- Inline src_id packing: peri_narrow → peri_xbar (add 1-bit src_id) ---- + // ---- Inline src_id packing: narrow mesh / peri_narrow → peri_xbar (add 1-bit src_id) ---- peri_xbar_req_chan_t [PeriXbarNumSrc-1:0] peri_xbar_src_req; logic [PeriXbarNumSrc-1:0] peri_xbar_src_req_valid; @@ -1375,26 +1558,28 @@ module cachepool_cluster logic [PeriXbarNumTgt-1:0] peri_xbar_tgt_rsp_valid; logic [PeriXbarNumTgt-1:0] peri_xbar_tgt_rsp_ready; - // Source [0]: HBM0 (src_id = 0) + // Source [0]: narrow L2 mesh (src_id = 0). The flit has no AMO field. always_comb begin peri_xbar_src_req[0] = '0; - peri_xbar_src_req[0].addr = hbm0_peri_narrow_req.q.addr; - peri_xbar_src_req[0].write = hbm0_peri_narrow_req.q.write; - peri_xbar_src_req[0].amo = hbm0_peri_narrow_req.q.amo; - peri_xbar_src_req[0].data = hbm0_peri_narrow_req.q.data; - peri_xbar_src_req[0].strb = hbm0_peri_narrow_req.q.strb; - peri_xbar_src_req[0].size = hbm0_peri_narrow_req.q.size; - peri_xbar_src_req[0].user = '{src_id: 1'b0, refill: hbm0_peri_narrow_req.q.user}; + peri_xbar_src_req[0].addr = peri_sbr_req.addr; + peri_xbar_src_req[0].write = peri_sbr_req.write; + peri_xbar_src_req[0].amo = reqrsp_pkg::AMONone; + peri_xbar_src_req[0].data = peri_sbr_req.data; + peri_xbar_src_req[0].strb = peri_sbr_req.strb; + peri_xbar_src_req[0].size = peri_sbr_req.size; + peri_xbar_src_req[0].user = '{src_id: 1'b0, narrow: peri_sbr_req.user}; end - assign peri_xbar_src_req_valid[0] = hbm0_peri_narrow_req.q_valid; - assign peri_xbar_src_rsp_ready[0] = hbm0_peri_narrow_req.p_ready; - - assign hbm0_peri_narrow_rsp.q_ready = peri_xbar_src_req_ready[0]; - assign hbm0_peri_narrow_rsp.p.data = peri_xbar_src_rsp[0].data; - assign hbm0_peri_narrow_rsp.p.error = peri_xbar_src_rsp[0].error; - assign hbm0_peri_narrow_rsp.p.write = peri_xbar_src_rsp[0].write; - assign hbm0_peri_narrow_rsp.p.user = peri_xbar_src_rsp[0].user.refill; - assign hbm0_peri_narrow_rsp.p_valid = peri_xbar_src_rsp_valid[0]; + assign peri_xbar_src_req_valid[0] = peri_sbr_req_valid; + assign peri_xbar_src_rsp_ready[0] = peri_sbr_rsp_ready; + + assign peri_sbr_req_ready = peri_xbar_src_req_ready[0]; + assign peri_sbr_rsp = '{ + data: peri_xbar_src_rsp[0].data, + error: peri_xbar_src_rsp[0].error, + write: peri_xbar_src_rsp[0].write, + user: peri_xbar_src_rsp[0].user.narrow + }; + assign peri_sbr_rsp_valid = peri_xbar_src_rsp_valid[0]; // Source [1]: TB/SoC external (src_id = 1) always_comb begin @@ -1405,7 +1590,7 @@ module cachepool_cluster peri_xbar_src_req[1].data = peri_ext_req_i.q.data; peri_xbar_src_req[1].strb = peri_ext_req_i.q.strb; peri_xbar_src_req[1].size = peri_ext_req_i.q.size; - peri_xbar_src_req[1].user = '{src_id: 1'b1, refill: peri_ext_req_i.q.user}; + peri_xbar_src_req[1].user = '{src_id: 1'b1, narrow: peri_ext_req_i.q.user}; end assign peri_xbar_src_req_valid[1] = peri_ext_req_i.q_valid; assign peri_xbar_src_rsp_ready[1] = peri_ext_req_i.p_ready; @@ -1414,19 +1599,35 @@ module cachepool_cluster assign peri_ext_rsp_o.p.data = peri_xbar_src_rsp[1].data; assign peri_ext_rsp_o.p.error = peri_xbar_src_rsp[1].error; assign peri_ext_rsp_o.p.write = peri_xbar_src_rsp[1].write; - assign peri_ext_rsp_o.p.user = peri_xbar_src_rsp[1].user.refill; + assign peri_ext_rsp_o.p.user = peri_xbar_src_rsp[1].user.narrow; assign peri_ext_rsp_o.p_valid = peri_xbar_src_rsp_valid[1]; // ---- Address decode per source ---- peri_tgt_sel_t [PeriXbarNumSrc-1:0] peri_slv_sel; - assign peri_slv_sel[0] = peri_addr_decode(hbm0_peri_narrow_req.q.addr); + assign peri_slv_sel[0] = peri_addr_decode(peri_sbr_req.addr); assign peri_slv_sel[1] = peri_addr_decode(peri_ext_req_i.q.addr); + // Core accesses only reach the CSRs and the UART; anything else is a software bug (stray + // pointer) that would silently read 0 from the CSR target, so flag it with its source. + `ifndef TARGET_SYNTHESIS + logic peri_core_addr_ok; + assign peri_core_addr_ok = + ((peri_sbr_req.addr >= tcdm_end_address) && + (peri_sbr_req.addr < tcdm_end_address + ClusterPeriphSize * 1024)) || + ((peri_sbr_req.addr >= UartAddr) && + (peri_sbr_req.addr < UartAddr + 32'h1_0000)); + `ASSERT(PeriCoreAddrMapped, peri_sbr_req_valid |-> peri_core_addr_ok, clk_i, !rst_ni, + $sformatf("cachepool_cluster: core access to unmapped address 0x%08x from L2 endpoint %0d tile %0d hart %0d", + peri_sbr_req.addr, peri_sbr_req.user.l2_src_id, peri_sbr_req.user.tile_id, + peri_sbr_req.user.hart_id)) + `ASSERT(PeriSingleWord, peri_sbr_req_valid |-> (peri_sbr_req.size <= reqrsp_pkg::size_t'(2)), + clk_i, !rst_ni, "cachepool_cluster: peripheral access wider than 32 bits") + `endif + // ---- Response routing per target: use src_id from user field ---- peri_src_sel_t [PeriXbarNumTgt-1:0] peri_mst_sel; assign peri_mst_sel[0] = peri_xbar_tgt_rsp[0].user.src_id; assign peri_mst_sel[1] = peri_xbar_tgt_rsp[1].user.src_id; - assign peri_mst_sel[2] = peri_xbar_tgt_rsp[2].user.src_id; // ---- 2×3 REQRSP Xbar ---- reqrsp_xbar #( @@ -1458,57 +1659,7 @@ module cachepool_cluster .mst_sel_i ( peri_mst_sel ) ); - // ---- Target [0]: BootROM → reqrsp_to_reg → bootrom ---- - - `REG_BUS_TYPEDEF_ALL(reg_bootrom, addr_t, narrow_data_t, narrow_strb_t) - - peri_xbar_req_t peri_bootrom_req; - peri_xbar_rsp_t peri_bootrom_rsp; - - // Bundle xbar split channels into REQRSP struct - assign peri_bootrom_req.q = peri_xbar_tgt_req[PeriTgtBootROM]; - assign peri_bootrom_req.q_valid = peri_xbar_tgt_req_valid[PeriTgtBootROM]; - assign peri_bootrom_req.p_ready = peri_xbar_tgt_rsp_ready[PeriTgtBootROM]; - assign peri_xbar_tgt_req_ready[PeriTgtBootROM] = peri_bootrom_rsp.q_ready; - assign peri_xbar_tgt_rsp[PeriTgtBootROM] = peri_bootrom_rsp.p; - assign peri_xbar_tgt_rsp_valid[PeriTgtBootROM] = peri_bootrom_rsp.p_valid; - - reg_bootrom_req_t bootrom_reg_req; - reg_bootrom_rsp_t bootrom_reg_rsp; - - reqrsp_to_reg #( - .AddrWidth ( AxiAddrWidth ), - .DataWidth ( 32 ), - .UserWidth ( $bits(peri_xbar_user_t) ), - .FifoDepth ( 2 ), - .ShiftResponse( 1'b0 ), - .user_t ( peri_xbar_user_t ), - .reqrsp_req_t ( peri_xbar_req_t ), - .reqrsp_rsp_t ( peri_xbar_rsp_t ), - .reg_req_t ( reg_bootrom_req_t ), - .reg_rsp_t ( reg_bootrom_rsp_t ) - ) i_reqrsp_to_reg_bootrom ( - .clk_i ( clk_i ), - .rst_ni ( rst_ni ), - .reqrsp_req_i ( peri_bootrom_req ), - .reqrsp_rsp_o ( peri_bootrom_rsp ), - .reg_req_o ( bootrom_reg_req ), - .reg_rsp_i ( bootrom_reg_rsp ) - ); - - bootrom #( - .DataWidth ( 32 ), - .AddrWidth ( AxiAddrWidth ) - ) i_bootrom ( - .clk_i ( clk_i ), - .req_i ( bootrom_reg_req.valid ), - .addr_i ( addr_t'(bootrom_reg_req.addr) ), - .rdata_o ( bootrom_reg_rsp.rdata ) - ); - `FF(bootrom_reg_rsp.ready, bootrom_reg_req.valid, 1'b0) - assign bootrom_reg_rsp.error = 1'b0; - - // ---- Target [1]: CSR → reqrsp_to_reg → cachepool_peripheral ---- + // ---- Target [0]: CSR → reqrsp_to_reg → cachepool_peripheral ---- `REG_BUS_TYPEDEF_ALL(reg_csr, addr_t, narrow_data_t, narrow_strb_t) @@ -1530,7 +1681,6 @@ module cachepool_cluster .AddrWidth ( AxiAddrWidth ), .DataWidth ( 32 ), .UserWidth ( $bits(peri_xbar_user_t) ), - .FifoDepth ( 2 ), .ShiftResponse( 1'b0 ), .user_t ( peri_xbar_user_t ), .reqrsp_req_t ( peri_xbar_req_t ), @@ -1554,8 +1704,7 @@ module cachepool_cluster .NumBarrierSlots ( NumBarrierSlots ), .PrivateWidth ( L1PrivWidth ), .reg_req_t ( reg_csr_req_t ), - .reg_rsp_t ( reg_csr_rsp_t ), - .cache_insn_t ( cache_insn_t ) + .reg_rsp_t ( reg_csr_rsp_t ) ) i_cachepool_cluster_peripheral ( .clk_i ( clk_i ), .rst_ni ( rst_ni ), @@ -1564,21 +1713,19 @@ module cachepool_cluster .reg_rsp_o ( reg_rsp ), .tcdm_start_address_i ( tcdm_start_address ), .tcdm_end_address_i ( tcdm_end_address ), - .icache_prefetch_enable_o ( icache_prefetch_enable ), .cluster_hart_base_id_i ( HartBaseId ), .cluster_probe_o ( cluster_probe_o ), - .dynamic_offset_o ( dynamic_offset ), - .private_start_addr_o ( private_start_addr ), .l1d_spm_size_o ( ), - .l1d_private_o ( l1d_private ), - .l1d_insn_o ( l1d_insn ), - .l1d_insn_valid_o ( l1d_insn_valid ), - .l1d_insn_ready_i ( l1d_insn_ready ), .l1d_busy_o ( l1d_busy ), + .cfg_start_o ( cfg_start ), + .cfg_reg_o ( cfg_reg ), + .cfg_data_o ( cfg_data ), + .cfg_tile_sel_o ( cfg_tile_sel ), + .cfg_done_i ( cfg_done ), .barrier_part_mask_o ( barrier_part_mask ) ); - // ---- Target [2]: UART → reqrsp_to_axi → axi_narrow_req_o ---- + // ---- Target [1]: UART → reqrsp_to_axi → axi_narrow_req_o ---- peri_xbar_req_t peri_uart_req; peri_xbar_rsp_t peri_uart_rsp; diff --git a/hardware/src/cachepool_group.sv b/hardware/src/cachepool_group.sv index ccde038b..e95ccc81 100644 --- a/hardware/src/cachepool_group.sv +++ b/hardware/src/cachepool_group.sv @@ -6,6 +6,7 @@ `include "axi/typedef.svh" `include "common_cells/registers.svh" +`include "common_cells/assertions.svh" `include "register_interface/typedef.svh" /// Group implementation for CachePool @@ -136,9 +137,12 @@ module cachepool_group input axi_addr_t cluster_base_addr_i, /// Partitioning address input axi_addr_t private_start_addr_i, - /// L2 refill reqrsp port (single merged output: cache refill + iCache/peripheral) + /// L2 refill reqrsp port (single merged output: cache refill + iCache) output l2_req_t l2_req_o, input l2_rsp_t l2_rsp_i, + /// Narrow peripheral reqrsp port (single merged output of the tiles' core accesses) + output peri_narrow_req_t periph_req_o, + input peri_narrow_rsp_t periph_rsp_i, /// This group's L2 refill mesh endpoint ID, stamped into refill_user_t.l2_src_id so the /// HBM chimney routes responses back without an in-order src_id FIFO. input floo_cachepool_noc_pkg::id_t l2_group_id_i, @@ -268,8 +272,8 @@ module cachepool_group cache_trans_rsp_t [NumL1CacheCtrlLocal-1:0] cache_refill_rsp; // Per-tile peripheral REQRSP (narrow 32b, from tile core mux) - periph_req_t [NumTilesPerGroup-1:0] tile_periph_req; - periph_rsp_t [NumTilesPerGroup-1:0] tile_periph_rsp; + peri_narrow_req_t [NumTilesPerGroup-1:0] tile_periph_req; + peri_narrow_rsp_t [NumTilesPerGroup-1:0] tile_periph_rsp; // L2 Group ICache AXI master output (from axi_hier_interco) axi_mst_cache_req_t axi_l2icache_mst_req; @@ -364,12 +368,11 @@ module cachepool_group // style-waive: COMMENT_BLOCK: port layout table // --------------------- - // Wiring: assemble flat xbar input from refill + peripheral + iCache paths + // Wiring: assemble flat xbar input from refill + iCache paths // --------------------- - // Port layout per tile (NumClusterMst = NumL1CtrlTile + 2): + // Port layout per tile (NumClusterMst = NumL1CtrlTile + 1): // p = 0..NumL1CtrlTile-1 → refill (bank_id = p, set by tile) - // p = NumL1CtrlTile → peripheral (bank_id = BankIdPeriph) - // p = NumL1CtrlTile+1 → iCache (bank_id = BankIdICache, only tile 0) + // p = NumL1CtrlTile → iCache (bank_id = BankIdICache, only tile 0) always_comb begin for (int t = 0; t < NumTilesPerGroup; t++) begin for (int p = 0; p < NumClusterMst; p++) begin @@ -389,32 +392,6 @@ module cachepool_group tile_req_chan [xbar_idx].user.tile_id = t; tile_req_chan [xbar_idx].user.l2_src_id = l2_group_id_i; - end else if (p == BankIdPeriph) begin - // Peripheral path: narrow REQRSP directly connected (data width mismatch — placeholder) - tile_req_chan [xbar_idx] = '0; - tile_req_chan [xbar_idx].addr = scrambleAddr(tile_periph_req[t].q.addr); - tile_req_chan [xbar_idx].write = tile_periph_req[t].q.write; - tile_req_chan [xbar_idx].amo = tile_periph_req[t].q.amo; - tile_req_chan [xbar_idx].size = tile_periph_req[t].q.size; - tile_req_chan [xbar_idx].data = axi_wide_data_t'(tile_periph_req[t].q.data); - tile_req_chan [xbar_idx].strb = axi_wide_strb_t'(tile_periph_req[t].q.strb); - tile_req_chan [xbar_idx].user.tile_id = t; - tile_req_chan [xbar_idx].user.bank_id = BankIdPeriph; - tile_req_chan [xbar_idx].user.l2_src_id = l2_group_id_i; - // Borrow the info field to carry req_id through the mesh - tile_req_chan [xbar_idx].user.info = cache_info_t'(tile_periph_req[t].q.user.req_id); - tile_req_valid [xbar_idx] = tile_periph_req[t].q_valid; - tile_periph_rsp[t].q_ready = tile_req_ready[xbar_idx]; - - tile_periph_rsp[t].p.data = tile_rsp_chan[xbar_idx].data[31:0]; - tile_periph_rsp[t].p.error = tile_rsp_chan[xbar_idx].error; - tile_periph_rsp[t].p.write = tile_rsp_chan[xbar_idx].write; - // Restore req_id from the info field; zero other tcdm_user fields - tile_periph_rsp[t].p.user = '0; - tile_periph_rsp[t].p.user.req_id = reqid_t'(tile_rsp_chan[xbar_idx].user.info); - tile_periph_rsp[t].p_valid = tile_rsp_valid[xbar_idx]; - tile_rsp_ready [xbar_idx] = tile_periph_req[t].p_ready; - end else begin // iCache path: only tile 0 active if (t == 0) begin @@ -513,6 +490,67 @@ module cachepool_group end + // --------------------- + // Peripheral mux: tiles' narrow core accesses -> one narrow port. Each request is stamped with + // its tile and this group's endpoint ID; responses are routed back by user.tile_id. + // --------------------- + typedef logic [cf_math_pkg::idx_width(NumTilesPerGroup)-1:0] periph_tile_sel_t; + + peri_narrow_req_chan_t [NumTilesPerGroup-1:0] periph_tile_req; + logic [NumTilesPerGroup-1:0] periph_tile_req_valid, periph_tile_req_ready; + peri_narrow_rsp_chan_t [NumTilesPerGroup-1:0] periph_tile_rsp; + logic [NumTilesPerGroup-1:0] periph_tile_rsp_valid, periph_tile_rsp_ready; + periph_tile_sel_t periph_rsp_sel; + + for (genvar t = 0; t < NumTilesPerGroup; t++) begin : gen_periph_tile + always_comb begin + periph_tile_req[t] = tile_periph_req[t].q; + periph_tile_req[t].user.tile_id = t; + periph_tile_req[t].user.l2_src_id = l2_group_id_i; + end + assign periph_tile_req_valid[t] = tile_periph_req[t].q_valid; + assign periph_tile_rsp_ready[t] = tile_periph_req[t].p_ready; + assign tile_periph_rsp[t].q_ready = periph_tile_req_ready[t]; + assign tile_periph_rsp[t].p = periph_tile_rsp[t]; + assign tile_periph_rsp[t].p_valid = periph_tile_rsp_valid[t]; + end + + assign periph_rsp_sel = periph_rsp_i.p.user.tile_id; + + reqrsp_xbar #( + .NumInp ( NumTilesPerGroup ), + .NumOut ( 1 ), + .PipeReg ( 1'b1 ), + .tcdm_req_chan_t ( peri_narrow_req_chan_t ), + .tcdm_rsp_chan_t ( peri_narrow_rsp_chan_t ), + .mst_sel_t ( periph_tile_sel_t ) + ) i_periph_xbar ( + .clk_i ( clk_i ), + .rst_ni ( rst_ni ), + .slv_req_i ( periph_tile_req ), + .slv_rr_i ( '0 ), + .slv_req_valid_i ( periph_tile_req_valid ), + .slv_req_ready_o ( periph_tile_req_ready ), + .slv_rsp_o ( periph_tile_rsp ), + .slv_rsp_valid_o ( periph_tile_rsp_valid ), + .slv_rsp_ready_i ( periph_tile_rsp_ready ), + .slv_sel_i ( '0 ), + .slv_selected_o ( /* unused */ ), + .mst_req_o ( periph_req_o.q ), + .mst_req_valid_o ( periph_req_o.q_valid ), + .mst_req_ready_i ( periph_rsp_i.q_ready ), + .mst_rsp_i ( periph_rsp_i.p ), + .mst_rr_i ( '0 ), + .mst_rsp_valid_i ( periph_rsp_i.p_valid ), + .mst_rsp_ready_o ( periph_req_o.p_ready ), + .mst_sel_i ( periph_rsp_sel ) + ); + + // The narrow L2 mesh carries no AMO field: atomics to the peripherals are a software bug. + `ASSERT(PeriphNoAmo, periph_req_o.q_valid |-> (periph_req_o.q.amo == reqrsp_pkg::AMONone), + clk_i, !rst_ni, $sformatf("cachepool_group: atomic to peripheral address 0x%08x", + periph_req_o.q.addr)) + // Tile remote access signals // In/Out relative to the tile (out--leave a tile; in--enter a tile) // Tile-side flat layout: index = j + r*NrTCDMPortsPerCore (j=xbar idx, r=remote slot within xbar) diff --git a/hardware/src/cachepool_group_cfg.sv b/hardware/src/cachepool_group_cfg.sv new file mode 100644 index 00000000..b236630d --- /dev/null +++ b/hardware/src/cachepool_group_cfg.sv @@ -0,0 +1,141 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// Solderpad Hardware License, Version 0.51, see LICENSE for details. +// SPDX-License-Identifier: SHL-0.51 + +`include "common_cells/registers.svh" +`include "common_cells/assertions.svh" + +/// style-waive: COMMENT_BLOCK: function description +/// Group-level configuration register. Receives single-register writes (group_cfg_reg_e) from the +/// cluster peripheral over the narrow L2 mesh (subordinate port of the group chimney) and drives +/// the L1D/xbar configuration of this group's tiles. A GroupCfgL1dInsn write starts its L1D +/// instruction in the selected tiles and is only answered once all of them have finished; any +/// other write is answered at once. The request is always accepted right away, so a pending flush +/// never stalls the mesh. +module cachepool_group_cfg + import cachepool_pkg::*; +#( + parameter type req_chan_t = logic, + parameter type rsp_chan_t = logic, + parameter type user_t = logic +) ( + input logic clk_i, + input logic rst_ni, + // Subordinate port from the L2 mesh chimney + input req_chan_t req_i, + input logic req_valid_i, + output logic req_ready_o, + output rsp_chan_t rsp_o, + output logic rsp_valid_o, + input logic rsp_ready_i, + // Configuration of the group's tiles + output xbar_offset_t dynamic_offset_o, + output logic [L1PrivWidth-1:0] l1d_private_o, + output axi_addr_t private_start_addr_o, + output logic icache_prefetch_enable_o, + output cache_insn_t l1d_insn_o, + output logic l1d_insn_valid_o, + input logic [NumTilesPerGroup-1:0] l1d_insn_ready_i +); + + // style-waive: COMMENT_BLOCK: FSM transition table + // Idle -(GroupCfgL1dInsn write)-> Flush -(selected tiles done)-> Resp -(rsp_ready)-> Idle + // Idle -(other write)----------------------------------------> Resp + typedef enum logic [1:0] { + Idle, + Flush, + Resp + } state_e; + + state_e state_d, state_q; + l1d_cfg_t cfg_d, cfg_q; + cache_insn_t insn_d, insn_q; + logic insn_valid_d, insn_valid_q; + // Tiles that still have to finish the current instruction + logic [NumTilesPerGroup-1:0] lock_d, lock_q; + user_t user_d, user_q; + + // Register index and value of the incoming write + group_cfg_reg_e req_reg; + logic [31:0] req_data; + cache_insn_t req_insn; + assign req_reg = group_cfg_reg_e'(req_i.addr[2 +: $bits(group_cfg_reg_e)]); + assign req_data = req_i.data[31:0]; + assign req_insn = req_data[$bits(cache_insn_t)-1:0]; + + always_comb begin : cfg_fsm + state_d = state_q; + cfg_d = cfg_q; + insn_d = insn_q; + insn_valid_d = 1'b0; + lock_d = lock_q & ~l1d_insn_ready_i; + user_d = user_q; + req_ready_o = 1'b0; + rsp_valid_o = 1'b0; + + unique case (state_q) + Idle: begin + req_ready_o = 1'b1; + if (req_valid_i) begin + user_d = req_i.user; + state_d = Resp; + unique case (req_reg) + GroupCfgXbarOffset: cfg_d.xbar_offset = req_data[$bits(xbar_offset_t)-1:0]; + GroupCfgL1dPrivate: cfg_d.l1d_private = req_data[L1PrivWidth-1:0]; + GroupCfgL1dAddr: cfg_d.private_addr = req_data; + GroupCfgPrefetch: cfg_d.prefetch_en = req_data[0]; + GroupCfgL1dInsn: begin + insn_d = req_insn; + insn_valid_d = 1'b1; + // Only private flushes target a subset of the tiles + lock_d = (req_insn.insn == 2'b00) ? req_insn.tile_sel : '1; + state_d = Flush; + end + default: ; + endcase + end + end + Flush: begin + if (lock_q == '0) state_d = Resp; + end + Resp: begin + rsp_valid_o = 1'b1; + if (rsp_ready_i) state_d = Idle; + end + default: state_d = Idle; + endcase + end + + always_comb begin : rsp_pack + rsp_o = '0; + rsp_o.write = 1'b1; + rsp_o.user = user_q; + end + + `FF(state_q, state_d, Idle, clk_i, rst_ni) + `FF(cfg_q, cfg_d, L1dCfgRst, clk_i, rst_ni) + `FF(insn_q, insn_d, '0, clk_i, rst_ni) + `FF(insn_valid_q, insn_valid_d, 1'b0, clk_i, rst_ni) + `FF(lock_q, lock_d, '0, clk_i, rst_ni) + `FF(user_q, user_d, '0, clk_i, rst_ni) + + assign dynamic_offset_o = cfg_q.xbar_offset; + assign l1d_private_o = cfg_q.l1d_private; + assign private_start_addr_o = cfg_q.private_addr; + assign icache_prefetch_enable_o = cfg_q.prefetch_en; + assign l1d_insn_o = insn_q; + assign l1d_insn_valid_o = insn_valid_q; + + `ASSERT_INIT(CheckRegsFitStride, GroupCfgStride >= 4 * (2 ** $bits(group_cfg_reg_e)), + "cachepool_group_cfg: GroupCfgStride is too small for the register block") + `ASSERT_INIT(CheckInsnFitsWord, $bits(cache_insn_t) <= 32, + "cachepool_group_cfg: cache_insn_t does not fit into one 32-bit register") + `ASSERT(OnlyCfgWrites, req_valid_i |-> req_i.write, clk_i, !rst_ni, + "cachepool_group_cfg: only writes from the cluster peripheral are expected") + `ASSERT(CfgAddrInRange, req_valid_i |-> (req_i.addr >= GroupCfgBase), clk_i, !rst_ni, + "cachepool_group_cfg: request address below GroupCfgBase") + `ASSERT(KnownCfgReg, req_valid_i |-> (req_reg inside {GroupCfgXbarOffset, GroupCfgL1dPrivate, + GroupCfgL1dAddr, GroupCfgPrefetch, GroupCfgL1dInsn}), clk_i, !rst_ni, + "cachepool_group_cfg: write to an unknown register") + +endmodule diff --git a/hardware/src/cachepool_group_cfg_sender.sv b/hardware/src/cachepool_group_cfg_sender.sv new file mode 100644 index 00000000..85714fe2 --- /dev/null +++ b/hardware/src/cachepool_group_cfg_sender.sv @@ -0,0 +1,127 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// Solderpad Hardware License, Version 0.51, see LICENSE for details. +// SPDX-License-Identifier: SHL-0.51 + +`include "common_cells/registers.svh" +`include "common_cells/assertions.svh" + +/// style-waive: COMMENT_BLOCK: function description +/// Forwards one configuration batch of the cluster peripheral, a single group-level register, to +/// every group-level configuration register (cachepool_group_cfg) as sequential unicast writes +/// over the narrow L2 mesh, and pulses done_o once all NumGroups responses are back. Group g is written +/// at GroupCfgBase + g * GroupCfgStride + 4 * reg_i; an L1D instruction carries the group's own +/// tile_sel slice. The batch inputs must stay stable from start_i until done_o (the peripheral +/// blocks new batches meanwhile). +module cachepool_group_cfg_sender + import cachepool_pkg::*; +#( + parameter type req_chan_t = logic, + parameter type rsp_chan_t = logic, + parameter type user_t = logic, + /// User field of every request; carries this endpoint's l2_src_id for response routing + parameter user_t ReqUser = '0 +) ( + input logic clk_i, + input logic rst_ni, + // Batch from the cluster peripheral + input logic start_i, + input group_cfg_reg_e reg_i, + input logic [31:0] data_i, + input logic [NumTiles-1:0] tile_sel_i, + output logic done_o, + // Manager port into the L2 mesh chimney + output req_chan_t req_o, + output logic req_valid_o, + input logic req_ready_i, + input rsp_chan_t rsp_i, + input logic rsp_valid_i, + output logic rsp_ready_o +); + + localparam int unsigned CntWidth = $clog2(NumGroups + 1); + typedef logic [CntWidth-1:0] cnt_t; + + logic busy_d, busy_q; + cnt_t send_cnt_d, send_cnt_q; + cnt_t rsp_cnt_d, rsp_cnt_q; + axi_addr_t addr_d, addr_q; + // Tile selection of the remaining groups; the current group's slice sits at the LSBs + logic [NumTiles-1:0] tile_sel_d, tile_sel_q; + + logic req_fire; + cache_insn_t group_insn; + logic [31:0] group_data; + + assign req_valid_o = busy_q && (send_cnt_q != NumGroups); + assign req_fire = req_valid_o && req_ready_i; + assign rsp_ready_o = 1'b1; + + // An L1D instruction carries the tile selection of the group it is sent to + always_comb begin : group_data_pack + group_insn.insn = data_i[1:0]; + group_insn.tile_sel = tile_sel_q[NumTilesPerGroup-1:0]; + group_data = data_i; + if (reg_i == GroupCfgL1dInsn) begin + group_data = '0; + group_data[$bits(cache_insn_t)-1:0] = group_insn; + end + end + + always_comb begin : req_pack + req_o = '0; + req_o.addr = addr_q; + req_o.write = 1'b1; + req_o.data[31:0] = group_data; + req_o.strb = '1; + req_o.user = ReqUser; + end + + always_comb begin : batch_ctrl + busy_d = busy_q; + send_cnt_d = send_cnt_q; + rsp_cnt_d = rsp_cnt_q; + addr_d = addr_q; + tile_sel_d = tile_sel_q; + done_o = 1'b0; + + if (!busy_q) begin + if (start_i) begin + busy_d = 1'b1; + send_cnt_d = '0; + rsp_cnt_d = '0; + addr_d = GroupCfgBase + {reg_i, 2'b00}; + tile_sel_d = tile_sel_i; + end + end else begin + if (req_fire) begin + send_cnt_d = send_cnt_q + 1'b1; + addr_d = addr_q + GroupCfgStride; + tile_sel_d = tile_sel_q >> NumTilesPerGroup; + end + if (rsp_valid_i) begin + rsp_cnt_d = rsp_cnt_q + 1'b1; + end + if (rsp_cnt_q == NumGroups) begin + busy_d = 1'b0; + done_o = 1'b1; + end + end + end + + `FF(busy_q, busy_d, 1'b0, clk_i, rst_ni) + `FF(send_cnt_q, send_cnt_d, '0, clk_i, rst_ni) + `FF(rsp_cnt_q, rsp_cnt_d, '0, clk_i, rst_ni) + `FF(addr_q, addr_d, '0, clk_i, rst_ni) + `FF(tile_sel_q, tile_sel_d, '0, clk_i, rst_ni) + + `ASSERT_INIT(CheckNumGroups, NumGroups > 1, "cachepool_group_cfg_sender: NumGroups must be > 1") + `ASSERT_INIT(CheckInsnFitsWord, $bits(cache_insn_t) <= 32, + "cachepool_group_cfg_sender: cache_insn_t does not fit into one 32-bit register") + `ASSERT(NoRspWhenIdle, rsp_valid_i |-> busy_q, clk_i, !rst_ni, + "cachepool_group_cfg_sender: response without an outstanding batch") + `ASSERT(NoExtraRsp, rsp_valid_i |-> (rsp_cnt_q < NumGroups), clk_i, !rst_ni, + "cachepool_group_cfg_sender: more responses than groups") + `ASSERT(NoStartWhenBusy, start_i |-> !busy_q, clk_i, !rst_ni, + "cachepool_group_cfg_sender: new batch while the previous one is in flight") + +endmodule diff --git a/hardware/src/cachepool_group_noc_wrapper.sv b/hardware/src/cachepool_group_noc_wrapper.sv index 9f881660..00bf276c 100644 --- a/hardware/src/cachepool_group_noc_wrapper.sv +++ b/hardware/src/cachepool_group_noc_wrapper.sv @@ -10,6 +10,7 @@ `include "common_cells/registers.svh" +`include "common_cells/assertions.svh" `include "axi/typedef.svh" module cachepool_group_noc_wrapper @@ -75,7 +76,6 @@ module cachepool_group_noc_wrapper input logic [9:0] hart_base_id_i, input logic [TileIDWidth-1:0] tile_base_id_i, input axi_addr_t cluster_base_addr_i, - input axi_addr_t private_start_addr_i, // L2 refill TCDM mesh: 4 directions (North=0, East=1, South=2, West=3) output l2_noc_req_t [3:0] l2_req_o, output logic [3:0] l2_req_valid_o, @@ -89,16 +89,25 @@ module cachepool_group_noc_wrapper input l2_noc_rsp_t [3:0] l2_rsp_i, input logic [3:0] l2_rsp_valid_i, output logic [3:0] l2_rsp_ready_o, - // L2 mesh: endpoint ID and source-routing table from floogen + // Narrow L2 mesh (peripheral accesses, group configuration): same 4 directions + output peri_noc_req_t [3:0] peri_req_o, + output logic [3:0] peri_req_valid_o, + input logic [3:0] peri_req_ready_i, + input peri_noc_req_t [3:0] peri_req_i, + input logic [3:0] peri_req_valid_i, + output logic [3:0] peri_req_ready_o, + output peri_noc_rsp_t [3:0] peri_rsp_o, + output logic [3:0] peri_rsp_valid_o, + input logic [3:0] peri_rsp_ready_i, + input peri_noc_rsp_t [3:0] peri_rsp_i, + input logic [3:0] peri_rsp_valid_i, + output logic [3:0] peri_rsp_ready_o, + // L2 mesh (wide and narrow): endpoint ID and source-routing table from floogen input floo_cachepool_noc_pkg::id_t l2_id_i, input floo_cachepool_noc_pkg::route_t [floo_cachepool_noc_pkg::RouteCfg.NumRoutes-1:0] l2_route_table_i, output icache_l1_events_t [NumCC-1:0] icache_events_o, - input logic icache_prefetch_enable_i, - input logic [$clog2(AxiAddrWidth)-1:0] dynamic_offset_i, - input logic [L1PrivWidth-1:0] l1d_private_i, - input cache_insn_t l1d_insn_i, - input logic l1d_insn_valid_i, - output logic [NumTilesPerGroup-1:0] l1d_insn_ready_o, + // Flush interlock from the cluster peripheral; the rest of the L1D/xbar configuration + // arrives over the narrow L2 mesh at the group configuration register. input logic [NumTilesPerGroup-1:0] l1d_busy_i, input impl_in_t [NrSramCfg-1:0] impl_i, output logic error_o, @@ -156,32 +165,24 @@ module cachepool_group_noc_wrapper remote_group_rsp_t [NumRemoteGroupPortGroup-1:0] remote_group_rsp_to_group; - // Input cuts on some registered signals. The barrier link is not cut + // Input cut on the flush interlock. The barrier link is not cut // here -- it is optionally cut inside cachepool_group itself, controlled // by RegisterBarrier, on top of the group_barrier FSM's own registered state. - logic [$clog2(AxiAddrWidth)-1:0] dynamic_offset_d, dynamic_offset_q; - logic [L1PrivWidth-1:0] l1d_private_d, l1d_private_q; - cache_insn_t l1d_insn_d, l1d_insn_q; - logic l1d_insn_valid_d, l1d_insn_valid_q; - logic [NumTilesPerGroup-1:0] l1d_insn_ready_d, l1d_insn_ready_q; logic [NumTilesPerGroup-1:0] l1d_busy_d, l1d_busy_q; - axi_addr_t private_start_addr_d, private_start_addr_q; - `FF(dynamic_offset_q, dynamic_offset_d, '0, clk_i, rst_ni) - `FF(l1d_private_q, l1d_private_d, '0, clk_i, rst_ni) - `FF(l1d_insn_q, l1d_insn_d, '0, clk_i, rst_ni) - `FF(l1d_insn_valid_q, l1d_insn_valid_d, '0, clk_i, rst_ni) - `FF(l1d_insn_ready_q, l1d_insn_ready_d, '0, clk_i, rst_ni) `FF(l1d_busy_q, l1d_busy_d, '0, clk_i, rst_ni) - `FF(private_start_addr_q, private_start_addr_d, '0, clk_i, rst_ni) - assign dynamic_offset_d = dynamic_offset_i; - assign l1d_private_d = l1d_private_i; - assign l1d_insn_d = l1d_insn_i; - assign l1d_insn_valid_d = l1d_insn_valid_i; - assign l1d_insn_ready_o = l1d_insn_ready_q; assign l1d_busy_d = l1d_busy_i; - assign private_start_addr_d = private_start_addr_i; + + // L1D/xbar configuration of the tiles, held by the group configuration register (i_group_cfg) + // and written by the cluster peripheral over the narrow L2 mesh. + xbar_offset_t dynamic_offset_q; + logic [L1PrivWidth-1:0] l1d_private_q; + axi_addr_t private_start_addr_q; + logic icache_prefetch_enable; + cache_insn_t l1d_insn_q; + logic l1d_insn_valid_q; + logic [NumTilesPerGroup-1:0] l1d_insn_ready; // ------------------------------------------------------------------------- // Mesh signals [tile][ch][dir=3:0] and transposition to/from ports @@ -567,8 +568,8 @@ module cachepool_group_noc_wrapper // ------------------------------------------------------------------------- // L2 path: floo_tcdm_chimney (MgrPort) + floo_router (source routing) // ------------------------------------------------------------------------- - // The chimney maps addresses to destination IDs (SAM), looks up source routes and - // packs/unpacks flits; two floo_routers (req + rsp) form the 5-port source-routed mesh. + // Chimney: address -> destination ID (SAM) -> source route; two floo_routers form the mesh. + // It only carries the group's own refills (no subordinate port); peripherals use the narrow mesh. // Internal L2 reqrsp between group and chimney l2_req_t l2_group_req; @@ -589,7 +590,7 @@ module cachepool_group_noc_wrapper logic [floo_pkg::Eject:floo_pkg::North] l2_router_rsp_valid_out; logic [floo_pkg::Eject:floo_pkg::North] l2_router_rsp_ready_in; - // Manager chimney: group REQRSP ↔ flit inject/eject at router Eject port + // Chimney at the router Eject port: manager for the group's REQRSP. floo_tcdm_chimney #( .RouteCfg ( floo_cachepool_noc_pkg::RouteCfg ), .EnMgrPort ( 1'b1 ), @@ -631,11 +632,11 @@ module cachepool_group_noc_wrapper .floo_req_o ( l2_router_req_in[floo_pkg::Eject] ), .floo_req_valid_o ( l2_router_req_valid_in[floo_pkg::Eject] ), .floo_req_ready_i ( l2_router_req_ready_out[floo_pkg::Eject] ), - // Request flit: eject from request router (unused for MgrPort) + // Request flit: eject from request router (none expected, drained) .floo_req_i ( l2_router_req_out[floo_pkg::Eject] ), .floo_req_valid_i ( l2_router_req_valid_out[floo_pkg::Eject] ), .floo_req_ready_o ( l2_router_req_ready_in[floo_pkg::Eject] ), - // Response flit: inject into response router (unused for MgrPort) + // Response flit: no subordinate port, nothing injected .floo_rsp_o ( l2_router_rsp_in[floo_pkg::Eject] ), .floo_rsp_valid_o ( l2_router_rsp_valid_in[floo_pkg::Eject] ), .floo_rsp_ready_i ( l2_router_rsp_ready_out[floo_pkg::Eject] ), @@ -645,6 +646,9 @@ module cachepool_group_noc_wrapper .floo_rsp_ready_o ( l2_router_rsp_ready_in[floo_pkg::Eject] ) ); + `ASSERT(L2NoReqEject, !l2_router_req_valid_out[floo_pkg::Eject], clk_i, !rst_ni, + "cachepool_group_noc_wrapper: request ejected on the wide L2 mesh") + // Request router (source routing, 5 ports: N/E/S/W/Eject) floo_router #( .NumRoutes ( 5 ), @@ -724,6 +728,204 @@ module cachepool_group_noc_wrapper assign l2_rsp_ready_o[d] = l2_router_rsp_ready_out[d]; end + // ------------------------------------------------------------------------- + // Narrow L2 path: floo_tcdm_chimney (MgrPort + SbrPort) + floo_router pair + // ------------------------------------------------------------------------- + // Same topology, IDs and routes as the wide mesh. Manager port: the group's core accesses to + // the cluster peripheral; subordinate port: its writes to the group configuration register. + + // Narrow reqrsp between group and chimney + peri_narrow_req_t periph_group_req; + peri_narrow_rsp_t periph_group_rsp; + peri_noc_req_chan_t periph_mgr_req; + + // Router port signals [Eject:North] = [4:0] + peri_noc_req_t [floo_pkg::Eject:floo_pkg::North] peri_router_req_in; + logic [floo_pkg::Eject:floo_pkg::North] peri_router_req_valid_in; + logic [floo_pkg::Eject:floo_pkg::North] peri_router_req_ready_out; + peri_noc_req_t [floo_pkg::Eject:floo_pkg::North] peri_router_req_out; + logic [floo_pkg::Eject:floo_pkg::North] peri_router_req_valid_out; + logic [floo_pkg::Eject:floo_pkg::North] peri_router_req_ready_in; + + peri_noc_rsp_t [floo_pkg::Eject:floo_pkg::North] peri_router_rsp_in; + logic [floo_pkg::Eject:floo_pkg::North] peri_router_rsp_valid_in; + logic [floo_pkg::Eject:floo_pkg::North] peri_router_rsp_ready_out; + peri_noc_rsp_t [floo_pkg::Eject:floo_pkg::North] peri_router_rsp_out; + logic [floo_pkg::Eject:floo_pkg::North] peri_router_rsp_valid_out; + logic [floo_pkg::Eject:floo_pkg::North] peri_router_rsp_ready_in; + + // Group configuration register on the chimney's subordinate port + peri_noc_req_chan_t cfg_sbr_req; + logic cfg_sbr_req_valid; + logic cfg_sbr_req_ready; + peri_narrow_rsp_chan_t cfg_sbr_rsp; + logic cfg_sbr_rsp_valid; + logic cfg_sbr_rsp_ready; + + // The flit has no AMO field (checked in cachepool_group) + assign periph_mgr_req = '{ + addr: periph_group_req.q.addr, + write: periph_group_req.q.write, + data: periph_group_req.q.data, + strb: periph_group_req.q.strb, + size: periph_group_req.q.size, + user: periph_group_req.q.user + }; + + floo_tcdm_chimney #( + .RouteCfg ( PeriCoreRouteCfg ), + .EnMgrPort ( 1'b1 ), + .EnSbrPort ( 1'b1 ), + .id_t ( floo_cachepool_noc_pkg::id_t ), + .route_t ( floo_cachepool_noc_pkg::route_t ), + .dst_t ( floo_cachepool_noc_pkg::route_t ), + .hdr_t ( l2_noc_hdr_t ), + .req_chan_t ( peri_noc_req_chan_t ), + .rsp_chan_t ( peri_narrow_rsp_chan_t ), + .floo_req_t ( peri_noc_req_t ), + .floo_rsp_t ( peri_noc_rsp_t ), + .addr_t ( axi_addr_t ), + .sam_rule_t ( floo_cachepool_noc_pkg::sam_rule_t ), + .Sam ( PeriCoreSam ) + ) i_peri_chimney ( + .clk_i ( clk_i ), + .rst_ni ( rst_ni ), + .test_enable_i ( 1'b0 ), + // Manager port: group core accesses + .mgr_req_i ( periph_mgr_req ), + .mgr_req_valid_i ( periph_group_req.q_valid ), + .mgr_req_ready_o ( periph_group_rsp.q_ready ), + .mgr_rsp_o ( periph_group_rsp.p ), + .mgr_rsp_valid_o ( periph_group_rsp.p_valid ), + .mgr_rsp_ready_i ( periph_group_req.p_ready ), + // Subordinate port: group configuration register, responses return to the requester + .sbr_req_o ( cfg_sbr_req ), + .sbr_req_valid_o ( cfg_sbr_req_valid ), + .sbr_req_ready_i ( cfg_sbr_req_ready ), + .sbr_rsp_i ( cfg_sbr_rsp ), + .sbr_rsp_valid_i ( cfg_sbr_rsp_valid ), + .sbr_rsp_ready_o ( cfg_sbr_rsp_ready ), + .sbr_txn_id_i ( cfg_sbr_rsp.user.l2_src_id ), + // Routing + .id_i ( l2_id_i ), + .route_table_i ( l2_route_table_i ), + // Request flits + .floo_req_o ( peri_router_req_in[floo_pkg::Eject] ), + .floo_req_valid_o ( peri_router_req_valid_in[floo_pkg::Eject] ), + .floo_req_ready_i ( peri_router_req_ready_out[floo_pkg::Eject] ), + .floo_req_i ( peri_router_req_out[floo_pkg::Eject] ), + .floo_req_valid_i ( peri_router_req_valid_out[floo_pkg::Eject] ), + .floo_req_ready_o ( peri_router_req_ready_in[floo_pkg::Eject] ), + // Response flits + .floo_rsp_o ( peri_router_rsp_in[floo_pkg::Eject] ), + .floo_rsp_valid_o ( peri_router_rsp_valid_in[floo_pkg::Eject] ), + .floo_rsp_ready_i ( peri_router_rsp_ready_out[floo_pkg::Eject] ), + .floo_rsp_i ( peri_router_rsp_out[floo_pkg::Eject] ), + .floo_rsp_valid_i ( peri_router_rsp_valid_out[floo_pkg::Eject] ), + .floo_rsp_ready_o ( peri_router_rsp_ready_in[floo_pkg::Eject] ) + ); + + cachepool_group_cfg #( + .req_chan_t ( peri_noc_req_chan_t ), + .rsp_chan_t ( peri_narrow_rsp_chan_t ), + .user_t ( peri_narrow_user_t ) + ) i_group_cfg ( + .clk_i ( clk_i ), + .rst_ni ( rst_ni ), + .req_i ( cfg_sbr_req ), + .req_valid_i ( cfg_sbr_req_valid ), + .req_ready_o ( cfg_sbr_req_ready ), + .rsp_o ( cfg_sbr_rsp ), + .rsp_valid_o ( cfg_sbr_rsp_valid ), + .rsp_ready_i ( cfg_sbr_rsp_ready ), + .dynamic_offset_o ( dynamic_offset_q ), + .l1d_private_o ( l1d_private_q ), + .private_start_addr_o ( private_start_addr_q ), + .icache_prefetch_enable_o ( icache_prefetch_enable ), + .l1d_insn_o ( l1d_insn_q ), + .l1d_insn_valid_o ( l1d_insn_valid_q ), + .l1d_insn_ready_i ( l1d_insn_ready ) + ); + + // Narrow request router (source routing, 5 ports: N/E/S/W/Eject) + floo_router #( + .NumRoutes ( 5 ), + .NumVirtChannels( 1 ), + .InFifoDepth ( 2 ), + .OutFifoDepth ( 2 ), + .RouteAlgo ( floo_pkg::SourceRouting ), + .id_t ( floo_cachepool_noc_pkg::id_t), + .flit_t ( peri_noc_req_t ), + .hdr_t ( l2_noc_hdr_t ), + .addr_rule_t ( logic ), + .red_req_t ( logic ), + .red_rsp_t ( logic ) + ) i_peri_req_router ( + .clk_i ( clk_i ), + .rst_ni ( rst_ni ), + .test_enable_i ( 1'b0 ), + .xy_id_i ( '0 ), + .id_route_map_i ( '0 ), + .valid_i ( peri_router_req_valid_in ), + .ready_o ( peri_router_req_ready_out ), + .data_i ( peri_router_req_in ), + .credit_o ( ), + .valid_o ( peri_router_req_valid_out ), + .ready_i ( peri_router_req_ready_in ), + .data_o ( peri_router_req_out ), + .credit_i ( '0 ), + .offload_req_o ( ), + .offload_rsp_i ( '0 ) + ); + + // Narrow response router (source routing, 5 ports: N/E/S/W/Eject) + floo_router #( + .NumRoutes ( 5 ), + .NumVirtChannels( 1 ), + .InFifoDepth ( 2 ), + .OutFifoDepth ( 2 ), + .RouteAlgo ( floo_pkg::SourceRouting ), + .id_t ( floo_cachepool_noc_pkg::id_t), + .flit_t ( peri_noc_rsp_t ), + .hdr_t ( l2_noc_hdr_t ), + .addr_rule_t ( logic ), + .red_req_t ( logic ), + .red_rsp_t ( logic ) + ) i_peri_rsp_router ( + .clk_i ( clk_i ), + .rst_ni ( rst_ni ), + .test_enable_i ( 1'b0 ), + .xy_id_i ( '0 ), + .id_route_map_i ( '0 ), + .valid_i ( peri_router_rsp_valid_in ), + .ready_o ( peri_router_rsp_ready_out ), + .data_i ( peri_router_rsp_in ), + .credit_o ( ), + .valid_o ( peri_router_rsp_valid_out ), + .ready_i ( peri_router_rsp_ready_in ), + .data_o ( peri_router_rsp_out ), + .credit_i ( '0 ), + .offload_req_o ( ), + .offload_rsp_i ( '0 ) + ); + + // Expose [West:North] narrow router ports (indices 3:0) to cluster for mesh wiring + for (genvar d = 0; d < 4; d++) begin : gen_peri_mesh_ports + assign peri_req_o[d] = peri_router_req_out[d]; + assign peri_req_valid_o[d] = peri_router_req_valid_out[d]; + assign peri_router_req_ready_in[d] = peri_req_ready_i[d]; + assign peri_router_req_in[d] = peri_req_i[d]; + assign peri_router_req_valid_in[d] = peri_req_valid_i[d]; + assign peri_req_ready_o[d] = peri_router_req_ready_out[d]; + + assign peri_rsp_o[d] = peri_router_rsp_out[d]; + assign peri_rsp_valid_o[d] = peri_router_rsp_valid_out[d]; + assign peri_router_rsp_ready_in[d] = peri_rsp_ready_i[d]; + assign peri_router_rsp_in[d] = peri_rsp_i[d]; + assign peri_router_rsp_valid_in[d] = peri_rsp_valid_i[d]; + assign peri_rsp_ready_o[d] = peri_router_rsp_ready_out[d]; + end + // ------------------------------------------------------------------------- // Group instantiation // ------------------------------------------------------------------------- @@ -781,6 +983,8 @@ module cachepool_group_noc_wrapper .l2_req_o ( l2_group_req ), .l2_rsp_i ( l2_group_rsp ), .l2_group_id_i ( l2_id_i ), + .periph_req_o ( periph_group_req ), + .periph_rsp_i ( periph_group_rsp ), .remote_group_req_o ( remote_group_req_from_group ), .remote_group_rsp_i ( remote_group_rsp_to_group ), .remote_group_req_i ( remote_group_req_to_group ), @@ -788,12 +992,12 @@ module cachepool_group_noc_wrapper .group_barrier_o ( group_barrier_o ), .barrier_done_i ( barrier_done_i ), .icache_events_o ( icache_events_o ), - .icache_prefetch_enable_i ( icache_prefetch_enable_i ), + .icache_prefetch_enable_i ( icache_prefetch_enable ), .dynamic_offset_i ( dynamic_offset_q ), .l1d_private_i ( l1d_private_q ), .l1d_insn_i ( l1d_insn_q ), .l1d_insn_valid_i ( l1d_insn_valid_q ), - .l1d_insn_ready_o ( l1d_insn_ready_d ), + .l1d_insn_ready_o ( l1d_insn_ready ), .l1d_busy_i ( l1d_busy_q ) ); diff --git a/hardware/src/cachepool_pkg.sv b/hardware/src/cachepool_pkg.sv index 79481eff..4e63a262 100644 --- a/hardware/src/cachepool_pkg.sv +++ b/hardware/src/cachepool_pkg.sv @@ -64,6 +64,8 @@ package cachepool_pkg; /////////////////// // How many cores for each tile? localparam int unsigned NumCoresTile = NumCC / NumTiles; + // Harts per tile (NumScalarPerCC harts share one CC) + localparam int unsigned NumHartsTile = NumCoresTile * NumScalarPerCC; // Intra-group remote ports per core (to other tiles in the same group). localparam int unsigned NumLGPortCore = `ifdef LG_PORT_PER_CORE `LG_PORT_PER_CORE `else 0 `endif; @@ -124,10 +126,25 @@ package cachepool_pkg; localparam int unsigned PeriStartAddr = 32'hC000_0000; localparam int unsigned BootAddr = 32'h1000; + // Bootrom word width, equal to the L2 refill width (checked in cachepool_cluster) + localparam int unsigned BootromDataWidth = `ifdef BOOTROM_DATA_WIDTH `BOOTROM_DATA_WIDTH `else 0 `endif; + // Bootrom address window + localparam int unsigned BootromSize = 32'h1_0000; // UART Configuration localparam int unsigned UartAddr = 32'hC001_0000; + // Group-level configuration registers: group g owns the block at GroupCfgBase + g * GroupCfgStride. + // Only the cluster peripheral writes them, over the L2 mesh; cores never access them. + localparam logic [31:0] GroupCfgBase = `ifdef GROUP_CFG_BASE 32'd `GROUP_CFG_BASE `else 32'hD000_0000 `endif; + localparam int unsigned GroupCfgStride = `ifdef GROUP_CFG_STRIDE `GROUP_CFG_STRIDE `else 32'h100 `endif; + + // Reset values shared by the cluster peripheral and the group-level configuration registers. + localparam int unsigned L1dXbarOffsetRst = `ifdef L1D_XBAR_OFFSET_RST `L1D_XBAR_OFFSET_RST `else 14 `endif; + localparam int unsigned L1dPrivateRst = `ifdef L1D_PRIVATE_RST `L1D_PRIVATE_RST `else 0 `endif; + localparam logic [31:0] L1dPrivateAddrRst = `ifdef L1D_PRIVATE_ADDR_RST 32'd `L1D_PRIVATE_ADDR_RST `else 32'hA000_0000 `endif; + localparam bit ICachePrefetchRst = `ifdef ICACHE_PREFETCH_RST `ICACHE_PREFETCH_RST `else 1'b1 `endif; + /////////////////////// // CACHEPOOL L1 CFG // /////////////////////// @@ -180,8 +197,8 @@ package cachepool_pkg; // Core id width within a tile => tile ID will be calculated separatly localparam int unsigned CoreIDWidth = idx_width(NumCoresTile); localparam int unsigned TileIDWidth = idx_width(NumTiles); - // Each bank inside a tile needs an unique id, plus two reserved for iCache and peripheral - localparam int unsigned BankIDWidth = idx_width(NumL1CtrlTile + 2); + // Each bank inside a tile needs an unique id, plus one reserved for the iCache + localparam int unsigned BankIDWidth = idx_width(NumL1CtrlTile + 1); localparam int unsigned RefillDataWidth = `ifdef REFILL_DATA_WIDTH `REFILL_DATA_WIDTH `else 0 `endif; localparam int unsigned RefillStrbWidth = RefillDataWidth / 8; @@ -213,12 +230,11 @@ package cachepool_pkg; // AXI ID field structure // ----------------------- // ClusterAxiIdWidth = [cluster_route_bits][tile_index_bits][tile_local_bits] - // Per-tile xbar ports: NumL1CtrlTile refill + 1 peripheral + 1 iCache - localparam int unsigned NumClusterMst = 2 + NumL1CtrlTile; + // Per-tile xbar ports: NumL1CtrlTile refill + 1 iCache (peripherals are on the narrow mesh) + localparam int unsigned NumClusterMst = 1 + NumL1CtrlTile; // Bank ID constants for the refill xbar response demux - // Layout per tile: [0..NumL1CtrlTile-1] = refill, NumL1CtrlTile = peripheral, NumL1CtrlTile+1 = iCache - localparam int unsigned BankIdPeriph = NumL1CtrlTile; - localparam int unsigned BankIdICache = NumL1CtrlTile + 1; + // Layout per tile: [0..NumL1CtrlTile-1] = refill, NumL1CtrlTile = iCache + localparam int unsigned BankIdICache = NumL1CtrlTile; localparam int unsigned ClusterRouteIdWidth = $clog2(NumClusterMst); @@ -394,19 +410,47 @@ package cachepool_pkg; } burst_req_t; // style-waive: COMMENT_BLOCK: instruction encoding table - // Cache flush/invalidation instruction issued by the peripheral flush controller. + // Cache flush/invalidation instruction issued by the group configuration register. // insn encoding: // 2'b00 : flush private banks only // 2'b01 : flush shared banks only // 2'b10 : flush all banks // 2'b11 : invalidate (init) all banks - // tile_sel: one-hot mask over NumTiles. For insn != 2'b00 the peripheral - // sets tile_sel to '1 (all tiles) for consistency. + // tile_sel: one-hot mask over the tiles of the group. For insn != 2'b00 the + // cluster peripheral sets tile_sel to '1 (all tiles) for consistency. typedef struct packed { - logic [1:0] insn; - logic [NumTiles-1:0] tile_sel; + logic [1:0] insn; + logic [NumTilesPerGroup-1:0] tile_sel; } cache_insn_t; + // Bit position of the address selecting the cache bank (cache xbar offset). + typedef logic [$clog2(SpatzAxiAddrWidth)-1:0] xbar_offset_t; + + // Committed L1D/xbar configuration, identical in every group. + typedef struct packed { + logic [L1PrivWidth-1:0] l1d_private; + axi_addr_t private_addr; + xbar_offset_t xbar_offset; + logic prefetch_en; + } l1d_cfg_t; + + localparam l1d_cfg_t L1dCfgRst = '{ + l1d_private: L1dPrivateRst, + private_addr: L1dPrivateAddrRst, + xbar_offset: L1dXbarOffsetRst, + prefetch_en: ICachePrefetchRst + }; + + // Registers of a group-level configuration register block, one 32-bit word each at + // GroupCfgBase + g * GroupCfgStride + 4 * index. Every write carries one register. + typedef enum logic [2:0] { + GroupCfgXbarOffset = 3'd0, + GroupCfgL1dPrivate = 3'd1, + GroupCfgL1dAddr = 3'd2, + GroupCfgPrefetch = 3'd3, + GroupCfgL1dInsn = 3'd4 + } group_cfg_reg_e; + typedef struct packed { logic for_write_pend; cache_ways_entry_ptr_t depth; @@ -517,6 +561,16 @@ package cachepool_pkg; } l2_noc_hdr_t; // l2_noc_req_t / l2_noc_rsp_t defined after REQRSP_TYPEDEF_ALL macros below. + // User field of the narrow (peripheral) L2 mesh: only what routes a response back to its hart. + // tile_id is the tile inside the source group, hart_id the hart inside that tile. + typedef struct packed { + floo_cachepool_noc_pkg::id_t l2_src_id; + logic [idx_width(NumTilesPerGroup)-1:0] tile_id; + logic [idx_width(NumHartsTile)-1:0] hart_id; + logic is_fpu; + reqid_t req_id; + } peri_narrow_user_t; + // Group ICache (L2 read-only cache control) // 2 rules: DRAM (cacheable) + BootROM (cacheable, avoids Bypass path) localparam int unsigned ROCacheNumAddrRules = 2; @@ -583,16 +637,33 @@ package cachepool_pkg; l2_noc_hdr_t hdr; } l2_noc_rsp_t; - // REQRSP: peripheral path (narrow 32b, tcdm_user_t) - `REQRSP_TYPEDEF_ALL (periph, axi_addr_t, narrow_data_t, narrow_strb_t, tcdm_user_t) + // REQRSP: narrow 32b peripheral path (core accesses to CSR/UART, group configuration writes) + `REQRSP_TYPEDEF_ALL (peri_narrow, axi_addr_t, narrow_data_t, narrow_strb_t, peri_narrow_user_t) + + // Narrow L2 mesh flit types. The request carries no AMO: atomics never target the peripherals. + typedef struct packed { + axi_addr_t addr; + logic write; + narrow_data_t data; + narrow_strb_t strb; + reqrsp_pkg::size_t size; + peri_narrow_user_t user; + } peri_noc_req_chan_t; + + typedef struct packed { + peri_noc_req_chan_t payload; + l2_noc_hdr_t hdr; + } peri_noc_req_t; - // REQRSP: narrow 32b with refill_user_t (DW converter output, peripheral path) - `REQRSP_TYPEDEF_ALL (peri_narrow, axi_addr_t, narrow_data_t, narrow_strb_t, refill_user_t) + typedef struct packed { + peri_narrow_rsp_chan_t payload; + l2_noc_hdr_t hdr; + } peri_noc_rsp_t; - // Peripheral xbar user: refill_user_t + 1-bit source ID (0=NoC, 1=TB) + // Peripheral xbar user: narrow user + 1-bit source ID (0=NoC, 1=TB) typedef struct packed { - logic src_id; - refill_user_t refill; + logic src_id; + peri_narrow_user_t narrow; } peri_xbar_user_t; // REQRSP: peripheral xbar path (narrow 32b, peri_xbar_user_t) @@ -732,4 +803,57 @@ package cachepool_pkg; end endfunction + // L2 mesh endpoint of the cluster peripheral: the West chimney at gy=0, next to group 0. Its + // channel ID follows the HBM placement (see gen_hbm_west in cachepool_cluster). + localparam int unsigned NumHbmAllSides = 2 * NumGroupsY + 2 * NumGroupsX; + localparam int unsigned HbmPerSide = NumHbmAllSides / 4; + localparam int unsigned L2PeriphChannel = (NumL2Channel == NumHbmAllSides) ? HbmPerSide : 0; + localparam int unsigned PeriEndpointId = NumGroups + L2PeriphChannel; + + // Narrow L2 mesh route config and SAM of the group side: every core access goes to the cluster + // peripheral endpoint, which decodes CSR/UART itself. + localparam int unsigned PeriCoreSamNumRules = 1; + typedef floo_cachepool_noc_pkg::sam_rule_t [PeriCoreSamNumRules-1:0] peri_core_sam_t; + + function automatic floo_pkg::route_cfg_t gen_peri_core_route_cfg(); + floo_pkg::route_cfg_t cfg; + cfg = floo_cachepool_noc_pkg::RouteCfg; + cfg.NumSamRules = PeriCoreSamNumRules; + return cfg; + endfunction + + localparam floo_pkg::route_cfg_t PeriCoreRouteCfg = gen_peri_core_route_cfg(); + localparam peri_core_sam_t PeriCoreSam = '{ + '{idx: floo_cachepool_noc_pkg::id_t'(PeriEndpointId), + start_addr: 32'h0000_0000, + end_addr: 32'hFFFF_FFFF} + }; + + // Narrow L2 mesh route config and SAM of the cluster peripheral's manager port. It only ever targets + // group configuration words, so its SAM holds just those rules; they stay out of the generated + // Sam so core requests can never be routed to a group configuration register. + localparam int unsigned PeriSamNumRules = NumGroups; + typedef floo_cachepool_noc_pkg::sam_rule_t [PeriSamNumRules-1:0] peri_sam_t; + + function automatic floo_pkg::route_cfg_t gen_peri_route_cfg(); + floo_pkg::route_cfg_t cfg; + cfg = floo_cachepool_noc_pkg::RouteCfg; + cfg.NumSamRules = PeriSamNumRules; + return cfg; + endfunction + + function automatic peri_sam_t gen_peri_sam(); + peri_sam_t sam; + for (int unsigned g = 0; g < NumGroups; g++) begin + // Group index g is row-major (gy * NumGroupsX + gx); L2 mesh endpoint IDs are column-major. + sam[g].idx = floo_cachepool_noc_pkg::id_t'((g % NumGroupsX) * NumGroupsY + g / NumGroupsX); + sam[g].start_addr = GroupCfgBase + g * GroupCfgStride; + sam[g].end_addr = GroupCfgBase + (g + 1) * GroupCfgStride; + end + return sam; + endfunction + + localparam floo_pkg::route_cfg_t PeriRouteCfg = gen_peri_route_cfg(); + localparam peri_sam_t PeriSam = gen_peri_sam(); + endpackage : cachepool_pkg diff --git a/hardware/src/cachepool_tile.sv b/hardware/src/cachepool_tile.sv index e8ee8253..a673148e 100644 --- a/hardware/src/cachepool_tile.sv +++ b/hardware/src/cachepool_tile.sv @@ -152,9 +152,9 @@ module cachepool_tile /// Cache Refill ports output cache_trans_req_t [NumL1CtrlTile-1:0] cache_refill_req_o, input cache_trans_rsp_t [NumL1CtrlTile-1:0] cache_refill_rsp_i, - /// Peripheral REQRSP port (narrow, 32b, bypasses wide AXI xbar) - output periph_req_t periph_req_o, - input periph_rsp_t periph_rsp_i, + /// Peripheral REQRSP port (narrow, 32b, to the narrow L2 mesh) + output peri_narrow_req_t periph_req_o, + input peri_narrow_rsp_t periph_rsp_i, /// Wide AXI port: iCache L2 refill only (BootROM at cluster level) output axi_out_req_t [TileWideAxiPorts-1:0] axi_wide_req_o, input axi_out_resp_t [TileWideAxiPorts-1:0] axi_wide_rsp_i, @@ -972,12 +972,12 @@ module cachepool_tile // l1d_insn_ready_o pulses as cache_flush_d hits zero (no loop: it only feeds a peripheral reg). // Determine whether this tile should act on the incoming instruction. - // Private flush (insn==00): only if our tile_id bit is set in tile_sel. + // Private flush (insn==00): only if our bit within the group is set in tile_sel. // All other modes: always act. logic tile_insn_active; always_comb begin : gen_tile_active if (l1d_insn_i.insn == 2'b00) - tile_insn_active = l1d_insn_i.tile_sel[tile_id_i]; + tile_insn_active = l1d_insn_i.tile_sel[TileIdxInGroup]; else tile_insn_active = 1'b1; end @@ -1688,30 +1688,76 @@ module cachepool_tile .cluster_periph_start_address_i (cluster_periph_start_address) ); - reqrsp_req_t core_to_periph_req; - reqrsp_rsp_t core_to_periph_rsp; - - reqrsp_mux #( - .NrPorts (NrHarts ), - .AddrWidth (AxiAddrWidth ), - .DataWidth (NarrowDataWidth ), - .UserWidth ($bits(tcdm_user_t)), - .req_t (reqrsp_req_t ), - .rsp_t (reqrsp_rsp_t ), - .RespDepth (2 ) - ) i_reqrsp_mux_core ( - .clk_i (clk_i ), - .rst_ni (rst_ni ), - .slv_req_i (filtered_core_req ), - .slv_rsp_o (filtered_core_rsp ), - .mst_req_o (core_to_periph_req ), - .mst_rsp_i (core_to_periph_rsp ), - .idx_o (/*unused*/ ) + // Peripheral accesses of all harts merge into one narrow port. Every request carries its hart + // index so the response is routed back by user.hart_id; the harts accept out-of-order + // responses by user.req_id, so no ordering is kept here. + typedef logic [cf_math_pkg::idx_width(NrHarts)-1:0] periph_hart_sel_t; + + peri_narrow_req_chan_t [NrHarts-1:0] periph_hart_req; + logic [NrHarts-1:0] periph_hart_req_valid, periph_hart_req_ready; + peri_narrow_rsp_chan_t [NrHarts-1:0] periph_hart_rsp; + logic [NrHarts-1:0] periph_hart_rsp_valid, periph_hart_rsp_ready; + periph_hart_sel_t periph_rsp_sel; + + for (genvar i = 0; i < NrHarts; i++) begin : gen_periph_hart + always_comb begin + periph_hart_req[i] = '0; + periph_hart_req[i].addr = filtered_core_req[i].q.addr; + periph_hart_req[i].write = filtered_core_req[i].q.write; + periph_hart_req[i].amo = filtered_core_req[i].q.amo; + periph_hart_req[i].data = filtered_core_req[i].q.data; + periph_hart_req[i].strb = filtered_core_req[i].q.strb; + periph_hart_req[i].size = filtered_core_req[i].q.size; + periph_hart_req[i].user.hart_id = i; + periph_hart_req[i].user.is_fpu = filtered_core_req[i].q.user.is_fpu; + periph_hart_req[i].user.req_id = filtered_core_req[i].q.user.req_id; + + filtered_core_rsp[i] = '0; + filtered_core_rsp[i].q_ready = periph_hart_req_ready[i]; + filtered_core_rsp[i].p.data = periph_hart_rsp[i].data; + filtered_core_rsp[i].p.error = periph_hart_rsp[i].error; + filtered_core_rsp[i].p.write = periph_hart_rsp[i].write; + filtered_core_rsp[i].p.user.is_fpu = periph_hart_rsp[i].user.is_fpu; + filtered_core_rsp[i].p.user.req_id = periph_hart_rsp[i].user.req_id; + filtered_core_rsp[i].p_valid = periph_hart_rsp_valid[i]; + end + assign periph_hart_req_valid[i] = filtered_core_req[i].q_valid; + assign periph_hart_rsp_ready[i] = filtered_core_req[i].p_ready; + end + + assign periph_rsp_sel = periph_rsp_i.p.user.hart_id; + + reqrsp_xbar #( + .NumInp ( NrHarts ), + .NumOut ( 1 ), + .PipeReg ( 1'b0 ), + .tcdm_req_chan_t ( peri_narrow_req_chan_t ), + .tcdm_rsp_chan_t ( peri_narrow_rsp_chan_t ), + .mst_sel_t ( periph_hart_sel_t ) + ) i_periph_xbar ( + .clk_i ( clk_i ), + .rst_ni ( rst_ni ), + .slv_req_i ( periph_hart_req ), + .slv_rr_i ( '0 ), + .slv_req_valid_i ( periph_hart_req_valid ), + .slv_req_ready_o ( periph_hart_req_ready ), + .slv_rsp_o ( periph_hart_rsp ), + .slv_rsp_valid_o ( periph_hart_rsp_valid ), + .slv_rsp_ready_i ( periph_hart_rsp_ready ), + .slv_sel_i ( '0 ), + .slv_selected_o ( /* unused */ ), + .mst_req_o ( periph_req_o.q ), + .mst_req_valid_o ( periph_req_o.q_valid ), + .mst_req_ready_i ( periph_rsp_i.q_ready ), + .mst_rsp_i ( periph_rsp_i.p ), + .mst_rr_i ( '0 ), + .mst_rsp_valid_i ( periph_rsp_i.p_valid ), + .mst_rsp_ready_o ( periph_req_o.p_ready ), + .mst_sel_i ( periph_rsp_sel ) ); - // Peripheral REQRSP: directly exposed to group level (no AXI conversion) - assign periph_req_o = core_to_periph_req; - assign core_to_periph_rsp = periph_rsp_i; + `ASSERT_INIT(CheckPeriphHartWidth, NrHarts == NumHartsTile, + "cachepool_tile: hart count does not match cachepool_pkg::NumHartsTile") // ------------- // Sanity Checks diff --git a/hardware/src/reqrsp_to_reg.sv b/hardware/src/reqrsp_to_reg.sv index 0cafe260..c71eba26 100644 --- a/hardware/src/reqrsp_to_reg.sv +++ b/hardware/src/reqrsp_to_reg.sv @@ -4,8 +4,9 @@ // // Based on periph_to_reg from register_interface. // Converts REQRSP protocol to register interface with 1-cycle response latency. -// Assumes in-order, constant-latency target (e.g., BootROM). -// A FIFO preserves the REQRSP user field from request to response. +// Assumes the target answers in the cycle it raises ready (e.g., regtool CSRs, BootROM). +// A request is only issued when the response register is free or drained in the same +// cycle, so a response is held until its handshake and carries its own user field. `include "common_cells/registers.svh" `include "common_cells/assertions.svh" @@ -14,8 +15,6 @@ module reqrsp_to_reg #( parameter int unsigned AddrWidth = 32, parameter int unsigned DataWidth = 32, parameter int unsigned UserWidth = 1, - // FIFO depth for buffering user field (should match max outstanding) - parameter int unsigned FifoDepth = 2, // Shift response data right by the request byte offset so the addressed bytes land at // [31:0], for regbus targets wider than the requester (e.g. 512b BootROM, 32b loads). parameter bit ShiftResponse = 1'b0, @@ -35,31 +34,32 @@ module reqrsp_to_reg #( localparam int unsigned ByteOffset = $clog2(DataWidth / 8); - // Request grant: accepted when both sides are valid/ready - logic gnt; - assign gnt = reqrsp_req_i.q_valid & reg_rsp_i.ready; + // Response register state + logic rsp_valid_q; + logic [DataWidth-1:0] rsp_rdata_d, rsp_rdata_q; + logic rsp_error_q; + logic rsp_write_q; + user_t rsp_user_q; - // Forward request to regbus (combinational) + // The response register can take a new response this cycle + logic rsp_free; + assign rsp_free = ~rsp_valid_q | reqrsp_req_i.p_ready; + + // Request grant: target ready and room for the response + logic req_valid, gnt; + assign req_valid = reqrsp_req_i.q_valid & rsp_free; + assign gnt = req_valid & reg_rsp_i.ready; + + // Forward request to regbus. valid is gated as well, so a write cannot take effect + // while its grant is held back. assign reg_req_o.addr = reqrsp_req_i.q.addr; assign reg_req_o.write = reqrsp_req_i.q.write; assign reg_req_o.wdata = reqrsp_req_i.q.data; assign reg_req_o.wstrb = reqrsp_req_i.q.strb; - assign reg_req_o.valid = reqrsp_req_i.q_valid; + assign reg_req_o.valid = req_valid; - // REQRSP q_ready: grant the request assign reqrsp_rsp_o.q_ready = gnt; - // Register response (1-cycle latency, same as periph_to_reg) - logic rsp_valid_q; - logic [DataWidth-1:0] rsp_rdata_d, rsp_rdata_q; - logic rsp_error_q; - logic rsp_write_q; - - `FF(rsp_valid_q, gnt, 1'b0) - `FF(rsp_rdata_q, rsp_rdata_d, '0) - `FF(rsp_error_q, reg_rsp_i.error, 1'b0) - `FF(rsp_write_q, reqrsp_req_i.q.write, 1'b0) - // Shift rdata by the combinational request address. Safe for constant-latency targets: // q.addr is held until q_ready (= gnt), which fires in the cycle rdata becomes valid. if (ShiftResponse) begin : gen_shift @@ -68,44 +68,27 @@ module reqrsp_to_reg #( assign rsp_rdata_d = reg_rsp_i.rdata; end - // User FIFO: push on grant, pop when response is consumed - user_t user_fifo_out; - logic user_fifo_full, user_fifo_empty; - logic user_fifo_push, user_fifo_pop; - - assign user_fifo_push = gnt; - assign user_fifo_pop = rsp_valid_q & reqrsp_req_i.p_ready; - - fifo_v3 #( - .FALL_THROUGH ( 1'b0 ), - .DEPTH ( FifoDepth ), - .dtype ( user_t ) - ) i_user_fifo ( - .clk_i, - .rst_ni, - .flush_i ( 1'b0 ), - .testmode_i ( 1'b0 ), - .full_o ( user_fifo_full ), - .empty_o ( user_fifo_empty ), - .usage_o ( /* unused */ ), - .data_i ( reqrsp_req_i.q.user ), - .push_i ( user_fifo_push ), - .data_o ( user_fifo_out ), - .pop_i ( user_fifo_pop ) - ); + // Set on grant, cleared on handshake + `FF(rsp_valid_q, gnt | (rsp_valid_q & ~reqrsp_req_i.p_ready), 1'b0) + `FFL(rsp_rdata_q, rsp_rdata_d, gnt, '0) + `FFL(rsp_error_q, reg_rsp_i.error, gnt, 1'b0) + `FFL(rsp_write_q, reqrsp_req_i.q.write, gnt, 1'b0) + `FFL(rsp_user_q, reqrsp_req_i.q.user, gnt, '0) // REQRSP response channel assign reqrsp_rsp_o.p.data = rsp_rdata_q; assign reqrsp_rsp_o.p.error = rsp_error_q; assign reqrsp_rsp_o.p.write = rsp_write_q; - assign reqrsp_rsp_o.p.user = user_fifo_out; + assign reqrsp_rsp_o.p.user = rsp_user_q; assign reqrsp_rsp_o.p_valid = rsp_valid_q; - // Target is read-only when ShiftResponse is enabled (e.g., BootROM) `ifndef TARGET_SYNTHESIS + // Target is read-only when ShiftResponse is enabled (e.g., BootROM) if (ShiftResponse) begin : gen_no_write_assert `ASSERT(NoWrite, !(reg_req_o.valid && reg_req_o.write), clk_i, rst_ni) end + `ASSERT(RspStable, reqrsp_rsp_o.p_valid && !reqrsp_req_i.p_ready |=> + reqrsp_rsp_o.p_valid && $stable(reqrsp_rsp_o.p), clk_i, rst_ni) `endif endmodule diff --git a/software/CMakeLists.txt b/software/CMakeLists.txt index 64b8ed7e..91bc2691 100644 --- a/software/CMakeLists.txt +++ b/software/CMakeLists.txt @@ -22,6 +22,17 @@ enable_testing() set(NUM_SCALAR_PER_CORE "1" CACHE STRING "Snitch harts sharing one Spatz per Core Complex") add_compile_definitions(SNRT_NUM_SCALAR_PER_CORE=${NUM_SCALAR_PER_CORE}) +# Memory system shape from config.mk, for kernels that lay data out for it +set(L2_CHANNEL "4" CACHE STRING "Number of HBM channels") +set(L2_INTERLEAVE "16" CACHE STRING "HBM channel interleave, in L2 bank words") +set(L2_BANK_WIDTH "512" CACHE STRING "L2 bank word width in bits") +set(L1D_CACHELINE_WIDTH "512" CACHE STRING "L1D cacheline width in bits") +add_compile_definitions( + SNRT_L2_CHANNEL=${L2_CHANNEL} + SNRT_L2_INTERLEAVE=${L2_INTERLEAVE} + SNRT_L2_BANK_WIDTH=${L2_BANK_WIDTH} + SNRT_L1D_CACHELINE_WIDTH=${L1D_CACHELINE_WIDTH}) + add_subdirectory(${SNITCH_SOFTWARE_DIR}/snRuntime snRuntime) # add_subdirectory(${SPATZ_SOFTWARE_DIR}/riscvTests riscvTests) # add_subdirectory(${SPATZ_SOFTWARE_DIR}/spatzBenchmarks spatzBenchmarks) @@ -38,6 +49,7 @@ if (ENABLE_CACHEPOOL_TESTS) add_subdirectory(${CACHEPOOL_DIR}/software/sync sync) add_subdirectory(${CACHEPOOL_DIR}/software/kernels/fp kernels_fp) add_subdirectory(${CACHEPOOL_DIR}/software/kernels/int kernels_int) + add_subdirectory(${CACHEPOOL_DIR}/software/control control) add_subdirectory(${CACHEPOOL_DIR}/software/rlc rlc) add_subdirectory(${CACHEPOOL_DIR}/software/tests CachePoolTests) endif() diff --git a/software/README.md b/software/README.md index 589e53d8..0a929d36 100644 --- a/software/README.md +++ b/software/README.md @@ -30,7 +30,10 @@ make sw config= # build all registered tests ``` Binaries land in `software/build/CachePoolTests/`, named -`test-`. Run one against the RTL simulator with: +`test-`, except the RLC and control-algorithm groups, named +`rlc-` and `ctrl-` (each group sets +`SNITCH_TEST_BIN_PREFIX` in its `CMakeLists.txt`). Run one against the RTL +simulator with: ``` ./sim/bin/cachepool_cluster.vsim software/build/CachePoolTests/test- diff --git a/software/cache/CMakeLists.txt b/software/cache/CMakeLists.txt index 4c3c6282..aa97700e 100644 --- a/software/cache/CMakeLists.txt +++ b/software/cache/CMakeLists.txt @@ -12,3 +12,15 @@ add_spatz_test_zeroParam(cache-rlc-mimic cache-rlc-mimic/main.c) add_spatz_test_zeroParam(cache-vector-rw cache-vector-rw/main.c) add_spatz_test_zeroParam(cache-coverage cache-coverage/main.c) add_spatz_test_zeroParam(cache-coverage-min cache-coverage-min/main.c) + +# Private-partition flush: cache-fold-flush__x_ +foreach(v "tile;0;6;512" "tile;0;0;512" "tile;0;6;128" "group;1;6;512" "shared;2;6;512") + list(GET v 0 fname) + list(GET v 1 fold) + list(GET v 2 xbar) + list(GET v 3 kib) + set(ff_name cache-fold-flush_${fname}_x${xbar}_${kib}) + add_spatz_test_zeroParam(${ff_name} cache-fold-flush/main.c) + target_compile_definitions(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}${ff_name} PUBLIC + FF_FOLD=${fold} FF_XBAR=${xbar} FF_KIB=${kib}) +endforeach() diff --git a/software/cache/cache-fold-flush/main.c b/software/cache/cache-fold-flush/main.c new file mode 100644 index 00000000..b7dde78e --- /dev/null +++ b/software/cache/cache-fold-flush/main.c @@ -0,0 +1,98 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// SPDX-License-Identifier: Apache-2.0 + +// Flush of a private (folded) L1 partition. Core 0 writes FF_KIB KiB of a +// pattern with vector stores, all cores flush, then every core checks a +// slice from its own tile. With FF_KIB above the tile capacity, dirty lines +// are evicted during the writes as well as by the flush. +// +// FF_FOLD: 0 tile, 1 group, 2 shared (no private partition) +// FF_XBAR: l1d_xbar_config offset (0 keeps the default) + +#include +#include +#include +#include "printf.h" + +#ifndef FF_KIB +#define FF_KIB 512 +#endif +#ifndef FF_FOLD +#define FF_FOLD 0 +#endif +#ifndef FF_XBAR +#define FF_XBAR 6 +#endif + +#define FF_WORDS (FF_KIB * 1024 / 4) +#define FF_VL 64 +#define FF_MAGIC 0x5a000000u +#define MAX_CORES 1024 + +static uint32_t buf[FF_WORDS] __attribute__((section(".dram"), aligned(64))); +static uint32_t ramp[FF_VL] __attribute__((section(".data"), aligned(64))); +static uint32_t errors[MAX_CORES] __attribute__((section(".data"))); +static uint32_t first_bad[MAX_CORES] __attribute__((section(".data"))); +static uint32_t first_val[MAX_CORES] __attribute__((section(".data"))); + +int main() { + const uint32_t cid = snrt_cluster_vpu_idx(); + const uint32_t ncores = snrt_cluster_vpu_num(); + + if (cid == 0) + for (uint32_t i = 0; i < FF_VL; ++i) + ramp[i] = i; + snrt_cluster_hw_barrier(); + if (FF_XBAR) + l1d_xbar_config(FF_XBAR); + if (FF_FOLD != 2) + l1d_part_folded(snrt_cluster_vpu_per_tile(), + FF_FOLD ? L1D_FOLD_GROUP : L1D_FOLD_TILE); + + if (cid == 0) { + uint32_t vl; + asm volatile("vsetvli %0, %1, e32, m4, ta, ma" : "=r"(vl) : "r"(FF_VL)); + asm volatile("vle32.v v0, (%0)" ::"r"(ramp)); + for (uint32_t p = 0; p < FF_WORDS; p += FF_VL) { + asm volatile("vadd.vx v4, v0, %0" ::"r"(FF_MAGIC + p)); + asm volatile("vse32.v v4, (%0)" ::"r"(buf + p) : "memory"); + } + snrt_fence_spatz(); + printf("cache-fold-flush: %u KiB, fold %d, xbar %d, written\n", FF_KIB, + FF_FOLD, FF_XBAR); + } + + l1d_cluster_flush(); + + const uint32_t i0 = FF_WORDS / ncores * cid; + const uint32_t i1 = FF_WORDS / ncores * (cid + 1); + uint32_t err = 0, bad = 0, val = 0; + for (uint32_t i = i0; i < i1; ++i) { + uint32_t got = ((volatile uint32_t *)buf)[i]; + if (got != FF_MAGIC + i) { + if (err == 0) { + bad = i; + val = got; + } + ++err; + } + } + errors[cid] = err; + first_bad[cid] = bad; + first_val[cid] = val; + // errors[] lives in each core's private partition until flushed + l1d_cluster_flush(); + + uint32_t total = 0; + if (cid == 0) { + for (uint32_t c = 0; c < ncores; ++c) { + if (errors[c]) + printf("core %u: %u errors, first at 0x%08x = 0x%08x\n", c, errors[c], + (uint32_t)(uintptr_t)(buf + first_bad[c]), first_val[c]); + total += errors[c]; + } + printf(total ? "[FAIL] %u errors\n" : "[PASS]\n", total); + } + snrt_cluster_hw_barrier(); + return total ? -1 : 0; +} diff --git a/software/cmake/CachePoolTests.cmake b/software/cmake/CachePoolTests.cmake index fef08168..5acd8b07 100644 --- a/software/cmake/CachePoolTests.cmake +++ b/software/cmake/CachePoolTests.cmake @@ -32,26 +32,26 @@ set(SNITCH_TEST_PREFIX "") macro(add_spatz_test_zeroParam name file) set(target_name ${name}) add_snitch_test(${target_name} ${file}) - target_link_libraries(test-${SNITCH_TEST_PREFIX}${target_name} benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) + target_link_libraries(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}${target_name} benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) endmacro() macro(add_spatz_test_oneParam name file param1) set(target_name ${name}_M${param1}) add_snitch_test(${target_name} ${file}) - target_link_libraries(test-${SNITCH_TEST_PREFIX}${target_name} benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) - target_compile_definitions(test-${SNITCH_TEST_PREFIX}${target_name} PUBLIC DATAHEADER="data/data_${param1}.h") + target_link_libraries(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}${target_name} benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) + target_compile_definitions(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}${target_name} PUBLIC DATAHEADER="data/data_${param1}.h") endmacro() macro(add_spatz_test_twoParam name file param1 param2) set(target_name ${name}_M${param1}_N${param2}) add_snitch_test(${target_name} ${file}) - target_link_libraries(test-${SNITCH_TEST_PREFIX}${target_name} benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) - target_compile_definitions(test-${SNITCH_TEST_PREFIX}${target_name} PUBLIC DATAHEADER="data/data_${param1}_${param2}.h") + target_link_libraries(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}${target_name} benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) + target_compile_definitions(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}${target_name} PUBLIC DATAHEADER="data/data_${param1}_${param2}.h") endmacro() macro(add_spatz_test_threeParam name file param1 param2 param3) set(target_name ${name}_M${param1}_N${param2}_K${param3}) add_snitch_test(${target_name} ${file}) - target_link_libraries(test-${SNITCH_TEST_PREFIX}${target_name} benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) - target_compile_definitions(test-${SNITCH_TEST_PREFIX}${target_name} PUBLIC DATAHEADER="data/data_${param1}_${param2}_${param3}.h") + target_link_libraries(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}${target_name} benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) + target_compile_definitions(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}${target_name} PUBLIC DATAHEADER="data/data_${param1}_${param2}_${param3}.h") endmacro() diff --git a/software/cmake/SnitchUtilities.cmake b/software/cmake/SnitchUtilities.cmake index ec96cf41..a8d25d3e 100644 --- a/software/cmake/SnitchUtilities.cmake +++ b/software/cmake/SnitchUtilities.cmake @@ -10,6 +10,9 @@ set(LLVM_PATH "/home/spatz" CACH PATH "Path to the LLVM RISCV installation") set(GCC_PATH "/home/spatz" CACHE PATH "Path to the GCC RISCV installation") set(RUNTIME_TRACE OFF CACHE BOOL "Enable runtime trace output") set(SNITCH_TEST_PREFIX "") +# Leading part of every test binary name; a test group may replace it (e.g. +# rlc-, ctrl-) in its own directory scope +set(SNITCH_TEST_BIN_PREFIX "test-") if (SNITCH_SIMULATOR) message(STATUS "Using RTL simulator: ${SNITCH_SIMULATOR}") endif() @@ -48,7 +51,7 @@ endmacro() macro(add_snitch_test_executable name) if (BUILD_TESTS) - add_snitch_executable(test-${SNITCH_TEST_PREFIX}${name} ${ARGN}) + add_snitch_executable(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}${name} ${ARGN}) endif() endmacro() @@ -62,7 +65,7 @@ macro(add_snitch_raw_test_rtl test_name target_name) endmacro() macro(add_snitch_test_rtl name) - add_snitch_raw_test_rtl(${SNITCH_TEST_PREFIX}rtl-${name} test-${SNITCH_TEST_PREFIX}${name}) + add_snitch_raw_test_rtl(${SNITCH_TEST_PREFIX}rtl-${name} ${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}${name}) endmacro() macro(add_snitch_test name) diff --git a/software/control/CMakeLists.txt b/software/control/CMakeLists.txt new file mode 100644 index 00000000..3c6b7c2f --- /dev/null +++ b/software/control/CMakeLists.txt @@ -0,0 +1,84 @@ +# Copyright 2025 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + +# Control algorithm kernels (see README.md). Each kernel lives in its +# own subdirectory with main.c, kernel/, script/ and data/; kernel sources are +# pulled into each test through main.c's #include, as in software/rlc. + +# Binaries of this group are named ctrl- instead of test-. +set(SNITCH_TEST_BIN_PREFIX "ctrl-") + +# ctrl-__C[_]: /main.c built with +# data/data_.h and ACTIVE_CORES = . Optional extra arguments: +# a variant suffix, then compile definitions for it. +macro(add_control_test name dir data cores) + set(target_name ${name}_${data}_C${cores}) + set(extra_defs ${ARGN}) + if(extra_defs) + list(GET extra_defs 0 variant) + list(REMOVE_AT extra_defs 0) + set(target_name ${target_name}_${variant}) + endif() + add_snitch_test(${target_name} ${dir}/main.c) + target_link_libraries(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}${target_name} benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) + target_compile_definitions(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}${target_name} PUBLIC + DATAHEADER="data/data_${data}.h" ACTIVE_CORES=${cores} ${extra_defs}) + target_include_directories(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}${target_name} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/common) +endmacro() + +# Kernel 1: initial U/R and V/G products +add_control_test(uv-init uv-init 8_16_const 1) +add_control_test(uv-init uv-init 8_16_const 4) +add_control_test(uv-init uv-init 9_21 1) +add_control_test(uv-init uv-init 9_21 4) +add_control_test(uv-init uv-init 9_21 8) +foreach(cores 1 2 4 8 16) + add_control_test(uv-init uv-init 64_128 ${cores}) +endforeach() +# L1 variants (all-private): tile-folded banks, columns split across tiles +# above four cores; group-folded banks; bank-select offsets 6 and 7 +foreach(xbar 6 7) + foreach(cores 1 4 8 16) + add_control_test(uv-init uv-init 64_128 ${cores} tile_x${xbar} + L1D_FOLD=L1D_FOLD_TILE XBAR_OFFSET=${xbar}) + endforeach() + foreach(cores 8 16) + add_control_test(uv-init uv-init 64_128 ${cores} group_x${xbar} + L1D_FOLD=L1D_FOLD_GROUP XBAR_OFFSET=${xbar}) + endforeach() +endforeach() + +# Gain = ln(1 + P * G) and the selection of the winner +add_control_test(gain-select gain-select 63_37 1) +add_control_test(gain-select gain-select 63_37 4) +foreach(cores 1 2 4 8 16) + add_control_test(gain-select gain-select 64_128 ${cores}) +endforeach() + +# The full algorithm (products, every iteration, finalization), one group +add_control_test(full full 9_21 1) +add_control_test(full full 9_21 4) +add_control_test(full full 63_37 4) +add_control_test(full full 63_37_mix 4) +foreach(cores 8 16) + add_control_test(full full 64_128 ${cores}) +endforeach() +# Shorter runs: a warm-up of 2 iterations, then 8, 16 or 32 measured +foreach(iters 8 16 32) + foreach(cores 8 16) + add_control_test(full full 64_128_i${iters}w ${cores}) + endforeach() +endforeach() +# Per-core phase profile of the iterations +foreach(cores 8 16) + add_control_test(full full 64_128_i16w ${cores} prof IT_PROF=1) +endforeach() +# Cycles of the parts of the selection +add_control_test(full full 64_128_i16w 16 cprof IT_CPROF=1) + +# Power update of the winner and finalization (one core) +add_control_test(power-update power-update 9_21 1) +add_control_test(power-update power-update 9_24 1) +add_control_test(power-update power-update 64_128 1) diff --git a/software/control/README.md b/software/control/README.md new file mode 100644 index 00000000..9e133b15 --- /dev/null +++ b/software/control/README.md @@ -0,0 +1,266 @@ +# Control algorithm + +Greedy subcarrier and power allocation for an uplink OFDMA system, based on +K. Kim et al., "Iterative and greedy resource allocation in an uplink OFDMA +system", PIMRC 2004. The benchmark extends the paper's algorithm with +synthetic matrix stages (R and G below) that make the gains of every user +change with each allocation. The algorithm runs once per 0.5 ms TTI, which is +500 000 cycles at 1 GHz; the target is to fit that budget with as few cores as +possible. + +## Inputs and outputs + +| Name | Shape | Range | +| --- | --- | --- | +| M | scalar | [1, 80] users | +| N | scalar | [1, 100] subcarriers | +| R1 | N x M | [-50, 50] | +| R2 | M x N | 0 or 1 | +| A | M x N | [-3000, 3000] | +| C | N x N | [-3000, 3000] | +| D | N | [-3000, 3000] | + +All values are integers. The outputs are the subcarrier allocation +`ResAllocMap[N]` and `UserResAllocMap[M][N]` (0 or 1) and the power allocation +`P[M][N]`. + +## Algorithm + +Everything is 32-bit integer arithmetic. Power values (P and the per-user +water level Delta) are fixed point, scaled by 100, and every division +truncates. + +``` +P = 100, Delta = 100, ResAllocMap = 0, UserResAllocMap = 0 +for j in 0 .. N-1, while some subcarrier is free: + R = clamp(R1 * R2 + C, MinR, MaxR) N x N + G = clamp((A * R + D) >> 7, MinG, MaxG) M x N + S = P .* G, Gain = ln(1 + S) + (w, k*) = the user and free subcarrier with the largest Gain + allocate k* to w and update P[w][:] (power update) + R2[w][:] = (R2[w][:] + j) % 2 +finalize: recompute P of every allocated (user, subcarrier) +``` + +| Parameter | Value | +| --- | --- | +| MinR, MaxR | -4096, 4096 | +| MinG, MaxG | 1, 2^20 | + +Ties go to the lowest subcarrier within a user, then to the lowest user. + +### Gain + +Gain is `ln(1 + S)` in Q16, computed with 32-bit integer operations: + +``` +1 + S = 2^e * m, m in [1, 2) exponent and mantissa bits of float(1 + S) +u = (m - 1.5) * 2/3 |u| <= 1/3 +ln m = ln 1.5 + u - u^2/2 + u^3/3 (degree-3 Taylor series, Horner in Q30) +Gain = e * ln 2 + ln m (summed in Q24, rounded to Q16) +``` + +The int-to-float conversion is the only floating-point operation; it finds +the exponent, as Spatz has no count-leading-zeros instruction. The error is +below 4.3e-3 against the exact ln(1 + S). + +With these ranges and N <= 128, every intermediate fits in int32: +`|R1 * R2 + C| <= 7000`, `|A * R + D| <= 1.58e9` and `S <= 1.05e8`. The shift +by 7 keeps G well inside [MinG, MaxG]. + +## Benchmark size + +The kernels are benchmarked at M = 64 users and N = 128 subcarriers. N is +above the range's 100, so this is more work than any allowed input (the +initial products are 2.1 M multiply-accumulates against 1.6 M at M = 80, +N = 100). The power-of-two sizes split into whole four-row blocks over 1, 2, +4, 8 and 16 cores, so every core gets the same work. + +### Power update + +Each user has a total power of 1 (100 scaled). With `inv(g) = 100 / g`, `n` +the number of subcarriers of user w after the allocation and `g` its row of +G: + +``` +sum = 100 + sum of inv(g[k]) over the subcarriers k of w +Delta[w] = sum / n +P[w][k*] = max(0, Delta[w] - inv(g[k*])) +for every free subcarrier k: + P[w][k] = max(0, (sum + inv(g[k])) / (n + 1) - inv(g[k])) +``` + +Finalization applies the first two lines to every user with at least one +subcarrier, using the final G. Users with no subcarriers keep their P row. + +On the paper's example (two users, three subcarriers, gains 8 4 2 and 4 2 1) +this gives P = 69, 31 and 100 and Delta = 81 and 150, against the exact +68.75, 31.25, 100, 81.25 and 150. + +## Restructured algorithm + +R and G do not depend on the user. `(x + j) % 2` keeps R2 for even j and +flips row w for odd j, so R2 changes in one row only after odd iterations, by +`s = +1` or `-1` per column. The kernels therefore keep the unclamped products +and update them instead of recomputing: + +``` +U = R1 * R2 + C R = clamp(U) +V = A * R + D G = clamp(V >> 7) +after an odd iteration with winner w: + U += R1[:, w] * s^T + V += (A * R1[:, w]) * s^T, in the columns where no R clamp is involved + V[:, k] = A * R[:, k] + D[k], in the other columns +``` + +For M = 64 and N = 128 this is 2.1 M multiply-accumulates once, then about +33 k operations per odd iteration, instead of 2.1 M per iteration. + +## Kernels + +| Kernel | Directory | Work | +| --- | --- | --- | +| Full algorithm | `full` | the whole algorithm on up to 16 cores of one group, one barrier per iteration (the benchmark) | +| Products | `uv-init` | U, R, V and G from the inputs (two int32 GEMMs with clamps), alone | +| Gain and selection | `gain-select` | Gain = ln(1 + P .* G) over all of M x N and the argmax over free subcarriers, alone | +| Power | `power-update` | power update of one winner and finalization, alone | + +`full` uses the products of `uv-init`, the integer log of `gain-select` and +the power update tables of `power-update`; the stage kernels measure those +parts on their own. Cycle counts and utilization are in `RESULTS.md`, the +best results in `RESULTS_SUMMARY.md`. + +In `gain-select`, the selection uses neither masks nor reductions: each Gain +is packed with its user into `key = Gain << 7 | (127 - i)`, an element-wise +`vmax` over the rows keeps the best key of every column, and a scalar scan +over the columns skips the allocated subcarriers. `full` evaluates the log +only where needed (lazy gain, below). + +## Full kernel + +`full` runs the whole algorithm on up to 16 cores of one group. Core c owns a +block of U rows (subcarriers, `[u0, u1)`) and a block of V, P and G rows +(users, `[v0, v1)`). All the data stays in the L1 of the group, so nothing is +flushed. + +### Lazy gain + +Only the selection uses Gain, and `ln(1 + S)` as computed above is +non-decreasing in S (checked for every S below 2^27, which covers all allowed +inputs). Its widest step is below 1.6e-5 * S, so every element whose Gain +equals the largest Gain has + +``` +S >= Smax - (Smax >> 14) - 2, Smax = largest S over the free subcarriers +``` + +Each core therefore keeps only `colS[k]`, the maximum of S over its rows in +column k, and evaluates the log exactly only in the candidate columns, whose +colS reaches that threshold. The winner is the same as with the Gain of every +element, ties included. + +### One iteration + +``` +every core: read the per-core keys of iteration j - 1, pick the winner + (w, k*), mark k* in its own tables +if j - 1 was odd (pass iteration): + owner of w: power update of row w (or copy of its speculative row) + every core: update pass on its rows: V += AR1[:, w] * s^T and colS of its + rows (U is updated one iteration later, see below) +else: + helper: power update of row w and the exact key of row w + owner of w: colS of its rows without row w + the others: late U update of the last pass +every core: candidates and exact keys of its rows (or its key of iteration + j - 1 if its best column was not k*), stored in its own slot; + before a pass iteration, the speculative power update +one barrier +if a U element crossed the margin (rare): recompute the affected V columns, + colS and the selection +``` + +`AR1 = A * R1` is computed with the products, so the V update is a rank-1 +update too. The margin is `MaxR - 50`: as long as no U element passes it, no +R element can clamp, and the rank-1 update of V is exact. The U rows whose +bound `C[l][k] + [sum of the negative, sum of the positive R1[l][i]]` stays +inside the margin can never clamp; they are found once after the products and +skipped by the pass. + +Three changes move work off the critical path, without changing the result: + +- **Late U update.** One pass changes U by at most 50, so the V update of a + pass is exact as long as U was inside the margin before it. The U update + of a pass and its margin check are therefore done in the next iteration + (which has no pass), by the cores other than the owner and the helper, + split into strips of half a row so that the shares differ by at most one + strip. This needs U inside the margin from the start, which is checked + once after the products; if it is not, the U update stays in the pass. +- **Helper core.** After an iteration without a pass only row w of P has + changed, and the next iteration rebuilds colS anyway. The owner of w + skips row w; a helper core (the first one whose best column was not k*) + runs the power update and the exact key of row w. The other cores whose + best column was not k* keep their key from the previous iteration. +- **Speculative power update.** An iteration without a pass is followed by + one with a pass, whose owner would run the power update before its pass. + Instead, in the iteration without a pass, each core other than the owner + and the helper computes the power update of its own best key into a + buffer while the others finish, unless a larger key of that iteration is + already published. The winner always qualifies, so its owner only copies + the buffer into its P row in the pass iteration. + +### Memory layout + +The L1 of a group interleaves 64 B lines over its 16 cache controllers, so +addresses 1 KiB apart share a controller. A row of N = 128 words is 512 B, +so every V and P row starts at controller 0 or 8. V and P are therefore +placed 256 B apart modulo 1 KiB, the per-core buffers (colS and the others) +get a pitch of N + 16 words, and the odd cores process the second strip of +columns of the pass first. The cores then spread their loads over all the +controllers instead of queueing at the same ones. + +### Selection + +The packed key is `key = S >> 7 << 7 | kidx[k]` for the columns and +`Gain << 7 | (127 - i)` for the exact keys of the candidate users. Each core +keeps its own table `kidx[k] = 127 - k`, and sets its sign bit when k is +allocated, so allocated columns get negative keys without a separate load +of ResAllocMap. Two `vredmax` reductions give the best and second-best column key of a core (the +second over the other lanes, with the best one masked out); the results are +read with `vmv.x.s`. If the second is below the threshold of the best, the +best column is the only candidate and the exact keys are computed in that +column only. The winner is the best key over the cores. With `IT_REDMAX 0` +the maxima use `vslidedown` + `vmax` trees instead, for a VFU without the +reduction tree (where `vredmax` is serial and slower). + +### Build options + +The steps above can be switched off for comparison with compile +definitions of `full`: `IT_U_LATE`, `IT_HELPER`, `IT_SPEC`, `IT_LAYOUT`, +`IT_STAGGER` and `IT_REDMAX` (all 1 by default). `IT_PROF=1` prints the +cycles of each phase per core, the owner of the winner and the helper, and +`IT_CPROF=1` the cycles of the parts of the selection. + +## Files + +- `common/golden.py`: reference model, written directly from the algorithm. +- `common/fast_model.py`: the restructured algorithm, checked bit-exactly + against `golden.py`. +- `common/control.h`: constants shared by the kernels. +- `full/main.c`: the full kernel (data layout, roles of the cores, iteration + loop, checks and printout). +- `full/kernel/iter.c`: its iteration kernels (update pass, late U update, + colS, selection, power update and its speculative version, column fix-up). +- `uv-init`, `gain-select`, `power-update`: the stage kernels. +- `/script/gen_data.py`: generates `data/data_.h` (inputs and + expected outputs) from `script/data_.json` with `make gen-data`. +- `RESULTS.md`: cycle counts and utilization of all versions; + `RESULTS_SUMMARY.md`: the best results only. + +Tests are named `ctrl-__C[_]`, where `` +is the number of cores the kernel uses. The `full` configurations are +`64_128` (all 128 iterations, run twice: first with the inputs not yet in +the L1, then hot), `64_128_iw` (a warm-up run of 2 iterations, then +`` = 8, 16 or 32 iterations hot, for shorter simulations), `63_37`, +`63_37_mix` (inputs that make most passes cross the clamp margin, a stress +test of the slow path) and `9_21`. The variants are `prof` and `cprof`. diff --git a/software/control/common/control.h b/software/control/common/control.h new file mode 100644 index 00000000..4e53a1a0 --- /dev/null +++ b/software/control/common/control.h @@ -0,0 +1,27 @@ +// Copyright 2025 ETH Zurich and University of Bologna. +// Licensed under the Apache License, Version 2.0, see LICENSE for details. +// SPDX-License-Identifier: Apache-2.0 + +// Constants shared by the control-algorithm kernels; they must match +// common/golden.py (see ../README.md for why these values). + +#ifndef CONTROL_H +#define CONTROL_H + +#define CTRL_MIN_R (-4096) +#define CTRL_MAX_R 4096 +#define CTRL_MIN_G 1 +#define CTRL_MAX_G (1 << 20) +#define CTRL_G_SHIFT 7 +#define CTRL_P_SCALE 100 + +// uv_matmul output modes +#define UV_MODE_U 0 +#define UV_MODE_V 1 +// out only, no clamped aux (the full kernel recomputes G where it is used) +#define UV_MODE_RAW 2 + +// e32 elements in one LMUL-4 register group (VLEN = 512) +#define UV_VLMAX 64 + +#endif diff --git a/software/control/common/fast_model.py b/software/control/common/fast_model.py new file mode 100644 index 00000000..c4651ae8 --- /dev/null +++ b/software/control/common/fast_model.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +# Copyright 2025 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + +# Restructured control algorithm, checked bit-exactly against golden.py: +# - R and G are computed once per iteration (they do not depend on the user) +# - R2 only changes after odd iterations: (x + j) % 2 keeps x for even j and +# flips the whole row for odd j +# - the unclamped U = R1 * R2 + C and V = A * R + D are updated in place: +# flipping row w of R2 adds s[k] * R1[:, w] to column k of U (s = +1 for a +# 0 -> 1 flip, -1 otherwise); while no R clamp binds, V changes by the +# rank-1 term (A * R1[:, w]) s^T, and columns whose clamp state changes are +# recomputed in full. + +import numpy as np +import golden as g + + +def control_algo_fast(R1, R2, A, C, D, stats=None): + n, m = R1.shape + R2 = R2.copy() + U = R1.astype(np.int64) @ R2 + C + R = np.clip(U, g.MIN_R, g.MAX_R) + V = A.astype(np.int64) @ R + D + G = np.clip(V >> g.G_SHIFT, g.MIN_G, g.MAX_G) + ops = {"init_mac": n * m * n + m * n * n, "u_add": 0, "v_mac": 0, + "col_fix": 0, "g_updates": 0} + + P = np.full((m, n), g.P_INIT, dtype=np.int64) + res_alloc = np.zeros(n, dtype=np.int64) + user_alloc = np.zeros((m, n), dtype=np.int64) + delta = [g.P_SCALE] * m + winners = [] + + for j in range(n): + if res_alloc.all(): + break + sel = g.select(P, G, res_alloc) + if sel is None: + break + s_max, w, k_star = sel + winners.append((w, k_star, int(s_max))) + g.update_winner(P, G, res_alloc, user_alloc, delta, w, k_star) + if j % 2: + s = 1 - 2 * R2[w] + R2[w] = 1 - R2[w] + U += np.outer(R1[:, w], s) + R_new = np.clip(U, g.MIN_R, g.MAX_R) + dR = R_new - R + rank1 = np.outer(R1[:, w], s) + exact = np.all(dR == rank1, axis=0) + # rank-1 part for columns where no clamp is involved + V += np.outer(A @ R1[:, w], s * exact) + ops["u_add"] += n * n + ops["v_mac"] += m * n + m * n + # full recompute of the other columns + for k in np.nonzero(~exact)[0]: + V[:, k] = A @ R_new[:, k] + D[k] + ops["col_fix"] += 1 + ops["v_mac"] += m * n + R = R_new + G = np.clip(V >> g.G_SHIFT, g.MIN_G, g.MAX_G) + ops["g_updates"] += 1 + else: + R2[w] = (R2[w] + j) % 2 + + g.finalize(P, G, user_alloc, delta) + if stats is not None: + stats.update(ops) + return {"P": P, "ResAllocMap": res_alloc, "UserResAllocMap": user_alloc, + "R2": R2, "winners": winners} + + +if __name__ == "__main__": + for (m, n) in [(1, 1), (2, 3), (1, 100), (8, 16), (16, 64), (48, 100), (80, 100)]: + for seed in range(3): + inp = g.gen_inputs(m, n, seed) + ref = g.control_algo(**inp) + st = {} + out = control_algo_fast(**inp, stats=st) + for key in ["P", "ResAllocMap", "UserResAllocMap", "R2"]: + assert np.array_equal(ref[key], out[key]), (m, n, seed, key) + assert ref["winners"] == out["winners"], (m, n, seed) + print(f"M={m:3d} N={n:3d}: exact over 3 seeds; last seed ops {st}") diff --git a/software/control/common/golden.py b/software/control/common/golden.py new file mode 100644 index 00000000..4af60dcf --- /dev/null +++ b/software/control/common/golden.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +# Copyright 2025 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + +# Golden model of the control algorithm (see ../README.md), written literally +# from the algorithm description. Everything is int32: P and Delta are scaled +# by 100 and every division truncates like C integer division. + +import numpy as np + +MIN_R, MAX_R = -4096, 4096 +MIN_G, MAX_G = 1, 1 << 20 +# G = clamp((A[i] * R + D) >> G_SHIFT, MinG, MaxG): A * R + D has a standard +# deviation of about 3e7, so without the shift S = P * G could not stay in 32 +# bits without saturating most of G. +G_SHIFT = 7 +P_SCALE = 100 +P_INIT = 100 + +INT32_MIN, INT32_MAX = -(1 << 31), (1 << 31) - 1 + + +def check_i32(x, what): + x = np.asarray(x, dtype=np.int64) + if x.size and (x.min() < INT32_MIN or x.max() > INT32_MAX): + raise OverflowError(f"{what} leaves the int32 range") + return x + + +def gen_inputs(m, n, seed): + rng = np.random.default_rng(seed) + return { + "R1": rng.integers(-50, 51, size=(n, m)), + "R2": rng.integers(0, 2, size=(m, n)), + "A": rng.integers(-3000, 3001, size=(m, n)), + "C": rng.integers(-3000, 3001, size=(n, n)), + "D": rng.integers(-3000, 3001, size=(n,)), + } + + +def calc_r(R1, R2, C): + u = check_i32(R1.astype(np.int64) @ R2.astype(np.int64) + C, "R1 * R2 + C") + return np.clip(u, MIN_R, MAX_R) + + +def calc_g(A, R, D): + g = check_i32(A.astype(np.int64) @ R + D, "A * R + D") + return np.clip(g >> G_SHIFT, MIN_G, MAX_G) + + +def div(a, b): + # C integer division (truncates toward zero); here both operands are >= 0 + assert a >= 0 and b > 0 + return a // b + + +def inv_g(g): + # 1/g scaled by 100 + return div(P_SCALE, int(g)) + + +def recip_sum(g_row, alloc_row): + # 100 * (1 + sum of 1/g over the allocated k) + return P_SCALE + sum(inv_g(g_row[k]) for k in np.nonzero(alloc_row)[0]) + + +def power(delta, ig): + # max(0, delta - 1/g), everything scaled by 100 + return max(0, delta - ig) + + +# ln(1 + S) in Q16 with 32-bit integer operations, as the gain-select kernel +# computes it (only the int -> float conversion that finds the exponent uses +# the FPU): +# 1 + S = 2^e * m, m in [1, 2), from the bits of float(1 + S) +# ln m = ln 1.5 + ln(1 + u), u = (m - 1.5) * 2/3, |u| <= 1/3 +# ln(1 + u) = u * sum (-1)^k u^k / (k + 1), LOG_TERMS terms, Horner in Q30 +# Gain = (e * ln 2 + ln m) in Q24, rounded to Q16 +LOG_TERMS = 3 +LOG_Q30 = 1 << 30 +LOG_COEF = [int(round((-1) ** k / (k + 1) * LOG_Q30)) for k in range(LOG_TERMS)] +LOG_LN15_Q30 = int(round(np.log(1.5) * LOG_Q30)) +LOG_LN2_Q24 = int(round(np.log(2) * (1 << 24))) +LOG_TWO_THIRDS_U32 = int(round(2 / 3 * 2 ** 32)) + + +def _mulh(a, b): + # high 32 bits of the 64-bit product (vmulh, vmulhsu) + return (np.asarray(a, dtype=np.int64) * np.int64(b)) >> 32 + + +def ilog(S): + """Gain = ln(1 + S) in Q16 for an int array S >= 0.""" + x = np.asarray(S, dtype=np.int64) + 1 + assert x.max() < (1 << 32) + y = x.astype(np.uint32).astype(np.float32).view(np.uint32).astype(np.int64) + e = (y >> 23) - 127 + f = y & 0x7FFFFF + u = _mulh((f << 9) - (1 << 31), LOG_TWO_THIRDS_U32) + p = np.full_like(u, LOG_COEF[-1]) + for c in LOG_COEF[-2::-1]: + p = c + _mulh(u, p) + lnm = LOG_LN15_Q30 + _mulh(u, p) + return check_i32((e * LOG_LN2_Q24 + (lnm >> 6) + 128) >> 8, "Gain") + + +def select(P, G, res_alloc): + """Per-user masked argmax of Gain = ln(1 + P * G) (lowest k on ties), then + the best user with a strict > (lowest user on ties). Returns (Gain, w, k) + or None.""" + m, n = P.shape + S = ilog(check_i32(P.astype(np.int64) * G, "P * G")) + max_gain, w, kw = -1, -1, -1 + for i in range(m): + best, bk = -1, -1 + for k in range(n): + if res_alloc[k] == 0 and S[i][k] > best: + best, bk = S[i][k], k + if bk >= 0 and best > max_gain: + max_gain, w, kw = best, i, bk + if w < 0: + return None + return max_gain, w, kw + + +def update_winner(P, G, res_alloc, user_alloc, delta, w, k_star): + res_alloc[k_star] = 1 + user_alloc[w][k_star] = 1 + n_w = int(user_alloc[w].sum()) + s = recip_sum(G[w], user_alloc[w]) + delta[w] = div(s, n_w) + P[w][k_star] = power(delta[w], inv_g(G[w][k_star])) + for k in range(P.shape[1]): + if res_alloc[k] == 0: + ig = inv_g(G[w][k]) + P[w][k] = power(div(s + ig, n_w + 1), ig) + + +def finalize(P, G, user_alloc, delta): + m, n = P.shape + for i in range(m): + n_i = int(user_alloc[i].sum()) + if n_i == 0: + continue + delta[i] = div(recip_sum(G[i], user_alloc[i]), n_i) + for k in range(n): + if user_alloc[i][k]: + P[i][k] = power(delta[i], inv_g(G[i][k])) + + +def control_algo(R1, R2, A, C, D, trace=None, iters=None): + """Literal algorithm with the synthetic R/G stages. Returns the outputs + and, per iteration, the winner (w, k, S). iters < N stops the greedy + loop early (shorter benchmark runs of the same shape).""" + n, m = R1.shape + R2 = R2.copy() + P = np.full((m, n), P_INIT, dtype=np.int64) + res_alloc = np.zeros(n, dtype=np.int64) + user_alloc = np.zeros((m, n), dtype=np.int64) + delta = [P_SCALE] * m + winners = [] + + for j in range(n if iters is None else iters): + if res_alloc.all(): + break + # R does not depend on the user, so it is computed once here + R = calc_r(R1, R2, C) + G = calc_g(A, R, D) + sel = select(P, G, res_alloc) + if sel is None: + break + s_max, w, k_star = sel + winners.append((w, k_star, int(s_max))) + update_winner(P, G, res_alloc, user_alloc, delta, w, k_star) + R2[w] = (R2[w] + j) % 2 + + G = calc_g(A, calc_r(R1, R2, C), D) + finalize(P, G, user_alloc, delta) + return {"P": P, "ResAllocMap": res_alloc, "UserResAllocMap": user_alloc, + "R2": R2, "winners": winners} + + +def paper_example(): + """The example of Kim et al.: M = 2, N = 3 with fixed gains, integer x100 + arithmetic.""" + G = np.array([[8, 4, 2], [4, 2, 1]]) + P = np.full((2, 3), P_INIT, dtype=np.int64) + res_alloc = np.zeros(3, dtype=np.int64) + user_alloc = np.zeros((2, 3), dtype=np.int64) + delta = [P_SCALE] * 2 + order = [] + while not res_alloc.all(): + _, w, k_star = select(P, G, res_alloc) + order.append((w, k_star)) + if res_alloc.sum() == len(res_alloc) - 1: + res_alloc[k_star] = 1 + user_alloc[w][k_star] = 1 + break + update_winner(P, G, res_alloc, user_alloc, delta, w, k_star) + finalize(P, G, user_alloc, delta) + return order, P, delta + + +def self_test(): + # exact values x100 are 68.75, 31.25, 100, 81.25 and 150; integer + # truncation of each 1/g and each division may lose up to a few units + order, P, delta = paper_example() + assert order == [(0, 0), (1, 1), (0, 2)], order + expect = {(0, 0): 68.75, (0, 2): 31.25, (1, 1): 100} + for (i, k), v in expect.items(): + assert abs(P[i][k] - v) <= 2, (i, k, P[i][k], v) + assert abs(delta[0] - 81.25) <= 2 and delta[1] == 150, delta + return P, delta + + +if __name__ == "__main__": + print('paper example (x100):', *self_test()) + for (m, n) in [(2, 3), (8, 16), (48, 100), (80, 100)]: + out = control_algo(**gen_inputs(m, n, seed=1)) + print(f"M={m:3d} N={n:3d}: {len(out['winners'])} iterations, " + f"P sum {int(out['P'].sum())}, first winners {out['winners'][:3]}") + print("golden model self-test passed") diff --git a/software/control/full/data/.gitignore b/software/control/full/data/.gitignore new file mode 100644 index 00000000..226b8c26 --- /dev/null +++ b/software/control/full/data/.gitignore @@ -0,0 +1,2 @@ +# Generated data headers — reproduced by `make gen-data` +*.h diff --git a/software/control/full/kernel/iter.c b/software/control/full/kernel/iter.c new file mode 100644 index 00000000..582e8847 --- /dev/null +++ b/software/control/full/kernel/iter.c @@ -0,0 +1,885 @@ +// Copyright 2025 ETH Zurich and University of Bologna. +// Licensed under the Apache License, Version 2.0, see LICENSE for details. +// SPDX-License-Identifier: Apache-2.0 + +// Iteration kernels of the full control algorithm (see ../../README.md), +// with the gain evaluated lazily. +// +// Gain = ilog(S), S = P * G, is only used by the selection, and ilog is +// non-decreasing in S (checked exhaustively for 0 <= S < 2^27, which holds S +// for every allowed input). Its widest step is below 1.6e-5 * S, so with +// Smax the largest S over the free subcarriers, every element whose Gain +// equals the largest Gain has S >= Smax - (Smax >> 14) - 2. The kernels +// therefore keep only the column maxima of S, and evaluate ilog exactly on +// the elements of the candidate columns (column maximum at or above that +// threshold). The winner is the same as with the gain of every element, +// ties included (largest Gain, then the lowest user, then the lowest +// subcarrier). +// +// Per core and column, colS[c][k] = max of S over the core's rows. The +// selection then runs on all cores: +// +// it_cands every core: Smax of its own colS over the free columns +// (slide tree), then its candidate columns and the exact +// keys Gain << 7 | (127 - i) of its rows in those columns; +// best key and column per core. The winner is the best key +// over the cores, so no merge of the cores' colS is needed. +// +// The maxima use vredmax (IT_REDMAX), or vslidedown + vmax trees on a VFU +// without the reduction tree. + +#include + +#include "control.h" +#include "../../gain-select/kernel/gain_select.c" +#include "../../power-update/kernel/power_update.c" + +static inline unsigned int it_vl(unsigned int n) { + unsigned int avl = n < UV_VLMAX ? n : UV_VLMAX; + unsigned int vl; + asm volatile("vsetvli %0, %1, e32, m4, ta, ma" : "=r"(vl) : "r"(avl)); + return vl; +} + +static inline unsigned int it_up4(unsigned int n) { return (n + 3) & ~3u; } + +// Largest |U| that cannot make an R clamp bind across one update pass +// (|R1| <= 50) +#define IT_MARGIN (CTRL_MAX_R - 50) + +// s = 1 - 2 * R2[w][p ..] into vs +#define IT_SIGN(vs, r2w) \ + do { \ + asm volatile("vle32.v " vs ", (%0)" ::"r"(r2w)); \ + asm volatile("vadd.vv " vs ", " vs ", " vs); \ + asm volatile("vrsub.vi " vs ", " vs ", 1"); \ + } while (0) + +// U row += c * s, then fold it into the max (v8) and min (v12) trackers +#define IT_U_ROW(vu, vs, row, c) \ + do { \ + asm volatile("vle32.v " vu ", (%0)" ::"r"(row)); \ + asm volatile("vmacc.vx " vu ", %0, " vs ::"r"(c)); \ + asm volatile("vse32.v " vu ", (%0)" ::"r"(row) : "memory"); \ + asm volatile("vmax.vv v8, v8, " vu); \ + asm volatile("vmin.vv v12, v12, " vu); \ + } while (0) + +// 1 if one of the n words of trk is above the clamp margin. The loads are +// independent, so IT_SCAN of them are kept in flight. +#define IT_SCAN 8 +static inline int it_flag_scan(const int32_t *trk, unsigned int n) { + const volatile int32_t *t = trk; + int flag = 0; + unsigned int k = 0; + for (; k + IT_SCAN <= n; k += IT_SCAN) { + int32_t x[IT_SCAN]; +#pragma clang loop unroll(full) + for (unsigned int j = 0; j < IT_SCAN; ++j) + x[j] = t[k + j]; +#pragma clang loop unroll(full) + for (unsigned int j = 0; j < IT_SCAN; ++j) + flag |= x[j] > IT_MARGIN; + } + for (; k < n; ++k) + flag |= t[k] > IT_MARGIN; + return flag; +} + +// R2[w] = 1 - R2[w]; one core, after every core has read the old row +void it_flip_r2(int32_t *R2, unsigned int w, unsigned int N) { + int32_t *r2w = R2 + w * N; + for (unsigned int p = 0; p < N;) { + const unsigned int vl = it_vl(N - p); + asm volatile("vle32.v v0, (%0)" ::"r"(r2w + p)); + asm volatile("vrsub.vi v0, v0, 1"); + asm volatile("vse32.v v0, (%0)" ::"r"(r2w + p) : "memory"); + p += vl; + } +} + +// Every vector op of the selection runs at a vl that is a multiple of 4 (the +// IPU lanes): with other lengths, the slide trees below gave wrong maxima on +// Spatz, and vloxei32 hangs (see power_update.c). + +// Lanes [n, it_up4(n)) of vk set to -1 (neutral for the maxima here), at vl +// it_up4(n); lidx[i] = i, v4 scratch +#define IT_PAD(vk, n, lidx) \ + do { \ + if ((n) & 3) { \ + asm volatile("vle32.v v4, (%0)" ::"r"(lidx)); \ + asm volatile("vsub.vx v4, v4, %0" ::"r"(n)); \ + asm volatile("vsra.vi v4, v4, 31"); \ + asm volatile("vxor.vi v4, v4, -1"); \ + asm volatile("vor.vv " vk ", " vk ", v4"); \ + } \ + } while (0) + +// IT_REDMAX 1: the maxima use vredmax (the reduction tree of the VFU) +// instead of the slide trees below. IT_RED_VMV 1 reads the result with +// vmv.x.s, 0 through memory. +#ifndef IT_REDMAX +#define IT_REDMAX 1 +#endif +#ifndef IT_RED_VMV +#define IT_RED_VMV 1 +#endif + +// Max of the first n elements of v0 with vredmax (v0 is its own start +// value); v4 is scratch, tmp one word +static inline int32_t it_red_max(unsigned int n, int32_t *tmp) { + it_vl(n); + asm volatile("vredmax.vs v4, v0, v0"); + if (IT_RED_VMV) { + int32_t r; + asm volatile("vmv.x.s %0, v4" : "=r"(r)); + return r; + } + it_vl(1); + asm volatile("vse32.v v4, (%0)" ::"r"(tmp) : "memory"); + snrt_fence_spatz(); + return *(const volatile int32_t *)tmp; +} + +// Max of the first n elements of v0 (n a multiple of 4, 4 <= n <= UV_VLMAX), +// through tmp (four words); v4 is scratch. Halving steps of multiples of 4 +// down to four lanes, then a scalar max. The slide runs at vl = n, so its +// whole source is below vl; vslidedown reads ahead of the element it writes +// and must not chain on its producer (prevent_chaining in +// spatz_controller.sv). +static inline int32_t it_tree_max(unsigned int n, int32_t *tmp) { + if (IT_REDMAX) + return it_red_max(n, tmp); + while (n > 4) { + const unsigned int h = it_up4(n / 2); + it_vl(n); + asm volatile("vslidedown.vx v4, v0, %0" ::"r"(h)); + it_vl(n - h); + asm volatile("vmax.vv v0, v0, v4"); + n = h; + } + it_vl(4); + asm volatile("vse32.v v0, (%0)" ::"r"(tmp) : "memory"); + snrt_fence_spatz(); + const volatile int32_t *t = tmp; + int32_t a = t[0], b = t[1], c = t[2], d = t[3]; + a = a > b ? a : b; + c = c > d ? c : d; + return a > c ? a : c; +} + +// Largest and second largest of the first n lanes of v0 and v8 together, +// where every lane of v8 is at most the same lane of v0 and the lanes of v0 +// are distinct or -1 (n as in it_tree_max). v4, v12 and v16 are scratch, +// tmp eight words. +static inline int32_t it_tree_top2(unsigned int n, int32_t *tmp, + int32_t *second) { + if (IT_REDMAX) { + // top, then the max of v8 and of v0 without the top lane (the lanes of + // v0 are distinct, so only the top lane is set, to 0: a second of 0 + // only makes it_cands take the general path) + const int32_t m = it_red_max(n, tmp); + it_vl(n); + asm volatile("vsub.vx v4, v0, %0" ::"r"(m)); + asm volatile("vsra.vi v4, v4, 31"); + asm volatile("vand.vv v0, v0, v4"); + asm volatile("vmax.vv v0, v0, v8"); + *second = it_red_max(n, tmp); + return m; + } + while (n > 4) { + const unsigned int h = it_up4(n / 2); + it_vl(n); + asm volatile("vslidedown.vx v4, v0, %0" ::"r"(h)); + asm volatile("vslidedown.vx v12, v8, %0" ::"r"(h)); + it_vl(n - h); + asm volatile("vmin.vv v16, v0, v4"); + asm volatile("vmax.vv v0, v0, v4"); + asm volatile("vmax.vv v12, v8, v12"); + asm volatile("vmax.vv v8, v16, v12"); + n = h; + } + it_vl(4); + asm volatile("vse32.v v0, (%0)" ::"r"(tmp) : "memory"); + asm volatile("vse32.v v8, (%0)" ::"r"(tmp + 4) : "memory"); + snrt_fence_spatz(); + const volatile int32_t *t = tmp; + int32_t a[4], b[4]; + for (int i = 0; i < 4; ++i) { + a[i] = t[i]; + b[i] = t[4 + i]; + } + int32_t m = a[0] > a[1] ? a[0] : a[1]; + const int32_t m23 = a[2] > a[3] ? a[2] : a[3]; + m = m > m23 ? m : m23; + int32_t s = b[0] > b[1] ? b[0] : b[1]; + const int32_t s23 = b[2] > b[3] ? b[2] : b[3]; + s = s > s23 ? s : s23; + for (int i = 0; i < 4; ++i) + if (a[i] != m && a[i] > s) + s = a[i]; + *second = s; + return m; +} + +// G = clamp(V >> G_SHIFT, MinG, MaxG) in place +#define IT_G_OF_V(vv) \ + do { \ + asm volatile("vsra.vi " vv ", " vv ", %0" ::"i"(CTRL_G_SHIFT)); \ + asm volatile("vmax.vx " vv ", " vv ", %0" ::"r"(CTRL_MIN_G)); \ + asm volatile("vmin.vx " vv ", " vv ", %0" ::"r"(CTRL_MAX_G)); \ + } while (0) + +// V row += a * s, then S = P * clamp(V >> G_SHIFT) folded into the column +// maxima acc. G is not stored: the pass is bound by the L1 bandwidth of the +// group, and G follows from V wherever it is needed (it_g_rows). (Loading +// the next row before this row's stores, with two buffers, was slower.) +#define IT_VS_ROW(acc, vs, vrow, prow, a) \ + do { \ + asm volatile("vle32.v v4, (%0)" ::"r"(vrow)); \ + asm volatile("vle32.v v0, (%0)" ::"r"(prow)); \ + asm volatile("vmacc.vx v4, %0, " vs ::"r"(a)); \ + asm volatile("vse32.v v4, (%0)" ::"r"(vrow) : "memory"); \ + IT_G_OF_V("v4"); \ + asm volatile("vmul.vv v0, v0, v4"); \ + asm volatile("vmax.vv " acc ", " acc ", v0"); \ + } while (0) + +// G of one V element (scalar) +static inline int32_t it_g_of(int32_t v) { + int32_t g = v >> CTRL_G_SHIFT; + g = g < CTRL_MIN_G ? CTRL_MIN_G : g; + return g > CTRL_MAX_G ? CTRL_MAX_G : g; +} + +// The front of one strip of it_pu_update at vl n: inv(G) of V row strip vr +// into vinv (the gather at vl rounded up to 4, see pu_update), and the +// ResAllocMap and P strips into vra and vp +#define IT_PU_FRONT(vi, vinv, vra, vp, vr, ra, pr, n) \ + do { \ + it_vl(it_up4(n)); \ + asm volatile("vle32.v " vi ", (%0)" ::"r"(vr)); \ + asm volatile("vsra.vi " vi ", " vi ", %0" ::"i"(CTRL_G_SHIFT)); \ + asm volatile("vmax.vx " vi ", " vi ", %0" ::"r"(CTRL_MIN_G)); \ + asm volatile("vminu.vx " vi ", " vi ", %0" ::"r"(PU_TAB_SIZE - 1)); \ + asm volatile("vsll.vi " vi ", " vi ", 2"); \ + asm volatile("vloxei32.v " vinv ", (%0), " vi ::"r"(pu_inv_tab)); \ + it_vl(n); \ + asm volatile("vle32.v " vra ", (%0)" ::"r"(ra)); \ + asm volatile("vle32.v " vp ", (%0)" ::"r"(pr)); \ + } while (0) + +// The back of one strip: P = max(0, (sum + inv) / (n + 1) - inv) where free, +// the old P where allocated (vres scratch) +#define IT_PU_BACK(vinv, vra, vp, vres, pr, sum, m, n) \ + do { \ + it_vl(n); \ + asm volatile("vadd.vx " vres ", " vinv ", %0" ::"r"(sum)); \ + asm volatile("vmulhu.vx " vres ", " vres ", %0" ::"r"(m)); \ + asm volatile("vsub.vv " vres ", " vres ", " vinv); \ + asm volatile("vmax.vx " vres ", " vres ", zero"); \ + asm volatile("vsub.vv " vp ", " vp ", " vres); \ + asm volatile("vmacc.vv " vres ", " vra ", " vp); \ + asm volatile("vse32.v " vres ", (%0)" ::"r"(pr) : "memory"); \ + } while (0) + +// pu_update (power_update.c) with G taken from the V row of w, so the owner +// does not first store a G row: the table index min(G, 101) is +// min(max(V >> G_SHIFT, MinG), 101). The two strips of the row (N <= 2 * +// UV_VLMAX) use separate registers, so their chains overlap, and the +// division for Delta waits in the shadow of the vector work. +// The vector part of it_pu_update: the new P row of a user with the n - 1 +// allocated subcarriers klist[0 .. n - 2] and the new one k (res_alloc[k] +// already 1) into dst (the P row itself, or another buffer), from the old P +// row p_row. Returns Delta and the new P[k] in *pk; dst[k] is not set. +static inline int32_t it_pu_calc(int32_t *dst, const int32_t *p_row, + const int32_t *v_row, + const int32_t *res_alloc, + const int32_t *klist, int32_t n, + unsigned int k, unsigned int N, + int32_t *pk) { + const unsigned int n0 = N < UV_VLMAX ? N : UV_VLMAX, n1 = N - n0; + IT_PU_FRONT("v0", "v4", "v8", "v12", v_row, res_alloc, p_row, n0); + if (n1) + IT_PU_FRONT("v16", "v20", "v24", "v28", v_row + n0, res_alloc + n0, + p_row + n0, n1); + + int32_t sum = CTRL_P_SCALE + pu_inv(it_g_of(v_row[k])); + for (int32_t j = 0; j < n - 1; ++j) + sum += pu_inv(it_g_of(v_row[klist[j]])); + const uint32_t m = 0xFFFFFFFFu / (uint32_t)(n + 1) + 1; + IT_PU_BACK("v4", "v8", "v12", "v0", dst, sum, m, n0); + if (n1) + IT_PU_BACK("v20", "v24", "v28", "v16", dst + n0, sum, m, n1); + + const int32_t delta = sum / n; + const int32_t p = delta - pu_inv(it_g_of(v_row[k])); + *pk = p > 0 ? p : 0; + return delta; +} + +int32_t it_pu_update(int32_t *p_row, const int32_t *v_row, int32_t *res_alloc, + int32_t *ua_row, int32_t *klist, int32_t *nalloc, + unsigned int k, unsigned int N) { + res_alloc[k] = 1; + ua_row[k] = 1; + const int32_t n = *nalloc + 1; + klist[n - 1] = k; + *nalloc = n; + // the vector loads below must see ResAllocMap[k] = 1 + snrt_fence_snitch(); + int32_t pk; + const int32_t delta = + it_pu_calc(p_row, p_row, v_row, res_alloc, klist, n, k, N, &pk); + // P[w][k] after the vector store of the same row + snrt_fence_spatz(); + p_row[k] = pk; + return delta; +} + +// it_pu_update in two steps (IT_SPEC): it_pu_spec computes the new P row of +// a possible winner (user of p_row, subcarrier k) into spec, without +// changing any state (res_alloc[k] is set during the call only; it is this +// core's copy); it_pu_commit applies it once the user has won, if res_alloc +// and the P and V rows are still the same. +int32_t it_pu_spec(int32_t *spec, const int32_t *p_row, const int32_t *v_row, + int32_t *res_alloc, const int32_t *klist, + const int32_t *nalloc, unsigned int k, unsigned int N, + int32_t *pk) { + res_alloc[k] = 1; + snrt_fence_snitch(); + const int32_t delta = + it_pu_calc(spec, p_row, v_row, res_alloc, klist, *nalloc + 1, k, N, pk); + snrt_fence_spatz(); + res_alloc[k] = 0; + return delta; +} + +void it_pu_commit(int32_t *p_row, const int32_t *spec, int32_t *ua_row, + int32_t *klist, int32_t *nalloc, unsigned int k, + unsigned int N, int32_t pk) { + ua_row[k] = 1; + const int32_t n = *nalloc + 1; + klist[n - 1] = k; + *nalloc = n; + for (unsigned int p = 0; p < N;) { + const unsigned int vl = it_vl(N - p); + asm volatile("vle32.v v0, (%0)" ::"r"(spec + p)); + asm volatile("vse32.v v0, (%0)" ::"r"(p_row + p) : "memory"); + p += vl; + } + snrt_fence_spatz(); + p_row[k] = pk; +} + +// The U rows of [u0, u1) that can come near the clamp: U = R1 * R2 + C with +// R2 in {0, 1}, so U[l][k] stays within C[l][k] + [sum of the negative, +// sum of the positive R1[l][i]]. Rows whose bound stays within the margin +// never clamp nor raise the margin flag, so the pass skips them and their U +// goes stale (main.c recomputes it before the first column fix-up). The +// other rows go into rows[]. Returns the number of listed rows. +unsigned int it_u_rows(int32_t *rows, const int32_t *C, + const int32_t *R1, unsigned int u0, unsigned int u1, + unsigned int M, unsigned int ldr, unsigned int N, + int32_t *bnd) { + // The bounds of up to UV_VLMAX rows at a time, one row per lane: strided + // loads down the columns of R1 and C (vl padded to a multiple of 4; the + // padded lanes read past row u1 and are not used). bnd: 2 * UV_VLMAX words. + const unsigned int r1stride = ldr * sizeof(int32_t); + const unsigned int cstride = N * sizeof(int32_t); + unsigned int n = 0; + for (unsigned int l0 = u0; l0 < u1; l0 += UV_VLMAX) { + const unsigned int nr = u1 - l0 < UV_VLMAX ? u1 - l0 : UV_VLMAX; + it_vl(it_up4(nr)); + asm volatile("vxor.vv v0, v0, v0"); + asm volatile("vxor.vv v4, v4, v4"); + for (unsigned int i = 0; i < M; ++i) { + asm volatile("vlse32.v v16, (%0), %1" ::"r"(R1 + l0 * ldr + i), + "r"(r1stride)); + asm volatile("vmax.vx v20, v16, zero"); + asm volatile("vadd.vv v0, v0, v20"); + asm volatile("vmin.vx v20, v16, zero"); + asm volatile("vadd.vv v4, v4, v20"); + } + asm volatile("vlse32.v v8, (%0), %1" ::"r"(C + l0 * N), "r"(cstride)); + asm volatile("vadd.vv v12, v8, v4"); + asm volatile("vadd.vv v8, v8, v0"); + for (unsigned int k = 1; k < N; ++k) { + asm volatile("vlse32.v v16, (%0), %1" ::"r"(C + l0 * N + k), + "r"(cstride)); + asm volatile("vadd.vv v20, v16, v0"); + asm volatile("vmax.vv v8, v8, v20"); + asm volatile("vadd.vv v20, v16, v4"); + asm volatile("vmin.vv v12, v12, v20"); + } + // v8 = largest, v12 = smallest reachable U of each row + asm volatile("vse32.v v8, (%0)" ::"r"(bnd) : "memory"); + asm volatile("vse32.v v12, (%0)" ::"r"(bnd + UV_VLMAX) : "memory"); + snrt_fence_spatz(); + for (unsigned int r = 0; r < nr; ++r) { + const int safe = + bnd[r] <= IT_MARGIN && bnd[UV_VLMAX + r] >= -IT_MARGIN; + if (!safe) + rows[n++] = (int32_t)(l0 + r); + } + } + snrt_fence_snitch(); + return n; +} + +// U rows += R1[l][w] * s on the listed rows, with s of both strips in v24 / +// v28, and v8 = max(Umax, -Umin) of those rows +static inline void it_u_rows_add(int32_t *U, const int32_t *R1, + unsigned int w, const int32_t *rows, + unsigned int n, unsigned int ldr, + unsigned int N) { + it_vl(N); + asm volatile("vxor.vv v8, v8, v8"); + asm volatile("vxor.vv v12, v12, v12"); + for (unsigned int p = 0, j = 0; p < N; ++j) { + const unsigned int vl = it_vl(N - p); + for (unsigned int r = 0; r < n; ++r) { + const unsigned int l = (unsigned int)rows[r]; + if (j) + IT_U_ROW("v0", "v28", U + l * N + p, R1[l * ldr + w]); + else + IT_U_ROW("v0", "v24", U + l * N + p, R1[l * ldr + w]); + } + p += vl; + } + it_vl(N); + asm volatile("vrsub.vi v12, v12, 0"); + asm volatile("vmax.vv v8, v8, v12"); +} + +// U part of the pass on the listed rows only: s of both strips into v24 / +// v28, then it_u_rows_add +static inline void it_u_part(int32_t *U, const int32_t *R1, + const int32_t *r2w, unsigned int w, + const int32_t *rows, unsigned int n, + unsigned int ldr, unsigned int N) { + const unsigned int vl0 = it_vl(N); + IT_SIGN("v24", r2w); + if (N > vl0) { + it_vl(N - vl0); + IT_SIGN("v28", r2w + vl0); + } + it_u_rows_add(U, R1, w, rows, n, ldr, N); +} + +// 1 if an element of the n listed U rows is outside the clamp margin (v8, +// v12 and v16 scratch, trk as in it_pass) +int it_u_check(const int32_t *U, const int32_t *rows, unsigned int n, + unsigned int N, int32_t *trk) { + if (!n) + return 0; + it_vl(N); + asm volatile("vxor.vv v8, v8, v8"); + asm volatile("vxor.vv v12, v12, v12"); + for (unsigned int p = 0; p < N;) { + const unsigned int vl = it_vl(N - p); + for (unsigned int r = 0; r < n; ++r) { + const int32_t *u = U + (unsigned int)rows[r] * N + p; + asm volatile("vle32.v v16, (%0)" ::"r"(u)); + asm volatile("vmax.vv v8, v8, v16"); + asm volatile("vmin.vv v12, v12, v16"); + } + p += vl; + } + it_vl(N); + asm volatile("vrsub.vi v12, v12, 0"); + asm volatile("vmax.vv v8, v8, v12"); + asm volatile("vse32.v v8, (%0)" ::"r"(trk) : "memory"); + snrt_fence_spatz(); + return it_flag_scan(trk, N < UV_VLMAX ? N : UV_VLMAX); +} + +// s of both strips (v24 / v28, as left by it_pass) into sbuf +static inline void it_s_save(int32_t *sbuf, unsigned int N) { + const unsigned int vl0 = it_vl(N); + asm volatile("vse32.v v24, (%0)" ::"r"(sbuf) : "memory"); + if (N > vl0) { + it_vl(N - vl0); + asm volatile("vse32.v v28, (%0)" ::"r"(sbuf + vl0) : "memory"); + } +} + +// Strips of columns per U row in it_u_late +#define IT_U_STRIPS(N) ((N) > UV_VLMAX ? 2u : 1u) + +// The U part of a pass one iteration later (see main.c), with s from sbuf +// (it_s_save) as R2[w] may already be flipped; trk as in it_pass. The work +// is split in units of one strip of one row: unit u is strip +// u % IT_U_STRIPS(N) of row rows[u / IT_U_STRIPS(N)], and this core updates +// units [u0, u1), so the cores' shares differ by at most one strip instead +// of one row. Returns 1 if a U element crossed the clamp margin. +int it_u_late(int32_t *U, const int32_t *R1, const int32_t *sbuf, + unsigned int w, const int32_t *rows, unsigned int u0, + unsigned int u1, unsigned int ldr, unsigned int N, + int32_t *trk) { + if (u0 >= u1) + return 0; + const unsigned int ns = IT_U_STRIPS(N); + const unsigned int vl0 = it_vl(N); + asm volatile("vle32.v v24, (%0)" ::"r"(sbuf)); + if (N > vl0) { + it_vl(N - vl0); + asm volatile("vle32.v v28, (%0)" ::"r"(sbuf + vl0)); + } + it_vl(N); + asm volatile("vxor.vv v8, v8, v8"); + asm volatile("vxor.vv v12, v12, v12"); + for (unsigned int j = 0; j < ns; ++j) { + it_vl(N - j * vl0); + for (unsigned int u = u0 + (j + ns - u0 % ns) % ns; u < u1; u += ns) { + const unsigned int l = (unsigned int)rows[u / ns]; + if (j) + IT_U_ROW("v0", "v28", U + l * N + vl0, R1[l * ldr + w]); + else + IT_U_ROW("v0", "v24", U + l * N, R1[l * ldr + w]); + } + } + it_vl(N); + asm volatile("vrsub.vi v12, v12, 0"); + asm volatile("vmax.vv v8, v8, v12"); + asm volatile("vse32.v v8, (%0)" ::"r"(trk) : "memory"); + snrt_fence_spatz(); + return it_flag_scan(trk, N < UV_VLMAX ? N : UV_VLMAX); +} + +// Update pass after an odd iteration won by user w: the nu U rows listed in +// urows (it_u_rows), V rows [v0, v1) and colS of those rows (G is left +// stale, see IT_VS_ROW). R1 and AR1 +// have row pitch ldr. trk is a scratch of UV_VLMAX words. If sbuf is not +// NULL, s is stored there for it_u_late. Returns 1 if a U element crossed +// the clamp margin (then the caller fixes columns and calls it_s_rows). +// rev 1 runs the second strip of columns first, so that the cores do not all +// stream the same L1 lines (controllers) at the same time. +int it_pass(int32_t *U, int32_t *V, const int32_t *P, + int32_t *colS, const int32_t *R1, const int32_t *AR1, + const int32_t *R2, unsigned int w, const int32_t *urows, + unsigned int nu, unsigned int v0, unsigned int v1, + unsigned int ldr, unsigned int N, int32_t *trk, + int32_t *sbuf, unsigned int rev) { + const int32_t *r2w = R2 + w * N; + if (nu) { + it_u_part(U, R1, r2w, w, urows, nu, ldr, N); + asm volatile("vse32.v v8, (%0)" ::"r"(trk) : "memory"); + } else { + // s of both strips for the V part + const unsigned int vl0 = it_vl(N); + IT_SIGN("v24", r2w); + if (N > vl0) { + it_vl(N - vl0); + IT_SIGN("v28", r2w + vl0); + } + } + if (sbuf) + it_s_save(sbuf, N); + const unsigned int vl0 = N < UV_VLMAX ? N : UV_VLMAX; + for (unsigned int jj = 0; jj < 2; ++jj) { + const unsigned int j = jj ^ (rev & 1), p = j ? vl0 : 0; + if (p >= N) + continue; + it_vl(N - p); + if (j == 0) { + asm volatile("vxor.vv v16, v16, v16"); + asm volatile("vadd.vi v16, v16, -1"); + } else { + asm volatile("vxor.vv v20, v20, v20"); + asm volatile("vadd.vi v20, v20, -1"); + } + for (unsigned int i = v0; i < v1; ++i) { + const unsigned int o = i * N + p; + const int32_t a = AR1[i * ldr + w]; + if (j == 0) + IT_VS_ROW("v16", "v24", V + o, P + o, a); + else + IT_VS_ROW("v20", "v28", V + o, P + o, a); + } + if (j == 0) + asm volatile("vse32.v v16, (%0)" ::"r"(colS + p) : "memory"); + else + asm volatile("vse32.v v20, (%0)" ::"r"(colS + p) : "memory"); + } + snrt_fence_spatz(); + return nu ? it_flag_scan(trk, N < UV_VLMAX ? N : UV_VLMAX) : 0; +} + +// colS of rows [r0, r1) from P and V (no update), without row skip (none if +// skip is outside [r0, r1)); rev as in it_pass +void it_s_rows(int32_t *colS, const int32_t *P, const int32_t *V, + unsigned int r0, unsigned int r1, unsigned int skip, + unsigned int N, unsigned int rev) { + const unsigned int vl0 = N < UV_VLMAX ? N : UV_VLMAX; + for (unsigned int jj = 0; jj < 2; ++jj) { + const unsigned int p = (jj ^ (rev & 1)) ? vl0 : 0; + if (p >= N) + continue; + it_vl(N - p); + asm volatile("vxor.vv v16, v16, v16"); + asm volatile("vadd.vi v16, v16, -1"); + for (unsigned int i = r0; i < r1; ++i) { + if (i == skip) + continue; + asm volatile("vle32.v v0, (%0)" ::"r"(P + i * N + p)); + asm volatile("vle32.v v4, (%0)" ::"r"(V + i * N + p)); + IT_G_OF_V("v4"); + asm volatile("vmul.vv v0, v0, v4"); + asm volatile("vmax.vv v16, v16, v0"); + } + asm volatile("vse32.v v16, (%0)" ::"r"(colS + p) : "memory"); + } + snrt_fence_spatz(); +} + +// G rows [r0, r1) from V, for the scalar code that reads G (power update, +// finalization) +void it_g_rows(int32_t *G, const int32_t *V, unsigned int r0, + unsigned int r1, unsigned int N) { + for (unsigned int i = r0; i < r1; ++i) + for (unsigned int p = 0; p < N;) { + unsigned int vl = it_vl(N - p); + asm volatile("vle32.v v4, (%0)" ::"r"(V + i * N + p)); + IT_G_OF_V("v4"); + asm volatile("vse32.v v4, (%0)" ::"r"(G + i * N + p) : "memory"); + p += vl; + } + snrt_fence_spatz(); +} + +// Recompute V[:, k] = A * clamp(U[:, k]) + D[k] for every column +// k in [k0, k1) with some |U| above the margin (A has row pitch lda). Slow +// path; every U row must be exact (see main.c). +void it_fix_cols(int32_t *V, const int32_t *U, const int32_t *A, + const int32_t *D, unsigned int k0, unsigned int k1, + unsigned int M, unsigned int N, unsigned int lda) { + const unsigned int stride = N * sizeof(int32_t); + const unsigned int astride = lda * sizeof(int32_t); + for (unsigned int k = k0; k < k1; ++k) { + int need = 0; + for (unsigned int l = 0; l < N; ++l) { + int32_t u = U[l * N + k]; + need |= (u > IT_MARGIN) | (u < -IT_MARGIN); + } + if (!need) + continue; + for (unsigned int i = 0; i < M;) { + unsigned int vl = it_vl(M - i); + asm volatile("vxor.vv v16, v16, v16"); + asm volatile("vadd.vx v16, v16, %0" ::"r"(D[k])); + for (unsigned int l = 0; l < N; ++l) { + int32_t r = U[l * N + k]; + r = r < CTRL_MIN_R ? CTRL_MIN_R : r; + r = r > CTRL_MAX_R ? CTRL_MAX_R : r; + asm volatile("vlse32.v v20, (%0), %1" ::"r"(A + i * lda + l), + "r"(astride)); + asm volatile("vmacc.vx v16, %0, v20" ::"r"(r)); + } + asm volatile("vsse32.v v16, (%0), %1" ::"r"(V + i * N + k), "r"(stride) + : "memory"); + i += vl; + } + } + snrt_fence_spatz(); +} + +// Scratch of it_cands: candidate keys, one word, the gains of a column +#define IT_SCR (3 * UV_VLMAX + 16) + +// Columns take the low IT_COL_BITS bits of a packed key. The per-core table +// kidx holds 127 - k for a free column k and INT32_MIN | (127 - k) for an +// allocated one (it_take), so the key of an allocated column is negative. +#define IT_COL_BITS 7 + +// Packed key of one strip of columns into vk: colS[k] >> 7 << 7 | kidx[k] +// (v4 scratch) +#define IT_PACK_KEY(vk, cs, ki) \ + do { \ + asm volatile("vle32.v " vk ", (%0)" ::"r"(cs)); \ + asm volatile("vle32.v v4, (%0)" ::"r"(ki)); \ + asm volatile("vand.vx " vk ", " vk ", %0" ::"r"(-(1 << IT_COL_BITS))); \ + asm volatile("vor.vv " vk ", " vk ", v4"); \ + } while (0) + +// Candidate keys of one strip into vk: kidx[k] where colS[k] >= thr, else +// -1 (negative for the allocated columns either way; v4 scratch) +#define IT_CAND_KEY(vk, cs, ki, thr) \ + do { \ + asm volatile("vle32.v " vk ", (%0)" ::"r"(cs)); \ + asm volatile("vle32.v v4, (%0)" ::"r"(ki)); \ + asm volatile("vsub.vx " vk ", " vk ", %0" ::"r"(thr)); \ + asm volatile("vsra.vi " vk ", " vk ", 31"); \ + asm volatile("vor.vv " vk ", " vk ", v4"); \ + } while (0) + +// Column k allocated in this core's table +static inline void it_take(int32_t *kidx, unsigned int k) { + kidx[k] = INT32_MIN | (127 - (int32_t)k); +} + +// Keys of vk below lim kept, the others set to -1 (v4 scratch) +#define IT_KEY_BELOW(vk, lim) \ + do { \ + asm volatile("vsub.vx v4, " vk ", %0" ::"r"(lim)); \ + asm volatile("vsra.vi v4, v4, 31"); \ + asm volatile("vand.vv " vk ", " vk ", v4"); \ + asm volatile("vxor.vi v4, v4, -1"); \ + asm volatile("vor.vv " vk ", " vk ", v4"); \ + } while (0) + +// IT_CPROF 1: cycles of the parts of it_cands per core, summed while +// it_cprof_on: top-2 of the column keys, the candidate keys (several +// candidate columns only), the exact keys; then the calls, the columns with +// exact keys and the calls with a single candidate column +#ifndef IT_CPROF +#define IT_CPROF 0 +#endif +enum { CP_BEST, CP_CKEY, CP_EXACT, CP_CALLS, CP_COLS, CP_SINGLE, CP_NUM }; +#if IT_CPROF +static unsigned int it_cprof[16][CP_NUM] __attribute__((section(".data"))); +static volatile int it_cprof_on[16] __attribute__((section(".data"))); +#define CP_LAP(ph) \ + do { \ + if (it_cprof_on[cpc]) { \ + const unsigned int cpn = benchmark_get_cycle(); \ + it_cprof[cpc][ph] += cpn - cpt; \ + cpt = cpn; \ + } \ + } while (0) +#define CP_ADD(ph, n) \ + do { \ + if (it_cprof_on[cpc]) \ + it_cprof[cpc][ph] += (n); \ + } while (0) +#else +#define CP_LAP(ph) \ + do { \ + } while (0) +#define CP_ADD(ph, n) \ + do { \ + } while (0) +#endif + +// Selection on one core over its rows [r0, r1) (1 <= r1 - r0 <= UV_VLMAX): +// Smax of its colS over the free columns, then its candidate columns and +// their exact keys. Smax of the core's own rows is at most the global one, +// so its threshold is too, and the core's best key is exact; the winner is +// the best key over the cores. kidx as in it_take, uidx[i] = 127 - i, +// lidx[i] = i. scr is a scratch of IT_SCR words. The padded lanes read up to +// 3 words past colS, kidx and uidx, and up to 3 rows past row r1 +// of P and V. Row skip (none if outside [r0, r1)) is left out of the exact +// keys. Returns the best key (-1 if none) and its column in *kb. +// (Scalar passes over the S of the free columns instead of the trees were +// slower: the scalar loads wait for the L1 one by one.) +int32_t it_cands(const int32_t *colS, const int32_t *P, const int32_t *V, + const int32_t *kidx, + const int32_t *uidx, const int32_t *lidx, unsigned int r0, + unsigned int r1, unsigned int skip, unsigned int N, + int32_t *scr, int *kb) { + const unsigned int vl0 = N < UV_VLMAX ? N : UV_VLMAX; + const unsigned int vl1 = N - vl0; + const unsigned int vl0p = it_up4(vl0), vl1p = it_up4(vl1); + int32_t *ckey = scr, *tmp = scr + 2 * UV_VLMAX, *gscr = tmp + 16; + const unsigned int gstride = N * sizeof(int32_t); +#if IT_CPROF + const unsigned int cpc = snrt_cluster_vpu_idx(); + unsigned int cpt = benchmark_get_cycle(); + CP_ADD(CP_CALLS, 1); +#endif + + // The two largest packed keys of the free columns: the first gives a + // candidate column and T = Smax >> 7 << 7, so thr = T - (T >> 14) - 2 is + // at most the threshold of Smax. If the second key's column cannot reach + // thr, that column is the only candidate. + it_vl(vl0p); + IT_PACK_KEY("v0", colS, kidx); + IT_PAD("v0", vl0, lidx); + if (!vl1) { + asm volatile("vxor.vv v8, v8, v8"); + asm volatile("vadd.vi v8, v8, -1"); + } else { + it_vl(vl1p); + IT_PACK_KEY("v20", colS + vl0, kidx + vl0); + IT_PAD("v20", vl1, lidx); + asm volatile("vmin.vv v8, v0, v20"); + asm volatile("vmax.vv v0, v0, v20"); + } + int32_t second; + const int32_t top = it_tree_top2(vl0p, tmp, &second); + *kb = -1; + if (top < 0) + return -1; + const int32_t tb = top & ~((1 << IT_COL_BITS) - 1); + const int32_t thr = tb - (tb >> 14) - 2; + const int single = (second | ((1 << IT_COL_BITS) - 1)) < thr; + CP_ADD(CP_SINGLE, single); + CP_LAP(CP_BEST); + + // Otherwise the candidate keys of both strips, kept in ckey; strip 1 + // folded into v0 + if (!single) { + it_vl(vl0p); + IT_CAND_KEY("v0", colS, kidx, thr); + IT_PAD("v0", vl0, lidx); + asm volatile("vse32.v v0, (%0)" ::"r"(ckey) : "memory"); + if (vl1) { + it_vl(vl1p); + IT_CAND_KEY("v8", colS + vl0, kidx + vl0, thr); + IT_PAD("v8", vl1, lidx); + asm volatile("vse32.v v8, (%0)" ::"r"(ckey + UV_VLMAX) : "memory"); + asm volatile("vmax.vv v0, v0, v8"); + } + } + + const unsigned int rows = r1 - r0, rowsp = it_up4(rows); + int32_t best = -1; + int32_t kc = + single ? (top & ((1 << IT_COL_BITS) - 1)) : it_tree_max(vl0p, tmp); + CP_LAP(CP_CKEY); + while (kc >= 0) { + CP_ADD(CP_COLS, 1); + const unsigned int k = 127 - kc; + // exact keys of rows [r0, r1) in column k, best one into tmp + it_vl(rowsp); + asm volatile("vlse32.v v0, (%0), %1" ::"r"(P + r0 * N + k), "r"(gstride)); + asm volatile("vlse32.v v4, (%0), %1" ::"r"(V + r0 * N + k), "r"(gstride)); + IT_G_OF_V("v4"); + GAIN_CORE_R("v0", "v4", "v12", gscr, 0); + asm volatile("vle32.v v4, (%0)" ::"r"(uidx + r0)); + asm volatile("vadd.vv v0, v12, v4"); + IT_PAD("v0", rows, lidx); + if (skip >= r0 && skip < r1) { + // lane skip - r0 to -1: (d | -d) >> 31 is 0 there and -1 elsewhere + asm volatile("vle32.v v4, (%0)" ::"r"(lidx)); + asm volatile("vsub.vx v4, v4, %0" ::"r"(skip - r0)); + asm volatile("vrsub.vi v12, v4, 0"); + asm volatile("vor.vv v4, v4, v12"); + asm volatile("vsra.vi v4, v4, 31"); + asm volatile("vand.vv v0, v0, v4"); + asm volatile("vxor.vi v4, v4, -1"); + asm volatile("vor.vv v0, v0, v4"); + } + const int32_t key = it_tree_max(rowsp, tmp); + // strict > in increasing k keeps the lowest column on ties + if (key > best) { + best = key; + *kb = (int)k; + } + if (single) + break; + // next candidate: the keys below kc (higher columns) + it_vl(vl0p); + asm volatile("vle32.v v0, (%0)" ::"r"(ckey)); + IT_KEY_BELOW("v0", kc); + if (vl1) { + it_vl(vl1p); + asm volatile("vle32.v v8, (%0)" ::"r"(ckey + UV_VLMAX)); + IT_KEY_BELOW("v8", kc); + asm volatile("vmax.vv v0, v0, v8"); + } + kc = it_tree_max(vl0p, tmp); + } + CP_LAP(CP_EXACT); + return best; +} diff --git a/software/control/full/main.c b/software/control/full/main.c new file mode 100644 index 00000000..10a500e1 --- /dev/null +++ b/software/control/full/main.c @@ -0,0 +1,763 @@ +// Copyright 2025 ETH Zurich and University of Bologna. +// Licensed under the Apache License, Version 2.0, see LICENSE for details. +// SPDX-License-Identifier: Apache-2.0 + +// The full control algorithm on ACTIVE_CORES cores of one group, from the +// inputs to P, ResAllocMap and UserResAllocMap (see ../README.md): +// +// products U/R, V and AR1 = A * R1 (uv_matmul), colS of every core, +// and the U rows that can come near the clamp (it_u_rows; the +// update pass skips the others) +// iteration j every core picks the winner w of iteration j - 1 from the +// per-core keys and marks its subcarrier; the owner of row w +// updates its power (it_pu_update). After an odd iteration, +// the update pass (it_pass), which the owner runs after its +// pu_update; after an even one, the owner recomputes its +// colS (P changed in row w only). Then every core: Smax of +// its own colS, its candidate columns and their exact keys +// (it_cands), and one barrier. If a pass crossed the clamp +// margin (rare), the slow path fixes the V columns +// (it_fix_cols) and redoes colS and the selection. Core 0 +// flips R2[w] one iteration after the pass. +// finalization the update for the last winner, then the users split over +// the cores (pu_finalize) +// +// One barrier of the active cores per iteration: the selection of a core +// only reads its own colS, P and V rows and its copy of ResAllocMap. The L1 is +// private to the group (L1D_FOLD_GROUP), so no data leaves the group and +// nothing is flushed. The state is restored before every run, outside the +// timed region; run 0 is the first touch of the inputs, the later runs are +// hot. Each active core checks its rows of P and UserResAllocMap, core 0 +// ResAllocMap and the winner of every iteration. + +#include +#include +#include + +#include DATAHEADER +#include "kernel/iter.c" +#include "../uv-init/kernel/uv_init.c" + +#ifndef ACTIVE_CORES +#define ACTIVE_CORES 1 +#endif +#ifndef L1D_FOLD +#define L1D_FOLD L1D_FOLD_GROUP +#endif +#ifndef XBAR_OFFSET +#define XBAR_OFFSET 6 +#endif + +#define NUM_RUNS 2 +// IT_HELPER 1: in an iteration without a pass (only row w of the winner w +// changed, and column k* was taken), a helper core runs the power update of +// row w and the exact key of row w, while the owner of row w selects over +// its other rows only. Its colS then leaves out row w, which is fine as +// the next iteration has a pass and rebuilds every colS. The helper is the +// first core after the owner whose best key of the previous iteration is +// not in column k*, so that key is still exact; every such core reuses its +// key instead of selecting again. +#ifndef IT_HELPER +#define IT_HELPER 1 +#endif +// IT_U_LATE 1: the U part of a pass runs in the next iteration (without a +// pass), on the cores other than the owner of its winner, which otherwise +// wait at the barrier. The rank-1 update of V in a pass is exact as long as +// U was within the clamp margin before it (one pass moves U by at most +// max |R1| = 50 = MaxR - margin), so the U update and its margin check are +// only needed before the next pass. Once a late check crosses the margin, +// or if U is already outside it after the products, the passes update U +// first again (and fix the V columns, as before). +#ifndef IT_U_LATE +#define IT_U_LATE 1 +#endif +// IT_SPEC 1: in an iteration without a pass, the cores other than the owner +// and the helper, which wait at the barrier, compute the power update of +// their own best key into pspec (it_pu_spec). The next iteration has a pass, +// and if one of those keys wins, its owner only copies the row +// (it_pu_commit) before its pass instead of running the power update. +#ifndef IT_SPEC +#define IT_SPEC 1 +#endif +// Iterations of run 0. Fewer than CTRL_ITERS make run 0 a short warm-up of +// the caches (the products and a few iterations); run 1 is measured in full. +#ifndef CTRL_WARM_ITERS +#define CTRL_WARM_ITERS CTRL_ITERS +#endif +#define IPU_LANES 4 +// barrier slot of the active cores (slot 0 is the cluster barrier) +#define ACT_BARRIER_SLOT 1 +// U = R1 * R2, V = A * R and AR1 = A * R1 +#define MAC_PROD \ + (CTRL_N * CTRL_M * CTRL_N + CTRL_M * CTRL_N * CTRL_N + \ + CTRL_M * CTRL_N * CTRL_M) + +#define ARR(name, n) \ + static int32_t name[n] __attribute__((section(".data"), aligned(64))) + +ARR(ctrl_U, CTRL_N *CTRL_N); +ARR(ctrl_R, CTRL_N *CTRL_N); +// IT_LAYOUT 1: the L1 interleaves 64 B lines over the 16 cache controllers +// of the group (xbar offset 6), so addresses 1 KiB apart share a controller. +// A row of N = 128 words is 512 B: every row then starts at controller 0 or +// 8 and, with V and P at the same offset, the strips of all cores queue at +// the same 8 controllers. V and P go into one block with P 256 B further +// modulo 1 KiB, and the per-core buffers get a pitch of 16 words more than +// N, so they start at different controllers. +#ifndef IT_LAYOUT +#define IT_LAYOUT 1 +#endif +// IT_STAGGER 1: the odd cores run the second strip of columns first in the +// pass and in the colS rebuilds, so that half the cores stream the other +// half of the controllers (see IT_LAYOUT) +#ifndef IT_STAGGER +#define IT_STAGGER 1 +#endif +#if IT_LAYOUT +static struct { + int32_t v[CTRL_M * CTRL_N]; + int32_t pad[(1024 - (CTRL_M * CTRL_N * 4) % 1024 + 256) % 1024 / 4]; + int32_t p[CTRL_M * CTRL_N]; +} vp __attribute__((section(".data"), aligned(1024))); +#define ctrl_V vp.v +#define ctrl_P vp.p +#define IT_CP (CTRL_N + 16) +#define IT_TP (UV_VLMAX + 16) +#else +ARR(ctrl_V, CTRL_M *CTRL_N); +ARR(ctrl_P, CTRL_M *CTRL_N); +#define IT_CP CTRL_N +#define IT_TP UV_VLMAX +#endif +ARR(ctrl_G, CTRL_M *CTRL_N); +ARR(ctrl_R2, CTRL_M *CTRL_N); +ARR(ctrl_AR1, CTRL_M *CTRL_LDA_R1); +ARR(zero_row, CTRL_LDA_R1); +ARR(ctrl_ua, CTRL_M *CTRL_N); +ARR(klist, CTRL_M *CTRL_N); +ARR(nalloc, CTRL_M); +ARR(delta, CTRL_M); +// Per-core copies of ResAllocMap and of the index tables of it_cands, so +// that the cores do not all load the same lines every iteration (each core +// marks k* in its own copy; core 0's copy is checked). IT_TBL words each, +// for N <= 128 and the 3 words read past the end. +#define IT_TBL 144 +ARR(res_alloc, ACTIVE_CORES *IT_TBL); +ARR(colS, ACTIVE_CORES *IT_CP); +ARR(urows, CTRL_N); +// number of listed U rows of each core +ARR(unum, ACTIVE_CORES); +// the listed U rows of all cores, in order: the pass shares them out evenly +ARR(brows, CTRL_N); +ARR(ubnd, ACTIVE_CORES * 2 * UV_VLMAX); +ARR(kidx, ACTIVE_CORES *IT_TBL); +ARR(uidx, ACTIVE_CORES *IT_TBL); +ARR(lidx, ACTIVE_CORES *IT_TBL); +ARR(scr, ACTIVE_CORES *IT_SCR); +ARR(trk, ACTIVE_CORES *IT_TP); +ARR(winners, CTRL_N * 3); + +// One word per core, on its own cache line +typedef struct { + volatile int32_t v; + int32_t pad[15]; +} it_word_t; + +// Set by a core whose pass crossed the clamp margin, read after the barrier +// of that iteration. Passes alternate between the two words; core 0 clears +// the other word at the start of a pass (its last reads were before the +// previous barrier, its next set is a pass later). +static it_word_t uflag[2] __attribute__((section(".data"))); +// Set by a core whose late U update (IT_U_LATE) crossed the clamp margin; +// read at the next pass, never cleared within a run +static it_word_t ulate __attribute__((section(".data"))); +// s of the last pass of each core, for its late U update +ARR(sbuf, ACTIVE_CORES *IT_CP); +// speculative P row of each core (IT_SPEC) +ARR(pspec, ACTIVE_CORES *IT_CP); +// colS of row w alone, for the helper (IT_HELPER) +ARR(wcolS, ACTIVE_CORES *IT_CP); +// Best key and its column of every core, one cache line per core: every +// core reads all keys after the barrier, and with the keys on one line all +// those loads queued at one L1 bank. Iteration j writes buffer j & 1, which +// is read after its barrier and written again only after the next one. +typedef struct { + volatile int32_t key, k; + // iteration of key and k + 1 (IT_SPEC: which keys are already published) + volatile int32_t it; + int32_t pad[13]; +} it_slot_t; +static it_slot_t slot[2][ACTIVE_CORES] + __attribute__((section(".data"), aligned(64))); +static int errors[ACTIVE_CORES] __attribute__((section(".data"))); +// first mismatch of each core: matrix, offset, got, expected +static int32_t bad[ACTIVE_CORES][4] __attribute__((section(".data"))); + +// cycles of each run: products, iterations with / without an update pass, +// the last update with the finalization, total +enum { T_PROD, T_A_ODD, T_A_EVEN, T_FIN, T_TOTAL, T_NUM }; +// IT_PROF 1: cycles of each phase of the hot iterations, per core, split +// by iterations with and without an update pass, and summed separately over +// the iterations where the core owns the winner +#ifndef IT_PROF +#define IT_PROF 0 +#endif +enum { PH_PU, PH_PASS, PH_CANDS, PH_BAR, PH_NUM }; +static unsigned int prof[2][ACTIVE_CORES][PH_NUM] + __attribute__((section(".data"))); +static unsigned int prof_own[2][ACTIVE_CORES][PH_NUM + 1] + __attribute__((section(".data"))); +// the same for the iterations where the core is the helper (IT_HELPER) +static unsigned int prof_hlp[ACTIVE_CORES][PH_NUM + 1] + __attribute__((section(".data"))); +static unsigned int cycles[NUM_RUNS][T_NUM] __attribute__((section(".data"))); + +static void split(unsigned int n, unsigned int cid, unsigned int cores, + unsigned int *x0, unsigned int *x1) { + *x0 = n * cid / cores; + *x1 = n * (cid + 1) / cores; +} + +// the core whose rows of split(n, ., cores) hold row i +static unsigned int split_owner(unsigned int n, unsigned int i, + unsigned int cores) { + unsigned int c = 0; + while (c + 1 < cores && n * (c + 1) / cores <= i) + ++c; + return c; +} + +typedef struct { + uint32_t core_mask, tile_mask; +} active_mask_t; + +// Masks of the barrier of harts 0 .. ACTIVE_CORES - 1 (all in group 0) +static active_mask_t active_mask(void) { + const unsigned int cpt = snrt_cluster_core_per_tile(); + active_mask_t m = {0, 0}; + for (unsigned int h = 0; h < ACTIVE_CORES && h < cpt; ++h) + m.core_mask |= 1u << h; + const unsigned int tiles = (ACTIVE_CORES + cpt - 1) / cpt; + m.tile_mask = tiles > 1 ? (1u << tiles) - 1 : SNRT_BARRIER_TILE_MASK_SELF; + return m; +} + +static inline void abar(active_mask_t m) { + snrt_cluster_group_barrier(m.core_mask, m.tile_mask, 1, ACT_BARRIER_SLOT); +} + +static int check(int32_t *first, int32_t name, const int32_t *got, + const int32_t *exp, unsigned int i0, unsigned int i1) { + int err = 0; + for (unsigned int i = i0; i < i1; ++i) { + if (got[i] != exp[i]) { + if (err == 0 && first[0] == 0) { + first[0] = name; + first[1] = (int32_t)i; + first[2] = got[i]; + first[3] = exp[i]; + } + ++err; + } + } + return err; +} + +int main() { + const unsigned int cid = snrt_cluster_vpu_idx(); + const int active = snrt_cluster_is_primary() && cid < ACTIVE_CORES; + const active_mask_t amask = active_mask(); + unsigned int u0, u1, v0, v1, k0, k1; + split(CTRL_N, cid, ACTIVE_CORES, &u0, &u1); + split(CTRL_M, cid, ACTIVE_CORES, &v0, &v1); + split(CTRL_N, cid, ACTIVE_CORES, &k0, &k1); + int32_t *my_colS = colS + cid * IT_CP; + const unsigned int stag = IT_STAGGER ? (cid & 1) : 0; + + snrt_cluster_hw_barrier(); + if (cid == 0 && + ACTIVE_CORES > snrt_cluster_vpu_num() / snrt_cluster_group_num()) + printf("warning: %u cores span more than one group\n", ACTIVE_CORES); + if (XBAR_OFFSET) + l1d_xbar_config(XBAR_OFFSET); + l1d_part_folded(snrt_cluster_vpu_per_tile(), L1D_FOLD); + int32_t *my_ra = res_alloc + cid * IT_TBL, *my_kidx = kidx + cid * IT_TBL; + int32_t *my_uidx = uidx + cid * IT_TBL, *my_lidx = lidx + cid * IT_TBL; + if (cid == 0) + pu_init_tab(); + if (active) { + for (unsigned int i = 0; i < IT_TBL; ++i) { + my_kidx[i] = 127 - (int32_t)i; + my_uidx[i] = 127 - (int32_t)i; + my_lidx[i] = (int32_t)i; + } + snrt_fence_snitch(); + } + snrt_cluster_hw_barrier(); + +#pragma clang loop unroll(disable) + for (int run = 0; run < NUM_RUNS; ++run) { + // Restore the input state of this core's rows + if (active) { + for (unsigned int i = v0 * CTRL_N; i < v1 * CTRL_N; ++i) { + ctrl_R2[i] = ctrl_R2_in[i]; + ctrl_P[i] = CTRL_P_SCALE; + ctrl_ua[i] = 0; + } + for (unsigned int i = v0; i < v1; ++i) { + nalloc[i] = 0; + delta[i] = CTRL_P_SCALE; + } + for (unsigned int k = 0; k < IT_TBL; ++k) { + my_ra[k] = 0; + my_kidx[k] = 127 - (int32_t)k; + } + if (cid == 0) { + uflag[0].v = 0; + uflag[1].v = 0; + for (unsigned int c = 0; c < ACTIVE_CORES; ++c) { + slot[0][c].it = 0; + slot[1][c].it = 0; + } + ulate.v = 0; + } + snrt_fence_snitch(); + } + snrt_cluster_hw_barrier(); + + if (active) { + // per core, so the laps of the cores do not race; core 0 reports + unsigned int cyc[T_NUM] = {0}; + unsigned int t = benchmark_get_cycle(), t0 = t, tn; +#define LAP(slot) \ + do { \ + tn = benchmark_get_cycle(); \ + cyc[slot] += tn - t; \ + t = tn; \ + } while (0) + + // Products and the column maxima of S + uv_matmul(ctrl_U, ctrl_R, ctrl_R1, ctrl_R2, ctrl_C, CTRL_N, u0, u1, 0, + CTRL_N, CTRL_M, CTRL_LDA_R1, CTRL_N, UV_MODE_U); + // this core's U rows that can come near the clamp + unsigned int nu = it_u_rows(urows + u0, ctrl_C, ctrl_R1, u0, u1, + CTRL_M, CTRL_LDA_R1, CTRL_N, + ubnd + cid * 2 * UV_VLMAX); + unum[cid] = (int32_t)nu; + // the late U updates assume U within the margin before the first pass + if (IT_U_LATE && it_u_check(ctrl_U, urows + u0, nu, CTRL_N, + trk + cid * IT_TP)) + ulate.v = 1; + int u_exact = 0; + snrt_fence(); + abar(amask); + // V and AR1 without their clamped copies (G is recomputed from V + // where it is used, see it_g_rows) + uv_matmul(ctrl_V, ctrl_V, ctrl_A, ctrl_R, ctrl_D, 0, v0, v1, 0, CTRL_N, + CTRL_N, CTRL_LDA_A, CTRL_N, UV_MODE_RAW); + uv_matmul(ctrl_AR1, ctrl_AR1, ctrl_A, ctrl_R1, zero_row, 0, v0, v1, 0, + CTRL_M, CTRL_N, CTRL_LDA_A, CTRL_LDA_R1, UV_MODE_RAW); + it_s_rows(my_colS, ctrl_P, ctrl_V, v0, v1, CTRL_M, CTRL_N, stag); + // this core's share [b0, b1) of all the listed U rows: their number + // differs between the blocks, so the pass would be unbalanced with + // the own rows only (each core fills its own part of brows) + unsigned int nall = 0; + for (unsigned int c = 0; c < ACTIVE_CORES; ++c) + nall += (unsigned int)unum[c]; + unsigned int b0, b1; + split(nall, cid, ACTIVE_CORES, &b0, &b1); + for (unsigned int c = 0, base = 0; c < ACTIVE_CORES; ++c) { + const unsigned int n = (unsigned int)unum[c]; + unsigned int c0, c1; + split(CTRL_N, c, ACTIVE_CORES, &c0, &c1); + for (unsigned int i = 0; i < n; ++i) + if (base + i >= b0 && base + i < b1) + brows[base + i] = urows[c0 + i]; + base += n; + } + const int32_t *prows = brows + b0; + unsigned int pn = b1 - b0; + // U updated in the pass itself (else one iteration later), and a late + // U update pending for the pass won by user late_w + int u_eager = !IT_U_LATE; + int late = 0; + unsigned int late_w = 0; + // the late U updates read the shares of the other cores + snrt_fence_snitch(); + abar(amask); + LAP(T_PROD); + + // The winner w of iteration j - 1 is applied in iteration j: its power, + // and after an odd iteration the update pass, whose R2[w] flip waits + // for the next iteration (every core has read R2[w] by then). + unsigned int q = 0, flip_w = 0; + int flip = 0; + // the speculative power update of this core (IT_SPEC): user, column + // (-1 if none), Delta and P[w][k] + int spec_k = -1; + unsigned int spec_w = 0; + int32_t spec_d = 0, spec_pk = 0; + int32_t *my_spec = pspec + cid * IT_CP; + const unsigned int iters = run ? CTRL_ITERS : CTRL_WARM_ITERS; +#pragma clang loop unroll(disable) + for (unsigned int j = 0; j <= iters; ++j) { + const int pass = j > 0 && ((j - 1) & 1); + int was_owner = 0; + unsigned int pt = benchmark_get_cycle(); +#if IT_CPROF + it_cprof_on[cid] = run == NUM_RUNS - 1 && j > 0 && j < iters; +#endif +#define PROF(ph) \ + do { \ + if (IT_PROF && run == NUM_RUNS - 1 && j > 0 && j < iters) { \ + const unsigned int pn = benchmark_get_cycle(); \ + prof[pass][cid][ph] += pn - pt; \ + if (was_owner) \ + prof_own[pass][cid][ph] += pn - pt; \ + if (helper) \ + prof_hlp[cid][ph] += pn - pt; \ + pt = pn; \ + } \ + } while (0) + if (flip && cid == 0) + it_flip_r2(ctrl_R2, flip_w, CTRL_N); + flip = 0; + // roles in an iteration without a pass (IT_HELPER): this core's key + // of the previous iteration is reused, the owner leaves out row + // skip_w, the helper adds the key of row w + int reuse = 0, helper = 0, owner_now = 0; + int32_t reuse_key = -1; + int reuse_k = -1; + unsigned int skip_w = CTRL_M, row_w = 0; + if (j > 0) { + // the cores hold different users, so their keys never tie + const it_slot_t *ks = slot[(j - 1) & 1]; + int32_t kv[ACTIVE_CORES]; +#pragma clang loop unroll(full) + for (unsigned int c = 0; c < ACTIVE_CORES; ++c) + kv[c] = ks[c].key; + int32_t best = kv[0]; + unsigned int bc = 0; +#pragma clang loop unroll(full) + for (unsigned int c = 1; c < ACTIVE_CORES; ++c) + if (kv[c] > best) { + best = kv[c]; + bc = c; + } + const int bk = ks[bc].k; + const unsigned int w = 127 - (best & 127); + if (cid == 0) { + winners[3 * (j - 1)] = (int32_t)w; + winners[3 * (j - 1) + 1] = bk; + winners[3 * (j - 1) + 2] = best >> KEY_USER_BITS; + } + // every core marks k* itself (the owner's pu_update stores the + // same 1), so the selection below needs no barrier + my_ra[bk] = 1; + it_take(my_kidx, (unsigned int)bk); + const int owner = w >= v0 && w < v1; + owner_now = owner; + const unsigned int oc = split_owner(CTRL_M, w, ACTIVE_CORES); + int hc = -1; + if (IT_HELPER && !pass && ACTIVE_CORES > 1) { + for (unsigned int d = 1; d < ACTIVE_CORES; ++d) { + const unsigned int c = (oc + d) % ACTIVE_CORES; + if (ks[c].k != bk) { + hc = (int)c; + break; + } + } + reuse = !owner && ks[cid].k != bk; + reuse_key = ks[cid].key; + reuse_k = ks[cid].k; + helper = hc == (int)cid; + if (owner && hc >= 0) + skip_w = w; + row_w = w; + } + if (owner && pass && spec_k == bk && spec_w == w) { + it_pu_commit(ctrl_P + w * CTRL_N, my_spec, ctrl_ua + w * CTRL_N, + klist + w * CTRL_N, &nalloc[w], bk, CTRL_N, spec_pk); + delta[w] = spec_d; + } else if (hc >= 0 ? helper : owner) { + delta[w] = it_pu_update(ctrl_P + w * CTRL_N, ctrl_V + w * CTRL_N, + my_ra, ctrl_ua + w * CTRL_N, + klist + w * CTRL_N, &nalloc[w], bk, + CTRL_N); + } + // P[w][k] and res_alloc[k*] are scalar stores; vector loads follow + snrt_fence_snitch(); + was_owner = owner; + if (IT_PROF && owner && run == NUM_RUNS - 1 && j < iters) + prof_own[pass][cid][PH_NUM] += 1; + if (IT_PROF && helper && run == NUM_RUNS - 1 && j < iters) + prof_hlp[cid][PH_NUM] += 1; + PROF(PH_PU); + if (pass) { + // flag word q & 1; the other one was last read before the + // previous barrier + if (cid == 0) + uflag[(q & 1) ^ 1].v = 0; + // a late check crossed the margin: from now on as before + if (!u_eager && ulate.v) + u_eager = 1; + // the owner's pass reads the new P row after its pu_update + if (it_pass(ctrl_U, ctrl_V, ctrl_P, my_colS, ctrl_R1, ctrl_AR1, + ctrl_R2, w, prows, u_eager ? pn : 0, v0, v1, + CTRL_LDA_R1, CTRL_N, trk + cid * IT_TP, + u_eager ? 0 : sbuf + cid * IT_CP, stag)) + uflag[q & 1].v = 1; + late = !u_eager; + late_w = w; + flip = 1; + flip_w = w; + } else { + if (owner && j < iters) + it_s_rows(my_colS, ctrl_P, ctrl_V, v0, v1, skip_w, CTRL_N, stag); + // the late U update of the last pass, all listed rows shared + // over the cores other than the owner (U is not used after the + // last iteration) + if (late && j < iters && + ((!owner && !helper) || ACTIVE_CORES == 1)) { + // ranks of the cores other than the owner and the helper + unsigned int parts = 0, rank = 0; + for (unsigned int c = 0; c < ACTIVE_CORES; ++c) + if (c != oc && (int)c != hc) { + rank += c < cid; + ++parts; + } + if (!parts) { + parts = 1; + rank = 0; + } + unsigned int r0, r1; + split(nall * IT_U_STRIPS(CTRL_N), rank, parts, &r0, &r1); + if (it_u_late(ctrl_U, ctrl_R1, sbuf + cid * IT_CP, late_w, + brows, r0, r1, CTRL_LDA_R1, CTRL_N, + trk + cid * IT_TP)) + ulate.v = 1; + } + late = 0; + } + PROF(PH_PASS); + } + + // Candidates and exact keys of this core's rows: they only read + // this core's colS, P and V rows and res_alloc + int kb = -1; + int32_t key = -1; + if (j < iters && v1 > v0) { + if (reuse) { + key = reuse_key; + kb = reuse_k; + } else { + key = it_cands(my_colS, ctrl_P, ctrl_V, my_kidx, my_uidx, + my_lidx, v0, v1, skip_w, CTRL_N, + scr + cid * IT_SCR, &kb); + } + } + if (helper && j < iters) { + // the exact key of row w, from its new P + int32_t *wc = wcolS + cid * IT_CP; + int kw = -1; + it_s_rows(wc, ctrl_P, ctrl_V, row_w, row_w + 1, CTRL_M, CTRL_N, stag); + const int32_t keyw = + it_cands(wc, ctrl_P, ctrl_V, my_kidx, my_uidx, my_lidx, + row_w, row_w + 1, CTRL_M, CTRL_N, scr + cid * IT_SCR, + &kw); + if (keyw > key) { + key = keyw; + kb = kw; + } + } + slot[j & 1][cid].key = key; + slot[j & 1][cid].k = kb; + slot[j & 1][cid].it = (int32_t)(j + 1); + snrt_fence(); + spec_k = -1; + // only for an odd j: the next iteration has a pass. Only if no key + // published so far in this iteration is larger: the winner always + // passes, and most other cores skip the work. + int spec = IT_SPEC && (j & 1) && j < iters && key >= 0 && !owner_now && + !helper; + if (spec) { + const it_slot_t *ks = slot[j & 1]; + for (unsigned int c = 0; c < ACTIVE_CORES; ++c) + if (ks[c].it == (int32_t)(j + 1) && ks[c].key > key) + spec = 0; + } + if (spec) { + spec_w = 127 - (unsigned int)(key & 127); + spec_k = kb; + spec_d = it_pu_spec(my_spec, ctrl_P + spec_w * CTRL_N, + ctrl_V + spec_w * CTRL_N, my_ra, + klist + spec_w * CTRL_N, &nalloc[spec_w], + (unsigned int)kb, CTRL_N, &spec_pk); + } + PROF(PH_CANDS); + abar(amask); + PROF(PH_BAR); + + // Slow path (rare): a U element crossed the clamp margin in this + // pass, so fix the V columns and redo colS and the selection + if (pass && uflag[q & 1].v) { + if (!u_exact) { + // first fix-up: the skipped U rows are stale, so recompute this + // core's rows (R2 before the flip of row w, then the update of + // this pass) and update all of them from now on + uv_matmul(ctrl_U, ctrl_R, ctrl_R1, ctrl_R2, ctrl_C, CTRL_N, u0, + u1, 0, CTRL_N, CTRL_M, CTRL_LDA_R1, CTRL_N, UV_MODE_U); + nu = 0; + for (unsigned int l = u0; l < u1; ++l) + urows[u0 + nu++] = (int32_t)l; + it_u_part(ctrl_U, ctrl_R1, ctrl_R2 + flip_w * CTRL_N, flip_w, + urows + u0, nu, CTRL_LDA_R1, CTRL_N); + prows = urows + u0; + pn = nu; + snrt_fence(); + u_exact = 1; + abar(amask); + } + it_fix_cols(ctrl_V, ctrl_U, ctrl_A, ctrl_D, k0, k1, CTRL_M, CTRL_N, + CTRL_LDA_A); + abar(amask); + if (j < iters) { + it_s_rows(my_colS, ctrl_P, ctrl_V, v0, v1, CTRL_M, CTRL_N, stag); + if (v1 > v0) + key = it_cands(my_colS, ctrl_P, ctrl_V, my_kidx, + my_uidx, my_lidx, v0, v1, CTRL_M, CTRL_N, + scr + cid * IT_SCR, &kb); + slot[j & 1][cid].key = key; + slot[j & 1][cid].k = kb; + snrt_fence(); + abar(amask); + } + } + if (pass) + ++q; + if (j == iters) + break; + if (pass) + LAP(T_A_ODD); + else + LAP(T_A_EVEN); + } + + // Finalization of this core's users (with the last update above) + it_g_rows(ctrl_G, ctrl_V, v0, v1, CTRL_N); + pu_finalize(ctrl_P, ctrl_G, klist, nalloc, delta, v0, v1, CTRL_N); + snrt_fence_snitch(); + abar(amask); + LAP(T_FIN); + cyc[T_TOTAL] = t - t0; + if (cid == 0) + for (int i = 0; i < T_NUM; ++i) + cycles[run][i] = cyc[i]; +#undef LAP + } + snrt_cluster_hw_barrier(); + } + + if (cid == 0) { + // iteration j applies the winner of j - 1: an update pass for j - 1 odd + const unsigned int odd = (CTRL_ITERS - 1) / 2; + const unsigned int even = CTRL_ITERS - odd; + printf("control algorithm, M %u N %u, %u iterations, %u cores, L1 %s " + "xbar %d\n", + CTRL_M, CTRL_N, CTRL_ITERS, ACTIVE_CORES, + L1D_FOLD == L1D_FOLD_TILE ? "tile" : "group", XBAR_OFFSET); + for (int run = 0; run < NUM_RUNS; ++run) { + unsigned int *c = cycles[run]; + if (!run && CTRL_WARM_ITERS != CTRL_ITERS) { + printf("run 0 (warm-up, %u iterations): products %u cycles\n", + CTRL_WARM_ITERS, c[T_PROD]); + continue; + } + printf("run %d (%s): total %u cycles (TTI 500000)\n", run, + run ? "hot" : "first touch", c[T_TOTAL]); + printf(" products %u (IPU util %u permille), finalization %u\n", + c[T_PROD], + (unsigned int)(1000ULL * MAC_PROD / + ((unsigned long long)c[T_PROD] * IPU_LANES * + ACTIVE_CORES)), + c[T_FIN]); + printf(" per iteration: %u with an update pass, %u without\n", + odd ? c[T_A_ODD] / odd : 0, even ? c[T_A_EVEN] / even : 0); + } + if (IT_PROF) { + // hot iterations 1 .. CTRL_ITERS - 1 (iteration 0 has no winner) + const unsigned int np = (CTRL_ITERS - 1) / 2, nn = CTRL_ITERS / 2; + for (int ps = 1; ps >= 0; --ps) { + const unsigned int ni = ps ? np : nn; + printf("%s pass, cycles per iteration: core pu pass cands barrier\n", + ps ? "with" : "without"); + for (unsigned int c = 0; c < ACTIVE_CORES; ++c) + printf(" %2u %5u %5u %5u %5u\n", c, prof[ps][c][PH_PU] / ni, + prof[ps][c][PH_PASS] / ni, prof[ps][c][PH_CANDS] / ni, + prof[ps][c][PH_BAR] / ni); + unsigned int own[PH_NUM + 1] = {0}; + for (unsigned int c = 0; c < ACTIVE_CORES; ++c) + for (int h = 0; h <= PH_NUM; ++h) + own[h] += prof_own[ps][c][h]; + if (own[PH_NUM]) + printf(" owner %5u %5u %5u %5u (%u iterations)\n", + own[PH_PU] / own[PH_NUM], own[PH_PASS] / own[PH_NUM], + own[PH_CANDS] / own[PH_NUM], own[PH_BAR] / own[PH_NUM], + own[PH_NUM]); + } + unsigned int hlp[PH_NUM + 1] = {0}; + for (unsigned int c = 0; c < ACTIVE_CORES; ++c) + for (int h = 0; h <= PH_NUM; ++h) + hlp[h] += prof_hlp[c][h]; + if (hlp[PH_NUM]) + printf(" helper %5u %5u %5u %5u (%u iterations)\n", + hlp[PH_PU] / hlp[PH_NUM], hlp[PH_PASS] / hlp[PH_NUM], + hlp[PH_CANDS] / hlp[PH_NUM], hlp[PH_BAR] / hlp[PH_NUM], + hlp[PH_NUM]); + } +#if IT_CPROF + printf("it_cands per call: core calls top2 ckeys exact cols%% single%%\n"); + for (unsigned int c = 0; c < ACTIVE_CORES; ++c) { + const unsigned int *cp = it_cprof[c], n = cp[CP_CALLS] ? cp[CP_CALLS] : 1; + printf(" %2u %4u %5u %5u %5u %4u %4u\n", c, cp[CP_CALLS], + cp[CP_BEST] / n, cp[CP_CKEY] / n, cp[CP_EXACT] / n, + 100 * cp[CP_COLS] / n, 100 * cp[CP_SINGLE] / n); + } +#endif + } + + // Each active core checks its rows; core 0 also the global results + if (active) { + int32_t *b = bad[cid]; + int err = check(b, 'P', ctrl_P, ctrl_P_exp, v0 * CTRL_N, v1 * CTRL_N); + err += check(b, 'U', ctrl_ua, ctrl_ua_exp, v0 * CTRL_N, v1 * CTRL_N); + if (cid == 0) { + err += check(b, 'A', res_alloc, ctrl_res_exp, 0, CTRL_N); + err += check(b, 'W', winners, ctrl_win_exp, 0, 3 * CTRL_ITERS); + } + errors[cid] = err; + snrt_fence_snitch(); + } + snrt_cluster_hw_barrier(); + + int total = 0; + if (cid == 0) { + for (unsigned int c = 0; c < ACTIVE_CORES; ++c) { + if (errors[c]) { + const int32_t o = bad[c][1]; + if (bad[c][0] == 'W') + printf("core %u: %d mismatches, first winner of iteration %d " + "(field %d) = %d, expected %d\n", + c, errors[c], o / 3, o % 3, bad[c][2], bad[c][3]); + else + printf("core %u: %d mismatches, first %c[%d] = %d, expected %d " + "(U = UserResAllocMap, A = ResAllocMap)\n", + c, errors[c], (char)bad[c][0], o, bad[c][2], bad[c][3]); + } + total += errors[c]; + } + printf(total ? "Check Failed!\n" : "Check passed.\n"); + } + snrt_cluster_hw_barrier(); + return total ? -1 : 0; +} diff --git a/software/control/full/script/data_63_37.json b/software/control/full/script/data_63_37.json new file mode 100644 index 00000000..551f76d1 --- /dev/null +++ b/software/control/full/script/data_63_37.json @@ -0,0 +1,6 @@ +// Rows and columns that do not split evenly +{ + M: 63, + N: 37, + seed: 2 +} diff --git a/software/control/full/script/data_63_37_mix.json b/software/control/full/script/data_63_37_mix.json new file mode 100644 index 00000000..b2202e73 --- /dev/null +++ b/software/control/full/script/data_63_37_mix.json @@ -0,0 +1,8 @@ +// Odd sizes with clamping in half of the U rows: the update pass skips the +// others, and the column fix-up rebuilds their U +{ + M: 63, + N: 37, + seed: 2, + init: "mix" +} diff --git a/software/control/full/script/data_64_128.json b/software/control/full/script/data_64_128.json new file mode 100644 index 00000000..be252632 --- /dev/null +++ b/software/control/full/script/data_64_128.json @@ -0,0 +1,6 @@ +// Benchmark size: M = 64 users, N = 128 subcarriers (see ../../README.md) +{ + M: 64, + N: 128, + seed: 1 +} diff --git a/software/control/full/script/data_64_128_i16w.json b/software/control/full/script/data_64_128_i16w.json new file mode 100644 index 00000000..5570a74f --- /dev/null +++ b/software/control/full/script/data_64_128_i16w.json @@ -0,0 +1,9 @@ +// Benchmark size M = 64, N = 128 with the first 16 iterations, and a short +// warm-up run (products and 2 iterations) before the measured run +{ + M: 64, + N: 128, + seed: 1, + iters: 16, + warm_iters: 2 +} diff --git a/software/control/full/script/data_64_128_i32w.json b/software/control/full/script/data_64_128_i32w.json new file mode 100644 index 00000000..a4e72d6f --- /dev/null +++ b/software/control/full/script/data_64_128_i32w.json @@ -0,0 +1,9 @@ +// Benchmark size M = 64, N = 128 with the first 32 iterations, and a short +// warm-up run (products and 2 iterations) before the measured run +{ + M: 64, + N: 128, + seed: 1, + iters: 32, + warm_iters: 2 +} diff --git a/software/control/full/script/data_64_128_i8w.json b/software/control/full/script/data_64_128_i8w.json new file mode 100644 index 00000000..918d9187 --- /dev/null +++ b/software/control/full/script/data_64_128_i8w.json @@ -0,0 +1,9 @@ +// Benchmark size M = 64, N = 128 with the first 8 iterations, and a short +// warm-up run (products and 2 iterations) before the measured run +{ + M: 64, + N: 128, + seed: 1, + iters: 8, + warm_iters: 2 +} diff --git a/software/control/full/script/data_9_21.json b/software/control/full/script/data_9_21.json new file mode 100644 index 00000000..b4ee175d --- /dev/null +++ b/software/control/full/script/data_9_21.json @@ -0,0 +1,6 @@ +// Small case for quick runs +{ + M: 9, + N: 21, + seed: 1 +} diff --git a/software/control/full/script/gen_data.py b/software/control/full/script/gen_data.py new file mode 100644 index 00000000..04f7a9db --- /dev/null +++ b/software/control/full/script/gen_data.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +# Copyright 2025 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + +# Generates data/.h for the full control algorithm: the inputs (R1 and A +# with the padded row pitch of uv-init) and the expected P, ResAllocMap, +# UserResAllocMap and winner of every iteration from common/golden.py. + +import argparse +import pathlib +import sys + +import hjson +import numpy as np + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[2] / "common")) +import golden # noqa: E402 + +LINE = 16 + + +def pitch(k): + lines = (k + LINE - 1) // LINE + return LINE * (lines if lines % 2 else lines + 1) + + +def padded(a, lda): + out = np.zeros((a.shape[0], lda), dtype=np.int64) + out[:, :a.shape[1]] = a + return out + + +def c_array(name, a, section=".data"): + vals = ", ".join(str(int(v)) for v in np.asarray(a).flat) + return (f'static int32_t {name}[{np.asarray(a).size}] ' + f'__attribute__((section("{section}"), aligned(64))) = {{{vals}}};\n\n') + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("-c", "--cfg", type=pathlib.Path, required=True) + args = parser.parse_args() + cfg = hjson.loads(args.cfg.read_text()) + m, n = cfg["M"], cfg["N"] + assert m <= 64 and n <= 128 + + inp = golden.gen_inputs(m, n, cfg.get("seed", 1)) + if cfg.get("init", "random") == "mix": + # every other row of R1 from [30, 50] (still inside [-50, 50]): those + # U rows grow into the clamp, the others stay clear of it, so both + # the skipped rows and the column fix-up are exercised + rng = np.random.default_rng(cfg.get("seed", 1) + 1000) + inp["R1"][0::2] = rng.integers(30, 51, size=inp["R1"][0::2].shape) + ref = golden.control_algo(**inp, iters=cfg.get("iters")) + win = np.zeros((n, 3), dtype=np.int64) + for j, (w, k, gain) in enumerate(ref["winners"]): + win[j] = (w, k, gain) + + out = ("// Copyright 2025 ETH Zurich and University of Bologna.\n" + "// Licensed under the Apache License, Version 2.0, see LICENSE for details.\n" + "// SPDX-License-Identifier: Apache-2.0\n\n" + "// This file was generated automatically by script/gen_data.py.\n\n" + "#include \n\n" + f"#define CTRL_M {m}\n#define CTRL_N {n}\n" + f"#define CTRL_LDA_R1 {pitch(m)}\n#define CTRL_LDA_A {pitch(n)}\n" + f"#define CTRL_ITERS {len(ref['winners'])}\n") + if "warm_iters" in cfg: + out += f"#define CTRL_WARM_ITERS {cfg['warm_iters']}\n" + out += "\n" + out += c_array("ctrl_R1", padded(inp["R1"], pitch(m))) + out += c_array("ctrl_A", padded(inp["A"], pitch(n))) + out += c_array("ctrl_R2_in", inp["R2"]) + for key in ["C", "D"]: + out += c_array(f"ctrl_{key}", inp[key]) + out += c_array("ctrl_P_exp", ref["P"]) + out += c_array("ctrl_res_exp", ref["ResAllocMap"]) + out += c_array("ctrl_ua_exp", ref["UserResAllocMap"]) + out += c_array("ctrl_win_exp", win) + + dst = pathlib.Path(__file__).resolve().parents[1] / "data" / (args.cfg.stem + ".h") + dst.write_text(out) + + +if __name__ == "__main__": + main() diff --git a/software/control/gain-select/data/.gitignore b/software/control/gain-select/data/.gitignore new file mode 100644 index 00000000..226b8c26 --- /dev/null +++ b/software/control/gain-select/data/.gitignore @@ -0,0 +1,2 @@ +# Generated data headers — reproduced by `make gen-data` +*.h diff --git a/software/control/gain-select/kernel/gain_select.c b/software/control/gain-select/kernel/gain_select.c new file mode 100644 index 00000000..1a35619b --- /dev/null +++ b/software/control/gain-select/kernel/gain_select.c @@ -0,0 +1,175 @@ +// Copyright 2025 ETH Zurich and University of Bologna. +// Licensed under the Apache License, Version 2.0, see LICENSE for details. +// SPDX-License-Identifier: Apache-2.0 + +// Gain and selection kernel of the control algorithm (see ../../README.md): +// +// Gain[i][k] = ln(1 + P[i][k] * G[i][k]) (Q16, integer, as ilog in +// common/golden.py) +// (w, k*) = the user and free subcarrier with the largest Gain; ties go +// to the lowest user, then the lowest subcarrier +// +// ln is evaluated with 32-bit integer vector operations: the exponent and +// mantissa of 1 + S come from the bits of its float conversion, ln of the +// mantissa from a degree-3 Taylor series around 1.5 in Q30 (Horner with +// vmulh). No precision is specified; the error is below 4.3e-3 in ln. +// +// The selection needs neither masks nor reductions: every row's Gain is +// packed with its user into key = Gain << 7 | (127 - i), and an element-wise +// vmax over rows keeps the best key of every column (largest Gain, lowest +// user). Allocation is per column: subtracting ResAllocMap << 30 pushes the +// allocated columns below every valid key, and a scalar scan over the column +// keys picks the winner. + +#include + +#include "control.h" + +#define LOG_LN15_Q30 435364845 +#define LOG_LN2_Q24 11629080 +#define LOG_TWO_THIRDS_U32 2863311531u +// rounding and the -127 exponent bias folded into one constant: +// 128 - 127 * LN2_Q24 +#define LOG_BIAS (128 - 127 * LOG_LN2_Q24) +// ln(1 + u) = u * (C0 + C1 u + C2 u^2), Taylor coefficients 1, -1/2, 1/3 in +// Q30 +#define LOG_C0 1073741824 +#define LOG_C1 (-536870912) +#define LOG_C2 357913941 + +#define KEY_USER_BITS 7 + +// One Horner step in v12: p = c + mulh(u, p), u in vu +#define LOG_HORNER(vu, c) \ + do { \ + asm volatile("vmulh.vv v12, " vu ", v12"); \ + asm volatile("vadd.vx v12, v12, %0" ::"r"(c)); \ + } while (0) + +// Gain of one row strip from P in vp and G in vg (both clobbered), into v12 +// and stored to `gain`; folds the row's keys into the column-best register +// group `acc`. v8 and v12 are scratch. +#define GAIN_CORE_R(vp, vg, acc, gain, ukey) \ + do { \ + /* x = P * G + 1 and its float bits */ \ + asm volatile("vmul.vv " vp ", " vp ", " vg); \ + asm volatile("vadd.vi " vp ", " vp ", 1"); \ + asm volatile("vfcvt.f.xu.v " vg ", " vp); \ + /* v8 = biased exponent, vg = u = (m - 1.5) * 2/3 in Q32 */ \ + asm volatile("vsrl.vi v8, " vg ", 23"); \ + asm volatile("vand.vx " vg ", " vg ", %0" ::"r"(0x7FFFFF)); \ + asm volatile("vsll.vi " vg ", " vg ", 9"); \ + asm volatile("vadd.vx " vg ", " vg ", %0" ::"r"(INT32_MIN)); \ + asm volatile("vmulhsu.vx " vg ", " vg ", %0" ::"r"(LOG_TWO_THIRDS_U32)); \ + /* v12 = ln(1 + u) / u, then ln m, in Q30 */ \ + asm volatile("vmulh.vx v12, " vg ", %0" ::"r"(LOG_C2)); \ + asm volatile("vadd.vx v12, v12, %0" ::"r"(LOG_C1)); \ + LOG_HORNER(vg, LOG_C0); \ + LOG_HORNER(vg, LOG_LN15_Q30); \ + /* Gain = (e * ln 2 in Q24 + ln m in Q24 + 128) >> 8 */ \ + asm volatile("vsra.vi v12, v12, 6"); \ + asm volatile("vmacc.vx v12, %0, v8" ::"r"(LOG_LN2_Q24)); \ + asm volatile("vadd.vx v12, v12, %0" ::"r"(LOG_BIAS)); \ + asm volatile("vsra.vi v12, v12, 8"); \ + asm volatile("vse32.v v12, (%0)" ::"r"(gain) : "memory"); \ + /* key = Gain << 7 | (127 - i) */ \ + asm volatile("vsll.vi v12, v12, %0" ::"i"(KEY_USER_BITS)); \ + asm volatile("vadd.vx v12, v12, %0" ::"r"(ukey)); \ + asm volatile("vmax.vv " acc ", " acc ", v12"); \ + } while (0) + +#define GAIN_CORE(acc, gain, ukey) GAIN_CORE_R("v0", "v4", acc, gain, ukey) + +#define GAIN_ROW(acc, prow, grow, gain, ukey) \ + do { \ + asm volatile("vle32.v v0, (%0)" ::"r"(prow)); \ + asm volatile("vle32.v v4, (%0)" ::"r"(grow)); \ + GAIN_CORE(acc, gain, ukey); \ + } while (0) + +static inline unsigned int gs_vl(unsigned int n) { + unsigned int avl = n < UV_VLMAX ? n : UV_VLMAX; + unsigned int vl; + asm volatile("vsetvli %0, %1, e32, m4, ta, ma" : "=r"(vl) : "r"(avl)); + return vl; +} + +// Gain of rows [r0, r1) and their column-best keys, stored to colkey[N] +// (N <= 128: the two strips use v16 and v20) +void gain_rows(int32_t *gain, int32_t *colkey, const int32_t *P, + const int32_t *G, unsigned int r0, unsigned int r1, + unsigned int N) { + for (unsigned int p = 0, j = 0; p < N; ++j) { + unsigned int vl = gs_vl(N - p); + if (j == 0) { + asm volatile("vxor.vv v16, v16, v16"); + asm volatile("vadd.vi v16, v16, -1"); + } else { + asm volatile("vxor.vv v20, v20, v20"); + asm volatile("vadd.vi v20, v20, -1"); + } + for (unsigned int i = r0; i < r1; ++i) { + const unsigned int o = i * N + p; + const int32_t ukey = (1 << KEY_USER_BITS) - 1 - i; + if (j == 0) + GAIN_ROW("v16", P + o, G + o, gain + o, ukey); + else + GAIN_ROW("v20", P + o, G + o, gain + o, ukey); + } + if (j == 0) + asm volatile("vse32.v v16, (%0)" ::"r"(colkey + p) : "memory"); + else + asm volatile("vse32.v v20, (%0)" ::"r"(colkey + p) : "memory"); + p += vl; + } +} + +// Merge the column keys of `cores` cores (rows of colkey, stride N) with +// vmax and push allocated columns below every valid key (key - 2^30 < 0), +// then scan. Returns the best key and its column in *k_best (-1 if every +// subcarrier is allocated). The scan keeps GS_SCAN loads in flight, as a +// dependent scalar load costs at least the L1 latency. +#define GS_SCAN 8 +int32_t gain_select(int32_t *colkey, const int32_t *res_alloc, + unsigned int cores, unsigned int N, int *k_best) { + for (unsigned int p = 0; p < N;) { + unsigned int vl = gs_vl(N - p); + asm volatile("vle32.v v16, (%0)" ::"r"(colkey + p)); + for (unsigned int c = 1; c < cores; ++c) { + asm volatile("vle32.v v20, (%0)" ::"r"(colkey + c * N + p)); + asm volatile("vmax.vv v16, v16, v20"); + } + asm volatile("vle32.v v20, (%0)" ::"r"(res_alloc + p)); + asm volatile("vsll.vi v20, v20, 30"); + asm volatile("vsub.vv v16, v16, v20"); + asm volatile("vse32.v v16, (%0)" ::"r"(colkey + p) : "memory"); + p += vl; + } + snrt_fence_spatz(); + + // strict > in increasing k keeps the lowest column on ties + int32_t best = -1; + int kb = -1; + unsigned int k = 0; + for (; k + GS_SCAN <= N; k += GS_SCAN) { + int32_t key[GS_SCAN]; +#pragma clang loop unroll(full) + for (unsigned int j = 0; j < GS_SCAN; ++j) + key[j] = ((volatile int32_t *)colkey)[k + j]; +#pragma clang loop unroll(full) + for (unsigned int j = 0; j < GS_SCAN; ++j) { + if (key[j] > best) { + best = key[j]; + kb = k + j; + } + } + } + for (; k < N; ++k) { + if (colkey[k] > best) { + best = colkey[k]; + kb = k; + } + } + *k_best = kb; + return best; +} diff --git a/software/control/gain-select/main.c b/software/control/gain-select/main.c new file mode 100644 index 00000000..89ad3f99 --- /dev/null +++ b/software/control/gain-select/main.c @@ -0,0 +1,139 @@ +// Copyright 2025 ETH Zurich and University of Bologna. +// Licensed under the Apache License, Version 2.0, see LICENSE for details. +// SPDX-License-Identifier: Apache-2.0 + +// Gain and selection kernel of the control algorithm, on the state of one +// iteration: Gain = ln(1 + P * G) for all M x N elements, then the best user +// and free subcarrier. +// +// 1. ACTIVE_CORES cores split the M user rows; each computes the Gain of +// its rows and the best key of every column, then a cluster barrier. +// 2. Core 0 merges the cores' column keys and scans the free columns. +// +// Runs NUM_RUNS times (cold, then hot L1). All cores of the cluster check the Gain +// matrix against the golden model; core 0 checks the winner. +// +// IPU utilization of phase 1 = element operations / (cycles * 4 * cores), +// with GAIN_OPS vector operations per element. + +#include +#include +#include + +#include DATAHEADER +#include "kernel/gain_select.c" + +#ifndef ACTIVE_CORES +#define ACTIVE_CORES 1 +#endif + +#define NUM_RUNS 3 +#define MAX_CORES 1024 +#define IPU_LANES 4 +// vmul, vadd, vfcvt, vsrl, vand, vsll, vadd, vmulhsu, 3 x (vmulh, vadd), +// vsra, vmacc, vadd, vsra, vsll, vadd, vmax +#define GAIN_OPS 21 + +static int32_t colkey[ACTIVE_CORES][CTRL_N] + __attribute__((section(".data"), aligned(64))); +static int errors[MAX_CORES] __attribute__((section(".data"))); +static unsigned int cycles[NUM_RUNS][2] __attribute__((section(".data"))); + +static void split(unsigned int n, unsigned int cid, unsigned int cores, + unsigned int *x0, unsigned int *x1) { + *x0 = n * cid / cores; + *x1 = n * (cid + 1) / cores; +} + +static unsigned int util(unsigned int ops, unsigned int cycles, + unsigned int cores) { + return (unsigned int)(1000ULL * ops / + ((unsigned long long)cycles * IPU_LANES * cores)); +} + +int main() { + const unsigned int cid = snrt_cluster_vpu_idx(); + const unsigned int num_cores = snrt_cluster_vpu_num(); + const int active = snrt_cluster_is_primary() && cid < ACTIVE_CORES; + unsigned int r0, r1; + split(CTRL_M, cid, ACTIVE_CORES, &r0, &r1); + int w = -1, k = -1, best = -1; + + // Not unrolled: every run executes the same (warm) instructions +#pragma clang loop unroll(disable) + for (int run = 0; run < NUM_RUNS; ++run) { + snrt_cluster_hw_barrier(); + unsigned int t0 = benchmark_get_cycle(); + + // Phase 1: Gain of this core's rows and its column-best keys + if (active) { + gain_rows(ctrl_gain, colkey[cid], ctrl_P, ctrl_G, r0, r1, CTRL_N); + snrt_fence_spatz(); + } + snrt_cluster_hw_barrier(); + unsigned int t1 = benchmark_get_cycle(); + + // Phase 2: merge the column keys and pick the winner + if (cid == 0) { + int32_t key = gain_select(&colkey[0][0], ctrl_res_alloc, ACTIVE_CORES, + CTRL_N, &k); + best = key >> KEY_USER_BITS; + w = (1 << KEY_USER_BITS) - 1 - (key & ((1 << KEY_USER_BITS) - 1)); + unsigned int t2 = benchmark_get_cycle(); + cycles[run][0] = t1 - t0; + cycles[run][1] = t2 - t1; + } + // keep the other cores (and the check below) off the memory system + // while core 0 selects + snrt_cluster_hw_barrier(); + } + + // Printed after all runs, so that no run starts while the UART drains + if (cid == 0) { + for (int run = 0; run < NUM_RUNS; ++run) { + printf("run %d (%s): %u cores, M %u N %u\n", run, run ? "hot" : "cold", + ACTIVE_CORES, CTRL_M, CTRL_N); + printf(" gain %u cycles, IPU util %u permille\n", cycles[run][0], + util(GAIN_OPS * CTRL_M * CTRL_N, cycles[run][0], ACTIVE_CORES)); + printf(" select %u cycles, total %u cycles\n", cycles[run][1], + cycles[run][0] + cycles[run][1]); + } + } + + // Every core of the cluster checks a slice of the Gain matrix + if (snrt_cluster_is_primary()) { + unsigned int i0, i1; + split(CTRL_M * CTRL_N, cid, num_cores, &i0, &i1); + int err = 0; + for (unsigned int i = i0; i < i1; ++i) { + if (ctrl_gain[i] != ctrl_gain_exp[i]) { + if (err == 0) + printf("Gain[%u][%u] = %d, expected %d\n", i / CTRL_N, i % CTRL_N, + ctrl_gain[i], ctrl_gain_exp[i]); + ++err; + } + } + errors[cid] = err; + snrt_fence_snitch(); + } + snrt_cluster_hw_barrier(); + + int total = 0; + if (cid == 0) { + for (unsigned int c = 0; c < num_cores; ++c) { + if (errors[c]) + printf("core %u: %d Gain mismatches\n", c, errors[c]); + total += errors[c]; + } + if (w != CTRL_W_EXP || k != CTRL_K_EXP || best != CTRL_GAIN_EXP) { + printf("winner user %d subcarrier %d gain %d, expected %d %d %d\n", w, + k, best, CTRL_W_EXP, CTRL_K_EXP, CTRL_GAIN_EXP); + ++total; + } else { + printf("winner user %d subcarrier %d gain %d\n", w, k, best); + } + printf(total ? "Check Failed!\n" : "Check passed.\n"); + } + snrt_cluster_hw_barrier(); + return total ? -1 : 0; +} diff --git a/software/control/gain-select/script/data_63_37.json b/software/control/gain-select/script/data_63_37.json new file mode 100644 index 00000000..d6f41bc9 --- /dev/null +++ b/software/control/gain-select/script/data_63_37.json @@ -0,0 +1,7 @@ +// Odd sizes: partial strip, rows not divisible by the core count +{ + M: 63, + N: 37, + J: 12, + seed: 2 +} diff --git a/software/control/gain-select/script/data_64_128.json b/software/control/gain-select/script/data_64_128.json new file mode 100644 index 00000000..42ced0a0 --- /dev/null +++ b/software/control/gain-select/script/data_64_128.json @@ -0,0 +1,7 @@ +// Benchmark size (see ../../README.md), state at the start of iteration 40 +{ + M: 64, + N: 128, + J: 40, + seed: 1 +} diff --git a/software/control/gain-select/script/gen_data.py b/software/control/gain-select/script/gen_data.py new file mode 100644 index 00000000..b03e0ec7 --- /dev/null +++ b/software/control/gain-select/script/gen_data.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +# Copyright 2025 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + +# Generates data/.h for the gain-select kernel: the state at the start +# of iteration J of the golden algorithm (P, the current G and ResAllocMap), +# the expected Gain = ln(1 + P * G) in Q16 and the expected winner. + +import argparse +import pathlib +import sys + +import hjson +import numpy as np + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[2] / "common")) +import golden # noqa: E402 + + +def c_array(name, a): + a = np.asarray(a) + vals = ", ".join(str(int(v)) for v in a.flat) + return (f'static int32_t {name}[{a.size}] ' + f'__attribute__((section(".data"), aligned(64))) = {{{vals}}};\n\n') + + +def state_at(inp, iters): + """P, G and ResAllocMap at the start of iteration `iters`.""" + n, m = inp["R1"].shape + R2 = inp["R2"].copy() + P = np.full((m, n), golden.P_INIT, dtype=np.int64) + res_alloc = np.zeros(n, dtype=np.int64) + user_alloc = np.zeros((m, n), dtype=np.int64) + delta = [golden.P_SCALE] * m + for j in range(iters): + G = golden.calc_g(inp["A"], golden.calc_r(inp["R1"], R2, inp["C"]), inp["D"]) + _, w, k = golden.select(P, G, res_alloc) + golden.update_winner(P, G, res_alloc, user_alloc, delta, w, k) + R2[w] = (R2[w] + j) % 2 + G = golden.calc_g(inp["A"], golden.calc_r(inp["R1"], R2, inp["C"]), inp["D"]) + return P, G, res_alloc + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("-c", "--cfg", type=pathlib.Path, required=True) + args = parser.parse_args() + cfg = hjson.loads(args.cfg.read_text()) + m, n = cfg["M"], cfg["N"] + assert m <= 128 and n <= 128, "7-bit user field in the key, two vector strips" + + P, G, res_alloc = state_at(golden.gen_inputs(m, n, cfg.get("seed", 1)), cfg["J"]) + gain = golden.ilog(P * G) + best, w, k = golden.select(P, G, res_alloc) + print(f"{args.cfg.stem}: J {cfg['J']}, {int(res_alloc.sum())} allocated, " + f"winner user {w} subcarrier {k} gain {best}") + + out = ("// Copyright 2025 ETH Zurich and University of Bologna.\n" + "// Licensed under the Apache License, Version 2.0, see LICENSE for details.\n" + "// SPDX-License-Identifier: Apache-2.0\n\n" + "// This file was generated automatically by script/gen_data.py.\n\n" + "#include \n\n" + f"#define CTRL_M {m}\n#define CTRL_N {n}\n\n" + f"#define CTRL_W_EXP {w}\n#define CTRL_K_EXP {k}\n#define CTRL_GAIN_EXP {best}\n\n") + out += c_array("ctrl_P", P) + out += c_array("ctrl_G", G) + out += c_array("ctrl_res_alloc", res_alloc) + out += c_array("ctrl_gain", np.zeros_like(gain)) + out += c_array("ctrl_gain_exp", gain) + + dst = pathlib.Path(__file__).resolve().parents[1] / "data" / (args.cfg.stem + ".h") + dst.write_text(out) + + +if __name__ == "__main__": + main() diff --git a/software/control/power-update/data/.gitignore b/software/control/power-update/data/.gitignore new file mode 100644 index 00000000..226b8c26 --- /dev/null +++ b/software/control/power-update/data/.gitignore @@ -0,0 +1,2 @@ +# Generated data headers — reproduced by `make gen-data` +*.h diff --git a/software/control/power-update/kernel/power_update.c b/software/control/power-update/kernel/power_update.c new file mode 100644 index 00000000..2a390a38 --- /dev/null +++ b/software/control/power-update/kernel/power_update.c @@ -0,0 +1,126 @@ +// Copyright 2025 ETH Zurich and University of Bologna. +// Licensed under the Apache License, Version 2.0, see LICENSE for details. +// SPDX-License-Identifier: Apache-2.0 + +// Power update of the control algorithm (see ../../README.md), integer and +// scaled by 100. After user w wins subcarrier k*, with n its subcarrier +// count and inv(g) = 100 / g: +// +// sum = 100 + sum of inv(G[w][k]) over the subcarriers of w +// Delta[w] = sum / n +// P[w][k*] = max(0, Delta[w] - inv(G[w][k*])) +// P[w][k] = max(0, (sum + inv(G[w][k])) / (n + 1) - inv(G[w][k])) free k +// +// No division in the vector part: inv(g) is 0 for g > 100, so it is a +// lookup in pu_inv_tab with index min(g, 101); dividing by n + 1 is a +// vmulhu by m = (2^32 - 1) / (n + 1) + 1, exact for numerators up to +// 100 + 128 * 100 + 100 and n + 1 <= 129. Allocated columns keep their P +// through a blend with ResAllocMap instead of a masked store. +// +// The sum runs over the user's subcarrier list (klist, nalloc) in scalar +// code: users hold few subcarriers, so this avoids a vector reduction. + +#include + +#include "control.h" + +#define PU_TAB_SIZE 102 + +// pu_inv_tab[g] = 100 / g for 1 <= g <= 100, 0 at 0 and 101 +static int32_t pu_inv_tab[PU_TAB_SIZE] + __attribute__((section(".data"), aligned(64))); + +void pu_init_tab(void) { + pu_inv_tab[0] = 0; + for (int g = 1; g <= 100; ++g) + pu_inv_tab[g] = CTRL_P_SCALE / g; + pu_inv_tab[101] = 0; +} + +static inline int32_t pu_inv(int32_t g) { + return g <= 100 ? pu_inv_tab[g] : 0; +} + +static inline unsigned int pu_vl(unsigned int n) { + unsigned int avl = n < UV_VLMAX ? n : UV_VLMAX; + unsigned int vl; + asm volatile("vsetvli %0, %1, e32, m4, ta, ma" : "=r"(vl) : "r"(avl)); + return vl; +} + +// Update after user w wins subcarrier k. p_row, g_row, ua_row and klist are +// row w of P, G, UserResAllocMap and of the subcarrier lists; nalloc points +// to w's count. Returns the new Delta[w]. +int32_t pu_update(int32_t *p_row, const int32_t *g_row, int32_t *res_alloc, + int32_t *ua_row, int32_t *klist, int32_t *nalloc, + unsigned int k, unsigned int N) { + res_alloc[k] = 1; + ua_row[k] = 1; + const int32_t n = *nalloc + 1; + klist[n - 1] = k; + *nalloc = n; + // the vector loads below must see ResAllocMap[k] = 1 + snrt_fence_snitch(); + + int32_t sum = CTRL_P_SCALE; + for (int32_t j = 0; j < n; ++j) + sum += pu_inv(g_row[klist[j]]); + const int32_t delta = sum / n; + const uint32_t m = 0xFFFFFFFFu / (uint32_t)(n + 1) + 1; + + for (unsigned int p = 0; p < N;) { + unsigned int vl = pu_vl(N - p); + // v4 = inv(g). Spatz hangs on vloxei32 with vl % 4 != 0, so the gather + // runs on vl rounded up to 4 (up to 3 words past the row are read, only + // as clamped table indices) and the rest on vl. + if (vl & 3) + pu_vl((vl + 3) & ~3u); + asm volatile("vle32.v v0, (%0)" ::"r"(g_row + p)); + asm volatile("vminu.vx v0, v0, %0" ::"r"(PU_TAB_SIZE - 1)); + asm volatile("vsll.vi v0, v0, 2"); + asm volatile("vloxei32.v v4, (%0), v0" ::"r"(pu_inv_tab)); + if (vl & 3) + pu_vl(vl); + // v8 = max(0, (sum + inv) / (n + 1) - inv) + asm volatile("vadd.vx v8, v4, %0" ::"r"(sum)); + asm volatile("vmulhu.vx v8, v8, %0" ::"r"(m)); + asm volatile("vsub.vv v8, v8, v4"); + asm volatile("vmax.vx v8, v8, zero"); + // keep P where allocated: v8 += alloc * (P - v8) + asm volatile("vle32.v v12, (%0)" ::"r"(res_alloc + p)); + asm volatile("vle32.v v16, (%0)" ::"r"(p_row + p)); + asm volatile("vsub.vv v16, v16, v8"); + asm volatile("vmacc.vv v8, v12, v16"); + asm volatile("vse32.v v8, (%0)" ::"r"(p_row + p) : "memory"); + p += vl; + } + // P[w][k] after the vector store of the same row + snrt_fence_spatz(); + int32_t pk = delta - pu_inv(g_row[k]); + p_row[k] = pk > 0 ? pk : 0; + return delta; +} + +// Finalization of users [i0, i1): Delta[i] = sum_i / n_i and +// P[i][k] = max(0, Delta[i] - inv(G[i][k])) for the subcarriers of i. +// Users without subcarriers keep their P row and Delta. +void pu_finalize(int32_t *P, const int32_t *G, const int32_t *klist, + const int32_t *nalloc, int32_t *delta, unsigned int i0, + unsigned int i1, unsigned int N) { + for (unsigned int i = i0; i < i1; ++i) { + const int32_t n = nalloc[i]; + if (n == 0) + continue; + const int32_t *kl = klist + i * N; + const int32_t *g = G + i * N; + int32_t sum = CTRL_P_SCALE; + for (int32_t j = 0; j < n; ++j) + sum += pu_inv(g[kl[j]]); + const int32_t d = sum / n; + delta[i] = d; + for (int32_t j = 0; j < n; ++j) { + int32_t pk = d - pu_inv(g[kl[j]]); + P[i * N + kl[j]] = pk > 0 ? pk : 0; + } + } +} diff --git a/software/control/power-update/main.c b/software/control/power-update/main.c new file mode 100644 index 00000000..5dd163f7 --- /dev/null +++ b/software/control/power-update/main.c @@ -0,0 +1,114 @@ +// Copyright 2025 ETH Zurich and University of Bologna. +// Licensed under the Apache License, Version 2.0, see LICENSE for details. +// SPDX-License-Identifier: Apache-2.0 + +// Power-update kernel of the control algorithm, on one core (it touches one +// row of N elements per iteration, less work than a barrier): +// +// - CTRL_CASES updates taken from a golden run at early, middle and late +// iterations. Each case restores its input state outside the timed +// region and runs NUM_RUNS times; the last run is hot. +// - The finalization of all M users on the final state. +// +// Both are checked against the golden model. + +#include +#include +#include + +#include DATAHEADER +#include "kernel/power_update.c" + +#define NUM_RUNS 2 + +static int32_t p_row[CTRL_N] __attribute__((section(".data"), aligned(64))); +static int32_t res_alloc[CTRL_N] __attribute__((section(".data"), aligned(64))); +static int32_t ua_row[CTRL_N] __attribute__((section(".data"), aligned(64))); +static int32_t klist[CTRL_N] __attribute__((section(".data"), aligned(64))); +static int32_t delta_fin[CTRL_M] __attribute__((section(".data"))); + +static void copy(int32_t *dst, const int32_t *src, unsigned int n) { + for (unsigned int i = 0; i < n; ++i) + dst[i] = src[i]; +} + +static int check(const char *name, const int32_t *got, const int32_t *exp, + unsigned int n) { + int err = 0; + for (unsigned int i = 0; i < n; ++i) { + if (got[i] != exp[i]) { + if (err == 0) + printf("%s[%u] = %d, expected %d\n", name, i, got[i], exp[i]); + ++err; + } + } + return err; +} + +int main() { + const unsigned int cid = snrt_cluster_vpu_idx(); + int err = 0; + + if (cid == 0) { + pu_init_tab(); + printf("power update, M %u N %u, 1 core\n", CTRL_M, CTRL_N); + + for (unsigned int c = 0; c < CTRL_CASES; ++c) { + const unsigned int k = ctrl_case_k[c]; + int32_t nalloc = 0, delta = 0; + unsigned int cyc = 0; +#pragma clang loop unroll(disable) + for (int run = 0; run < NUM_RUNS; ++run) { + copy(p_row, ctrl_case_p + c * CTRL_N, CTRL_N); + copy(res_alloc, ctrl_case_res + c * CTRL_N, CTRL_N); + copy(klist, ctrl_case_kl + c * CTRL_N, CTRL_N); + for (unsigned int i = 0; i < CTRL_N; ++i) + ua_row[i] = 0; + nalloc = ctrl_case_nal[c]; + snrt_fence(); + + unsigned int t0 = benchmark_get_cycle(); + delta = pu_update(p_row, ctrl_case_g + c * CTRL_N, res_alloc, ua_row, + klist, &nalloc, k, CTRL_N); + snrt_fence(); + cyc = benchmark_get_cycle() - t0; + } + + int e = check("P", p_row, ctrl_case_p_exp + c * CTRL_N, CTRL_N); + if (delta != ctrl_case_d_exp[c]) { + printf("Delta = %d, expected %d\n", delta, ctrl_case_d_exp[c]); + ++e; + } + if (!res_alloc[k] || !ua_row[k] || nalloc != ctrl_case_nal[c] + 1 || + klist[nalloc - 1] != (int32_t)k) { + printf("allocation maps not updated\n"); + ++e; + } + printf("iteration %3d: user %2d subcarrier %3d, n %2d -> %2d: %u cycles%s\n", + ctrl_case_j[c], ctrl_case_w[c], k, ctrl_case_nal[c], nalloc, cyc, + e ? " MISMATCH" : ""); + err += e; + } + + // Finalization of all users, NUM_RUNS times on the same input + unsigned int cyc = 0; +#pragma clang loop unroll(disable) + for (int run = 0; run < NUM_RUNS; ++run) { + copy(delta_fin, ctrl_fin_delta, CTRL_M); + snrt_fence(); + unsigned int t0 = benchmark_get_cycle(); + pu_finalize(ctrl_fin_P, ctrl_fin_G, ctrl_fin_klist, ctrl_fin_nalloc, + delta_fin, 0, CTRL_M, CTRL_N); + snrt_fence(); + cyc = benchmark_get_cycle() - t0; + } + int e = check("final P", ctrl_fin_P, ctrl_fin_P_exp, CTRL_M * CTRL_N); + e += check("final Delta", delta_fin, ctrl_fin_delta_exp, CTRL_M); + printf("finalization of %u users: %u cycles%s\n", CTRL_M, cyc, + e ? " MISMATCH" : ""); + err += e; + printf(err ? "Check Failed!\n" : "Check passed.\n"); + } + snrt_cluster_hw_barrier(); + return err ? -1 : 0; +} diff --git a/software/control/power-update/script/data_64_128.json b/software/control/power-update/script/data_64_128.json new file mode 100644 index 00000000..a9b33544 --- /dev/null +++ b/software/control/power-update/script/data_64_128.json @@ -0,0 +1,8 @@ +// Benchmark size (see ../../README.md): updates at early, middle and late +// iterations, and the finalization +{ + M: 64, + N: 128, + seed: 1, + cases: [0, 1, 2, 40, 41, 90, 126, 127] +} diff --git a/software/control/power-update/script/data_9_21.json b/software/control/power-update/script/data_9_21.json new file mode 100644 index 00000000..e2b31954 --- /dev/null +++ b/software/control/power-update/script/data_9_21.json @@ -0,0 +1,7 @@ +// Odd sizes: partial vector strip +{ + M: 9, + N: 21, + seed: 2, + cases: [0, 1, 7, 19, 20] +} diff --git a/software/control/power-update/script/data_9_24.json b/software/control/power-update/script/data_9_24.json new file mode 100644 index 00000000..56bf6f6f --- /dev/null +++ b/software/control/power-update/script/data_9_24.json @@ -0,0 +1,7 @@ +// vl = 24 (a multiple of the 4 lanes), against data_9_21 (vl = 21) +{ + M: 9, + N: 24, + seed: 2, + cases: [0, 1, 7, 19, 20] +} diff --git a/software/control/power-update/script/gen_data.py b/software/control/power-update/script/gen_data.py new file mode 100644 index 00000000..702f21e4 --- /dev/null +++ b/software/control/power-update/script/gen_data.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +# Copyright 2025 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + +# Generates data/.h for the power-update kernel from one run of the +# golden algorithm: +# - for each iteration j in cfg "cases": the winner (w, k), the state the +# update reads (row w of P and G, ResAllocMap, the subcarrier list of w) +# and the expected row w of P and Delta[w] after the update; +# - the state before finalization (P, final G, subcarrier lists) and the +# expected P and Delta after it. + +import argparse +import pathlib +import sys + +import hjson +import numpy as np + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[2] / "common")) +import golden # noqa: E402 + + +def c_array(name, a): + a = np.asarray(a) + vals = ", ".join(str(int(v)) for v in a.flat) + return (f'static int32_t {name}[{a.size}] ' + f'__attribute__((section(".data"), aligned(64))) = {{{vals}}};\n\n') + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("-c", "--cfg", type=pathlib.Path, required=True) + args = parser.parse_args() + cfg = hjson.loads(args.cfg.read_text()) + m, n = cfg["M"], cfg["N"] + cases = sorted(cfg["cases"]) + + inp = golden.gen_inputs(m, n, cfg.get("seed", 1)) + R2 = inp["R2"].copy() + P = np.full((m, n), golden.P_INIT, dtype=np.int64) + res_alloc = np.zeros(n, dtype=np.int64) + user_alloc = np.zeros((m, n), dtype=np.int64) + delta = [golden.P_SCALE] * m + klist = [[] for _ in range(m)] + rec = {k: [] for k in ["w", "k", "p", "g", "res", "kl", "nal", "p_exp", "d_exp"]} + + for j in range(n): + if res_alloc.all(): + break + G = golden.calc_g(inp["A"], golden.calc_r(inp["R1"], R2, inp["C"]), inp["D"]) + _, w, k = golden.select(P, G, res_alloc) + if j in cases: + kl = np.zeros(n, dtype=np.int64) + kl[:len(klist[w])] = klist[w] + for key, val in [("w", w), ("k", k), ("p", P[w].copy()), ("g", G[w].copy()), + ("res", res_alloc.copy()), ("kl", kl), ("nal", len(klist[w]))]: + rec[key].append(val) + golden.update_winner(P, G, res_alloc, user_alloc, delta, w, k) + klist[w].append(k) + if j in cases: + rec["p_exp"].append(P[w].copy()) + rec["d_exp"].append(delta[w]) + R2[w] = (R2[w] + j) % 2 + + G = golden.calc_g(inp["A"], golden.calc_r(inp["R1"], R2, inp["C"]), inp["D"]) + fin_kl = np.zeros((m, n), dtype=np.int64) + for i in range(m): + fin_kl[i, :len(klist[i])] = klist[i] + fin_nal = [len(x) for x in klist] + fin_p, fin_d = P.copy(), list(delta) + golden.finalize(P, G, user_alloc, delta) + + # the outputs must equal the plain golden run + ref = golden.control_algo(**inp) + assert np.array_equal(ref["P"], P) + + print(f"{args.cfg.stem}: cases {cases}, n before update " + f"{rec['nal']}, users with subcarriers {sum(1 for x in fin_nal if x)}") + + out = ("// Copyright 2025 ETH Zurich and University of Bologna.\n" + "// Licensed under the Apache License, Version 2.0, see LICENSE for details.\n" + "// SPDX-License-Identifier: Apache-2.0\n\n" + "// This file was generated automatically by script/gen_data.py.\n\n" + "#include \n\n" + f"#define CTRL_M {m}\n#define CTRL_N {n}\n#define CTRL_CASES {len(cases)}\n\n") + out += c_array("ctrl_case_j", cases) + for key in ["w", "k", "nal", "d_exp"]: + out += c_array(f"ctrl_case_{key}", rec[key]) + for key in ["p", "g", "res", "kl", "p_exp"]: + out += c_array(f"ctrl_case_{key}", np.array(rec[key])) + out += c_array("ctrl_fin_P", fin_p) + out += c_array("ctrl_fin_G", G) + out += c_array("ctrl_fin_klist", fin_kl) + out += c_array("ctrl_fin_nalloc", fin_nal) + out += c_array("ctrl_fin_delta", fin_d) + out += c_array("ctrl_fin_P_exp", P) + out += c_array("ctrl_fin_delta_exp", delta) + + dst = pathlib.Path(__file__).resolve().parents[1] / "data" / (args.cfg.stem + ".h") + dst.write_text(out) + + +if __name__ == "__main__": + main() diff --git a/software/control/uv-init/data/.gitignore b/software/control/uv-init/data/.gitignore new file mode 100644 index 00000000..226b8c26 --- /dev/null +++ b/software/control/uv-init/data/.gitignore @@ -0,0 +1,2 @@ +# Generated data headers — reproduced by `make gen-data` +*.h diff --git a/software/control/uv-init/kernel/uv_init.c b/software/control/uv-init/kernel/uv_init.c new file mode 100644 index 00000000..6871e2c0 --- /dev/null +++ b/software/control/uv-init/kernel/uv_init.c @@ -0,0 +1,142 @@ +// Copyright 2025 ETH Zurich and University of Bologna. +// Licensed under the Apache License, Version 2.0, see LICENSE for details. +// SPDX-License-Identifier: Apache-2.0 + +// Kernel 1 of the control algorithm (see ../../README.md): the initial int32 +// products +// +// U = R1 * R2 + C, R = clamp(U, MinR, MaxR) (N x N) +// V = A * R + D, G = clamp(V >> G_SHIFT, MinG, MaxG) (M x N) +// +// Both are out = bias + a * b with a row-major (rows x K, row pitch lda) and +// b row-major (K x P). The scalar loads of a step through four rows at +// once; a pitch of an odd number of cache lines puts the four rows in +// different L1 banks (see kernels/fp/fmatmul-32b/main.c). Each call computes rows [r0, r1) and columns [c0, c1), four rows +// at a time with vmacc.vx, in strips of at most one LMUL-4 register group. +// The accumulators start from the bias rows (C rows for U, D for every row of +// V), so there is no vector move. U and V are stored unclamped because the +// later rank-1 updates need them; R and G are stored clamped. + +#include + +#include "control.h" + +// Bias row of out row r: C row r (stride P) for U, D (stride 0) for V +#define UV_BIAS(bias, stride, r) ((bias) + (r) * (stride)) + +// Store the accumulator group `acc` to out_row, clamp it (U -> R, or +// V -> G) and store it to aux_row +#define UV_EPILOGUE(acc, out_row, aux_row) \ + do { \ + asm volatile("vse32.v " acc ", (%0)" ::"r"(out_row) : "memory"); \ + if (mode == UV_MODE_RAW) { \ + } else if (mode == UV_MODE_U) { \ + asm volatile("vmax.vx " acc ", " acc ", %0" ::"r"(CTRL_MIN_R)); \ + asm volatile("vmin.vx " acc ", " acc ", %0" ::"r"(CTRL_MAX_R)); \ + } else { \ + asm volatile("vsra.vi " acc ", " acc ", %0" ::"i"(CTRL_G_SHIFT)); \ + asm volatile("vmax.vx " acc ", " acc ", %0" ::"r"(CTRL_MIN_G)); \ + asm volatile("vmin.vx " acc ", " acc ", %0" ::"r"(CTRL_MAX_G)); \ + } \ + if (mode != UV_MODE_RAW) \ + asm volatile("vse32.v " acc ", (%0)" ::"r"(aux_row) : "memory"); \ + } while (0) + +// Four rows: accumulators v0, v4, v8, v12; b rows alternate between v16 and +// v20 so the next row loads while the current one is consumed. The a +// elements for the next k are loaded while the vmacc of this k is issued. +static inline void uv_rows4(int32_t *out, int32_t *aux, const int32_t *a, + const int32_t *b, const int32_t *bias, + unsigned int bias_stride, unsigned int r, + unsigned int K, unsigned int lda, unsigned int P, + unsigned int p, int mode) { + const int32_t *a0 = a + r * lda; + const int32_t *a1 = a0 + lda; + const int32_t *a2 = a1 + lda; + const int32_t *a3 = a2 + lda; + const int32_t *bp = b + p; + + asm volatile("vle32.v v0, (%0)" ::"r"(UV_BIAS(bias, bias_stride, r) + p)); + asm volatile("vle32.v v4, (%0)" ::"r"(UV_BIAS(bias, bias_stride, r + 1) + p)); + asm volatile("vle32.v v8, (%0)" ::"r"(UV_BIAS(bias, bias_stride, r + 2) + p)); + asm volatile("vle32.v v12, (%0)" ::"r"(UV_BIAS(bias, bias_stride, r + 3) + p)); + asm volatile("vle32.v v16, (%0)" ::"r"(bp)); + + int32_t t0 = a0[0], t1 = a1[0], t2 = a2[0], t3 = a3[0]; + unsigned int k = 0; + while (1) { + // k is even here; b row k is in v16 + unsigned int kn = k + 1; + if (kn < K) + asm volatile("vle32.v v20, (%0)" ::"r"(bp + kn * P)); + unsigned int kl = kn < K ? kn : k; + asm volatile("vmacc.vx v0, %0, v16" ::"r"(t0)); + t0 = a0[kl]; + asm volatile("vmacc.vx v4, %0, v16" ::"r"(t1)); + t1 = a1[kl]; + asm volatile("vmacc.vx v8, %0, v16" ::"r"(t2)); + t2 = a2[kl]; + asm volatile("vmacc.vx v12, %0, v16" ::"r"(t3)); + t3 = a3[kl]; + if (kn == K) + break; + + // b row kn is in v20 + k = kn + 1; + if (k < K) + asm volatile("vle32.v v16, (%0)" ::"r"(bp + k * P)); + kl = k < K ? k : kn; + asm volatile("vmacc.vx v0, %0, v20" ::"r"(t0)); + t0 = a0[kl]; + asm volatile("vmacc.vx v4, %0, v20" ::"r"(t1)); + t1 = a1[kl]; + asm volatile("vmacc.vx v8, %0, v20" ::"r"(t2)); + t2 = a2[kl]; + asm volatile("vmacc.vx v12, %0, v20" ::"r"(t3)); + t3 = a3[kl]; + if (k == K) + break; + } + + int32_t *o = out + r * P + p; + int32_t *x = aux + r * P + p; + UV_EPILOGUE("v0", o, x); + UV_EPILOGUE("v4", o + P, x + P); + UV_EPILOGUE("v8", o + 2 * P, x + 2 * P); + UV_EPILOGUE("v12", o + 3 * P, x + 3 * P); +} + +// One row, for the rows left over when r1 - r0 is not a multiple of four +static inline void uv_rows1(int32_t *out, int32_t *aux, const int32_t *a, + const int32_t *b, const int32_t *bias, + unsigned int bias_stride, unsigned int r, + unsigned int K, unsigned int lda, unsigned int P, + unsigned int p, int mode) { + const int32_t *a0 = a + r * lda; + const int32_t *bp = b + p; + asm volatile("vle32.v v0, (%0)" ::"r"(UV_BIAS(bias, bias_stride, r) + p)); + for (unsigned int k = 0; k < K; ++k) { + asm volatile("vle32.v v16, (%0)" ::"r"(bp + k * P)); + asm volatile("vmacc.vx v0, %0, v16" ::"r"(a0[k])); + } + UV_EPILOGUE("v0", out + r * P + p, aux + r * P + p); +} + +void uv_matmul(int32_t *out, int32_t *aux, const int32_t *a, const int32_t *b, + const int32_t *bias, unsigned int bias_stride, unsigned int r0, + unsigned int r1, unsigned int c0, unsigned int c1, + unsigned int K, unsigned int lda, unsigned int P, int mode) { + for (unsigned int p = c0; p < c1;) { + // Always an explicit AVL that fits one LMUL-4 group (see spatz quirks) + unsigned int avl = c1 - p < UV_VLMAX ? c1 - p : UV_VLMAX; + unsigned int vl; + asm volatile("vsetvli %0, %1, e32, m4, ta, ma" : "=r"(vl) : "r"(avl)); + unsigned int r = r0; + for (; r + 4 <= r1; r += 4) + uv_rows4(out, aux, a, b, bias, bias_stride, r, K, lda, P, p, mode); + for (; r < r1; ++r) + uv_rows1(out, aux, a, b, bias, bias_stride, r, K, lda, P, p, mode); + p += vl; + } +} + diff --git a/software/control/uv-init/main.c b/software/control/uv-init/main.c new file mode 100644 index 00000000..4a6e322c --- /dev/null +++ b/software/control/uv-init/main.c @@ -0,0 +1,199 @@ +// Copyright 2025 ETH Zurich and University of Bologna. +// Licensed under the Apache License, Version 2.0, see LICENSE for details. +// SPDX-License-Identifier: Apache-2.0 + +// Kernel 1 of the control algorithm: U = R1 * R2 + C, R = clamp(U), then +// V = A * R + D, G = clamp(V >> G_SHIFT). +// +// The L1 is all-private (L1D_PRIVATE = 1, default) with the private banks +// folded per tile or per group (L1D_FOLD), or all-shared. Private data is +// only visible inside one folding domain, so the work is split by columns +// across domains and by rows inside a domain: V[:, cols] only needs +// R[:, cols], which the same domain computes. R never crosses a domain and +// the barrier between the two products needs no flush. The final check +// flushes the L1 so that every core sees the results. +// +// Runs twice: the first run starts from a cold L1 (the inputs change every +// TTI), the second is hot. Both runs write the same outputs, which all cores +// of the cluster check against the golden model after the second run. +// +// IPU utilization = multiply-accumulates / (cycles * 4 lanes * cores). + +#include +#include +#include + +#include DATAHEADER +#include "kernel/uv_init.c" + +#ifndef ACTIVE_CORES +#define ACTIVE_CORES 1 +#endif + +#define NUM_RUNS 2 + +// L1 configuration, as in kernels/fp/fmatmul-32b/main.c +#ifndef L1D_PRIVATE +#define L1D_PRIVATE 1 +#endif +#ifndef L1D_FOLD +#define L1D_FOLD L1D_FOLD_TILE +#endif +// 0 keeps the default bank-select offset +#ifndef XBAR_OFFSET +#define XBAR_OFFSET 0 +#endif + +// Multiply-accumulates of U = R1 * R2 + C and of V = A * R + D +#define MAC_U (CTRL_N * CTRL_N * CTRL_M) +#define MAC_V (CTRL_M * CTRL_N * CTRL_N) +#define IPU_LANES 4 + +static int errors[1024] __attribute__((section(".data"))); +// first mismatch of each core: offset, got, expected, matrix name +static int32_t bad[1024][4] __attribute__((section(".data"))); +static int32_t *first_bad; + +// Rows [r0, r1) of `rows` for core lc of `cores`, in blocks of four rows +static void split_rows(unsigned int rows, unsigned int lc, unsigned int cores, + unsigned int *r0, unsigned int *r1) { + unsigned int blocks = (rows + 3) / 4; + unsigned int b0 = blocks * lc / cores; + unsigned int b1 = blocks * (lc + 1) / cores; + *r0 = snrt_min(4 * b0, rows); + *r1 = snrt_min(4 * b1, rows); +} + +// Compare the block rows [r0, r1) x columns [c0, c1) of got and exp. Each +// core checks the block it computed, which its own L1 domain holds, so the +// check needs no flush of the private partition. +static int check(const char *name, const int32_t *got, const int32_t *exp, + unsigned int r0, unsigned int r1, unsigned int c0, + unsigned int c1, unsigned int cols) { + int err = 0; + for (unsigned int i = r0; i < r1; ++i) { + for (unsigned int j = c0; j < c1; ++j) { + const unsigned int o = i * cols + j; + if (got[o] != exp[o]) { + if (err == 0) { + first_bad[0] = (int32_t)o; + first_bad[1] = got[o]; + first_bad[2] = exp[o]; + first_bad[3] = name[0]; + } + ++err; + } + } + } + return err; +} + +// Utilization in per mille of the peak of `cores` cores +static unsigned int util(unsigned int mac, unsigned int cycles, + unsigned int cores) { + return (unsigned int)(1000ULL * mac / ((unsigned long long)cycles * + IPU_LANES * cores)); +} + +int main() { + const unsigned int cid = snrt_cluster_vpu_idx(); + const unsigned int num_cores = snrt_cluster_vpu_num(); + const int active = snrt_cluster_is_primary() && cid < ACTIVE_CORES; + + // Cores per folding domain; the active cores fill whole domains + const unsigned int domain = + !L1D_PRIVATE ? ACTIVE_CORES + : L1D_FOLD == L1D_FOLD_TILE + ? snrt_cluster_vpu_per_tile() + : snrt_cluster_vpu_num() / snrt_cluster_group_num(); + const unsigned int dcores = snrt_min(ACTIVE_CORES, domain); + const unsigned int ndom = ACTIVE_CORES / dcores; + const unsigned int d = cid / dcores, lc = cid % dcores; + unsigned int u0, u1, v0, v1; + split_rows(CTRL_N, lc, dcores, &u0, &u1); + split_rows(CTRL_M, lc, dcores, &v0, &v1); + const unsigned int c0 = CTRL_N * d / ndom, c1 = CTRL_N * (d + 1) / ndom; + + // Both calls flush, so run 0 starts from a cold L1 either way + snrt_cluster_hw_barrier(); + if (XBAR_OFFSET) + l1d_xbar_config(XBAR_OFFSET); + if (L1D_PRIVATE) + l1d_part_folded(snrt_cluster_vpu_per_tile(), L1D_FOLD); + + // Not unrolled: every run executes the same (warm) instructions +#pragma clang loop unroll(disable) + for (int run = 0; run < NUM_RUNS; ++run) { + snrt_cluster_hw_barrier(); + unsigned int t0 = benchmark_get_cycle(); + if (cid == 0) + start_kernel(); + + // U = R1 * R2 + C, rows of U split over the cores + // The outputs are vector stores only: drain Spatz before each barrier + if (active) + uv_matmul(ctrl_U, ctrl_R, ctrl_R1, ctrl_R2, ctrl_C, CTRL_N, u0, u1, c0, + c1, CTRL_M, CTRL_LDA_R1, CTRL_N, UV_MODE_U); + snrt_fence_spatz(); + snrt_cluster_hw_barrier(); + unsigned int t1 = benchmark_get_cycle(); + + // V = A * R + D, rows of V split over the cores + if (active) + uv_matmul(ctrl_V, ctrl_G, ctrl_A, ctrl_R, ctrl_D, 0, v0, v1, c0, c1, + CTRL_N, CTRL_LDA_A, CTRL_N, UV_MODE_V); + snrt_fence_spatz(); + snrt_cluster_hw_barrier(); + unsigned int t2 = benchmark_get_cycle(); + + if (cid == 0) { + stop_kernel(); + printf("run %d (%s): %u cores, U %ux%ux%u, V %ux%ux%u, L1 %s xbar %d\n", + run, run ? "hot" : "cold", ACTIVE_CORES, CTRL_N, CTRL_M, CTRL_N, + CTRL_M, CTRL_N, CTRL_N, + L1D_PRIVATE ? (L1D_FOLD == L1D_FOLD_TILE ? "tile" : "group") + : "shared", + XBAR_OFFSET); + printf(" U+R %u cycles, IPU util %u permille\n", t1 - t0, + util(MAC_U, t1 - t0, ACTIVE_CORES)); + printf(" V+G %u cycles, IPU util %u permille\n", t2 - t1, + util(MAC_V, t2 - t1, ACTIVE_CORES)); + printf(" total %u cycles, IPU util %u permille\n", t2 - t0, + util(MAC_U + MAC_V, t2 - t0, ACTIVE_CORES)); + } + } + + // Write the private banks back so that every checking core sees the + // results (must be called by all cores) + // Each active core checks its own block (see check); the per-core error + // counts are then read by core 0, which needs a flush in private mode + if (active) { + first_bad = bad[cid]; + int err = check("U", ctrl_U, ctrl_U_exp, u0, u1, c0, c1, CTRL_N); + err += check("R", ctrl_R, ctrl_R_exp, u0, u1, c0, c1, CTRL_N); + err += check("V", ctrl_V, ctrl_V_exp, v0, v1, c0, c1, CTRL_N); + err += check("G", ctrl_G, ctrl_G_exp, v0, v1, c0, c1, CTRL_N); + errors[cid] = err; + snrt_fence_snitch(); + } + // errors[] and bad[] leave one dirty line per set, which the flush writes + // back (it loses lines when several ways of a set are dirty) + if (L1D_PRIVATE) + l1d_cluster_flush(); + else + snrt_cluster_hw_barrier(); + + int total = 0; + if (cid == 0) { + for (unsigned int c = 0; c < num_cores; ++c) { + if (errors[c]) + printf("core %u: %d mismatches, first %c[%d][%d] = %d, expected %d\n", + c, errors[c], (char)bad[c][3], bad[c][0] / CTRL_N, + bad[c][0] % CTRL_N, bad[c][1], bad[c][2]); + total += errors[c]; + } + printf(total ? "Check Failed!\n" : "Check passed.\n"); + } + snrt_cluster_hw_barrier(); + return total ? -1 : 0; +} diff --git a/software/control/uv-init/script/data_64_128.json b/software/control/uv-init/script/data_64_128.json new file mode 100644 index 00000000..be252632 --- /dev/null +++ b/software/control/uv-init/script/data_64_128.json @@ -0,0 +1,6 @@ +// Benchmark size: M = 64 users, N = 128 subcarriers (see ../../README.md) +{ + M: 64, + N: 128, + seed: 1 +} diff --git a/software/control/uv-init/script/data_8_16_const.json b/software/control/uv-init/script/data_8_16_const.json new file mode 100644 index 00000000..6e78f2f2 --- /dev/null +++ b/software/control/uv-init/script/data_8_16_const.json @@ -0,0 +1,6 @@ +// Small constant-data smoke test: U = M everywhere, V = N * M +{ + M: 8, + N: 16, + init: "const" +} diff --git a/software/control/uv-init/script/data_9_21.json b/software/control/uv-init/script/data_9_21.json new file mode 100644 index 00000000..1ffa2bfb --- /dev/null +++ b/software/control/uv-init/script/data_9_21.json @@ -0,0 +1,6 @@ +// Odd sizes: exercises the one-row tail and a partial vector strip +{ + M: 9, + N: 21, + seed: 2 +} diff --git a/software/control/uv-init/script/gen_data.py b/software/control/uv-init/script/gen_data.py new file mode 100644 index 00000000..7ee409f4 --- /dev/null +++ b/software/control/uv-init/script/gen_data.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +# Copyright 2025 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + +# Generates data/.h for kernel 1 (uv-init): the control-algorithm inputs +# and the expected U, R, V and G from common/golden.py. + +import argparse +import pathlib +import sys + +import hjson +import numpy as np + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[2] / "common")) +import golden # noqa: E402 + + +# Row pitch (in int32) of the a matrices: a whole number of 64-byte cache +# lines, and an odd one, so that the four rows a core reads at once fall in +# different L1 banks +LINE = 16 + + +def pitch(k): + lines = (k + LINE - 1) // LINE + return LINE * (lines if lines % 2 else lines + 1) + + +def padded(a, lda): + out = np.zeros((a.shape[0], lda), dtype=np.int64) + out[:, :a.shape[1]] = a + return out + + +def c_array(name, a, section=".data"): + vals = ", ".join(str(int(v)) for v in np.asarray(a).flat) + return (f'static int32_t {name}[{np.asarray(a).size}] ' + f'__attribute__((section("{section}"), aligned(64))) = {{{vals}}};\n\n') + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("-c", "--cfg", type=pathlib.Path, required=True) + args = parser.parse_args() + cfg = hjson.loads(args.cfg.read_text()) + m, n = cfg["M"], cfg["N"] + + if cfg.get("init", "random") == "const": + # small exact values, easy to read back in a failing run + inp = {"R1": np.ones((n, m), int), "R2": np.ones((m, n), int), + "A": np.ones((m, n), int), "C": np.zeros((n, n), int), + "D": np.zeros(n, int)} + else: + inp = golden.gen_inputs(m, n, cfg.get("seed", 1)) + + U = inp["R1"] @ inp["R2"] + inp["C"] + R = golden.calc_r(inp["R1"], inp["R2"], inp["C"]) + V = golden.check_i32(inp["A"] @ R + inp["D"], "V") + G = golden.calc_g(inp["A"], R, inp["D"]) + + out = ("// Copyright 2025 ETH Zurich and University of Bologna.\n" + "// Licensed under the Apache License, Version 2.0, see LICENSE for details.\n" + "// SPDX-License-Identifier: Apache-2.0\n\n" + "// This file was generated automatically by script/gen_data.py.\n\n" + "#include \n\n" + f"#define CTRL_M {m}\n#define CTRL_N {n}\n" + f"#define CTRL_LDA_R1 {pitch(m)}\n#define CTRL_LDA_A {pitch(n)}\n\n") + out += c_array("ctrl_R1", padded(inp["R1"], pitch(m))) + out += c_array("ctrl_A", padded(inp["A"], pitch(n))) + for key in ["R2", "C", "D"]: + out += c_array(f"ctrl_{key}", inp[key]) + for key, val in [("U", U), ("R", R), ("V", V), ("G", G)]: + out += c_array(f"ctrl_{key}", np.zeros_like(val)) + out += c_array(f"ctrl_{key}_exp", val) + + dst = pathlib.Path(__file__).resolve().parents[1] / "data" / (args.cfg.stem + ".h") + dst.write_text(out) + + +if __name__ == "__main__": + main() diff --git a/software/kernels/fp/fdotp-32b/kernel/fdotp.c b/software/kernels/fp/fdotp-32b/kernel/fdotp.c index e57ae15f..77409f96 100644 --- a/software/kernels/fp/fdotp-32b/kernel/fdotp.c +++ b/software/kernels/fp/fdotp-32b/kernel/fdotp.c @@ -15,152 +15,171 @@ // limitations under the License. // Author: Matteo Perotti +// Author: Diyou Shen #include "fdotp.h" -static inline int fp_check(const float a, const float b) { - const float threshold = 0.01f; - - // Absolute value - float comp = a - b; - if (comp < 0) - comp = -comp; - - return comp > threshold; -} - -// 32-bit dot-product: a * b -float fdotp_v32b_lmul8(const float *a, const float *b, const unsigned int offset, const unsigned int avl, const unsigned int rounds) { - unsigned int vl; - unsigned int iter = rounds; - - float red; - - // Set the vl - asm volatile("vsetvli %0, %1, e32, m8, ta, ma" : "=r"(vl) : "r"(avl)); - - asm volatile("vle32.v v8, (%0)" ::"r"(a)); - iter --; - asm volatile("vle32.v v16, (%0)" ::"r"(b)); - asm volatile("vfmul.vv v24, v8, v16"); - - // Stripmine and accumulate a partial reduced vector - while (iter > 0) { - // Load chunk a and b - a += offset; - asm volatile("vle32.v v8, (%0)" ::"r"(a)); - b += offset; - asm volatile("vle32.v v16, (%0)" ::"r"(b)); - iter --; - - // Multiply and accumulate - asm volatile("vfmacc.vv v24, v8, v16"); +// Every round loads `vl` elements of a and b, `offset` floats after the +// previous one, and accumulates a * b into v24 (at LMUL L). vl is always set +// explicitly: Spatz maps `vsetvli rd, x0` to the LMUL 8 VLMAX whatever the +// requested LMUL. + +// One register set: a in v8, b in v16 +#define FDOTP_ACC_U1(L) \ + static inline void fdotp_acc_u1_m##L(const float *a, const float *b, \ + const uint32_t vl, \ + const uint32_t offset, \ + uint32_t rounds) { \ + asm volatile("vsetvli zero, %0, e32, m" #L ", ta, ma" ::"r"(vl)); \ + asm volatile("vle32.v v8, (%0)" ::"r"(a)); \ + asm volatile("vle32.v v16, (%0)" ::"r"(b)); \ + asm volatile("vfmul.vv v24, v8, v16"); \ + while (--rounds) { \ + a += offset; \ + asm volatile("vle32.v v8, (%0)" ::"r"(a)); \ + b += offset; \ + asm volatile("vle32.v v16, (%0)" ::"r"(b)); \ + asm volatile("vfmacc.vv v24, v8, v16"); \ + } \ } - // Reduce and return - asm volatile("vfredusum.vs v0, v24, v0"); - asm volatile("vfmv.f.s %0, v0" : "=f"(red)); - return red; -} - -// 32-bit dot-product: a * b -float fdotp_v32b_lmul4(const float *a, const float *b, const unsigned int offset, const unsigned int avl, const unsigned int rounds) { - unsigned int vl; - unsigned int iter = rounds; - - float red; - - // Set the vl - asm volatile("vsetvli %0, %1, e32, m4, ta, ma" : "=r"(vl) : "r"(avl)); - - asm volatile("vle32.v v8, (%0)" ::"r"(a)); - iter --; - asm volatile("vle32.v v16, (%0)" ::"r"(b)); - asm volatile("vfmul.vv v24, v8, v16"); - - // Stripmine and accumulate a partial reduced vector - do { - // Load chunk a and b - a += offset; - asm volatile("vle32.v v8, (%0)" ::"r"(a)); - b += offset; - asm volatile("vle32.v v16, (%0)" ::"r"(b)); - iter --; - - // Multiply and accumulate - asm volatile("vfmacc.vv v24, v8, v16"); - } while (iter > 0); +// Two register sets, (v0, v4) and (v8, v12): the next round is loaded into +// one set while the other one is multiplied. LMUL 4 at most. +#define FDOTP_ACC_U2(L) \ + static inline void fdotp_acc_u2_m##L(const float *a, const float *b, \ + const uint32_t vl, \ + const uint32_t offset, \ + const uint32_t rounds) { \ + asm volatile("vsetvli zero, %0, e32, m" #L ", ta, ma" ::"r"(vl)); \ + asm volatile("vle32.v v0, (%0)" ::"r"(a)); \ + asm volatile("vle32.v v4, (%0)" ::"r"(b)); \ + if (rounds == 1) { \ + asm volatile("vfmul.vv v24, v0, v4"); \ + return; \ + } \ + a += offset; \ + b += offset; \ + asm volatile("vle32.v v8, (%0)" ::"r"(a)); \ + asm volatile("vle32.v v12, (%0)" ::"r"(b)); \ + asm volatile("vfmul.vv v24, v0, v4"); \ + uint32_t r = 2; \ + while (r < rounds) { \ + a += offset; \ + b += offset; \ + asm volatile("vle32.v v0, (%0)" ::"r"(a)); \ + asm volatile("vle32.v v4, (%0)" ::"r"(b)); \ + asm volatile("vfmacc.vv v24, v8, v12"); \ + if (++r == rounds) { \ + asm volatile("vfmacc.vv v24, v0, v4"); \ + return; \ + } \ + a += offset; \ + b += offset; \ + asm volatile("vle32.v v8, (%0)" ::"r"(a)); \ + asm volatile("vle32.v v12, (%0)" ::"r"(b)); \ + asm volatile("vfmacc.vv v24, v0, v4"); \ + ++r; \ + } \ + asm volatile("vfmacc.vv v24, v8, v12"); \ + } - // Reduce and return - asm volatile("vfredusum.vs v0, v24, v0"); - asm volatile("vfmv.f.s %0, v0" : "=f"(red)); - return red; +FDOTP_ACC_U1(8) +FDOTP_ACC_U1(4) +FDOTP_ACC_U1(2) +FDOTP_ACC_U1(1) +FDOTP_ACC_U2(4) +FDOTP_ACC_U2(2) +FDOTP_ACC_U2(1) + +// Spatz reduces one element per FPU latency, so the accumulator is first +// halved with full-width vfadd down to LMUL 1. vl is the accumulator length. +static inline void fdotp_fold_m1(const uint32_t lmul, uint32_t vl) { + if (lmul >= 8) { + vl >>= 1; + asm volatile("vsetvli zero, %0, e32, m4, ta, ma" ::"r"(vl)); + asm volatile("vfadd.vv v24, v24, v28"); + } + if (lmul >= 4) { + vl >>= 1; + asm volatile("vsetvli zero, %0, e32, m2, ta, ma" ::"r"(vl)); + asm volatile("vfadd.vv v24, v24, v26"); + } + if (lmul >= 2) { + vl >>= 1; + asm volatile("vsetvli zero, %0, e32, m1, ta, ma" ::"r"(vl)); + asm volatile("vfadd.vv v24, v24, v25"); + } } -// 32-bit dot-product: a * b -float fdotp_v32b_lmul2(const float *a, const float *b, const unsigned int offset, const unsigned int avl, const unsigned int rounds) { - unsigned int vl; - unsigned int iter = rounds; - +// TMP-DOTP-SEED: 0 seeds v0 with vmv.s.x, 1 with vfmv.s.f, 2 not at all +#ifndef FDOTP_SEED +#define FDOTP_SEED 1 +#endif +#if FDOTP_SEED == 0 +#define FDOTP_SEED_V0() asm volatile("vmv.s.x v0, zero") +#elif FDOTP_SEED == 2 +#define FDOTP_SEED_V0() do { } while (0) +#else +#define FDOTP_SEED_V0() \ + do { \ + float zero_f; \ + asm volatile("fmv.w.x %0, zero" : "=f"(zero_f)); \ + asm volatile("vfmv.s.f v0, %0" ::"f"(zero_f)); \ + } while (0) +#endif + +// Reduce the v24 accumulator (LMUL lmul, vl elements) to a scalar +static inline float fdotp_reduce(const uint32_t lmul, const uint32_t vl, + const uint32_t vred) { float red; - // Set the vl - asm volatile("vsetvli %0, %1, e32, m2, ta, ma" : "=r"(vl) : "r"(avl)); - - asm volatile("vle32.v v8, (%0)" ::"r"(a)); - iter --; - asm volatile("vle32.v v16, (%0)" ::"r"(b)); - asm volatile("vfmul.vv v24, v8, v16"); - - // Stripmine and accumulate a partial reduced vector - do { - // Load chunk a and b - a += offset; - asm volatile("vle32.v v8, (%0)" ::"r"(a)); - b += offset; - asm volatile("vle32.v v16, (%0)" ::"r"(b)); - iter --; - - // Multiply and accumulate - asm volatile("vfmacc.vv v24, v8, v16"); - } while (iter > 0); + if (vred == FDOTP_VRED_FULL) { + FDOTP_SEED_V0(); + asm volatile("vfredusum.vs v0, v24, v0"); + asm volatile("vfmv.f.s %0, v0" : "=f"(red)); + return red; + } - // Reduce and return - asm volatile("vfredusum.vs v0, v24, v0"); - asm volatile("vfmv.f.s %0, v0" : "=f"(red)); + fdotp_fold_m1(lmul, vl); + + if (vred == FDOTP_VRED_FOLD) { + FDOTP_SEED_V0(); + asm volatile("vfredusum.vs v0, v24, v0"); + asm volatile("vfmv.f.s %0, v0" : "=f"(red)); + } else { + // Halve the remaining LMUL 1 vector until one element is left + for (uint32_t n = vl / lmul / 2; n > 0; n >>= 1) { + asm volatile("vsetvli zero, %0, e32, m1, ta, ma" ::"r"(n)); + asm volatile("vslidedown.vx v25, v24, %0" ::"r"(n)); + asm volatile("vfadd.vv v24, v24, v25"); + } + asm volatile("vfmv.f.s %0, v24" : "=f"(red)); + } return red; } -// 32-bit dot-product: a * b -float fdotp_v32b_lmul1(const float *a, const float *b, const unsigned int offset, const unsigned int avl, const unsigned int rounds) { - unsigned int vl; - unsigned int iter = rounds; - - float red; - - // Set the vl - asm volatile("vsetvli %0, %1, e32, m1, ta, ma" : "=r"(vl) : "r"(avl)); - - asm volatile("vle32.v v8, (%0)" ::"r"(a)); - iter --; - asm volatile("vle32.v v16, (%0)" ::"r"(b)); - asm volatile("vfmul.vv v24, v8, v16"); - - // Stripmine and accumulate a partial reduced vector - do { - // Load chunk a and b - a += offset; - asm volatile("vle32.v v8, (%0)" ::"r"(a)); - b += offset; - asm volatile("vle32.v v16, (%0)" ::"r"(b)); - iter --; - - // Multiply and accumulate - asm volatile("vfmacc.vv v24, v8, v16"); - } while (iter > 0); - - // Reduce and return - asm volatile("vfredusum.vs v0, v24, v0"); - asm volatile("vfmv.f.s %0, v0" : "=f"(red)); - return red; +// 32-bit dot-product partial of one core: `rounds` chunks of vl elements at +// `lmul`, `offset` floats apart +float fdotp_v32b(const float *a, const float *b, const uint32_t vl, + const uint32_t offset, const uint32_t rounds, + const uint32_t lmul, const uint32_t unroll, + const uint32_t vred) { + if (unroll == 2 && lmul <= 4) { + if (lmul == 4) + fdotp_acc_u2_m4(a, b, vl, offset, rounds); + else if (lmul == 2) + fdotp_acc_u2_m2(a, b, vl, offset, rounds); + else + fdotp_acc_u2_m1(a, b, vl, offset, rounds); + } else { + if (lmul == 8) + fdotp_acc_u1_m8(a, b, vl, offset, rounds); + else if (lmul == 4) + fdotp_acc_u1_m4(a, b, vl, offset, rounds); + else if (lmul == 2) + fdotp_acc_u1_m2(a, b, vl, offset, rounds); + else + fdotp_acc_u1_m1(a, b, vl, offset, rounds); + } + return fdotp_reduce(lmul, vl, vred); } diff --git a/software/kernels/fp/fdotp-32b/kernel/fdotp.h b/software/kernels/fp/fdotp-32b/kernel/fdotp.h index dee09584..008e9a15 100644 --- a/software/kernels/fp/fdotp-32b/kernel/fdotp.h +++ b/software/kernels/fp/fdotp-32b/kernel/fdotp.h @@ -15,19 +15,24 @@ // limitations under the License. // Author: Matteo Perotti +// Author: Diyou Shen #ifndef _FDOTPROD_H_ #define _FDOTPROD_H_ -inline float fdotp_v32b_lmul8(const float *a, const float *b, const unsigned int offset, const unsigned int avl, const unsigned int rounds) - __attribute__((always_inline)); +#include -inline float fdotp_v32b_lmul4(const float *a, const float *b, const unsigned int offset, const unsigned int avl, const unsigned int rounds) - __attribute__((always_inline)); +// In-vector reduction of the accumulator (FDOTP_VRED): +// 0: one vfredusum over the whole accumulator +// 1: fold to LMUL 1 with vfadd, then vfredusum +// 2: fold to one element with vfadd and vslidedown +#define FDOTP_VRED_FULL 0 +#define FDOTP_VRED_FOLD 1 +#define FDOTP_VRED_SLIDE 2 -inline float fdotp_v32b_lmul2(const float *a, const float *b, const unsigned int offset, const unsigned int avl, const unsigned int rounds) - __attribute__((always_inline)); +float fdotp_v32b(const float *a, const float *b, const uint32_t vl, + const uint32_t offset, const uint32_t rounds, + const uint32_t lmul, const uint32_t unroll, + const uint32_t vred); -inline float fdotp_v32b_lmul1(const float *a, const float *b, const unsigned int offset, const unsigned int avl, const unsigned int rounds) - __attribute__((always_inline)); #endif diff --git a/software/kernels/fp/fdotp-32b/main.c b/software/kernels/fp/fdotp-32b/main.c index 2237fe19..3124d4de 100644 --- a/software/kernels/fp/fdotp-32b/main.c +++ b/software/kernels/fp/fdotp-32b/main.c @@ -17,6 +17,7 @@ // Author: Diyou Shen #include +#include #include #include #include @@ -24,159 +25,257 @@ #include DATAHEADER #include "kernel/fdotp.c" -int main() { - const uint32_t num_cores = snrt_cluster_vpu_num(); - const uint32_t cid = snrt_cluster_vpu_idx(); - const int is_primary = snrt_cluster_is_primary(); +// sum(a * b) over M floats. Core c owns chunks c, c + cores, c + 2 * cores, ... +// of LMUL * VLEN / 32 elements each, reduces its partial inside the vector register and +// then across cores. + +// LMUL of the accumulation (1, 2, 4 or 8) +#ifndef DOTP_LMUL +#define DOTP_LMUL 8 +#endif +// 1: one register set; 2: two register sets, LMUL 4 at most +#ifndef DOTP_UNROLL +#define DOTP_UNROLL 1 +#endif +// In-vector reduction, see kernel/fdotp.h +#ifndef DOTP_VRED +#define DOTP_VRED FDOTP_VRED_FOLD +#endif +// Cross-core reduction. 0: groups of 4, then core 0 adds the group sums; +// 1: binary tree over per-core slots, neighbours first. 1 polls slots that +// other cores write in the same L1 line and gives wrong sums on the current +// L1, so 0 is the default. +#ifndef DOTP_XRED +#define DOTP_XRED 0 +#endif +// Number of private banks per tile (0 = all-shared) and how they are folded +#ifndef L1D_PART +#define L1D_PART 0 +#endif +#ifndef L1D_FOLD +#define L1D_FOLD L1D_FOLD_TILE +#endif + +#define MEASURE_ITER 3 + +// Upper bound on cores, for the per-core slots +#define MAX_CORES 256 + +// Empty slot marker: a NaN no partial sum of valid data produces +#define SLOT_EMPTY 0xFFFFFFFFu + +// Tree reduction mailboxes: core c publishes its subtree sum in slot c, and +// its parent empties it again after reading. +static volatile uint32_t dotp_slot[MAX_CORES] + __attribute__((section(".data"))) = {[0 ... MAX_CORES - 1] = SLOT_EMPTY}; +// Partials of the two-level reduction +static float dotp_part[MAX_CORES] __attribute__((section(".data"))); + +static int num_errors __attribute__((section(".data"))); + +static inline int fp_check(const float a, const float b) { + const float threshold = 0.01f; + + float comp = a - b; + if (comp < 0) + comp = -comp; + + return comp > threshold; +} - const uint32_t measure_iter = 3; +static inline uint32_t f2u(const float f) { + union { float f; uint32_t u; } v = {.f = f}; + return v.u; +} - /*** DRAM Parameters for Optimization ***/ - const uint32_t l2_interleave = 16; - const uint32_t l2_channel = 4; - const uint32_t l2_lanewidth = 512; - const uint32_t l2_block_size = l2_lanewidth * l2_interleave; - const uint32_t l2_block_elem = l2_block_size / 32; +static inline float u2f(const uint32_t u) { + union { float f; uint32_t u; } v = {.u = u}; + return v.f; +} - /*** Core Parameters for Optimization ***/ - const uint32_t elem_per_core = dotp_l.M / num_cores; - const uint32_t lmul_m1 = 512; - const uint32_t lmul_m1_elem = lmul_m1 / 32; - - const uint32_t lmul_max = elem_per_core / lmul_m1_elem; - uint32_t lmul; - if (lmul_max >= 8) { - lmul = 8; - } else if (lmul_max >= 4) { - lmul = 4; - } else if (lmul_max >= 2) { - lmul = 2; - } else if (lmul_max == 1) { - lmul = 1; - } else { - if (is_primary && cid == 0) { - printf("FATAL: Problem size too small!\n"); - return -2; +// Binary tree: at level s, core c with bit s set hands its sum to core c - s. +// Returns the full sum on core 0. +static inline float reduce_tree(float acc, const uint32_t cid, + const uint32_t num_cores) { + for (uint32_t s = 1; s < num_cores; s <<= 1) { + if (cid & s) { + dotp_slot[cid] = f2u(acc); + break; + } + if (cid + s < num_cores) { + uint32_t v; + while ((v = dotp_slot[cid + s]) == SLOT_EMPTY) + ; + dotp_slot[cid + s] = SLOT_EMPTY; + acc += u2f(v); } } + return acc; +} - uint32_t elem_per_round = lmul * lmul_m1_elem; - uint32_t rounds = dotp_l.M / elem_per_round / num_cores; +// Groups of 4 under a barrier, then core 0 adds the group sums. Every +// partial is already in dotp_part, behind a barrier. +static inline float reduce_two_level(float acc, const uint32_t cid, + const uint32_t num_cores) { + const uint32_t red_group = 4; + + if (cid % red_group == 0) { + for (uint32_t i = 1; i < red_group && (cid + i) < num_cores; ++i) + acc += dotp_part[cid + i]; + dotp_part[cid] = acc; + snrt_fence_snitch(); + } - if ((elem_per_round * num_cores) < (l2_block_elem * l2_channel)) { - if (is_primary && cid == 0) { - printf("Warning: Current scheme cannot utilize all bandwidth!\n"); - } + snrt_cluster_host0_barrier(SNRT_HOST_BARRIER_SLOT); + + if (cid == 0) + for (uint32_t g = red_group; g < num_cores; g += red_group) + acc += dotp_part[g]; + return acc; +} + +// 2 * M operations against 4 FPUs per core, 2 operations each +static void print_perf(const char *name, const uint32_t cycles, + const uint32_t m, const uint32_t num_cores) { + const uint32_t perf = 1000 * 2 * m / cycles; + printf("%s: %u cycles, %u OP/1000cycle (%u%%o utilization).\n", name, + cycles, perf, perf / (2 * num_cores * 4)); +} + +int main() { + const uint32_t num_cores = snrt_cluster_vpu_num(); + const uint32_t cores_per_tile = snrt_cluster_vpu_per_tile(); + const uint32_t cid = snrt_cluster_vpu_idx(); + const int is_primary = snrt_cluster_is_primary(); + + // VLEN in floats + uint32_t vlenb; + asm volatile("csrr %0, vlenb" : "=r"(vlenb)); + const uint32_t vlen_elem = vlenb / sizeof(float); + + const uint32_t elem_per_core = dotp_l.M / num_cores; + + const uint32_t lmul = DOTP_LMUL; + const uint32_t unroll = (DOTP_UNROLL == 2 && lmul <= 4) ? 2 : 1; + const uint32_t elem_per_round = lmul * vlen_elem; + const uint32_t rounds = elem_per_core / elem_per_round; + const uint32_t offset = elem_per_round * num_cores; + + if (num_cores > MAX_CORES || rounds == 0 || + rounds * elem_per_round != elem_per_core) { + if (is_primary && cid == 0) + printf("FATAL: M %u does not split into %u cores x %u elements\n", + dotp_l.M, num_cores, elem_per_round); + return -2; } - const uint32_t l1_scramble_bits = 31 - __builtin_clz(elem_per_round*32/8); + // One stripe of all cores' chunks should cover every HBM channel + const uint32_t l2_stripe = SNRT_L2_CHANNEL * SNRT_L2_INTERLEAVE * + SNRT_L2_BANK_WIDTH / 8; + if (is_primary && cid == 0 && offset * sizeof(float) < l2_stripe) + printf("Warning: a stripe of %u B is below the %u B HBM stripe\n", + offset * (uint32_t)sizeof(float), l2_stripe); + + // A core's chunk is one L1 controller block, so core k of a tile streams + // through controller k. + const uint32_t xbar_offset = 31 - __builtin_clz(elem_per_round * sizeof(float)); // Must be called by all cores, so it stays above the is_primary guard below. - l1d_xbar_config(l1_scramble_bits); + l1d_xbar_config(xbar_offset); +#ifndef ISO_NO_PART + l1d_part_folded(L1D_PART, L1D_FOLD); +#endif // Host 1 never touches Spatz or the shared result/timer state in this // kernel, so the rest of main() is gated by one check instead of many. if (is_primary) { - uint32_t elem_jump_per_round = elem_per_round * num_cores; - - uint32_t timer = (uint32_t)-1; - uint32_t timer_tmp, timer_iter1; - - float *a_int = dotp_A_dram + cid * elem_per_round; - float *b_int = dotp_B_dram + cid * elem_per_round; - - if (cid == 0) { - printf("lmul:%u, elem:%u, offs:%u, iter:%u\n", lmul, elem_per_round, elem_jump_per_round, rounds); - } + const float *a = (const float *)dotp_A_dram + cid * elem_per_round; + const float *b = (const float *)dotp_B_dram + cid * elem_per_round; + + // Cycles up to the per-core partials, and in total with the cross-core + // reduction + uint32_t part_cold = 0; + uint32_t part_hot = (uint32_t)-1; + uint32_t timer_cold = 0; + uint32_t timer_hot = (uint32_t)-1; + int errors = 0; + + if (cid == 0) + printf("lmul %u, unroll %u, vred %u, xred %u, L1 part %u fold %u, " + "xbar %u, elem %u, offs %u, rounds %u\n", + lmul, unroll, DOTP_VRED, DOTP_XRED, L1D_PART, L1D_FOLD, + xbar_offset, elem_per_round, offset, rounds); // LOCKED mode gives host 0 unarbitrated Spatz access; FREE mode would // round-robin-arbitrate with host 1 even though it never issues here. spatz_lock_acquire(); - for (int iter = 0; iter < measure_iter; iter ++) { + for (uint32_t iter = 0; iter < MEASURE_ITER; iter++) { if (cid == 0) start_kernel(); + // The fence publishes the slots emptied in the previous iteration + asm volatile("fence" ::: "memory"); snrt_cluster_host0_barrier(SNRT_HOST_BARRIER_SLOT); - timer_tmp = benchmark_get_cycle(); - - float acc; - - if (lmul >= 8) - acc = fdotp_v32b_lmul8(a_int, b_int, elem_jump_per_round, elem_per_round, rounds); - else if (lmul >= 4) - acc = fdotp_v32b_lmul4(a_int, b_int, elem_jump_per_round, elem_per_round, rounds); - else if (lmul >= 2) - acc = fdotp_v32b_lmul2(a_int, b_int, elem_jump_per_round, elem_per_round, rounds); - else if (lmul >= 1) - acc = fdotp_v32b_lmul1(a_int, b_int, elem_jump_per_round, elem_per_round, rounds); - else - return -3; + uint32_t timer = benchmark_get_cycle(); - result[cid] = acc; + float acc = fdotp_v32b(a, b, elem_per_round, offset, rounds, lmul, + unroll, DOTP_VRED); - snrt_fence_spatz(); + // Every core has its partial: the end of the part that scales with M + dotp_part[cid] = acc; + snrt_fence_snitch(); snrt_cluster_host0_barrier(SNRT_HOST_BARRIER_SLOT); + const uint32_t part = benchmark_get_cycle() - timer; - if (cid == 0) { - timer_tmp = benchmark_get_cycle() - timer_tmp; - timer = (timer < timer_tmp) ? timer : timer_tmp; - if (iter == 0) - timer_iter1 = timer; + if (DOTP_XRED) + acc = reduce_tree(acc, cid, num_cores); + else + acc = reduce_two_level(acc, cid, num_cores); + // Core 0 holds the full sum, so every core has finished its part + if (cid == 0) { + timer = benchmark_get_cycle() - timer; stop_kernel(); - } - - // Two-level reduction tree, group size 4. - const uint32_t red_group = 4; - if (cid % red_group == 0) { - for (uint32_t i = 1; i < red_group && (cid + i) < num_cores; ++i) - acc += result[cid + i]; - result[cid] = acc; - } - - snrt_cluster_host0_barrier(SNRT_HOST_BARRIER_SLOT); - - if (cid == 0) { - for (uint32_t g = red_group; g < num_cores; g += red_group) - acc += result[g]; - result[0] = acc; + if (iter == 0) { + part_cold = part; + timer_cold = timer; + } else { + if (part < part_hot) + part_hot = part; + if (timer < timer_hot) + timer_hot = timer; + } + + if (fp_check(acc, dotp_result)) { + errors++; + printf("Iteration %u: check failed. Calc:", iter); + snrt_printf_float(acc); + printf(", Exp:"); + snrt_printf_float(dotp_result); + printf("\n"); + } } } if (cid == 0) { - // timer excludes the reduction above. - uint32_t performance = 1000 * 2 * dotp_l.M / timer; - uint32_t perf_iter1 = 1000 * 2 * dotp_l.M / timer_iter1; - uint32_t utilization = performance / (2 * num_cores * 4); - uint32_t util_iter1 = perf_iter1 / (2 * num_cores * 4); - write_cyc(timer); - - printf("\n----- (%d) sp fdotp -----\n", dotp_l.M); - printf("The 1st execution took %u cycles.\n", timer_iter1); - printf("The performance is %u OP/1000cycle (%u%%o utilization).\n", - perf_iter1 , util_iter1); - printf("The execution took %u cycles.\n", timer); - printf("The performance is %u OP/1000cycle (%u%%o utilization).\n", - performance, utilization); - } - - if (cid == 0) { - if (fp_check(result[0], dotp_result*measure_iter)) { + write_cyc(timer_hot); + + printf("\n----- (%u) sp fdotp -----\n", dotp_l.M); + // Up to the per-core partials: the measure for a long vector, where + // the cross-core reduction is negligible + print_perf("Partials, 1st execution", part_cold, dotp_l.M, num_cores); + print_perf("Partials", part_hot, dotp_l.M, num_cores); + // With the cross-core reduction: the cost of one dot product + print_perf("Total, 1st execution", timer_cold, dotp_l.M, num_cores); + print_perf("Total", timer_hot, dotp_l.M, num_cores); + if (errors) printf("Check Failed!\n"); - printf("Calc:"); - snrt_printf_float(result[0]); - printf(", Exp:"); - snrt_printf_float((float)(dotp_result * measure_iter)); - printf("\n"); - return -1; - } + num_errors = errors; } - snrt_cluster_host0_barrier(SNRT_HOST_BARRIER_SLOT); - // main()'s epilogue restores fs0 unconditionally, even for host 1. spatz_lock_release(); } @@ -184,5 +283,5 @@ int main() { // Explicit closing barrier instead of relying on the implicit post-main() one. snrt_cluster_hw_barrier(); - return 0; + return (num_errors > 0) ? -1 : 0; } diff --git a/software/kernels/fp/fdotp-32b/script/gen_data.py b/software/kernels/fp/fdotp-32b/script/gen_data.py index 8513133a..d27eb606 100644 --- a/software/kernels/fp/fdotp-32b/script/gen_data.py +++ b/software/kernels/fp/fdotp-32b/script/gen_data.py @@ -49,7 +49,7 @@ def emit_header_file(layer_type: str, **kwargs): + "// This file was generated automatically.\n\n" ) - file = file_path / ("data_" + str(kwargs["M"]) + ".h") + file = file_path / (kwargs["cfg_name"] + ".h") emit_str += emit_dotp_layer(**kwargs) with file.open("w") as f: f.write(emit_str) @@ -88,11 +88,6 @@ def emit_dotp_layer(name="dotp", **kwargs): + array_to_cstr(result) + ";\n\n\n" ) - layer_str += '// Global memory location to hold reduction results for each core\n' - layer_str += ( - f'{dtype} result [64] __attribute__((section(".data"))) = {{0}}' - + ";\n\n\n" - ) else: layer_str += ( f"static {dtype} {name}_A_dram [{m}] = " @@ -164,8 +159,16 @@ def main(): with args.cfg.open() as f: param = hjson.loads(f.read()) - vec_A, bits_A = rand_data_generator((param["M"], 1), param["prec"]) - vec_B, bits_B = rand_data_generator((param["M"], 1), param["prec"]) + if param.get("init", "rand") == "const": + # Small integers: every partial sum is exact in fp32, so the result + # must match exactly, and a dropped or doubled chunk shows up. + idx = torch.arange(param["M"]).reshape(param["M"], 1) + vec_A = (idx % 4 + 1).to(torch.float32) + vec_B = (idx % 3 + 1).to(torch.float32) + bits_A, bits_B = {}, {} + else: + vec_A, bits_A = rand_data_generator((param["M"], 1), param["prec"]) + vec_B, bits_B = rand_data_generator((param["M"], 1), param["prec"]) result = dotp(vec_A, vec_B) kwargs = { @@ -177,6 +180,7 @@ def main(): "expand": param["expand"], "bits_A": bits_A, "bits_B": bits_B, + "cfg_name": args.cfg.stem, } emit_header_file("dotp", **kwargs) diff --git a/software/rlc/CMakeLists.txt b/software/rlc/CMakeLists.txt index 798b8c37..38de9f14 100644 --- a/software/rlc/CMakeLists.txt +++ b/software/rlc/CMakeLists.txt @@ -10,6 +10,9 @@ # The shared test macros and the benchmark/spin_lock/mcs_lock libraries come # from software/cmake/CachePoolTests.cmake. +# Binaries of this group are named rlc- instead of test-. +set(SNITCH_TEST_BIN_PREFIX "rlc-") + # add_snitch_test(multi_producer_single_consumer_double_linked_list multi_producer_single_consumer_double_linked_list/main.c) # add_spatz_test_threeParam(multi_producer_single_consumer_double_linked_list multi_producer_single_consumer_double_linked_list/main.c 1 1350 1000) # add_spatz_test_threeParam(multi_producer_single_consumer_double_linked_list multi_producer_single_consumer_double_linked_list/main.c 1 2044 100) @@ -28,13 +31,13 @@ add_spatz_test_threeParam(multi_producer_single_consumer_double_linked_list mult macro(add_spatz_test_rlc_am_wq name file param1 param2 param3 pcore ccore grant verify) set(target_name ${name}_M${param1}_N${param2}_K${param3}_P${pcore}_C${ccore}_am_G${grant}_wq) add_snitch_test(${target_name} ${file}) - target_link_libraries(test-${SNITCH_TEST_PREFIX}${target_name} benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) - target_compile_definitions(test-${SNITCH_TEST_PREFIX}${target_name} PUBLIC + target_link_libraries(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}${target_name} benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) + target_compile_definitions(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}${target_name} PUBLIC DATAHEADER="data/data_${param1}_${param2}_${param3}.h" PRODUCER_CORE_NUM=${pcore} CONSUMER_CORE_NUM=${ccore} RLC_TB_MODE=1 RLC_GRANT_BYTES=${grant} RLC_AM_WORKQ=1) if(${verify}) - target_compile_definitions(test-${SNITCH_TEST_PREFIX}${target_name} PUBLIC + target_compile_definitions(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}${target_name} PUBLIC RLC_PLAN_VERIFY=1 RLC_SELF_CHECK=1) endif() endmacro() @@ -42,15 +45,15 @@ endmacro() macro(add_spatz_test_rlc_am name file param1 param2 param3 pcore ccore grant verify) set(target_name ${name}_M${param1}_N${param2}_K${param3}_P${pcore}_C${ccore}_am_G${grant}) add_snitch_test(${target_name} ${file}) - target_link_libraries(test-${SNITCH_TEST_PREFIX}${target_name} benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) - target_compile_definitions(test-${SNITCH_TEST_PREFIX}${target_name} PUBLIC + target_link_libraries(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}${target_name} benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) + target_compile_definitions(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}${target_name} PUBLIC DATAHEADER="data/data_${param1}_${param2}_${param3}.h" PRODUCER_CORE_NUM=${pcore} CONSUMER_CORE_NUM=${ccore} RLC_TB_MODE=1 RLC_GRANT_BYTES=${grant}) if(${verify}) # Diff the vectorised planner against the scalar reference, and decode # every assembled transport block back to check the execute stage. - target_compile_definitions(test-${SNITCH_TEST_PREFIX}${target_name} PUBLIC + target_compile_definitions(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}${target_name} PUBLIC RLC_PLAN_VERIFY=1 RLC_SELF_CHECK=1) endif() endmacro() @@ -58,8 +61,8 @@ endmacro() macro(add_spatz_test_rlc name file param1 param2 param3 pcore ccore) set(target_name ${name}_M${param1}_N${param2}_K${param3}_P${pcore}_C${ccore}) add_snitch_test(${target_name} ${file}) - target_link_libraries(test-${SNITCH_TEST_PREFIX}${target_name} benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) - target_compile_definitions(test-${SNITCH_TEST_PREFIX}${target_name} PUBLIC + target_link_libraries(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}${target_name} benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) + target_compile_definitions(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}${target_name} PUBLIC DATAHEADER="data/data_${param1}_${param2}_${param3}.h" PRODUCER_CORE_NUM=${pcore} CONSUMER_CORE_NUM=${ccore}) endmacro() @@ -88,8 +91,8 @@ add_spatz_test_rlc_am(multi_producer_single_consumer_double_linked_list multi_pr macro(add_spatz_test_rlc_probe suffix name file param1 param2 param3) set(target_name ${name}_M${param1}_N${param2}_K${param3}${suffix}_probe) add_snitch_test(${target_name} ${file}) - target_link_libraries(test-${SNITCH_TEST_PREFIX}${target_name} benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) - target_compile_definitions(test-${SNITCH_TEST_PREFIX}${target_name} PUBLIC + target_link_libraries(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}${target_name} benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) + target_compile_definitions(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}${target_name} PUBLIC DATAHEADER="data/data_${param1}_${param2}_${param3}.h" RLC_PROBE=1 ${ARGN}) endmacro() add_spatz_test_rlc_probe("" multi_producer_single_consumer_double_linked_list multi_producer_single_consumer_double_linked_list/main.c 1 1350 100) @@ -116,8 +119,8 @@ macro(add_spatz_test_rlc_full layout variant name file param1 param2 param3) set(target_name ${name}_M${param1}_N${param2}_K${param3}_${layout}${variant}) endif() add_snitch_test(${target_name} ${file}) - target_link_libraries(test-${SNITCH_TEST_PREFIX}${target_name} benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) - target_compile_definitions(test-${SNITCH_TEST_PREFIX}${target_name} PUBLIC + target_link_libraries(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}${target_name} benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) + target_compile_definitions(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}${target_name} PUBLIC DATAHEADER="data/data_${param1}_${param2}_${param3}_${layout_lc}.h" MM_POOL_PAGES=${param3} RLC_PROBE=${probe} ${ARGN}) endforeach() @@ -202,9 +205,9 @@ endforeach() # in one run. add_snitch_test(multi_producer_single_consumer_double_linked_list_M1_N1350_K100_P2_C2_am_G8192_novec multi_producer_single_consumer_double_linked_list/main.c) -target_link_libraries(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M1_N1350_K100_P2_C2_am_G8192_novec +target_link_libraries(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M1_N1350_K100_P2_C2_am_G8192_novec benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) -target_compile_definitions(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M1_N1350_K100_P2_C2_am_G8192_novec PUBLIC +target_compile_definitions(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M1_N1350_K100_P2_C2_am_G8192_novec PUBLIC DATAHEADER="data/data_1_1350_100.h" PRODUCER_CORE_NUM=2 CONSUMER_CORE_NUM=2 RLC_TB_MODE=1 RLC_GRANT_BYTES=8192 RLC_PLAN_IMPL=0 RLC_SELF_CHECK=1) @@ -216,8 +219,8 @@ target_compile_definitions(test-${SNITCH_TEST_PREFIX}multi_producer_single_consu macro(add_rlc_am_bisect suffix vecmask) set(tn multi_producer_single_consumer_double_linked_list_M1_N1350_K100_P2_C2_am_G8192_${suffix}) add_snitch_test(${tn} multi_producer_single_consumer_double_linked_list/main.c) - target_link_libraries(test-${SNITCH_TEST_PREFIX}${tn} benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) - target_compile_definitions(test-${SNITCH_TEST_PREFIX}${tn} PUBLIC + target_link_libraries(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}${tn} benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) + target_compile_definitions(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}${tn} PUBLIC DATAHEADER="data/data_1_1350_100.h" PRODUCER_CORE_NUM=2 CONSUMER_CORE_NUM=2 RLC_TB_MODE=1 RLC_GRANT_BYTES=8192 RLC_SELF_CHECK=1 @@ -239,9 +242,9 @@ add_rlc_am_bisect(v4 4) # + first-crossing only # GVSoC can no longer answer, since its cross-core shared-data path is unsound. add_snitch_test(multi_producer_single_consumer_double_linked_list_M1_N1350_K10_P2_C2_am_G8192_tbchk multi_producer_single_consumer_double_linked_list/main.c) -target_link_libraries(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M1_N1350_K10_P2_C2_am_G8192_tbchk +target_link_libraries(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M1_N1350_K10_P2_C2_am_G8192_tbchk benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) -target_compile_definitions(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M1_N1350_K10_P2_C2_am_G8192_tbchk PUBLIC +target_compile_definitions(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M1_N1350_K10_P2_C2_am_G8192_tbchk PUBLIC DATAHEADER="data/data_1_1350_10.h" PRODUCER_CORE_NUM=2 CONSUMER_CORE_NUM=2 RLC_TB_MODE=1 RLC_GRANT_BYTES=8192 RLC_SELF_CHECK=1) @@ -252,9 +255,9 @@ target_compile_definitions(test-${SNITCH_TEST_PREFIX}multi_producer_single_consu # of TC2's work. RLC_SELF_CHECK only, for the same reason as the tbchk target. add_snitch_test(multi_producer_single_consumer_double_linked_list_M8_N1350_K24_P2_C2_am_G8192_tbchk multi_producer_single_consumer_double_linked_list/main.c) -target_link_libraries(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M8_N1350_K24_P2_C2_am_G8192_tbchk +target_link_libraries(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M8_N1350_K24_P2_C2_am_G8192_tbchk benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) -target_compile_definitions(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M8_N1350_K24_P2_C2_am_G8192_tbchk PUBLIC +target_compile_definitions(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M8_N1350_K24_P2_C2_am_G8192_tbchk PUBLIC DATAHEADER="data/data_8_1350_24.h" PRODUCER_CORE_NUM=2 CONSUMER_CORE_NUM=2 RLC_TB_MODE=1 RLC_GRANT_BYTES=8192 RLC_SELF_CHECK=1) @@ -265,9 +268,9 @@ target_compile_definitions(test-${SNITCH_TEST_PREFIX}multi_producer_single_consu # entity count; if it also wedges, entity count is implicated on its own. add_snitch_test(multi_producer_single_consumer_double_linked_list_M48_N1350_K24_P2_C2_am_G8192_tbchk multi_producer_single_consumer_double_linked_list/main.c) -target_link_libraries(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M48_N1350_K24_P2_C2_am_G8192_tbchk +target_link_libraries(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M48_N1350_K24_P2_C2_am_G8192_tbchk benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) -target_compile_definitions(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M48_N1350_K24_P2_C2_am_G8192_tbchk PUBLIC +target_compile_definitions(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M48_N1350_K24_P2_C2_am_G8192_tbchk PUBLIC DATAHEADER="data/data_48_1350_24.h" PRODUCER_CORE_NUM=2 CONSUMER_CORE_NUM=2 RLC_TB_MODE=1 RLC_GRANT_BYTES=8192 RLC_SELF_CHECK=1) @@ -278,9 +281,9 @@ target_compile_definitions(test-${SNITCH_TEST_PREFIX}multi_producer_single_consu # byte-compares in the verifier -- minutes, not hours. add_snitch_test(multi_producer_single_consumer_double_linked_list_M1_N1350_K4_P2_C2_am_G8192_tbchk multi_producer_single_consumer_double_linked_list/main.c) -target_link_libraries(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M1_N1350_K4_P2_C2_am_G8192_tbchk +target_link_libraries(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M1_N1350_K4_P2_C2_am_G8192_tbchk benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) -target_compile_definitions(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M1_N1350_K4_P2_C2_am_G8192_tbchk PUBLIC +target_compile_definitions(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M1_N1350_K4_P2_C2_am_G8192_tbchk PUBLIC DATAHEADER="data/data_1_1350_4.h" PRODUCER_CORE_NUM=2 CONSUMER_CORE_NUM=2 RLC_TB_MODE=1 RLC_GRANT_BYTES=8192 RLC_SELF_CHECK=1) @@ -292,18 +295,18 @@ target_compile_definitions(test-${SNITCH_TEST_PREFIX}multi_producer_single_consu # entirely -- so F5 is the enabler here, not an optimisation. add_snitch_test(multi_producer_single_consumer_double_linked_list_M8_N1350_K8_P2_C2_am_G8192_wqchk multi_producer_single_consumer_double_linked_list/main.c) -target_link_libraries(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M8_N1350_K8_P2_C2_am_G8192_wqchk +target_link_libraries(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M8_N1350_K8_P2_C2_am_G8192_wqchk benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) -target_compile_definitions(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M8_N1350_K8_P2_C2_am_G8192_wqchk PUBLIC +target_compile_definitions(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M8_N1350_K8_P2_C2_am_G8192_wqchk PUBLIC DATAHEADER="data/data_8_1350_8.h" PRODUCER_CORE_NUM=2 CONSUMER_CORE_NUM=2 RLC_TB_MODE=1 RLC_GRANT_BYTES=8192 RLC_SELF_CHECK=1 RLC_AM_WORKQ=1) add_snitch_test(multi_producer_single_consumer_double_linked_list_M48_N1350_K8_P2_C2_am_G8192_wqchk multi_producer_single_consumer_double_linked_list/main.c) -target_link_libraries(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M48_N1350_K8_P2_C2_am_G8192_wqchk +target_link_libraries(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M48_N1350_K8_P2_C2_am_G8192_wqchk benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) -target_compile_definitions(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M48_N1350_K8_P2_C2_am_G8192_wqchk PUBLIC +target_compile_definitions(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M48_N1350_K8_P2_C2_am_G8192_wqchk PUBLIC DATAHEADER="data/data_48_1350_8.h" PRODUCER_CORE_NUM=2 CONSUMER_CORE_NUM=2 RLC_TB_MODE=1 RLC_GRANT_BYTES=8192 RLC_SELF_CHECK=1 RLC_AM_WORKQ=1) @@ -314,18 +317,18 @@ target_compile_definitions(test-${SNITCH_TEST_PREFIX}multi_producer_single_consu # GVSoC ladder's axis, at a workload RTL can actually finish. add_snitch_test(multi_producer_single_consumer_double_linked_list_M8_N1350_K8_P2_C2_am_G8192_tbchk multi_producer_single_consumer_double_linked_list/main.c) -target_link_libraries(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M8_N1350_K8_P2_C2_am_G8192_tbchk +target_link_libraries(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M8_N1350_K8_P2_C2_am_G8192_tbchk benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) -target_compile_definitions(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M8_N1350_K8_P2_C2_am_G8192_tbchk PUBLIC +target_compile_definitions(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M8_N1350_K8_P2_C2_am_G8192_tbchk PUBLIC DATAHEADER="data/data_8_1350_8.h" PRODUCER_CORE_NUM=2 CONSUMER_CORE_NUM=2 RLC_TB_MODE=1 RLC_GRANT_BYTES=8192 RLC_SELF_CHECK=1) add_snitch_test(multi_producer_single_consumer_double_linked_list_M48_N1350_K8_P2_C2_am_G8192_tbchk multi_producer_single_consumer_double_linked_list/main.c) -target_link_libraries(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M48_N1350_K8_P2_C2_am_G8192_tbchk +target_link_libraries(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M48_N1350_K8_P2_C2_am_G8192_tbchk benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) -target_compile_definitions(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M48_N1350_K8_P2_C2_am_G8192_tbchk PUBLIC +target_compile_definitions(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M48_N1350_K8_P2_C2_am_G8192_tbchk PUBLIC DATAHEADER="data/data_48_1350_8.h" PRODUCER_CORE_NUM=2 CONSUMER_CORE_NUM=2 RLC_TB_MODE=1 RLC_GRANT_BYTES=8192 RLC_SELF_CHECK=1) @@ -337,9 +340,9 @@ target_compile_definitions(test-${SNITCH_TEST_PREFIX}multi_producer_single_consu # so_next chaining becomes the thing under test. add_snitch_test(multi_producer_single_consumer_double_linked_list_M1_N1350_K4_P2_C2_am_G1024_tbchk multi_producer_single_consumer_double_linked_list/main.c) -target_link_libraries(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M1_N1350_K4_P2_C2_am_G1024_tbchk +target_link_libraries(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M1_N1350_K4_P2_C2_am_G1024_tbchk benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) -target_compile_definitions(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M1_N1350_K4_P2_C2_am_G1024_tbchk PUBLIC +target_compile_definitions(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M1_N1350_K4_P2_C2_am_G1024_tbchk PUBLIC DATAHEADER="data/data_1_1350_4.h" PRODUCER_CORE_NUM=2 CONSUMER_CORE_NUM=2 RLC_TB_MODE=1 RLC_GRANT_BYTES=1024 RLC_SELF_CHECK=1) @@ -352,18 +355,18 @@ target_compile_definitions(test-${SNITCH_TEST_PREFIX}multi_producer_single_consu # M32 fails under either -> success is layout-contingent add_snitch_test(multi_producer_single_consumer_double_linked_list_M32_N1350_K24_P2_C2_am_G8192_padshift multi_producer_single_consumer_double_linked_list/main.c) -target_link_libraries(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M32_N1350_K24_P2_C2_am_G8192_padshift +target_link_libraries(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M32_N1350_K24_P2_C2_am_G8192_padshift benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) -target_compile_definitions(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M32_N1350_K24_P2_C2_am_G8192_padshift PUBLIC +target_compile_definitions(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M32_N1350_K24_P2_C2_am_G8192_padshift PUBLIC DATAHEADER="data/data_32_1350_24.h" PRODUCER_CORE_NUM=2 CONSUMER_CORE_NUM=2 RLC_TB_MODE=1 RLC_GRANT_BYTES=8192 RLC_SELF_CHECK=1 RLC_LAYOUT_PAD=320) add_snitch_test(multi_producer_single_consumer_double_linked_list_M32_N1350_K24_P2_C2_am_G8192_entpad multi_producer_single_consumer_double_linked_list/main.c) -target_link_libraries(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M32_N1350_K24_P2_C2_am_G8192_entpad +target_link_libraries(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M32_N1350_K24_P2_C2_am_G8192_entpad benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) -target_compile_definitions(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M32_N1350_K24_P2_C2_am_G8192_entpad PUBLIC +target_compile_definitions(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M32_N1350_K24_P2_C2_am_G8192_entpad PUBLIC DATAHEADER="data/data_32_1350_24.h" PRODUCER_CORE_NUM=2 CONSUMER_CORE_NUM=2 RLC_TB_MODE=1 RLC_GRANT_BYTES=8192 RLC_SELF_CHECK=1 RLC_AM_ENT_PADDED=1) @@ -374,27 +377,27 @@ target_compile_definitions(test-${SNITCH_TEST_PREFIX}multi_producer_single_consu # both fail -> neither; look elsewhere add_snitch_test(multi_producer_single_consumer_double_linked_list_M16_N1350_K24_P2_C2_am_G8192_pad0 multi_producer_single_consumer_double_linked_list/main.c) -target_link_libraries(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M16_N1350_K24_P2_C2_am_G8192_pad0 +target_link_libraries(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M16_N1350_K24_P2_C2_am_G8192_pad0 benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) -target_compile_definitions(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M16_N1350_K24_P2_C2_am_G8192_pad0 PUBLIC +target_compile_definitions(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M16_N1350_K24_P2_C2_am_G8192_pad0 PUBLIC DATAHEADER="data/data_16_1350_24.h" PRODUCER_CORE_NUM=2 CONSUMER_CORE_NUM=2 RLC_TB_MODE=1 RLC_GRANT_BYTES=8192 RLC_SELF_CHECK=1) add_snitch_test(multi_producer_single_consumer_double_linked_list_M16_N1350_K24_P2_C2_am_G8192_padshift multi_producer_single_consumer_double_linked_list/main.c) -target_link_libraries(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M16_N1350_K24_P2_C2_am_G8192_padshift +target_link_libraries(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M16_N1350_K24_P2_C2_am_G8192_padshift benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) -target_compile_definitions(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M16_N1350_K24_P2_C2_am_G8192_padshift PUBLIC +target_compile_definitions(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M16_N1350_K24_P2_C2_am_G8192_padshift PUBLIC DATAHEADER="data/data_16_1350_24.h" PRODUCER_CORE_NUM=2 CONSUMER_CORE_NUM=2 RLC_TB_MODE=1 RLC_GRANT_BYTES=8192 RLC_SELF_CHECK=1 RLC_LAYOUT_PAD=320) add_snitch_test(multi_producer_single_consumer_double_linked_list_M16_N1350_K24_P2_C2_am_G8192_entpad multi_producer_single_consumer_double_linked_list/main.c) -target_link_libraries(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M16_N1350_K24_P2_C2_am_G8192_entpad +target_link_libraries(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M16_N1350_K24_P2_C2_am_G8192_entpad benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) -target_compile_definitions(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M16_N1350_K24_P2_C2_am_G8192_entpad PUBLIC +target_compile_definitions(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M16_N1350_K24_P2_C2_am_G8192_entpad PUBLIC DATAHEADER="data/data_16_1350_24.h" PRODUCER_CORE_NUM=2 CONSUMER_CORE_NUM=2 RLC_TB_MODE=1 RLC_GRANT_BYTES=8192 RLC_SELF_CHECK=1 RLC_AM_ENT_PADDED=1) @@ -404,36 +407,36 @@ target_compile_definitions(test-${SNITCH_TEST_PREFIX}multi_producer_single_consu # gradual one implies accumulation. Those are different bugs. add_snitch_test(multi_producer_single_consumer_double_linked_list_M12_N1350_K24_P2_C2_am_G8192_tbchk multi_producer_single_consumer_double_linked_list/main.c) -target_link_libraries(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M12_N1350_K24_P2_C2_am_G8192_tbchk +target_link_libraries(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M12_N1350_K24_P2_C2_am_G8192_tbchk benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) -target_compile_definitions(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M12_N1350_K24_P2_C2_am_G8192_tbchk PUBLIC +target_compile_definitions(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M12_N1350_K24_P2_C2_am_G8192_tbchk PUBLIC DATAHEADER="data/data_12_1350_24.h" PRODUCER_CORE_NUM=2 CONSUMER_CORE_NUM=2 RLC_TB_MODE=1 RLC_GRANT_BYTES=8192 RLC_SELF_CHECK=1) add_snitch_test(multi_producer_single_consumer_double_linked_list_M16_N1350_K24_P2_C2_am_G8192_tbchk multi_producer_single_consumer_double_linked_list/main.c) -target_link_libraries(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M16_N1350_K24_P2_C2_am_G8192_tbchk +target_link_libraries(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M16_N1350_K24_P2_C2_am_G8192_tbchk benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) -target_compile_definitions(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M16_N1350_K24_P2_C2_am_G8192_tbchk PUBLIC +target_compile_definitions(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M16_N1350_K24_P2_C2_am_G8192_tbchk PUBLIC DATAHEADER="data/data_16_1350_24.h" PRODUCER_CORE_NUM=2 CONSUMER_CORE_NUM=2 RLC_TB_MODE=1 RLC_GRANT_BYTES=8192 RLC_SELF_CHECK=1) add_snitch_test(multi_producer_single_consumer_double_linked_list_M24_N1350_K24_P2_C2_am_G8192_tbchk multi_producer_single_consumer_double_linked_list/main.c) -target_link_libraries(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M24_N1350_K24_P2_C2_am_G8192_tbchk +target_link_libraries(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M24_N1350_K24_P2_C2_am_G8192_tbchk benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) -target_compile_definitions(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M24_N1350_K24_P2_C2_am_G8192_tbchk PUBLIC +target_compile_definitions(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M24_N1350_K24_P2_C2_am_G8192_tbchk PUBLIC DATAHEADER="data/data_24_1350_24.h" PRODUCER_CORE_NUM=2 CONSUMER_CORE_NUM=2 RLC_TB_MODE=1 RLC_GRANT_BYTES=8192 RLC_SELF_CHECK=1) add_snitch_test(multi_producer_single_consumer_double_linked_list_M32_N1350_K24_P2_C2_am_G8192_tbchk multi_producer_single_consumer_double_linked_list/main.c) -target_link_libraries(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M32_N1350_K24_P2_C2_am_G8192_tbchk +target_link_libraries(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M32_N1350_K24_P2_C2_am_G8192_tbchk benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) -target_compile_definitions(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M32_N1350_K24_P2_C2_am_G8192_tbchk PUBLIC +target_compile_definitions(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M32_N1350_K24_P2_C2_am_G8192_tbchk PUBLIC DATAHEADER="data/data_32_1350_24.h" PRODUCER_CORE_NUM=2 CONSUMER_CORE_NUM=2 RLC_TB_MODE=1 RLC_GRANT_BYTES=8192 RLC_SELF_CHECK=1) @@ -443,9 +446,9 @@ target_compile_definitions(test-${SNITCH_TEST_PREFIX}multi_producer_single_consu # alignment-sensitive vector copy from wrong offset arithmetic. add_snitch_test(multi_producer_single_consumer_double_linked_list_M1_N1350_K100_P2_C2_am_G8192_scpy multi_producer_single_consumer_double_linked_list/main.c) -target_link_libraries(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M1_N1350_K100_P2_C2_am_G8192_scpy +target_link_libraries(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M1_N1350_K100_P2_C2_am_G8192_scpy benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) -target_compile_definitions(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M1_N1350_K100_P2_C2_am_G8192_scpy PUBLIC +target_compile_definitions(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M1_N1350_K100_P2_C2_am_G8192_scpy PUBLIC DATAHEADER="data/data_1_1350_100.h" PRODUCER_CORE_NUM=2 CONSUMER_CORE_NUM=2 RLC_TB_MODE=1 RLC_GRANT_BYTES=8192 RLC_SELF_CHECK=1 RLC_DL_COPY_SCALAR=1) @@ -455,9 +458,9 @@ target_compile_definitions(test-${SNITCH_TEST_PREFIX}multi_producer_single_consu # shrink the list). Anything that reproduces here is not a concurrency bug. add_snitch_test(multi_producer_single_consumer_double_linked_list_M1_N1350_K100_P1_C1_am_G8192 multi_producer_single_consumer_double_linked_list/main.c) -target_link_libraries(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M1_N1350_K100_P1_C1_am_G8192 +target_link_libraries(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M1_N1350_K100_P1_C1_am_G8192 benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) -target_compile_definitions(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M1_N1350_K100_P1_C1_am_G8192 PUBLIC +target_compile_definitions(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M1_N1350_K100_P1_C1_am_G8192 PUBLIC DATAHEADER="data/data_1_1350_100_p1c1.h" PRODUCER_CORE_NUM=1 CONSUMER_CORE_NUM=1 RLC_TB_MODE=1 RLC_GRANT_BYTES=8192 RLC_PLAN_VERIFY=1 RLC_SELF_CHECK=1) @@ -472,9 +475,9 @@ target_compile_definitions(test-${SNITCH_TEST_PREFIX}multi_producer_single_consu foreach(M 8 16 24 32) add_snitch_test(multi_producer_single_consumer_double_linked_list_M${M}_N1350_K24_P2_C2_am_G8192_scpy multi_producer_single_consumer_double_linked_list/main.c) - target_link_libraries(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M${M}_N1350_K24_P2_C2_am_G8192_scpy + target_link_libraries(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M${M}_N1350_K24_P2_C2_am_G8192_scpy benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) - target_compile_definitions(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M${M}_N1350_K24_P2_C2_am_G8192_scpy PUBLIC + target_compile_definitions(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M${M}_N1350_K24_P2_C2_am_G8192_scpy PUBLIC DATAHEADER="data/data_${M}_1350_24.h" PRODUCER_CORE_NUM=2 CONSUMER_CORE_NUM=2 RLC_TB_MODE=1 RLC_GRANT_BYTES=8192 RLC_SELF_CHECK=1 RLC_DL_COPY_SCALAR=1) @@ -493,9 +496,9 @@ endforeach() # --------------------------------------------------------------------------- macro(add_rlc_ul name source dataheader prod cons exec tc tb suffix) add_snitch_test(${name}_P${prod}_C${cons}_ul_${suffix} ${source}) - target_link_libraries(test-${SNITCH_TEST_PREFIX}${name}_P${prod}_C${cons}_ul_${suffix} + target_link_libraries(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}${name}_P${prod}_C${cons}_ul_${suffix} benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) - target_compile_definitions(test-${SNITCH_TEST_PREFIX}${name}_P${prod}_C${cons}_ul_${suffix} PUBLIC + target_compile_definitions(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}${name}_P${prod}_C${cons}_ul_${suffix} PUBLIC DATAHEADER="${dataheader}" PRODUCER_CORE_NUM=${prod} CONSUMER_CORE_NUM=${cons} RLC_UL_MODE=1 RLC_UL_EXEC=${exec} RLC_UL_TC=${tc} RLC_UL_TB_BYTES=${tb} @@ -533,7 +536,7 @@ add_rlc_ul(multi_producer_single_consumer_double_linked_list add_rlc_ul(multi_producer_single_consumer_double_linked_list multi_producer_single_consumer_double_linked_list/main.c "data/data_1_1350_100_p1c3.h" 1 3 0 1 2048 ci) -target_compile_definitions(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_P1_C3_ul_ci PUBLIC +target_compile_definitions(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_P1_C3_ul_ci PUBLIC RLC_UL_SDUS=256) # Same cases with the bitmap work queue (RLC_AM_WORKQ=1), so the scan-free path @@ -548,5 +551,5 @@ add_spatz_test_rlc(multi_producer_single_consumer_double_linked_list multi_produ add_spatz_test_rlc(multi_producer_single_consumer_double_linked_list multi_producer_single_consumer_double_linked_list/main.c 256 800 300 128 128) # TC2 with the payload self-check enabled (src vs tgt compare after the barrier) add_snitch_test(multi_producer_single_consumer_double_linked_list_M48_N800_K300_sc multi_producer_single_consumer_double_linked_list/main.c) -target_link_libraries(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M48_N800_K300_sc benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) -target_compile_definitions(test-${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M48_N800_K300_sc PUBLIC DATAHEADER="data/data_48_800_300.h" RLC_SELF_CHECK=1) +target_link_libraries(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M48_N800_K300_sc benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) +target_compile_definitions(${SNITCH_TEST_BIN_PREFIX}${SNITCH_TEST_PREFIX}multi_producer_single_consumer_double_linked_list_M48_N800_K300_sc PUBLIC DATAHEADER="data/data_48_800_300.h" RLC_SELF_CHECK=1) diff --git a/software/snRuntime/README.md b/software/snRuntime/README.md index af85f57e..1997949c 100644 --- a/software/snRuntime/README.md +++ b/software/snRuntime/README.md @@ -137,8 +137,12 @@ across groups. // Requires a flush before changing while valid data is cached. void l1d_addr(uint32_t addr); -// Poll the peripheral until the current flush instruction completes. -// Used by the low-level flush functions; not normally needed in application code. +// Poll the peripheral until the last committed flush instruction and every changed +// configuration (partition, address boundary, xbar offset, iCache prefetch) has been +// applied by every group. Configuration registers need no commit: a changed value is +// applied on its own, only flush/invalidate instructions are committed. +// Used by the low-level flush and configuration functions; not normally needed in +// application code. void l1d_wait(); ``` diff --git a/software/snRuntime/include/cachepool_peripheral.h b/software/snRuntime/include/cachepool_peripheral.h index 750388fb..e4f31d74 100644 --- a/software/snRuntime/include/cachepool_peripheral.h +++ b/software/snRuntime/include/cachepool_peripheral.h @@ -102,15 +102,16 @@ extern "C" { #define CACHEPOOL_PERIPHERAL_L1D_SPM_COMMIT_REG_OFFSET 0x38 #define CACHEPOOL_PERIPHERAL_L1D_SPM_COMMIT_COMMIT_BIT 0 -// Controls the L1 DCache flushing and invalidation. +// Starts the L1D instruction in CFG_L1D_INSN on the tiles selected by #define CACHEPOOL_PERIPHERAL_L1D_INSN_COMMIT_REG_OFFSET 0x3c #define CACHEPOOL_PERIPHERAL_L1D_INSN_COMMIT_COMMIT_BIT 0 -// Indicate the status of flushing +// Busy while a committed L1D instruction or a changed setting is still being #define CACHEPOOL_PERIPHERAL_L1D_FLUSH_STATUS_REG_OFFSET 0x40 #define CACHEPOOL_PERIPHERAL_L1D_FLUSH_STATUS_STATUS_BIT 0 -// L1D partitioning: private bank count and how it is folded +// L1D partitioning: private bank count and how it is folded. Applied when +// the #define CACHEPOOL_PERIPHERAL_L1D_PRIVATE_REG_OFFSET 0x44 #define CACHEPOOL_PERIPHERAL_L1D_PRIVATE_PARTITION_MASK 0x7 #define CACHEPOOL_PERIPHERAL_L1D_PRIVATE_PARTITION_OFFSET 0 @@ -120,10 +121,11 @@ extern "C" { .index = CACHEPOOL_PERIPHERAL_L1D_PRIVATE_PARTITION_OFFSET}) #define CACHEPOOL_PERIPHERAL_L1D_PRIVATE_GROUP_FOLD_BIT 3 -// Starting address of private L1D partition +// Starting address of private L1D partition. Applied when the value changes, +// no #define CACHEPOOL_PERIPHERAL_L1D_ADDR_REG_OFFSET 0x48 -// Cache xbar offset setting +// Cache xbar offset setting. Applied when the value changes, no commit #define CACHEPOOL_PERIPHERAL_XBAR_OFFSET_REG_OFFSET 0x4c #define CACHEPOOL_PERIPHERAL_XBAR_OFFSET_OFFSET_MASK 0x1f #define CACHEPOOL_PERIPHERAL_XBAR_OFFSET_OFFSET_OFFSET 0 @@ -132,10 +134,6 @@ extern "C" { .index = \ CACHEPOOL_PERIPHERAL_XBAR_OFFSET_OFFSET_OFFSET}) -// Cache xbar offset setting -#define CACHEPOOL_PERIPHERAL_XBAR_OFFSET_COMMIT_REG_OFFSET 0x50 -#define CACHEPOOL_PERIPHERAL_XBAR_OFFSET_COMMIT_COMMIT_BIT 0 - #ifdef __cplusplus } // extern "C" #endif diff --git a/software/snRuntime/include/l1cache.h b/software/snRuntime/include/l1cache.h index aec2c73c..3b0a87f5 100644 --- a/software/snRuntime/include/l1cache.h +++ b/software/snRuntime/include/l1cache.h @@ -12,7 +12,6 @@ extern __thread struct snrt_team *_snrt_team_current; // Used to configure cache xbar's arbitration algorithm // Size in Bytes void l1d_xbar_config(uint32_t offset); -void l1d_xbar_commit(); void l1d_commit(); void l1d_init(uint32_t size); diff --git a/software/snRuntime/src/l1cache.c b/software/snRuntime/src/l1cache.c index ca3a5107..6d51cff3 100644 --- a/software/snRuntime/src/l1cache.c +++ b/software/snRuntime/src/l1cache.c @@ -21,26 +21,20 @@ void l1d_xbar_config(uint32_t offset) { asm volatile("fence" ::: "memory"); snrt_cluster_hw_barrier(); if (snrt_cluster_core_idx() == 0) { + // Flush cache before changing the xbar offset: a changed offset is applied as soon as it + // is written, no commit needed + l1d_flush(); + l1d_wait(); uint32_t *cfg = (uint32_t *)(_snrt_team_current->root->cluster_mem.end + CACHEPOOL_PERIPHERAL_XBAR_OFFSET_REG_OFFSET); *cfg = offset; - // Flush cache before committing xbar changes - l1d_flush(); + // Wait until every group has applied the new offset l1d_wait(); - l1d_xbar_commit(); } snrt_cluster_hw_barrier(); } - -void l1d_xbar_commit() { - uint32_t *commit = - (uint32_t *)(_snrt_team_current->root->cluster_mem.end + - CACHEPOOL_PERIPHERAL_XBAR_OFFSET_COMMIT_REG_OFFSET); - *commit = 1; -} - void l1d_commit() { uint32_t *commit = (uint32_t *)(_snrt_team_current->root->cluster_mem.end + @@ -161,8 +155,10 @@ void l1d_part_folded (uint32_t size, uint32_t fold) { volatile uint32_t *cfg_private = (uint32_t *)(_snrt_team_current->root->cluster_mem.end + CACHEPOOL_PERIPHERAL_L1D_PRIVATE_REG_OFFSET); + // Applied as soon as the value changes, no commit needed *cfg_private = (size & 0x7u) | ((fold & 0x1u) << 3); - l1d_commit(); + // Wait until every group has applied the new partition + l1d_wait(); } snrt_cluster_hw_barrier(); } @@ -182,12 +178,10 @@ void l1d_addr (uint32_t addr) { volatile uint32_t *cfg_private = (uint32_t *)(_snrt_team_current->root->cluster_mem.end + CACHEPOOL_PERIPHERAL_L1D_ADDR_REG_OFFSET); + // Applied as soon as the value changes, no commit needed *cfg_private = addr; - l1d_commit(); - // TODO: replace with a real completion status bit in the peripheral. - uint32_t start = read_csr(mcycle); - while ((read_csr(mcycle) - start) < 100) { - } + // Wait until every group has applied the new start address + l1d_wait(); } snrt_cluster_hw_barrier(); } diff --git a/software/snRuntime/src/platforms/standalone/start_snitch.S b/software/snRuntime/src/platforms/standalone/start_snitch.S index ed9a848c..a7a15851 100644 --- a/software/snRuntime/src/platforms/standalone/start_snitch.S +++ b/software/snRuntime/src/platforms/standalone/start_snitch.S @@ -32,6 +32,9 @@ _snrt_exit: mv s0, t0 # call l1d_flush mv a0, s0 + # Earlier UART/CSR writes may still be in flight: they reach different targets and are not + # ordered with the EOC write, so wait for all of them first. + fence call set_eoc 1: ret diff --git a/util/scripts/generate_bootrom.py b/util/scripts/generate_bootrom.py index 24442770..c0b192bf 100755 --- a/util/scripts/generate_bootrom.py +++ b/util/scripts/generate_bootrom.py @@ -120,8 +120,9 @@ def read_bin(): with open(filename + ".bin", "rb") as f: rom = bytes.hex(f.read()) rom = list(map("".join, zip(rom[::2], rom[1::2]))) - # align to 64 bit - align = (int((len(rom) + 7) / 8)) * 8 + # align to 64 bit and to the bootrom word + word = max(8, int(DataWidth / 8)) + align = (int((len(rom) + word - 1) / word)) * word for i in range(len(rom), align): rom.append("00") return rom