diff --git a/ci/testcases/unittest.yaml b/ci/testcases/unittest.yaml index 5a3893e535..5e579d279f 100644 --- a/ci/testcases/unittest.yaml +++ b/ci/testcases/unittest.yaml @@ -11,6 +11,9 @@ tests: - id: hw via: script run: "make -C hw/unittest" + - id: hw-kmu-arb + via: script + run: "make -C hw/unittest/kmu_arb run" # FPU arithmetic units (VX_fma/fdiv/fsqrt/fcvt/fdivsqrt): the `hw` lane above only # builds the unit tests; this lane executes them so a softfloat-divergent # regression actually fails CI (each driver checks bit-exact result+fflags over diff --git a/hw/rtl/VX_cluster.sv b/hw/rtl/VX_cluster.sv index ec8b7ac340..18b05ba7cf 100644 --- a/hw/rtl/VX_cluster.sv +++ b/hw/rtl/VX_cluster.sv @@ -94,6 +94,7 @@ module VX_cluster import VX_gpu_pkg::*; `endif VX_kmu_bus_if per_socket_kmu_bus_if[NUM_SOCKETS](); + wire kmu_fanout_pending; VX_kmu_arb #( .NUM_INPUTS (1), @@ -103,7 +104,8 @@ module VX_cluster import VX_gpu_pkg::*; .clk (clk), .reset (reset), .bus_in_if (kmu_bus_if), - .bus_out_if (per_socket_kmu_bus_if) + .bus_out_if (per_socket_kmu_bus_if), + .pending (kmu_fanout_pending) ); VX_gbar_bus_if per_socket_gbar_bus_if[NUM_SOCKETS](); @@ -521,10 +523,10 @@ module VX_cluster import VX_gpu_pkg::*; wire busy_r; `ifdef EXT_GFX_ANY_ENABLE - `BUFFER_EX(busy_r, dcr_bus_if.req_valid | (|per_socket_busy) | gfx_busy, 1'b1, 1, (NUM_SOCKETS > 1)); + `BUFFER_EX(busy_r, dcr_bus_if.req_valid | kmu_fanout_pending | (|per_socket_busy) | gfx_busy, 1'b1, 1, (NUM_SOCKETS > 1)); `else - `BUFFER_EX(busy_r, dcr_bus_if.req_valid | (|per_socket_busy), 1'b1, 1, (NUM_SOCKETS > 1)); + `BUFFER_EX(busy_r, dcr_bus_if.req_valid | kmu_fanout_pending | (|per_socket_busy), 1'b1, 1, (NUM_SOCKETS > 1)); `endif - assign busy = busy_r | dcr_bus_if.req_valid; + assign busy = busy_r | dcr_bus_if.req_valid | kmu_fanout_pending; endmodule diff --git a/hw/rtl/VX_socket.sv b/hw/rtl/VX_socket.sv index d6d8891b21..bea3b5e4d6 100644 --- a/hw/rtl/VX_socket.sv +++ b/hw/rtl/VX_socket.sv @@ -80,6 +80,7 @@ module VX_socket import VX_gpu_pkg::*; `endif VX_kmu_bus_if per_core_kmu_bus_if[`VX_CFG_SOCKET_SIZE](); + wire kmu_fanout_pending; VX_kmu_arb #( .NUM_INPUTS (1), @@ -89,7 +90,8 @@ module VX_socket import VX_gpu_pkg::*; .clk (clk), .reset (reset), .bus_in_if (kmu_bus_if), - .bus_out_if (per_core_kmu_bus_if) + .bus_out_if (per_core_kmu_bus_if), + .pending (kmu_fanout_pending) ); VX_gbar_bus_if per_core_gbar_bus_if[`VX_CFG_SOCKET_SIZE](); @@ -513,7 +515,7 @@ module VX_socket import VX_gpu_pkg::*; end wire busy_r; - `BUFFER_EX(busy_r, dcr_bus_if.req_valid | (|per_core_busy), 1'b1, 1, (`VX_CFG_SOCKET_SIZE > 1)); - assign busy = busy_r | dcr_bus_if.req_valid; + `BUFFER_EX(busy_r, dcr_bus_if.req_valid | kmu_fanout_pending | (|per_core_busy), 1'b1, 1, (`VX_CFG_SOCKET_SIZE > 1)); + assign busy = busy_r | dcr_bus_if.req_valid | kmu_fanout_pending; endmodule diff --git a/hw/rtl/Vortex.sv b/hw/rtl/Vortex.sv index 4b3d233abc..ab669f104c 100644 --- a/hw/rtl/Vortex.sv +++ b/hw/rtl/Vortex.sv @@ -181,6 +181,7 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*; ( wire [`VX_CFG_NUM_CLUSTERS-1:0] per_cluster_busy; VX_kmu_bus_if per_cluster_kmu_bus_if[`VX_CFG_NUM_CLUSTERS](); + wire kmu_fanout_pending; VX_kmu_arb #( .NUM_INPUTS (1), .NUM_OUTPUTS (`VX_CFG_NUM_CLUSTERS), @@ -189,7 +190,8 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*; ( .clk (clk), .reset (reset), .bus_in_if (kmu_bus_in), - .bus_out_if (per_cluster_kmu_bus_if) + .bus_out_if (per_cluster_kmu_bus_if), + .pending (kmu_fanout_pending) ); `ifdef VX_CFG_EXT_RASTER_ENABLE @@ -245,8 +247,8 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*; ( ); end wire busy_r; - `BUFFER_EX(busy_r, kmu_busy | dcr_bus_if.req_valid | (|per_cluster_busy), 1'b1, 1, (`VX_CFG_NUM_CLUSTERS > 1)); - assign busy = busy_r | kmu_busy | dcr_bus_if.req_valid; + `BUFFER_EX(busy_r, kmu_busy | kmu_fanout_pending | dcr_bus_if.req_valid | (|per_cluster_busy), 1'b1, 1, (`VX_CFG_NUM_CLUSTERS > 1)); + assign busy = busy_r | kmu_busy | kmu_fanout_pending | dcr_bus_if.req_valid; `ifdef PERF_ENABLE diff --git a/hw/rtl/core/VX_core.sv b/hw/rtl/core/VX_core.sv index 7b036a7e5a..ec2cbe6e8b 100644 --- a/hw/rtl/core/VX_core.sv +++ b/hw/rtl/core/VX_core.sv @@ -211,6 +211,7 @@ module VX_core import VX_gpu_pkg::*; #( VX_kmu_bus_if raster_frag_kmu_if(); // distributor → arb VX_kmu_bus_if kmu_arb_in_if[2](); VX_kmu_bus_if sched_kmu_arr_if[1](); // arb → scheduler + wire frag_kmu_merge_pending; // input 0 = device-KMU stream (the core's incoming kmu bus) assign kmu_arb_in_if[0].valid = kmu_bus_if.valid; @@ -230,7 +231,8 @@ module VX_core import VX_gpu_pkg::*; #( .clk (clk), .reset (reset), .bus_in_if (kmu_arb_in_if), - .bus_out_if (sched_kmu_arr_if) + .bus_out_if (sched_kmu_arr_if), + .pending (frag_kmu_merge_pending) ); VX_gfx_win_wr_if #(.NUM_LANES (`VX_CFG_NUM_SFU_LANES)) rast_win_if(); @@ -574,7 +576,7 @@ module VX_core import VX_gpu_pkg::*; #( `endif `ifdef VX_CFG_EXT_RASTER_ENABLE - assign busy = sched_busy || dcr_busy || ~(&lsu_sched_empty) || ~mem_unit_empty || raster_dispatch_busy || raster_packer_busy; + assign busy = sched_busy || dcr_busy || ~(&lsu_sched_empty) || ~mem_unit_empty || raster_dispatch_busy || raster_packer_busy || frag_kmu_merge_pending; `else assign busy = sched_busy || dcr_busy || ~(&lsu_sched_empty) || ~mem_unit_empty; `endif diff --git a/hw/rtl/core/VX_cta_dispatch.sv b/hw/rtl/core/VX_cta_dispatch.sv index f23b0341e8..3b69e45c64 100644 --- a/hw/rtl/core/VX_cta_dispatch.sv +++ b/hw/rtl/core/VX_cta_dispatch.sv @@ -486,9 +486,10 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( // the accept->DISPATCH transition is registered, so gating on state alone // leaves the accept cycle un-busy. With SOCKET_SIZE>1 the socket busy // aggregation is registered, so on the final CTA kmu_busy drops before the - // buffered per-core busy rises, opening a 1-cycle device-busy gap the host's - // edge-sensitive idle-wait latches as premature completion (cores>1 launch - // failure). Covering the accept cycle closes the gap. + // buffered per-core busy rises, opening a 1-cycle device-busy gap that the + // host idle wait treats as premature completion (cores>1 launch failure). + // Covering the accept cycle closes the dispatcher-local gap; + // registered upstream KMU fanouts keep their own pending state in busy. assign busy = (state == DISPATCH) || kmu_bus_if_fire; // ------------------------------------------------------------------------- diff --git a/hw/rtl/core/VX_kmu_arb.sv b/hw/rtl/core/VX_kmu_arb.sv index a97bfbba50..0b7de952da 100644 --- a/hw/rtl/core/VX_kmu_arb.sv +++ b/hw/rtl/core/VX_kmu_arb.sv @@ -27,7 +27,12 @@ module VX_kmu_arb import VX_gpu_pkg::*; #( VX_kmu_bus_if.slave bus_in_if [NUM_INPUTS], // output requests - VX_kmu_bus_if.master bus_out_if [NUM_OUTPUTS] + VX_kmu_bus_if.master bus_out_if [NUM_OUTPUTS], + + // A launch request is still in flight through this fan-out/arbiter. + // Parents include this in their busy chain so a buffered final CTA cannot + // create a false-idle window before the destination core accepts it. + output wire pending ); localparam DATAW = NUM_LANES * $bits(kmu_req_t); @@ -69,4 +74,6 @@ module VX_kmu_arb import VX_gpu_pkg::*; #( assign ready_out[i] = bus_out_if[i].ready; end + assign pending = (| valid_in) || (| valid_out); + endmodule diff --git a/hw/unittest/Makefile b/hw/unittest/Makefile index f30d1194e2..581f7510b3 100644 --- a/hw/unittest/Makefile +++ b/hw/unittest/Makefile @@ -9,6 +9,7 @@ all: $(MAKE) -C tcu_fedp $(MAKE) -C cta_dispatcher $(MAKE) -C kmu + $(MAKE) -C kmu_arb $(MAKE) -C dxa_core $(MAKE) -C tcu_unit $(MAKE) -C cp_arbiter @@ -37,6 +38,7 @@ run: $(MAKE) -C tcu_fedp run $(MAKE) -C cta_dispatcher run $(MAKE) -C kmu run + $(MAKE) -C kmu_arb run $(MAKE) -C dxa_core run $(MAKE) -C tcu_unit run $(MAKE) -C cp_arbiter run @@ -127,6 +129,7 @@ clean: $(MAKE) -C tcu_fedp clean $(MAKE) -C cta_dispatcher clean $(MAKE) -C kmu clean + $(MAKE) -C kmu_arb clean $(MAKE) -C dxa_core clean $(MAKE) -C tcu_unit clean $(MAKE) -C cp_arbiter clean diff --git a/hw/unittest/kmu_arb/Makefile b/hw/unittest/kmu_arb/Makefile new file mode 100644 index 0000000000..7856fcaf5b --- /dev/null +++ b/hw/unittest/kmu_arb/Makefile @@ -0,0 +1,26 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := kmu_arb + +RTL_DIR := $(VORTEX_HOME)/hw/rtl +DPI_DIR := $(VORTEX_HOME)/hw/dpi + +SRC_DIR := $(VORTEX_HOME)/hw/unittest/$(PROJECT) + +CXXFLAGS := -I$(SRC_DIR) -I$(VORTEX_HOME)/hw/unittest/common -I$(SW_COMMON_DIR) +CXXFLAGS += -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw + +SRCS := $(SRC_DIR)/main.cpp + +DBG_TRACE_FLAGS := + +RTL_PKGS := $(RTL_DIR)/VX_gpu_pkg.sv + +RTL_INCLUDE := -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw -I$(RTL_DIR) -I$(DPI_DIR) +RTL_INCLUDE += -I$(RTL_DIR)/libs -I$(RTL_DIR)/interfaces -I$(RTL_DIR)/core +RTL_INCLUDE += -I$(SRC_DIR) + +TOP := VX_kmu_arb_top + +include ../common.mk diff --git a/hw/unittest/kmu_arb/VX_kmu_arb_top.sv b/hw/unittest/kmu_arb/VX_kmu_arb_top.sv new file mode 100644 index 0000000000..8028bf2d4b --- /dev/null +++ b/hw/unittest/kmu_arb/VX_kmu_arb_top.sv @@ -0,0 +1,54 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +module VX_kmu_arb_top import VX_gpu_pkg::*; ( + input wire clk, + input wire reset, + + input wire in_valid, + output wire in_ready, + + input wire [1:0] out_ready, + output wire [1:0] out_valid, + + output wire pending +); + + VX_kmu_bus_if bus_in_if[1](); + VX_kmu_bus_if bus_out_if[2](); + + assign bus_in_if[0].valid = in_valid; + assign bus_in_if[0].data = '0; + assign in_ready = bus_in_if[0].ready; + + for (genvar i = 0; i < 2; ++i) begin : g_outputs + assign bus_out_if[i].ready = out_ready[i]; + assign out_valid[i] = bus_out_if[i].valid; + `UNUSED_VAR (bus_out_if[i].data) + end + + VX_kmu_arb #( + .NUM_INPUTS (1), + .NUM_OUTPUTS (2), + .OUT_BUF (3) + ) dut ( + .clk (clk), + .reset (reset), + .bus_in_if (bus_in_if), + .bus_out_if (bus_out_if), + .pending (pending) + ); + +endmodule diff --git a/hw/unittest/kmu_arb/main.cpp b/hw/unittest/kmu_arb/main.cpp new file mode 100644 index 0000000000..9d51a81709 --- /dev/null +++ b/hw/unittest/kmu_arb/main.cpp @@ -0,0 +1,119 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "vl_simulator.h" +#include "VVX_kmu_arb_top.h" + +#include +#include +#include + +static uint64_t timestamp = 0; +static bool trace_enabled = false; + +double sc_time_stamp() { + return timestamp; +} + +bool sim_trace_enabled() { + return trace_enabled; +} + +void sim_trace_enable(bool enable) { + trace_enabled = enable; +} + +#define EXPECT(cond, msg) do { \ + if (!(cond)) { \ + std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, msg); \ + std::exit(1); \ + } \ +} while (0) + +int main(int argc, char** argv) { + Verilated::commandArgs(argc, argv); + + vl_simulator sim; + uint64_t tick = 0; + + sim->in_valid = 0; + sim->out_ready = 0; + tick = sim.reset(tick); + sim->eval(); + + EXPECT(!sim->pending, "pending must be low after reset"); + EXPECT(sim->out_valid == 0, "no output may be valid after reset"); + + // An input that cannot yet enter the fanout is still an in-flight launch. + sim->in_valid = 1; + sim->out_ready = 0; + sim->eval(); + EXPECT(sim->pending, "a stalled input request must assert pending"); + EXPECT(!sim->in_ready, "input must stall while every destination is blocked"); + + sim->in_valid = 0; + sim->eval(); + EXPECT(!sim->pending, "pending must clear when the stalled input is withdrawn"); + + // Let the registered fanout accept one request, then backpressure the + // selected destination. This is the handoff that used to create a false + // device-idle cycle after the source dropped its own busy indication. + sim->in_valid = 1; + sim->out_ready = 0x3; + sim->eval(); + EXPECT(sim->pending, "the presented request must assert pending"); + EXPECT(sim->in_ready, "one destination must be able to accept the request"); + + tick = sim.step(tick, 2); + sim->in_valid = 0; + sim->out_ready = 0; + sim->eval(); + + const uint8_t held_output = sim->out_valid; + EXPECT(held_output == 0x1 || held_output == 0x2, + "exactly one output must hold the buffered request"); + std::printf("buffered handoff: in_valid=%u out_valid=0x%x pending=%u\n", + static_cast(sim->in_valid), + static_cast(held_output), + static_cast(sim->pending)); + EXPECT(sim->pending, + "pending must bridge the source-to-buffer ownership transfer"); + + // The request can remain buffered for an arbitrary number of cycles. + // pending must not pulse or depend on downstream readiness. + for (int cycle = 0; cycle < 3; ++cycle) { + tick = sim.step(tick, 2); + sim->eval(); + EXPECT(sim->out_valid == held_output, + "the same output must retain the backpressured request"); + EXPECT(sim->pending, + "a backpressured buffered request must keep pending asserted"); + } + + // Accept the held request. pending remains high in the transfer cycle and + // clears only after the destination has taken ownership. + sim->out_ready = held_output; + sim->eval(); + EXPECT((sim->out_valid & sim->out_ready) == held_output, + "the selected destination must complete the handshake"); + EXPECT(sim->pending, "pending must cover the output handshake cycle"); + + tick = sim.step(tick, 2); + sim->out_ready = 0; + sim->eval(); + EXPECT(sim->out_valid == 0, "the request must leave the fanout after acceptance"); + EXPECT(!sim->pending, "pending must clear after the fanout drains"); + + std::printf("PASSED: KMU fanout pending covers input, buffering, and output\n"); + return 0; +}