rtl: keep buffered KMU launches busy

ci/testcases/unittest.yaml

@@ -11,6 +11,9 @@ tests:
   - id: hw
     via: script
     run: "make -C hw/unittest"
+  - id: hw-kmu-arb
+    via: script
+    run: "make -C hw/unittest/kmu_arb run"
   # FPU arithmetic units (VX_fma/fdiv/fsqrt/fcvt/fdivsqrt): the `hw` lane above only
   # builds the unit tests; this lane executes them so a softfloat-divergent
   # regression actually fails CI (each driver checks bit-exact result+fflags over

hw/rtl/VX_cluster.sv

@@ -94,6 +94,7 @@ module VX_cluster import VX_gpu_pkg::*;
 `endif
 
     VX_kmu_bus_if per_socket_kmu_bus_if[NUM_SOCKETS]();
+    wire kmu_fanout_pending;
 
     VX_kmu_arb #(
         .NUM_INPUTS (1),
@@ -103,7 +104,8 @@ module VX_cluster import VX_gpu_pkg::*;
         .clk        (clk),
         .reset      (reset),
         .bus_in_if  (kmu_bus_if),
-        .bus_out_if (per_socket_kmu_bus_if)
+        .bus_out_if (per_socket_kmu_bus_if),
+        .pending    (kmu_fanout_pending)
     );
 
     VX_gbar_bus_if per_socket_gbar_bus_if[NUM_SOCKETS]();
@@ -521,10 +523,10 @@ module VX_cluster import VX_gpu_pkg::*;
 
     wire busy_r;
 `ifdef EXT_GFX_ANY_ENABLE
-    `BUFFER_EX(busy_r, dcr_bus_if.req_valid | (|per_socket_busy) | gfx_busy, 1'b1, 1, (NUM_SOCKETS > 1));
+    `BUFFER_EX(busy_r, dcr_bus_if.req_valid | kmu_fanout_pending | (|per_socket_busy) | gfx_busy, 1'b1, 1, (NUM_SOCKETS > 1));
 `else
-    `BUFFER_EX(busy_r, dcr_bus_if.req_valid | (|per_socket_busy), 1'b1, 1, (NUM_SOCKETS > 1));
+    `BUFFER_EX(busy_r, dcr_bus_if.req_valid | kmu_fanout_pending | (|per_socket_busy), 1'b1, 1, (NUM_SOCKETS > 1));
 `endif
-    assign busy = busy_r | dcr_bus_if.req_valid;
+    assign busy = busy_r | dcr_bus_if.req_valid | kmu_fanout_pending;
 
 endmodule

hw/rtl/VX_socket.sv

@@ -80,6 +80,7 @@ module VX_socket import VX_gpu_pkg::*;
 `endif
 
     VX_kmu_bus_if per_core_kmu_bus_if[`VX_CFG_SOCKET_SIZE]();
+    wire kmu_fanout_pending;
 
     VX_kmu_arb #(
         .NUM_INPUTS (1),
@@ -89,7 +90,8 @@ module VX_socket import VX_gpu_pkg::*;
         .clk        (clk),
         .reset      (reset),
         .bus_in_if  (kmu_bus_if),
-        .bus_out_if (per_core_kmu_bus_if)
+        .bus_out_if (per_core_kmu_bus_if),
+        .pending    (kmu_fanout_pending)
     );
 
     VX_gbar_bus_if per_core_gbar_bus_if[`VX_CFG_SOCKET_SIZE]();
@@ -513,7 +515,7 @@ module VX_socket import VX_gpu_pkg::*;
     end
 
     wire busy_r;
-    `BUFFER_EX(busy_r, dcr_bus_if.req_valid | (|per_core_busy), 1'b1, 1, (`VX_CFG_SOCKET_SIZE > 1));
-    assign busy = busy_r | dcr_bus_if.req_valid;
+    `BUFFER_EX(busy_r, dcr_bus_if.req_valid | kmu_fanout_pending | (|per_core_busy), 1'b1, 1, (`VX_CFG_SOCKET_SIZE > 1));
+    assign busy = busy_r | dcr_bus_if.req_valid | kmu_fanout_pending;
 
 endmodule

hw/rtl/Vortex.sv

@@ -181,6 +181,7 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*; (
     wire [`VX_CFG_NUM_CLUSTERS-1:0] per_cluster_busy;
 
     VX_kmu_bus_if per_cluster_kmu_bus_if[`VX_CFG_NUM_CLUSTERS]();
+    wire kmu_fanout_pending;
     VX_kmu_arb #(
         .NUM_INPUTS (1),
         .NUM_OUTPUTS (`VX_CFG_NUM_CLUSTERS),
@@ -189,7 +190,8 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*; (
         .clk        (clk),
         .reset      (reset),
         .bus_in_if  (kmu_bus_in),
-        .bus_out_if (per_cluster_kmu_bus_if)
+        .bus_out_if (per_cluster_kmu_bus_if),
+        .pending    (kmu_fanout_pending)
     );
 
 `ifdef VX_CFG_EXT_RASTER_ENABLE
@@ -245,8 +247,8 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*; (
         );
     end
     wire busy_r;
-    `BUFFER_EX(busy_r, kmu_busy | dcr_bus_if.req_valid | (|per_cluster_busy), 1'b1, 1, (`VX_CFG_NUM_CLUSTERS > 1));
-    assign busy = busy_r | kmu_busy | dcr_bus_if.req_valid;
+    `BUFFER_EX(busy_r, kmu_busy | kmu_fanout_pending | dcr_bus_if.req_valid | (|per_cluster_busy), 1'b1, 1, (`VX_CFG_NUM_CLUSTERS > 1));
+    assign busy = busy_r | kmu_busy | kmu_fanout_pending | dcr_bus_if.req_valid;
 
 `ifdef PERF_ENABLE
 

hw/rtl/core/VX_core.sv

@@ -211,6 +211,7 @@ module VX_core import VX_gpu_pkg::*; #(
     VX_kmu_bus_if raster_frag_kmu_if();   // distributor → arb
     VX_kmu_bus_if kmu_arb_in_if[2]();
     VX_kmu_bus_if sched_kmu_arr_if[1]();   // arb → scheduler
+    wire frag_kmu_merge_pending;
 
     // input 0 = device-KMU stream (the core's incoming kmu bus)
     assign kmu_arb_in_if[0].valid = kmu_bus_if.valid;
@@ -230,7 +231,8 @@ module VX_core import VX_gpu_pkg::*; #(
         .clk        (clk),
         .reset      (reset),
         .bus_in_if  (kmu_arb_in_if),
-        .bus_out_if (sched_kmu_arr_if)
+        .bus_out_if (sched_kmu_arr_if),
+        .pending    (frag_kmu_merge_pending)
     );
 
     VX_gfx_win_wr_if #(.NUM_LANES (`VX_CFG_NUM_SFU_LANES)) rast_win_if();
@@ -574,7 +576,7 @@ module VX_core import VX_gpu_pkg::*; #(
 `endif
 
 `ifdef VX_CFG_EXT_RASTER_ENABLE
-    assign busy = sched_busy || dcr_busy || ~(&lsu_sched_empty) || ~mem_unit_empty || raster_dispatch_busy || raster_packer_busy;
+    assign busy = sched_busy || dcr_busy || ~(&lsu_sched_empty) || ~mem_unit_empty || raster_dispatch_busy || raster_packer_busy || frag_kmu_merge_pending;
 `else
     assign busy = sched_busy || dcr_busy || ~(&lsu_sched_empty) || ~mem_unit_empty;
 `endif

hw/rtl/core/VX_cta_dispatch.sv

@@ -486,9 +486,10 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #(
     // the accept->DISPATCH transition is registered, so gating on state alone
     // leaves the accept cycle un-busy. With SOCKET_SIZE>1 the socket busy
     // aggregation is registered, so on the final CTA kmu_busy drops before the
-    // buffered per-core busy rises, opening a 1-cycle device-busy gap the host's
-    // edge-sensitive idle-wait latches as premature completion (cores>1 launch
-    // failure). Covering the accept cycle closes the gap.
+    // buffered per-core busy rises, opening a 1-cycle device-busy gap that the
+    // host idle wait treats as premature completion (cores>1 launch failure).
+    // Covering the accept cycle closes the dispatcher-local gap;
+    // registered upstream KMU fanouts keep their own pending state in busy.
     assign busy = (state == DISPATCH) || kmu_bus_if_fire;
 
     // -------------------------------------------------------------------------

hw/rtl/core/VX_kmu_arb.sv

@@ -27,7 +27,12 @@ module VX_kmu_arb import VX_gpu_pkg::*; #(
     VX_kmu_bus_if.slave  bus_in_if [NUM_INPUTS],
 
     // output requests
-    VX_kmu_bus_if.master bus_out_if [NUM_OUTPUTS]
+    VX_kmu_bus_if.master bus_out_if [NUM_OUTPUTS],
+
+    // A launch request is still in flight through this fan-out/arbiter.
+    // Parents include this in their busy chain so a buffered final CTA cannot
+    // create a false-idle window before the destination core accepts it.
+    output wire pending
 );
     localparam DATAW = NUM_LANES * $bits(kmu_req_t);
 
@@ -69,4 +74,6 @@ module VX_kmu_arb import VX_gpu_pkg::*; #(
         assign ready_out[i] = bus_out_if[i].ready;
     end
 
+    assign pending = (| valid_in) || (| valid_out);
+
 endmodule

hw/unittest/Makefile

@@ -9,6 +9,7 @@ all:
 	$(MAKE) -C tcu_fedp
 	$(MAKE) -C cta_dispatcher
 	$(MAKE) -C kmu
+	$(MAKE) -C kmu_arb
 	$(MAKE) -C dxa_core
 	$(MAKE) -C tcu_unit
 	$(MAKE) -C cp_arbiter
@@ -37,6 +38,7 @@ run:
 	$(MAKE) -C tcu_fedp run
 	$(MAKE) -C cta_dispatcher run
 	$(MAKE) -C kmu run
+	$(MAKE) -C kmu_arb run
 	$(MAKE) -C dxa_core run
 	$(MAKE) -C tcu_unit run
 	$(MAKE) -C cp_arbiter run
@@ -127,6 +129,7 @@ clean:
 	$(MAKE) -C tcu_fedp clean
 	$(MAKE) -C cta_dispatcher clean
 	$(MAKE) -C kmu clean
+	$(MAKE) -C kmu_arb clean
 	$(MAKE) -C dxa_core clean
 	$(MAKE) -C tcu_unit clean
 	$(MAKE) -C cp_arbiter clean

hw/unittest/kmu_arb/Makefile

@@ -0,0 +1,26 @@
+ROOT_DIR := $(realpath ../../..)
+include $(ROOT_DIR)/config.mk
+
+PROJECT := kmu_arb
+
+RTL_DIR := $(VORTEX_HOME)/hw/rtl
+DPI_DIR := $(VORTEX_HOME)/hw/dpi
+
+SRC_DIR := $(VORTEX_HOME)/hw/unittest/$(PROJECT)
+
+CXXFLAGS := -I$(SRC_DIR) -I$(VORTEX_HOME)/hw/unittest/common -I$(SW_COMMON_DIR)
+CXXFLAGS += -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw
+
+SRCS := $(SRC_DIR)/main.cpp
+
+DBG_TRACE_FLAGS :=
+
+RTL_PKGS := $(RTL_DIR)/VX_gpu_pkg.sv
+
+RTL_INCLUDE := -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw -I$(RTL_DIR) -I$(DPI_DIR)
+RTL_INCLUDE += -I$(RTL_DIR)/libs -I$(RTL_DIR)/interfaces -I$(RTL_DIR)/core
+RTL_INCLUDE += -I$(SRC_DIR)
+
+TOP := VX_kmu_arb_top
+
+include ../common.mk

hw/unittest/kmu_arb/VX_kmu_arb_top.sv

@@ -0,0 +1,54 @@
+// Copyright © 2019-2023
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+`include "VX_define.vh"
+
+module VX_kmu_arb_top import VX_gpu_pkg::*; (
+    input  wire       clk,
+    input  wire       reset,
+
+    input  wire       in_valid,
+    output wire       in_ready,
+
+    input  wire [1:0] out_ready,
+    output wire [1:0] out_valid,
+
+    output wire       pending
+);
+
+    VX_kmu_bus_if bus_in_if[1]();
+    VX_kmu_bus_if bus_out_if[2]();
+
+    assign bus_in_if[0].valid = in_valid;
+    assign bus_in_if[0].data  = '0;
+    assign in_ready = bus_in_if[0].ready;
+
+    for (genvar i = 0; i < 2; ++i) begin : g_outputs
+        assign bus_out_if[i].ready = out_ready[i];
+        assign out_valid[i] = bus_out_if[i].valid;
+        `UNUSED_VAR (bus_out_if[i].data)
+    end
+
+    VX_kmu_arb #(
+        .NUM_INPUTS  (1),
+        .NUM_OUTPUTS (2),
+        .OUT_BUF     (3)
+    ) dut (
+        .clk        (clk),
+        .reset      (reset),
+        .bus_in_if  (bus_in_if),
+        .bus_out_if (bus_out_if),
+        .pending    (pending)
+    );
+
+endmodule

hw/unittest/kmu_arb/main.cpp

@@ -0,0 +1,119 @@
+// Copyright © 2019-2023
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "vl_simulator.h"
+#include "VVX_kmu_arb_top.h"
+
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+
+static uint64_t timestamp = 0;
+static bool trace_enabled = false;
+
+double sc_time_stamp() {
+    return timestamp;
+}
+
+bool sim_trace_enabled() {
+    return trace_enabled;
+}
+
+void sim_trace_enable(bool enable) {
+    trace_enabled = enable;
+}
+
+#define EXPECT(cond, msg) do {                                             \
+    if (!(cond)) {                                                         \
+        std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, msg); \
+        std::exit(1);                                                      \
+    }                                                                      \
+} while (0)
+
+int main(int argc, char** argv) {
+    Verilated::commandArgs(argc, argv);
+
+    vl_simulator<VVX_kmu_arb_top> sim;
+    uint64_t tick = 0;
+
+    sim->in_valid = 0;
+    sim->out_ready = 0;
+    tick = sim.reset(tick);
+    sim->eval();
+
+    EXPECT(!sim->pending, "pending must be low after reset");
+    EXPECT(sim->out_valid == 0, "no output may be valid after reset");
+
+    // An input that cannot yet enter the fanout is still an in-flight launch.
+    sim->in_valid = 1;
+    sim->out_ready = 0;
+    sim->eval();
+    EXPECT(sim->pending, "a stalled input request must assert pending");
+    EXPECT(!sim->in_ready, "input must stall while every destination is blocked");
+
+    sim->in_valid = 0;
+    sim->eval();
+    EXPECT(!sim->pending, "pending must clear when the stalled input is withdrawn");
+
+    // Let the registered fanout accept one request, then backpressure the
+    // selected destination. This is the handoff that used to create a false
+    // device-idle cycle after the source dropped its own busy indication.
+    sim->in_valid = 1;
+    sim->out_ready = 0x3;
+    sim->eval();
+    EXPECT(sim->pending, "the presented request must assert pending");
+    EXPECT(sim->in_ready, "one destination must be able to accept the request");
+
+    tick = sim.step(tick, 2);
+    sim->in_valid = 0;
+    sim->out_ready = 0;
+    sim->eval();
+
+    const uint8_t held_output = sim->out_valid;
+    EXPECT(held_output == 0x1 || held_output == 0x2,
+           "exactly one output must hold the buffered request");
+    std::printf("buffered handoff: in_valid=%u out_valid=0x%x pending=%u\n",
+                static_cast<unsigned>(sim->in_valid),
+                static_cast<unsigned>(held_output),
+                static_cast<unsigned>(sim->pending));
+    EXPECT(sim->pending,
+           "pending must bridge the source-to-buffer ownership transfer");
+
+    // The request can remain buffered for an arbitrary number of cycles.
+    // pending must not pulse or depend on downstream readiness.
+    for (int cycle = 0; cycle < 3; ++cycle) {
+        tick = sim.step(tick, 2);
+        sim->eval();
+        EXPECT(sim->out_valid == held_output,
+               "the same output must retain the backpressured request");
+        EXPECT(sim->pending,
+               "a backpressured buffered request must keep pending asserted");
+    }
+
+    // Accept the held request. pending remains high in the transfer cycle and
+    // clears only after the destination has taken ownership.
+    sim->out_ready = held_output;
+    sim->eval();
+    EXPECT((sim->out_valid & sim->out_ready) == held_output,
+           "the selected destination must complete the handshake");
+    EXPECT(sim->pending, "pending must cover the output handshake cycle");
+
+    tick = sim.step(tick, 2);
+    sim->out_ready = 0;
+    sim->eval();
+    EXPECT(sim->out_valid == 0, "the request must leave the fanout after acceptance");
+    EXPECT(!sim->pending, "pending must clear after the fanout drains");
+
+    std::printf("PASSED: KMU fanout pending covers input, buffering, and output\n");
+    return 0;
+}