From e0ad0f14c272b3c8e2f559dfe3f7fd66283211cf Mon Sep 17 00:00:00 2001 From: tinebp Date: Tue, 6 Oct 2026 13:09:29 -0700 Subject: [PATCH 001/124] simx: deliver same-cycle channel events in the order they were sent LinkedList::push_back inserted before the current tail, because insert() places a node ahead of its position: pushing 1, 2, 3 produced 2, 3, 1. The timing wheel and the zero-delay event list are LinkedLists, so two packets sent into one SimChannel in the same cycle with the same delay arrived reversed. Models almost never send twice per channel per cycle, which kept it hidden; a two-wide fetch prototype exposed it as a younger instruction issuing ahead of its producer. push_back now appends at end(). The header also includes so it compiles on its own. tests/unittest/linked_list checks the order and fails against the old header. SimX legs of all 31 model_parity cases: identical retired instructions; cycles move by at most 0.7%, except rt_raycast +1.7%, toward rtlsim (gap 2.4% -> 0.8% against a current rtlsim run). rtlsim and the perf gate do not use this code. Co-Authored-By: Claude Opus 5.5 --- sw/common/linked_list.h | 3 +- tests/unittest/Makefile | 2 +- tests/unittest/linked_list/Makefile | 10 ++++++ tests/unittest/linked_list/main.cpp | 55 +++++++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 2 deletions(-) create mode 100644 tests/unittest/linked_list/Makefile create mode 100644 tests/unittest/linked_list/main.cpp diff --git a/sw/common/linked_list.h b/sw/common/linked_list.h index f75a6b8dae..38f175ed54 100644 --- a/sw/common/linked_list.h +++ b/sw/common/linked_list.h @@ -14,6 +14,7 @@ #pragma once #include +#include #include namespace vortex { @@ -238,7 +239,7 @@ class LinkedList { } void push_back(T *obj) { - insert(iterator(tail_), obj); + insert(end(), obj); } void push_front(T *obj) { diff --git a/tests/unittest/Makefile b/tests/unittest/Makefile index b344a6ff1a..0c561d642e 100644 --- a/tests/unittest/Makefile +++ b/tests/unittest/Makefile @@ -1,7 +1,7 @@ ROOT_DIR := $(realpath ../..) include $(ROOT_DIR)/config.mk -PROJECTS := vx_malloc dxa_ooo rvc_decompress gfx_binsort gfx_tex_sw gfx_rast_sw gfx_msaa gfx_om_mrt +PROJECTS := vx_malloc linked_list dxa_ooo rvc_decompress gfx_binsort gfx_tex_sw gfx_rast_sw gfx_msaa gfx_om_mrt all: $(foreach p,$(PROJECTS),$(MAKE) -C $(p) &&) true diff --git a/tests/unittest/linked_list/Makefile b/tests/unittest/linked_list/Makefile new file mode 100644 index 0000000000..293746789e --- /dev/null +++ b/tests/unittest/linked_list/Makefile @@ -0,0 +1,10 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := linked_list + +SRC_DIR := $(VORTEX_HOME)/tests/unittest/$(PROJECT) + +SRCS := $(SRC_DIR)/main.cpp + +include ../common.mk diff --git a/tests/unittest/linked_list/main.cpp b/tests/unittest/linked_list/main.cpp new file mode 100644 index 0000000000..9434ed22bd --- /dev/null +++ b/tests/unittest/linked_list/main.cpp @@ -0,0 +1,55 @@ +#include +#include +#include + +using namespace vortex; + +struct Node { + int value; + LinkedListNode hook; +}; + +using List = LinkedList; + +static bool expect(const List& list, const std::vector& want, const char* what) { + std::vector got; + for (auto& n : list) { + got.push_back(n.value); + } + if (got == want) + return true; + printf("Error: %s: got {", what); + for (size_t i = 0; i < got.size(); ++i) printf("%s%d", i ? ", " : "", got[i]); + printf("}, expected {"); + for (size_t i = 0; i < want.size(); ++i) printf("%s%d", i ? ", " : "", want[i]); + printf("}\n"); + return false; +} + +int main() { + Node n[6] = {{1, {}}, {2, {}}, {3, {}}, {4, {}}, {5, {}}, {6, {}}}; + List list; + bool ok = true; + + // The simulator's timing wheel delivers same-cycle events in push_back order. + list.push_back(&n[0]); + list.push_back(&n[1]); + list.push_back(&n[2]); + ok &= expect(list, {1, 2, 3}, "push_back"); + + list.push_front(&n[3]); + ok &= expect(list, {4, 1, 2, 3}, "push_front"); + + list.insert(++list.begin(), &n[4]); + ok &= expect(list, {4, 5, 1, 2, 3}, "insert before position"); + + list.erase(list.begin()); + list.pop_back(); + ok &= expect(list, {5, 1, 2}, "erase/pop_back"); + + list.push_back(&n[5]); + ok &= expect(list, {5, 1, 2, 6}, "push_back after erase"); + + printf("%s\n", ok ? "PASSED!" : "FAILED!"); + return ok ? 0 : 1; +} From cbd9cbee8c35d87fdeb017ac6f1de9f173ce343a Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 7 Oct 2026 18:51:08 -0700 Subject: [PATCH 002/124] sw, tests: make device index math portable to RV64 On RV64 an index computed in uint32_t wraps at 32 bits, so LLVM cannot widen it into a 64-bit induction variable or fold it into a pointer increment, and re-emits zero-extend + shift + add for every access (sgemm's unrolled body: 90 instructions on RV64 vs 35 on RV32). Indices that feed pointer arithmetic are now size_t, which on RV32 is the same type as uint32_t, so RV32 kernels compile to identical code; vm_stress uses int, where size_t moved the loop exit onto a divergent per-thread pointer compare. RV64 correctness: - vx_dxa.h: keep the wgather result (lane 0 carries the shared-memory address) at register width instead of truncating it to 32 bits. - vx_graphics.h: compute the OM aperture address in uintptr_t and pass it to the export at register width; on RV32 the old expression was a 64-bit sum whose dead carry cost an instruction per fragment. - vx_intrinsics.h, vx_tensor.h, packld_box: pack-load strides and ldm steps are size_t; the address unit reads the full register. - vx_serial.S: keep sp 16-byte aligned across the callback (frames 32/64 bytes on RV32/RV64); select the XLEN path with __riscv_xlen, as LOAD_IMMEDIATE64 in src/common.h now does. - vx_start.S: __dso_handle is pointer-sized on RV64. - vx_syscalls.c: use newlib's prototypes for _lseek/_read/_write/_sbrk. - dotproduct2: don't pass uninitialized upper bytes through the shuffle. Kernels whose RV64 cycles got worse with a wider index are left as they were: sgemv, demo, probe, fence, dogfood, dotproduct, softmax, vm_test, mpi_dotproduct, mpi_put_dotproduct, sgemm_tcu, sgemm_tcu_mx, sgemm_tcu_sp_mx and sgemm_tcu_wg. In most of them LLVM's loop strength reduction moves the loop exit onto a per-thread pointer compare, which needs predication every iteration. RV64 simx cycles: sgemm -55%, mstress -28%, sgemm_tcu_wg_mx -22%, sgemm2_dxa -21%, stencil3d -20%, sgemm_v1 -18%, sgemm2_v1 -16%, dxa_copy_mcast -16%, sgemm2 -12%, sgemm2_dxa_mcast -12%, jacobi -10%, sgemm2_tcu -6%; no measured kernel is slower (mpi_sgemm, mpi_blocked_sgemm and the graphics pipeline kernels print no cycle counts and were checked for correctness only). On RV32 only the OM export path and vx_serial's frame change; the affected graphics perf-gate baselines are regenerated with --update-baselines (om -1.3%/-1.9%, draw3d -0.4% cycles on rtlsim). Co-Authored-By: Claude Opus 5.5 --- ci/baselines/perf/graphics.json | 20 +-- sw/common/gfx_sw.h | 4 +- sw/gfx/gfx_frontend_k.h | 28 ++-- sw/kernel/include/vx_dxa.h | 144 +++++++++--------- sw/kernel/include/vx_graphics.h | 7 +- sw/kernel/include/vx_intrinsics.h | 4 +- sw/kernel/include/vx_tensor.h | 14 +- sw/kernel/src/common.h | 2 +- sw/kernel/src/vx_serial.S | 12 +- sw/kernel/src/vx_start.S | 6 + sw/kernel/src/vx_syscalls.c | 15 +- tests/mpi/mpi_blocked_sgemm/kernel.cpp | 4 +- tests/mpi/mpi_sgemm/kernel.cpp | 4 +- tests/regression/async_barrier/kernel.cpp | 14 +- tests/regression/basic/kernel.cpp | 6 +- tests/regression/dotproduct2/kernel.cpp | 4 +- tests/regression/dxa_copy_mcast/kernel.cpp | 12 +- tests/regression/dxa_kmajor_check/kernel.cpp | 6 +- .../regression/gfx_binsort_kernel/kernel.cpp | 14 +- .../regression/gfx_pipeline_kernel/kernel.cpp | 40 ++--- tests/regression/gfx_setup_kernel/kernel.cpp | 18 +-- tests/regression/jacobi/kernel.cpp | 8 +- tests/regression/module_reload/kernel.cpp | 6 +- tests/regression/mstress/kernel.cpp | 10 +- tests/regression/packld/kernel.cpp | 2 +- tests/regression/packld_box/kernel.cpp | 8 +- tests/regression/raycast/render.h | 6 +- tests/regression/reopen/kernel.cpp | 6 +- tests/regression/sgemm/kernel.cpp | 4 +- tests/regression/sgemm2/kernel.cpp | 22 +-- tests/regression/sgemm2_dxa/kernel.cpp | 30 ++-- tests/regression/sgemm2_dxa_mcast/kernel.cpp | 14 +- tests/regression/sgemm2_tcu/kernel.cpp | 26 ++-- tests/regression/sgemm2_v1/kernel.cpp | 22 +-- tests/regression/sgemm_tcu_wg_dxa/kernel.cpp | 8 +- tests/regression/sgemm_tcu_wg_mx/kernel.cpp | 22 +-- tests/regression/sgemm_tcu_wg_sp/kernel.cpp | 24 +-- tests/regression/sgemm_v1/kernel.cpp | 4 +- tests/regression/stencil3d/kernel.cpp | 2 +- tests/regression/vm_stress/kernel.cpp | 4 +- 40 files changed, 307 insertions(+), 299 deletions(-) diff --git a/ci/baselines/perf/graphics.json b/ci/baselines/perf/graphics.json index 39103db55e..3ca43a64f7 100644 --- a/ci/baselines/perf/graphics.json +++ b/ci/baselines/perf/graphics.json @@ -1,8 +1,8 @@ { "graphics:perf_gate-draw3d-mc:rtlsim": { "32": { - "cycles": 531658, - "instrs": 204102 + "cycles": 529553, + "instrs": 203136 }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", @@ -12,8 +12,8 @@ }, "graphics:perf_gate-draw3d-nt16:rtlsim": { "32": { - "cycles": 148046, - "instrs": 51475 + "cycles": 147521, + "instrs": 51232 }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", @@ -23,8 +23,8 @@ }, "graphics:perf_gate-draw3d-nt4:rtlsim": { "32": { - "cycles": 531210, - "instrs": 204102 + "cycles": 529126, + "instrs": 203136 }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", @@ -34,8 +34,8 @@ }, "graphics:perf_gate-om-nt16:rtlsim": { "32": { - "cycles": 192077, - "instrs": 69648 + "cycles": 189486, + "instrs": 68624 }, "app": "gfx_om", "args": "-w 128 -h 128 -r whitebox_128.png", @@ -45,8 +45,8 @@ }, "graphics:perf_gate-om-nt4:rtlsim": { "32": { - "cycles": 755028, - "instrs": 278544 + "cycles": 740760, + "instrs": 274448 }, "app": "gfx_om", "args": "-w 128 -h 128 -r whitebox_128.png", diff --git a/sw/common/gfx_sw.h b/sw/common/gfx_sw.h index 503b9721f9..96405a48d8 100644 --- a/sw/common/gfx_sw.h +++ b/sw/common/gfx_sw.h @@ -399,11 +399,11 @@ static inline __attribute__((always_inline)) bool om_sample_rmw( // stride gather. static inline __attribute__((always_inline)) uintptr_t msaa_color_addr( const om_state_t& s, uint32_t samples, uint32_t x, uint32_t y, uint32_t sample) { - return (uintptr_t)(s.cbuf_base + (uint64_t)y * s.cbuf_pitch + (uint64_t)(x * samples + sample) * om_color_bpp(s.color_format)); + return (uintptr_t)(s.cbuf_base + (uint64_t)y * s.cbuf_pitch + (uint64_t)((uintptr_t)x * samples + sample) * om_color_bpp(s.color_format)); } static inline __attribute__((always_inline)) uintptr_t msaa_depth_addr( const om_state_t& s, uint32_t samples, uint32_t x, uint32_t y, uint32_t sample) { - return (uintptr_t)(s.zbuf_base + (uint64_t)y * s.zbuf_pitch + (uint64_t)(x * samples + sample) * om_depth_bpp(s.depth_format)); + return (uintptr_t)(s.zbuf_base + (uint64_t)y * s.zbuf_pitch + (uint64_t)((uintptr_t)x * samples + sample) * om_depth_bpp(s.depth_format)); } // Box resolve of one pixel's S color samples → a single ARGB8888 value diff --git a/sw/gfx/gfx_frontend_k.h b/sw/gfx/gfx_frontend_k.h index 9e557ab7e3..e2ee763c97 100644 --- a/sw/gfx/gfx_frontend_k.h +++ b/sw/gfx/gfx_frontend_k.h @@ -85,8 +85,8 @@ __kernel void expand_k(expand_arg_t* __UNIFORM__ arg) { const uint32_t nv = arg->num_varyings; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; uint32_t gstride = gridDim.x * blockDim.x; - for (uint32_t i = gid; i < n; i += gstride) { - const uint8_t* rec = recs + (size_t)i * stride; + for (size_t i = gid; i < n; i += gstride) { + const uint8_t* rec = recs + i * stride; const float* pos = reinterpret_cast(rec); setup_vertex_t v; v.pos[0] = pos[0]; v.pos[1] = pos[1]; v.pos[2] = pos[2]; v.pos[3] = pos[3]; @@ -156,7 +156,7 @@ __kernel void setup_k(pipe_arg_t* __UNIFORM__ arg) { // traffic and removes the slot_prim/slot_bbox scratch entirely. uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; uint32_t gstride = gridDim.x * blockDim.x; - for (uint32_t t = gid; t < ntri; t += gstride) { + for (size_t t = gid; t < ntri; t += gstride) { rast_prim_t pr[SETUP_MAX_SUB]; setup_bbox_t bb[SETUP_MAX_SUB]; keep[t] = pipe_clip_and_setup(&verts[3 * t], W, H, arg->cull_mode, vpp, pr, bb); @@ -183,7 +183,7 @@ __kernel void setup_k(pipe_arg_t* __UNIFORM__ arg) { // here equals keep[t] from SETUP (same pure function, same inputs). uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; uint32_t gstride = gridDim.x * blockDim.x; - for (uint32_t t = gid; t < ntri; t += gstride) { + for (size_t t = gid; t < ntri; t += gstride) { rast_prim_t pr[SETUP_MAX_SUB]; setup_bbox_t bb[SETUP_MAX_SUB]; uint32_t kept = pipe_clip_and_setup(&verts[3 * t], W, H, arg->cull_mode, vpp, pr, bb); @@ -210,12 +210,12 @@ __kernel void setup_k(pipe_arg_t* __UNIFORM__ arg) { for (uint32_t c = 0; c < 4; ++c) { fw[2 + c] = pv[4 + c]; } for (uint32_t c = 0; c < 6; ++c) { fw[6 + c] = pv[10 + c]; } } - uint32_t w = offset[t]; + size_t w = offset[t]; for (uint32_t s = 0; s < kept; ++s) { prim[w] = pr[s]; bbox[w] = bb[s]; if (flat) { - uint32_t* dst = flat + (size_t)w * GFX_FS_FLAT_WORDS; + uint32_t* dst = flat + w * GFX_FS_FLAT_WORDS; for (uint32_t c = 0; c < GFX_FS_FLAT_WORDS; ++c) { dst[c] = fw[c]; } } ++w; @@ -250,7 +250,7 @@ __kernel void binning_k(pipe_arg_t* __UNIFORM__ arg) { uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; uint32_t gstride = gridDim.x * blockDim.x; uint32_t P = meta[0]; - for (uint32_t i = gid; i < P; i += gstride) { + for (size_t i = gid; i < P; i += gstride) { int bL, bR, bT, bB; pipe_bin_range(bbox[i], bL, bR, bT, bB); bcount[i] = (uint32_t)((bR - bL + 1) * (bB - bT + 1)); } @@ -275,9 +275,9 @@ __kernel void binning_k(pipe_arg_t* __UNIFORM__ arg) { uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; uint32_t gstride = gridDim.x * blockDim.x; uint32_t P = meta[0]; - for (uint32_t i = gid; i < P; i += gstride) { + for (size_t i = gid; i < P; i += gstride) { int bL, bR, bT, bB; pipe_bin_range(bbox[i], bL, bR, bT, bB); - uint32_t w = boffset[i]; + size_t w = boffset[i]; for (int by = bT; by <= bB; ++by) for (int bx = bL; bx <= bR; ++bx) keys[w++] = ((uint32_t)(by * arg->bin_cols + bx) << PIPE_PRIM_BITS) | i; @@ -289,11 +289,11 @@ __kernel void binning_k(pipe_arg_t* __UNIFORM__ arg) { uint32_t K = meta[1]; uint32_t kchunk = (K + T - 1) / T; uint32_t klo = pipe_umin(tid * kchunk, K), khi = pipe_umin(klo + kchunk, K); - for (uint32_t b = lo; b < hi; ++b) thist[tid * B + b] = 0; + for (uint32_t b = lo; b < hi; ++b) thist[(size_t)tid * B + b] = 0; __syncthreads(); - for (uint32_t k = klo; k < khi; ++k) { uint32_t b = keys[k] >> PIPE_PRIM_BITS; if (b >= lo && b < hi) thist[tid * B + b]++; } + for (uint32_t k = klo; k < khi; ++k) { uint32_t b = keys[k] >> PIPE_PRIM_BITS; if (b >= lo && b < hi) thist[(size_t)tid * B + b]++; } __syncthreads(); - for (uint32_t b = lo + tid; b < hi; b += T) { uint32_t s = 0; for (uint32_t t = 0; t < T; ++t) s += thist[t * B + b]; bincount[b] = s; } + for (uint32_t b = lo + tid; b < hi; b += T) { uint32_t s = 0; for (uint32_t t = 0; t < T; ++t) s += thist[(size_t)t * B + b]; bincount[b] = s; } } break; case PIPE_STAGE_BBASE: { @@ -341,11 +341,11 @@ __kernel void binning_k(pipe_arg_t* __UNIFORM__ arg) { auto pids = reinterpret_cast(tilebuf + B * sizeof(rast_bin_header_t)); uint32_t kchunk = (K + T - 1) / T; uint32_t klo = pipe_umin(tid * kchunk, K), khi = pipe_umin(klo + kchunk, K); - for (uint32_t b = lo + tid; b < hi; b += T) { uint32_t run = binbase[b]; for (uint32_t t = 0; t < T; ++t) { uint32_t c = thist[t * B + b]; thist[t * B + b] = run; run += c; } } + for (uint32_t b = lo + tid; b < hi; b += T) { uint32_t run = binbase[b]; for (uint32_t t = 0; t < T; ++t) { uint32_t c = thist[(size_t)t * B + b]; thist[(size_t)t * B + b] = run; run += c; } } __syncthreads(); for (uint32_t k = klo; k < khi; ++k) { uint32_t key = keys[k], b = key >> PIPE_PRIM_BITS; - if (b >= lo && b < hi) pids[thist[tid * B + b]++] = key & PIPE_PRIM_MASK; + if (b >= lo && b < hi) pids[thist[(size_t)tid * B + b]++] = key & PIPE_PRIM_MASK; } } break; diff --git a/sw/kernel/include/vx_dxa.h b/sw/kernel/include/vx_dxa.h index dbbc031e37..79babb18b5 100644 --- a/sw/kernel/include/vx_dxa.h +++ b/sw/kernel/include/vx_dxa.h @@ -55,10 +55,10 @@ inline void vx_dxa_issue_1d_wg(uint32_t desc_slot, const void* smem_addr, uint32_t coord0) { const uint32_t meta = vx_dxa_pack_meta(desc_slot, barrier_id); - const uint32_t a0 = (uint32_t)vx_wgather((size_t)(uintptr_t)smem_addr, - (size_t)meta, - (size_t)coord0, - (size_t)0u); + const size_t a0 = vx_wgather((size_t)(uintptr_t)smem_addr, + (size_t)meta, + (size_t)coord0, + (size_t)0u); __asm__ volatile ( ".insn r %0, 0, %1, x0, %2, x0\n\t" : @@ -73,10 +73,10 @@ inline void vx_dxa_issue_2d_wg(uint32_t desc_slot, uint32_t coord0, uint32_t coord1) { const uint32_t meta = vx_dxa_pack_meta(desc_slot, barrier_id); - const uint32_t a0 = (uint32_t)vx_wgather((size_t)(uintptr_t)smem_addr, - (size_t)meta, - (size_t)coord0, - (size_t)coord1); + const size_t a0 = vx_wgather((size_t)(uintptr_t)smem_addr, + (size_t)meta, + (size_t)coord0, + (size_t)coord1); __asm__ volatile ( ".insn r %0, 0, %1, x0, %2, x0\n\t" : @@ -92,14 +92,14 @@ inline void vx_dxa_issue_3d_wg(uint32_t desc_slot, uint32_t coord1, uint32_t coord2) { const uint32_t meta = vx_dxa_pack_meta(desc_slot, barrier_id); - const uint32_t a0 = (uint32_t)vx_wgather((size_t)(uintptr_t)smem_addr, - (size_t)meta, - (size_t)coord0, - (size_t)coord1); - const uint32_t a1 = (uint32_t)vx_wgather((size_t)coord2, - (size_t)0u, - (size_t)0u, - (size_t)0u); + const size_t a0 = vx_wgather((size_t)(uintptr_t)smem_addr, + (size_t)meta, + (size_t)coord0, + (size_t)coord1); + const size_t a1 = vx_wgather((size_t)coord2, + (size_t)0u, + (size_t)0u, + (size_t)0u); __asm__ volatile ( ".insn r %0, 0, %1, x0, %2, %3\n\t" : @@ -115,14 +115,14 @@ inline void vx_dxa_issue_4d_wg(uint32_t desc_slot, uint32_t coord2, uint32_t coord3) { const uint32_t meta = vx_dxa_pack_meta(desc_slot, barrier_id); - const uint32_t a0 = (uint32_t)vx_wgather((size_t)(uintptr_t)smem_addr, - (size_t)meta, - (size_t)coord0, - (size_t)coord1); - const uint32_t a1 = (uint32_t)vx_wgather((size_t)coord2, - (size_t)coord3, - (size_t)0u, - (size_t)0u); + const size_t a0 = vx_wgather((size_t)(uintptr_t)smem_addr, + (size_t)meta, + (size_t)coord0, + (size_t)coord1); + const size_t a1 = vx_wgather((size_t)coord2, + (size_t)coord3, + (size_t)0u, + (size_t)0u); __asm__ volatile ( ".insn r %0, 0, %1, x0, %2, %3\n\t" : @@ -139,14 +139,14 @@ inline void vx_dxa_issue_5d_wg(uint32_t desc_slot, uint32_t coord3, uint32_t coord4) { const uint32_t meta = vx_dxa_pack_meta(desc_slot, barrier_id); - const uint32_t a0 = (uint32_t)vx_wgather((size_t)(uintptr_t)smem_addr, - (size_t)meta, - (size_t)coord0, - (size_t)coord1); - const uint32_t a1 = (uint32_t)vx_wgather((size_t)coord2, - (size_t)coord3, - (size_t)coord4, - (size_t)0u); + const size_t a0 = vx_wgather((size_t)(uintptr_t)smem_addr, + (size_t)meta, + (size_t)coord0, + (size_t)coord1); + const size_t a1 = vx_wgather((size_t)coord2, + (size_t)coord3, + (size_t)coord4, + (size_t)0u); __asm__ volatile ( ".insn r %0, 0, %1, x0, %2, %3\n\t" : @@ -164,14 +164,14 @@ inline void vx_dxa_issue_1d_multicast_wg(uint32_t desc_slot, uint32_t coord0, uint32_t cta_mask) { const uint32_t meta = vx_dxa_pack_meta(desc_slot, barrier_id); - const uint32_t a0 = (uint32_t)vx_wgather((size_t)(uintptr_t)smem_addr, - (size_t)meta, - (size_t)coord0, - (size_t)0u); - const uint32_t a1 = (uint32_t)vx_wgather((size_t)0, - (size_t)0, - (size_t)0, - (size_t)cta_mask); + const size_t a0 = vx_wgather((size_t)(uintptr_t)smem_addr, + (size_t)meta, + (size_t)coord0, + (size_t)0u); + const size_t a1 = vx_wgather((size_t)0, + (size_t)0, + (size_t)0, + (size_t)cta_mask); __asm__ volatile ( ".insn r %0, 0, %1, x0, %2, %3\n\t" : @@ -187,14 +187,14 @@ inline void vx_dxa_issue_2d_multicast_wg(uint32_t desc_slot, uint32_t coord1, uint32_t cta_mask) { const uint32_t meta = vx_dxa_pack_meta(desc_slot, barrier_id); - const uint32_t a0 = (uint32_t)vx_wgather((size_t)(uintptr_t)smem_addr, - (size_t)meta, - (size_t)coord0, - (size_t)coord1); - const uint32_t a1 = (uint32_t)vx_wgather((size_t)0, - (size_t)0, - (size_t)0, - (size_t)cta_mask); + const size_t a0 = vx_wgather((size_t)(uintptr_t)smem_addr, + (size_t)meta, + (size_t)coord0, + (size_t)coord1); + const size_t a1 = vx_wgather((size_t)0, + (size_t)0, + (size_t)0, + (size_t)cta_mask); __asm__ volatile ( ".insn r %0, 0, %1, x0, %2, %3\n\t" : @@ -211,14 +211,14 @@ inline void vx_dxa_issue_3d_multicast_wg(uint32_t desc_slot, uint32_t coord2, uint32_t cta_mask) { const uint32_t meta = vx_dxa_pack_meta(desc_slot, barrier_id); - const uint32_t a0 = (uint32_t)vx_wgather((size_t)(uintptr_t)smem_addr, - (size_t)meta, - (size_t)coord0, - (size_t)coord1); - const uint32_t a1 = (uint32_t)vx_wgather((size_t)coord2, - (size_t)0, - (size_t)0, - (size_t)cta_mask); + const size_t a0 = vx_wgather((size_t)(uintptr_t)smem_addr, + (size_t)meta, + (size_t)coord0, + (size_t)coord1); + const size_t a1 = vx_wgather((size_t)coord2, + (size_t)0, + (size_t)0, + (size_t)cta_mask); __asm__ volatile ( ".insn r %0, 0, %1, x0, %2, %3\n\t" : @@ -236,14 +236,14 @@ inline void vx_dxa_issue_4d_multicast_wg(uint32_t desc_slot, uint32_t coord3, uint32_t cta_mask) { const uint32_t meta = vx_dxa_pack_meta(desc_slot, barrier_id); - const uint32_t a0 = (uint32_t)vx_wgather((size_t)(uintptr_t)smem_addr, - (size_t)meta, - (size_t)coord0, - (size_t)coord1); - const uint32_t a1 = (uint32_t)vx_wgather((size_t)coord2, - (size_t)coord3, - (size_t)0, - (size_t)cta_mask); + const size_t a0 = vx_wgather((size_t)(uintptr_t)smem_addr, + (size_t)meta, + (size_t)coord0, + (size_t)coord1); + const size_t a1 = vx_wgather((size_t)coord2, + (size_t)coord3, + (size_t)0, + (size_t)cta_mask); __asm__ volatile ( ".insn r %0, 0, %1, x0, %2, %3\n\t" : @@ -262,14 +262,14 @@ inline void vx_dxa_issue_5d_multicast_wg(uint32_t desc_slot, uint32_t coord4, uint32_t cta_mask) { const uint32_t meta = vx_dxa_pack_meta(desc_slot, barrier_id); - const uint32_t a0 = (uint32_t)vx_wgather((size_t)(uintptr_t)smem_addr, - (size_t)meta, - (size_t)coord0, - (size_t)coord1); - const uint32_t a1 = (uint32_t)vx_wgather((size_t)coord2, - (size_t)coord3, - (size_t)coord4, - (size_t)cta_mask); + const size_t a0 = vx_wgather((size_t)(uintptr_t)smem_addr, + (size_t)meta, + (size_t)coord0, + (size_t)coord1); + const size_t a1 = vx_wgather((size_t)coord2, + (size_t)coord3, + (size_t)coord4, + (size_t)cta_mask); __asm__ volatile ( ".insn r %0, 0, %1, x0, %2, %3\n\t" : diff --git a/sw/kernel/include/vx_graphics.h b/sw/kernel/include/vx_graphics.h index 0eb7b094a6..67664edf5b 100644 --- a/sw/kernel/include/vx_graphics.h +++ b/sw/kernel/include/vx_graphics.h @@ -68,8 +68,9 @@ inline unsigned vx_tex(unsigned stage, unsigned u, unsigned v, unsigned lod) { // XBITS/YBITS/RECORD_SHIFT come from the OM DCRs; the runtime programs them and // passes them to the kernel, so the shader just shifts and adds. #define VX_OM_APERTURE_ADDR_RT(xbits, ybits, record_shift, x, y, face, rt) \ - ((VX_MEM_OM_BASE_ADDR) + \ - (((((uint32_t)(rt) << 1 | (uint32_t)(face)) << ((xbits) + (ybits))) \ + ((uintptr_t)(VX_MEM_OM_BASE_ADDR) + \ + (uintptr_t)(((((uint32_t)(rt) << 1 | (uint32_t)(face)) \ + << ((xbits) + (ybits))) \ | ((uint32_t)(y) << (xbits)) \ | (uint32_t)(x)) << (record_shift))) @@ -84,7 +85,7 @@ inline unsigned vx_tex(unsigned stage, unsigned u, unsigned v, unsigned lod) { // into one ordinary store per word; the LSU never learns that OM exists. #define vx_om_export(addr, color, depth, mask) \ __asm__ volatile (".insn r4 %0, 3, %1, x0, %2, %3, %4" \ - :: "i"(RISCV_CUSTOM1), "i"(mask), "r"(addr), "r"(color), "r"(depth)) + :: "i"(RISCV_CUSTOM1), "i"(mask), "r"((uintptr_t)(addr)), "r"(color), "r"(depth)) // The three record shapes. #define vx_om_export_color(addr, color) vx_om_export(addr, color, 0, 1) diff --git a/sw/kernel/include/vx_intrinsics.h b/sw/kernel/include/vx_intrinsics.h index 11ee4081a8..255a6ec392 100644 --- a/sw/kernel/include/vx_intrinsics.h +++ b/sw/kernel/include/vx_intrinsics.h @@ -602,7 +602,7 @@ inline void vx_barrier_expect_tx(int barrier_id, int count) { // | (MEM8[base + 2 * stride] << 16) // | (MEM8[base + 3 * stride] << 24)); __attribute__((always_inline)) -inline float vx_packlb_f(const void* base, uint32_t stride) { +inline float vx_packlb_f(const void* base, size_t stride) { float out; __asm__ volatile ( ".insn r %1, 1, 4, %0, %2, %3" : "=f"(out) : "i"(RISCV_CUSTOM0), "r"(base), "r"(stride) : "memory" @@ -616,7 +616,7 @@ inline float vx_packlb_f(const void* base, uint32_t stride) { // fd = bitcast((MEM16[base + 0 * stride] << 0) // | (MEM16[base + 1 * stride] << 16)); __attribute__((always_inline)) -inline float vx_packlh_f(const void* base, uint32_t stride) { +inline float vx_packlh_f(const void* base, size_t stride) { float out; __asm__ volatile ( ".insn r %1, 2, 4, %0, %2, %3" : "=f"(out) : "i"(RISCV_CUSTOM0), "r"(base), "r"(stride) : "memory" diff --git a/sw/kernel/include/vx_tensor.h b/sw/kernel/include/vx_tensor.h index 4106404b6b..618a95b030 100644 --- a/sw/kernel/include/vx_tensor.h +++ b/sw/kernel/include/vx_tensor.h @@ -87,7 +87,7 @@ namespace detail { } } - static __attribute__((always_inline)) D pack_row(const Type *base, uint32_t ldm) { + static __attribute__((always_inline)) D pack_row(const Type *base, size_t ldm) { static_assert(sizeof(D) % sizeof(Type) == 0, "D must be a multiple of Type in size"); if constexpr (sizeof(Type) == 1 && sizeof(D) == 4) { // 4 × 1-byte strided loads → single pack-load byte instruction @@ -368,7 +368,7 @@ struct wmma_context { k_bases[0] = base; if constexpr (num_k_groups >= 2) { asm volatile("" : "+r"(k_bases[0])); // prevent reverse strength reduction - auto k_ldm_step = k_stride * (uint32_t)ldm; + size_t k_ldm_step = k_stride * ldm; detail::unroll_for([&](auto i) { k_bases[i + 1] = k_bases[i] + k_ldm_step; }); @@ -393,7 +393,7 @@ struct wmma_context { n_bases[0] = base; if constexpr (num_n_groups >= 2) { asm volatile("" : "+r"(n_bases[0])); // prevent reverse strength reduction - auto n_ldm_step = n_stride * (uint32_t)ldm; + size_t n_ldm_step = n_stride * ldm; detail::unroll_for([&](auto i) { n_bases[i + 1] = n_bases[i] + n_ldm_step; }); @@ -864,7 +864,7 @@ struct wgmma_context { uint32_t k_in_elem = (lane % tcK) * i_ratio; auto load_reg = [&](uint32_t m_blk, uint32_t k_blk) { - uint32_t elem_off; + size_t elem_off; if (ldm == 0) { // Block-major SMEM: blocks contiguous, k-block outer. constexpr uint32_t block_elems = is_sparse ? sparse_blk_elems : a_blk_elems; @@ -873,7 +873,7 @@ struct wgmma_context { + k_in_elem; } else { // Row-major SMEM: standard row*ldm + col. - elem_off = (m_blk * tcM + i_in) * uint32_t(ldm) + elem_off = (m_blk * tcM + i_in) * ldm + (k_blk * k_row_elems + k_in_elem); } return *reinterpret_cast( @@ -893,14 +893,14 @@ struct wgmma_context { uint32_t m_blk = r / a_reg_k_steps; uint32_t k_blk = r % a_reg_k_steps; if constexpr (k_steps == 1) { - uint32_t elem_off; + size_t elem_off; if (ldm == 0) { elem_off = m_blk * a_blk_elems + i_in * (fedpK * i_ratio) + k_blk * k_row_elems + k_in_elem; } else { - elem_off = (m_blk * tcM + i_in) * uint32_t(ldm) + elem_off = (m_blk * tcM + i_in) * ldm + (k_blk * k_row_elems + k_in_elem); } dst.data[r] = *reinterpret_cast( diff --git a/sw/kernel/src/common.h b/sw/kernel/src/common.h index af05d63fe2..6256141855 100644 --- a/sw/kernel/src/common.h +++ b/sw/kernel/src/common.h @@ -13,7 +13,7 @@ #define RISCV_CUSTOM0 0x0B -#ifdef VX_CFG_XLEN_64 +#if __riscv_xlen == 64 #define LOAD_IMMEDIATE64(rd, imm) \ li t0, (imm >> 32); \ slli t0, t0, 32; \ diff --git a/sw/kernel/src/vx_serial.S b/sw/kernel/src/vx_serial.S index 6d6d8ba827..0cea0704a7 100644 --- a/sw/kernel/src/vx_serial.S +++ b/sw/kernel/src/vx_serial.S @@ -18,8 +18,8 @@ .type vx_serial, @function .global vx_serial vx_serial: -#if (VX_CFG_XLEN == 64) - addi sp, sp, -56 +#if __riscv_xlen == 64 + addi sp, sp, -64 sd ra, 48(sp) sd s5, 40(sp) sd s4, 32(sp) @@ -28,7 +28,7 @@ vx_serial: sd s1, 8(sp) sd s0, 0(sp) #else - addi sp, sp, -28 + addi sp, sp, -32 sw ra, 24(sp) sw s5, 20(sp) sw s4, 16(sp) @@ -53,7 +53,7 @@ label_join: .insn r RISCV_CUSTOM0, 3, 0, x0, s5, x0 # join s5 addi s0, s0, 1 # index++ blt s0, s2, label_loop # loop back -#if (VX_CFG_XLEN == 64) +#if __riscv_xlen == 64 ld ra, 48(sp) ld s5, 40(sp) ld s4, 32(sp) @@ -61,7 +61,7 @@ label_join: ld s2, 16(sp) ld s1, 8(sp) ld s0, 0(sp) - addi sp, sp, 56 + addi sp, sp, 64 #else lw ra, 24(sp) lw s5, 20(sp) @@ -70,6 +70,6 @@ label_join: lw s2, 8(sp) lw s1, 4(sp) lw s0, 0(sp) - addi sp, sp, 28 + addi sp, sp, 32 #endif ret diff --git a/sw/kernel/src/vx_start.S b/sw/kernel/src/vx_start.S index 60d47d8125..5467253caa 100644 --- a/sw/kernel/src/vx_start.S +++ b/sw/kernel/src/vx_start.S @@ -261,5 +261,11 @@ init_tls_all: .section .data .weak __dso_handle +#if __riscv_xlen == 64 + .p2align 3 +__dso_handle: + .dword 0 +#else __dso_handle: .long 0 +#endif diff --git a/sw/kernel/src/vx_syscalls.c b/sw/kernel/src/vx_syscalls.c index b96ff6f17f..bef330323e 100644 --- a/sw/kernel/src/vx_syscalls.c +++ b/sw/kernel/src/vx_syscalls.c @@ -17,6 +17,7 @@ #include #include #include +#include #ifdef __cplusplus extern "C" { @@ -28,21 +29,21 @@ int _fstat(int file, struct stat *st) { return -1; } int _isatty(int file) { return 0; } -int _lseek(int file, int ptr, int dir) { return 0; } +off_t _lseek(int file, off_t ptr, int dir) { return 0; } int _open(const char *name, int flags, int mode) { return -1; } -int _read(int file, char *ptr, int len) { return -1; } +ssize_t _read(int file, void *ptr, size_t len) { return -1; } -caddr_t _sbrk(int incr) { +void *_sbrk(ptrdiff_t incr) { __asm__ __volatile__("ebreak"); return 0; } -int _write(int file, char *ptr, int len) { - int i; - for (i = 0; i < len; ++i) { - vx_putchar(*ptr++); +ssize_t _write(int file, const void *ptr, size_t len) { + const char *p = (const char *)ptr; + for (size_t i = 0; i < len; ++i) { + vx_putchar(*p++); } return len; } diff --git a/tests/mpi/mpi_blocked_sgemm/kernel.cpp b/tests/mpi/mpi_blocked_sgemm/kernel.cpp index 4634bae635..001651da5f 100644 --- a/tests/mpi/mpi_blocked_sgemm/kernel.cpp +++ b/tests/mpi/mpi_blocked_sgemm/kernel.cpp @@ -5,13 +5,13 @@ void kernel_body(kernel_arg_t* __UNIFORM__ arg) { auto A = reinterpret_cast(arg->A_addr); auto B = reinterpret_cast(arg->B_addr); auto C = reinterpret_cast(arg->C_addr); - auto size = arg->size; + size_t size = arg->size; int col = blockIdx.x; int row = blockIdx.y; TYPE sum(0); - for (uint32_t e = 0; e < size; ++e) { + for (size_t e = 0; e < size; ++e) { sum += A[row * size + e] * B[e * size + col]; } diff --git a/tests/mpi/mpi_sgemm/kernel.cpp b/tests/mpi/mpi_sgemm/kernel.cpp index 4634bae635..001651da5f 100644 --- a/tests/mpi/mpi_sgemm/kernel.cpp +++ b/tests/mpi/mpi_sgemm/kernel.cpp @@ -5,13 +5,13 @@ void kernel_body(kernel_arg_t* __UNIFORM__ arg) { auto A = reinterpret_cast(arg->A_addr); auto B = reinterpret_cast(arg->B_addr); auto C = reinterpret_cast(arg->C_addr); - auto size = arg->size; + size_t size = arg->size; int col = blockIdx.x; int row = blockIdx.y; TYPE sum(0); - for (uint32_t e = 0; e < size; ++e) { + for (size_t e = 0; e < size; ++e) { sum += A[row * size + e] * B[e * size + col]; } diff --git a/tests/regression/async_barrier/kernel.cpp b/tests/regression/async_barrier/kernel.cpp index 8fc25a4b61..0a1b3aa082 100644 --- a/tests/regression/async_barrier/kernel.cpp +++ b/tests/regression/async_barrier/kernel.cpp @@ -24,11 +24,11 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto local_B0 = local_A0 + blockDim.x * blockDim.y; auto local_u32 = reinterpret_cast(local_B0 + blockDim.x * blockDim.y); - auto size = arg->size; - auto tile_size = arg->tile_size; + size_t size = arg->size; + size_t tile_size = arg->tile_size; - auto g_row = blockIdx.x * blockDim.x + threadIdx.x; - auto g_col = blockIdx.y * blockDim.y + threadIdx.y; + size_t g_row = blockIdx.x * blockDim.x + threadIdx.x; + size_t g_col = blockIdx.y * blockDim.y + threadIdx.y; auto l_row = threadIdx.x; auto l_col = threadIdx.y; @@ -40,7 +40,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { uint32_t token_compute; - for (uint32_t k = 0; k < size; k += tile_size) { + for (size_t k = 0; k < size; k += tile_size) { if (k > 0) { bar1.wait(token_compute); } @@ -52,7 +52,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { TYPE my_val = local_A0[l_row * tile_size + l_col]; TYPE my_val_b = local_B0[l_row * tile_size + l_col]; - for (uint32_t iter = 0; iter < tile_size; ++iter) { + for (size_t iter = 0; iter < tile_size; ++iter) { my_val = sqrt(my_val * my_val + 0.01f); my_val_b = sqrt(my_val_b * my_val_b + 0.01f); } @@ -60,7 +60,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { bar1.wait(token_load); - for (uint32_t j = 0; j < tile_size; ++j) { + for (size_t j = 0; j < tile_size; ++j) { sum += local_A0[l_row * tile_size + j] * local_B0[j * tile_size + l_col]; } token_compute = bar1.arrive(); diff --git a/tests/regression/basic/kernel.cpp b/tests/regression/basic/kernel.cpp index 5e8f51a124..31eb93f18d 100644 --- a/tests/regression/basic/kernel.cpp +++ b/tests/regression/basic/kernel.cpp @@ -3,13 +3,13 @@ int main() { kernel_arg_t* __UNIFORM__ arg = (kernel_arg_t*)csr_read(VX_CSR_MSCRATCH); - uint32_t count = arg->count; + size_t count = arg->count; int32_t* src_ptr = (int32_t*)arg->src_addr; int32_t* dst_ptr = (int32_t*)arg->dst_addr; - uint32_t offset = vx_core_id() * count; + size_t offset = vx_core_id() * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { dst_ptr[offset + i] = src_ptr[offset + i]; } diff --git a/tests/regression/dotproduct2/kernel.cpp b/tests/regression/dotproduct2/kernel.cpp index c10a20c2bf..bf7ead6823 100644 --- a/tests/regression/dotproduct2/kernel.cpp +++ b/tests/regression/dotproduct2/kernel.cpp @@ -5,8 +5,8 @@ #define VX_CFG_NUM_THREADS 4 #endif -static inline size_t bits_from_f32(float x) { size_t u; __builtin_memcpy(&u, &x, 4); return u; } -static inline float f32_from_bits(size_t u) { float x; __builtin_memcpy(&x, &u, 4); return x; } +static inline size_t bits_from_f32(float x) { uint32_t u; __builtin_memcpy(&u, &x, 4); return u; } +static inline float f32_from_bits(size_t u) { uint32_t b = u; float x; __builtin_memcpy(&x, &b, 4); return x; } static inline float warp_reduce_sum(float x, uint32_t warp_size) { int clamp = warp_size - 1; diff --git a/tests/regression/dxa_copy_mcast/kernel.cpp b/tests/regression/dxa_copy_mcast/kernel.cpp index d56b2558c5..c25a792719 100644 --- a/tests/regression/dxa_copy_mcast/kernel.cpp +++ b/tests/regression/dxa_copy_mcast/kernel.cpp @@ -14,9 +14,9 @@ constexpr uint32_t kDescSrc = 0; __kernel void kernel_main(kernel_arg_t* arg) { - const uint32_t cta_id = get_local_group_id(); - const uint32_t tile_rows = arg->tile_rows; - const uint32_t tile_cols = arg->tile_cols; + const size_t cta_id = get_local_group_id(); + const size_t tile_rows = arg->tile_rows; + const size_t tile_cols = arg->tile_cols; const uint32_t num_recv = arg->num_recv; auto shmem = reinterpret_cast(__local_mem()); @@ -42,9 +42,9 @@ __kernel void kernel_main(kernel_arg_t* arg) { // Store the received tile to dst region for host verification. // Layout: dst[cta_id * tile_elems + r * tile_cols + c] auto dst = reinterpret_cast(arg->dst_addr); - const uint32_t tile_elems = tile_rows * tile_cols; - const uint32_t l_col = threadIdx.x; - for (uint32_t r = 0; r < tile_rows; ++r) { + const size_t tile_elems = tile_rows * tile_cols; + const size_t l_col = threadIdx.x; + for (size_t r = 0; r < tile_rows; ++r) { if (l_col < tile_cols) { dst[cta_id * tile_elems + r * tile_cols + l_col] = shmem[r * tile_cols + l_col]; diff --git a/tests/regression/dxa_kmajor_check/kernel.cpp b/tests/regression/dxa_kmajor_check/kernel.cpp index de381140da..d1bcc86f8c 100644 --- a/tests/regression/dxa_kmajor_check/kernel.cpp +++ b/tests/regression/dxa_kmajor_check/kernel.cpp @@ -14,7 +14,7 @@ constexpr uint32_t kDescA = 0; constexpr uint32_t kDescB = 1; __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { - const uint32_t tid = threadIdx.x; + const size_t tid = threadIdx.x; const uint32_t cta_id = blockIdx.x; // 1D grid — one CTA per (M-tile, N-tile) pair const uint32_t grid_n = arg->N / arg->tileN; const uint32_t cta_row = cta_id / grid_n; @@ -37,8 +37,8 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { // Copy both SMEM regions to dst[cta_id * cta_bytes ..] auto pDst = reinterpret_cast(arg->dst_addr) + cta_id * arg->cta_bytes; - const uint32_t total = arg->cta_bytes; - for (uint32_t i = tid; i < total; i += blockDim.x) { + const size_t total = arg->cta_bytes; + for (size_t i = tid; i < total; i += blockDim.x) { pDst[i] = A_smem[i]; // A_smem is followed by B_smem in memory } __syncthreads(); diff --git a/tests/regression/gfx_binsort_kernel/kernel.cpp b/tests/regression/gfx_binsort_kernel/kernel.cpp index 53e11bdc03..0791bdb204 100644 --- a/tests/regression/gfx_binsort_kernel/kernel.cpp +++ b/tests/regression/gfx_binsort_kernel/kernel.cpp @@ -34,9 +34,9 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto meta = reinterpret_cast(arg->meta_addr); const uint32_t n = arg->num_prims; - const uint32_t tid = threadIdx.x; - const uint32_t T = blockDim.x; - const uint32_t B = BINSORT_NUM_BINS; + const size_t tid = threadIdx.x; + const size_t T = blockDim.x; + const size_t B = BINSORT_NUM_BINS; switch (arg->stage) { @@ -69,7 +69,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { __syncthreads(); if (tid == 0) { uint32_t acc = 0; - for (uint32_t t = 0; t < T; ++t) { uint32_t v = tsum[t]; tsum[t] = acc; acc += v; } + for (size_t t = 0; t < T; ++t) { uint32_t v = tsum[t]; tsum[t] = acc; acc += v; } meta[0] = acc; offset[n] = acc; } __syncthreads(); @@ -82,7 +82,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { for (uint32_t i = lo; i < hi; ++i) { if (count[i] == 0) continue; // culled / empty bbox int bL, bR, bT, bB; bin_range(prims[i], bL, bR, bT, bB); - uint32_t w = offset[i]; + size_t w = offset[i]; for (int by = bT; by <= bB; ++by) for (int bx = bL; bx <= bR; ++bx) keys[w++] = ((uint32_t)(by * BINSORT_BIN_COLS + bx) << BINSORT_PRIM_BITS) | i; @@ -98,7 +98,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { __syncthreads(); for (uint32_t k = klo; k < khi; ++k) { uint32_t b = keys[k] >> BINSORT_PRIM_BITS; if (b >= lo && b < hi) thist[tid * B + b]++; } __syncthreads(); - for (uint32_t b = lo + tid; b < hi; b += T) { uint32_t s = 0; for (uint32_t t = 0; t < T; ++t) s += thist[t * B + b]; bincount[b] = s; } + for (uint32_t b = lo + tid; b < hi; b += T) { uint32_t s = 0; for (size_t t = 0; t < T; ++t) s += thist[t * B + b]; bincount[b] = s; } } break; case BINSORT_STAGE_BASE: { // single CTA: bin-base scan + headers @@ -123,7 +123,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { uint32_t kchunk = (P + T - 1) / T; uint32_t klo = umin(tid * kchunk, P), khi = umin(klo + kchunk, P); // owned hist columns -> stable write cursors (base + thread-exclusive prefix) - for (uint32_t b = lo + tid; b < hi; b += T) { uint32_t run = binbase[b]; for (uint32_t t = 0; t < T; ++t) { uint32_t c = thist[t * B + b]; thist[t * B + b] = run; run += c; } } + for (uint32_t b = lo + tid; b < hi; b += T) { uint32_t run = binbase[b]; for (size_t t = 0; t < T; ++t) { uint32_t c = thist[t * B + b]; thist[t * B + b] = run; run += c; } } __syncthreads(); for (uint32_t k = klo; k < khi; ++k) { uint32_t key = keys[k], b = key >> BINSORT_PRIM_BITS; diff --git a/tests/regression/gfx_pipeline_kernel/kernel.cpp b/tests/regression/gfx_pipeline_kernel/kernel.cpp index 765cfc8b16..1d17eb3703 100644 --- a/tests/regression/gfx_pipeline_kernel/kernel.cpp +++ b/tests/regression/gfx_pipeline_kernel/kernel.cpp @@ -62,9 +62,9 @@ __kernel void setup_k(kernel_arg_t* __UNIFORM__ arg) { switch (arg->stage) { case PIPE_STAGE_SETUP: { // multi-CTA, grid-stride - uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; - uint32_t gstride = gridDim.x * blockDim.x; - for (uint32_t t = gid; t < ntri; t += gstride) { + size_t gid = blockIdx.x * blockDim.x + threadIdx.x; + size_t gstride = gridDim.x * blockDim.x; + for (size_t t = gid; t < ntri; t += gstride) { rast_prim_t pr[SETUP_MAX_SUB]; setup_bbox_t bb[SETUP_MAX_SUB]; uint32_t kept = clip_and_setup(&verts[3 * t], W, H, pr, bb); @@ -91,12 +91,12 @@ __kernel void setup_k(kernel_arg_t* __UNIFORM__ arg) { } break; case PIPE_STAGE_EMIT: { // multi-CTA, grid-stride - uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; - uint32_t gstride = gridDim.x * blockDim.x; - for (uint32_t t = gid; t < ntri; t += gstride) { - uint32_t w = offset[t]; + size_t gid = blockIdx.x * blockDim.x + threadIdx.x; + size_t gstride = gridDim.x * blockDim.x; + for (size_t t = gid; t < ntri; t += gstride) { + size_t w = offset[t]; for (uint32_t s = 0; s < keep[t]; ++s) { - uint32_t slot = t * SETUP_MAX_SUB + s; + size_t slot = t * SETUP_MAX_SUB + s; prim[w] = slot_prim[slot]; bbox[w] = slot_bbox[slot]; ++w; @@ -120,17 +120,17 @@ __kernel void binning_k(kernel_arg_t* __UNIFORM__ arg) { auto tilebuf = reinterpret_cast(arg->tilebuf_addr); auto meta = reinterpret_cast(arg->meta_addr); - const uint32_t tid = threadIdx.x; - const uint32_t T = blockDim.x; - const uint32_t B = PIPE_NUM_BINS; + const size_t tid = threadIdx.x; + const size_t T = blockDim.x; + const size_t B = PIPE_NUM_BINS; switch (arg->stage) { case PIPE_STAGE_BCOUNT: { // multi-CTA, grid-stride - uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; - uint32_t gstride = gridDim.x * blockDim.x; + size_t gid = blockIdx.x * blockDim.x + threadIdx.x; + size_t gstride = gridDim.x * blockDim.x; uint32_t P = meta[0]; - for (uint32_t i = gid; i < P; i += gstride) { + for (size_t i = gid; i < P; i += gstride) { int bL, bR, bT, bB; bin_range(bbox[i], bL, bR, bT, bB); bcount[i] = (uint32_t)((bR - bL + 1) * (bB - bT + 1)); } @@ -152,12 +152,12 @@ __kernel void binning_k(kernel_arg_t* __UNIFORM__ arg) { } break; case PIPE_STAGE_BEMIT: { // multi-CTA, grid-stride - uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; - uint32_t gstride = gridDim.x * blockDim.x; + size_t gid = blockIdx.x * blockDim.x + threadIdx.x; + size_t gstride = gridDim.x * blockDim.x; uint32_t P = meta[0]; - for (uint32_t i = gid; i < P; i += gstride) { + for (size_t i = gid; i < P; i += gstride) { int bL, bR, bT, bB; bin_range(bbox[i], bL, bR, bT, bB); - uint32_t w = boffset[i]; + size_t w = boffset[i]; for (int by = bT; by <= bB; ++by) for (int bx = bL; bx <= bR; ++bx) keys[w++] = ((uint32_t)(by * PIPE_BIN_COLS + bx) << PIPE_PRIM_BITS) | i; @@ -173,7 +173,7 @@ __kernel void binning_k(kernel_arg_t* __UNIFORM__ arg) { __syncthreads(); for (uint32_t k = klo; k < khi; ++k) { uint32_t b = keys[k] >> PIPE_PRIM_BITS; if (b >= lo && b < hi) thist[tid * B + b]++; } __syncthreads(); - for (uint32_t b = lo + tid; b < hi; b += T) { uint32_t s = 0; for (uint32_t t = 0; t < T; ++t) s += thist[t * B + b]; bincount[b] = s; } + for (uint32_t b = lo + tid; b < hi; b += T) { uint32_t s = 0; for (size_t t = 0; t < T; ++t) s += thist[t * B + b]; bincount[b] = s; } } break; case PIPE_STAGE_BBASE: { // single CTA: tilebuf headers @@ -205,7 +205,7 @@ __kernel void binning_k(kernel_arg_t* __UNIFORM__ arg) { auto pids = reinterpret_cast(tilebuf + nb * sizeof(rast_tile_header_t)); uint32_t kchunk = (K + T - 1) / T; uint32_t klo = umin(tid * kchunk, K), khi = umin(klo + kchunk, K); - for (uint32_t b = lo + tid; b < hi; b += T) { uint32_t run = binbase[b]; for (uint32_t t = 0; t < T; ++t) { uint32_t c = thist[t * B + b]; thist[t * B + b] = run; run += c; } } + for (uint32_t b = lo + tid; b < hi; b += T) { uint32_t run = binbase[b]; for (size_t t = 0; t < T; ++t) { uint32_t c = thist[t * B + b]; thist[t * B + b] = run; run += c; } } __syncthreads(); for (uint32_t k = klo; k < khi; ++k) { uint32_t key = keys[k], b = key >> PIPE_PRIM_BITS; diff --git a/tests/regression/gfx_setup_kernel/kernel.cpp b/tests/regression/gfx_setup_kernel/kernel.cpp index 657311e37a..d20c43dd83 100644 --- a/tests/regression/gfx_setup_kernel/kernel.cpp +++ b/tests/regression/gfx_setup_kernel/kernel.cpp @@ -38,9 +38,9 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { switch (arg->stage) { case SETUP_STAGE_SETUP: { // multi-CTA, grid-stride - uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; - uint32_t gstride = gridDim.x * blockDim.x; - for (uint32_t t = gid; t < n; t += gstride) { + size_t gid = blockIdx.x * blockDim.x + threadIdx.x; + size_t gstride = gridDim.x * blockDim.x; + for (size_t t = gid; t < n; t += gstride) { clip_tri_t sub[SETUP_MAX_SUB]; int ns = clip_near(verts[3 * t + 0], verts[3 * t + 1], verts[3 * t + 2], sub); uint32_t kept = 0; @@ -49,7 +49,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { setup_bbox_t bbox{}; if (setup_triangle(sub[s].v[0], sub[s].v[1], sub[s].v[2], W, H, SETUP_NEAR, SETUP_FAR, prim, bbox, arg->cull_mode)) { - uint32_t slot = t * SETUP_MAX_SUB + kept; + size_t slot = t * SETUP_MAX_SUB + kept; slot_prim[slot] = prim; slot_bbox[slot] = bbox; slot_vtx[slot] = sub[s]; @@ -75,12 +75,12 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { } break; case SETUP_STAGE_EMIT: { // multi-CTA, grid-stride - uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; - uint32_t gstride = gridDim.x * blockDim.x; - for (uint32_t t = gid; t < n; t += gstride) { - uint32_t w = offset[t]; + size_t gid = blockIdx.x * blockDim.x + threadIdx.x; + size_t gstride = gridDim.x * blockDim.x; + for (size_t t = gid; t < n; t += gstride) { + size_t w = offset[t]; for (uint32_t s = 0; s < keep[t]; ++s) { - uint32_t slot = t * SETUP_MAX_SUB + s; + size_t slot = t * SETUP_MAX_SUB + s; out_prim[w] = slot_prim[slot]; out_bbox[w] = slot_bbox[slot]; out_vtx[w] = slot_vtx[slot]; diff --git a/tests/regression/jacobi/kernel.cpp b/tests/regression/jacobi/kernel.cpp index 5b16c63cf9..9e10782d50 100644 --- a/tests/regression/jacobi/kernel.cpp +++ b/tests/regression/jacobi/kernel.cpp @@ -6,15 +6,15 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto x_old = reinterpret_cast(arg->src1_addr); auto b = reinterpret_cast(arg->src2_addr); auto x_new = reinterpret_cast(arg->dst_addr); - auto n = arg->size; + size_t n = arg->size; - uint32_t i = blockIdx.x * blockDim.x + threadIdx.x; + size_t i = blockIdx.x * blockDim.x + threadIdx.x; if (i >= n) return; - uint32_t index = i * n; + size_t index = i * n; double sum = 0.0; - for (uint32_t j = 0; j < n; j++) { + for (size_t j = 0; j < n; j++) { if (j != i) sum += A[index + j] * x_old[j]; } diff --git a/tests/regression/module_reload/kernel.cpp b/tests/regression/module_reload/kernel.cpp index 1ebcfeddae..0491f994d1 100644 --- a/tests/regression/module_reload/kernel.cpp +++ b/tests/regression/module_reload/kernel.cpp @@ -3,13 +3,13 @@ int main() { kernel_arg_t* __UNIFORM__ arg = (kernel_arg_t*)csr_read(VX_CSR_MSCRATCH); - uint32_t count = arg->count; + size_t count = arg->count; int32_t* src_ptr = (int32_t*)arg->src_addr; int32_t* dst_ptr = (int32_t*)arg->dst_addr; - uint32_t offset = vx_core_id() * count; + size_t offset = vx_core_id() * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { dst_ptr[offset + i] = src_ptr[offset + i]; } diff --git a/tests/regression/mstress/kernel.cpp b/tests/regression/mstress/kernel.cpp index f3b5d2d212..8a66f2047c 100644 --- a/tests/regression/mstress/kernel.cpp +++ b/tests/regression/mstress/kernel.cpp @@ -2,18 +2,18 @@ #include "common.h" __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { - uint32_t stride = arg->stride; + size_t stride = arg->stride; uint32_t* addr_ptr = (uint32_t*)arg->src0_addr; float* src_ptr = (float*)arg->src1_addr; float* dst_ptr = (float*)arg->dst_addr; - uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; - uint32_t offset = gid * stride; + size_t gid = blockIdx.x * blockDim.x + threadIdx.x; + size_t offset = gid * stride; - for (uint32_t i = 0; i < stride; ++i) { + for (size_t i = 0; i < stride; ++i) { float value = 0.0f; for (uint32_t j = 0; j < NUM_LOADS; ++j) { - uint32_t addr = offset + i + j; + size_t addr = offset + i + j; uint32_t index = addr_ptr[addr]; value *= src_ptr[index]; } diff --git a/tests/regression/packld/kernel.cpp b/tests/regression/packld/kernel.cpp index 0d3bef2215..5473c06d22 100644 --- a/tests/regression/packld/kernel.cpp +++ b/tests/regression/packld/kernel.cpp @@ -14,7 +14,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto dst_lb = reinterpret_cast(arg->dst_lb_addr); auto dst_lh = reinterpret_cast(arg->dst_lh_addr); - uint32_t tid = blockIdx.x * blockDim.x + threadIdx.x; + size_t tid = blockIdx.x * blockDim.x + threadIdx.x; uint32_t stride = 1; // byte stride between consecutive elements for (uint32_t p = 0; p < NUM_POINTS; ++p) { diff --git a/tests/regression/packld_box/kernel.cpp b/tests/regression/packld_box/kernel.cpp index b407b12518..e41ef52e7f 100644 --- a/tests/regression/packld_box/kernel.cpp +++ b/tests/regression/packld_box/kernel.cpp @@ -15,7 +15,7 @@ // Register spilling silently erases the bug by doing proper NaN-boxing. // Uncomment the two lines to see all the tests pass. __attribute__((always_inline)) -inline uint64_t packlb_bits(const void* base, uint32_t stride) { +inline uint64_t packlb_bits(const void* base, size_t stride) { uint64_t bits; __asm__ volatile ( ".insn r %1, 1, 4, ft0, %2, %3\n\t" @@ -28,7 +28,7 @@ inline uint64_t packlb_bits(const void* base, uint32_t stride) { } __attribute__((always_inline)) -inline uint64_t packlh_bits(const void* base, uint32_t stride) { +inline uint64_t packlh_bits(const void* base, size_t stride) { uint64_t bits; __asm__ volatile ( ".insn r %1, 2, 4, ft0, %2, %3\n\t" @@ -53,8 +53,8 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto dst_lb = reinterpret_cast(arg->dst_lb_addr); auto dst_lh = reinterpret_cast(arg->dst_lh_addr); - uint32_t tid = blockIdx.x * blockDim.x + threadIdx.x; - uint32_t stride = 1; // byte stride between consecutive elements + size_t tid = blockIdx.x * blockDim.x + threadIdx.x; + size_t stride = 1; // byte stride between consecutive elements for (uint32_t p = 0; p < NUM_POINTS; ++p) { // 4 bytes at consecutive addresses → one packed float (PACKLB) diff --git a/tests/regression/raycast/render.h b/tests/regression/raycast/render.h index f793a6e946..b75d87fc84 100644 --- a/tests/regression/raycast/render.h +++ b/tests/regression/raycast/render.h @@ -78,7 +78,7 @@ void BVHIntersect(const ray_t &ray, const tri_t *triBuffer, ray_hit_t *hit) { uint32_t stack[BVH_STACK_SIZE]; - uint32_t stackPtr = 0; + size_t stackPtr = 0; stack[stackPtr++] = 0; // Push root node index while (stackPtr != 0) { @@ -86,7 +86,7 @@ void BVHIntersect(const ray_t &ray, const bvh_node_t &node = bvhBuffer[nodeIdx]; if (node.isLeaf()) { // Intersect leaf triangles - for (uint32_t i = 0; i < node.triCount; ++i) { + for (size_t i = 0; i < node.triCount; ++i) { uint32_t triIdx = triIdxBuffer[node.leftFirst + i]; float dist; float3_t bcoords; @@ -149,7 +149,7 @@ void TLASIntersect(const ray_t &ray, const tri_t *triBuffer, ray_hit_t *hit) { uint32_t stack[BVH_STACK_SIZE]; - uint32_t stackPtr = 0; + size_t stackPtr = 0; stack[stackPtr++] = tlas_root; while (stackPtr != 0) { diff --git a/tests/regression/reopen/kernel.cpp b/tests/regression/reopen/kernel.cpp index 1ebcfeddae..0491f994d1 100644 --- a/tests/regression/reopen/kernel.cpp +++ b/tests/regression/reopen/kernel.cpp @@ -3,13 +3,13 @@ int main() { kernel_arg_t* __UNIFORM__ arg = (kernel_arg_t*)csr_read(VX_CSR_MSCRATCH); - uint32_t count = arg->count; + size_t count = arg->count; int32_t* src_ptr = (int32_t*)arg->src_addr; int32_t* dst_ptr = (int32_t*)arg->dst_addr; - uint32_t offset = vx_core_id() * count; + size_t offset = vx_core_id() * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { dst_ptr[offset + i] = src_ptr[offset + i]; } diff --git a/tests/regression/sgemm/kernel.cpp b/tests/regression/sgemm/kernel.cpp index bafb73cb91..7b79bbfb11 100644 --- a/tests/regression/sgemm/kernel.cpp +++ b/tests/regression/sgemm/kernel.cpp @@ -5,14 +5,14 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto A = reinterpret_cast(arg->A_addr); auto B = reinterpret_cast(arg->B_addr); auto C = reinterpret_cast(arg->C_addr); - auto size = arg->size; + size_t size = arg->size; int col = blockIdx.x * blockDim.x + threadIdx.x; int row = blockIdx.y * blockDim.y + threadIdx.y; TYPE sum(0); #pragma unroll 8 - for (uint32_t e = 0; e < size; ++e) { + for (size_t e = 0; e < size; ++e) { sum += A[row * size + e] * B[e * size + col]; } diff --git a/tests/regression/sgemm2/kernel.cpp b/tests/regression/sgemm2/kernel.cpp index 17a8817f9a..afe6c336e8 100644 --- a/tests/regression/sgemm2/kernel.cpp +++ b/tests/regression/sgemm2/kernel.cpp @@ -6,15 +6,15 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto B_ptr = reinterpret_cast(arg->B_addr); auto C_ptr = reinterpret_cast(arg->C_addr); - auto size = arg->size; - auto tile_size = arg->tile_size; - auto chunk_k = arg->chunk_k; + size_t size = arg->size; + size_t tile_size = arg->tile_size; + size_t chunk_k = arg->chunk_k; // Global and local thread coordinates. - auto g_row = blockIdx.x * blockDim.x + threadIdx.x; - auto g_col = blockIdx.y * blockDim.y + threadIdx.y; - auto l_row = threadIdx.x; - auto l_col = threadIdx.y; + size_t g_row = blockIdx.x * blockDim.x + threadIdx.x; + size_t g_col = blockIdx.y * blockDim.y + threadIdx.y; + size_t l_row = threadIdx.x; + size_t l_col = threadIdx.y; // Shared memory layout: // local_A: [tile_size × chunk_k], row-major (row = l_row, col = k_local) @@ -26,19 +26,19 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { TYPE sum(0); - for (uint32_t k = 0; k < size; k += chunk_k) { + for (size_t k = 0; k < size; k += chunk_k) { // Cooperative load: each thread loads chunk_k/tile_size elements per tile. // Thread (l_row, l_col) loads columns l_col, l_col+tile_size, ... of its row. - for (uint32_t kk = l_col; kk < chunk_k; kk += tile_size) { + for (size_t kk = l_col; kk < chunk_k; kk += tile_size) { local_A[l_row * chunk_k + kk] = A_ptr[g_row * size + (k + kk)]; } - for (uint32_t kk = l_row; kk < chunk_k; kk += tile_size) { + for (size_t kk = l_row; kk < chunk_k; kk += tile_size) { local_B[kk * tile_size + l_col] = B_ptr[(k + kk) * size + g_col]; } __syncthreads(); - for (uint32_t j = 0; j < chunk_k; ++j) { + for (size_t j = 0; j < chunk_k; ++j) { sum += local_A[l_row * chunk_k + j] * local_B[j * tile_size + l_col]; } diff --git a/tests/regression/sgemm2_dxa/kernel.cpp b/tests/regression/sgemm2_dxa/kernel.cpp index f152f52275..7693034844 100644 --- a/tests/regression/sgemm2_dxa/kernel.cpp +++ b/tests/regression/sgemm2_dxa/kernel.cpp @@ -11,9 +11,9 @@ constexpr uint32_t kDescB = 1; // Accumulate partial sum from shared memory tiles. // shA layout: [tile_size × chunk_k], row-major (row = l_row, col = k) // shB layout: [chunk_k × tile_size], row-major (row = k, col = l_col) -static inline void gemm_accumulate(TYPE& sum, const TYPE* shA, const TYPE* shB, uint32_t tile_size, - uint32_t chunk_k, uint32_t l_row, uint32_t l_col) { - for (uint32_t k = 0; k < chunk_k; ++k) { +static inline void gemm_accumulate(TYPE& sum, const TYPE* shA, const TYPE* shB, size_t tile_size, + size_t chunk_k, size_t l_row, size_t l_col) { + for (size_t k = 0; k < chunk_k; ++k) { sum += shA[l_row * chunk_k + k] * shB[k * tile_size + l_col]; } } @@ -21,25 +21,25 @@ static inline void gemm_accumulate(TYPE& sum, const TYPE* shA, const TYPE* shB, __kernel void kernel_main(kernel_arg_t* arg) { auto C = reinterpret_cast(arg->C_addr); - const uint32_t size = arg->size; - const uint32_t tile_size = arg->tile_size; - const uint32_t chunk_k = arg->chunk_k; + const size_t size = arg->size; + const size_t tile_size = arg->tile_size; + const size_t chunk_k = arg->chunk_k; const uint32_t mode = arg->mode; // Global and local thread coordinates. - const uint32_t row_base = blockIdx.x * tile_size; - const uint32_t col_base = blockIdx.y * tile_size; - const uint32_t l_row = threadIdx.x; - const uint32_t l_col = threadIdx.y; - const uint32_t g_row = row_base + l_row; - const uint32_t g_col = col_base + l_col; + const size_t row_base = blockIdx.x * tile_size; + const size_t col_base = blockIdx.y * tile_size; + const size_t l_row = threadIdx.x; + const size_t l_col = threadIdx.y; + const size_t g_row = row_base + l_row; + const size_t g_col = col_base + l_col; TYPE sum(0); // SMEM tile layout: A tile is [tile_size x chunk_k], B tile is [chunk_k x tile_size]. - const uint32_t tile_elems_a = tile_size * chunk_k; - const uint32_t tile_elems_b = chunk_k * tile_size; - const uint32_t stage_elems = tile_elems_a + tile_elems_b; + const size_t tile_elems_a = tile_size * chunk_k; + const size_t tile_elems_b = chunk_k * tile_size; + const size_t stage_elems = tile_elems_a + tile_elems_b; // const uint32_t stage_count = (mode == 2) ? 2u : 1u; // Allocate shared memory for tile buffers. diff --git a/tests/regression/sgemm2_dxa_mcast/kernel.cpp b/tests/regression/sgemm2_dxa_mcast/kernel.cpp index 1127865e94..7552459835 100644 --- a/tests/regression/sgemm2_dxa_mcast/kernel.cpp +++ b/tests/regression/sgemm2_dxa_mcast/kernel.cpp @@ -26,16 +26,16 @@ constexpr uint32_t kDescB = 1; __kernel void kernel_main(kernel_arg_t* arg) { auto C = reinterpret_cast(arg->C_addr); - const uint32_t size = arg->size; - const uint32_t tile_size = arg->tile_size; - const uint32_t chunk_k = arg->chunk_k; + const size_t size = arg->size; + const size_t tile_size = arg->tile_size; + const size_t chunk_k = arg->chunk_k; const uint32_t mc_group_size = arg->mc_group_size; // CTA position. blockIdx.x walks rows; blockIdx.y walks column-blocks. // 4 CTAs (mc_group_size) in adjacent blockIdx.x values are co-resident // (= one local group) and share one B column-block via multicast. - const uint32_t row = blockIdx.x; - const uint32_t col_blk = blockIdx.y; + const size_t row = blockIdx.x; + const size_t col_blk = blockIdx.y; // SMEM tile layout: A row [chunk_k] then B column-block [chunk_k × tile_size]. auto shmem = reinterpret_cast(__local_mem()); @@ -68,9 +68,9 @@ __kernel void kernel_main(kernel_arg_t* arg) { local_B.arrive_and_wait(); // ── Compute: 1 row × tile_size cols ─────────────────────────────────────── - const uint32_t l_col = threadIdx.x; // 0..tile_size-1 (single warp) + const size_t l_col = threadIdx.x; // 0..tile_size-1 (single warp) TYPE sum(0); - for (uint32_t k = 0; k < chunk_k; ++k) { + for (size_t k = 0; k < chunk_k; ++k) { sum += shA[k] * shB[k * tile_size + l_col]; } diff --git a/tests/regression/sgemm2_tcu/kernel.cpp b/tests/regression/sgemm2_tcu/kernel.cpp index ff7c717dd0..3cc808bf0d 100644 --- a/tests/regression/sgemm2_tcu/kernel.cpp +++ b/tests/regression/sgemm2_tcu/kernel.cpp @@ -11,16 +11,16 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto pB = reinterpret_cast(arg->B_addr); auto pC = reinterpret_cast(arg->C_addr); - uint32_t N = arg->N; - uint32_t K = arg->K; + size_t N = arg->N; + size_t K = arg->K; uint32_t cta_M = arg->cta_M; uint32_t tid = threadIdx.x; uint32_t num_threads = blockDim.x; // num_warps * VX_CFG_NUM_THREADS uint32_t warp_rank = tid / VX_CFG_NUM_THREADS; - uint32_t tile_row = blockIdx.y * cta_M; - uint32_t tile_col = blockIdx.x * ctx::tileN; + size_t tile_row = blockIdx.y * cta_M; + size_t tile_col = blockIdx.x * ctx::tileN; // Shared memory: A [cta_M x tileK] row-major, B [tileK x tileN] row-major auto smem = reinterpret_cast(__local_mem()); @@ -32,23 +32,23 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { ctx::fragment_acc fragC; ctx::fill_fragment(fragC, 0); - for (uint32_t k = 0; k < K; k += ctx::tileK) { + for (size_t k = 0; k < K; k += ctx::tileK) { // Cooperatively load A tile [cta_M x tileK] from DRAM (row-major). uint32_t a_size = cta_M * ctx::tileK; - for (uint32_t i = 0; i < a_size; i += num_threads) { - uint32_t idx = i + tid; - uint32_t r = idx / ctx::tileK; - uint32_t c = idx % ctx::tileK; + for (size_t i = 0; i < a_size; i += num_threads) { + size_t idx = i + tid; + size_t r = idx / ctx::tileK; + size_t c = idx % ctx::tileK; A_smem[r * ctx::tileK + c] = pA[(tile_row + r) * K + (k + c)]; } // Cooperatively load B tile [tileK x tileN] from DRAM (col-major source) // and store row-major in shared memory. uint32_t b_size = ctx::tileK * ctx::tileN; - for (uint32_t i = 0; i < b_size; i += num_threads) { - uint32_t idx = i + tid; - uint32_t r = idx / ctx::tileN; - uint32_t c = idx % ctx::tileN; + for (size_t i = 0; i < b_size; i += num_threads) { + size_t idx = i + tid; + size_t r = idx / ctx::tileN; + size_t c = idx % ctx::tileN; B_smem[r * ctx::tileN + c] = pB[(tile_col + c) * K + (k + r)]; } diff --git a/tests/regression/sgemm2_v1/kernel.cpp b/tests/regression/sgemm2_v1/kernel.cpp index e26687a322..9f32b87f3d 100644 --- a/tests/regression/sgemm2_v1/kernel.cpp +++ b/tests/regression/sgemm2_v1/kernel.cpp @@ -6,15 +6,15 @@ void kernel_body(kernel_arg_t *arg) { auto B_ptr = reinterpret_cast(arg->B_addr); auto C_ptr = reinterpret_cast(arg->C_addr); - auto size = arg->size; - auto tile_size = arg->tile_size; - auto chunk_k = arg->chunk_k; + size_t size = arg->size; + size_t tile_size = arg->tile_size; + size_t chunk_k = arg->chunk_k; // Global and local thread coordinates. - auto g_row = blockIdx.x * blockDim.x + threadIdx.x; - auto g_col = blockIdx.y * blockDim.y + threadIdx.y; - auto l_row = threadIdx.x; - auto l_col = threadIdx.y; + size_t g_row = blockIdx.x * blockDim.x + threadIdx.x; + size_t g_col = blockIdx.y * blockDim.y + threadIdx.y; + size_t l_row = threadIdx.x; + size_t l_col = threadIdx.y; // Shared memory layout: // local_A: [tile_size × chunk_k], row-major (row = l_row, col = k_local) @@ -27,19 +27,19 @@ void kernel_body(kernel_arg_t *arg) { TYPE sum(0); - for (uint32_t k = 0; k < size; k += chunk_k) { + for (size_t k = 0; k < size; k += chunk_k) { // Cooperative load: each thread loads chunk_k/tile_size elements per tile. // Thread (l_row, l_col) loads columns l_col, l_col+tile_size, ... of its row. - for (uint32_t kk = l_col; kk < chunk_k; kk += tile_size) { + for (size_t kk = l_col; kk < chunk_k; kk += tile_size) { local_A[l_row * chunk_k + kk] = A_ptr[g_row * size + (k + kk)]; } - for (uint32_t kk = l_row; kk < chunk_k; kk += tile_size) { + for (size_t kk = l_row; kk < chunk_k; kk += tile_size) { local_B[kk * tile_size + l_col] = B_ptr[(k + kk) * size + g_col]; } __syncthreads(); - for (uint32_t j = 0; j < chunk_k; ++j) { + for (size_t j = 0; j < chunk_k; ++j) { sum += local_A[l_row * chunk_k + j] * local_B[j * tile_size + l_col]; } diff --git a/tests/regression/sgemm_tcu_wg_dxa/kernel.cpp b/tests/regression/sgemm_tcu_wg_dxa/kernel.cpp index 05dfd78059..281bd1aa77 100644 --- a/tests/regression/sgemm_tcu_wg_dxa/kernel.cpp +++ b/tests/regression/sgemm_tcu_wg_dxa/kernel.cpp @@ -99,10 +99,10 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { #ifdef SW_LOAD_A // Cooperative load A row-major (matches DXA row-major layout). uint32_t a_size = cta_M * ctx::tileK; - for (uint32_t i = 0; i < a_size; i += num_threads) { - uint32_t idx = i + tid; - uint32_t r = idx / ctx::tileK; - uint32_t c = idx % ctx::tileK; + for (size_t i = 0; i < a_size; i += num_threads) { + size_t idx = i + tid; + size_t r = idx / ctx::tileK; + size_t c = idx % ctx::tileK; A_smem[r * ctx::tileK + c] = pA[(tile_row + r) * K + (k + c)]; } #endif diff --git a/tests/regression/sgemm_tcu_wg_mx/kernel.cpp b/tests/regression/sgemm_tcu_wg_mx/kernel.cpp index f370bbd583..34bb9e5c9c 100644 --- a/tests/regression/sgemm_tcu_wg_mx/kernel.cpp +++ b/tests/regression/sgemm_tcu_wg_mx/kernel.cpp @@ -19,15 +19,15 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto pBTensorScale = reinterpret_cast(arg->B_tensor_scale_addr); #endif - uint32_t N = arg->N; - uint32_t K = arg->K; + size_t N = arg->N; + size_t K = arg->K; uint32_t tid = threadIdx.x; uint32_t num_threads = blockDim.x; uint32_t warp_rank = tid / VX_CFG_NUM_THREADS; uint32_t num_warps = num_threads / VX_CFG_NUM_THREADS; uint32_t cta_m = num_warps * ctx::xtileM; - uint32_t tile_row = blockIdx.y * cta_m; - uint32_t tile_col = blockIdx.x * ctx::xtileN; + size_t tile_row = blockIdx.y * cta_m; + size_t tile_col = blockIdx.x * ctx::xtileN; uint32_t num_k_tiles = K / ctx::tileK; auto A_smem = reinterpret_cast(__local_mem()); @@ -37,18 +37,18 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { ctx::fragment_acc fragC; ctx::fill_fragment(fragC, 0); - for (uint32_t k = 0; k < K; k += ctx::tileK) { + for (size_t k = 0; k < K; k += ctx::tileK) { uint32_t k_tile = k / ctx::tileK; uint32_t a_bytes = cta_m * ctx::tileK; - for (uint32_t idx = tid; idx < a_bytes; idx += num_threads) { - uint32_t row = idx / ctx::tileK; - uint32_t col = idx % ctx::tileK; + for (size_t idx = tid; idx < a_bytes; idx += num_threads) { + size_t row = idx / ctx::tileK; + size_t col = idx % ctx::tileK; A_smem[idx] = pA[(tile_row + row) * K + k + col]; } uint32_t b_bytes = ctx::xtileN * ctx::tileK; - for (uint32_t idx = tid; idx < b_bytes; idx += num_threads) { - uint32_t col = idx / ctx::tileK; - uint32_t k_byte = idx % ctx::tileK; + for (size_t idx = tid; idx < b_bytes; idx += num_threads) { + size_t col = idx / ctx::tileK; + size_t k_byte = idx % ctx::tileK; B_smem[idx] = pB[(tile_col + col) * K + k + k_byte]; } __syncthreads(); diff --git a/tests/regression/sgemm_tcu_wg_sp/kernel.cpp b/tests/regression/sgemm_tcu_wg_sp/kernel.cpp index 8c60045c2e..f1872cbed6 100644 --- a/tests/regression/sgemm_tcu_wg_sp/kernel.cpp +++ b/tests/regression/sgemm_tcu_wg_sp/kernel.cpp @@ -27,8 +27,8 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto pA = reinterpret_cast(arg->A_addr); // compressed A auto pB = reinterpret_cast(arg->B_addr); // dense B - uint32_t N = arg->N; - uint32_t K = arg->K; + size_t N = arg->N; + size_t K = arg->K; uint32_t tid = threadIdx.x; uint32_t lane = tid % VX_CFG_NUM_THREADS; @@ -38,8 +38,8 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { // CTA tile dimensions uint32_t cta_M = num_warps * ctx::xtileM; - uint32_t tile_row = blockIdx.y * cta_M; - uint32_t tile_col = blockIdx.x * ctx::xtileN; + size_t tile_row = blockIdx.y * cta_M; + size_t tile_col = blockIdx.x * ctx::xtileN; auto smem_base = reinterpret_cast(__local_mem()); uint32_t smem_b_off = @@ -51,28 +51,28 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { ctx::fragment_acc fragC; ctx::fill_fragment(fragC, 0); - uint32_t a_sp_stride = K / 2; + size_t a_sp_stride = K / 2; uint32_t num_k_tiles = K / ctx::tileK; uint32_t meta_words_per_tile = ctx::wg_meta_total_bytes / 4; - for (uint32_t k = 0; k < K; k += ctx::tileK) { + for (size_t k = 0; k < K; k += ctx::tileK) { uint32_t k_tile = k / ctx::tileK; // Each warp loads its compressed A tile into its private TBUF section. auto A_smem_w = reinterpret_cast( smem_base + warp_rank * per_warp_section); - for (uint32_t i = lane; i < smem_a_elems; i += VX_CFG_NUM_THREADS) { - uint32_t r = i / (ctx::tileK / 2); - uint32_t c = i % (ctx::tileK / 2); + for (size_t i = lane; i < smem_a_elems; i += VX_CFG_NUM_THREADS) { + size_t r = i / (ctx::tileK / 2); + size_t c = i % (ctx::tileK / 2); A_smem_w[i] = pA[(tile_row + warp_rank * ctx::xtileM + r) * a_sp_stride + (k / 2 + c)]; } // Cooperative load: K-major B, matching the TBUF's strided descriptor. - for (uint32_t i = tid; i < smem_b_elems; i += num_threads) { - uint32_t r = i / ctx::xtileN; - uint32_t c = i % ctx::xtileN; + for (size_t i = tid; i < smem_b_elems; i += num_threads) { + size_t r = i / ctx::xtileN; + size_t c = i % ctx::xtileN; B_smem[c * ctx::tileK + r] = pB[(k + r) * N + (tile_col + c)]; } diff --git a/tests/regression/sgemm_v1/kernel.cpp b/tests/regression/sgemm_v1/kernel.cpp index 4634bae635..001651da5f 100644 --- a/tests/regression/sgemm_v1/kernel.cpp +++ b/tests/regression/sgemm_v1/kernel.cpp @@ -5,13 +5,13 @@ void kernel_body(kernel_arg_t* __UNIFORM__ arg) { auto A = reinterpret_cast(arg->A_addr); auto B = reinterpret_cast(arg->B_addr); auto C = reinterpret_cast(arg->C_addr); - auto size = arg->size; + size_t size = arg->size; int col = blockIdx.x; int row = blockIdx.y; TYPE sum(0); - for (uint32_t e = 0; e < size; ++e) { + for (size_t e = 0; e < size; ++e) { sum += A[row * size + e] * B[e * size + col]; } diff --git a/tests/regression/stencil3d/kernel.cpp b/tests/regression/stencil3d/kernel.cpp index bab24af116..9c0c0c7bf4 100644 --- a/tests/regression/stencil3d/kernel.cpp +++ b/tests/regression/stencil3d/kernel.cpp @@ -5,7 +5,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto A = reinterpret_cast(arg->A_addr); auto B = reinterpret_cast(arg->B_addr); - auto size = arg->size; + size_t size = arg->size; int col = blockIdx.x * blockDim.x + threadIdx.x; int row = blockIdx.y * blockDim.y + threadIdx.y; diff --git a/tests/regression/vm_stress/kernel.cpp b/tests/regression/vm_stress/kernel.cpp index c1df67de35..dc241710a1 100644 --- a/tests/regression/vm_stress/kernel.cpp +++ b/tests/regression/vm_stress/kernel.cpp @@ -14,8 +14,8 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { uint32_t bias = phys_ptr[task_id % arg->phys_words]; for (uint32_t k = 0; k < arg->pages_per_task; ++k) { - uint32_t page = ((task_id * arg->pages_per_task + k) * arg->stride_pages) % arg->total_pages; - uint32_t word = page * WORDS_PER_PAGE + (task_id % WORDS_PER_PAGE); + int page = ((task_id * arg->pages_per_task + k) * arg->stride_pages) % arg->total_pages; + int word = page * WORDS_PER_PAGE + (task_id % WORDS_PER_PAGE); dst_ptr[word] = src_ptr[word] + bias; } } From 577a4afea074ea35f9c02beb5dad9ff7abf787cf Mon Sep 17 00:00:00 2001 From: tinebp Date: Fri, 9 Oct 2026 18:57:06 -0700 Subject: [PATCH 003/124] rtu: carry full-width scene and payload pointers on RV64 On RV64 the trace lost the upper half of both warp-uniform pointers. SimX narrowed the scene base to 32 bits, so a scene above 4 GiB was read from the wrong address and every ray missed. SimX and the RTL both carried the payload pointer as 32 bits and wrote it back through the single VX_RT_PAYLOAD_PTR_LO slot, so a callback shader got a truncated payload pointer. vx_rt_wtrace took uint32_t pointers and narrowed the gathered config, and every tests/raytracing kernel masked its device addresses to 32 bits, which kept all of it hidden: device buffers normally sit below 4 GiB. - The high word goes in slot VX_RT_PAYLOAD_PTR_LO + 1, which already sat unused next to the low word. It is not a new VX_types define: the perf-gate fingerprint hashes VX_types, so a define would mark every perf baseline stale without moving a cycle. - SimX keeps the scene base at the device address width and the payload at full width, and writes the high word on RV64. - RTL widens payload_ptr to XLEN across the RTU bus, the arm staging RAM and the slot descriptor. On RV64 the record write-back sends one more word (the high half) before the hit attribute; STATUS stays last. At XLEN=32 the logic is unchanged. - vx_raytrace.h takes uintptr_t pointers, keeps the gathered config at register width, and adds vx_rt_get_payload(), which reads both words on RV64. - The tests/raytracing kernels pass full-width pointers. On a 64-bit device rt_smoke_chs places its scene and payload above 4 GiB; it fails before this change (miss, payload never written) and passes after. - ci/testcases/raytracing.yaml adds rt_suite-x64, the suite on RV64 simx. - hw/unittest gfx and rtu_core tops follow the wider arm word. RV32: all 34 ray-tracing kernels compile to identical code; the three rt_raycast perf_gate cases reproduce their baseline cycles and instruction counts exactly, the rtlsim suite and model_parity pass, and every perf_gate config hash is unchanged. RV64: the simx suite passes 34/34, and rtlsim passes 31/32 plus rt_smoke_chs above 4 GiB. rt_raycast fails pixel comparison on RV64 rtlsim identically before and after this change (210/768 pixels); that is a separate, pre-existing RTL defect, so rt_suite-x64 stays simx-only until it is fixed. The extra RV64 write-back word moves RV64 rtlsim rt_raycast cycles by 0% to +5.5% depending on shape (NT4 +5.1%, NT16 0.0%, 2-core +0.7%, 8x8 +5.5%) against two added cycles per record; RV64 rtlsim has no perf gate. Co-Authored-By: Claude Opus 5.5 --- VX_types.toml | 1 + ci/testcases/raytracing.yaml | 17 ++++++++-- docs/designs/ray_tracing_architecture.md | 2 +- hw/rtl/rtu/VX_rtu_bus_arb.sv | 2 +- hw/rtl/rtu/VX_rtu_bus_if.sv | 2 +- hw/rtl/rtu/VX_rtu_bus_slice.sv | 2 +- hw/rtl/rtu/VX_rtu_core.sv | 34 ++++++++++--------- hw/rtl/rtu/VX_rtu_pkg.sv | 19 ++++++----- hw/rtl/rtu/VX_rtu_unit.sv | 2 +- hw/unittest/gfx/VX_gfx_top.sv | 2 +- hw/unittest/rtu_core/VX_rtu_core_top.sv | 2 +- sim/simx/rtu/rtu_core.cpp | 6 ++-- sim/simx/rtu/rtu_types.h | 2 +- sim/simx/rtu/rtu_unit.cpp | 22 +++++++++--- sim/simx/rtu/rtu_unit.h | 7 ++-- sw/kernel/include/vx_raytrace.h | 19 ++++++++--- tests/raytracing/rt_bvh_multinode/kernel.cpp | 4 +-- tests/raytracing/rt_raycast/kernel.cpp | 4 +-- tests/raytracing/rt_smoke/kernel.cpp | 4 +-- tests/raytracing/rt_smoke_ahs/kernel.cpp | 4 +-- tests/raytracing/rt_smoke_ahs_bvh/kernel.cpp | 4 +-- .../raytracing/rt_smoke_ahs_custom/kernel.cpp | 12 +++---- tests/raytracing/rt_smoke_ahs_geom/kernel.cpp | 12 +++---- .../raytracing/rt_smoke_ahs_mixed/kernel.cpp | 4 +-- .../rt_smoke_async_batch/kernel.cpp | 8 ++--- tests/raytracing/rt_smoke_bvh6/kernel.cpp | 4 +-- .../raytracing/rt_smoke_bvh_basic/kernel.cpp | 4 +-- .../rt_smoke_bvh_instanced/kernel.cpp | 4 +-- .../rt_smoke_bvh_multilevel/kernel.cpp | 4 +-- tests/raytracing/rt_smoke_chs/common.h | 2 +- tests/raytracing/rt_smoke_chs/kernel.cpp | 10 +++--- tests/raytracing/rt_smoke_chs/main.cpp | 26 +++++++++++--- .../raytracing/rt_smoke_cull_back/kernel.cpp | 4 +-- .../raytracing/rt_smoke_cull_mask/kernel.cpp | 4 +-- .../raytracing/rt_smoke_deep_stack/kernel.cpp | 4 +-- .../raytracing/rt_smoke_deep_tlas/kernel.cpp | 4 +-- tests/raytracing/rt_smoke_fat_leaf/kernel.cpp | 4 +-- tests/raytracing/rt_smoke_host_cfg/kernel.cpp | 4 +-- .../raytracing/rt_smoke_inst_flags/kernel.cpp | 4 +-- tests/raytracing/rt_smoke_is/kernel.cpp | 10 +++--- tests/raytracing/rt_smoke_miss/kernel.cpp | 10 +++--- tests/raytracing/rt_smoke_multitri/kernel.cpp | 4 +-- tests/raytracing/rt_smoke_proc/kernel.cpp | 4 +-- .../raytracing/rt_smoke_recursive/kernel.cpp | 10 +++--- tests/raytracing/rt_smoke_reform/kernel.cpp | 4 +-- .../raytracing/rt_smoke_reform_mw/kernel.cpp | 4 +-- .../raytracing/rt_smoke_reform_sbt/kernel.cpp | 2 +- tests/raytracing/rt_smoke_sbt/kernel.cpp | 10 +++--- tests/raytracing/rt_smoke_shadow/kernel.cpp | 4 +-- tests/raytracing/rt_smoke_tlas/kernel.cpp | 4 +-- .../rt_smoke_tlas_builder/kernel.cpp | 4 +-- 51 files changed, 204 insertions(+), 145 deletions(-) diff --git a/VX_types.toml b/VX_types.toml index 495afbf7cc..c00dd29521 100644 --- a/VX_types.toml +++ b/VX_types.toml @@ -442,6 +442,7 @@ VX_RT_STATUS = 26 # per-lane VX_RT_STS_*; WAIT completes on # Every slot is written by the RTU and read by the shader; nothing else writes the # window. The payload pointer rides the arm doorbell (warp-uniform) and the # hitAttribute rides the CONTINUE, so both reach the RTU without a window write. +# On RV64 the payload pointer's upper word follows in slot VX_RT_PAYLOAD_PTR_LO + 1. VX_RT_PAYLOAD_PTR_LO = 27 # the tracing shader's payload; the callbacks read it VX_RT_HIT_ATTR_0 = 29 # user hitAttributeEXT (an IS shader returns it) diff --git a/ci/testcases/raytracing.yaml b/ci/testcases/raytracing.yaml index 6ccbc74045..f4b42836d4 100644 --- a/ci/testcases/raytracing.yaml +++ b/ci/testcases/raytracing.yaml @@ -1,8 +1,7 @@ # Test cases for `raytracing` (RTU), ported from regression.sh.in raytracing(). # -# simx is the only backend that validates the RTU end-to-end today; rtlsim / -# opae / xrt RTU coverage lands when the RTL side comes online. xlen is pinned to -# 32 (the RTU SimX-parity reference build); 64-bit RTU coverage is a follow-on. +# xlen defaults to 32 (the RTU SimX-parity reference build); rt_suite-x64 runs +# the suite on RV64. category: raytracing defaults: xlen: @@ -30,6 +29,18 @@ tests: target: run-{driver} configs: -DVX_CFG_EXT_RTU_ENABLE +# The same suite on RV64. There rt_smoke_chs puts its scene and payload above +# 4 GiB, so a trace that narrows either pointer misses or loses its payload. +- id: rt_suite-x64 + via: make-run + xlen: + - 64 + drivers: + - simx + dir: tests/raytracing + target: run-{driver} + configs: -DVX_CFG_EXT_RTU_ENABLE + # Multi-core RTU coverage: same canary kernel (rt_smoke) under two 4-core # topologies so a topology-only break is a single failure. -n64 = full device # capacity (cores x warps x threads); 64 rays in flight through the cluster's diff --git a/docs/designs/ray_tracing_architecture.md b/docs/designs/ray_tracing_architecture.md index e1590aae68..efd89a0fbb 100644 --- a/docs/designs/ray_tracing_architecture.md +++ b/docs/designs/ray_tracing_architecture.md @@ -176,7 +176,7 @@ state as base+count bursts, so each span's members must abut. | 17..22 | object-space ray `origin`/`dir` (staged on an AHS/IS yield) | RTU writes | | 23 / 24 / 25 | candidate `cb_type` / `sbt_idx` / `cb_handle` | RTU writes | | 26 | **STATUS** (`VX_RT_STS_*`) | RTU writes **last** | -| 27 / 29 | payload pointer / user `hitAttribute` | RTU writes (payload at arm; attr on commit) | +| 27 (+28 on RV64) / 29 | payload pointer (low / high word) / user `hitAttribute` | RTU writes (payload with every record; attr on commit) | **STATUS is deliberately last.** A parked WAIT completes on the address-match write to slot 26, so every other result word must already be in the window when diff --git a/hw/rtl/rtu/VX_rtu_bus_arb.sv b/hw/rtl/rtu/VX_rtu_bus_arb.sv index f619044b3c..ec01860259 100644 --- a/hw/rtl/rtu/VX_rtu_bus_arb.sv +++ b/hw/rtl/rtu/VX_rtu_bus_arb.sv @@ -68,7 +68,7 @@ module VX_rtu_bus_arb import VX_gpu_pkg::*, VX_rtu_pkg::*; #( localparam SRC_IN_W = 1; localparam ARM_DATAW = NW_WIDTH + NUM_LANES - + `VX_CFG_MEM_ADDR_WIDTH + 16 + 16 + 32 + TAG_WIDTH; + + `VX_CFG_MEM_ADDR_WIDTH + 16 + 16 + `VX_CFG_XLEN + TAG_WIDTH; localparam REQ_DATAW = 1 + NW_WIDTH + NUM_LANES * 32 + NUM_LANES * RTU_CB_ACTION_BITS; // the win payload carries the sel-stripped tag on the core side localparam WIN_DATAW = 1 + NW_WIDTH + RTU_SLOT_BITS diff --git a/hw/rtl/rtu/VX_rtu_bus_if.sv b/hw/rtl/rtu/VX_rtu_bus_if.sv index 23b017e826..2f259fa299 100644 --- a/hw/rtl/rtu/VX_rtu_bus_if.sv +++ b/hw/rtl/rtu/VX_rtu_bus_if.sv @@ -69,7 +69,7 @@ interface VX_rtu_bus_if import VX_gpu_pkg::*, VX_rtu_pkg::*; #( logic [`VX_CFG_MEM_ADDR_WIDTH-1:0] scene_base; // device address of the scene logic [15:0] flags; // VX_RT_FLAG_* logic [15:0] cull_mask; - logic [31:0] payload_ptr; // the RTU stages it for the callbacks + logic [`VX_CFG_XLEN-1:0] payload_ptr; // the RTU stages it for the callbacks logic [TAG_WIDTH-1:0] tag; } arm_data_t; diff --git a/hw/rtl/rtu/VX_rtu_bus_slice.sv b/hw/rtl/rtu/VX_rtu_bus_slice.sv index d2601157a0..a85e27711c 100644 --- a/hw/rtl/rtu/VX_rtu_bus_slice.sv +++ b/hw/rtl/rtu/VX_rtu_bus_slice.sv @@ -54,7 +54,7 @@ module VX_rtu_bus_slice import VX_gpu_pkg::*, VX_rtu_pkg::*; #( // parameter cannot read $bits of a hierarchical interface member, so size // the elastic buffers from the field formula instead. localparam ARM_DATAW = SRC_WIDTH + NW_WIDTH + NUM_LANES - + `VX_CFG_MEM_ADDR_WIDTH + 16 + 16 + 32 + TAG_WIDTH; + + `VX_CFG_MEM_ADDR_WIDTH + 16 + 16 + `VX_CFG_XLEN + TAG_WIDTH; localparam REQ_DATAW = 1 + SRC_WIDTH + NW_WIDTH + NUM_LANES * 32 + NUM_LANES * RTU_CB_ACTION_BITS; localparam WIN_DATAW = 1 + NW_WIDTH + RTU_SLOT_BITS diff --git a/hw/rtl/rtu/VX_rtu_core.sv b/hw/rtl/rtu/VX_rtu_core.sv index f7bd8af8ea..425c167f7a 100644 --- a/hw/rtl/rtu/VX_rtu_core.sv +++ b/hw/rtl/rtu/VX_rtu_core.sv @@ -147,11 +147,11 @@ module VX_rtu_core import VX_gpu_pkg::*, VX_rtu_pkg::*; #( reg [NUM_SLOTS-1:0][2:0] tstate; reg [NUM_SLOTS-1:0][NUM_LANES-1:0] req_mask; - reg [NUM_SLOTS-1:0][TAG_WIDTH-1:0] req_tag; - reg [NUM_SLOTS-1:0][STG_IDX_W-1:0] req_stg; // the staging entry it launched from - reg [NUM_SLOTS-1:0][NW_WIDTH-1:0] req_wid; // its warp, for the window write - reg [NUM_SLOTS-1:0][31:0] req_payload; // warp-uniform; staged for the callbacks - reg [NUM_SLOTS-1:0] is_cand; + reg [NUM_SLOTS-1:0][TAG_WIDTH-1:0] req_tag; + reg [NUM_SLOTS-1:0][STG_IDX_W-1:0] req_stg; // the staging entry it launched from + reg [NUM_SLOTS-1:0][NW_WIDTH-1:0] req_wid; // its warp, for the window write + reg [NUM_SLOTS-1:0][`VX_CFG_XLEN-1:0] req_payload; // warp-uniform; staged for the callbacks + reg [NUM_SLOTS-1:0] is_cand; reg [NUM_SLOTS-1:0][NUM_LANES-1:0][RTU_CB_ACTION_BITS-1:0] cont_action; // ── the scheduler ───────────────────────────────────────────────── @@ -306,7 +306,7 @@ module VX_rtu_core import VX_gpu_pkg::*, VX_rtu_pkg::*; #( ); // warp-uniform scalars of a staged trace (read once, at the claim) - localparam SCLW = NUM_LANES + TAG_WIDTH + 32 + `VX_CFG_MEM_ADDR_WIDTH + 16 + 16; + localparam SCLW = NUM_LANES + TAG_WIDTH + `VX_CFG_XLEN + `VX_CFG_MEM_ADDR_WIDTH + 16 + 16; wire [SCLW-1:0] stg_scl_rdata; wire [SLOT_W-1:0] fill_slot_pick; wire [STG_IDX_W-1:0] fill_stg_pick; @@ -332,11 +332,11 @@ module VX_rtu_core import VX_gpu_pkg::*, VX_rtu_pkg::*; #( .raddr (fill_stg_pick), .rdata (stg_scl_rdata) ); - wire [NUM_LANES-1:0] scl_mask; - wire [TAG_WIDTH-1:0] scl_tag; - wire [31:0] scl_payload; + wire [NUM_LANES-1:0] scl_mask; + wire [TAG_WIDTH-1:0] scl_tag; + wire [`VX_CFG_XLEN-1:0] scl_payload; wire [`VX_CFG_MEM_ADDR_WIDTH-1:0] scl_scene; - wire [15:0] scl_flags, scl_cull; + wire [15:0] scl_flags, scl_cull; assign {scl_mask, scl_tag, scl_payload, scl_scene, scl_flags, scl_cull} = stg_scl_rdata; // ── the fill engine ─────────────────────────────────────────────── @@ -481,7 +481,7 @@ module VX_rtu_core import VX_gpu_pkg::*, VX_rtu_pkg::*; #( // 7..12 object ray (candidate only) — the staging RAM's beat rows // 13,14 cb_type (flag flops), sbt (window row) // 15 cb_handle (0) - // then payload (descriptor), hitAttribute (window row), status LAST + // then payload (descriptor, XLEN/32 words), hitAttribute (window row), status LAST localparam [2:0] WB_IDLE = 3'd0, WB_RD1 = 3'd1, WB_CAP1 = 3'd2, @@ -495,9 +495,10 @@ module VX_rtu_core import VX_gpu_pkg::*, VX_rtu_pkg::*; #( wire [RTU_IDX_BITS-1:0] n_attrs = is_cand[ws] ? RTU_IDX_BITS'(RTU_RES_CAND) : RTU_IDX_BITS'(RTU_RES_HIT); - wire wr_payload = (wr_idx == n_attrs); - wire wr_attr = (wr_idx == (n_attrs + RTU_IDX_BITS'(1))); - wire wr_status = (wr_idx == (n_attrs + RTU_IDX_BITS'(2))); + wire wr_pl_hi = (RTU_PAYLOAD_WORDS > 1) && (wr_idx == (n_attrs + RTU_IDX_BITS'(1))); + wire wr_payload = (wr_idx == n_attrs) || wr_pl_hi; + wire wr_attr = (wr_idx == (n_attrs + RTU_IDX_BITS'(RTU_PAYLOAD_WORDS))); + wire wr_status = (wr_idx == (n_attrs + RTU_IDX_BITS'(RTU_PAYLOAD_WORDS + 1))); wire wr_hitspan = (wr_idx < RTU_IDX_BITS'(RTU_RES_HIT)); wire wr_objray = is_cand[ws] && !wr_hitspan && (wr_idx < RTU_IDX_BITS'(13)); wire wr_cbtype = is_cand[ws] && (wr_idx == RTU_IDX_BITS'(13)); @@ -553,7 +554,8 @@ module VX_rtu_core import VX_gpu_pkg::*, VX_rtu_pkg::*; #( if (wr_status) begin win_word[i] = status_word[i]; end else if (wr_payload) begin - win_word[i] = req_payload[ws]; + win_word[i] = wr_pl_hi ? req_payload[ws][`VX_CFG_XLEN-1 -: 32] + : req_payload[ws][31:0]; end else if (wr_attr) begin win_word[i] = sch_attrv[wctx[i]] ? wb_d1[i*32 +: 32] : 32'd0; end else if (wr_hitspan) begin @@ -591,7 +593,7 @@ module VX_rtu_core import VX_gpu_pkg::*, VX_rtu_pkg::*; #( assign rtu_bus_w.win_data.mask = (is_cand[ws] && !wr_uniform) ? wr_cb_mask : req_mask[ws]; assign rtu_bus_w.win_data.slot = wr_status ? RTU_SLOT_BITS'(RTU_STATUS_SLOT) - : wr_payload ? RTU_SLOT_BITS'(RTU_PAYLOAD_SLOT) + : wr_payload ? RTU_SLOT_BITS'(RTU_PAYLOAD_SLOT) + RTU_SLOT_BITS'(wr_pl_hi) : wr_attr ? RTU_SLOT_BITS'(RTU_ATTR_SLOT) : RTU_SLOT_BITS'(RTU_RES_BASE) + RTU_SLOT_BITS'(wr_idx); diff --git a/hw/rtl/rtu/VX_rtu_pkg.sv b/hw/rtl/rtu/VX_rtu_pkg.sv index d21d89be69..0eb4a8a5a9 100644 --- a/hw/rtl/rtu/VX_rtu_pkg.sv +++ b/hw/rtl/rtu/VX_rtu_pkg.sv @@ -264,21 +264,22 @@ package VX_rtu_pkg; // // hit attrs [RES_BASE, +RES_HIT) written by every response // candidate [RES_BASE, +RES_CAND) + object ray, cb_type, sbt, handle - // payload PAYLOAD_SLOT written once, at arm (warp-uniform) + // payload PAYLOAD_SLOT(+1) XLEN/32 words, written per record (warp-uniform) // hit attr ATTR_SLOT the IS shader's hitAttribute, on commit // status STATUS_SLOT written LAST // // Status is last by design: writing it is what completes the warp's parked // WAIT, so every slot the warp may then read must already be in place. // ───────────────────────────────────────────────────────────────── - localparam RTU_RAY_BEATS = 8; // origin[3], dir[3], t_min, t_max - localparam RTU_RES_BASE = `VX_RT_HIT_T; - localparam RTU_RES_HIT = 7; // hit_t, hit_u, hit_v, prim, instance, geometry, custom - localparam RTU_RES_CAND = 16; // + object ray (6), cb_type, cb_sbt_idx, cb_handle - localparam RTU_STATUS_SLOT = `VX_RT_STATUS; - localparam RTU_PAYLOAD_SLOT = `VX_RT_PAYLOAD_PTR_LO; - localparam RTU_ATTR_SLOT = `VX_RT_HIT_ATTR_0; - localparam RTU_IDX_BITS = `CLOG2(RTU_RES_CAND + 1); + localparam RTU_RAY_BEATS = 8; // origin[3], dir[3], t_min, t_max + localparam RTU_RES_BASE = `VX_RT_HIT_T; + localparam RTU_RES_HIT = 7; // hit_t, hit_u, hit_v, prim, instance, geometry, custom + localparam RTU_RES_CAND = 16; // + object ray (6), cb_type, cb_sbt_idx, cb_handle + localparam RTU_STATUS_SLOT = `VX_RT_STATUS; + localparam RTU_PAYLOAD_SLOT = `VX_RT_PAYLOAD_PTR_LO; + localparam RTU_PAYLOAD_WORDS = `VX_CFG_XLEN / 32; + localparam RTU_ATTR_SLOT = `VX_RT_HIT_ATTR_0; + localparam RTU_IDX_BITS = `CLOG2(RTU_RES_CAND + 1); // ───────────────────────────────────────────────────────────────── // Window store — the per-slot result record, one NUM_LANES*32 row per word, diff --git a/hw/rtl/rtu/VX_rtu_unit.sv b/hw/rtl/rtu/VX_rtu_unit.sv index 07bc926c74..50781fcaf0 100644 --- a/hw/rtl/rtu/VX_rtu_unit.sv +++ b/hw/rtl/rtu/VX_rtu_unit.sv @@ -204,7 +204,7 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( assign rtu_bus_w.arm_data.wid = wid; assign rtu_bus_w.arm_data.mask = execute_if.data.header.tmask; assign rtu_bus_w.arm_data.scene_base = execute_if.data.rs1_data[CFG_L1][`VX_CFG_MEM_ADDR_WIDTH-1:0]; - assign rtu_bus_w.arm_data.payload_ptr = execute_if.data.rs1_data[CFG_L2][31:0]; + assign rtu_bus_w.arm_data.payload_ptr = execute_if.data.rs1_data[CFG_L2]; assign rtu_bus_w.arm_data.flags = execute_if.data.rs1_data[CFG_L3][15:0]; assign rtu_bus_w.arm_data.cull_mask = execute_if.data.rs1_data[CFG_L3][31:16]; assign rtu_bus_w.arm_data.tag = ($bits(rtu_bus_w.arm_data.tag))'(execute_if.data.header.uuid); diff --git a/hw/unittest/gfx/VX_gfx_top.sv b/hw/unittest/gfx/VX_gfx_top.sv index 53386bfd77..51aed0b6d5 100644 --- a/hw/unittest/gfx/VX_gfx_top.sv +++ b/hw/unittest/gfx/VX_gfx_top.sv @@ -99,7 +99,7 @@ module VX_gfx_top import VX_gpu_pkg::* input wire [NC-1:0][`VX_CFG_MEM_ADDR_WIDTH-1:0] rtu_arm_scene_base, input wire [NC-1:0][15:0] rtu_arm_flags, input wire [NC-1:0][15:0] rtu_arm_cull_mask, - input wire [NC-1:0][31:0] rtu_arm_payload_ptr, + input wire [NC-1:0][`VX_CFG_XLEN-1:0] rtu_arm_payload_ptr, input wire [NC-1:0][RTU_REQ_TAG_WIDTH-1:0] rtu_arm_tag, output wire [NC-1:0] rtu_arm_ready, diff --git a/hw/unittest/rtu_core/VX_rtu_core_top.sv b/hw/unittest/rtu_core/VX_rtu_core_top.sv index 33452de81b..f4405346dd 100644 --- a/hw/unittest/rtu_core/VX_rtu_core_top.sv +++ b/hw/unittest/rtu_core/VX_rtu_core_top.sv @@ -31,7 +31,7 @@ module VX_rtu_core_top import VX_gpu_pkg::*, VX_rtu_pkg::*; #( // Channel widths (mirror VX_rtu_bus_slice / VX_rtu_bus_if). parameter SRC_WIDTH = `UP(`CLOG2(NUM_SRCS)), parameter ARM_DATAW = SRC_WIDTH + NW_WIDTH + NUM_LANES - + `VX_CFG_MEM_ADDR_WIDTH + 16 + 16 + 32 + TAG_WIDTH, + + `VX_CFG_MEM_ADDR_WIDTH + 16 + 16 + `VX_CFG_XLEN + TAG_WIDTH, parameter REQ_DATAW = 1 + SRC_WIDTH + NW_WIDTH + NUM_LANES * 32 + NUM_LANES * RTU_CB_ACTION_BITS, parameter WIN_DATAW = 1 + NW_WIDTH + RTU_SLOT_BITS diff --git a/sim/simx/rtu/rtu_core.cpp b/sim/simx/rtu/rtu_core.cpp index 75492a860c..0e2db91fba 100644 --- a/sim/simx/rtu/rtu_core.cpp +++ b/sim/simx/rtu/rtu_core.cpp @@ -463,14 +463,14 @@ class RtuCore::Impl { } void bind_context(uint32_t ci, uint32_t slot, uint32_t lane, - uint32_t scene_root) { + uint64_t scene_root) { Context& cx = contexts_[ci]; cx = Context{}; cx.valid = true; cx.slot = slot; cx.lane = lane; - cx.base_line = uint64_t(scene_root) & kRtuLineMask; - cx.byte_off = uint32_t(uint64_t(scene_root) - cx.base_line); + cx.base_line = scene_root & kRtuLineMask; + cx.byte_off = uint32_t(scene_root - cx.base_line); // The ray opens on the reciprocal (1/dir) setup and the scene-header fetch, // which cost front-end states like any other step, then drain the reciprocal // pipeline before the first node can be tested. diff --git a/sim/simx/rtu/rtu_types.h b/sim/simx/rtu/rtu_types.h index 8d8858bce2..6e16c3f627 100644 --- a/sim/simx/rtu/rtu_types.h +++ b/sim/simx/rtu/rtu_types.h @@ -66,7 +66,7 @@ struct RtuReq { uint32_t slot_idx = 0; // Per-lane ray descriptor snapshot (TRACE_NEW only). - std::array scene_root = {}; + std::array scene_root = {}; std::array origin_x = {}; std::array origin_y = {}; std::array origin_z = {}; diff --git a/sim/simx/rtu/rtu_unit.cpp b/sim/simx/rtu/rtu_unit.cpp index 5b6a93ad8a..2d8cef624f 100644 --- a/sim/simx/rtu/rtu_unit.cpp +++ b/sim/simx/rtu/rtu_unit.cpp @@ -75,6 +75,10 @@ namespace { inline uint32_t fp_src_bits(const instr_trace_t* trace, uint32_t src, uint32_t t) { return static_cast(trace->src_data[src].at(t).u64 & 0xffffffffu); } + +// The scene base rides the arm doorbell as a device address, MEM_ADDR_WIDTH wide. +constexpr uint64_t kSceneAddrMask = (VX_CFG_MEM_ADDR_WIDTH >= 64) + ? ~uint64_t(0) : ((uint64_t(1) << VX_CFG_MEM_ADDR_WIDTH) - 1); } // namespace instr_trace_t* RtuUnit::process_wait(instr_trace_t* trace, uint32_t block_id) { @@ -260,7 +264,7 @@ void RtuUnit::apply_response(const RtuRsp& rsp) { auto& lregs = wregs.at(t); // Written by the RTU, not by any instruction: the payload pointer rode the // arm and the hitAttribute rode the CONTINUE that accepted this hit. - lregs[VX_RT_PAYLOAD_PTR_LO] = trace_payload_.at(rsp.warp_id); + stage_payload_ptr(lregs, rsp.warp_id); lregs[VX_RT_HIT_ATTR_0] = rsp.hit_attr[t]; lregs[VX_RT_HIT_T] = float_to_bits(rsp.hit_t[t]); lregs[VX_RT_HIT_BARY_U] = float_to_bits(rsp.hit_bary_u[t]); @@ -280,6 +284,14 @@ void RtuUnit::apply_response(const RtuRsp& rsp) { } } +void RtuUnit::stage_payload_ptr(RtuWindow::LaneRegs& lregs, uint32_t wid) const { + uint64_t payload = trace_payload_.at(wid); + lregs[VX_RT_PAYLOAD_PTR_LO] = uint32_t(payload); +#ifdef VX_CFG_XLEN_64 + lregs[VX_RT_PAYLOAD_PTR_LO + 1] = uint32_t(payload >> 32); +#endif +} + void RtuUnit::apply_callback_payload(const RtuRsp& rsp) { // Stage candidate-hit attrs + cb_type + cb_handle into the RTU regs // for the lanes whose rays yielded, so the dispatcher's vx_rt_get @@ -295,7 +307,7 @@ void RtuUnit::apply_callback_payload(const RtuRsp& rsp) { // The callback shaders read the tracing shader's payload pointer, and they // are a different shader — it cannot reach them any other way. A closest-hit // shader likewise reads the accepted hit's attribute. - lregs[VX_RT_PAYLOAD_PTR_LO] = trace_payload_.at(rsp.warp_id); + stage_payload_ptr(lregs, rsp.warp_id); lregs[VX_RT_HIT_ATTR_0] = rsp.hit_attr[t]; lregs[VX_RT_HIT_T] = float_to_bits(rsp.hit_t[t]); lregs[VX_RT_HIT_BARY_U] = float_to_bits(rsp.hit_bary_u[t]); @@ -437,7 +449,7 @@ instr_trace_t* RtuUnit::process_trace_uop(instr_trace_t* trace, uint32_t block_i case 0: { // GP config uop: the pool slot was claimed at issue (trace2_reserve_slot), // so this uop has no backpressure source. Unpack the lane-packed config - // (lane0=scene, lane1=payload, lane2=flags, lane3=cull — the implicit + // (lane1=scene, lane2=payload, lane3=flags|cull — the implicit // vx_wgather layout) and stage it. int32_t slot = trace_slot_.at(wid); assert(slot >= 0 && "TRACE uop0 issued without a reserved pool slot"); @@ -448,12 +460,12 @@ instr_trace_t* RtuUnit::process_trace_uop(instr_trace_t* trace, uint32_t block_i uint32_t flagscull = static_cast(cfg.at(3).u); // Warp-uniform: it rides the arm doorbell, so it is staged here, not written // into the window. The RTU writes the payload pointer back with the record. - trace_payload_.at(wid) = static_cast(cfg.at(2).u); + trace_payload_.at(wid) = cfg.at(2).u; trace_flags_.at(wid) = flagscull & 0xffffu; trace_cull_.at(wid) = flagscull >> 16; for (uint32_t t = 0; t < VX_CFG_NUM_THREADS; ++t) { if (!trace->tmask.test(t)) continue; - trace_scene_.at(wid)[t] = static_cast(cfg.at(1).u); + trace_scene_.at(wid)[t] = uint64_t(cfg.at(1).u) & kSceneAddrMask; trace->dst_data[t].u = uint32_t(slot); // handle returns early } return trace; diff --git a/sim/simx/rtu/rtu_unit.h b/sim/simx/rtu/rtu_unit.h index ab807f324c..978e7ea511 100644 --- a/sim/simx/rtu/rtu_unit.h +++ b/sim/simx/rtu/rtu_unit.h @@ -156,6 +156,9 @@ class RtuUnit { // code reads the right payload before issuing CONTINUE. void apply_callback_payload(const RtuRsp& rsp); + // The payload pointer is XLEN wide; the window holds it as 32-bit words. + void stage_payload_ptr(RtuWindow::LaneRegs& lregs, uint32_t wid) const; + // Representative slot of a candidate rsp (first active lane's cb_handle). static uint32_t candidate_slot(const RtuRsp& rsp); @@ -225,9 +228,9 @@ class RtuUnit { VX_CFG_NUM_WARPS> trace_ray_; // Warp-uniform half of the ray, staged by the config uop: it rides the arm // doorbell in HW, so it is not a window write either. - std::array, + std::array, VX_CFG_NUM_WARPS> trace_scene_; - std::array trace_payload_{}; + std::array trace_payload_{}; std::array trace_flags_{}; std::array trace_cull_{}; // The slot handle each lane's outstanding candidate came from, mirrored here diff --git a/sw/kernel/include/vx_raytrace.h b/sw/kernel/include/vx_raytrace.h index 0ff732b88c..baf3fe98da 100644 --- a/sw/kernel/include/vx_raytrace.h +++ b/sw/kernel/include/vx_raytrace.h @@ -50,6 +50,17 @@ extern "C" { : "memory"); \ __v; }) +// vx_rt_get_payload — the tracing shader's payload pointer, as the RTU staged it +// for this record (VX_RT_PAYLOAD_PTR_LO, plus the next slot on RV64). +static inline __attribute__((always_inline)) +uintptr_t vx_rt_get_payload(uint32_t tok) { + uintptr_t ptr = vx_rt_get_attr(VX_RT_PAYLOAD_PTR_LO, tok); +#if __riscv_xlen == 64 + ptr |= (uintptr_t)vx_rt_get_attr(VX_RT_PAYLOAD_PTR_LO + 1, tok) << 32; +#endif + return ptr; +} + // vx_rt_cb_ret — release the lane's parked context in the RTU with one of // VX_RT_CB_{ACCEPT,IGNORE,TERMINATE}, handing back the intersection shader's // computed hit distance and attribute in the same instruction. The callback @@ -141,13 +152,13 @@ _Static_assert(offsetof(vx_hit_t, instance_custom) == 6 * sizeof(float), "vx_hit // across the warp's lanes, and the f0..f7 ray window is per-lane. // scene_ptr / payload_ptr / ray_flags / cull_mask : per-trace warp-uniform // config, lane-packed into one register via an IMPLICIT vx_wgather -// (lane0=scene, lane1=payload, lane2=flags, lane3=cull) — pure register +// (lane1=scene, lane2=payload, lane3=flags|cull) — pure register // domain, no memory traffic; hoists out of a bounce loop when invariant. // ray : the per-thread geometry, pinned into the f0..f7 caller-saved window. // Returns the async ray handle in rd. Non-blocking: the RTU traverses // while the kernel runs independent work; vx_rt_wait is the sync point. static inline __attribute__((always_inline)) -uint32_t vx_rt_wtrace(uint32_t scene_ptr, uint32_t payload_ptr, +uint32_t vx_rt_wtrace(uintptr_t scene_ptr, uintptr_t payload_ptr, uint32_t ray_flags, uint32_t cull_mask, const vx_ray_t* ray) { // Pack config into the GATHERED operands (not the wgather self slot): the @@ -156,7 +167,7 @@ uint32_t vx_rt_wtrace(uint32_t scene_ptr, uint32_t payload_ptr, // lane3={flags,cull}; lane0 (self) unused. Keeps scene valid even when lane 0 // is masked (callback/recursion-narrowed traces). uint32_t flags_cull = (ray_flags & 0xffffu) | (cull_mask << 16); - uint32_t cfg = (uint32_t)vx_wgather(0u, scene_ptr, payload_ptr, flags_cull); + size_t cfg = vx_wgather(0u, scene_ptr, payload_ptr, flags_cull); register float r0 __asm__("f0") = ray->origin[0]; register float r1 __asm__("f1") = ray->origin[1]; register float r2 __asm__("f2") = ray->origin[2]; @@ -328,7 +339,7 @@ void vx_rt_get_objray(vx_objray_t* out) { // have independent work, it calls trace/wait separately and the compiler // schedules that work into the gap. static inline __attribute__((always_inline)) -uint32_t vx_rt_wtrace_sync(uint32_t scene_ptr, uint32_t payload_ptr, +uint32_t vx_rt_wtrace_sync(uintptr_t scene_ptr, uintptr_t payload_ptr, uint32_t ray_flags, uint32_t cull_mask, const vx_ray_t* ray, vx_hit_t* hit) { uint32_t h = vx_rt_wtrace(scene_ptr, payload_ptr, ray_flags, cull_mask, ray); diff --git a/tests/raytracing/rt_bvh_multinode/kernel.cpp b/tests/raytracing/rt_bvh_multinode/kernel.cpp index 0b31652e27..12eb9ff64a 100644 --- a/tests/raytracing/rt_bvh_multinode/kernel.cpp +++ b/tests/raytracing/rt_bvh_multinode/kernel.cpp @@ -29,8 +29,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { { r[3], r[4], r[5] }, r[6], r[7] }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); diff --git a/tests/raytracing/rt_raycast/kernel.cpp b/tests/raytracing/rt_raycast/kernel.cpp index 9b8013146c..0ffc6baf7d 100644 --- a/tests/raytracing/rt_raycast/kernel.cpp +++ b/tests/raytracing/rt_raycast/kernel.cpp @@ -57,8 +57,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { // Fire the ray on the RTU (v2 single-issue trace). vx_ray_t ray = { {ro.x, ro.y, ro.z}, {rd.x, rd.y, rd.z}, 0.001f, 1e30f }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); diff --git a/tests/raytracing/rt_smoke/kernel.cpp b/tests/raytracing/rt_smoke/kernel.cpp index 7271651877..dd265ad464 100644 --- a/tests/raytracing/rt_smoke/kernel.cpp +++ b/tests/raytracing/rt_smoke/kernel.cpp @@ -43,8 +43,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { // One trace + one wait. cull_mask = 0xff = no culling (the default). // payload = 0 (smoke test reads no payload). - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); diff --git a/tests/raytracing/rt_smoke_ahs/kernel.cpp b/tests/raytracing/rt_smoke_ahs/kernel.cpp index 0019a747e0..0acb3685fc 100644 --- a/tests/raytracing/rt_smoke_ahs/kernel.cpp +++ b/tests/raytracing/rt_smoke_ahs/kernel.cpp @@ -44,8 +44,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { // Fire ray + wait. Ray flags = 0 (no OPAQUE override; per-triangle flags // drive AHS). No payload. Any-hit candidates return to this warp; resume // with the decided action until a terminal status ends traversal. - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, 0u, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, 0u, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); while (vx_rt_sts_is_yield(sts)) { diff --git a/tests/raytracing/rt_smoke_ahs_bvh/kernel.cpp b/tests/raytracing/rt_smoke_ahs_bvh/kernel.cpp index 1953f25848..467439e216 100644 --- a/tests/raytracing/rt_smoke_ahs_bvh/kernel.cpp +++ b/tests/raytracing/rt_smoke_ahs_bvh/kernel.cpp @@ -43,8 +43,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { // Fire ray + wait for terminal. Ray flags = 0 (no OPAQUE override; // per-triangle flags drive AHS). No payload. - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, 0u, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, 0u, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); while (vx_rt_sts_is_yield(sts)) { diff --git a/tests/raytracing/rt_smoke_ahs_custom/kernel.cpp b/tests/raytracing/rt_smoke_ahs_custom/kernel.cpp index cfbe1c7e94..a476597782 100644 --- a/tests/raytracing/rt_smoke_ahs_custom/kernel.cpp +++ b/tests/raytracing/rt_smoke_ahs_custom/kernel.cpp @@ -33,20 +33,20 @@ __kernel void kernel_main(kernel_arg_t* arg) { // Ray flags = 0: the non-opaque triangle drives the opacity classifier so // the walker yields an AHS candidate callback. The candidate-capture buffer - // rides the trace as the payload pointer (VX_RT_PAYLOAD_PTR_LO), so the + // rides the trace as the payload pointer (vx_rt_get_payload), so the // loop can locate it from the window. - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t cand_lo = (uint32_t)(arg->cand_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, cand_lo, 0u, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uintptr_t capture_ptr = arg->cand_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, capture_ptr, 0u, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); while (vx_rt_sts_is_yield(sts)) { // Read the candidate instance attributes and the capture buffer pointer // (staged as the trace payload) from the register window, then stash. - uint32_t cand_ptr = vx_rt_get_attr(VX_RT_PAYLOAD_PTR_LO, sts); + uintptr_t cand_ptr = vx_rt_get_payload(sts); uint32_t cand_custom = vx_rt_get_attr(VX_RT_HIT_INSTANCE_CUSTOM, sts); uint32_t cand_inst = vx_rt_get_attr(VX_RT_HIT_INSTANCE_ID, sts); - uint32_t* cand = (uint32_t*)(uintptr_t)cand_ptr; + uint32_t* cand = (uint32_t*)cand_ptr; cand[0] = cand_custom; // cand->cand_instance_custom cand[1] = cand_inst; // cand->cand_instance_id sts = vx_rt_continue(h, VX_RT_CB_ACCEPT, hit.t, 0u, &hit); diff --git a/tests/raytracing/rt_smoke_ahs_geom/kernel.cpp b/tests/raytracing/rt_smoke_ahs_geom/kernel.cpp index f3bf38da34..9e7b7c7b43 100644 --- a/tests/raytracing/rt_smoke_ahs_geom/kernel.cpp +++ b/tests/raytracing/rt_smoke_ahs_geom/kernel.cpp @@ -33,20 +33,20 @@ __kernel void kernel_main(kernel_arg_t* arg) { // Ray flags = 0: the non-opaque triangle drives the opacity classifier so // the walker yields an AHS candidate callback. The candidate-capture buffer - // rides the trace as the payload pointer (VX_RT_PAYLOAD_PTR_LO), so the + // rides the trace as the payload pointer (vx_rt_get_payload), so the // loop can locate it from the window. - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t cand_lo = (uint32_t)(arg->cand_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, cand_lo, 0u, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uintptr_t capture_ptr = arg->cand_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, capture_ptr, 0u, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); while (vx_rt_sts_is_yield(sts)) { // Read the candidate geometry/instance attributes and the capture buffer // pointer (staged as the trace payload) from the register window, stash. - uint32_t cand_ptr = vx_rt_get_attr(VX_RT_PAYLOAD_PTR_LO, sts); + uintptr_t cand_ptr = vx_rt_get_payload(sts); uint32_t cand_geom = vx_rt_get_attr(VX_RT_HIT_GEOMETRY_INDEX, sts); uint32_t cand_inst = vx_rt_get_attr(VX_RT_HIT_INSTANCE_ID, sts); - uint32_t* cand = (uint32_t*)(uintptr_t)cand_ptr; + uint32_t* cand = (uint32_t*)cand_ptr; cand[0] = cand_geom; // cand->cand_geometry cand[1] = cand_inst; // cand->cand_instance_id sts = vx_rt_continue(h, VX_RT_CB_ACCEPT, hit.t, 0u, &hit); diff --git a/tests/raytracing/rt_smoke_ahs_mixed/kernel.cpp b/tests/raytracing/rt_smoke_ahs_mixed/kernel.cpp index a5525d56b0..33ca412327 100644 --- a/tests/raytracing/rt_smoke_ahs_mixed/kernel.cpp +++ b/tests/raytracing/rt_smoke_ahs_mixed/kernel.cpp @@ -37,8 +37,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { arg->tmax, }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, 0u, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, 0u, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); while (vx_rt_sts_is_yield(sts)) { diff --git a/tests/raytracing/rt_smoke_async_batch/kernel.cpp b/tests/raytracing/rt_smoke_async_batch/kernel.cpp index f7a0f9de1c..a9ce39b2db 100644 --- a/tests/raytracing/rt_smoke_async_batch/kernel.cpp +++ b/tests/raytracing/rt_smoke_async_batch/kernel.cpp @@ -41,10 +41,10 @@ __kernel void kernel_main(kernel_arg_t* arg) { // Issue RTU_ASYNC_NUM_BATCH traces back-to-back. No WAIT in // between — each ray runs async in the cluster's RtuCore. - uint32_t h0 = vx_rt_wtrace((uint32_t)(arg->scene_addr[0] & 0xffffffffu), 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); - uint32_t h1 = vx_rt_wtrace((uint32_t)(arg->scene_addr[1] & 0xffffffffu), 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); - uint32_t h2 = vx_rt_wtrace((uint32_t)(arg->scene_addr[2] & 0xffffffffu), 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); - uint32_t h3 = vx_rt_wtrace((uint32_t)(arg->scene_addr[3] & 0xffffffffu), 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uint32_t h0 = vx_rt_wtrace(arg->scene_addr[0], 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uint32_t h1 = vx_rt_wtrace(arg->scene_addr[1], 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uint32_t h2 = vx_rt_wtrace(arg->scene_addr[2], 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uint32_t h3 = vx_rt_wtrace(arg->scene_addr[3], 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); // Drain in declared order. Each WAIT blocks until its TERMINAL // delivers and applies the matching hit attrs. diff --git a/tests/raytracing/rt_smoke_bvh6/kernel.cpp b/tests/raytracing/rt_smoke_bvh6/kernel.cpp index 2b81b595bb..ef33e3ba4a 100644 --- a/tests/raytracing/rt_smoke_bvh6/kernel.cpp +++ b/tests/raytracing/rt_smoke_bvh6/kernel.cpp @@ -29,8 +29,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { {arg->ray_direction[0], arg->ray_direction[1], arg->ray_direction[2]}, arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); diff --git a/tests/raytracing/rt_smoke_bvh_basic/kernel.cpp b/tests/raytracing/rt_smoke_bvh_basic/kernel.cpp index 1fecb230e1..c6a731bb9b 100644 --- a/tests/raytracing/rt_smoke_bvh_basic/kernel.cpp +++ b/tests/raytracing/rt_smoke_bvh_basic/kernel.cpp @@ -26,8 +26,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { {arg->ray_direction[0], arg->ray_direction[1], arg->ray_direction[2]}, arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); diff --git a/tests/raytracing/rt_smoke_bvh_instanced/kernel.cpp b/tests/raytracing/rt_smoke_bvh_instanced/kernel.cpp index 5dc276e80a..4344855937 100644 --- a/tests/raytracing/rt_smoke_bvh_instanced/kernel.cpp +++ b/tests/raytracing/rt_smoke_bvh_instanced/kernel.cpp @@ -23,8 +23,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { {arg->ray_direction[0], arg->ray_direction[1], arg->ray_direction[2]}, arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); diff --git a/tests/raytracing/rt_smoke_bvh_multilevel/kernel.cpp b/tests/raytracing/rt_smoke_bvh_multilevel/kernel.cpp index 0154389871..00001a3025 100644 --- a/tests/raytracing/rt_smoke_bvh_multilevel/kernel.cpp +++ b/tests/raytracing/rt_smoke_bvh_multilevel/kernel.cpp @@ -23,8 +23,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { {arg->ray_direction[0], arg->ray_direction[1], arg->ray_direction[2]}, arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); diff --git a/tests/raytracing/rt_smoke_chs/common.h b/tests/raytracing/rt_smoke_chs/common.h index e6abfdea09..d501295419 100644 --- a/tests/raytracing/rt_smoke_chs/common.h +++ b/tests/raytracing/rt_smoke_chs/common.h @@ -47,7 +47,7 @@ typedef struct { typedef struct { uint64_t scene_addr; uint64_t results_addr; - uint64_t payload_addr; // CHS reads VX_RT_PAYLOAD_PTR_LO/HI + uint64_t payload_addr; // CHS reads it back with vx_rt_get_payload uint32_t reserved; uint32_t reserved2; float ray_origin[3]; diff --git a/tests/raytracing/rt_smoke_chs/kernel.cpp b/tests/raytracing/rt_smoke_chs/kernel.cpp index c555e3f6a6..bd164b4bc2 100644 --- a/tests/raytracing/rt_smoke_chs/kernel.cpp +++ b/tests/raytracing/rt_smoke_chs/kernel.cpp @@ -37,17 +37,17 @@ __kernel void kernel_main(kernel_arg_t* arg) { // The trace stages the payload pointer the CHS loop body reads via // vx_gfx_get. Opt into CHS dispatch. - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t payload = (uint32_t)(arg->payload_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, payload, VX_RT_FLAG_ENABLE_CHS, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uintptr_t payload = arg->payload_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, payload, VX_RT_FLAG_ENABLE_CHS, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); while (vx_rt_sts_is_yield(sts)) { // Read the committed hit_t bits and the payload pointer from the window, // write MAGIC ^ hit_t, then resume: DONE drains the CHS-shaded hit. uint32_t hit_t_bits = vx_rt_get_attr(VX_RT_HIT_T, sts); - uint32_t payload_ptr = vx_rt_get_attr(VX_RT_PAYLOAD_PTR_LO, sts); - *(uint32_t*)(uintptr_t)payload_ptr = RTU_CHS_MAGIC ^ hit_t_bits; + uintptr_t payload_ptr = vx_rt_get_payload(sts); + *(uint32_t*)payload_ptr = RTU_CHS_MAGIC ^ hit_t_bits; sts = vx_rt_continue(h, VX_RT_CB_DONE, hit.t, 0u, &hit); } diff --git a/tests/raytracing/rt_smoke_chs/main.cpp b/tests/raytracing/rt_smoke_chs/main.cpp index c5ee60896d..6e3b95c3ab 100644 --- a/tests/raytracing/rt_smoke_chs/main.cpp +++ b/tests/raytracing/rt_smoke_chs/main.cpp @@ -16,11 +16,13 @@ // Scene: one opaque triangle at z=5. Single lane fires a ray that // hits it. The kernel opts in to CHS via VX_RT_FLAG_ENABLE_CHS and // publishes a per-ray payload buffer pointer through -// VX_RT_PAYLOAD_PTR_LO. RtuCore commits the hit, then queues a +// VX_RT_PAYLOAD_PTR_LO/HI. RtuCore commits the hit, then queues a // CHS yield through the existing reformation/CB_YIELD path. The // dispatcher reads hit_t + payload pointer, writes MAGIC ^ hit_t_bits // to the payload, and exits via vx_rt_cb_ret(VX_RT_CB_DONE) + mret. // Oracle: status=HIT, payload[0] = MAGIC ^ bits(5.0f). +// On a 64-bit device the scene and the payload sit above 4 GiB, so the trace +// and the payload read-back must carry full-width pointers. #include #include @@ -52,6 +54,10 @@ vx_queue_h queue = nullptr; vx_module_h module_ = nullptr; vx_kernel_h kernel = nullptr; kernel_arg_t kernel_arg = {}; +bool high_mem = false; + +constexpr uint64_t HIGH_SCENE_ADDR = 0x140000000ull; +constexpr uint64_t HIGH_PAYLOAD_ADDR = 0x140010000ull; void cleanup() { if (device) { @@ -65,11 +71,23 @@ void cleanup() { } } +static vx_result_t create_buffer(uint64_t high_addr, uint64_t size, + uint32_t flags, vx_buffer_h* buf) { + if (high_mem) { + return vx_buffer_reserve(device, high_addr, size, flags, buf); + } + return vx_buffer_create(device, size, flags, buf); +} + int main(int /*argc*/, char* /*argv*/[]) { RT_CHECK(vx_device_open(0, &device)); vx_queue_info_t qi = { sizeof(qi), nullptr, VX_QUEUE_PRIORITY_NORMAL, 0 }; RT_CHECK(vx_queue_create(device, &qi, &queue)); + uint64_t isa_flags = 0; + RT_CHECK(vx_device_query(device, VX_CAPS_ISA_FLAGS, &isa_flags)); + high_mem = (VX_ISA_ARCH(isa_flags) == 64); + // Scene: 1 opaque triangle at z=5. std::vector scene_bytes(RTU_SCENE_HDR_BYTES + RTU_TRI_STRIDE_BYTES, 0); uint32_t* hdr = reinterpret_cast(scene_bytes.data()); @@ -83,7 +101,7 @@ int main(int /*argc*/, char* /*argv*/[]) { *tri_flags = RTU_TRI_FLAG_OPAQUE; uint32_t scene_bytes_sz = (uint32_t)scene_bytes.size(); - RT_CHECK(vx_buffer_create(device, scene_bytes_sz, VX_MEM_READ, &scene_buffer)); + RT_CHECK(create_buffer(HIGH_SCENE_ADDR, scene_bytes_sz, VX_MEM_READ, &scene_buffer)); RT_CHECK(vx_buffer_address(scene_buffer, &kernel_arg.scene_addr)); uint32_t res_size = sizeof(rtu_result_t); @@ -93,8 +111,8 @@ int main(int /*argc*/, char* /*argv*/[]) { // Payload buffer: 4 bytes. Pre-init with a sentinel so we can tell // "CHS never ran" from "CHS ran and wrote 0". uint32_t payload_init = 0xdeadbeef; - RT_CHECK(vx_buffer_create(device, sizeof(uint32_t), - VX_MEM_READ_WRITE, &payload_buffer)); + RT_CHECK(create_buffer(HIGH_PAYLOAD_ADDR, sizeof(uint32_t), + VX_MEM_READ_WRITE, &payload_buffer)); RT_CHECK(vx_buffer_address(payload_buffer, &kernel_arg.payload_addr)); RT_CHECK(vx_enqueue_write(queue, payload_buffer, 0, &payload_init, sizeof(payload_init), 0, nullptr, nullptr)); diff --git a/tests/raytracing/rt_smoke_cull_back/kernel.cpp b/tests/raytracing/rt_smoke_cull_back/kernel.cpp index 25d4c99be3..492bccdf0a 100644 --- a/tests/raytracing/rt_smoke_cull_back/kernel.cpp +++ b/tests/raytracing/rt_smoke_cull_back/kernel.cpp @@ -33,8 +33,8 @@ static inline uint32_t fire_ray(uint64_t scene_addr, ray.dir[2] = dir[2]; ray.tmin = tmin; ray.tmax = tmax; - uint32_t scene_lo = (uint32_t)(scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, + uintptr_t scene_ptr = scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE | VX_RT_FLAG_CULL_BACK_FACING, 0xffu, &ray); return vx_rt_wait(h, hit); diff --git a/tests/raytracing/rt_smoke_cull_mask/kernel.cpp b/tests/raytracing/rt_smoke_cull_mask/kernel.cpp index 4e8ce8137b..38723d6ad5 100644 --- a/tests/raytracing/rt_smoke_cull_mask/kernel.cpp +++ b/tests/raytracing/rt_smoke_cull_mask/kernel.cpp @@ -36,12 +36,12 @@ __kernel void kernel_main(kernel_arg_t* arg) { ray.tmin = arg->tmin; ray.tmax = arg->tmax; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); + uintptr_t scene_ptr = arg->scene_addr; rtu_result_t* results = (rtu_result_t*)((uintptr_t)arg->results_addr); // Each ray: fire+wait with a distinct cull mask, read results. for (uint32_t i = 0; i < RTU_NUM_RAYS; ++i) { - uint32_t h = vx_rt_wtrace(scene_lo, 0u, VX_RT_FLAG_OPAQUE, arg->ray_cull_mask[i], &ray); + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE, arg->ray_cull_mask[i], &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); results[tid].rays[i].status = sts; diff --git a/tests/raytracing/rt_smoke_deep_stack/kernel.cpp b/tests/raytracing/rt_smoke_deep_stack/kernel.cpp index 3f7da83385..f83047fcd8 100644 --- a/tests/raytracing/rt_smoke_deep_stack/kernel.cpp +++ b/tests/raytracing/rt_smoke_deep_stack/kernel.cpp @@ -25,8 +25,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { {arg->ray_direction[0], arg->ray_direction[1], arg->ray_direction[2]}, arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); diff --git a/tests/raytracing/rt_smoke_deep_tlas/kernel.cpp b/tests/raytracing/rt_smoke_deep_tlas/kernel.cpp index ef23db8f9f..68c1941e68 100644 --- a/tests/raytracing/rt_smoke_deep_tlas/kernel.cpp +++ b/tests/raytracing/rt_smoke_deep_tlas/kernel.cpp @@ -25,8 +25,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { {arg->ray_direction[0], arg->ray_direction[1], arg->ray_direction[2]}, arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); diff --git a/tests/raytracing/rt_smoke_fat_leaf/kernel.cpp b/tests/raytracing/rt_smoke_fat_leaf/kernel.cpp index 3f7da83385..f83047fcd8 100644 --- a/tests/raytracing/rt_smoke_fat_leaf/kernel.cpp +++ b/tests/raytracing/rt_smoke_fat_leaf/kernel.cpp @@ -25,8 +25,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { {arg->ray_direction[0], arg->ray_direction[1], arg->ray_direction[2]}, arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); diff --git a/tests/raytracing/rt_smoke_host_cfg/kernel.cpp b/tests/raytracing/rt_smoke_host_cfg/kernel.cpp index b508ae0760..cc94b8342b 100644 --- a/tests/raytracing/rt_smoke_host_cfg/kernel.cpp +++ b/tests/raytracing/rt_smoke_host_cfg/kernel.cpp @@ -27,8 +27,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { {arg->ray_direction[0], arg->ray_direction[1], arg->ray_direction[2]}, arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); diff --git a/tests/raytracing/rt_smoke_inst_flags/kernel.cpp b/tests/raytracing/rt_smoke_inst_flags/kernel.cpp index 2d078fc2cb..39ff1830e2 100644 --- a/tests/raytracing/rt_smoke_inst_flags/kernel.cpp +++ b/tests/raytracing/rt_smoke_inst_flags/kernel.cpp @@ -36,8 +36,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { // Ray flags = 0: the per-instance FORCE_NO_OPAQUE flag alone drives the // opacity classification, so the opaque triangle yields an AHS callback. - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, 0u, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, 0u, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); while (vx_rt_sts_is_yield(sts)) { diff --git a/tests/raytracing/rt_smoke_is/kernel.cpp b/tests/raytracing/rt_smoke_is/kernel.cpp index 0d273775b1..32b56087bd 100644 --- a/tests/raytracing/rt_smoke_is/kernel.cpp +++ b/tests/raytracing/rt_smoke_is/kernel.cpp @@ -32,19 +32,19 @@ __kernel void kernel_main(kernel_arg_t* arg) { arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t payload = (uint32_t)(arg->payload_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, payload, 0u, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uintptr_t payload = arg->payload_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, payload, 0u, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); while (vx_rt_sts_is_yield(sts)) { // Read the candidate's callback type + payload pointer from the window. // Write MAGIC for a procedural candidate, a ~MAGIC sentinel otherwise. uint32_t cb_type = vx_rt_get_attr(VX_RT_CB_TYPE, sts); - uint32_t payload_ptr = vx_rt_get_attr(VX_RT_PAYLOAD_PTR_LO, sts); + uintptr_t payload_ptr = vx_rt_get_payload(sts); uint32_t val = (cb_type == VX_RT_CB_TYPE_PROC) ? (uint32_t)RTU_IS_MAGIC : ~(uint32_t)RTU_IS_MAGIC; - *(uint32_t*)(uintptr_t)payload_ptr = val; + *(uint32_t*)payload_ptr = val; sts = vx_rt_continue(h, VX_RT_CB_ACCEPT, hit.t, 0u, &hit); } diff --git a/tests/raytracing/rt_smoke_miss/kernel.cpp b/tests/raytracing/rt_smoke_miss/kernel.cpp index fba90e7b70..b120f863bf 100644 --- a/tests/raytracing/rt_smoke_miss/kernel.cpp +++ b/tests/raytracing/rt_smoke_miss/kernel.cpp @@ -35,15 +35,15 @@ __kernel void kernel_main(kernel_arg_t* arg) { // The trace stages the payload pointer the MISS loop body reads via // vx_gfx_get. Opt into MISS dispatch. - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t payload = (uint32_t)(arg->payload_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, payload, VX_RT_FLAG_ENABLE_MISS, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uintptr_t payload = arg->payload_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, payload, VX_RT_FLAG_ENABLE_MISS, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); while (vx_rt_sts_is_yield(sts)) { // Read the payload pointer from the window, write MAGIC, then resume. - uint32_t payload_ptr = vx_rt_get_attr(VX_RT_PAYLOAD_PTR_LO, sts); - *(uint32_t*)(uintptr_t)payload_ptr = RTU_MISS_MAGIC; + uintptr_t payload_ptr = vx_rt_get_payload(sts); + *(uint32_t*)payload_ptr = RTU_MISS_MAGIC; sts = vx_rt_continue(h, VX_RT_CB_DONE, hit.t, 0u, &hit); } diff --git a/tests/raytracing/rt_smoke_multitri/kernel.cpp b/tests/raytracing/rt_smoke_multitri/kernel.cpp index 53b94f3172..bbc13fbff0 100644 --- a/tests/raytracing/rt_smoke_multitri/kernel.cpp +++ b/tests/raytracing/rt_smoke_multitri/kernel.cpp @@ -28,8 +28,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { {arg->ray_direction[0], arg->ray_direction[1], arg->ray_direction[2]}, arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, 0u, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, 0u, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); diff --git a/tests/raytracing/rt_smoke_proc/kernel.cpp b/tests/raytracing/rt_smoke_proc/kernel.cpp index 98e1c4a59d..17cf1d0967 100644 --- a/tests/raytracing/rt_smoke_proc/kernel.cpp +++ b/tests/raytracing/rt_smoke_proc/kernel.cpp @@ -35,9 +35,9 @@ __kernel void kernel_main(kernel_arg_t* arg) { arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); + uintptr_t scene_ptr = arg->scene_addr; // procedural primitive → IS decides (flags = 0), no payload. - uint32_t h = vx_rt_wtrace(scene_lo, 0u, 0u, 0xffu, &ray); + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, 0u, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); while (vx_rt_sts_is_yield(sts)) { diff --git a/tests/raytracing/rt_smoke_recursive/kernel.cpp b/tests/raytracing/rt_smoke_recursive/kernel.cpp index 3cc63860c0..521d7496c1 100644 --- a/tests/raytracing/rt_smoke_recursive/kernel.cpp +++ b/tests/raytracing/rt_smoke_recursive/kernel.cpp @@ -29,7 +29,7 @@ __kernel void kernel_main(kernel_arg_t* arg) { uint32_t tid = blockIdx.x; if (tid != 0) return; - uint32_t sub_scene = (uint32_t)(arg->sub_scene_addr & 0xffffffffu); + uintptr_t sub_scene = arg->sub_scene_addr; vx_ray_t ray = { { arg->ray_origin[0], arg->ray_origin[1], arg->ray_origin[2] }, @@ -37,11 +37,11 @@ __kernel void kernel_main(kernel_arg_t* arg) { arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); + uintptr_t scene_ptr = arg->scene_addr; // payload pointer for the CHS to write sub_status into; enable CHS for // the parent ray (so its closest hit returns a candidate). - uint32_t payload = (uint32_t)(arg->payload_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, payload, VX_RT_FLAG_ENABLE_CHS, + uintptr_t payload = arg->payload_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, payload, VX_RT_FLAG_ENABLE_CHS, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); @@ -51,7 +51,7 @@ __kernel void kernel_main(kernel_arg_t* arg) { vx_hit_t sub_hit; uint32_t sub_h = vx_rt_wtrace(sub_scene, 0u, 0u, 0xffu, &ray); uint32_t sub_status = vx_rt_wait(sub_h, &sub_hit); - *(volatile uint32_t*)(uintptr_t)payload = sub_status; + *(volatile uint32_t*)payload = sub_status; sts = vx_rt_continue(h, VX_RT_CB_DONE, hit.t, 0u, &hit); } diff --git a/tests/raytracing/rt_smoke_reform/kernel.cpp b/tests/raytracing/rt_smoke_reform/kernel.cpp index fde753bffa..2339db5a09 100644 --- a/tests/raytracing/rt_smoke_reform/kernel.cpp +++ b/tests/raytracing/rt_smoke_reform/kernel.cpp @@ -36,8 +36,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, 0u, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, 0u, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); while (vx_rt_sts_is_yield(sts)) { diff --git a/tests/raytracing/rt_smoke_reform_mw/kernel.cpp b/tests/raytracing/rt_smoke_reform_mw/kernel.cpp index ed2fca2489..ff89e784d2 100644 --- a/tests/raytracing/rt_smoke_reform_mw/kernel.cpp +++ b/tests/raytracing/rt_smoke_reform_mw/kernel.cpp @@ -35,8 +35,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, 0u, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, 0u, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); while (vx_rt_sts_is_yield(sts)) { diff --git a/tests/raytracing/rt_smoke_reform_sbt/kernel.cpp b/tests/raytracing/rt_smoke_reform_sbt/kernel.cpp index f260332c21..a7a213c3ca 100644 --- a/tests/raytracing/rt_smoke_reform_sbt/kernel.cpp +++ b/tests/raytracing/rt_smoke_reform_sbt/kernel.cpp @@ -32,7 +32,7 @@ __kernel void kernel_main(kernel_arg_t* arg) { // One shared (warp-uniform) scene; the per-lane ray aims at tri `tid`, so // each lane gets a distinct sbt_idx from the tri it hits — divergence rides // the ray, not the scene pointer. - uint32_t scene_addr = (uint32_t)(arg->scene_base_addr & 0xffffffffu); + uintptr_t scene_addr = arg->scene_base_addr; float ox = (float)tid * RTU_TRI_SPACING + RTU_RAY_XOFF; vx_ray_t ray = { diff --git a/tests/raytracing/rt_smoke_sbt/kernel.cpp b/tests/raytracing/rt_smoke_sbt/kernel.cpp index d9b407b5f1..bcd4ad849c 100644 --- a/tests/raytracing/rt_smoke_sbt/kernel.cpp +++ b/tests/raytracing/rt_smoke_sbt/kernel.cpp @@ -46,9 +46,9 @@ __kernel void kernel_main(kernel_arg_t* arg) { }; // The trace stages the payload pointer the SBT lookup reads via vx_gfx_get. - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t payload = (uint32_t)(arg->payload_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, payload, 0u, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uintptr_t payload = arg->payload_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, payload, 0u, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); while (vx_rt_sts_is_yield(sts)) { @@ -56,9 +56,9 @@ __kernel void kernel_main(kernel_arg_t* arg) { // cb_type slot, load the selected magic, write it to the payload. uint32_t cb_type = vx_rt_get_attr(VX_RT_CB_TYPE, sts); uint32_t sbt_idx = vx_rt_get_attr(VX_RT_HIT_SBT_IDX, sts); - uint32_t payload_ptr = vx_rt_get_attr(VX_RT_PAYLOAD_PTR_LO, sts); + uintptr_t payload_ptr = vx_rt_get_payload(sts); uint32_t magic = sbt[sbt_idx * 4 + (cb_type - 1)]; - *(uint32_t*)(uintptr_t)payload_ptr = magic; + *(uint32_t*)payload_ptr = magic; sts = vx_rt_continue(h, VX_RT_CB_ACCEPT, hit.t, 0u, &hit); } diff --git a/tests/raytracing/rt_smoke_shadow/kernel.cpp b/tests/raytracing/rt_smoke_shadow/kernel.cpp index 6b5e4b21d0..5723306124 100644 --- a/tests/raytracing/rt_smoke_shadow/kernel.cpp +++ b/tests/raytracing/rt_smoke_shadow/kernel.cpp @@ -27,8 +27,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { arg->tmin, arg->tmax }; // Shadow-ray flags: opaque-force + terminate-on-first-hit. - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE | VX_RT_FLAG_TERMINATE_ON_FIRST_HIT, 0xffu, &ray); vx_hit_t hit; diff --git a/tests/raytracing/rt_smoke_tlas/kernel.cpp b/tests/raytracing/rt_smoke_tlas/kernel.cpp index 653139aa08..1f20dd3b2b 100644 --- a/tests/raytracing/rt_smoke_tlas/kernel.cpp +++ b/tests/raytracing/rt_smoke_tlas/kernel.cpp @@ -25,8 +25,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { {arg->ray_direction[0], arg->ray_direction[1], arg->ray_direction[2]}, arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, 0u, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, 0u, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); diff --git a/tests/raytracing/rt_smoke_tlas_builder/kernel.cpp b/tests/raytracing/rt_smoke_tlas_builder/kernel.cpp index d083e6b25d..833667843f 100644 --- a/tests/raytracing/rt_smoke_tlas_builder/kernel.cpp +++ b/tests/raytracing/rt_smoke_tlas_builder/kernel.cpp @@ -25,8 +25,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { {arg->ray_direction[0], arg->ray_direction[1], arg->ray_direction[2]}, arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); From 9e90dab10ac17113edf04bab33abe539ae5592d1 Mon Sep 17 00:00:00 2001 From: tinebp Date: Fri, 9 Oct 2026 19:37:00 -0700 Subject: [PATCH 004/124] rtu: NaN-box single-precision slot reads into FP registers on RV64 On RV64 with the D extension an FP register is 64 bits, and a single-precision value in it must have its upper word all ones; anything else reads back as the canonical NaN. The core-side RTU unit zero-extended every slot read, so the hit t/u/v that vx_rt_wait (GETWF) lands in f0..f2 arrived as NaN, and the shading that used them went wrong on every hit. rt_raycast failed pixel comparison on RV64 rtlsim (210/768 pixels, maxdiff 172) while simx passed: SimX already NaN-boxes GETWF, as the LSU does for flw and the TCU for its results. The result path now NaN-boxes when the destination is an FP register, the same test the LSU uses; GP reads and RV32 are unchanged. rt_suite-x64 now also runs on rtlsim. RV64 rtlsim: rt_raycast passes with 0/768 mismatched pixels at NT4, NT16, 2-core and 8x8, and its retired instruction count (46136 at NT4) now matches simx. The ray-tracing CI category passes at xlen 64 on simx and rtlsim, and at xlen 32 on rtlsim (suite, model_parity, perf_gate unchanged). Co-Authored-By: Claude Opus 5.5 --- ci/testcases/raytracing.yaml | 4 +++- hw/rtl/rtu/VX_rtu_unit.sv | 15 ++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/ci/testcases/raytracing.yaml b/ci/testcases/raytracing.yaml index f4b42836d4..734ef894cf 100644 --- a/ci/testcases/raytracing.yaml +++ b/ci/testcases/raytracing.yaml @@ -30,13 +30,15 @@ tests: configs: -DVX_CFG_EXT_RTU_ENABLE # The same suite on RV64. There rt_smoke_chs puts its scene and payload above -# 4 GiB, so a trace that narrows either pointer misses or loses its payload. +# 4 GiB, so a trace that narrows either pointer misses or loses its payload, and +# the FP hit-window reads must come back NaN-boxed. - id: rt_suite-x64 via: make-run xlen: - 64 drivers: - simx + - rtlsim dir: tests/raytracing target: run-{driver} configs: -DVX_CFG_EXT_RTU_ENABLE diff --git a/hw/rtl/rtu/VX_rtu_unit.sv b/hw/rtl/rtu/VX_rtu_unit.sv index 50781fcaf0..cfeb22a315 100644 --- a/hw/rtl/rtu/VX_rtu_unit.sv +++ b/hw/rtl/rtu/VX_rtu_unit.sv @@ -391,8 +391,21 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( // write back zero. sfu_result_t rsp_data_out; assign rsp_data_out.header = s1_header; +`ifdef VX_CFG_XLEN_64 +`ifdef VX_CFG_EXT_F_ENABLE + // A single-precision slot read into a 64-bit FP register must be NaN-boxed. + wire rsp_is_float = (get_reg_type(s1_header.rd) == REG_TYPE_F); +`else + wire rsp_is_float = 1'b0; +`endif +`endif for (genvar i = 0; i < NUM_LANES; ++i) begin : g_rsp_data - assign rsp_data_out.data[i] = `VX_CFG_XLEN'(s1_from_ram ? core_rdata[i] : 32'd0); + wire [31:0] rsp_word = s1_from_ram ? core_rdata[i] : 32'd0; + `ifdef VX_CFG_XLEN_64 + assign rsp_data_out.data[i] = rsp_is_float ? {32'hffffffff, rsp_word} : 64'(rsp_word); + `else + assign rsp_data_out.data[i] = rsp_word; + `endif end assign result_if.valid = s1_valid; From 127b03927eee29d5f31aef1039ec09896cb4dd05 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 10 Oct 2026 03:14:20 -0700 Subject: [PATCH 005/124] gfx: store the depth plane modulo 2^32 instead of converting out of range Setup converts the screen-space depth plane Z = A'x + B'y + C' to Q7.24 with a plain float-to-int cast. C' is the plane at the screen origin, so a steep plane on a triangle away from the origin puts it past Q7.24's +-128 range even though the depth at every covered pixel is in range. That conversion is undefined: RISC-V fcvt saturates (wrong depth on the device), x86 cvttss2si returns INT_MIN. gfx_pipeline_kernel failed on primbuf[41] at both xlen 32 and 64 with dev C' = INT_MAX against ref C' = INT_MIN. Late-Z and the raster early-Z (SimX and RTL) all evaluate the plane mod 2^32, so a term stored modulo 2^32 gives the exact depth at the covered pixels. PlaneToFixed keeps the plain cast for in-range terms, wraps an out-of-range term through int64, and maps a non-finite or absurdly steep term (past 2^39, which would overflow int64) to 0 so every input is defined on host and device. gfx_setup_kernel, gfx_binsort_kernel and gfx_pipeline_kernel compare the device front end byte for byte against the host reference but were in no CI category; they now run in graphics at xlen 32 and 64. gfx_pipeline_kernel passes at xlen 32 and 64 (failed at both before). graphics simx 38/38 at xlen 32 and 64, vulkan simx 13/13 at xlen 32 and 64, graphics rtlsim 56/56 at xlen 32 including every perf_gate case unchanged. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/graphics.yaml | 5 +++++ sw/common/gfx_setup.h | 24 +++++++++++++++++++++--- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/ci/testcases/graphics.yaml b/ci/testcases/graphics.yaml index 7214c48ba0..493d96b9fd 100644 --- a/ci/testcases/graphics.yaml +++ b/ci/testcases/graphics.yaml @@ -547,6 +547,11 @@ tests: - {id: gfx_pipeline_tex, via: make-run, drivers: [simx], dir: tests/graphics/gfx_pipeline_tex, target: "run-{driver}"} - {id: gfx_pipeline_om, via: make-run, drivers: [simx], dir: tests/graphics/gfx_pipeline_om, target: "run-{driver}"} - {id: gfx_pipeline_raster,via: make-run, drivers: [simx], dir: tests/graphics/gfx_pipeline_raster,target: "run-{driver}"} +# The front-end setup, bin-sort and full pipeline kernels, each checked byte for +# byte against the host reference built from the same sw/gfx and sw/common headers. +- {id: gfx_setup_kernel, via: make-run, drivers: [simx], dir: tests/regression/gfx_setup_kernel, target: "run-{driver}"} +- {id: gfx_binsort_kernel, via: make-run, drivers: [simx], dir: tests/regression/gfx_binsort_kernel, target: "run-{driver}"} +- {id: gfx_pipeline_kernel, via: make-run, drivers: [simx], dir: tests/regression/gfx_pipeline_kernel, target: "run-{driver}"} # ── FF/SW routing matrix ───────────────────────────────────────────────────── # Each fixed-function unit dropped from the build (NO_TEX / NO_OM / NO_RASTER); diff --git a/sw/common/gfx_setup.h b/sw/common/gfx_setup.h index 3698cd84f7..7b9d619d00 100644 --- a/sw/common/gfx_setup.h +++ b/sw/common/gfx_setup.h @@ -155,6 +155,24 @@ static inline bool EdgeEquation(vec3f edges[3], const vec4f& v0, return true; } +// A depth-plane term in Q7.24, stored modulo 2^32. The constant term is the plane +// at the screen origin, which leaves Q7.24's +-128 range for a steep plane even +// though the depth at every covered pixel is in range, and an out-of-range float +// to int conversion is undefined (RISC-V saturates, x86 yields INT_MIN). Late-Z +// and early-Z both evaluate the plane mod 2^32, so the wrapped term is exact. A +// term past 2^39 (non-finite, or a degenerate sliver) has no usable depth. +static inline FloatA PlaneToFixed(float f) { + float scaled = f * 16777216.0f; // 2^24 + float mag = __builtin_fabsf(f); + if (mag < 128.0f) { + return FloatA::make((int32_t)scaled); + } + if (!(mag < 549755813888.0f)) { // 2^39: scaled would overflow int64 + return FloatA::make(0); + } + return FloatA::make((int32_t)(uint32_t)(uint64_t)(int64_t)scaled); +} + // Normalize the edge matrix by 1/maxVal and convert to Q15.16. static inline void EdgeToFixed(vec3e_t out[3], const vec3f in[3]) { float maxVal = __builtin_fabsf(in[0].x); @@ -271,9 +289,9 @@ static inline bool setup_triangle(const setup_vertex_t& v0, // C' evaluated so an integer-pixel MAC samples Z at the pixel center // (X+0.5, Y+0.5) — matching the +0.5 half-pixel offset the coverage edges use. float zc = ps0.z - za * ps0.x - zb * ps0.y + 0.5f * (za + zb); - out_prim.attribs.z.x = FloatA(za); - out_prim.attribs.z.y = FloatA(zb); - out_prim.attribs.z.z = FloatA(zc); + out_prim.attribs.z.x = PlaneToFixed(za); + out_prim.attribs.z.y = PlaneToFixed(zb); + out_prim.attribs.z.z = PlaneToFixed(zc); } // Perspective-correct varyings: premultiply each colour/texcoord by the // vertex's 1/w and carry a separate 1/w plane; the FS interpolates every plane From afd7acb2cc3aeccd035eb4289ef618220783c440 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 10 Oct 2026 03:24:52 -0700 Subject: [PATCH 006/124] kernel: check vx_printf formats, and print vx_putint without a float detour vx_printf and vx_vprintf carried no format attribute, so a kernel could pass a 64-bit value, a pointer or a size_t under a %d and the compiler said nothing; on RV64 that reads the wrong varargs slot. They are now declared format(printf, ...), so -Wall -Werror rejects a mismatched call. tinyprintf already implements the l/ll/z/j/t/h modifiers and %p the checker asks for. Every current caller (vx_spawn.c, tests/kernel/*, tests/regression/printf) is clean at xlen 32 and 64; a deliberately mismatched call is rejected at both. vx_putint copied its int argument into a float before itoa, so any value past 2^24 printed rounded (16777217 printed as 16777216). It now stays an int. Co-Authored-By: Claude Opus 5.5 --- sw/kernel/include/vx_print.h | 4 ++-- sw/kernel/src/vx_print.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sw/kernel/include/vx_print.h b/sw/kernel/include/vx_print.h index 696d297759..6df557009f 100644 --- a/sw/kernel/include/vx_print.h +++ b/sw/kernel/include/vx_print.h @@ -20,8 +20,8 @@ extern "C" { #endif -int vx_vprintf(const char* format, va_list va); -int vx_printf(const char * format, ...); +int vx_vprintf(const char* format, va_list va) __attribute__((format(printf, 1, 0))); +int vx_printf(const char * format, ...) __attribute__((format(printf, 1, 2))); void vx_putchar(int c); void vx_putint(int value, int base); diff --git a/sw/kernel/src/vx_print.c b/sw/kernel/src/vx_print.c index e23f0719ad..1056353031 100644 --- a/sw/kernel/src/vx_print.c +++ b/sw/kernel/src/vx_print.c @@ -43,7 +43,7 @@ typedef struct { static void __putint_cb(const putint_arg_t* arg) { char tmp[33]; - float value = arg->value; + int value = arg->value; int base = arg->base; itoa(value, tmp, base); for (int i = 0; i < 33; ++i) { From 2a8e29d92d9412a9e8604be7e0e8e1b0f571dba1 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 10 Oct 2026 03:37:39 -0700 Subject: [PATCH 007/124] sgemm_tcu_wg_dxa: widen the SW_LOAD_B index math, build SW_LOAD_A + SW_LOAD_B The SW_LOAD_B cooperative load still computed its indices in uint32_t, the pattern the RV64 index-portability change replaced in the SW_LOAD_A loop next to it; it now uses size_t (the same type as uint32_t on RV32). RV64 simx at the dxa sgemm_tcu_wg_dxa-1 config, 8 warps x 32 threads: SW_LOAD_A + SW_LOAD_B 28832 vs 29141 cycles (-1.1%), SW_LOAD_B alone 23167 vs 23160 (flat). With both loads in software no DXA transfer is issued, is_dxa_warp goes unused, and -Werror rejected the build at xlen 32 and 64; it is now [[maybe_unused]], like the descriptor ids above it. RV32 code is identical with SW_LOAD_B, and the default (DXA) build CI runs is identical at xlen 32 and 64. Co-Authored-By: Claude Opus 5.5 --- tests/regression/sgemm_tcu_wg_dxa/kernel.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/regression/sgemm_tcu_wg_dxa/kernel.cpp b/tests/regression/sgemm_tcu_wg_dxa/kernel.cpp index 281bd1aa77..7b2596dabb 100644 --- a/tests/regression/sgemm_tcu_wg_dxa/kernel.cpp +++ b/tests/regression/sgemm_tcu_wg_dxa/kernel.cpp @@ -75,7 +75,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { ctx::fill_fragment(fragC, 0); // Only the first warp in the CTA issues DXA commands. - const bool is_dxa_warp = (get_sub_group_id() == 0); + [[maybe_unused]] const bool is_dxa_warp = (get_sub_group_id() == 0); // Stage the tile at K-offset k into the {A_smem, B_smem} buffer guarded // by barrier id `bar`. All state is passed by value in registers. @@ -109,10 +109,10 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { #ifdef SW_LOAD_B // Cooperative load B block-major (matches DXA BlockMajor LAYOUT / bbuf). uint32_t b_size = ctx::tileK * ctx::xtileN; - for (uint32_t i = 0; i < b_size; i += num_threads) { - uint32_t idx = i + tid; - uint32_t r = idx / ctx::xtileN; - uint32_t c = idx % ctx::xtileN; + for (size_t i = 0; i < b_size; i += num_threads) { + size_t idx = i + tid; + size_t r = idx / ctx::xtileN; + size_t c = idx % ctx::xtileN; B_smem[ctx::b_blockmajor_idx(r, c)] = pB[(k + r) * N + (tile_col + c)]; } #endif From 5d224d8f68a030d5fd9dc218b53db0165ead5f5e Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 10 Oct 2026 06:27:34 -0700 Subject: [PATCH 008/124] tests: widen the index math of demo, fence, dogfood, softmax and sgemm_tcu These five kernels kept uint32_t indices in cbd9cbee8 because widening them made RV64 slower: LLVM's LoopTermFold moved each loop's uniform exit test onto a per-thread pointer, which needs divergence predication every iteration. llvm_vortex a81a5957f406 keeps a uniform exit uniform on targets with branch divergence, and with it the size_t form is the faster one on RV64 simx: demo -20.4%, fence -20.6%, dogfood -14.6%, softmax -10.7%, sgemm_tcu -8.6% cycles. RV32 code is identical (size_t is uint32_t there). This depends on that compiler change: with the current toolchain the same kernels are 1.4-3.8% slower on RV64 (demo +1.8%, fence +1.4%, dogfood +1.5%, softmax +3.8%, sgemm_tcu +2.1%). It must land after the toolchain prebuilt carrying a81a5957f406, together with the perf baselines that compiler moves. The other kernels left narrow in cbd9cbee8 stay so: sgemv (+5.0%), dotproduct (+1.5%), sgemm_tcu_mx (+3.7%), sgemm_tcu_sp_mx (+1.8%) and sgemm_tcu_wg (+1.3%) are still slower widened with the new compiler, and vm_test is unchanged. Co-Authored-By: Claude Opus 5.5 --- tests/regression/demo/kernel.cpp | 8 +-- tests/regression/dogfood/kernel.cpp | 90 +++++++++++++-------------- tests/regression/fence/kernel.cpp | 8 +-- tests/regression/sgemm_tcu/kernel.cpp | 10 +-- tests/regression/softmax/kernel.cpp | 10 +-- 5 files changed, 63 insertions(+), 63 deletions(-) diff --git a/tests/regression/demo/kernel.cpp b/tests/regression/demo/kernel.cpp index 2199d70fa0..a4f5467bb4 100644 --- a/tests/regression/demo/kernel.cpp +++ b/tests/regression/demo/kernel.cpp @@ -11,11 +11,11 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { // the flat tid = y*bdx+x reconstruction. uint32_t gx = blockIdx.x * blockDim.x + threadIdx.x; uint32_t gy = blockIdx.y * blockDim.y + threadIdx.y; - uint32_t gid = gy * arg->dim_x + gx; - uint32_t count = arg->task_size; - uint32_t offset = gid * count; + size_t gid = gy * arg->dim_x + gx; + size_t count = arg->task_size; + size_t offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { dst_ptr[offset+i] = src0_ptr[offset+i] + src1_ptr[offset+i]; } } diff --git a/tests/regression/dogfood/kernel.cpp b/tests/regression/dogfood/kernel.cpp index 889a063f50..a5ffa38a22 100644 --- a/tests/regression/dogfood/kernel.cpp +++ b/tests/regression/dogfood/kernel.cpp @@ -10,14 +10,14 @@ inline float __ieee754_sqrtf (float x) { } void kernel_iadd(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (int32_t*)arg->src0_addr; auto src1_ptr = (int32_t*)arg->src1_addr; auto dst_ptr = (int32_t*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { int32_t a = src0_ptr[offset+i]; int32_t b = src1_ptr[offset+i]; int32_t c = a + b; @@ -26,14 +26,14 @@ void kernel_iadd(kernel_arg_t* __UNIFORM__ arg) { } void kernel_imul(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (int32_t*)arg->src0_addr; auto src1_ptr = (int32_t*)arg->src1_addr; auto dst_ptr = (int32_t*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c = a * b; @@ -42,14 +42,14 @@ void kernel_imul(kernel_arg_t* __UNIFORM__ arg) { } void kernel_idiv(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (int32_t*)arg->src0_addr; auto src1_ptr = (int32_t*)arg->src1_addr; auto dst_ptr = (int32_t*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c = a / b; @@ -58,14 +58,14 @@ void kernel_idiv(kernel_arg_t* __UNIFORM__ arg) { } void kernel_idiv_mul(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (int32_t*)arg->src0_addr; auto src1_ptr = (int32_t*)arg->src1_addr; auto dst_ptr = (int32_t*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c = a / b; @@ -76,14 +76,14 @@ void kernel_idiv_mul(kernel_arg_t* __UNIFORM__ arg) { } void kernel_fadd(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { float a = src0_ptr[offset+i]; float b = src1_ptr[offset+i]; float c = a + b; @@ -92,14 +92,14 @@ void kernel_fadd(kernel_arg_t* __UNIFORM__ arg) { } void kernel_fsub(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c = a - b; @@ -108,14 +108,14 @@ void kernel_fsub(kernel_arg_t* __UNIFORM__ arg) { } void kernel_fmul(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c = a * b; @@ -124,14 +124,14 @@ void kernel_fmul(kernel_arg_t* __UNIFORM__ arg) { } void kernel_fmadd(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c = a * b + b; @@ -140,14 +140,14 @@ void kernel_fmadd(kernel_arg_t* __UNIFORM__ arg) { } void kernel_fmsub(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c = a * b - b; @@ -156,14 +156,14 @@ void kernel_fmsub(kernel_arg_t* __UNIFORM__ arg) { } void kernel_fnmadd(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c =-a * b - b; @@ -172,14 +172,14 @@ void kernel_fnmadd(kernel_arg_t* __UNIFORM__ arg) { } void kernel_fnmsub(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c =-a * b + b; @@ -188,14 +188,14 @@ void kernel_fnmsub(kernel_arg_t* __UNIFORM__ arg) { } void kernel_fnmadd_madd(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c =-a * b - b; @@ -206,14 +206,14 @@ void kernel_fnmadd_madd(kernel_arg_t* __UNIFORM__ arg) { } void kernel_fdiv(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c = a / b; @@ -222,14 +222,14 @@ void kernel_fdiv(kernel_arg_t* __UNIFORM__ arg) { } void kernel_fdiv2(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c = a / b; @@ -240,14 +240,14 @@ void kernel_fdiv2(kernel_arg_t* __UNIFORM__ arg) { } void kernel_fsqrt(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c = __ieee754_sqrtf(a * b); @@ -256,14 +256,14 @@ void kernel_fsqrt(kernel_arg_t* __UNIFORM__ arg) { } void kernel_ftoi(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (int32_t*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c = a + b; @@ -273,14 +273,14 @@ void kernel_ftoi(kernel_arg_t* __UNIFORM__ arg) { } void kernel_ftou(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (uint32_t*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c = a + b; @@ -290,14 +290,14 @@ void kernel_ftou(kernel_arg_t* __UNIFORM__ arg) { } void kernel_itof(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (int32_t*)arg->src0_addr; auto src1_ptr = (int32_t*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c = a + b; @@ -307,14 +307,14 @@ void kernel_itof(kernel_arg_t* __UNIFORM__ arg) { } void kernel_utof(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (int32_t*)arg->src0_addr; auto src1_ptr = (int32_t*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c = a + b; @@ -328,14 +328,14 @@ inline float fclamp(float a, float b, float c) { } void kernel_fclamp(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; dst_ptr[offset+i] = fclamp(1.0f, a, b); @@ -347,14 +347,14 @@ inline int iclamp(int a, int b, int c) { } void kernel_iclamp(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (int*)arg->src0_addr; auto src1_ptr = (int*)arg->src1_addr; auto dst_ptr = (int*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; dst_ptr[offset+i] = iclamp(1, a, b); @@ -362,14 +362,14 @@ void kernel_iclamp(kernel_arg_t* __UNIFORM__ arg) { } void kernel_trigo(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { - uint32_t j = offset + i; + for (size_t i = 0; i < count; ++i) { + size_t j = offset + i; auto a = src0_ptr[j]; auto b = src1_ptr[j]; auto c = a * b; diff --git a/tests/regression/fence/kernel.cpp b/tests/regression/fence/kernel.cpp index 8d0c8576ea..1e7a6ba47f 100644 --- a/tests/regression/fence/kernel.cpp +++ b/tests/regression/fence/kernel.cpp @@ -2,14 +2,14 @@ #include "common.h" __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { - uint32_t count = arg->task_size; + size_t count = arg->task_size; int32_t* src0_ptr = (int32_t*)arg->src0_addr; int32_t* src1_ptr = (int32_t*)arg->src1_addr; int32_t* dst_ptr = (int32_t*)arg->dst_addr; - uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; - uint32_t offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + size_t gid = blockIdx.x * blockDim.x + threadIdx.x; + size_t offset = gid * count; + for (size_t i = 0; i < count; ++i) { dst_ptr[offset+i] = src0_ptr[offset+i] + src1_ptr[offset+i]; } diff --git a/tests/regression/sgemm_tcu/kernel.cpp b/tests/regression/sgemm_tcu/kernel.cpp index 32cfd94d35..b3f8c8d2d9 100644 --- a/tests/regression/sgemm_tcu/kernel.cpp +++ b/tests/regression/sgemm_tcu/kernel.cpp @@ -11,16 +11,16 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto pB = reinterpret_cast(arg->B_addr); auto pC = reinterpret_cast(arg->C_addr); - uint32_t N = arg->N; - uint32_t K = arg->K; + size_t N = arg->N; + size_t K = arg->K; ctx::fragment_a fragA; ctx::fragment_b fragB; ctx::fragment_acc fragC; // calculate tile row & column based on block index - uint32_t tile_row = blockIdx.y * ctx::tileM; - uint32_t tile_col = blockIdx.x * ctx::tileN; + size_t tile_row = blockIdx.y * ctx::tileM; + size_t tile_col = blockIdx.x * ctx::tileN; // Initialize accumulator tile to zero ctx::fill_fragment(fragC, 0); @@ -29,7 +29,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { uint32_t cycles = 0; #endif - for (uint32_t i = 0; i < K; i += ctx::tileK) { + for (size_t i = 0; i < K; i += ctx::tileK) { auto pTileA = pA + tile_row * K + i; auto pTileB = pB + tile_col * K + i; diff --git a/tests/regression/softmax/kernel.cpp b/tests/regression/softmax/kernel.cpp index 92fec90970..4d2a0ea5ac 100644 --- a/tests/regression/softmax/kernel.cpp +++ b/tests/regression/softmax/kernel.cpp @@ -5,30 +5,30 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto src0_ptr = reinterpret_cast(arg->src0_addr); auto dst_ptr = reinterpret_cast(arg->dst_addr); - uint32_t num_cols = arg->num_cols; + size_t num_cols = arg->num_cols; uint32_t row = blockIdx.x * blockDim.x + threadIdx.x; if (row >= arg->num_rows) return; - uint32_t offset = row * num_cols; + size_t offset = row * num_cols; // 1. Find the max value TYPE max = 0.0; - for (uint32_t i = 0; i < num_cols; i++) { + for (size_t i = 0; i < num_cols; i++) { auto curr = src0_ptr[offset + i]; if (curr > max) max = curr; } // 2. Subtract max and compute exp TYPE sum = 0.0; - for (uint32_t i = 0; i < num_cols; i++) { + for (size_t i = 0; i < num_cols; i++) { auto computed = exp(src0_ptr[offset + i] - max); src0_ptr[offset + i] = computed; sum += computed; } // 3. Normalize - for (uint32_t i = 0; i < num_cols; i++) { + for (size_t i = 0; i < num_cols; i++) { dst_ptr[offset + i] = src0_ptr[offset + i] / sum; } } From 6c26b424ae66eccd522bcfd89e360254ce8f377e Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 10 Oct 2026 15:50:48 -0700 Subject: [PATCH 009/124] rtu: carry the full scene address through the ray log and rt_replay Code merged from master after the RTU pointer widening narrowed the scene again on RV64: - vx_rt_wtrace pinned the gathered trace config in a uint32_t x31 register, truncating the scene base it had just gathered at full width; - the SimX ray log stored scene_root in 32 bits, so a snapshot of a scene above 4 GiB read the wrong memory, and rt_replay replayed it against that address; - rt_smoke_ahs_multi masked its scene pointer to the low word. The ray-log record keeps its 112-byte size (scene_root takes one reserved word); the format version goes from 1 to 2 since the field offsets change. Co-Authored-By: Claude Opus 5.5 --- sim/simx/rtu/rtu_raylog.cpp | 10 +++++----- sim/simx/rtu/rtu_raylog.h | 6 +++--- sw/kernel/include/vx_raytrace.h | 2 +- tests/raytracing/rt_replay/common.h | 10 +++++----- tests/raytracing/rt_replay/kernel.cpp | 2 +- tests/raytracing/rt_replay/main.cpp | 6 +++--- tests/raytracing/rt_smoke_ahs_multi/kernel.cpp | 4 ++-- 7 files changed, 20 insertions(+), 20 deletions(-) diff --git a/sim/simx/rtu/rtu_raylog.cpp b/sim/simx/rtu/rtu_raylog.cpp index 4281972b50..42592e10ec 100644 --- a/sim/simx/rtu/rtu_raylog.cpp +++ b/sim/simx/rtu/rtu_raylog.cpp @@ -158,13 +158,13 @@ class Logger { static constexpr uint64_t kInstTableSpan = 64 * 1024; static constexpr uint64_t kMaxSceneBytes = 1ull << 30; - void snapshot_scene(uint32_t root) { + void snapshot_scene(uint64_t root) { if (ram_ == nullptr || full() || !snapped_.insert(root).second) return; const RAM& ram = *ram_; uint32_t scene_bytes = 0; for (int i = 0; i < 4; ++i) scene_bytes |= uint32_t(ram[root + 8 + i]) << (8 * i); const uint64_t lo = (root > kInstTableSpan ? root - kInstTableSpan : 0) & kRtuLineMask; - const uint64_t hi = uint64_t(root) + std::min(scene_bytes, kMaxSceneBytes); + const uint64_t hi = root + std::min(scene_bytes, kMaxSceneBytes); uint64_t added = 0; for (uint64_t a = lo; a < hi; a += VX_CFG_MEM_BLOCK_SIZE) { if (image_.count(a)) continue; @@ -184,12 +184,12 @@ class Logger { std::fwrite(&r, sizeof(r), 1, fp_); ++added; } - std::fprintf(stderr, "[rtu-raylog] scene 0x%x: %u bytes, snapshot %llu lines\n", - root, scene_bytes, (unsigned long long)added); + std::fprintf(stderr, "[rtu-raylog] scene 0x%llx: %u bytes, snapshot %llu lines\n", + (unsigned long long)root, scene_bytes, (unsigned long long)added); } const RAM* ram_ = nullptr; - std::unordered_set snapped_; + std::unordered_set snapped_; FILE* fp_ = nullptr; unsigned long long max_rays_ = 4ull << 20; unsigned long long every_ = 1; diff --git a/sim/simx/rtu/rtu_raylog.h b/sim/simx/rtu/rtu_raylog.h index d9b284cee1..b73b2d834b 100644 --- a/sim/simx/rtu/rtu_raylog.h +++ b/sim/simx/rtu/rtu_raylog.h @@ -47,7 +47,7 @@ class RAM; namespace rtu { namespace raylog { constexpr uint32_t kMagic = 0x4C525856; // "VXRL" -constexpr uint32_t kVersion = 1; +constexpr uint32_t kVersion = 2; enum RecType : uint32_t { REC_LINE = 1, @@ -80,7 +80,7 @@ struct RaylogRay { uint32_t info; uint32_t seq; uint32_t epoch; - uint32_t scene_root; + uint64_t scene_root; uint32_t ray_flags; uint32_t cull_mask; uint32_t warp_lane; // lane | warp << 8 | slot << 16 @@ -97,7 +97,7 @@ struct RaylogRay { uint32_t inst_id; uint32_t inst_custom; uint32_t hit_attr; - uint32_t reserved[3]; + uint32_t reserved[2]; }; static_assert(sizeof(RaylogRay) == 112, "RaylogRay layout"); diff --git a/sw/kernel/include/vx_raytrace.h b/sw/kernel/include/vx_raytrace.h index ea9e63e5bb..fe092afd1a 100644 --- a/sw/kernel/include/vx_raytrace.h +++ b/sw/kernel/include/vx_raytrace.h @@ -184,7 +184,7 @@ uint32_t vx_rt_wtrace(uintptr_t scene_ptr, uintptr_t payload_ptr, // the field references stable across the long register-binding list. // The RTU reads the packed config lanes regardless of the thread mask, so it // reads them from the warp-gather register itself (see __VX_WGATHER_IN). - register uint32_t cfg_reg __asm__("x31") = cfg; + register size_t cfg_reg __asm__("x31") = cfg; __asm__ volatile (".insn r %[op], 7, 0, %[hnd], %[cfg], x0" : [hnd]"=r"(handle) : [op]"i"(RISCV_CUSTOM1), [cfg]"r"(cfg_reg), diff --git a/tests/raytracing/rt_replay/common.h b/tests/raytracing/rt_replay/common.h index 8f94077e7b..e696d13641 100644 --- a/tests/raytracing/rt_replay/common.h +++ b/tests/raytracing/rt_replay/common.h @@ -23,10 +23,9 @@ // One ray as the kernel consumes it. Each warp's rays share scene / flags / // cull: those ride the warp-uniform half of vx_rt_wtrace. typedef struct { - uint32_t scene; + uint64_t scene; uint32_t flags; uint32_t cull; - uint32_t pad; float origin[3]; float dir[3]; float tmin; @@ -57,7 +56,7 @@ typedef struct { #ifdef __cplusplus // Ray-log stream layout (mirror of sim/simx/rtu/rtu_raylog.h). #define RAYLOG_MAGIC 0x4C525856u -#define RAYLOG_VERSION 1u +#define RAYLOG_VERSION 2u #define RAYLOG_REC_LINE 1u #define RAYLOG_REC_RAY 2u #define RAYLOG_INFO_REPLAYABLE (1u << 8) @@ -69,11 +68,12 @@ struct RaylogHeader { struct RaylogRay { uint32_t type, info, seq, epoch; - uint32_t scene_root, ray_flags, cull_mask, warp_lane; + uint64_t scene_root; + uint32_t ray_flags, cull_mask, warp_lane; uint32_t origin[3], dir[3]; uint32_t tmin, tmax; uint32_t status, hit_t, hit_u, hit_v, prim, geom, inst_id, inst_custom, hit_attr; - uint32_t reserved[3]; + uint32_t reserved[2]; }; static_assert(sizeof(RaylogRay) == 112, "RaylogRay layout"); #endif diff --git a/tests/raytracing/rt_replay/kernel.cpp b/tests/raytracing/rt_replay/kernel.cpp index a6cf6922c5..24b6434c95 100644 --- a/tests/raytracing/rt_replay/kernel.cpp +++ b/tests/raytracing/rt_replay/kernel.cpp @@ -27,7 +27,7 @@ __kernel void kernel_main(kernel_arg_t* arg) { vx_ray_t ray = { {r.origin[0], r.origin[1], r.origin[2]}, {r.dir[0], r.dir[1], r.dir[2]}, r.tmin, r.tmax }; - uint32_t h = vx_rt_wtrace(r.scene, 0u, r.flags, r.cull, &ray); + uint32_t h = vx_rt_wtrace((uintptr_t)r.scene, 0u, r.flags, r.cull, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); uint32_t yielded = 0; diff --git a/tests/raytracing/rt_replay/main.cpp b/tests/raytracing/rt_replay/main.cpp index e326bcecfe..cf0dd113c7 100644 --- a/tests/raytracing/rt_replay/main.cpp +++ b/tests/raytracing/rt_replay/main.cpp @@ -164,9 +164,9 @@ bool matches(const RaylogRay& ref, const replay_result_t& got) { } void print_ray(FILE* fp, uint64_t idx, const RaylogRay& r, const replay_result_t& g, const char* cat) { - fprintf(fp, "%llu seq=%u cat=%s epoch=%u scene=0x%x flags=0x%x cull=0x%x" + fprintf(fp, "%llu seq=%u cat=%s epoch=%u scene=0x%llx flags=0x%x cull=0x%x" " o=(%a,%a,%a) d=(%a,%a,%a) tmin=%a tmax=%a\n", - (unsigned long long)idx, r.seq, cat, r.epoch, r.scene_root, r.ray_flags, r.cull_mask, + (unsigned long long)idx, r.seq, cat, r.epoch, (unsigned long long)r.scene_root, r.ray_flags, r.cull_mask, fbits(r.origin[0]), fbits(r.origin[1]), fbits(r.origin[2]), fbits(r.dir[0]), fbits(r.dir[1]), fbits(r.dir[2]), fbits(r.tmin), fbits(r.tmax)); @@ -338,7 +338,7 @@ int main(int argc, char** argv) { // Group by the warp-uniform trace config; pad each group to whole warps // with copies of its last ray (their results are dropped). - std::map, std::vector> groups; + std::map, std::vector> groups; for (size_t k = pos; k < end; ++k) { const RaylogRay& r = rays[sel[k]]; groups[{r.scene_root, r.ray_flags & ~(VX_RT_FLAG_ENABLE_CHS | VX_RT_FLAG_ENABLE_MISS), r.cull_mask}].push_back(sel[k]); diff --git a/tests/raytracing/rt_smoke_ahs_multi/kernel.cpp b/tests/raytracing/rt_smoke_ahs_multi/kernel.cpp index fdbe122e8d..96caf8da71 100644 --- a/tests/raytracing/rt_smoke_ahs_multi/kernel.cpp +++ b/tests/raytracing/rt_smoke_ahs_multi/kernel.cpp @@ -35,8 +35,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { rtu_result_t* res = (rtu_result_t*)((uintptr_t)arg->results_addr) + tid; uint32_t n = 0; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, 0u, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, 0u, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); while (vx_rt_sts_is_yield(sts)) { From 78dbecfa79315c2ace2febbb04fbfbe6cd288195 Mon Sep 17 00:00:00 2001 From: tinebp Date: Tue, 6 Oct 2026 02:12:40 -0700 Subject: [PATCH 010/124] config: hardware profiles shared by every flow A profile (profiles/.json) names a complete machine: per-core knobs, chip-level knobs, the platform around it, the measurements it is calibrated against, and the sources of every value. gen_config.py layers it between the VX_config.toml defaults and explicit -D flags, validates each knob against the TOML (unknown knobs, enum values, bool/int types, required sources) and folds it into CONFIGS with --merge-configs, one -D per macro so -Werror builds never see a redefinition. Profiles inherit with "extends". config.mk does the merge once, so every flow that includes it -- simulators, runtimes, kernels, tests and the synthesis Makefiles -- builds the same machine without per-flow changes. configure --profile sets a build directory's default and bakes it into the generated headers and the config stamp; blackbox.sh --profile selects one per run. Without a profile the generated flags and headers are byte-identical to before. Adds profiles/rtx_5080.json (GB203 at base clock) with the knobs the model has today. Co-Authored-By: Claude Opus 5.5 --- ci/blackbox.sh | 3 +- ci/gen_config.py | 147 +++++++++++++++++++++++++++++++++++++++++ config.mk.in | 11 +++ configure | 16 ++++- profiles/rtx_5080.json | 45 +++++++++++++ 5 files changed, 219 insertions(+), 3 deletions(-) create mode 100644 profiles/rtx_5080.json diff --git a/ci/blackbox.sh b/ci/blackbox.sh index f4072e9ada..21215167a0 100755 --- a/ci/blackbox.sh +++ b/ci/blackbox.sh @@ -19,7 +19,7 @@ ROOT_DIR=$SCRIPT_DIR/.. show_usage() { echo "Vortex BlackBox Test Driver v1.0" - echo "Usage: $0 [[--clusters=#n] [--cores=#n] [--warps=#n] [--threads=#n] [--l2cache] [--l3cache] [[--driver=#name] [--target=#name] [--app=#app] [--args=#args] [--debug=#level] [--scope] [--vcd] [--saif] [--perf=#class] [--vcd_file=#file] [--saif_file=#file] [--log=logfile] [--nohup] [--help]]" + echo "Usage: $0 [[--profile=#name] [--clusters=#n] [--cores=#n] [--warps=#n] [--threads=#n] [--l2cache] [--l3cache] [[--driver=#name] [--target=#name] [--app=#app] [--args=#args] [--debug=#level] [--scope] [--vcd] [--saif] [--perf=#class] [--vcd_file=#file] [--saif_file=#file] [--log=logfile] [--nohup] [--help]]" } show_help() @@ -66,6 +66,7 @@ parse_args() { for i in "$@"; do case $i in --driver=*) DRIVER=${i#*=} ;; + --profile=*) export PROFILE=${i#*=} ;; --target=*) TARGET=${i#*=} ;; --app=*) APP=${i#*=} ;; --clusters=*) CONFIGS=$(add_option "$CONFIGS" "-DVX_CFG_NUM_CLUSTERS=${i#*=}") ;; diff --git a/ci/gen_config.py b/ci/gen_config.py index 1dd4c83ced..ea3436c49d 100755 --- a/ci/gen_config.py +++ b/ci/gen_config.py @@ -39,6 +39,9 @@ definitions that can be overridden from -D flags. - Resolved mode (-r/--resolved): fully evaluates expressions and emits resolved constants (always enabled for cflags). + - Hardware profiles (--profile): a named machine in profiles/.json layered + between the TOML defaults and explicit -D flags. --merge-configs prints the + caller's CONFIGS with the profile folded in, one -D per macro. """ from __future__ import annotations @@ -46,6 +49,7 @@ import argparse import ast import copy +import json import os import re import shlex @@ -468,6 +472,124 @@ def set_enum(name: str, val: Any) -> None: return overrides, explicit +# ----------------------------- +# Hardware profiles +# ----------------------------- + +PROFILE_SECTIONS = ("core", "chip", "platform") +PROFILE_FIELDS = {"name", "extends", "description", "reference", "sources"} | set(PROFILE_SECTIONS) + + +@dataclass +class Profile: + name: str + chain: List[str] # base first + values: Dict[str, Any] # VX_CFG_* knob -> literal value + sections: Dict[str, Set[str]] # section -> knobs it sets + reference: Dict[str, Any] + sources: Any + + +def _profile_path(name: str, profiles_dir: str) -> str: + if name.endswith(".json") and os.path.isfile(name): + return name + return os.path.join(profiles_dir, f"{name}.json") + + +def load_profile(name: str, profiles_dir: str, _seen: Tuple[str, ...] = ()) -> Profile: + if name in _seen: + raise ValueError(f"Profile inheritance cycle: {' -> '.join(_seen + (name,))}") + path = _profile_path(name, profiles_dir) + if not os.path.isfile(path): + raise ValueError(f"Unknown profile '{name}' (no {path})") + with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + unknown = set(data.keys()) - PROFILE_FIELDS + if unknown: + raise ValueError(f"Profile '{name}': unknown field(s) {sorted(unknown)} (allowed: {sorted(PROFILE_FIELDS)})") + + if "extends" in data: + base = load_profile(data["extends"], profiles_dir, _seen + (name,)) + else: + base = Profile(name="", chain=[], values={}, sections={s: set() for s in PROFILE_SECTIONS}, + reference={}, sources=None) + + values = dict(base.values) + sections = {s: set(base.sections[s]) for s in PROFILE_SECTIONS} + for sec in PROFILE_SECTIONS: + for k, v in data.get(sec, {}).items(): + for other in PROFILE_SECTIONS: + sections[other].discard(k) + values[k] = v + sections[sec].add(k) + reference = dict(base.reference) + reference.update(data.get("reference", {})) + sources = data.get("sources", base.sources) + return Profile(name=data.get("name", name), chain=base.chain + [name], values=values, + sections=sections, reference=reference, sources=sources) + + +def validate_profile(prof: Profile, toml_defs: Dict[str, Any], enums: Dict[str, EnumSpec], + read_only: Set[str]) -> None: + if not prof.sources: + raise ValueError(f"Profile '{prof.name}': 'sources' is required (where every value comes from)") + for k, v in prof.values.items(): + if k in read_only: + raise ValueError(f"Profile '{prof.name}': {k} is a read-only [[param]]/[[builtin]] symbol") + if k not in toml_defs and k not in enums: + raise ValueError(f"Profile '{prof.name}': unknown knob {k} (not in VX_config.toml)") + if not _has_public_scope(k): + raise ValueError(f"Profile '{prof.name}': {k} is a private TOML local") + if k in enums: + if v not in enums[k].values: + raise ValueError(f"Profile '{prof.name}': invalid {k}={v!r} (allowed: {enums[k].values})") + elif k.endswith("_ENABLE"): + if not isinstance(v, bool): + raise ValueError(f"Profile '{prof.name}': {k} must be true or false") + elif isinstance(v, bool) or not isinstance(v, (int, str)): + raise ValueError(f"Profile '{prof.name}': {k} must be an integer or a string literal") + elif isinstance(v, str) and _is_expr_string(v): + raise ValueError(f"Profile '{prof.name}': {k} must be a literal; expressions belong in VX_config.toml") + + +def profile_defines(prof: Profile) -> List[str]: + toks: List[str] = [f"-DVX_PROFILE={prof.name}"] + for k in sorted(prof.values): + v = prof.values[k] + if isinstance(v, bool): + toks.append(f"-D{k}" if v else f"-D{k[:-len('_ENABLE')]}_DISABLE") + else: + toks.append(f"-D{k}={v}") + return toks + + +def _canonical_define(tok: str, enums: Dict[str, EnumSpec]) -> Optional[str]: + # The knob a -D token sets, so X_DISABLE and X_ENABLE, or X_ and X=, + # are recognised as the same macro. + defs = _parse_defines([tok]) + if not defs: + return None + d = defs[0] + if d.name.endswith("_DISABLE"): + return d.name[:-len("_DISABLE")] + "_ENABLE" + if d.name in enums: + return d.name + if d.value is None and "_" in d.name: + base, _ = d.name.rsplit("_", 1) + if base in enums: + return base + return d.name + + +def merge_configs(prof: Profile, cflags: str, enums: Dict[str, EnumSpec]) -> str: + explicit = shlex.split(cflags) if cflags else [] + given = {_canonical_define(t, enums) for t in explicit} + given.discard(None) + merged = [t for t in profile_defines(prof) if _canonical_define(t, enums) not in given] + # The caller's flags pass through verbatim, quoting included. + return " ".join(merged + ([cflags.strip()] if cflags.strip() else [])) + + # ----------------------------- # Dialect helpers # ----------------------------- @@ -1489,6 +1611,11 @@ def main(argv: List[str]) -> int: ap.add_argument("--output", "-o", default=None) ap.add_argument("--resolved", "-r", action="store_true", help="Emit resolved constants for cpp/verilog (cflags is always resolved)") + ap.add_argument("--profile", default="", + help="hardware profile: profiles/.json next to the config, or a .json path") + ap.add_argument("--profiles-dir", default=None, help="profile directory (default: /profiles)") + ap.add_argument("--merge-configs", action="store_true", + help="print --cflags with the profile folded in (explicit -D flags win) and exit") args, unknown = ap.parse_known_args(argv) hex_meta = _scan_hex_literals(args.config) @@ -1501,6 +1628,26 @@ def main(argv: List[str]) -> int: toml_defs, layout = _flatten_with_layout(toml_data) + prof: Optional[Profile] = None + if args.profile and args.profile != "none": + profiles_dir = args.profiles_dir or os.path.join(os.path.dirname(os.path.abspath(args.config)), "profiles") + prof = load_profile(args.profile, profiles_dir) + validate_profile(prof, toml_defs, enums, params_set | set(builtins.keys())) + + if args.merge_configs: + out = merge_configs(prof, args.cflags, enums) if prof else args.cflags + sys.stdout.write(out.strip() + "\n") + return 0 + + if prof is not None: + if args.format == "cflags" or args.resolved: + # Resolved outputs see the profile as the lowest-priority overrides. + args.cflags = merge_configs(prof, args.cflags, enums) + else: + # Unresolved headers carry the profile as their overridable defaults. + for k, v in prof.values.items(): + toml_defs[k] = v + defs: List[Define] = [] defs += _parse_defines_from_cflags(args.cflags) defs += _parse_defines(unknown) diff --git a/config.mk.in b/config.mk.in index 8fe66bbd0d..1dc8f4efbd 100644 --- a/config.mk.in +++ b/config.mk.in @@ -55,3 +55,14 @@ SIM_COMMON_DIR ?= $(VORTEX_HOME)/sim/common # V80 runtime install prefix; only consulted by the aved backend when it # targets real hardware or the vendor simulator. VRT_HOME ?= /usr/local + +# Hardware profile (profiles/.json), folded into CONFIGS once here so every +# flow that includes config.mk builds the same machine. Explicit -D flags in +# CONFIGS win over the profile. +PROFILE ?= @PROFILE@ +ifneq ($(strip $(PROFILE)),) +override CONFIGS := $(shell python3 $(VORTEX_HOME)/ci/gen_config.py --config=$(VORTEX_HOME)/VX_config.toml --profile=$(PROFILE) --merge-configs --cflags='$(CONFIGS)') +ifneq ($(.SHELLSTATUS),0) +$(error invalid hardware profile '$(PROFILE)') +endif +endif diff --git a/configure b/configure index f9340d12ee..c5e1e21147 100755 --- a/configure +++ b/configure @@ -77,7 +77,7 @@ copy_files() { continue fi mkdir -p "$dest_dir" - sed "s|@VORTEX_HOME@|$SOURCE_DIR|g; s|@XLEN@|$XLEN|g; s|@TOOLDIR@|$TOOLDIR|g; s|@OSVERSION@|$OSVERSION|g; s|@INSTALLDIR@|$PREFIX|g; s|@BUILDDIR@|$CURRENT_DIR|g; s|@TOOLCHAIN_REV@|$TOOLCHAIN_REV|g; s|@VORTEX_VERSION@|$VORTEX_VERSION|g; s|@GEM5_REV@|$GEM5_REV|g" "$file" > "$dest_file" + sed "s|@VORTEX_HOME@|$SOURCE_DIR|g; s|@XLEN@|$XLEN|g; s|@TOOLDIR@|$TOOLDIR|g; s|@OSVERSION@|$OSVERSION|g; s|@INSTALLDIR@|$PREFIX|g; s|@BUILDDIR@|$CURRENT_DIR|g; s|@TOOLCHAIN_REV@|$TOOLCHAIN_REV|g; s|@VORTEX_VERSION@|$VORTEX_VERSION|g; s|@GEM5_REV@|$GEM5_REV|g; s|@PROFILE@|$PROFILE|g" "$file" > "$dest_file" # apply permissions to bash scripts read -r firstline < "$dest_file" if [[ "$firstline" =~ ^#!.*bash ]]; then @@ -140,6 +140,7 @@ default_xlen=32 default_tooldir=$HOME/tools default_osversion=$(detect_osversion) default_prefix=$CURRENT_DIR/install +default_profile= # load default configuration parameters from existing config.mk if [ -f "config.mk" ]; then @@ -152,6 +153,7 @@ if [ -f "config.mk" ]; then TOOLDIR\ ?*) default_tooldir=${value//\?=/} ;; OSVERSION\ ?*) default_osversion=${value//\?=/} ;; PREFIX\ ?*) default_prefix=${value//\?=/} ;; + PROFILE\ ?*) default_profile=${value//\?=/} ;; esac done < config.mk fi @@ -161,6 +163,7 @@ XLEN=${XLEN:=$default_xlen} TOOLDIR=${TOOLDIR:=$default_tooldir} OSVERSION=${OSVERSION:=$default_osversion} PREFIX=${PREFIX:=$default_prefix} +PROFILE=${PROFILE-$default_profile} # parse command line arguments usage() { @@ -169,6 +172,7 @@ usage() { echo " --tooldir= Set the TOOLDIR path (default: $HOME/tools)" echo " --osversion= Set the OS Version (default: $(detect_osversion))" echo " --prefix=, --installdir= Set installation directory" + echo " --profile= Default hardware profile, profiles/.json (default: none)" exit 1 } while [[ "$#" -gt 0 ]]; do @@ -177,12 +181,19 @@ while [[ "$#" -gt 0 ]]; do --tooldir=*) TOOLDIR="${1#*=}" ;; --osversion=*) OSVERSION="${1#*=}" ;; --prefix=*|--installdir=*) PREFIX="${1#*=}" ;; + --profile=*) PROFILE="${1#*=}" ;; -h|--help) usage ;; *) echo "Unknown parameter passed: $1"; usage ;; esac shift done +# check the profile before anything is generated from it +if [ -n "$PROFILE" ] && [ ! -f "$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/profiles/$PROFILE.json" ]; then + echo "Error: unknown profile '$PROFILE' (no profiles/$PROFILE.json)" + exit -1 +fi + # check OS if [ "$OSVERSION" == "unsupported" ]; then echo "Error: Unsupported OS." @@ -202,7 +213,7 @@ source "$SOURCE_DIR/VERSION" # stamp file tracks config parameter changes; touched only when params change CONFIG_STAMP="$CURRENT_DIR/.config.stamp" -CONFIG_SIG="XLEN=$XLEN TOOLDIR=$TOOLDIR OSVERSION=$OSVERSION PREFIX=$PREFIX SOURCE_DIR=$SOURCE_DIR BUILDDIR=$CURRENT_DIR" +CONFIG_SIG="XLEN=$XLEN TOOLDIR=$TOOLDIR OSVERSION=$OSVERSION PREFIX=$PREFIX PROFILE=$PROFILE SOURCE_DIR=$SOURCE_DIR BUILDDIR=$CURRENT_DIR" if [ ! -f "$CONFIG_STAMP" ] || [ "$(cat "$CONFIG_STAMP" 2>/dev/null)" != "$CONFIG_SIG" ]; then echo "$CONFIG_SIG" > "$CONFIG_STAMP" fi @@ -238,6 +249,7 @@ for toml in "$SOURCE_DIR"/*.toml; do extra="" case "$name" in VX_types) extra="--resolved" ;; + VX_config) [ -n "$PROFILE" ] && extra="--profile=$PROFILE" ;; esac gen_header "$toml" "$CURRENT_DIR/hw/$name.vh" verilog "$extra" gen_header "$toml" "$CURRENT_DIR/sw/$name.h" cpp "$extra" diff --git a/profiles/rtx_5080.json b/profiles/rtx_5080.json new file mode 100644 index 0000000000..8dc5906364 --- /dev/null +++ b/profiles/rtx_5080.json @@ -0,0 +1,45 @@ +{ + "name": "rtx_5080", + "description": "NVIDIA GeForce RTX 5080 (GB203, Blackwell, sm_120) at its 2295 MHz base clock: one Vortex core per SM.", + "core": { + "VX_CFG_NUM_WARPS": 48, + "VX_CFG_NUM_THREADS": 32, + "VX_CFG_ISSUE_WIDTH": 4, + "VX_CFG_DCACHE_SIZE": 131072, + "VX_CFG_DCACHE_MSHR_SIZE": 24 + }, + "chip": { + "VX_CFG_NUM_CLUSTERS": 1, + "VX_CFG_NUM_CORES": 84, + "VX_CFG_L2_ENABLE": true, + "VX_CFG_L2_SIZE": 67108864, + "VX_CFG_L2_NUM_WAYS": 16, + "VX_CFG_L2_NUM_BANKS": 32, + "VX_CFG_L2_MSHR_SIZE": 192 + }, + "platform": { + "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 32, + "VX_CFG_PLATFORM_CLOCK_RATE": 2295, + "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 960000 + }, + "reference": { + "clock_mhz": 2295, + "l1_hit_cycles": 44.1, + "l2_hit_cycles": 354.4, + "dram_cycles": 802, + "dram_peak_bytes_per_cycle": 423, + "stream_efficiency": { "4MiB": 0.795, "16MiB": 0.894, "64MiB": 0.919 }, + "sgemm_cycles": { "128": 24193, "256": 45476, "512": 270396, "1024": 1845287 }, + "stream_cycles": { "4MiB": 25569, "16MiB": 90741, "64MiB": 427878 } + }, + "sources": { + "NUM_CORES, NUM_WARPS, NUM_THREADS, ISSUE_WIDTH": "84 SMs, 48 resident warps/SM, 4 schedulers x 32 lanes (NVIDIA RTX Blackwell GPU Architecture whitepaper, App. B; cudaGetDeviceProperties on the user's RTX 5080)", + "DCACHE_SIZE": "128 KB L1/shared per SM (whitepaper App. B)", + "L2_SIZE, L2_NUM_WAYS": "64 MB L2 (whitepaper App. B); 16 ways as in GPGPU-Sim's Ampere/Hopper L2 configs", + "L2_NUM_BANKS": "32 banks as the nearest power of two to the 48 L2 slices measured by arXiv 2507.10789", + "DCACHE_MSHR_SIZE, L2_MSHR_SIZE": "384 L1 MSHR entries per SM (24 x 16 banks) and 192 per L2 slice, GPGPU-Sim SM86_RTX3070 (-gpgpu_cache:dl1 A:384:48, dl2 A:192:4)", + "PLATFORM_MEMORY_NUM_BANKS": "8 x 32-bit GDDR7 devices = 32 independent channels (whitepaper App. B)", + "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "2295 MHz base clock, 960 GB/s (whitepaper App. B)", + "reference": "Nsight Compute on the user's RTX 5080, clocks locked to base, cold L2, 2026-10-05 (docs/reports/simx_vs_rtx5080_mlp_report.md)" + } +} From 7b91e0e94a367cbf99a9b8d0ce204d14eae53cd4 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 10 Oct 2026 22:17:16 -0700 Subject: [PATCH 011/124] build: restore the zero-warnings policy everywhere Warnings had become non-fatal again: a June RTU change re-added Verilator -Wno-fatal to rtlsim after it had been removed, the AFU simulators copied it, and several C/C++ trees (OpenCL/HIP host code, unit tests, kernel library) never had -Werror. Warnings then accumulated unseen. Every C/C++ build (simx, rtlsim, xrtsim, opaesim, avedsim, runtime, kernel library, hw/sw unit tests, OpenCL/HIP/Vulkan apps) is now -Werror and Verilator warnings are fatal, with the code fixed rather than silenced: - RTL: VX_find_first, VX_onehot_encoder (MODEL1) and VX_pipe_buffer rewritten loop-free so Verilator -O0 does not report UNOPTFLAT (each proven equivalent to the original with an sv2v+yosys miter, with negative controls); width fixes in the tex bus arbiter, RTU scheduler config, PLRU replacement, the no-F CSR data path and the TFR MX scale; unused cluster/AFU ports removed or tied off; the early-Z y-pitch add widened explicitly (WIDTHEXPAND), and the RTU scheduler's issue block reads the memory grant instead of mem_fire, which is derived from the block's own mem_issue output (UNOPTFLAT). - Host code: 21 OpenCL apps, HIP, Vulkan, the kernel library and the DXA/FMA testbenches; -Wno-maybe-uninitialized/-Wno-array-bounds dropped; MAKEFLAGS cleared around verilator --build (jobserver warning). - vx_spawn.c printed uint32_t with %d, which the vx_printf format attribute rejects under GCC's newlib (uint32_t is unsigned long there). - The riscv-tests benchmark patch cast the unsuffixed COUT address straight to a pointer (-Wint-to-pointer-cast on RV64), and the riscv-tests stamp did not depend on the patch, so an edited patch was never re-applied. Remaining exceptions compile code that is not ours: the yosys gate-level netlist sims, the cln28hpc vendor macro testbench, and the FireSim driver, whose upstream bridge and platform objects share our CXXFLAGS. Verified: from-clean build of every group at XLEN 32 and 64 with 0 warnings; all 252 CI Verilator commands verilate with 0 warnings; the CI matrix cells run so far (perf_gate, amo, cache, config1 on rtlsim/simx) build warning-free. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/VX_cluster.sv | 2 - hw/rtl/Vortex.sv | 1 - hw/rtl/afu/opae/vortex_afu.sv | 3 + hw/rtl/cache/VX_cache_repl.sv | 1 + hw/rtl/core/VX_csr_data.sv | 2 +- hw/rtl/fpu/VX_fma_unit_rtl.sv | 6 +- hw/rtl/libs/VX_find_first.sv | 42 ++-- hw/rtl/libs/VX_onehot_encoder.sv | 49 +++-- hw/rtl/libs/VX_pipe_buffer.sv | 6 +- hw/rtl/raster/VX_raster_earlyz.sv | 2 +- hw/rtl/rtu/VX_rtu_scheduler.sv | 31 +-- hw/rtl/tcu/tfr/VX_tcu_tfr_shared_mul.sv | 4 + hw/rtl/tex/VX_tex_bus_arb.sv | 27 ++- hw/unittest/common.mk | 8 +- hw/unittest/dxa_core/main.cpp | 6 +- hw/unittest/fma_unit/main.cpp | 3 + miscs/patches/riscv-benchmarks.patch | 2 +- sim/avedsim/Makefile | 8 +- sim/opaesim/Makefile | 10 +- sim/rtlsim/Makefile | 14 +- sim/simx/Makefile | 2 +- sim/xrtsim/Makefile | 10 +- sw/kernel/Makefile | 1 + sw/kernel/src/vx_spawn.c | 7 +- sw/kernel/src/vx_syscalls.c | 45 +++- sw/runtime/simx/Makefile | 2 +- tests/hip/common.mk | 2 +- tests/hip/sgemm/main.cpp | 4 +- tests/hip/vecadd/main.cpp | 4 +- tests/opencl/bfs/CLHelper.h | 15 +- tests/opencl/bfs/main.cc | 38 ++-- tests/opencl/bfs/util.h | 2 + .../blackscholes/oclBlackScholes_launcher.cpp | 5 +- tests/opencl/blackscholes/shrUtils.cpp | 57 ++--- tests/opencl/cfd/main.cc | 12 +- tests/opencl/common.mk | 9 +- tests/opencl/conv3/main.cc | 2 +- tests/opencl/dotproduct/main.cc | 1 - tests/opencl/dotproduct/shrUtils.cpp | 57 ++--- tests/opencl/guassian/clutils.cpp | 30 +-- tests/opencl/guassian/main.cc | 20 +- tests/opencl/image_get/main.cc | 9 +- tests/opencl/image_int/main.cc | 17 +- tests/opencl/image_linear/main.cc | 11 +- tests/opencl/kmeans/kmeans_clustering.c | 6 +- tests/opencl/kmeans/main.cc | 17 +- tests/opencl/kmeans/read_input.c | 16 +- tests/opencl/kmeans/rmse.c | 6 +- tests/opencl/lbm/args.c | 5 +- tests/opencl/lbm/lbm.c | 21 +- tests/opencl/lbm/ocl.c | 2 +- tests/opencl/lbm/parboil_opencl.c | 204 ++---------------- tests/opencl/myocyte/myocyte_model.h | 4 +- tests/opencl/nearn/clutils.cpp | 31 +-- tests/opencl/nearn/main.cc | 10 +- tests/opencl/oclprintf/main.cc | 2 +- tests/opencl/psort/main.cc | 2 +- tests/opencl/saxpy/main.cc | 6 +- tests/opencl/sfilter/main.cc | 6 +- tests/opencl/spmv/args.c | 5 +- tests/opencl/spmv/convert_dataset.c | 17 +- tests/opencl/spmv/file.c | 43 +++- tests/opencl/spmv/main.cc | 15 +- tests/opencl/spmv/mmio.c | 12 +- tests/opencl/spmv/ocl.c | 2 +- tests/opencl/spmv/parboil_opencl.c | 81 ++----- tests/opencl/srad/main.cc | 4 +- tests/opencl/stencil/args.c | 5 +- tests/opencl/stencil/file.c | 10 +- tests/opencl/stencil/main.cc | 18 +- tests/opencl/stencil/ocl.c | 2 +- tests/opencl/stencil/parboil_opencl.c | 80 ++----- tests/opencl/transpose/main.cc | 5 +- tests/opencl/transpose/shrUtils.cpp | 57 ++--- tests/riscv/common.mk | 5 +- tests/unittest/common.mk | 2 +- tests/vulkan/common.mk | 2 +- tests/vulkan/subgroup_vs/main.c | 2 +- 78 files changed, 593 insertions(+), 701 deletions(-) diff --git a/hw/rtl/VX_cluster.sv b/hw/rtl/VX_cluster.sv index fa68f07398..86fc979410 100644 --- a/hw/rtl/VX_cluster.sv +++ b/hw/rtl/VX_cluster.sv @@ -43,8 +43,6 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; `endif `ifdef VX_CFG_VM_ENABLE - // Device MMU sideband from the top DCR surface. - input wire [`VX_CFG_XLEN-1:0] mmu_satp, // Device-level walker: the L2 TLB's miss bus, exported to the device. VX_tlb_bus_if.master dev_ptw_if, // Walker PTE fetches, entering as an L2 client when this cluster's L2 diff --git a/hw/rtl/Vortex.sv b/hw/rtl/Vortex.sv index 762e1e0200..2b4b49df89 100644 --- a/hw/rtl/Vortex.sv +++ b/hw/rtl/Vortex.sv @@ -346,7 +346,6 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( `endif `ifdef VX_CFG_VM_ENABLE - .mmu_satp (mmu_satp), .dev_ptw_if (per_cluster_dev_ptw_if[cluster_id]), .dev_ptw_mem_if (per_cluster_ptw_mem_if[cluster_id]), .mmu_flush_req (mmu_flush_req), diff --git a/hw/rtl/afu/opae/vortex_afu.sv b/hw/rtl/afu/opae/vortex_afu.sv index e9ba15f39e..61ad02a8e4 100644 --- a/hw/rtl/afu/opae/vortex_afu.sv +++ b/hw/rtl/afu/opae/vortex_afu.sv @@ -356,6 +356,9 @@ module vortex_afu import ccip_if_pkg::*; import local_mem_cfg_pkg::*; import VX_ .axil_s (cp_axil), .axi_host (cp_axi_host), .axi_dev (cp_axi_dev), + // CCI-P has no AXI host port, so there are no drain counters to report. + .dbg_host_w_counts ('0), + .dbg_host_r_counts ('0), .gpu_if (cp_gpu_if), .irq (cp_interrupt) ); diff --git a/hw/rtl/cache/VX_cache_repl.sv b/hw/rtl/cache/VX_cache_repl.sv index 5aa92f7d5a..b639ac33ac 100644 --- a/hw/rtl/cache/VX_cache_repl.sv +++ b/hw/rtl/cache/VX_cache_repl.sv @@ -108,6 +108,7 @@ module VX_cache_repl #( if (REPL_POLICY == `CS_REPL_PLRU) begin : g_plru // Pseudo Least Recently Used replacement policy `UNUSED_VAR (repl_line) // read uses repl_line_n (look-ahead) + `UNUSED_VAR (repl_valid) // state advances on lookup hits only localparam LRU_WIDTH = `UP(NUM_WAYS-1); wire [LRU_WIDTH-1:0] plru_rdata; diff --git a/hw/rtl/core/VX_csr_data.sv b/hw/rtl/core/VX_csr_data.sv index 51efaa9762..f09c17bb7e 100644 --- a/hw/rtl/core/VX_csr_data.sv +++ b/hw/rtl/core/VX_csr_data.sv @@ -338,7 +338,7 @@ import VX_fpu_pkg::*; end end `else - `UNUSED_VAR (write_sel[WR_FFLAGS +: 3]) + `UNUSED_VAR ({write_sel[WR_FCSR], write_sel[WR_FRM], write_sel[WR_FFLAGS]}) `endif `ifdef VX_CFG_VM_ENABLE diff --git a/hw/rtl/fpu/VX_fma_unit_rtl.sv b/hw/rtl/fpu/VX_fma_unit_rtl.sv index cddf2b4889..02b986d826 100644 --- a/hw/rtl/fpu/VX_fma_unit_rtl.sv +++ b/hw/rtl/fpu/VX_fma_unit_rtl.sv @@ -115,9 +115,9 @@ module VX_fma_unit_rtl import VX_gpu_pkg::*, VX_fpu_pkg::*; #( // DAZ: flush a subnormal operand to signed zero when subnormals are disabled. localparam DAZ = (SUBNORM_ENABLE == 0); - function automatic [FLOAT_BITS-1:0] daz_f(input [FLOAT_BITS-1:0] v); - daz_f = (DAZ && (v[FLOAT_BITS-2:MAN_BITS] == '0) && (|v[MAN_BITS-1:0])) - ? {v[FLOAT_BITS-1], {(FLOAT_BITS-1){1'b0}}} : v; + function automatic [FLOAT_BITS-1:0] daz_f(input [FLOAT_BITS-1:0] val); + daz_f = (DAZ && (val[FLOAT_BITS-2:MAN_BITS] == '0) && (|val[MAN_BITS-1:0])) + ? {val[FLOAT_BITS-1], {(FLOAT_BITS-1){1'b0}}} : val; endfunction wire [FLOAT_BITS-1:0] op_a = daz_f(dataa); diff --git a/hw/rtl/libs/VX_find_first.sv b/hw/rtl/libs/VX_find_first.sv index b7fc3e2596..92ad0f2d20 100644 --- a/hw/rtl/libs/VX_find_first.sv +++ b/hw/rtl/libs/VX_find_first.sv @@ -28,32 +28,30 @@ module VX_find_first #( localparam TL = (1 << LOGN) - 1; localparam TN = (1 << (LOGN+1)) - 1; - wire s_n [TN]; - wire [DATAW-1:0] d_n [TN]; - - for (genvar i = 0; i < N; ++i) begin : g_fill - assign s_n[TL+i] = REVERSE ? valid_in[N-1-i] : valid_in[i]; - assign d_n[TL+i] = REVERSE ? data_in[N-1-i] : data_in[i]; - end - - if (TL < (TN-N)) begin : g_padding - for (genvar i = TL+N; i < TN; ++i) begin : g_i - assign s_n[i] = 0; - assign d_n[i] = '0; + reg valid_r; + reg [DATAW-1:0] data_r; + + // Binary tree: node k reduces children 2k+1 (first) and 2k+2; leaves start at TL. + // The tree is local to one block so no node variable feeds itself. + always @(*) begin : g_scan + reg [TN-1:0] s_n; + reg [TN*DATAW-1:0] d_n; + s_n = '0; + d_n = '0; + for (integer i = 0; i < N; ++i) begin + s_n[TL+i] = REVERSE ? valid_in[N-1-i] : valid_in[i]; + d_n[(TL+i)*DATAW +: DATAW] = REVERSE ? data_in[N-1-i] : data_in[i]; end - end - - for (genvar j = 0; j < LOGN; ++j) begin : g_scan - localparam I = 1 << j; - for (genvar i = 0; i < I; ++i) begin : g_i - localparam K = I+i-1; - assign s_n[K] = s_n[2*K+2] | s_n[2*K+1]; - assign d_n[K] = s_n[2*K+1] ? d_n[2*K+1] : d_n[2*K+2]; + for (integer k = TL-1; k >= 0; --k) begin + s_n[k] = s_n[2*k+2] | s_n[2*k+1]; + d_n[k*DATAW +: DATAW] = s_n[2*k+1] ? d_n[(2*k+1)*DATAW +: DATAW] : d_n[(2*k+2)*DATAW +: DATAW]; end + valid_r = s_n[0]; + data_r = d_n[0 +: DATAW]; end - assign valid_out = s_n[0]; - assign data_out = d_n[0]; + assign valid_out = valid_r; + assign data_out = data_r; endmodule `TRACING_ON diff --git a/hw/rtl/libs/VX_onehot_encoder.sv b/hw/rtl/libs/VX_onehot_encoder.sv index 273a9cfe26..2bbf58d377 100644 --- a/hw/rtl/libs/VX_onehot_encoder.sv +++ b/hw/rtl/libs/VX_onehot_encoder.sv @@ -39,31 +39,38 @@ module VX_onehot_encoder #( end else if (MODEL == 1) begin : g_model1 localparam M = 1 << LN; - wire [M-1:0] addr [LN]; - wire [M-1:0] v [LN+1]; - - // base case, also handle padding for non-power of two inputs - assign v[0] = REVERSE ? (M'(data_in) << (M - N)) : M'(data_in); - - for (genvar lvl = 1; lvl < (LN+1); ++lvl) begin : g_scan_l - localparam SN = 1 << (LN - lvl); - localparam SI = M / SN; - for (genvar s = 0; s < SN; ++s) begin : g_scan_s - wire [1:0] vs = {v[lvl-1][s*SI+(SI>>1)], v[lvl-1][s*SI]}; - assign v[lvl][s*SI] = (| vs); - if (lvl == 1) begin : g_lvl_1 - assign addr[lvl-1][s*SI +: lvl] = vs[!REVERSE]; - end else begin : g_lvl_n - assign addr[lvl-1][s*SI +: lvl] = { - vs[!REVERSE], - addr[lvl-2][s*SI +: lvl-1] | addr[lvl-2][s*SI+(SI>>1) +: lvl-1] - }; + + reg [LN-1:0] addr_r; + reg valid_r; + + // Level lvl merges pairs of level lvl-1 groups of size 2^(lvl-1); each + // level has its own slice of v/addr, local to one block, so no + // variable feeds itself. + always @(*) begin : g_scan + reg [(LN+1)*M-1:0] v; + reg [LN*M-1:0] addr; + reg [1:0] vs; + v = '0; + addr = '0; + // base case, also handle padding for non-power of two inputs + v[0 +: M] = REVERSE ? (M'(data_in) << (M - N)) : M'(data_in); + for (integer lvl = 1; lvl <= LN; ++lvl) begin + for (integer s = 0; s < (1 << (LN - lvl)); ++s) begin + vs = {v[(lvl-1)*M + s*(1 << lvl) + (1 << (lvl-1))], v[(lvl-1)*M + s*(1 << lvl)]}; + v[lvl*M + s*(1 << lvl)] = (| vs); + addr[(lvl-1)*M + s*(1 << lvl) + (lvl-1)] = vs[!REVERSE]; + for (integer b = 0; b < lvl-1; ++b) begin + addr[(lvl-1)*M + s*(1 << lvl) + b] = addr[(lvl-2)*M + s*(1 << lvl) + b] + | addr[(lvl-2)*M + s*(1 << lvl) + (1 << (lvl-1)) + b]; + end end end + addr_r = addr[(LN-1)*M +: LN]; + valid_r = v[LN*M]; end - assign data_out = addr[LN-1][LN-1:0]; - assign valid_out = v[LN][0]; + assign data_out = addr_r; + assign valid_out = valid_r; end else if (MODEL == 2 && REVERSE == 0) begin : g_model2 diff --git a/hw/rtl/libs/VX_pipe_buffer.sv b/hw/rtl/libs/VX_pipe_buffer.sv index 0847c0bbe9..e72e8d2b35 100644 --- a/hw/rtl/libs/VX_pipe_buffer.sv +++ b/hw/rtl/libs/VX_pipe_buffer.sv @@ -40,7 +40,7 @@ module VX_pipe_buffer #( assign data_out = data_in; end else begin : g_register wire [DEPTH:0] valid; - wire ready [DEPTH+1]; + wire [DEPTH-1:0] ready; wire [DEPTH:0][DATAW-1:0] data; assign valid[0] = valid_in; @@ -48,7 +48,8 @@ module VX_pipe_buffer #( assign ready_in = ready[0]; for (genvar i = 0; i < DEPTH; ++i) begin : g_pipe_regs - assign ready[i] = (ready[i+1] || ~valid[i+1]); + // Stage i advances when any later stage has a hole or the output drains. + assign ready[i] = ready_out || ~(& valid[DEPTH:i+1]); VX_pipe_register #( .DATAW (1 + DATAW), .RESETW (1 + RESETW) @@ -63,7 +64,6 @@ module VX_pipe_buffer #( assign valid_out = valid[DEPTH]; assign data_out = data[DEPTH]; - assign ready[DEPTH] = ready_out; end endmodule diff --git a/hw/rtl/raster/VX_raster_earlyz.sv b/hw/rtl/raster/VX_raster_earlyz.sv index 59ae4e853d..2cb0529650 100644 --- a/hw/rtl/raster/VX_raster_earlyz.sv +++ b/hw/rtl/raster/VX_raster_earlyz.sv @@ -141,7 +141,7 @@ module VX_raster_earlyz import VX_gpu_pkg::*; import VX_raster_pkg::*; import VX assign pix_cand_n[p] = zbits[31] ? 24'd0 : (|zbits[30:24]) ? 24'(OM_DEPTH_MASK) : zbits[23:0]; - wire [31:0] y_pitch = quad_ypitch[q] + (i[1] ? dcrs.zbuf_pitch : 32'd0); + wire [31:0] y_pitch = quad_ypitch[q] + (i[1] ? 32'(dcrs.zbuf_pitch) : 32'd0); wire [W_ADDR_BITS-1:0] baddr = {dcrs.zbuf_addr, 4'b0} + W_ADDR_BITS'(px); assign pix_waddr_n[p] = baddr + W_ADDR_BITS'(y_pitch[31:2]); `UNUSED_VAR (y_pitch) diff --git a/hw/rtl/rtu/VX_rtu_scheduler.sv b/hw/rtl/rtu/VX_rtu_scheduler.sv index 8477773ae5..46f1676251 100644 --- a/hw/rtl/rtu/VX_rtu_scheduler.sv +++ b/hw/rtl/rtu/VX_rtu_scheduler.sv @@ -119,8 +119,8 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( localparam LINES = FLAT ? RTU_FLAT_LINES : RTU_NODE_LINES; localparam LB = `CLOG2(LINES + 1); localparam BUF_BITS = LINES * LINE_BITS; - localparam IDXW = `CLOG2(`UP(RTU_BVH_WIDTH)); localparam NODE_W = `UP(RTU_BVH_WIDTH); + localparam IDXW = `LOG2UP(NODE_W); localparam ADDRW = `VX_CFG_MEM_ADDR_WIDTH; `STATIC_ASSERT(((NUM_LANES * NUM_SLOTS) == NUM_CTX), @@ -461,8 +461,6 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( wire recip_obj; wire recip_valid_out, recip_last_out; wire [CTX_TAG_W-1:0] recip_tag_out; - wire [1:0] recip_axis_out; - wire [31:0] recip_result; wire [95:0] recip_rdata; // ── short stack (BVH only) ──────────────────────────────────────── @@ -792,6 +790,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( assign ins_shift[oc] = ~ins_le[oc-1]; end end else begin : g_no_ins_slots + `UNUSED_VAR ({coll_ordt_sel, coll_ordcnt_sel}) assign {ins_le, ins_here, ins_shift} = '0; end wire coll_pushable = box_hit && (box_childoff != 32'd0); @@ -847,6 +846,8 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( wire [1:0] recip_axis; wire [31:0] recip_din = recip_obj ? word_q.obj_d[recip_axis] : ray_q.dir[recip_axis]; if (!FLAT) begin : g_recip_pe + wire [1:0] recip_axis_out; + wire [31:0] recip_result; VX_rtu_recip #( .LATENCY (RTU_FDIV_LAT), .DSP_SEED (`VX_CFG_RTU_RECIP_DSP_SEED) @@ -890,8 +891,6 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( assign recip_valid_out = 1'b0; assign recip_last_out = 1'b0; assign recip_tag_out = '0; - assign recip_axis_out = '0; - assign recip_result = '0; assign recip_rdata = '0; `UNUSED_VAR ({recip_issue, recip_obj, recip_din, recip_q}) end @@ -1292,7 +1291,11 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( reg [LVLW-1:0] path_wlvl_r; reg [PATHW-1:0] path_wdata_r; - wire mem_fire = x_valid && mem_issue && mem_req_ready; + // The issue block below reads the grant only where it has just set + // mem_issue, so it uses mem_grant: reading mem_fire would feed its own + // output back into itself (a combinational loop to Verilator). + wire mem_grant = x_valid && mem_req_ready; + wire mem_fire = mem_grant && mem_issue; wire [RTU_CHILD_BITS-1:0] last_child = node.n_children - RTU_CHILD_BITS'(1); @@ -1355,7 +1358,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( end CS_HDR_REQ: begin mem_issue = 1'b1; - if (mem_fire) begin + if (mem_grant) begin word_n.cstate = CS_HDR_WAIT; end else begin wake_self = 1'b1; @@ -1400,7 +1403,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( end CS_REQ0: begin mem_issue = 1'b1; - if (mem_fire) begin + if (mem_grant) begin word_n.cstate = CS_RSP0; end else begin wake_self = 1'b1; @@ -1435,7 +1438,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( CS_REQN: begin mem_issue = 1'b1; mem_fslot = word_x.f_idx; - if (mem_fire) begin + if (mem_grant) begin word_n.cstate = CS_RSPN; end else begin wake_self = 1'b1; @@ -1598,7 +1601,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( end CS_LTRI_REQ0: begin mem_issue = 1'b1; - if (mem_fire) begin + if (mem_grant) begin word_n.f_total = tri_rec_lines; word_n.cstate = CS_LTRI_RSP0; end else begin @@ -1619,7 +1622,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( CS_LTRI_REQN: begin mem_issue = 1'b1; mem_fslot = word_x.f_idx; - if (mem_fire) begin + if (mem_grant) begin word_n.cstate = CS_LTRI_RSPN; end else begin wake_self = 1'b1; @@ -1758,7 +1761,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( end CS_INST_REQ: begin mem_issue = 1'b1; - if (mem_fire) begin + if (mem_grant) begin word_n.f_total = inst_lines; word_n.cstate = CS_INST_RSP0; end else begin @@ -1777,7 +1780,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( CS_INST_REQN: begin mem_issue = 1'b1; mem_fslot = word_x.f_idx; - if (mem_fire) begin + if (mem_grant) begin word_n.cstate = CS_INST_RSPN; end else begin wake_self = 1'b1; @@ -1862,7 +1865,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( end CS_BHDR_REQ: begin mem_issue = 1'b1; - if (mem_fire) begin + if (mem_grant) begin word_n.cstate = CS_BHDR_WAIT; end else begin wake_self = 1'b1; diff --git a/hw/rtl/tcu/tfr/VX_tcu_tfr_shared_mul.sv b/hw/rtl/tcu/tfr/VX_tcu_tfr_shared_mul.sv index dec2159349..12953c50f3 100644 --- a/hw/rtl/tcu/tfr/VX_tcu_tfr_shared_mul.sv +++ b/hw/rtl/tcu/tfr/VX_tcu_tfr_shared_mul.sv @@ -55,6 +55,10 @@ module VX_tcu_tfr_shared_mul import VX_tcu_pkg::*; #( `UNUSED_SPARAM (INSTANCE_ID) `ifndef VX_CFG_TCU_MX_ENABLE `UNUSED_PARAM (SF) +`else + if (!`VX_CFG_TCU_FP8_ENABLED && !`VX_CFG_TCU_FP4_ENABLED) begin : g_sf_unused + `UNUSED_VAR ({sf_a, sf_b}) + end `endif `UNUSED_VAR ({clk, req_id, valid_in}) diff --git a/hw/rtl/tex/VX_tex_bus_arb.sv b/hw/rtl/tex/VX_tex_bus_arb.sv index 319570f9c6..5041b81a11 100644 --- a/hw/rtl/tex/VX_tex_bus_arb.sv +++ b/hw/rtl/tex/VX_tex_bus_arb.sv @@ -32,6 +32,8 @@ module VX_tex_bus_arb import VX_gpu_pkg::*, VX_tex_pkg::*; #( ); localparam LOG_NUM_REQS = `ARB_SEL_BITS(NUM_INPUTS, NUM_OUTPUTS); + localparam SEL_COUNT = `MIN(NUM_INPUTS, NUM_OUTPUTS); + localparam SEL_W = `UP(`CLOG2(`CDIV(`MAX(NUM_INPUTS, NUM_OUTPUTS), SEL_COUNT))); localparam REQ_DATAW = TAG_WIDTH + NUM_LANES * (1 + 2 * 32 + TEX_LODF_BITS) + TEX_STAGE_BITS; localparam RSP_DATAW = TAG_WIDTH + NUM_LANES * 32; @@ -43,7 +45,7 @@ module VX_tex_bus_arb import VX_gpu_pkg::*, VX_tex_pkg::*; #( wire [NUM_OUTPUTS-1:0] req_valid_out; wire [NUM_OUTPUTS-1:0][REQ_DATAW-1:0] req_data_out; - wire [NUM_OUTPUTS-1:0][`UP(LOG_NUM_REQS)-1:0] req_sel_out; + wire [SEL_COUNT-1:0][SEL_W-1:0] req_sel_out; wire [NUM_OUTPUTS-1:0] req_ready_out; for (genvar i = 0; i < NUM_INPUTS; ++i) begin : g_req_data_in @@ -72,15 +74,20 @@ module VX_tex_bus_arb import VX_gpu_pkg::*, VX_tex_pkg::*; #( for (genvar i = 0; i < NUM_OUTPUTS; ++i) begin : g_bus_out_if wire [TAG_WIDTH-1:0] req_tag_out; - VX_bits_insert #( - .N (TAG_WIDTH), - .S (LOG_NUM_REQS), - .POS (TAG_SEL_IDX) - ) bits_insert ( - .data_in (req_tag_out), - .ins_in (req_sel_out[i]), - .data_out (bus_out_if[i].req_data.tag) - ); + if (NUM_INPUTS > NUM_OUTPUTS) begin : g_req_tag_sel_out + VX_bits_insert #( + .N (TAG_WIDTH), + .S (LOG_NUM_REQS), + .POS (TAG_SEL_IDX) + ) bits_insert ( + .data_in (req_tag_out), + .ins_in (req_sel_out[i]), + .data_out (bus_out_if[i].req_data.tag) + ); + end else begin : g_req_tag_out + `UNUSED_VAR (req_sel_out) + assign bus_out_if[i].req_data.tag = req_tag_out; + end assign bus_out_if[i].req_valid = req_valid_out[i]; assign {req_tag_out, bus_out_if[i].req_data.mask, bus_out_if[i].req_data.coords, bus_out_if[i].req_data.lod, bus_out_if[i].req_data.stage} = req_data_out[i]; assign req_ready_out[i] = bus_out_if[i].req_ready; diff --git a/hw/unittest/common.mk b/hw/unittest/common.mk index 3729fadce2..25ca8e25ee 100644 --- a/hw/unittest/common.mk +++ b/hw/unittest/common.mk @@ -3,8 +3,8 @@ DESTDIR ?= . CONFIGS += PARAMS += -CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Wno-array-bounds -CXXFLAGS += -fPIC -Wno-maybe-uninitialized +CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Werror +CXXFLAGS += -fPIC CXXFLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) CXXFLAGS += $(CONFIGS) @@ -67,8 +67,10 @@ endif all: $(DESTDIR)/$(PROJECT) +# verilator --build runs its own make (parallelized by -j in VL_FLAGS) that +# cannot reach this make's jobserver, so it gets a clean MAKEFLAGS. $(DESTDIR)/$(PROJECT): $(SRCS) $(RTL_SRCS) - $(VERILATOR_PATH)/bin/verilator --build $(VL_FLAGS) $(SRCS) -CFLAGS '$(CXXFLAGS)' -LDFLAGS '$(LDFLAGS)' -MMD -o ../$@ + MAKEFLAGS= $(VERILATOR_PATH)/bin/verilator --build $(VL_FLAGS) $(SRCS) -CFLAGS '$(CXXFLAGS)' -LDFLAGS '$(LDFLAGS)' -MMD -o ../$@ run: $(DESTDIR)/$(PROJECT) $(DESTDIR)/$(PROJECT) $(OPTS) diff --git a/hw/unittest/dxa_core/main.cpp b/hw/unittest/dxa_core/main.cpp index 91b74ff863..6f796cd0aa 100644 --- a/hw/unittest/dxa_core/main.cpp +++ b/hw/unittest/dxa_core/main.cpp @@ -19,6 +19,7 @@ #include #include #include "VVX_dxa_core_top.h" +#include "VX_config.h" #include "verilated.h" #ifdef VCD_OUTPUT #include "verilated_vcd_c.h" @@ -93,9 +94,10 @@ int main(int argc, char** argv) { dut->dcr_req_valid = 0; dut->dcr_req_rw = 0; dut->dxa_req_valid = 0; - dut->gmem_req_ready = ~0ULL; // all ready + auto all_ready = [](uint32_t ports) { return (ports >= 32) ? ~0u : ((1u << ports) - 1); }; + dut->gmem_req_ready = all_ready(VX_CFG_NUM_DXA_CORES); dut->gmem_rsp_valid = 0; - dut->lmem_req_ready = ~0ULL; // all ready + dut->lmem_req_ready = all_ready(VX_CFG_SOCKET_SIZE); reset_dut(); diff --git a/hw/unittest/fma_unit/main.cpp b/hw/unittest/fma_unit/main.cpp index 2ff05c8a90..7f0c07b672 100644 --- a/hw/unittest/fma_unit/main.cpp +++ b/hw/unittest/fma_unit/main.cpp @@ -74,6 +74,9 @@ void encode(int op, uint32_t a, uint32_t b, uint32_t c, case FMSUB: op_type = OP_MADD; fmt = 2; break; // a*b - c case FNMADD: op_type = OP_NMADD; fmt = 0; break; // -(a*b) - c case FNMSUB: op_type = OP_NMADD; fmt = 2; break; // -(a*b) + c + default: + std::fprintf(stderr, "encode: unknown op %d\n", op); + std::abort(); } } diff --git a/miscs/patches/riscv-benchmarks.patch b/miscs/patches/riscv-benchmarks.patch index 3f4011d2c7..4db8b67a94 100644 --- a/miscs/patches/riscv-benchmarks.patch +++ b/miscs/patches/riscv-benchmarks.patch @@ -120,7 +120,7 @@ index 2a2a8f3..e967138 100644 - syscall(SYS_write, HTIF_DEV_CONSOLE, (uintptr_t)&c, HTIF_CONSOLE_CMD_PUTC); - + unsigned tid = read_csr(mhartid) & (VX_MEM_IO_COUT_SLOTS - 1); -+ *(volatile char*)(VX_MEM_IO_COUT_ADDR + tid) = (char)ch; ++ *(volatile char*)(uintptr_t)(VX_MEM_IO_COUT_ADDR + tid) = (char)ch; return 0; } diff --git a/sim/avedsim/Makefile b/sim/avedsim/Makefile index 539c3922ef..6dd3b52bf2 100644 --- a/sim/avedsim/Makefile +++ b/sim/avedsim/Makefile @@ -10,8 +10,8 @@ XRTSIM_DIR := $(VORTEX_HOME)/sim/xrtsim AFU_DIR := $(RTL_DIR)/afu/aved AFU_COMMON_DIR := $(RTL_DIR)/afu/common -CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Wno-array-bounds -CXXFLAGS += -fPIC -Wno-maybe-uninitialized +CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Werror +CXXFLAGS += -fPIC # GCC 13 turns the generated per-word copy of a wide signal into a memcpy and # inlines it with aligned 16-byte stores, faulting on a destination that is only # 8-byte aligned in the root class (NUM_THREADS=16 + RTU crashes in reset). @@ -213,8 +213,10 @@ $(DESTDIR)/vortex.json: $(CONFIG_FILE) $(DESTDIR)/scope.json: $(DESTDIR)/vortex.json $(SCRIPT_DIR)/scope.py $^ -o $@ +# verilator --build runs its own make (parallelized by -j in VL_FLAGS) that +# cannot reach this make's jobserver, so it gets a clean MAKEFLAGS. $(DESTDIR)/$(PROJECT): $(SRCS) $(SCOPE_JSON) $(CONFIG_FILE) $(RTL_SRCS) - $(VERILATOR_PATH)/bin/verilator --build --exe $(VL_FLAGS) --cc $(TOP) --top-module $(TOP) $(SRCS) -CFLAGS '$(CXXFLAGS)' -LDFLAGS '$(LDFLAGS)' --Mdir $@.obj_dir -o $@ + MAKEFLAGS= $(VERILATOR_PATH)/bin/verilator --build --exe $(VL_FLAGS) --cc $(TOP) --top-module $(TOP) $(SRCS) -CFLAGS '$(CXXFLAGS)' -LDFLAGS '$(LDFLAGS)' --Mdir $@.obj_dir -o $@ # updates the timestamp when flags changed. $(CONFIG_FILE): force diff --git a/sim/opaesim/Makefile b/sim/opaesim/Makefile index 5fc6e6aaee..61fc9aa0d8 100644 --- a/sim/opaesim/Makefile +++ b/sim/opaesim/Makefile @@ -6,8 +6,8 @@ CONFIG_FILE := $(DESTDIR)/opaesim_config.stamp SRC_DIR := $(VORTEX_HOME)/sim/opaesim AFU_DIR := $(RTL_DIR)/afu/opae -CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Wno-array-bounds -CXXFLAGS += -fPIC -Wno-maybe-uninitialized +CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Werror +CXXFLAGS += -fPIC # GCC 13 turns the generated per-word copy of a wide signal into a memcpy and # inlines it with aligned 16-byte stores, faulting on a destination that is only # 8-byte aligned in the root class (NUM_THREADS=16 + RTU crashes in reset). @@ -141,7 +141,7 @@ RTL_PKGS += $(RTL_DIR)/cp/VX_cp_pkg.sv TOP = vortex_afu_shim VL_FLAGS += --language 1800-2012 --assert -Wall -Wpedantic -VL_FLAGS += -Wno-DECLFILENAME -Wno-REDEFMACRO -Wno-fatal +VL_FLAGS += -Wno-DECLFILENAME -Wno-REDEFMACRO VL_FLAGS += --x-initial unique --x-assign unique VL_FLAGS += verilator.vlt VL_FLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) @@ -216,8 +216,10 @@ $(DESTDIR)/vortex.json: $(CONFIG_FILE) $(DESTDIR)/scope.json: $(DESTDIR)/vortex.json $(SCRIPT_DIR)/scope.py $^ -o $@ +# verilator --build runs its own make (parallelized by -j in VL_FLAGS) that +# cannot reach this make's jobserver, so it gets a clean MAKEFLAGS. $(DESTDIR)/$(PROJECT): $(SRCS) $(SCOPE_JSON) $(CONFIG_FILE) $(RTL_SRCS) - $(VERILATOR_PATH)/bin/verilator --build --exe $(VL_FLAGS) --cc $(TOP) --top-module $(TOP) $(SRCS) -CFLAGS '$(CXXFLAGS)' -LDFLAGS '$(LDFLAGS)' --Mdir $@.obj_dir -o $@ + MAKEFLAGS= $(VERILATOR_PATH)/bin/verilator --build --exe $(VL_FLAGS) --cc $(TOP) --top-module $(TOP) $(SRCS) -CFLAGS '$(CXXFLAGS)' -LDFLAGS '$(LDFLAGS)' --Mdir $@.obj_dir -o $@ # updates the timestamp when flags changed. $(CONFIG_FILE): force diff --git a/sim/rtlsim/Makefile b/sim/rtlsim/Makefile index 9856159b26..f6b8ca3cf8 100644 --- a/sim/rtlsim/Makefile +++ b/sim/rtlsim/Makefile @@ -5,8 +5,8 @@ DESTDIR ?= $(CURDIR) CONFIG_FILE = $(DESTDIR)/rtlsim_config.stamp SRC_DIR = $(VORTEX_HOME)/sim/rtlsim -CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Wno-array-bounds -CXXFLAGS += -fPIC -Wno-maybe-uninitialized +CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Werror +CXXFLAGS += -fPIC # GCC 13 turns the generated per-word copy of a wide signal into a memcpy and # inlines it with aligned 16-byte stores, faulting on a destination that is only # 8-byte aligned in the root class (NUM_THREADS=16 + RTU crashes in reset). @@ -132,9 +132,7 @@ TOP = rtlsim_shim VL_FLAGS = --exe VL_FLAGS += --language 1800-2012 --assert -Wall -Wpedantic -# Keep lint warnings visible but non-fatal: newer Verilator surfaces latent -# UNUSED*/PINCONNECTEMPTY warnings in existing IP that older releases did not. -VL_FLAGS += -Wno-DECLFILENAME -Wno-REDEFMACRO -Wno-fatal +VL_FLAGS += -Wno-DECLFILENAME -Wno-REDEFMACRO VL_FLAGS += --x-initial unique --x-assign unique VL_FLAGS += verilator.vlt VL_FLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) @@ -201,11 +199,13 @@ PROJECT := rtlsim all: $(DESTDIR)/$(PROJECT) +# verilator --build runs its own make (parallelized by -j in VL_FLAGS) that +# cannot reach this make's jobserver, so it gets a clean MAKEFLAGS. $(DESTDIR)/$(PROJECT): $(SRCS) $(SRC_DIR)/main.cpp $(CONFIG_FILE) $(RTL_SRCS) - $(VERILATOR_PATH)/bin/verilator --build $(VL_FLAGS) $(SRCS) $(SRC_DIR)/main.cpp -CFLAGS '$(CXXFLAGS)' -LDFLAGS '$(LDFLAGS)' --MMD --Mdir $@.obj_dir -o $@ + MAKEFLAGS= $(VERILATOR_PATH)/bin/verilator --build $(VL_FLAGS) $(SRCS) $(SRC_DIR)/main.cpp -CFLAGS '$(CXXFLAGS)' -LDFLAGS '$(LDFLAGS)' --MMD --Mdir $@.obj_dir -o $@ $(DESTDIR)/lib$(PROJECT).so: $(SRCS) $(CONFIG_FILE) $(RTL_SRCS) - $(VERILATOR_PATH)/bin/verilator --build $(VL_FLAGS) $(SRCS) -CFLAGS '$(CXXFLAGS)' -LDFLAGS '-shared $(LDFLAGS)' --MMD --Mdir $@.obj_dir -o $@ + MAKEFLAGS= $(VERILATOR_PATH)/bin/verilator --build $(VL_FLAGS) $(SRCS) -CFLAGS '$(CXXFLAGS)' -LDFLAGS '-shared $(LDFLAGS)' --MMD --Mdir $@.obj_dir -o $@ # updates the timestamp when flags changed. $(CONFIG_FILE): force diff --git a/sim/simx/Makefile b/sim/simx/Makefile index 63581b4b8f..a9638e5ac7 100644 --- a/sim/simx/Makefile +++ b/sim/simx/Makefile @@ -17,7 +17,7 @@ CONFIG_FILE = $(DESTDIR)/simx_config.stamp SRC_DIR = $(VORTEX_HOME)/sim/simx CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Werror -CXXFLAGS += -fPIC -Wno-maybe-uninitialized +CXXFLAGS += -fPIC CXXFLAGS += -I$(SRC_DIR) -I$(SRC_DIR)/dxa -I$(SRC_DIR)/tex -I$(SRC_DIR)/om -I$(SRC_DIR)/raster -I$(SRC_DIR)/rtu -I$(SRC_DIR)/kmu -I$(SRC_DIR)/mem -I$(SRC_DIR)/dtm -I$(SW_COMMON_DIR) -I$(SIM_COMMON_DIR) -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw -I$(THIRD_PARTY_DIR) CXXFLAGS += -I$(THIRD_PARTY_DIR)/softfloat/source/include CXXFLAGS += -I$(THIRD_PARTY_DIR)/ramulator/ext/spdlog/include diff --git a/sim/xrtsim/Makefile b/sim/xrtsim/Makefile index b2e9be960a..231d745806 100644 --- a/sim/xrtsim/Makefile +++ b/sim/xrtsim/Makefile @@ -7,8 +7,8 @@ SRC_DIR := $(VORTEX_HOME)/sim/xrtsim AFU_DIR := $(RTL_DIR)/afu/xrt AFU_COMMON_DIR := $(RTL_DIR)/afu/common -CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Wno-array-bounds -CXXFLAGS += -fPIC -Wno-maybe-uninitialized +CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Werror +CXXFLAGS += -fPIC # GCC 13 turns the generated per-word copy of a wide signal into a memcpy and # inlines it with aligned 16-byte stores, faulting on a destination that is only # 8-byte aligned in the root class (NUM_THREADS=16 + RTU crashes in reset). @@ -141,7 +141,7 @@ RTL_PKGS += $(RTL_DIR)/cp/VX_cp_pkg.sv TOP = vortex_afu_shim VL_FLAGS += --language 1800-2012 --assert -Wall -Wpedantic -VL_FLAGS += -Wno-DECLFILENAME -Wno-REDEFMACRO -Wno-fatal +VL_FLAGS += -Wno-DECLFILENAME -Wno-REDEFMACRO VL_FLAGS += --x-initial unique --x-assign unique VL_FLAGS += verilator.vlt VL_FLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) @@ -216,8 +216,10 @@ $(DESTDIR)/vortex.json: $(CONFIG_FILE) $(DESTDIR)/scope.json: $(DESTDIR)/vortex.json $(SCRIPT_DIR)/scope.py $^ -o $@ +# verilator --build runs its own make (parallelized by -j in VL_FLAGS) that +# cannot reach this make's jobserver, so it gets a clean MAKEFLAGS. $(DESTDIR)/$(PROJECT): $(SRCS) $(SCOPE_JSON) $(CONFIG_FILE) $(RTL_SRCS) - $(VERILATOR_PATH)/bin/verilator --build --exe $(VL_FLAGS) --cc $(TOP) --top-module $(TOP) $(SRCS) -CFLAGS '$(CXXFLAGS)' -LDFLAGS '$(LDFLAGS)' --Mdir $@.obj_dir -o $@ + MAKEFLAGS= $(VERILATOR_PATH)/bin/verilator --build --exe $(VL_FLAGS) --cc $(TOP) --top-module $(TOP) $(SRCS) -CFLAGS '$(CXXFLAGS)' -LDFLAGS '$(LDFLAGS)' --Mdir $@.obj_dir -o $@ # updates the timestamp when flags changed. $(CONFIG_FILE): force diff --git a/sw/kernel/Makefile b/sw/kernel/Makefile index a00ea753a8..668b6f8c18 100644 --- a/sw/kernel/Makefile +++ b/sw/kernel/Makefile @@ -29,6 +29,7 @@ DP = $(RISCV_TOOLCHAIN_PATH)/bin/$(RISCV_PREFIX)-objdump CP = $(RISCV_TOOLCHAIN_PATH)/bin/$(RISCV_PREFIX)-objcopy CFLAGS += -O3 -mcmodel=medany -fno-exceptions -fdata-sections -ffunction-sections -D__VORTEX__ +CFLAGS += -Wall -Wextra -Wfatal-errors -Werror CFLAGS += -I$(INC_DIR) -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw XCONFIGS := $(shell python3 $(ROOT_DIR)/ci/gen_config.py --config=$(VORTEX_HOME)/VX_config.toml --cflags='$(CONFIGS) -DVX_CFG_XLEN=$(XLEN)') CFLAGS += $(XCONFIGS) diff --git a/sw/kernel/src/vx_spawn.c b/sw/kernel/src/vx_spawn.c index af8d9c2c65..fecb6fdfb7 100644 --- a/sw/kernel/src/vx_spawn.c +++ b/sw/kernel/src/vx_spawn.c @@ -14,6 +14,7 @@ #include #include #include +#include #ifdef __cplusplus extern "C" { @@ -144,7 +145,7 @@ static void __attribute__ ((noinline)) process_thread_groups() { for (uint32_t group_id = start_group; group_id < end_group; group_id += group_stride) { blockIdx.x = group_id % gridDim_x; blockIdx.y = (group_id / gridDim_x) % gridDim_y; - blockIdx.z = group_id / (gridDim_x * gridDim_y); + blockIdx.z = group_id / gridDim_xy; callback((void*)arg); } } @@ -155,7 +156,7 @@ static void __attribute__ ((noinline)) process_thread_groups_stub() { uint32_t remaining_mask = targs->remaining_mask; uint32_t warp_id = vx_warp_id(); uint32_t group_warp_id = warp_id % warps_per_group; - uint32_t threads_mask = (group_warp_id == warps_per_group-1) ? remaining_mask : -1; + uint32_t threads_mask = (group_warp_id == warps_per_group-1) ? remaining_mask : ~0u; // activate threads vx_tmc(threads_mask); @@ -192,7 +193,7 @@ int vx_spawn_threads(uint32_t dimension, // check group size uint32_t threads_per_core = warps_per_core * threads_per_warp; if (threads_per_core < group_size) { - vx_printf("error: group_size > threads_per_core (%d,%d)\n", group_size, threads_per_core); + vx_printf("error: group_size > threads_per_core (%" PRIu32 ",%" PRIu32 ")\n", group_size, threads_per_core); return -1; } diff --git a/sw/kernel/src/vx_syscalls.c b/sw/kernel/src/vx_syscalls.c index bef330323e..4f34771e32 100644 --- a/sw/kernel/src/vx_syscalls.c +++ b/sw/kernel/src/vx_syscalls.c @@ -23,24 +23,51 @@ extern "C" { #endif -int _close(int file) { return -1; } +int _close(int file) { + (void)file; + return -1; +} -int _fstat(int file, struct stat *st) { return -1; } +int _fstat(int file, struct stat *st) { + (void)file; + (void)st; + return -1; +} -int _isatty(int file) { return 0; } +int _isatty(int file) { + (void)file; + return 0; +} -off_t _lseek(int file, off_t ptr, int dir) { return 0; } +off_t _lseek(int file, off_t ptr, int dir) { + (void)file; + (void)ptr; + (void)dir; + return 0; +} -int _open(const char *name, int flags, int mode) { return -1; } +int _open(const char *name, int flags, int mode) { + (void)name; + (void)flags; + (void)mode; + return -1; +} -ssize_t _read(int file, void *ptr, size_t len) { return -1; } +ssize_t _read(int file, void *ptr, size_t len) { + (void)file; + (void)ptr; + (void)len; + return -1; +} void *_sbrk(ptrdiff_t incr) { + (void)incr; __asm__ __volatile__("ebreak"); return 0; } ssize_t _write(int file, const void *ptr, size_t len) { + (void)file; const char *p = (const char *)ptr; for (size_t i = 0; i < len; ++i) { vx_putchar(*p++); @@ -48,7 +75,11 @@ ssize_t _write(int file, const void *ptr, size_t len) { return len; } -int _kill(int pid, int sig) { return -1; } +int _kill(int pid, int sig) { + (void)pid; + (void)sig; + return -1; +} int _getpid() { return vx_hart_id(); diff --git a/sw/runtime/simx/Makefile b/sw/runtime/simx/Makefile index 65ec2c4c7a..39c40cc70f 100644 --- a/sw/runtime/simx/Makefile +++ b/sw/runtime/simx/Makefile @@ -6,7 +6,7 @@ CONFIG_FILE := $(DESTDIR)/simx_rt_config.stamp SRC_DIR := $(VORTEX_HOME)/sw/runtime/simx CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Werror -CXXFLAGS += -fPIC -Wno-maybe-uninitialized +CXXFLAGS += -fPIC CXXFLAGS += -I$(INC_DIR) -I../common -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw -I$(SIM_DIR)/simx -I$(SIM_COMMON_DIR) -I$(SW_COMMON_DIR) -I$(RT_COMMON_DIR) CXXFLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) CXXFLAGS += $(CONFIGS) diff --git a/tests/hip/common.mk b/tests/hip/common.mk index 0b315896d2..77c80baa6e 100644 --- a/tests/hip/common.mk +++ b/tests/hip/common.mk @@ -83,7 +83,7 @@ POCL_CC_FLAGS += POCL_VORTEX_BINTOOL="$(VX_BINTOOL)" POCL_CC_FLAGS += POCL_VORTEX_CFLAGS="$(VX_CFLAGS)" POCL_CC_FLAGS += POCL_VORTEX_LDFLAGS="$(VX_LDFLAGS)" -HIPCC_FLAGS += -std=c++17 +HIPCC_FLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Werror HIPCC_FLAGS += $(CONFIGS) # The embedded .hipInfo OFFLOAD_TRIPLE selects the primary SPIR-V pointer # width. hipcc rewrites it to match XLEN so the emitted SPIR-V is diff --git a/tests/hip/sgemm/main.cpp b/tests/hip/sgemm/main.cpp index 26de4d7233..99b63c0da8 100644 --- a/tests/hip/sgemm/main.cpp +++ b/tests/hip/sgemm/main.cpp @@ -89,8 +89,8 @@ int main(int argc, char** argv) { srand(50); for (uint32_t i = 0; i < size_sq; ++i) { - h_a[i] = static_cast(rand()) / RAND_MAX; - h_b[i] = static_cast(rand()) / RAND_MAX; + h_a[i] = static_cast(rand()) / static_cast(RAND_MAX); + h_b[i] = static_cast(rand()) / static_cast(RAND_MAX); } printf("Allocate device buffers\n"); diff --git a/tests/hip/vecadd/main.cpp b/tests/hip/vecadd/main.cpp index 9532b98eec..5e8e8b7e4b 100644 --- a/tests/hip/vecadd/main.cpp +++ b/tests/hip/vecadd/main.cpp @@ -71,8 +71,8 @@ int main(int argc, char** argv) { srand(50); for (uint32_t i = 0; i < size; ++i) { - h_a[i] = static_cast(rand()) / RAND_MAX; - h_b[i] = static_cast(rand()) / RAND_MAX; + h_a[i] = static_cast(rand()) / static_cast(RAND_MAX); + h_b[i] = static_cast(rand()) / static_cast(RAND_MAX); } printf("Allocate device buffers\n"); diff --git a/tests/opencl/bfs/CLHelper.h b/tests/opencl/bfs/CLHelper.h index e4106845bd..eafce0d076 100755 --- a/tests/opencl/bfs/CLHelper.h +++ b/tests/opencl/bfs/CLHelper.h @@ -254,7 +254,6 @@ free(allPlatforms);*/ // read kernel binary from file uint8_t *kernel_bin = NULL; size_t kernel_size; - cl_int binary_status = 0; if (0 != read_kernel_file("kernel.cl", &kernel_bin, &kernel_size)) std::abort(); @@ -374,7 +373,7 @@ free(allPlatforms);*/ void _clRelease() { char errorFlag = false; - for (int nKernel = 0; nKernel < oclHandles.kernel.size(); nKernel++) { + for (size_t nKernel = 0; nKernel < oclHandles.kernel.size(); nKernel++) { if (oclHandles.kernel[nKernel] != NULL) { cl_int resultCL = clReleaseKernel(oclHandles.kernel[nKernel]); if (resultCL != CL_SUCCESS) { @@ -511,7 +510,9 @@ cl_mem _clCreateAndCpyPinnedMem(int size, float *h_mem_source) { throw(string("excpetion in _clCreateAndCpyMem() -> clEnqueueMapBuffer")); #endif int element_number = size / sizeof(float); +#ifdef _OPENMP #pragma omp parallel for +#endif for (int i = 0; i < element_number; i++) { h_mem_pinned[i] = h_mem_source[i]; } @@ -690,8 +691,8 @@ void _clInvokeKernel(int kernel_id, int work_items, // cannot be divided by work_group_size work_items = work_items + (work_group_size - (work_items % work_group_size)); - size_t local_work_size[] = {work_group_size, 1}; - size_t global_work_size[] = {work_items, 1}; + size_t local_work_size[] = {static_cast(work_group_size), 1}; + size_t global_work_size[] = {static_cast(work_items), 1}; oclHandles.cl_status = clEnqueueNDRangeKernel( oclHandles.queue, oclHandles.kernel[kernel_id], work_dim, 0, global_work_size, local_work_size, 0, 0, NULL); @@ -757,8 +758,10 @@ void _clInvokeKernel(int kernel_id, int work_items, void _clInvokeKernel2D(int kernel_id, int range_x, int range_y, int group_x, int group_y) { cl_uint work_dim = WORK_DIM; - size_t local_work_size[] = {group_x, group_y}; - size_t global_work_size[] = {range_x, range_y}; + size_t local_work_size[] = {static_cast(group_x), + static_cast(group_y)}; + size_t global_work_size[] = {static_cast(range_x), + static_cast(range_y)}; //cl_event e[1]; /*if(work_items%work_group_size != 0) //process situations that work_items cannot be divided by work_group_size diff --git a/tests/opencl/bfs/main.cc b/tests/opencl/bfs/main.cc index cd55f5b3fb..c676a40c53 100755 --- a/tests/opencl/bfs/main.cc +++ b/tests/opencl/bfs/main.cc @@ -29,10 +29,9 @@ struct Node { //--date: 26/01/2011 //--note: width is changed to the new_width //---------------------------------------------------------- -void run_bfs_cpu(int no_of_nodes, Node *h_graph_nodes, int edge_list_size, - int *h_graph_edges, char *h_graph_mask, - char *h_updating_graph_mask, char *h_graph_visited, - int *h_cost_ref) { +void run_bfs_cpu(int no_of_nodes, Node *h_graph_nodes, int *h_graph_edges, + char *h_graph_mask, char *h_updating_graph_mask, + char *h_graph_visited, int *h_cost_ref) { char stop; int k = 0; do { @@ -186,8 +185,9 @@ int main(int argc, char *argv[]) { int no_of_nodes; int edge_list_size; FILE *fp; - Node *h_graph_nodes; - char *h_graph_mask, *h_updating_graph_mask, *h_graph_visited; + Node *h_graph_nodes = NULL; + char *h_graph_mask = NULL, *h_updating_graph_mask = NULL, + *h_graph_visited = NULL; if (argc < 2) { printf("graph file missing!\n"); @@ -206,17 +206,22 @@ int main(int argc, char *argv[]) { printf("Reading File completed!\n"); + auto read_int = [fp](int *value) { + if (fscanf(fp, "%d", value) != 1) { + fclose(fp); + throw(std::string("error parsing graph file")); + } + }; + int source = 0; - fscanf(fp, "%d", &no_of_nodes); + read_int(&no_of_nodes); - int num_of_blocks = 1; int num_of_threads_per_block = no_of_nodes; // Make execution Parameters according to the number of nodes // Distribute threads across multiple Blocks if necessary if (no_of_nodes > MAX_THREADS_PER_BLOCK) { - num_of_blocks = (int)ceil(no_of_nodes / (double)MAX_THREADS_PER_BLOCK); num_of_threads_per_block = MAX_THREADS_PER_BLOCK; } work_group_size = num_of_threads_per_block; @@ -229,7 +234,8 @@ int main(int argc, char *argv[]) { int start, edgeno; // initalize the memory for (int i = 0; i < no_of_nodes; i++) { - fscanf(fp, "%d %d", &start, &edgeno); + read_int(&start); + read_int(&edgeno); h_graph_nodes[i].starting = start; h_graph_nodes[i].no_of_edges = edgeno; h_graph_mask[i] = false; @@ -237,17 +243,17 @@ int main(int argc, char *argv[]) { h_graph_visited[i] = false; } // read the source node from the file - fscanf(fp, "%d", &source); + read_int(&source); source = 0; // set the source node as true in the mask h_graph_mask[source] = true; h_graph_visited[source] = true; - fscanf(fp, "%d", &edge_list_size); + read_int(&edge_list_size); int id, cost; int *h_graph_edges = (int *)malloc(sizeof(int) * edge_list_size); for (int i = 0; i < edge_list_size; i++) { - fscanf(fp, "%d", &id); - fscanf(fp, "%d", &cost); + read_int(&id); + read_int(&cost); h_graph_edges[i] = id; } @@ -278,7 +284,7 @@ int main(int argc, char *argv[]) { source = 0; h_graph_mask[source] = true; h_graph_visited[source] = true; - run_bfs_cpu(no_of_nodes, h_graph_nodes, edge_list_size, h_graph_edges, + run_bfs_cpu(no_of_nodes, h_graph_nodes, h_graph_edges, h_graph_mask, h_updating_graph_mask, h_graph_visited, h_cost_ref); //--------------------------------------------------------- @@ -290,7 +296,7 @@ int main(int argc, char *argv[]) { free(h_updating_graph_mask); free(h_graph_visited); } catch (std::string msg) { - printf("--cambine: exception in main ->%s\n", msg); + printf("--cambine: exception in main ->%s\n", msg.c_str()); // release host memory free(h_graph_nodes); free(h_graph_mask); diff --git a/tests/opencl/bfs/util.h b/tests/opencl/bfs/util.h index 4d1d3e27b9..0941d18071 100755 --- a/tests/opencl/bfs/util.h +++ b/tests/opencl/bfs/util.h @@ -35,7 +35,9 @@ template void verify_array(const datatype *cpuResults, const datatype *gpuResults, const int size){ char passed = true; +#ifdef _OPENMP #pragma omp parallel for +#endif for (int i=0; i MAX_RELATIVE_ERROR){ passed = false; diff --git a/tests/opencl/blackscholes/oclBlackScholes_launcher.cpp b/tests/opencl/blackscholes/oclBlackScholes_launcher.cpp index 41209e3637..19b89b6c73 100644 --- a/tests/opencl/blackscholes/oclBlackScholes_launcher.cpp +++ b/tests/opencl/blackscholes/oclBlackScholes_launcher.cpp @@ -37,9 +37,8 @@ static int read_kernel_file(const char* filename, uint8_t** data, size_t* size) return CL_SUCCESS; } -extern "C" void initBlackScholes(cl_context cxGPUContext, cl_command_queue cqParamCommandQueue, const char **argv){ +extern "C" void initBlackScholes(cl_context cxGPUContext, cl_command_queue cqParamCommandQueue, const char ** /*argv*/){ cl_int ciErrNum; - size_t kernelLength; /*shrLog("...loading BlackScholes.cl\n"); char *cPathAndName = shrFindFilePath("BlackScholes.cl", argv[0]); @@ -50,8 +49,6 @@ extern "C" void initBlackScholes(cl_context cxGPUContext, cl_command_queue cqPar shrLog("...creating BlackScholes program\n"); uint8_t *kernel_bin = NULL; size_t kernel_size; - cl_int binary_status = 0; - cl_device_id device_id = oclGetFirstDev(cxGPUContext); ciErrNum = read_kernel_file("kernel.cl", &kernel_bin, &kernel_size); shrCheckError(ciErrNum, CL_SUCCESS); diff --git a/tests/opencl/blackscholes/shrUtils.cpp b/tests/opencl/blackscholes/shrUtils.cpp index cf0d2c3e7e..fb4bea436f 100644 --- a/tests/opencl/blackscholes/shrUtils.cpp +++ b/tests/opencl/blackscholes/shrUtils.cpp @@ -203,7 +203,6 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list { static FILE* pFileStream0 = NULL; static FILE* pFileStream1 = NULL; - size_t szNumWritten = 0; char cFileMode [3]; // if the sample log file is closed and the call incudes a "write-to-file", open file for writing @@ -318,12 +317,12 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list // print string to console if flagged if (iLogMode & LOGCONSOLE) { - szNumWritten = printf ("\n !!! Error # %i at ", iErrNum); // console + printf ("\n !!! Error # %i at ", iErrNum); // console } // print string to file if flagged if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, "\n !!! Error # %i at ", iErrNum); // sample log file + fprintf (pFileStream0, "\n !!! Error # %i at ", iErrNum); // sample log file } } @@ -347,14 +346,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list // character is not '%', so print it verbatim to console and/or files as flagged if (iLogMode & LOGCONSOLE) { - szNumWritten = putc(*pStr, stdout); // console + putc(*pStr, stdout); // console } if (iLogMode & LOGFILE) { - szNumWritten = putc(*pStr, pFileStream0); // sample log file + putc(*pStr, pFileStream0); // sample log file if (iLogMode & MASTER) { - szNumWritten = putc(*pStr, pFileStream1); // master log file + putc(*pStr, pFileStream1); // master log file } } } @@ -399,16 +398,26 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list { case '%': // special handling for string of %%%% { + // collapse each "%%" to a literal '%', as printf would + std::string sLiteral; + for (size_t i = 0; i < sFormatSpec.size(); ++i) + { + sLiteral += sFormatSpec[i]; + if ((sFormatSpec[i] == '%') && (i + 1 < sFormatSpec.size()) && (sFormatSpec[i + 1] == '%')) + { + ++i; + } + } if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str()); // console + fputs(sLiteral.c_str(), stdout); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str()); // sample log file + fputs(sLiteral.c_str(), pFileStream0); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str()); // master log file + fputs(sLiteral.c_str(), pFileStream1); // master log file } } continue; @@ -420,14 +429,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list cArg = va_arg(vaArgList, char*); if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str(), cArg); // console + printf(sFormatSpec.c_str(), cArg); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str(), cArg); // sample log file + fprintf (pFileStream0, sFormatSpec.c_str(), cArg); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str(), cArg); // master log file + fprintf(pFileStream1, sFormatSpec.c_str(), cArg); // master log file } } continue; @@ -439,14 +448,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list iArg = va_arg(vaArgList, int); if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str(), iArg); // console + printf(sFormatSpec.c_str(), iArg); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str(), iArg); // sample log file + fprintf (pFileStream0, sFormatSpec.c_str(), iArg); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str(), iArg); // master log file + fprintf(pFileStream1, sFormatSpec.c_str(), iArg); // master log file } } continue; @@ -460,14 +469,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list uiArg = va_arg(vaArgList, unsigned int); if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str(), uiArg); // console + printf(sFormatSpec.c_str(), uiArg); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str(), uiArg); // sample log file + fprintf (pFileStream0, sFormatSpec.c_str(), uiArg); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str(), uiArg); // master log file + fprintf(pFileStream1, sFormatSpec.c_str(), uiArg); // master log file } } continue; @@ -484,14 +493,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list dArg = va_arg(vaArgList, double); if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str(), dArg); // console + printf(sFormatSpec.c_str(), dArg); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str(), dArg); // sample log file + fprintf (pFileStream0, sFormatSpec.c_str(), dArg); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str(), dArg); // master log file + fprintf(pFileStream1, sFormatSpec.c_str(), dArg); // master log file } } continue; @@ -501,14 +510,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list // print arg of unknown/unsupported type to console and/or file if flagged if (iLogMode & LOGCONSOLE) // console { - szNumWritten = putc(*pStr, stdout); + putc(*pStr, stdout); } if (iLogMode & LOGFILE) { - szNumWritten = putc(*pStr, pFileStream0); // sample log file + putc(*pStr, pFileStream0); // sample log file if (iLogMode & MASTER) { - szNumWritten = putc(*pStr, pFileStream1); // master log file + putc(*pStr, pFileStream1); // master log file } } } diff --git a/tests/opencl/cfd/main.cc b/tests/opencl/cfd/main.cc index 330d68ec66..1d8d11c741 100644 --- a/tests/opencl/cfd/main.cc +++ b/tests/opencl/cfd/main.cc @@ -158,9 +158,8 @@ static inline float compute_pressure(float density, float density_energy, float static inline float compute_speed_of_sound(float density, float pressure) { return sqrtf(GAMMA * pressure / density); } -static inline void compute_flux_contribution(float density, FLOAT3 momentum, float density_energy, - float pressure, FLOAT3 velocity, - FLOAT3* fc_momentum_x, FLOAT3* fc_momentum_y, +static inline void compute_flux_contribution(FLOAT3 momentum, float density_energy, float pressure, + FLOAT3 velocity, FLOAT3* fc_momentum_x, FLOAT3* fc_momentum_y, FLOAT3* fc_momentum_z, FLOAT3* fc_density_energy) { fc_momentum_x->x = velocity.x * momentum.x + pressure; fc_momentum_x->y = velocity.x * momentum.y; @@ -216,7 +215,7 @@ static void cpu_compute_flux(int nelr, const int* esse, const float* normals, float pressure_i = compute_pressure(density_i, density_energy_i, speed_sqd_i); float speed_of_sound_i = compute_speed_of_sound(density_i, pressure_i); FLOAT3 fc_i_momentum_x, fc_i_momentum_y, fc_i_momentum_z, fc_i_density_energy; - compute_flux_contribution(density_i, momentum_i, density_energy_i, pressure_i, velocity_i, + compute_flux_contribution(momentum_i, density_energy_i, pressure_i, velocity_i, &fc_i_momentum_x, &fc_i_momentum_y, &fc_i_momentum_z, &fc_i_density_energy); float flux_i_density = 0.0f; @@ -244,7 +243,7 @@ static void cpu_compute_flux(int nelr, const int* esse, const float* normals, speed_sqd_nb = compute_speed_sqd(velocity_nb); pressure_nb = compute_pressure(density_nb, density_energy_nb, speed_sqd_nb); speed_of_sound_nb = compute_speed_of_sound(density_nb, pressure_nb); - compute_flux_contribution(density_nb, momentum_nb, density_energy_nb, pressure_nb, velocity_nb, + compute_flux_contribution(momentum_nb, density_energy_nb, pressure_nb, velocity_nb, &fc_nb_momentum_x, &fc_nb_momentum_y, &fc_nb_momentum_z, &fc_nb_density_energy); factor = -normal_len * smoothing_coefficient * 0.5f * @@ -352,8 +351,7 @@ int main(int argc, char** argv) { ff_momentum.x = h_ff_variable[VAR_MOMENTUM + 0]; ff_momentum.y = h_ff_variable[VAR_MOMENTUM + 1]; ff_momentum.z = h_ff_variable[VAR_MOMENTUM + 2]; - compute_flux_contribution(h_ff_variable[VAR_DENSITY], ff_momentum, - h_ff_variable[VAR_DENSITY_ENERGY], ff_pressure, ff_velocity, + compute_flux_contribution(ff_momentum, h_ff_variable[VAR_DENSITY_ENERGY], ff_pressure, ff_velocity, &h_ff_fc_momentum_x, &h_ff_fc_momentum_y, &h_ff_fc_momentum_z, &h_ff_fc_density_energy); } diff --git a/tests/opencl/common.mk b/tests/opencl/common.mk index e35673e00e..84774d3e2d 100644 --- a/tests/opencl/common.mk +++ b/tests/opencl/common.mk @@ -51,6 +51,7 @@ VX_CFLAGS += -Xclang -target-feature -Xclang +xvortex VX_CFLAGS += -Xclang -target-feature -Xclang +zicond VX_LDFLAGS += -fuse-ld=lld VX_CFLAGS += -mllvm -disable-loop-idiom-all # disable memset/memcpy loop replacement +VX_CFLAGS += -Wall -Wextra -Wfatal-errors -Werror VX_LDFLAGS += -Wl,-z,norelro #VX_CFLAGS += -mllvm -vortex-branch-divergence=0 #VX_CFLAGS += -mllvm -debug -mllvm -print-after-all @@ -59,10 +60,11 @@ VX_LDFLAGS += -Wl,-Bstatic,--gc-sections,-T$(VORTEX_HOME)/sw/kernel/scripts/link VX_BINTOOL += OBJCOPY=$(LLVM_PATH)/bin/llvm-objcopy $(VORTEX_HOME)/sw/kernel/scripts/vxbin.py -CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -CXXFLAGS += -Wno-deprecated-declarations -Wno-unused-parameter -Wno-narrowing +CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Werror CXXFLAGS += -pthread CXXFLAGS += -I$(POCL_PATH)/include +# The tests are written against the OpenCL 1.2 host API. +CXXFLAGS += -DCL_TARGET_OPENCL_VERSION=120 CXXFLAGS += $(CONFIGS) POCL_CC_FLAGS += LLVM_PREFIX=$(LLVM_PATH) POCL_VORTEX_BINTOOL="$(VX_BINTOOL)" POCL_VORTEX_CFLAGS="$(VX_CFLAGS)" POCL_VORTEX_LDFLAGS="$(VX_LDFLAGS)" @@ -99,8 +101,9 @@ all: $(PROJECT) %.cpp.o: $(SRC_DIR)/%.cpp $(CXX) $(CXXFLAGS) -c $< -o $@ +# Host C sources share the C++ flags except the language standard. %.c.o: $(SRC_DIR)/%.c - $(CC) $(CXXFLAGS) -c $< -o $@ + $(CC) $(filter-out -std=c++%,$(CXXFLAGS)) -c $< -o $@ $(VORTEX_KN_PATH)/libvortex.a: $(MAKE) -C $(VORTEX_KN_PATH) diff --git a/tests/opencl/conv3/main.cc b/tests/opencl/conv3/main.cc index 1220dabdb2..44bfe64886 100644 --- a/tests/opencl/conv3/main.cc +++ b/tests/opencl/conv3/main.cc @@ -180,7 +180,7 @@ int main (int argc, char **argv) { // Create kernel kernel = CL_CHECK2(clCreateKernel(program, KERNEL_NAME, &_err)); - size_t global_size[2] = {size, size}; + size_t global_size[2] = {static_cast(size), static_cast(size)}; size_t local_size[2] = {1, 1}; // Set kernel arguments diff --git a/tests/opencl/dotproduct/main.cc b/tests/opencl/dotproduct/main.cc index 789b41491b..fe4418228b 100644 --- a/tests/opencl/dotproduct/main.cc +++ b/tests/opencl/dotproduct/main.cc @@ -165,7 +165,6 @@ int main(int argc, char **argv) // Create the program shrLog("clCreateProgramWithSource...\n"); - cl_int binary_status; cl_program program; program = clCreateProgramWithSource( diff --git a/tests/opencl/dotproduct/shrUtils.cpp b/tests/opencl/dotproduct/shrUtils.cpp index cf0d2c3e7e..fb4bea436f 100644 --- a/tests/opencl/dotproduct/shrUtils.cpp +++ b/tests/opencl/dotproduct/shrUtils.cpp @@ -203,7 +203,6 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list { static FILE* pFileStream0 = NULL; static FILE* pFileStream1 = NULL; - size_t szNumWritten = 0; char cFileMode [3]; // if the sample log file is closed and the call incudes a "write-to-file", open file for writing @@ -318,12 +317,12 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list // print string to console if flagged if (iLogMode & LOGCONSOLE) { - szNumWritten = printf ("\n !!! Error # %i at ", iErrNum); // console + printf ("\n !!! Error # %i at ", iErrNum); // console } // print string to file if flagged if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, "\n !!! Error # %i at ", iErrNum); // sample log file + fprintf (pFileStream0, "\n !!! Error # %i at ", iErrNum); // sample log file } } @@ -347,14 +346,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list // character is not '%', so print it verbatim to console and/or files as flagged if (iLogMode & LOGCONSOLE) { - szNumWritten = putc(*pStr, stdout); // console + putc(*pStr, stdout); // console } if (iLogMode & LOGFILE) { - szNumWritten = putc(*pStr, pFileStream0); // sample log file + putc(*pStr, pFileStream0); // sample log file if (iLogMode & MASTER) { - szNumWritten = putc(*pStr, pFileStream1); // master log file + putc(*pStr, pFileStream1); // master log file } } } @@ -399,16 +398,26 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list { case '%': // special handling for string of %%%% { + // collapse each "%%" to a literal '%', as printf would + std::string sLiteral; + for (size_t i = 0; i < sFormatSpec.size(); ++i) + { + sLiteral += sFormatSpec[i]; + if ((sFormatSpec[i] == '%') && (i + 1 < sFormatSpec.size()) && (sFormatSpec[i + 1] == '%')) + { + ++i; + } + } if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str()); // console + fputs(sLiteral.c_str(), stdout); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str()); // sample log file + fputs(sLiteral.c_str(), pFileStream0); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str()); // master log file + fputs(sLiteral.c_str(), pFileStream1); // master log file } } continue; @@ -420,14 +429,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list cArg = va_arg(vaArgList, char*); if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str(), cArg); // console + printf(sFormatSpec.c_str(), cArg); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str(), cArg); // sample log file + fprintf (pFileStream0, sFormatSpec.c_str(), cArg); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str(), cArg); // master log file + fprintf(pFileStream1, sFormatSpec.c_str(), cArg); // master log file } } continue; @@ -439,14 +448,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list iArg = va_arg(vaArgList, int); if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str(), iArg); // console + printf(sFormatSpec.c_str(), iArg); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str(), iArg); // sample log file + fprintf (pFileStream0, sFormatSpec.c_str(), iArg); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str(), iArg); // master log file + fprintf(pFileStream1, sFormatSpec.c_str(), iArg); // master log file } } continue; @@ -460,14 +469,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list uiArg = va_arg(vaArgList, unsigned int); if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str(), uiArg); // console + printf(sFormatSpec.c_str(), uiArg); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str(), uiArg); // sample log file + fprintf (pFileStream0, sFormatSpec.c_str(), uiArg); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str(), uiArg); // master log file + fprintf(pFileStream1, sFormatSpec.c_str(), uiArg); // master log file } } continue; @@ -484,14 +493,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list dArg = va_arg(vaArgList, double); if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str(), dArg); // console + printf(sFormatSpec.c_str(), dArg); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str(), dArg); // sample log file + fprintf (pFileStream0, sFormatSpec.c_str(), dArg); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str(), dArg); // master log file + fprintf(pFileStream1, sFormatSpec.c_str(), dArg); // master log file } } continue; @@ -501,14 +510,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list // print arg of unknown/unsupported type to console and/or file if flagged if (iLogMode & LOGCONSOLE) // console { - szNumWritten = putc(*pStr, stdout); + putc(*pStr, stdout); } if (iLogMode & LOGFILE) { - szNumWritten = putc(*pStr, pFileStream0); // sample log file + putc(*pStr, pFileStream0); // sample log file if (iLogMode & MASTER) { - szNumWritten = putc(*pStr, pFileStream1); // master log file + putc(*pStr, pFileStream1); // master log file } } } diff --git a/tests/opencl/guassian/clutils.cpp b/tests/opencl/guassian/clutils.cpp index 2881028489..53a25ff70d 100755 --- a/tests/opencl/guassian/clutils.cpp +++ b/tests/opencl/guassian/clutils.cpp @@ -71,9 +71,6 @@ static cl_uint numPlatforms; static cl_device_id* devices = NULL; static cl_uint* numDevices = NULL; -//! The chosen OpenCL platform -static cl_platform_id platform = NULL; - //! The chosen OpenCL device static cl_device_id device = NULL; @@ -82,11 +79,6 @@ static cl_context context = NULL; //! OpenCL command queue static cl_command_queue commandQueue = NULL; -static cl_command_queue commandQueueProf = NULL; -static cl_command_queue commandQueueNoProf = NULL; - -//! Global status of events -static bool eventsEnabled = false; //------------------------------------------------------- // Initialization and Cleanup @@ -260,9 +252,7 @@ static int read_kernel_file(const char* filename, uint8_t** data, size_t* size) } -cl_context cl_init_context(int platform, int dev,int quiet) { - int printInfo=1; - if (platform >= 0 && dev >= 0) printInfo = 0; +cl_context cl_init_context(int /*platform*/, int /*dev*/, int /*quiet*/) { cl_int status; // Used to iterate through the platforms and devices, respectively @@ -401,7 +391,6 @@ cl_context cl_init_context(int platform, int dev,int quiet) { numDevices[0] = 1; devices = (cl_device_id*)malloc(sizeof(cl_device_id)*numDevices[0]); - int platform_touse = 0; int device_touse = 0; status = clGetPlatformIDs(numPlatforms, platforms, NULL); @@ -644,7 +633,11 @@ cl_mem cl_allocImage(size_t height, size_t width, char type, cl_mem_flags flags) allocationSize += height*width*elemSize; // Create the image - mem = clCreateImage2D(context, flags, &format, width, height, 0, NULL, &status); + cl_image_desc desc = {}; + desc.image_type = CL_MEM_OBJECT_IMAGE2D; + desc.image_width = width; + desc.image_height = height; + mem = clCreateImage(context, flags, &format, &desc, NULL, &status); //cl_errChk(status, "creating image", true); if(status != CL_SUCCESS) { @@ -726,7 +719,7 @@ void cl_copyBufferToHost(void* dst, cl_mem src, size_t mem_size, cl_bool blockin void cl_copyBufferToImage(cl_mem buffer, cl_mem image, int height, int width) { size_t origin[3] = {0, 0, 0}; - size_t region[3] = {width, height, 1}; + size_t region[3] = {static_cast(width), static_cast(height), 1}; cl_int status; status = clEnqueueCopyBufferToImage(commandQueue, buffer, image, 0, @@ -833,12 +826,9 @@ Compile Opencl source file into a cl_program. The cl_program will be made into a \param compileoptions Compilation options \param verbosebuild Switch to enable verbose Output */ -cl_program cl_compileProgram(char* kernelPath, char* compileoptions, bool verbosebuild ) +cl_program cl_compileProgram(char* /*kernelPath*/, char* compileoptions, bool verbosebuild ) { cl_int status; - FILE *fp = NULL; - char *source = NULL; - long int size; /*printf("\t%s\n", kernelPath); @@ -879,7 +869,6 @@ cl_program cl_compileProgram(char* kernelPath, char* compileoptions, bool verbos // read kernel binary from file uint8_t *kernel_bin = NULL; size_t kernel_size; - cl_int binary_status = 0; cl_program clProgramReturn; // Create the program object @@ -1421,8 +1410,7 @@ char* catStringWithInt(const char* string, int integer) { char tmp[6]; - strcpy(eventStr, string); - strncat(eventStr, itoa_portable(integer, tmp, 10), 5); + snprintf(eventStr, strLen, "%s%s", string, itoa_portable(integer, tmp, 10)); return eventStr; } diff --git a/tests/opencl/guassian/main.cc b/tests/opencl/guassian/main.cc index fbb99576e4..a06ebbbdcd 100755 --- a/tests/opencl/guassian/main.cc +++ b/tests/opencl/guassian/main.cc @@ -31,7 +31,10 @@ int main(int argc, char *argv[]) { if (size == 0) { fp = fopen(filename, "r"); - fscanf(fp, "%d", &size); + if (fscanf(fp, "%d", &size) != 1) { + printf("Error reading matrix size from %s\n", filename); + exit(1); + } printf("using %dx%d input matrix\n", size, size); @@ -200,8 +203,9 @@ void ForwardSub(cl_context context, float *a, float *b, float *m, int size, writeMB = (float)(sizeof(float) * size * (size + size + 1) / 1e6); // 3. Determine block sizes - size_t globalWorksizeFan1[1] = {size}; - size_t globalWorksizeFan2[2] = {size, size}; + size_t globalWorksizeFan1[1] = {static_cast(size)}; + size_t globalWorksizeFan2[2] = {static_cast(size), + static_cast(size)}; size_t localWorksizeFan1[1] = {1}; size_t localWorksizeFan2[2] = {1, 1}; @@ -425,7 +429,10 @@ void InitMat(FILE *fp, int size, float *ary, int nrow, int ncol) { for (i = 0; i < nrow; i++) { for (j = 0; j < ncol; j++) { - fscanf(fp, "%f", ary + size * i + j); + if (fscanf(fp, "%f", ary + size * i + j) != 1) { + printf("Error reading input matrix\n"); + exit(1); + } } } } @@ -438,7 +445,10 @@ void InitAry(FILE *fp, float *ary, int ary_size) { int i; for (i = 0; i < ary_size; i++) { - fscanf(fp, "%f", &ary[i]); + if (fscanf(fp, "%f", &ary[i]) != 1) { + printf("Error reading input array\n"); + exit(1); + } } } /*------------------------------------------------------ diff --git a/tests/opencl/image_get/main.cc b/tests/opencl/image_get/main.cc index 8ceae1749f..3ec48fb636 100644 --- a/tests/opencl/image_get/main.cc +++ b/tests/opencl/image_get/main.cc @@ -21,9 +21,12 @@ static int read_kernel_file(const char* fn, uint8_t** data, size_t* size) { cl_device_id device_id = NULL; cl_context context = NULL; cl_command_queue queue = NULL; cl_program program = NULL; cl_kernel kernel = NULL; cl_mem image = NULL, out_buf = NULL; static void cleanup() { - if (queue) clReleaseCommandQueue(queue); if (kernel) clReleaseKernel(kernel); - if (program) clReleaseProgram(program); if (image) clReleaseMemObject(image); - if (out_buf) clReleaseMemObject(out_buf); if (context) clReleaseContext(context); + if (queue) { clReleaseCommandQueue(queue); } + if (kernel) { clReleaseKernel(kernel); } + if (program) { clReleaseProgram(program); } + if (image) { clReleaseMemObject(image); } + if (out_buf) { clReleaseMemObject(out_buf); } + if (context) { clReleaseContext(context); } } int main(int argc, char** argv) { diff --git a/tests/opencl/image_int/main.cc b/tests/opencl/image_int/main.cc index 2665fa6ee5..af04df8148 100644 --- a/tests/opencl/image_int/main.cc +++ b/tests/opencl/image_int/main.cc @@ -23,10 +23,13 @@ cl_device_id device_id = NULL; cl_context context = NULL; cl_command_queue queue cl_program program = NULL; cl_kernel kernel = NULL; cl_sampler sampler = NULL; cl_mem src_image = NULL, dst_image = NULL; static void cleanup() { - if (queue) clReleaseCommandQueue(queue); if (kernel) clReleaseKernel(kernel); - if (program) clReleaseProgram(program); if (sampler) clReleaseSampler(sampler); - if (src_image) clReleaseMemObject(src_image); if (dst_image) clReleaseMemObject(dst_image); - if (context) clReleaseContext(context); + if (queue) { clReleaseCommandQueue(queue); } + if (kernel) { clReleaseKernel(kernel); } + if (program) { clReleaseProgram(program); } + if (sampler) { clReleaseSampler(sampler); } + if (src_image) { clReleaseMemObject(src_image); } + if (dst_image) { clReleaseMemObject(dst_image); } + if (context) { clReleaseContext(context); } } int main(int argc, char** argv) { @@ -66,7 +69,11 @@ int main(int argc, char** argv) { int errors=0; for (size_t i=0;i(n_points), 1, 1}; /// Ke Wang adjustable local group size 2013/08/07 10:37:33 size_t local_work_size = BLOCK_SIZE; // work group size is defined by // RD_WG_SIZE_0 or RD_WG_SIZE_0_0 @@ -335,10 +332,10 @@ int kmeansOCL(float **feature, /* in: [npoints][nfeatures] */ float **clusters, int *new_centers_len, float **new_centers) { int delta = 0; - int i, j, k; + int i, j; cl_int err = 0; - size_t global_work[3] = {n_points, 1, 1}; + size_t global_work[3] = {static_cast(n_points), 1, 1}; /// Ke Wang adjustable local group size 2013/08/07 10:37:33 size_t local_work_size = BLOCK_SIZE2; // work group size is defined by diff --git a/tests/opencl/kmeans/read_input.c b/tests/opencl/kmeans/read_input.c index f70e7fb300..1dfc2df346 100755 --- a/tests/opencl/kmeans/read_input.c +++ b/tests/opencl/kmeans/read_input.c @@ -126,7 +126,6 @@ int setup(int argc, char **argv) { int best_nclusters = 0; int nfeatures = 16; /* small self-contained default */ int npoints = 512; /* small self-contained default */ - float len; float **features; float **cluster_centres = NULL; @@ -192,8 +191,11 @@ int setup(int argc, char **argv) { fprintf(stderr, "Error: no such file (%s)\n", filename); exit(1); } - fread(&npoints, 1, sizeof(int), infile); - fread(&nfeatures, 1, sizeof(int), infile); + if (fread(&npoints, 1, sizeof(int), infile) != sizeof(int) || + fread(&nfeatures, 1, sizeof(int), infile) != sizeof(int)) { + fprintf(stderr, "Error: failed to read header of (%s)\n", filename); + exit(1); + } // allocate space for features[][] and read attributes of all objects buf = (float *)malloc(npoints * nfeatures * sizeof(float)); @@ -202,7 +204,11 @@ int setup(int argc, char **argv) { for (i = 1; i < npoints; i++) { features[i] = features[i - 1] + nfeatures; } - fread(buf, 1, npoints * nfeatures * sizeof(float), infile); + if (fread(buf, 1, npoints * nfeatures * sizeof(float), infile) != + npoints * nfeatures * sizeof(float)) { + fprintf(stderr, "Error: failed to read features of (%s)\n", filename); + exit(1); + } fclose(infile); } else { FILE *infile; @@ -320,8 +326,6 @@ int setup(int argc, char **argv) { } } - len = (float)((max_nclusters - min_nclusters + 1) * nloops); - printf("Number of Iteration: %d\n", nloops); // printf("Time for I/O: %.5fsec\n", io_timing); // printf("Time for Entire Clustering: %.5fsec\n", cluster_timing); diff --git a/tests/opencl/kmeans/rmse.c b/tests/opencl/kmeans/rmse.c index 03d614a624..2f6cee3c61 100755 --- a/tests/opencl/kmeans/rmse.c +++ b/tests/opencl/kmeans/rmse.c @@ -42,7 +42,7 @@ int find_nearest_point(float *pt, /* [nfeatures] */ float **pts, /* [npts][nfeatures] */ int npts) { - int index, i; + int index = 0, i; float max_dist=FLT_MAX; /* find the cluster center id with min distance to pt */ @@ -70,11 +70,13 @@ float rms_err (float **feature, /* [npoints][nfeatures] */ float ret; /* return value */ /* calculate and sum the sqaure of euclidean distance*/ +#ifdef _OPENMP #pragma omp parallel for \ shared(feature,cluster_centres) \ firstprivate(npoints,nfeatures,nclusters) \ private(i, nearest_cluster_index) \ - schedule (static) + schedule (static) +#endif for (i=0; iasync_markers != NULL) && - (timers->async_markers->timerID != INVALID_TIMERID); -} - -static struct pb_async_time_marker_list * -get_last_async(struct pb_TimerSet* timers) -{ - /* Find the last event recorded thus far */ - struct pb_async_time_marker_list * last_event = timers->async_markers; - if(last_event != NULL && last_event->timerID != INVALID_TIMERID) { - while(last_event->next != NULL && - last_event->next->timerID != INVALID_TIMERID) - last_event = last_event->next; - return last_event; - } else - return NULL; -} - -static void insert_marker(struct pb_TimerSet* tset, enum pb_TimerID timer) -{ - cl_int ciErrNum = CL_SUCCESS; - struct pb_async_time_marker_list ** new_event = &(tset->async_markers); - - while(*new_event != NULL && (*new_event)->timerID != INVALID_TIMERID) { - new_event = &((*new_event)->next); - } - - if(*new_event == NULL) { - *new_event = (struct pb_async_time_marker_list *) - malloc(sizeof(struct pb_async_time_marker_list)); - (*new_event)->marker = calloc(1, sizeof(cl_event)); - /* - // I don't think this is needed at all. I believe clEnqueueMarker 'creates' the event -#if ( __OPENCL_VERSION__ >= CL_VERSION_1_1 ) -fprintf(stderr, "Creating Marker [%d]\n", timer); - *((cl_event *)((*new_event)->marker)) = clCreateUserEvent(*clContextPtr, &ciErrNum); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Creating User Event Object!\n"); - } - ciErrNum = clSetUserEventStatus(*((cl_event *)((*new_event)->marker)), CL_QUEUED); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Setting User Event Status!\n"); - } -#endif -*/ - (*new_event)->next = NULL; - } - - /* valid event handle now aquired: insert the event record */ - (*new_event)->label = NULL; - (*new_event)->timerID = timer; - ciErrNum = clEnqueueMarker(*clCommandQueuePtr, (cl_event *)(*new_event)->marker); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Enqueueing Marker!\n"); - } - -} - -static void insert_submarker(struct pb_TimerSet* tset, char *label, enum pb_TimerID timer) -{ - cl_int ciErrNum = CL_SUCCESS; - struct pb_async_time_marker_list ** new_event = &(tset->async_markers); - - while(*new_event != NULL && (*new_event)->timerID != INVALID_TIMERID) { - new_event = &((*new_event)->next); - } - - if(*new_event == NULL) { - *new_event = (struct pb_async_time_marker_list *) - malloc(sizeof(struct pb_async_time_marker_list)); - (*new_event)->marker = calloc(1, sizeof(cl_event)); - /* -#if ( __OPENCL_VERSION__ >= CL_VERSION_1_1 ) -fprintf(stderr, "Creating SubMarker %s[%d]\n", label, timer); - *((cl_event *)((*new_event)->marker)) = clCreateUserEvent(*clContextPtr, &ciErrNum); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Creating User Event Object!\n"); - } - ciErrNum = clSetUserEventStatus(*((cl_event *)((*new_event)->marker)), CL_QUEUED); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Setting User Event Status!\n"); - } -#endif -*/ - (*new_event)->next = NULL; - } - - /* valid event handle now aquired: insert the event record */ - (*new_event)->label = label; - (*new_event)->timerID = timer; - ciErrNum = clEnqueueMarker(*clCommandQueuePtr, (cl_event *)(*new_event)->marker); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Enqueueing Marker!\n"); - } - -} - - -/* Assumes that all recorded events have completed */ -static pb_Timestamp record_async_times(struct pb_TimerSet* tset) -{ - struct pb_async_time_marker_list * next_interval = NULL; - struct pb_async_time_marker_list * last_marker = get_last_async(tset); - pb_Timestamp total_async_time = 0; - enum pb_TimerID timer; - - for(next_interval = tset->async_markers; next_interval != last_marker; - next_interval = next_interval->next) { - cl_ulong command_start=0, command_end=0; - cl_int ciErrNum = CL_SUCCESS; - - ciErrNum = clGetEventProfilingInfo(*((cl_event *)next_interval->marker), CL_PROFILING_COMMAND_END, sizeof(cl_ulong), &command_start, NULL); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error getting first EventProfilingInfo: %d\n", ciErrNum); - } - - ciErrNum = clGetEventProfilingInfo(*((cl_event *)next_interval->next->marker), CL_PROFILING_COMMAND_END, sizeof(cl_ulong), &command_end, NULL); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error getting second EventProfilingInfo: %d\n", ciErrNum); - } - - pb_Timestamp interval = (pb_Timestamp) (((double)(command_end - command_start)) / 1e3); - tset->timers[next_interval->timerID].elapsed += interval; - if (next_interval->label != NULL) { - struct pb_SubTimer *subtimer = tset->sub_timer_list[next_interval->timerID]->subtimer_list; - while (subtimer != NULL) { - if ( strcmp(subtimer->label, next_interval->label) == 0) { - subtimer->timer.elapsed += interval; - break; - } - subtimer = subtimer->next; - } - } - total_async_time += interval; - next_interval->timerID = INVALID_TIMERID; - } - - if(next_interval != NULL) - next_interval->timerID = INVALID_TIMERID; - - return total_async_time; -} - -static void -accumulate_time(pb_Timestamp *accum, - pb_Timestamp start, - pb_Timestamp end) -{ -//#if _POSIX_VERSION >= 200112L - *accum += end - start; -//#else -//# error "Timestamps not implemented for this system" -//#endif -} - -//#if _POSIX_VERSION >= 200112L -static pb_Timestamp get_time() -{ - //struct timeval tv; - //gettimeofday(&tv, NULL); - //return (pb_Timestamp) (tv.tv_sec * 1000000LL + tv.tv_usec); - return 0; -} -//#else -//# error "no supported time libraries are available on this platform" -//#endif - void pb_ResetTimer(struct pb_Timer *timer) { @@ -222,6 +40,7 @@ pb_ResetTimer(struct pb_Timer *timer) void pb_StartTimer(struct pb_Timer *timer) { + (void)timer; /*#ifndef DISABLE_PARBOIL_TIMER if (timer->state != pb_Timer_STOPPED) { fputs("Ignoring attempt to start a running timer\n", stderr); @@ -245,6 +64,8 @@ pb_StartTimer(struct pb_Timer *timer) void pb_StartTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) { + (void)timer; + (void)subtimer; /*#ifndef DISABLE_PARBOIL_TIMER unsigned int numNotStopped = 0x3; // 11 @@ -287,6 +108,7 @@ pb_StartTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) void pb_StopTimer(struct pb_Timer *timer) { + (void)timer; /*#ifndef DISABLE_PARBOIL_TIMER pb_Timestamp fini; @@ -315,6 +137,8 @@ pb_StopTimer(struct pb_Timer *timer) } void pb_StopTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) { + (void)timer; + (void)subtimer; /*#ifndef DISABLE_PARBOIL_TIMER pb_Timestamp fini; @@ -364,6 +188,7 @@ void pb_StopTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) double pb_GetElapsedTime(struct pb_Timer *timer) { + (void)timer; /*double ret; #ifndef DISABLE_PARBOIL_TIMER @@ -384,6 +209,7 @@ pb_GetElapsedTime(struct pb_Timer *timer) void pb_InitializeTimerSet(struct pb_TimerSet *timers) { + (void)timers; /*#ifndef DISABLE_PARBOIL_TIMER int n; @@ -406,6 +232,9 @@ void pb_SetOpenCL(void *p_clContextPtr, void *p_clCommandQueuePtr) { void pb_AddSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID pb_Category) { + (void)timers; + (void)label; + (void)pb_Category; /*#ifndef DISABLE_PARBOIL_TIMER struct pb_SubTimer *subtimer = (struct pb_SubTimer *) malloc @@ -440,6 +269,8 @@ pb_AddSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID pb_Categ void pb_SwitchToTimer(struct pb_TimerSet *timers, enum pb_TimerID timer) { + (void)timers; + (void)timer; #if 0 #ifndef DISABLE_PARBOIL_TIMER @@ -560,6 +391,9 @@ pb_SwitchToTimer(struct pb_TimerSet *timers, enum pb_TimerID timer) void pb_SwitchToSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID category) { + (void)timers; + (void)label; + (void)category; #if 0 #ifndef DISABLE_PARBOIL_TIMER struct pb_SubTimerList *subtimerlist = timers->sub_timer_list[timers->current]; @@ -708,6 +542,7 @@ pb_SwitchToSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID cat void pb_PrintTimerSet(struct pb_TimerSet *timers) { + (void)timers; #if 0 #ifndef DISABLE_PARBOIL_TIMER pb_Timestamp wall_end = 0; //get_time(); @@ -1198,6 +1033,7 @@ pb_GetPlatform(struct pb_PlatformParam *platform) { pb_Context* pb_InitOpenCLContext(struct pb_Parameters* parameters) { + (void)parameters; #if 0 pb_Platform* ps = pb_GetPlatform(parameters->platform); if (!ps) return NULL; @@ -1254,11 +1090,13 @@ pb_InitOpenCLContext(struct pb_Parameters* parameters) { void pb_ReleaseOpenCLContext(pb_Context* c) { + (void)c; pb_ReleasePlatforms(); } void pb_PrintPlatformInfo(pb_Context* c) { + (void)c; /*pb_Platform** ps = pb_GetPlatforms(); if (!ps) { fprintf (stderr, "No platform found"); @@ -1370,7 +1208,7 @@ pb_sig_int(char* c, int* p, int sz) { void pb_sig_uchar(char* c, unsigned char* p, unsigned int sz) { - int i; + unsigned int i; unsigned long long int s = 0; for (i = 0; i < sz; i++) s += p[i]; printf ("[Signature] %s = %lld\n", c, s); diff --git a/tests/opencl/myocyte/myocyte_model.h b/tests/opencl/myocyte/myocyte_model.h index e4212ecc55..2aff0285f7 100644 --- a/tests/opencl/myocyte/myocyte_model.h +++ b/tests/opencl/myocyte/myocyte_model.h @@ -922,7 +922,7 @@ kernel_ecc( fp timeinst, case 2: V_hold = -55; V_test = 0; - if(timeinst>0.5 & timeinst<200.5){ + if((timeinst>0.5) & (timeinst<200.5)){ V_clamp = V_test; } else{ @@ -951,7 +951,7 @@ kernel_ecc( fp timeinst, //========================================================================================================================================================================================================200 void -kernel_cam( fp timeinst, +kernel_cam( fp /*timeinst*/, GLOBAL fp *d_initvalu, GLOBAL fp *d_finavalu, int valu_offset, diff --git a/tests/opencl/nearn/clutils.cpp b/tests/opencl/nearn/clutils.cpp index b411b19b15..8cd8065195 100755 --- a/tests/opencl/nearn/clutils.cpp +++ b/tests/opencl/nearn/clutils.cpp @@ -71,9 +71,6 @@ static cl_uint numPlatforms; static cl_device_id* devices = NULL; static cl_uint* numDevices = NULL; -//! The chosen OpenCL platform -static cl_platform_id platform = NULL; - //! The chosen OpenCL device static cl_device_id device = NULL; @@ -82,11 +79,6 @@ static cl_context context = NULL; //! OpenCL command queue static cl_command_queue commandQueue = NULL; -static cl_command_queue commandQueueProf = NULL; -static cl_command_queue commandQueueNoProf = NULL; - -//! Global status of events -static bool eventsEnabled = false; //------------------------------------------------------- // Initialization and Cleanup @@ -259,9 +251,7 @@ static int read_kernel_file(const char* filename, uint8_t** data, size_t* size) return 0; } -cl_context cl_init_context(int platform, int dev,int quiet) { - int printInfo=1; - if (platform >= 0 && dev >= 0) printInfo = 0; +cl_context cl_init_context(int /*platform*/, int /*dev*/, int /*quiet*/) { cl_int status; // Used to iterate through the platforms and devices, respectively @@ -400,7 +390,6 @@ cl_context cl_init_context(int platform, int dev,int quiet) { numDevices[0] = 1; devices = (cl_device_id*)malloc(sizeof(cl_device_id)*numDevices[0]); - int platform_touse = 0; int device_touse = 0; status = clGetPlatformIDs(numPlatforms, platforms, NULL); @@ -643,7 +632,12 @@ cl_mem cl_allocImage(size_t height, size_t width, char type, cl_mem_flags flags) allocationSize += height*width*elemSize; // Create the image - mem = clCreateImage2D(context, flags, &format, width, height, 0, NULL, &status); + cl_image_desc desc; + memset(&desc, 0, sizeof(desc)); + desc.image_type = CL_MEM_OBJECT_IMAGE2D; + desc.image_width = width; + desc.image_height = height; + mem = clCreateImage(context, flags, &format, &desc, NULL, &status); //cl_errChk(status, "creating image", true); if(status != CL_SUCCESS) { @@ -725,7 +719,7 @@ void cl_copyBufferToHost(void* dst, cl_mem src, size_t mem_size, cl_bool blockin void cl_copyBufferToImage(cl_mem buffer, cl_mem image, int height, int width) { size_t origin[3] = {0, 0, 0}; - size_t region[3] = {width, height, 1}; + size_t region[3] = {static_cast(width), static_cast(height), 1}; cl_int status; status = clEnqueueCopyBufferToImage(commandQueue, buffer, image, 0, @@ -832,12 +826,9 @@ Compile Opencl source file into a cl_program. The cl_program will be made into a \param compileoptions Compilation options \param verbosebuild Switch to enable verbose Output */ -cl_program cl_compileProgram(char* kernelPath, char* compileoptions, bool verbosebuild ) +cl_program cl_compileProgram(char* /*kernelPath*/, char* compileoptions, bool verbosebuild ) { cl_int status; - FILE *fp = NULL; - char *source = NULL; - long int size; /*printf("\t%s\n", kernelPath); @@ -878,7 +869,6 @@ cl_program cl_compileProgram(char* kernelPath, char* compileoptions, bool verbos // read kernel binary from file uint8_t *kernel_bin = NULL; size_t kernel_size; - cl_int binary_status = 0; cl_program clProgramReturn; // Create the program object @@ -1420,8 +1410,7 @@ char* catStringWithInt(const char* string, int integer) { char tmp[6]; - strcpy(eventStr, string); - strncat(eventStr, itoa_portable(integer, tmp, 10), 5); + snprintf(eventStr, strLen, "%s%s", string, itoa_portable(integer, tmp, 10)); return eventStr; } diff --git a/tests/opencl/nearn/main.cc b/tests/opencl/nearn/main.cc index 973dcd9c8c..ea8238e167 100755 --- a/tests/opencl/nearn/main.cc +++ b/tests/opencl/nearn/main.cc @@ -209,8 +209,6 @@ int loadData(char *filename, std::vector &records, /**Main processing **/ - int q = 0; - flist = fopen(filename, "r"); while (!feof(flist)) { /** @@ -232,7 +230,13 @@ int loadData(char *filename, std::vector &records, while (!feof(fp)) { Record record; LatLong latLong; - fgets(record.recString, 49, fp); + if (fgets(record.recString, 49, fp) == NULL) { + if (ferror(fp)) { + printf("error reading a db\n"); + exit(1); + } + break; + } fgetc(fp); // newline if (feof(fp)) break; diff --git a/tests/opencl/oclprintf/main.cc b/tests/opencl/oclprintf/main.cc index c23e6dec08..73dc536716 100644 --- a/tests/opencl/oclprintf/main.cc +++ b/tests/opencl/oclprintf/main.cc @@ -147,7 +147,7 @@ int main (int argc, char **argv) { CL_CHECK(clEnqueueWriteBuffer(commandQueue, a_memobj, CL_TRUE, 0, nbytes, h_a, 0, NULL, NULL)); printf("Execute the kernel\n"); - size_t global_work_size[1] = {size}; + size_t global_work_size[1] = {static_cast(size)}; size_t local_work_size[1] = {1}; auto time_start = std::chrono::high_resolution_clock::now(); CL_CHECK(clEnqueueNDRangeKernel(commandQueue, kernel, 1, NULL, global_work_size, local_work_size, 0, NULL, NULL)); diff --git a/tests/opencl/psort/main.cc b/tests/opencl/psort/main.cc index 8ecfdc5239..0e72599ece 100644 --- a/tests/opencl/psort/main.cc +++ b/tests/opencl/psort/main.cc @@ -167,7 +167,7 @@ int main (int argc, char **argv) { CL_CHECK(clEnqueueWriteBuffer(commandQueue, a_memobj, CL_TRUE, 0, nbytes, h_a, 0, NULL, NULL)); printf("Execute the kernel\n"); - size_t global_work_size[1] = {size}; + size_t global_work_size[1] = {static_cast(size)}; size_t local_work_size[1] = {1}; auto time_start = std::chrono::high_resolution_clock::now(); CL_CHECK(clEnqueueNDRangeKernel(commandQueue, kernel, 1, NULL, global_work_size, local_work_size, 0, NULL, NULL)); diff --git a/tests/opencl/saxpy/main.cc b/tests/opencl/saxpy/main.cc index 2397c720e4..2cb4b71ba8 100644 --- a/tests/opencl/saxpy/main.cc +++ b/tests/opencl/saxpy/main.cc @@ -53,8 +53,8 @@ _ret; \ }) -void pfn_notify(const char *errinfo, const void *private_info, size_t cb, - void *user_data) { +void pfn_notify(const char *errinfo, const void * /*private_info*/, size_t /*cb*/, + void * /*user_data*/) { fprintf(stderr, "OpenCL Error (via pfn_notify): %s\n", errinfo); } @@ -198,7 +198,7 @@ int main(int argc, char **argv) { CL_CHECK(clSetKernelArg(kernel, 2, sizeof(factor), &factor)); size_t global_offset[1] = {0}; - size_t global_work_size[1] = {size}; + size_t global_work_size[1] = {static_cast(size)}; size_t local_work_size[1] = {1}; printf("initialize buffers\n"); diff --git a/tests/opencl/sfilter/main.cc b/tests/opencl/sfilter/main.cc index 97cfb689e7..c56d26ffdc 100644 --- a/tests/opencl/sfilter/main.cc +++ b/tests/opencl/sfilter/main.cc @@ -56,8 +56,8 @@ _ret; \ }) -void pfn_notify(const char *errinfo, const void *private_info, size_t cb, - void *user_data) { +void pfn_notify(const char *errinfo, const void * /*private_info*/, size_t /*cb*/, + void * /*user_data*/) { fprintf(stderr, "OpenCL Error (via pfn_notify): %s\n", errinfo); } @@ -220,7 +220,7 @@ int main(int argc, char **argv) { CL_CHECK(clSetKernelArg(kernel, 11, sizeof(m8), (&m8))); size_t global_offset[2] = {1, 1}; - size_t global_work_size[2] = {size - 2, size - 2}; + size_t global_work_size[2] = {static_cast(size - 2), static_cast(size - 2)}; size_t local_work_size[2] = {1, 1}; // {size-2,1} printf("enqueue write buffer\n"); diff --git a/tests/opencl/spmv/args.c b/tests/opencl/spmv/args.c index 9d751e2940..42637fe1c5 100644 --- a/tests/opencl/spmv/args.c +++ b/tests/opencl/spmv/args.c @@ -225,7 +225,7 @@ read_platform_param(char *str) /* The platform name is either the entire string, or all characters before * the separator */ - int name_length = separator_index == -1 ? strlen(str) : separator_index; + int name_length = separator_index == -1 ? (int)strlen(str) : separator_index; char *name_str = (char *)malloc(name_length + 1); memcpy(name_str, str, name_length); name_str[name_length] = 0; @@ -410,6 +410,8 @@ parse_input_files(struct argparse *ap, struct pb_Parameters *params) static result parse_end_options(struct argparse *ap, struct pb_Parameters *params) { + (void)ap; + (void)params; return ARGPARSE_DONE; } @@ -500,7 +502,6 @@ is_last_option(struct option *op) static int pb_ParseParameters (struct pb_Parameters *ret, int *_argc, char **argv) { - char *err_message; struct argparse ap; /* Each argument */ diff --git a/tests/opencl/spmv/convert_dataset.c b/tests/opencl/spmv/convert_dataset.c index aba9c3b3de..8aab7b32d4 100644 --- a/tests/opencl/spmv/convert_dataset.c +++ b/tests/opencl/spmv/convert_dataset.c @@ -83,7 +83,6 @@ int coo_to_jds(char *mtx_filename, int pad_rows, int warp_size, int pack_size, FILE *f; int nz; int i; - float *val; mat_entry *entries; row_stats *stats; int rows, cols; @@ -126,10 +125,16 @@ int coo_to_jds(char *mtx_filename, int pad_rows, int warp_size, int pack_size, for (i = 0; i < nz; i++, cur_i++) { if (!binary) { - fscanf(f, "%d %d %f\n", &entries[cur_i].row, &entries[cur_i].col, - &entries[cur_i].val); + if (fscanf(f, "%d %d %f\n", &entries[cur_i].row, &entries[cur_i].col, + &entries[cur_i].val) != 3) { + printf("Could not read matrix entry %d.\n", i); + exit(1); + } } else { - fscanf(f, "%d %d\n", &entries[cur_i].row, &entries[cur_i].col); + if (fscanf(f, "%d %d\n", &entries[cur_i].row, &entries[cur_i].col) != 2) { + printf("Could not read matrix entry %d.\n", i); + exit(1); + } entries[cur_i].val = 1.0; } entries[cur_i].row--; @@ -216,7 +221,7 @@ int coo_to_jds(char *mtx_filename, int pad_rows, int warp_size, int pack_size, if (debug_level >= 1) { printf("Padding data....%d rows, %d groups\n", rows, *nz_count_len); } - int pad_to, total_padding = 0, pack_to; + int pad_to = 0, total_padding = 0, pack_to; pad_rows *= pack_size; // change padding to account for packed items for (i = 0; i < rows; i++) { // record JDS to real row number @@ -281,7 +286,7 @@ int coo_to_jds(char *mtx_filename, int pad_rows, int warp_size, int pack_size, */ irow = 0; // keep track of which row we are in inside the fubini-ed array int idata = 0; // position within final data array - int entry_index, j; + int entry_index; int ipack; // used in internal loop for writing packed values mat_entry entry; while (1) { diff --git a/tests/opencl/spmv/file.c b/tests/opencl/spmv/file.c index a98451e56b..23709bad19 100644 --- a/tests/opencl/spmv/file.c +++ b/tests/opencl/spmv/file.c @@ -28,27 +28,50 @@ void inputData(char* fName, int* len, int* depth, int* dim,int *nzcnt_len,int *p exit(-1); } - fscanf(fid, "%d %d %d %d %d\n",len,depth,nzcnt_len,dim,pad); + if (fscanf(fid, "%d %d %d %d %d\n",len,depth,nzcnt_len,dim,pad) != 5) + { + fprintf(stderr, "Error reading input file header\n"); + exit(-1); + } int _len=len[0]; int _depth=depth[0]; int _dim=dim[0]; - int _pad=pad[0]; int _nzcnt_len=nzcnt_len[0]; *h_data = (float *) malloc(_len * sizeof (float)); - fread (*h_data, sizeof (float), _len, fid); + if (fread (*h_data, sizeof (float), _len, fid) != (size_t)_len) + { + fprintf(stderr, "Error reading input file data\n"); + exit(-1); + } *h_indices = (int *) malloc(_len * sizeof (int)); - fread (*h_indices, sizeof (int), _len, fid); + if (fread (*h_indices, sizeof (int), _len, fid) != (size_t)_len) + { + fprintf(stderr, "Error reading input file indices\n"); + exit(-1); + } *h_ptr = (int *) malloc(_depth * sizeof (int)); - fread (*h_ptr, sizeof (int), _depth, fid); + if (fread (*h_ptr, sizeof (int), _depth, fid) != (size_t)_depth) + { + fprintf(stderr, "Error reading input file row pointers\n"); + exit(-1); + } *h_perm = (int *) malloc(_dim * sizeof (int)); - fread (*h_perm, sizeof (int), _dim, fid); + if (fread (*h_perm, sizeof (int), _dim, fid) != (size_t)_dim) + { + fprintf(stderr, "Error reading input file permutation\n"); + exit(-1); + } *h_nzcnt = (int *) malloc(_nzcnt_len * sizeof (int)); - fread (*h_nzcnt, sizeof (int), _nzcnt_len, fid); + if (fread (*h_nzcnt, sizeof (int), _nzcnt_len, fid) != (size_t)_nzcnt_len) + { + fprintf(stderr, "Error reading input file nonzero counts\n"); + exit(-1); + } fclose (fid); } @@ -56,7 +79,11 @@ void inputData(char* fName, int* len, int* depth, int* dim,int *nzcnt_len,int *p void input_vec(char *fName,float *h_vec,int dim) { FILE* fid = fopen(fName, "rb"); - fread (h_vec, sizeof (float), dim, fid); + if (fread (h_vec, sizeof (float), dim, fid) != (size_t)dim) + { + fprintf(stderr, "Error reading input vector file\n"); + exit(-1); + } fclose(fid); } diff --git a/tests/opencl/spmv/main.cc b/tests/opencl/spmv/main.cc index 0be7565cd0..fada19c156 100644 --- a/tests/opencl/spmv/main.cc +++ b/tests/opencl/spmv/main.cc @@ -50,7 +50,7 @@ static float* read_output_file(const char* filename, int* out_size) { return NULL; } // Read the float data - if (fread(floats, sizeof(float), size, file) != size) { + if (fread(floats, sizeof(float), size, file) != (size_t)size) { fclose(file); free(floats); perror("Error reading floats from file"); @@ -109,15 +109,6 @@ static int read_kernel_file(const char* filename, uint8_t** data, size_t* size) return CL_SUCCESS; } -static int generate_vector(float *x_vector, int dim) { - srand(54321); - int i; - for (i = 0; i < dim; i++) { - x_vector[i] = (rand() / (float)RAND_MAX); - } - return 0; -} - int main(int argc, char **argv) { struct pb_TimerSet timers; struct pb_Parameters *parameters; @@ -146,7 +137,6 @@ int main(int argc, char **argv) { } cl_device_id clDevice = (cl_device_id)pb_context->clDeviceId; - cl_platform_id clPlatform = (cl_platform_id)pb_context->clPlatformId; cl_context clContext = (cl_context)pb_context->clContext; cl_command_queue clCommandQueue = clCreateCommandQueue( clContext, clDevice, CL_QUEUE_PROFILING_ENABLE, &clStatus); @@ -159,7 +149,6 @@ int main(int argc, char **argv) { // clCreateProgramWithSource(clContext,1,clSource,NULL,&clStatus); uint8_t *kernel_bin = NULL; size_t kernel_size; - cl_int binary_status = 0; cl_program clProgram; clStatus = read_kernel_file("kernel.cl", &kernel_bin, &kernel_size); @@ -169,7 +158,7 @@ int main(int argc, char **argv) { CHECK_ERROR("clCreateProgramWithSource") char clOptions[50]; - sprintf(clOptions, ""); + clOptions[0] = '\0'; clStatus = clBuildProgram(clProgram, 1, &clDevice, clOptions, NULL, NULL); CHECK_ERROR("clBuildProgram") diff --git a/tests/opencl/spmv/mmio.c b/tests/opencl/spmv/mmio.c index a356471851..c5eb01eb6f 100644 --- a/tests/opencl/spmv/mmio.c +++ b/tests/opencl/spmv/mmio.c @@ -72,7 +72,12 @@ int mm_read_unsymmetric_sparse(const char *fname, int *M_, int *N_, int *nz_, for (i=0; i #endif -#include "perfmon.h" cl_context *clContextPtr; cl_command_queue *clCommandQueuePtr; @@ -56,46 +55,6 @@ get_last_async(struct pb_TimerSet* timers) return NULL; } -static void insert_marker(struct pb_TimerSet* tset, enum pb_TimerID timer) -{ - cl_int ciErrNum = CL_SUCCESS; - struct pb_async_time_marker_list ** new_event = &(tset->async_markers); - - while(*new_event != NULL && (*new_event)->timerID != INVALID_TIMERID) { - new_event = &((*new_event)->next); - } - - if(*new_event == NULL) { - *new_event = (struct pb_async_time_marker_list *) - malloc(sizeof(struct pb_async_time_marker_list)); - (*new_event)->marker = calloc(1, sizeof(cl_event)); - /* - // I don't think this is needed at all. I believe clEnqueueMarker 'creates' the event -#if ( __OPENCL_VERSION__ >= CL_VERSION_1_1 ) -fprintf(stderr, "Creating Marker [%d]\n", timer); - *((cl_event *)((*new_event)->marker)) = clCreateUserEvent(*clContextPtr, &ciErrNum); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Creating User Event Object!\n"); - } - ciErrNum = clSetUserEventStatus(*((cl_event *)((*new_event)->marker)), CL_QUEUED); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Setting User Event Status!\n"); - } -#endif -*/ - (*new_event)->next = NULL; - } - - /* valid event handle now aquired: insert the event record */ - (*new_event)->label = NULL; - (*new_event)->timerID = timer; - ciErrNum = clEnqueueMarker(*clCommandQueuePtr, (cl_event *)(*new_event)->marker); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Enqueueing Marker!\n"); - } - -} - static void insert_submarker(struct pb_TimerSet* tset, char *label, enum pb_TimerID timer) { cl_int ciErrNum = CL_SUCCESS; @@ -128,7 +87,7 @@ fprintf(stderr, "Creating SubMarker %s[%d]\n", label, timer); /* valid event handle now aquired: insert the event record */ (*new_event)->label = label; (*new_event)->timerID = timer; - ciErrNum = clEnqueueMarker(*clCommandQueuePtr, (cl_event *)(*new_event)->marker); + ciErrNum = clEnqueueMarkerWithWaitList(*clCommandQueuePtr, 0, NULL, (cl_event *)(*new_event)->marker); if (ciErrNum != CL_SUCCESS) { fprintf(stderr, "Error Enqueueing Marker!\n"); } @@ -142,7 +101,6 @@ static pb_Timestamp record_async_times(struct pb_TimerSet* tset) struct pb_async_time_marker_list * next_interval = NULL; struct pb_async_time_marker_list * last_marker = get_last_async(tset); pb_Timestamp total_async_time = 0; - enum pb_TimerID timer; for(next_interval = tset->async_markers; next_interval != last_marker; next_interval = next_interval->next) { @@ -193,18 +151,6 @@ accumulate_time(pb_Timestamp *accum, //#endif } -//#if _POSIX_VERSION >= 200112L -static pb_Timestamp get_time() -{ - //struct timeval tv; - //gettimeofday(&tv, NULL); - //return (pb_Timestamp) (tv.tv_sec * 1000000LL + tv.tv_usec); - return 0; -} -//#else -//# error "no supported time libraries are available on this platform" -//#endif - void pb_ResetTimer(struct pb_Timer *timer) { @@ -222,6 +168,7 @@ pb_ResetTimer(struct pb_Timer *timer) void pb_StartTimer(struct pb_Timer *timer) { + (void)timer; /*#ifndef DISABLE_PARBOIL_TIMER if (timer->state != pb_Timer_STOPPED) { fputs("Ignoring attempt to start a running timer\n", stderr); @@ -245,6 +192,8 @@ pb_StartTimer(struct pb_Timer *timer) void pb_StartTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) { + (void)timer; + (void)subtimer; /*#ifndef DISABLE_PARBOIL_TIMER unsigned int numNotStopped = 0x3; // 11 @@ -287,6 +236,7 @@ pb_StartTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) void pb_StopTimer(struct pb_Timer *timer) { + (void)timer; /*#ifndef DISABLE_PARBOIL_TIMER pb_Timestamp fini; @@ -315,6 +265,8 @@ pb_StopTimer(struct pb_Timer *timer) } void pb_StopTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) { + (void)timer; + (void)subtimer; /*#ifndef DISABLE_PARBOIL_TIMER pb_Timestamp fini; @@ -364,6 +316,7 @@ void pb_StopTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) double pb_GetElapsedTime(struct pb_Timer *timer) { + (void)timer; /*double ret; #ifndef DISABLE_PARBOIL_TIMER @@ -384,6 +337,7 @@ pb_GetElapsedTime(struct pb_Timer *timer) void pb_InitializeTimerSet(struct pb_TimerSet *timers) { + (void)timers; /*#ifndef DISABLE_PARBOIL_TIMER int n; @@ -405,7 +359,10 @@ void pb_SetOpenCL(void *p_clContextPtr, void *p_clCommandQueuePtr) { } void -pb_AddSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID pb_Category) { +pb_AddSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID pb_Category) { + (void)timers; + (void)label; + (void)pb_Category; /*#ifndef DISABLE_PARBOIL_TIMER struct pb_SubTimer *subtimer = (struct pb_SubTimer *) malloc @@ -440,6 +397,8 @@ pb_AddSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID pb_Categ void pb_SwitchToTimer(struct pb_TimerSet *timers, enum pb_TimerID timer) { + (void)timers; + (void)timer; #if 0 #ifndef DISABLE_PARBOIL_TIMER @@ -732,8 +691,9 @@ pb_PrintTimerSet(struct pb_TimerSet *timers) maxSubLength = 0; while (sub != NULL) { // Find longest SubTimer label - if (strlen(sub->label) > maxSubLength) { - maxSubLength = strlen(sub->label); + int labelLength = (int)strlen(sub->label); + if (labelLength > maxSubLength) { + maxSubLength = labelLength; } sub = sub->next; } @@ -1194,6 +1154,7 @@ pb_GetPlatform(struct pb_PlatformParam *platform) { pb_Context* pb_InitOpenCLContext(struct pb_Parameters* parameters) { + (void)parameters; #if 0 pb_Platform* ps = pb_GetPlatform(parameters->platform); if (!ps) return NULL; @@ -1235,11 +1196,13 @@ pb_InitOpenCLContext(struct pb_Parameters* parameters) { void pb_ReleaseOpenCLContext(pb_Context* c) { + (void)c; pb_ReleasePlatforms(); } void pb_PrintPlatformInfo(pb_Context* c) { + (void)c; /*pb_Platform** ps = pb_GetPlatforms(); if (!ps) { fprintf (stderr, "No platform found"); @@ -1351,7 +1314,7 @@ pb_sig_int(char* c, int* p, int sz) { void pb_sig_uchar(char* c, unsigned char* p, unsigned int sz) { - int i; + unsigned int i; unsigned long long int s = 0; for (i = 0; i < sz; i++) s += p[i]; printf ("[Signature] %s = %lld\n", c, s); diff --git a/tests/opencl/srad/main.cc b/tests/opencl/srad/main.cc index 3ef2cfa1a1..c15d1deb8f 100644 --- a/tests/opencl/srad/main.cc +++ b/tests/opencl/srad/main.cc @@ -129,7 +129,7 @@ static void parse_args(int argc, char** argv) { // Serial CPU reference: identical single-precision SRAD math over the same // (extracted) image. Mirrors extract -> [stats, srad, srad2]*niter -> compress. -static void srad_cpu(std::vector& I, int Nr, int Nc, long Ne, int niter, +static void srad_cpu(std::vector& I, int Nr, long Ne, int niter, float lambda, long NeROI, const std::vector& iN, const std::vector& iS, const std::vector& jE, const std::vector& jW) { @@ -342,7 +342,7 @@ int main(int argc, char** argv) { // CPU golden reference over the same seeded input. std::vector h_ref(h_image); - srad_cpu(h_ref, Nr, Nc, Ne, niter, lambda, NeROI, iN, iS, jE, jW); + srad_cpu(h_ref, Nr, Ne, niter, lambda, NeROI, iN, iS, jE, jW); // Compare with a mixed absolute/relative float tolerance (exp/log + iterated // accumulation across host and device libm differ by a few ULP). diff --git a/tests/opencl/stencil/args.c b/tests/opencl/stencil/args.c index 9d751e2940..42637fe1c5 100644 --- a/tests/opencl/stencil/args.c +++ b/tests/opencl/stencil/args.c @@ -225,7 +225,7 @@ read_platform_param(char *str) /* The platform name is either the entire string, or all characters before * the separator */ - int name_length = separator_index == -1 ? strlen(str) : separator_index; + int name_length = separator_index == -1 ? (int)strlen(str) : separator_index; char *name_str = (char *)malloc(name_length + 1); memcpy(name_str, str, name_length); name_str[name_length] = 0; @@ -410,6 +410,8 @@ parse_input_files(struct argparse *ap, struct pb_Parameters *params) static result parse_end_options(struct argparse *ap, struct pb_Parameters *params) { + (void)ap; + (void)params; return ARGPARSE_DONE; } @@ -500,7 +502,6 @@ is_last_option(struct option *op) static int pb_ParseParameters (struct pb_Parameters *ret, int *_argc, char **argv) { - char *err_message; struct argparse ap; /* Each argument */ diff --git a/tests/opencl/stencil/file.c b/tests/opencl/stencil/file.c index a9b39b1935..4b44376df6 100644 --- a/tests/opencl/stencil/file.c +++ b/tests/opencl/stencil/file.c @@ -26,9 +26,13 @@ void inputData(char* fName, int* nx, int* ny, int* nz) exit(-1); } - fread(nx, sizeof(int ),1,fid); - fread(ny, sizeof(int ),1,fid); - fread(nz, sizeof(int ),1,fid); + if (fread(nx, sizeof(int ),1,fid) != 1 || + fread(ny, sizeof(int ),1,fid) != 1 || + fread(nz, sizeof(int ),1,fid) != 1) + { + fprintf(stderr, "Error reading input file\n"); + exit(-1); + } fclose (fid); } diff --git a/tests/opencl/stencil/main.cc b/tests/opencl/stencil/main.cc index 0018d00d70..30dee81bd0 100644 --- a/tests/opencl/stencil/main.cc +++ b/tests/opencl/stencil/main.cc @@ -35,7 +35,10 @@ static int read_data(float *A0, int nx,int ny,int nz,FILE *fp) { for(k=0;kclPlatformId; clContext = (cl_context) pb_context->clContext; clDevice = (cl_device_id) pb_context->clDeviceId; @@ -208,7 +209,6 @@ int main(int argc, char** argv) { //cl_program clProgram = clCreateProgramWithSource(clContext,1,clSource,NULL,&clStatus); uint8_t *kernel_bin = NULL; size_t kernel_size; - cl_int binary_status = 0; cl_program clProgram; clStatus = read_kernel_file("kernel.cl", &kernel_bin, &kernel_size); @@ -241,7 +241,10 @@ int main(int argc, char** argv) { h_Anext=(float*)malloc(sizeof(float)*size); pb_SwitchToTimer(&timers, pb_TimerID_IO); FILE *fp = fopen(parameters->inpFiles[0], "rb"); - read_data(h_A0, nx,ny,nz, fp); + if (read_data(h_A0, nx,ny,nz, fp) != 0) { + fclose(fp); + return -1; + } fclose(fp); memcpy (h_Anext,h_A0,sizeof(float)*size); @@ -264,9 +267,8 @@ int main(int argc, char** argv) { //only use 1D thread block int tx = 128; size_t block[3] = {1,1,1}; // {tx,1,1} - size_t grid[3] = {(nx-2+tx-1)/tx*tx,ny-2,nz-2}; + size_t grid[3] = {static_cast((nx-2+tx-1)/tx*tx),static_cast(ny-2),static_cast(nz-2)}; //size_t grid[3] = {nx-2,ny-2,nz-2}; - size_t offset[3] = {1,1,1}; printf("grid size in x/y/z = %ld %ld %ld\n",grid[0],grid[1],grid[2]); printf("block size in x/y/z = %ld %ld %ld\n",block[0],block[1],block[2]); diff --git a/tests/opencl/stencil/ocl.c b/tests/opencl/stencil/ocl.c index 9ce9a2f5d2..08b3aa06b5 100644 --- a/tests/opencl/stencil/ocl.c +++ b/tests/opencl/stencil/ocl.c @@ -27,7 +27,7 @@ char* readFile(const char* fileName) } size_t res = fread(buffer,1,size,fp); - if(res != size) + if(res != (size_t)size) { printf("Error 3!\n"); fclose(fp); diff --git a/tests/opencl/stencil/parboil_opencl.c b/tests/opencl/stencil/parboil_opencl.c index 7348dc2c0d..3caf4967de 100644 --- a/tests/opencl/stencil/parboil_opencl.c +++ b/tests/opencl/stencil/parboil_opencl.c @@ -56,46 +56,6 @@ get_last_async(struct pb_TimerSet* timers) return NULL; } -static void insert_marker(struct pb_TimerSet* tset, enum pb_TimerID timer) -{ - cl_int ciErrNum = CL_SUCCESS; - struct pb_async_time_marker_list ** new_event = &(tset->async_markers); - - while(*new_event != NULL && (*new_event)->timerID != INVALID_TIMERID) { - new_event = &((*new_event)->next); - } - - if(*new_event == NULL) { - *new_event = (struct pb_async_time_marker_list *) - malloc(sizeof(struct pb_async_time_marker_list)); - (*new_event)->marker = calloc(1, sizeof(cl_event)); - /* - // I don't think this is needed at all. I believe clEnqueueMarker 'creates' the event -#if ( __OPENCL_VERSION__ >= CL_VERSION_1_1 ) -fprintf(stderr, "Creating Marker [%d]\n", timer); - *((cl_event *)((*new_event)->marker)) = clCreateUserEvent(*clContextPtr, &ciErrNum); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Creating User Event Object!\n"); - } - ciErrNum = clSetUserEventStatus(*((cl_event *)((*new_event)->marker)), CL_QUEUED); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Setting User Event Status!\n"); - } -#endif -*/ - (*new_event)->next = NULL; - } - - /* valid event handle now aquired: insert the event record */ - (*new_event)->label = NULL; - (*new_event)->timerID = timer; - ciErrNum = clEnqueueMarker(*clCommandQueuePtr, (cl_event *)(*new_event)->marker); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Enqueueing Marker!\n"); - } - -} - static void insert_submarker(struct pb_TimerSet* tset, char *label, enum pb_TimerID timer) { cl_int ciErrNum = CL_SUCCESS; @@ -128,7 +88,7 @@ fprintf(stderr, "Creating SubMarker %s[%d]\n", label, timer); /* valid event handle now aquired: insert the event record */ (*new_event)->label = label; (*new_event)->timerID = timer; - ciErrNum = clEnqueueMarker(*clCommandQueuePtr, (cl_event *)(*new_event)->marker); + ciErrNum = clEnqueueMarkerWithWaitList(*clCommandQueuePtr, 0, NULL, (cl_event *)(*new_event)->marker); if (ciErrNum != CL_SUCCESS) { fprintf(stderr, "Error Enqueueing Marker!\n"); } @@ -142,7 +102,6 @@ static pb_Timestamp record_async_times(struct pb_TimerSet* tset) struct pb_async_time_marker_list * next_interval = NULL; struct pb_async_time_marker_list * last_marker = get_last_async(tset); pb_Timestamp total_async_time = 0; - enum pb_TimerID timer; for(next_interval = tset->async_markers; next_interval != last_marker; next_interval = next_interval->next) { @@ -193,18 +152,6 @@ accumulate_time(pb_Timestamp *accum, //#endif } -//#if _POSIX_VERSION >= 200112L -static pb_Timestamp get_time() -{ - //struct timeval tv; - //gettimeofday(&tv, NULL); - //return (pb_Timestamp) (tv.tv_sec * 1000000LL + tv.tv_usec); - return 0; -} -//#else -//# error "no supported time libraries are available on this platform" -//#endif - void pb_ResetTimer(struct pb_Timer *timer) { @@ -222,6 +169,7 @@ pb_ResetTimer(struct pb_Timer *timer) void pb_StartTimer(struct pb_Timer *timer) { + (void)timer; /*#ifndef DISABLE_PARBOIL_TIMER if (timer->state != pb_Timer_STOPPED) { fputs("Ignoring attempt to start a running timer\n", stderr); @@ -245,6 +193,8 @@ pb_StartTimer(struct pb_Timer *timer) void pb_StartTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) { + (void)timer; + (void)subtimer; /*#ifndef DISABLE_PARBOIL_TIMER unsigned int numNotStopped = 0x3; // 11 @@ -287,6 +237,7 @@ pb_StartTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) void pb_StopTimer(struct pb_Timer *timer) { + (void)timer; /*#ifndef DISABLE_PARBOIL_TIMER pb_Timestamp fini; @@ -315,6 +266,8 @@ pb_StopTimer(struct pb_Timer *timer) } void pb_StopTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) { + (void)timer; + (void)subtimer; /*#ifndef DISABLE_PARBOIL_TIMER pb_Timestamp fini; @@ -364,6 +317,7 @@ void pb_StopTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) double pb_GetElapsedTime(struct pb_Timer *timer) { + (void)timer; /*double ret; #ifndef DISABLE_PARBOIL_TIMER @@ -384,6 +338,7 @@ pb_GetElapsedTime(struct pb_Timer *timer) void pb_InitializeTimerSet(struct pb_TimerSet *timers) { + (void)timers; /*#ifndef DISABLE_PARBOIL_TIMER int n; @@ -405,7 +360,10 @@ void pb_SetOpenCL(void *p_clContextPtr, void *p_clCommandQueuePtr) { } void -pb_AddSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID pb_Category) { +pb_AddSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID pb_Category) { + (void)timers; + (void)label; + (void)pb_Category; /*#ifndef DISABLE_PARBOIL_TIMER struct pb_SubTimer *subtimer = (struct pb_SubTimer *) malloc @@ -440,6 +398,8 @@ pb_AddSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID pb_Categ void pb_SwitchToTimer(struct pb_TimerSet *timers, enum pb_TimerID timer) { + (void)timers; + (void)timer; #if 0 #ifndef DISABLE_PARBOIL_TIMER @@ -732,8 +692,9 @@ pb_PrintTimerSet(struct pb_TimerSet *timers) maxSubLength = 0; while (sub != NULL) { // Find longest SubTimer label - if (strlen(sub->label) > maxSubLength) { - maxSubLength = strlen(sub->label); + int labelLength = (int)strlen(sub->label); + if (labelLength > maxSubLength) { + maxSubLength = labelLength; } sub = sub->next; } @@ -1194,6 +1155,7 @@ pb_GetPlatform(struct pb_PlatformParam *platform) { pb_Context* pb_InitOpenCLContext(struct pb_Parameters* parameters) { + (void)parameters; #if 0 pb_Platform* ps = pb_GetPlatform(parameters->platform); if (!ps) return NULL; @@ -1235,11 +1197,13 @@ pb_InitOpenCLContext(struct pb_Parameters* parameters) { void pb_ReleaseOpenCLContext(pb_Context* c) { + (void)c; pb_ReleasePlatforms(); } void pb_PrintPlatformInfo(pb_Context* c) { + (void)c; /*pb_Platform** ps = pb_GetPlatforms(); if (!ps) { fprintf (stderr, "No platform found"); @@ -1351,7 +1315,7 @@ pb_sig_int(char* c, int* p, int sz) { void pb_sig_uchar(char* c, unsigned char* p, unsigned int sz) { - int i; + unsigned int i; unsigned long long int s = 0; for (i = 0; i < sz; i++) s += p[i]; printf ("[Signature] %s = %lld\n", c, s); diff --git a/tests/opencl/transpose/main.cc b/tests/opencl/transpose/main.cc index 93c3a4fdbd..8a2148b4c3 100644 --- a/tests/opencl/transpose/main.cc +++ b/tests/opencl/transpose/main.cc @@ -62,7 +62,7 @@ static double transposeGPU(const char* kernelName, bool useLocalMem, cl_uint ci { cl_mem d_odata[MAX_GPU_COUNT]; cl_mem d_idata[MAX_GPU_COUNT]; - cl_kernel ckKernel[MAX_GPU_COUNT]; + cl_kernel ckKernel[MAX_GPU_COUNT] = {}; size_t szGlobalWorkSize[2]; size_t szLocalWorkSize[2]; @@ -317,7 +317,6 @@ int runTest( const int argc, const char** argv) // (const char **)&source, &program_length, &ciErrNum); uint8_t *kernel_bin = NULL; size_t kernel_size; - cl_int binary_status = 0; ciErrNum = read_kernel_file("kernel.cl", &kernel_bin, &kernel_size); if (ciErrNum != CL_SUCCESS) { @@ -349,7 +348,7 @@ int runTest( const int argc, const char** argv) //double sharedCopyTime = transposeGPU("shared_copy", true, ciDeviceCount, h_idata, h_odata, size_x, size_y); #endif - double naiveTime = transposeGPU("transpose_naive", false, ciDeviceCount, h_idata, h_odata, size_x, size_y); + transposeGPU("transpose_naive", false, ciDeviceCount, h_idata, h_odata, size_x, size_y); //double optimizedTime = transposeGPU("transpose", true, ciDeviceCount, h_idata, h_odata, size_x, size_y); #ifdef GPU_PROFILING diff --git a/tests/opencl/transpose/shrUtils.cpp b/tests/opencl/transpose/shrUtils.cpp index 23e4488d27..89c0b7169b 100644 --- a/tests/opencl/transpose/shrUtils.cpp +++ b/tests/opencl/transpose/shrUtils.cpp @@ -203,7 +203,6 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list { static FILE* pFileStream0 = NULL; static FILE* pFileStream1 = NULL; - size_t szNumWritten = 0; char cFileMode [3]; // if the sample log file is closed and the call incudes a "write-to-file", open file for writing @@ -318,12 +317,12 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list // print string to console if flagged if (iLogMode & LOGCONSOLE) { - szNumWritten = printf ("\n !!! Error # %i at ", iErrNum); // console + printf ("\n !!! Error # %i at ", iErrNum); // console } // print string to file if flagged if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, "\n !!! Error # %i at ", iErrNum); // sample log file + fprintf (pFileStream0, "\n !!! Error # %i at ", iErrNum); // sample log file } } @@ -347,14 +346,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list // character is not '%', so print it verbatim to console and/or files as flagged if (iLogMode & LOGCONSOLE) { - szNumWritten = putc(*pStr, stdout); // console + putc(*pStr, stdout); // console } if (iLogMode & LOGFILE) { - szNumWritten = putc(*pStr, pFileStream0); // sample log file + putc(*pStr, pFileStream0); // sample log file if (iLogMode & MASTER) { - szNumWritten = putc(*pStr, pFileStream1); // master log file + putc(*pStr, pFileStream1); // master log file } } } @@ -399,16 +398,26 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list { case '%': // special handling for string of %%%% { + // collapse each "%%" to a literal '%', as printf would + std::string sLiteral; + for (size_t i = 0; i < sFormatSpec.size(); ++i) + { + sLiteral += sFormatSpec[i]; + if ((sFormatSpec[i] == '%') && (i + 1 < sFormatSpec.size()) && (sFormatSpec[i + 1] == '%')) + { + ++i; + } + } if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str()); // console + fputs(sLiteral.c_str(), stdout); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str()); // sample log file + fputs(sLiteral.c_str(), pFileStream0); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str()); // master log file + fputs(sLiteral.c_str(), pFileStream1); // master log file } } continue; @@ -420,14 +429,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list cArg = va_arg(vaArgList, char*); if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str(), cArg); // console + printf(sFormatSpec.c_str(), cArg); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str(), cArg); // sample log file + fprintf (pFileStream0, sFormatSpec.c_str(), cArg); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str(), cArg); // master log file + fprintf(pFileStream1, sFormatSpec.c_str(), cArg); // master log file } } continue; @@ -439,14 +448,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list iArg = va_arg(vaArgList, int); if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str(), iArg); // console + printf(sFormatSpec.c_str(), iArg); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str(), iArg); // sample log file + fprintf (pFileStream0, sFormatSpec.c_str(), iArg); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str(), iArg); // master log file + fprintf(pFileStream1, sFormatSpec.c_str(), iArg); // master log file } } continue; @@ -460,14 +469,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list uiArg = va_arg(vaArgList, unsigned int); if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str(), uiArg); // console + printf(sFormatSpec.c_str(), uiArg); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str(), uiArg); // sample log file + fprintf (pFileStream0, sFormatSpec.c_str(), uiArg); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str(), uiArg); // master log file + fprintf(pFileStream1, sFormatSpec.c_str(), uiArg); // master log file } } continue; @@ -484,14 +493,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list dArg = va_arg(vaArgList, double); if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str(), dArg); // console + printf(sFormatSpec.c_str(), dArg); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str(), dArg); // sample log file + fprintf (pFileStream0, sFormatSpec.c_str(), dArg); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str(), dArg); // master log file + fprintf(pFileStream1, sFormatSpec.c_str(), dArg); // master log file } } continue; @@ -501,14 +510,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list // print arg of unknown/unsupported type to console and/or file if flagged if (iLogMode & LOGCONSOLE) // console { - szNumWritten = putc(*pStr, stdout); + putc(*pStr, stdout); } if (iLogMode & LOGFILE) { - szNumWritten = putc(*pStr, pFileStream0); // sample log file + putc(*pStr, pFileStream0); // sample log file if (iLogMode & MASTER) { - szNumWritten = putc(*pStr, pFileStream1); // master log file + putc(*pStr, pFileStream1); // master log file } } } diff --git a/tests/riscv/common.mk b/tests/riscv/common.mk index ef9ec7332b..b3c3a9413d 100644 --- a/tests/riscv/common.mk +++ b/tests/riscv/common.mk @@ -41,8 +41,9 @@ BENCH_LIST := median qsort rsort towers memcpy multiply dhrystone spmv # (VX_MEM_IO_EXIT_CODE, VX_MEM_IO_COUT_*) via the patched crt.S + # syscalls.c, so the stamp must invalidate when those values change. # Listing $(ROOT_DIR)/sw/VX_types.h as a prerequisite forces a rebuild -# after a memory-map shift (e.g. a COUT region size change). -$(RISCV_TESTS_STAMP): $(ROOT_DIR)/sw/VX_types.h +# after a memory-map shift (e.g. a COUT region size change), and listing +# the patch re-applies an edited patch to a fresh clone. +$(RISCV_TESTS_STAMP): $(ROOT_DIR)/sw/VX_types.h $(VORTEX_HOME)/miscs/patches/riscv-benchmarks.patch rm -rf $(RISCV_TESTS_DIR) git clone $(RISCV_TESTS_REPO) $(RISCV_TESTS_DIR) cd $(RISCV_TESTS_DIR) && git checkout --quiet $(RISCV_TESTS_COMMIT) && git submodule update --init --recursive diff --git a/tests/unittest/common.mk b/tests/unittest/common.mk index 4a0139e677..07887cbafd 100644 --- a/tests/unittest/common.mk +++ b/tests/unittest/common.mk @@ -1,7 +1,7 @@ ROOT_DIR := $(realpath ../../..) -CXXFLAGS += -std=c++17 -Wall -Wextra -pedantic -Wfatal-errors +CXXFLAGS += -std=c++17 -Wall -Wextra -pedantic -Wfatal-errors -Werror CXXFLAGS += -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw -I$(SW_COMMON_DIR) CXXFLAGS += $(CONFIGS) diff --git a/tests/vulkan/common.mk b/tests/vulkan/common.mk index a3a6453258..ccdaeabd90 100644 --- a/tests/vulkan/common.mk +++ b/tests/vulkan/common.mk @@ -70,7 +70,7 @@ CONFIGS += -DVX_CFG_EXT_A_ENABLE GLSLC ?= glslc CC ?= cc -CFLAGS += -O2 -Wall +CFLAGS += -O2 -Wall -Wextra -Wfatal-errors -Werror LDLIBS += -lvulkan # One SPIR-V module per shader source, stage kept in the name so a diff --git a/tests/vulkan/subgroup_vs/main.c b/tests/vulkan/subgroup_vs/main.c index ae42a7f810..1538dafc3d 100644 --- a/tests/vulkan/subgroup_vs/main.c +++ b/tests/vulkan/subgroup_vs/main.c @@ -443,7 +443,7 @@ main(int argc, char **argv) return 1; } - struct sg_report r = { 0, 0, 0, 0 }; + struct sg_report r = { 0 }; int rc = render_subgroup(dev, queue, qf, &mp, vs, fs, &r); vkDestroyShaderModule(dev, vs, NULL); From 57f28d2f31e549226ca2d5f59c44de50c02f2671 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 10 Oct 2026 22:17:50 -0700 Subject: [PATCH 012/124] gen_config: accept enum flag overrides whose value contains '_' A bare -D_ override was recognised by splitting the flag at its last underscore, so a value that itself contains one (e.g. -DVX_CFG_PLATFORM_MEMORY_TYPE_HBM2_A100) never matched its enum: the generator silently kept the default, and the profile merge treated the flag and the profile's entry as different knobs and passed both. The enum is now matched by name prefix and the rest checked against its declared values, in both places. Output is byte-identical to before for all 247 CONFIGS strings in ci/testcases at both XLENs. Co-Authored-By: Claude Opus 5.5 --- ci/gen_config.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/ci/gen_config.py b/ci/gen_config.py index ea3436c49d..316c8d0a97 100755 --- a/ci/gen_config.py +++ b/ci/gen_config.py @@ -313,6 +313,13 @@ class EnumSpec: values: List[Any] +def _enum_of_flag(name: str, enums: Dict[str, EnumSpec]) -> Optional[str]: + # The enum a bare -D_ flag selects. A value may itself contain + # '_', so the enum name is matched as a prefix instead of splitting the flag. + owners = [e for e in enums if name.startswith(e + "_")] + return max(owners, key=len) if owners else None + + def _load_enums(toml_data: Dict[str, Any]) -> Dict[str, EnumSpec]: enums = toml_data.get("enum", None) if enums is None: @@ -454,10 +461,10 @@ def set_enum(name: str, val: Any) -> None: set_enum(d.name, _scalar(d.value)) continue - if "_" in d.name and d.value is None: - base, suffix = d.name.rsplit("_", 1) - if base in enums: - set_enum(base, _scalar(suffix)) + if d.value is None: + base = _enum_of_flag(d.name, enums) + if base is not None: + set_enum(base, _scalar(d.name[len(base) + 1:])) overrides[d.name] = True explicit.add(d.name) continue @@ -574,9 +581,9 @@ def _canonical_define(tok: str, enums: Dict[str, EnumSpec]) -> Optional[str]: return d.name[:-len("_DISABLE")] + "_ENABLE" if d.name in enums: return d.name - if d.value is None and "_" in d.name: - base, _ = d.name.rsplit("_", 1) - if base in enums: + if d.value is None: + base = _enum_of_flag(d.name, enums) + if base is not None: return base return d.name From 4b924f24d0f4ad3e7e398c61bd8ee947a11f8524 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 10 Oct 2026 22:18:07 -0700 Subject: [PATCH 013/124] caps: report exact warp, core and cluster counts GPU_DEV_CAPS encoded threads, warps, socket/cluster sizes and clusters as 3-bit log2 fields, so a machine that is not a power of two (48 warps, 84 or 108 cores) reported the wrong shape to the runtime. The spare bits [63:42] now carry exact warps (7 bits), sockets per cluster (8) and clusters (7), produced identically by the RTL CP regfile and the C++ CP model. The decoder prefers them and falls back to the log2 fields only when they read zero, so devices built before this change keep reporting correctly. Verified: every simx/rtlsim test in the matrix so far decodes these caps (NUM_WARPS/NUM_CORES drive the kernel launch), and the A100-shaped 108-core SimX run reports 108 cores. Co-Authored-By: Claude Opus 5.5 --- docs/designs/command_processor.md | 11 +++++++++-- hw/rtl/cp/VX_cp_axil_regfile.sv | 7 ++++++- sim/common/cmd_processor.cpp | 8 +++++++- sw/runtime/common/caps.h | 20 ++++++++++++++------ 4 files changed, 36 insertions(+), 10 deletions(-) diff --git a/docs/designs/command_processor.md b/docs/designs/command_processor.md index c7f7e93962..118c5dbdcb 100644 --- a/docs/designs/command_processor.md +++ b/docs/designs/command_processor.md @@ -419,10 +419,17 @@ CP regfile is the only producer on every target. | `[33:26]` | 8 | log2 local_mem_size | `VX_CAPS_LOCAL_MEM_SIZE` | | `[36:34]` | 3 | log2 num_mem_banks | `VX_CAPS_NUM_MEM_BANKS` | | `[41:37]` | 5 | mem_bank_size, biased by 20 | `VX_CAPS_MEM_BANK_SIZE` | -| `[63:42]` | 22 | reserved | — | +| `[48:42]` | 7 | warps/core | `VX_CAPS_NUM_WARPS` | +| `[56:49]` | 8 | cluster_size | (only via `NUM_CORES`) | +| `[63:57]` | 7 | num_clusters | `VX_CAPS_NUM_CLUSTERS` | + +The exact counts in `[63:42]` let a machine that is not a power of two +(e.g. 48 warps, 84 cores) report its true shape; the decoder prefers them and +falls back to the log2 fields only when they read zero, as on devices built +before they existed. `VX_CAPS_NUM_CORES` is derived, not a field: -`num_clusters × cluster_size × socket_size`. `CACHE_LINE_SIZE`, +`num_clusters × cluster_size × socket_size`, from the exact counts. `CACHE_LINE_SIZE`, `GLOBAL_MEM_SIZE`, `CLOCK_RATE`, `PEAK_MEM_BW` and the VM ids are not encoded here — `decode_caps` returns false and the backend resolves them. diff --git a/hw/rtl/cp/VX_cp_axil_regfile.sv b/hw/rtl/cp/VX_cp_axil_regfile.sv index db425cbb2c..806a996da9 100644 --- a/hw/rtl/cp/VX_cp_axil_regfile.sv +++ b/hw/rtl/cp/VX_cp_axil_regfile.sv @@ -128,8 +128,13 @@ module VX_cp_axil_regfile localparam int GPU_BANK_ADDR_W = `VX_CFG_PLATFORM_MEMORY_ADDR_WIDTH - `CLOG2(`VX_CFG_PLATFORM_MEMORY_NUM_BANKS); + `STATIC_ASSERT(`VX_CFG_NUM_WARPS < 128 && `VX_CFG_NUM_CLUSTERS < 128 && GPU_CLUSTER_SIZE < 256, + ("GPU_DEV_CAPS exact-count fields overflow")) + wire [63:0] gpu_dev_caps = { - 22'b0, + 7'(`VX_CFG_NUM_CLUSTERS), + 8'(GPU_CLUSTER_SIZE), + 7'(`VX_CFG_NUM_WARPS), 5'(GPU_BANK_ADDR_W - 20), 3'($clog2(`VX_CFG_PLATFORM_MEMORY_NUM_BANKS)), 8'(`VX_CFG_LMEM_ENABLED ? `VX_CFG_LMEM_LOG_SIZE : 0), diff --git a/sim/common/cmd_processor.cpp b/sim/common/cmd_processor.cpp index 85083ef49d..0972440954 100644 --- a/sim/common/cmd_processor.cpp +++ b/sim/common/cmd_processor.cpp @@ -26,6 +26,9 @@ constexpr unsigned cp_clog2(uint64_t n) { } uint64_t gpu_dev_caps() { const unsigned cluster_size = VX_CFG_NUM_CORES / VX_CFG_SOCKET_SIZE; + static_assert(VX_CFG_NUM_WARPS < 128 && VX_CFG_NUM_CLUSTERS < 128 + && VX_CFG_NUM_CORES / VX_CFG_SOCKET_SIZE < 256, + "GPU_DEV_CAPS exact-count fields overflow"); const unsigned bank_addr_w = VX_CFG_PLATFORM_MEMORY_ADDR_WIDTH - cp_clog2(VX_CFG_PLATFORM_MEMORY_NUM_BANKS); return (uint64_t(VX_ISA_IMPL_ID) & 0xFF) @@ -37,7 +40,10 @@ uint64_t gpu_dev_caps() { | ((uint64_t(cp_clog2(VX_CFG_ISSUE_WIDTH)) & 0x7) << 23) | ((uint64_t(VX_CFG_LMEM_ENABLED ? VX_CFG_LMEM_LOG_SIZE : 0) & 0xFF) << 26) | ((uint64_t(cp_clog2(VX_CFG_PLATFORM_MEMORY_NUM_BANKS)) & 0x7) << 34) - | ((uint64_t(bank_addr_w - 20) & 0x1F) << 37); + | ((uint64_t(bank_addr_w - 20) & 0x1F) << 37) + | ((uint64_t(VX_CFG_NUM_WARPS) & 0x7F) << 42) + | ((uint64_t(cluster_size) & 0xFF) << 49) + | ((uint64_t(VX_CFG_NUM_CLUSTERS) & 0x7F) << 57); } uint64_t gpu_isa_caps() { return (uint64_t(VX_CFG_MISA_EXT) << 32) diff --git a/sw/runtime/common/caps.h b/sw/runtime/common/caps.h index ca30664017..ddb1def115 100644 --- a/sw/runtime/common/caps.h +++ b/sw/runtime/common/caps.h @@ -37,6 +37,14 @@ static constexpr uint32_t CP_REG_GPU_DEV_CAPS_HI = 0x01C; static constexpr uint32_t CP_REG_GPU_ISA_CAPS_LO = 0x020; static constexpr uint32_t CP_REG_GPU_ISA_CAPS_HI = 0x024; +// Warps, sockets per cluster and clusters carry an exact count, so a machine +// that is not a power of two reports truly; the field is zero on devices +// built before it existed, which report only the 3-bit log2 field. +inline uint64_t exact_or_log2(uint64_t dev_caps, uint32_t exact_lsb, uint64_t exact_mask, uint32_t log2_lsb) { + uint64_t exact = (dev_caps >> exact_lsb) & exact_mask; + return exact ? exact : (1ull << ((dev_caps >> log2_lsb) & 0x7)); +} + // Decode a VX_CAPS_* id from the two CP capability words. Returns true // and writes *out when the id is encoded in the words; returns false // for ids the caller must resolve itself (CACHE_LINE_SIZE, @@ -51,20 +59,20 @@ inline bool decode_caps(uint64_t dev_caps, uint64_t isa_caps, *out = 1ull << ((dev_caps >> 8) & 0x7); return true; case VX_CAPS_NUM_WARPS: - *out = 1ull << ((dev_caps >> 11) & 0x7); + *out = exact_or_log2(dev_caps, 42, 0x7F, 11); return true; case VX_CAPS_NUM_CORES: { - uint32_t socket_size = 1u << ((dev_caps >> 14) & 0x7); - uint32_t cluster_size = 1u << ((dev_caps >> 17) & 0x7); - uint32_t num_clusters = 1u << ((dev_caps >> 20) & 0x7); - *out = uint64_t(num_clusters) * cluster_size * socket_size; + uint64_t socket_size = 1ull << ((dev_caps >> 14) & 0x7); + uint64_t cluster_size = exact_or_log2(dev_caps, 49, 0xFF, 17); + uint64_t num_clusters = exact_or_log2(dev_caps, 57, 0x7F, 20); + *out = num_clusters * cluster_size * socket_size; return true; } case VX_CAPS_SOCKET_SIZE: *out = 1ull << ((dev_caps >> 14) & 0x7); return true; case VX_CAPS_NUM_CLUSTERS: - *out = 1ull << ((dev_caps >> 20) & 0x7); + *out = exact_or_log2(dev_caps, 57, 0x7F, 20); return true; case VX_CAPS_ISSUE_WIDTH: *out = 1ull << ((dev_caps >> 23) & 0x7); From 5f6d3e147d99c17f97a6f619b2ca57052f8d8d27 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 10 Oct 2026 22:22:43 -0700 Subject: [PATCH 014/124] runtime: VORTEX_VM_MEGAPAGE_HEAP maps the heap with superpages With 4 KB pages a working set beyond the TLB reach thrashes: sgemm at N=512 on a VM-enabled RTX 5080-shaped machine ran 18x slower (1% L1 TLB hit rate, the memory port blocked behind 2 requests parked per in-flight page). GPU drivers back allocations with 2 MB pages. VORTEX_VM_MEGAPAGE_HEAP=1 maps the whole heap once with superpages at its identity VA, which the page walker and the L2 TLB's megapage array already support, so a buffer costs no TLB entry per 4 KB (in that study VM then cost 1-4% instead of 18x). Per-buffer access flags are not enforced in this mode and it cannot be combined with VORTEX_RANDOMIZE_VA; the default mapping is unchanged. Adds vm-megapage-heap (sgemm, simx + rtlsim). Verified: vm-megapage-heap passes on simx (XLEN 64); the rtlsim variant runs in the CI matrix vm cell. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/vm.yaml | 11 ++++++++++ docs/designs/virtual_memory_subsystem.md | 4 ++++ sw/runtime/common/vm.cpp | 26 ++++++++++++++++++++++++ sw/runtime/common/vm.h | 3 +++ 4 files changed, 44 insertions(+) diff --git a/ci/testcases/vm.yaml b/ci/testcases/vm.yaml index 738eb3851f..2c4d2b7c2b 100644 --- a/ci/testcases/vm.yaml +++ b/ci/testcases/vm.yaml @@ -170,6 +170,17 @@ tests: target: run-{driver} vars: OPTS: -t4 -n4 -p1 +# The heap mapped once with superpages (VORTEX_VM_MEGAPAGE_HEAP): every buffer +# translates through the TLBs' megapage path at its identity VA. +- id: vm-megapage-heap + via: make-run + drivers: + - simx + - rtlsim + dir: tests/regression/sgemm + target: run-{driver} + env: + VORTEX_VM_MEGAPAGE_HEAP: 1 - id: tlb-concurrent via: make-run drivers: diff --git a/docs/designs/virtual_memory_subsystem.md b/docs/designs/virtual_memory_subsystem.md index 5a667d44a0..8b6a34a6e4 100644 --- a/docs/designs/virtual_memory_subsystem.md +++ b/docs/designs/virtual_memory_subsystem.md @@ -653,6 +653,10 @@ MMU moved out of the core there is no `sysmem_perf` slot to carry it, so the in-core view reads zero on rtlsim / FPGA (§13). Testing knobs: `VORTEX_RANDOMIZE_VA=0|1` and `VORTEX_VA_SEED=N` (§8.4); +`VORTEX_VM_MEGAPAGE_HEAP=1` maps the whole heap once with superpages at its +identity VA, as GPU drivers back allocations with 2 MB pages, so a buffer costs +no TLB entry per 4 KB page (per-buffer access flags are not enforced, and it +cannot be combined with `VORTEX_RANDOMIZE_VA`); `VX_CFG_VM_ADDR_MODE=BARE` in `CONFIGS` builds the VM hardware but runs untranslated. diff --git a/sw/runtime/common/vm.cpp b/sw/runtime/common/vm.cpp index 3c539fd521..0922d8cfd1 100644 --- a/sw/runtime/common/vm.cpp +++ b/sw/runtime/common/vm.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -59,6 +60,12 @@ VMManager::VMManager(DeviceMemIO* dev_io) std::cout << "[VM] Virtual address randomization ENABLED (seed=0x" << std::hex << seed << std::dec << ")" << std::endl; } + + const char* megapage_env = std::getenv("VORTEX_VM_MEGAPAGE_HEAP"); + megapage_heap_ = (megapage_env != nullptr && std::atoi(megapage_env) != 0); + if (megapage_heap_) { + std::cout << "[VM] Heap mapped with superpages" << std::endl; + } } VMManager::~VMManager() { @@ -114,6 +121,22 @@ int VMManager::init() { GLOBAL_MEM_SIZE - VX_MEM_PAGE_TABLE_BASE_ADDR), { return err; }); + if (megapage_heap_) { + // Superpages over the whole heap, as GPU drivers back allocations with + // 2 MB pages: a buffer keeps its identity VA and no longer costs a TLB + // entry per 4 KB. Per-buffer access flags are not enforced in this mode. + if (randomize_va_) { + std::cerr << "[VM] VORTEX_VM_MEGAPAGE_HEAP and VORTEX_RANDOMIZE_VA are mutually exclusive" << std::endl; + return -1; + } + uint64_t heap_end = VX_MEM_PAGE_TABLE_BASE_ADDR; +#if VX_VM_ADDR_MODE == SV32 + heap_end = std::min(heap_end, 0x100000000ULL); +#endif + CHECK_ERR(install_identity_map(ALLOC_BASE_ADDR, heap_end - ALLOC_BASE_ADDR), { + return err; + }); + } } // install_identity_map() already flushed; an extra flush here is a // cheap no-op if nothing else dirtied the shadow. @@ -183,6 +206,9 @@ uint64_t VMManager::map_p2v(uint64_t ppn, uint32_t flags) { int VMManager::phy_to_virt_map(uint64_t size, uint64_t* dev_pAddr, uint32_t flags) { if (!need_trans(*dev_pAddr)) return 0; + if (megapage_heap_ && *dev_pAddr >= ALLOC_BASE_ADDR && *dev_pAddr < VX_MEM_PAGE_TABLE_BASE_ADDR) { + return 0; // identity-mapped by init() + } uint64_t init_pAddr = *dev_pAddr; // Round up: a sub-page allocation still needs one PTE. A plain diff --git a/sw/runtime/common/vm.h b/sw/runtime/common/vm.h index e922a580aa..581ecd43fe 100644 --- a/sw/runtime/common/vm.h +++ b/sw/runtime/common/vm.h @@ -126,6 +126,9 @@ class VMManager { // Randomized VA allocation (VORTEX_RANDOMIZE_VA / VORTEX_VA_SEED). std::mt19937_64 rng_; bool randomize_va_; + + // Heap mapped once with superpages (VORTEX_VM_MEGAPAGE_HEAP). + bool megapage_heap_; }; } // namespace vortex From 4dbb6154a749c22a6633c00d060a54a173a1759c Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 10 Oct 2026 23:17:22 -0700 Subject: [PATCH 015/124] simx: widen the cache config's input count and pipeline latency Cache::Config stored the input count and the pipeline latency in uint8_t, so any machine whose L2 sees more than 255 inputs (an 84-132 core chip: 16 L1 requests per socket) or whose L2 latency exceeds 255 cycles failed to build with a narrowing error. The RTL caches take both as unbounded parameters; the model now matches. Verified: the 84-core rtx5080 profile builds and runs vecadd on simx; the a100/h100/v100/rtx1080 profiles build (1,728 L2 inputs on a100). Co-Authored-By: Claude Opus 5.5 --- sim/simx/mem/cache.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sim/simx/mem/cache.h b/sim/simx/mem/cache.h index b3ffe9c8be..70ad9e6e3f 100644 --- a/sim/simx/mem/cache.h +++ b/sim/simx/mem/cache.h @@ -35,12 +35,12 @@ class Cache : public SimObject { uint8_t A; // log2 associativity uint8_t B; // log2 number of banks uint8_t addr_width; // word address bits - uint8_t num_inputs; // number of inputs + uint16_t num_inputs; // number of inputs uint8_t mem_ports; // memory ports bool write_back; // is write-back bool write_reponse; // enable write response uint16_t mshr_size; // MSHR buffer size - uint8_t latency; // pipeline latency + uint16_t latency; // pipeline latency uint8_t repl_policy; // replacement policy (ReplPolicy) bool is_llc; // is last-level cache }; From 325d685dbe4ab62985001da5916ef61948093c58 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 10 Oct 2026 23:17:30 -0700 Subject: [PATCH 016/124] lsu: declare the LMEM window parameters only when LMEM exists With local memory disabled (e.g. the tinygpu profile) LMEM_BLK_BASE was never read, and Verilator's UNUSEDPARAM warning, now fatal, stopped the rtlsim build. No CI configuration disables LMEM, so the zero-warnings sweep did not reach it. The window parameters and their static check now sit under the same VX_CFG_LMEM_ENABLE guard as their only use. Verified: 1 core, 2 warps x 2 threads, LMEM disabled builds warning-free on rtlsim and passes sgemm -n16 (15,240 instructions, as on simx). Co-Authored-By: Claude Opus 5.5 --- hw/rtl/core/VX_lsu_slice.sv | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hw/rtl/core/VX_lsu_slice.sv b/hw/rtl/core/VX_lsu_slice.sv index dba90d2856..affb932298 100644 --- a/hw/rtl/core/VX_lsu_slice.sv +++ b/hw/rtl/core/VX_lsu_slice.sv @@ -42,9 +42,11 @@ module VX_lsu_slice import VX_gpu_pkg::*; #( // window: the LMEM base is asserted aligned to the capacity, so the prefix // compare is exact and needs no end address -- which is not representable // when the window ends at the top of the address space. +`ifdef VX_CFG_LMEM_ENABLE localparam LMEM_BLK_LSB = `VX_CFG_LMEM_LOG_SIZE - MEM_ASHIFT; localparam [MEM_ADDRW-1:0] LMEM_BLK_BASE = MEM_ADDRW'(`VX_CFG_XLEN'(`VX_MEM_LMEM_BASE_ADDR) >> MEM_ASHIFT); `STATIC_ASSERT (MEM_ADDRW > LMEM_BLK_LSB, ("LMEM window wider than the block address space")) +`endif `UNUSED_PARAM (CORE_ID) `UNUSED_SPARAM (INSTANCE_ID) From 1642289411ea1d2c51828ea92593e7d2c483daf7 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 10 Oct 2026 23:21:31 -0700 Subject: [PATCH 017/124] tests: guard the global index in vecadd, relu, dropout and minimal vx_device_max_occupancy_grid sizes a 1D block to a whole core and rounds the grid up, so grid x block exceeds the problem size whenever a core has more threads than there are points. These four kernels indexed with the global id unguarded, so the surplus threads read and wrote past their buffers. On a core with 1,024 threads vecadd -n256 overwrote its kernel-argument block and later warps faulted on a float loaded as the destination pointer ("misaligned memory access", every app on the 20-132 core profiles). Each kernel now returns when the index is past num_points, the size the host already passes, as any kernel on a rounded-up grid must. The other tests that launch through the helper were audited: they guard the index, use a grid-stride loop, or size the problem as an exact multiple of the block. Verified: the four pass on simx at the default config and at 32 warps x 32 threads, including sizes that do not divide the block (vecadd -n64/-n100/ -n256/-n16384, relu -n64/-n100, dropout -n1000/-n1024, minimal -n1); the unguarded vecadd reproduces the fault at 32x32 -n256. The guard adds instructions, so the perf-gate baselines are re-recorded after this series. Co-Authored-By: Claude Opus 5.5 --- tests/regression/dropout/kernel.cpp | 3 +++ tests/regression/minimal/kernel.cpp | 3 +++ tests/regression/relu/kernel.cpp | 3 +++ tests/regression/vecadd/kernel.cpp | 3 +++ 4 files changed, 12 insertions(+) diff --git a/tests/regression/dropout/kernel.cpp b/tests/regression/dropout/kernel.cpp index 95e77d8189..c9cd35b5fe 100644 --- a/tests/regression/dropout/kernel.cpp +++ b/tests/regression/dropout/kernel.cpp @@ -10,6 +10,9 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto multiplier = arg->multiplier; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; + if (gid >= arg->num_points) { + return; + } float rand_value = RandomFloat(WangHash(gid)); TYPE scaled_value = src0_ptr[gid] * multiplier; dst_ptr[gid] = (rand_value < dropout_p) ? 0.0 : scaled_value; diff --git a/tests/regression/minimal/kernel.cpp b/tests/regression/minimal/kernel.cpp index 0d04cd2563..9cdf8cf52b 100644 --- a/tests/regression/minimal/kernel.cpp +++ b/tests/regression/minimal/kernel.cpp @@ -7,5 +7,8 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto dst_ptr = reinterpret_cast(arg->dst_addr); uint32_t idx = blockIdx.x * blockDim.x + threadIdx.x; + if (idx >= arg->num_points) { + return; + } dst_ptr[idx] = MINIMAL_MAGIC | idx; } diff --git a/tests/regression/relu/kernel.cpp b/tests/regression/relu/kernel.cpp index 55335af652..b70730b79a 100644 --- a/tests/regression/relu/kernel.cpp +++ b/tests/regression/relu/kernel.cpp @@ -6,6 +6,9 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto dst_ptr = reinterpret_cast(arg->dst_addr); uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; + if (gid >= arg->num_points) { + return; + } TYPE value = src0_ptr[gid]; dst_ptr[gid] = (value < 0) ? TYPE(0) : value; } diff --git a/tests/regression/vecadd/kernel.cpp b/tests/regression/vecadd/kernel.cpp index a84972e50e..3660c77e59 100644 --- a/tests/regression/vecadd/kernel.cpp +++ b/tests/regression/vecadd/kernel.cpp @@ -6,5 +6,8 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto src1_ptr = reinterpret_cast(arg->src1_addr); auto dst_ptr = reinterpret_cast(arg->dst_addr); uint32_t idx = blockIdx.x * blockDim.x + threadIdx.x; + if (idx >= arg->num_points) { + return; + } dst_ptr[idx] = src0_ptr[idx] + src1_ptr[idx]; } From 997ee24e094da45151d0dccaa729312442adf659 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 10 Oct 2026 23:50:54 -0700 Subject: [PATCH 018/124] cache: admit LLC fills against the AMO writeback queue's room The LLC AMO commit engine queues each result in a 4-entry writeback queue that drains through the bank's core-request slot, last in the input priority. Its sizing assumed commit_busy serializes AMOs, but commit_busy only gates core requests: a burst of fills for different lines, each line's chain ending in one AMO, replays an AMO per fill while the fill/replay stream holds the input slot, so nothing drains and the fifth result overflows the queue (amo:zacas:rtlsim, exposed by the per-channel DRAM model's tighter fill bursts; latent on master). - The engine counts the AMOs still able to reach the queue (S0, S0..stC, the compute stage) and flags when the queue cannot take one more; the bank admits a fill only while it can. A fill is admitted only between replay chains and a chain carries at most one AMO, so replays never wait mid-chain, and the drain still fires between chains as before. The flag is registered (the sum grows by at most one a cycle, so it tests one entry early). - req_input_defer missed the cycle a missing AMO finalizes at S1 (the S0 pulse has passed, the MSHR probe does not see the entry yet), letting a same-line request link behind it. A registered S1 allocation term closes it, in both the LLC and passthrough guards, and the MSHR asserts that no request links behind a pending AMO at the LLC. - chain_stall now paces any AMO behind a busy compute stage, not only a same-line one: a core AMO admitted while a chain's head leaves through the forward port could otherwise be overwritten by the chain's storing AMO. - A flush no longer starts while AMO results are still queued (bank_empty now includes commit_busy). SimX models no writeback queue; cycles change only when a fill would overflow it. Verified: amo and cache cells on rtlsim, XLEN 32 and 64, all pass (zacas included). Co-Authored-By: Claude Opus 5.5 --- hw/rtl/cache/VX_cache_amo.sv | 69 ++++++++++++++++++++++++++++------- hw/rtl/cache/VX_cache_bank.sv | 15 +++++--- hw/rtl/cache/VX_cache_mshr.sv | 5 +++ 3 files changed, 71 insertions(+), 18 deletions(-) diff --git a/hw/rtl/cache/VX_cache_amo.sv b/hw/rtl/cache/VX_cache_amo.sv index 23c33a60cd..63a857d330 100644 --- a/hw/rtl/cache/VX_cache_amo.sv +++ b/hw/rtl/cache/VX_cache_amo.sv @@ -101,8 +101,9 @@ module VX_cache_amo import VX_gpu_pkg::*; #( // commit outputs (tied off when IS_LLC=0) output wire amo_hit_st1, // AMO commits locally at S1 output wire commit_busy, // commit in flight - output wire chain_stall, // pace same-line chained AMO + output wire chain_stall, // pace a request behind the compute stage output wire wb_pending, // writeback request live + output wire wbq_full, // no queue room for another chain's AMO output wire [WORD_WIDTH-1:0] rsp_data, // response word on amo_hit_st1 // Read forward: a request reading the AMO'd word while the result is still // queued/settling must observe those bytes (replays are admitted during @@ -152,6 +153,18 @@ module VX_cache_amo import VX_gpu_pkg::*; #( end wire st0_ld_alloc_any = mshr_allocate_st0 && ~pipe_stall && ~amo_st0.amo_valid && ~rw_st0; wire st0_amo_alloc_any = mshr_allocate_st0 && ~pipe_stall && amo_st0.amo_valid; + // The allocation one stage on: at S1 it finalizes, and the MSHR probe + // only sees it the cycle after, so this covers the cycle in between. + reg st1_ld_alloc_r, st1_amo_alloc_r; + always @(posedge clk) begin + if (reset) begin + st1_ld_alloc_r <= 1'b0; + st1_amo_alloc_r <= 1'b0; + end else if (~pipe_stall) begin + st1_ld_alloc_r <= mshr_allocate_st0 && ~amo_st0.amo_valid && ~rw_st0; + st1_amo_alloc_r <= mshr_allocate_st0 && amo_st0.amo_valid; + end + end if (IS_LLC != 0) begin : g_commit // ---------------------------------------------------------------- @@ -184,8 +197,8 @@ module VX_cache_amo import VX_gpu_pkg::*; #( // bounded by the core-port fan-in (<= 4 for this cache family), not the // full line. Sizing to that bound (min 2, cap 4) halves the coalesce/ // forward/enqueue fanout vs a full-line queue; the cap is a constant - // here because the engine has no NUM_REQS parameter, and the overflow - // assertion guards it if a wider core-port config ever exceeds 4. + // here because the engine has no NUM_REQS parameter. Replays bypass + // commit_busy, so fills are admitted against queue room (wbq_full). localparam WBQ_TARGETS = WORDS_PER_LINE * NUM_LANES; localparam WBQ_SIZE = (WBQ_TARGETS < 4) ? ((WBQ_TARGETS < 2) ? 2 : WBQ_TARGETS) : 4; localparam WBQ_CNTW = `CLOG2(WBQ_SIZE+1); @@ -228,9 +241,9 @@ module VX_cache_amo import VX_gpu_pkg::*; #( wire post_wb_valid = (post_wb_age != 2'd0); // Compute stage: S1 latches the aligned operands, the RMW ALU + the - // re-align shift run the next cycle, off the S1 critical path. AMO - // commits are serialized by commit_busy (the bank holds off core - // requests), so the stage holds at most one operation; the old operand + // re-align shift run the next cycle, off the S1 critical path. The + // stage holds at most one operation (commit_busy serializes core AMOs, + // chain_stall paces any AMO behind a busy stage); the old operand // is byte-forwarded from the writeback queue when a prior AMO to the same // bytes has not yet reached the array (see line_lane_st1). reg cmp_valid; @@ -750,7 +763,8 @@ module VX_cache_amo import VX_gpu_pkg::*; #( // line AMOs back to back); those are paced instead by chain_stall. assign commit_busy = do_store_st0 || amo_inflight || do_store_st1 || cmp_valid || wb_pending_r; // Pace any same-line request sitting behind an in-flight compute by one - // cycle, so the result lands in wb_data_r and forwards cleanly. Gated on + // cycle, so the result lands in wb_data_r and forwards cleanly, and any + // AMO, which needs the compute stage free. Gated on // cmp_valid (an AMO is computing), so it never fires for baseline traffic. // The address match is registered from the look-ahead operands: both only // move on ~pipe_stall, so a held match stays valid across a stall. The @@ -765,7 +779,34 @@ module VX_cache_amo import VX_gpu_pkg::*; #( chain_match_r <= (cmp_addr_n == res_addr_n); end end - assign chain_stall = cmp_valid && valid_st1 && is_creq_st1 && chain_match_r; + assign chain_stall = cmp_valid && valid_st1 && is_creq_st1 && (chain_match_r || amo_st1.amo_valid); + + // AMOs that may reach the queue: at S0, between S0 and stC, and in the + // compute stage. A fill is admitted only while one more fits. The sum + // grows by at most one a cycle (one AMO enters S0), so the registered + // flag tests one entry early and stays exact a cycle later. + localparam AMO_CNTW = `CLOG2(PIPE_EX + 2); + wire st0_amo = amo_st0.amo_valid && valid_st0 && is_creq_st0; + wire st1_amo = amo_st1.amo_valid && valid_st1 && is_creq_st1; + wire amo_cnt_inc = st0_amo && ~pipe_stall; + wire amo_cnt_dec = st1_amo && ~pipe_stall; + reg [AMO_CNTW-1:0] amo_pipe_cnt; + reg wbq_full_r; + always @(posedge clk) begin + if (reset) begin + amo_pipe_cnt <= '0; + wbq_full_r <= 1'b0; + end else begin + if (amo_cnt_inc != amo_cnt_dec) begin + amo_pipe_cnt <= amo_cnt_inc ? (amo_pipe_cnt + AMO_CNTW'(1)) + : (amo_pipe_cnt - AMO_CNTW'(1)); + end + wbq_full_r <= (32'(wbq_count) + 32'(amo_pipe_cnt) + 32'(cmp_valid) + 32'(st0_amo) + 1) >= WBQ_SIZE; + end + end + assign wbq_full = wbq_full_r; + `RUNTIME_ASSERT (~(amo_cnt_dec && ~amo_cnt_inc && (amo_pipe_cnt == '0)), + ("%t: AMO pipe count underflow", $time)) // Invariants: a store-bearing AMO is only ever accepted into a free // compute stage (the queue absorbs different-line writebacks behind it), @@ -808,8 +849,9 @@ module VX_cache_amo import VX_gpu_pkg::*; #( // covers it from S0 onward. Consumes only the registered probe (see // the header block) so the CAM stays out of core_req_ready. assign req_input_defer = core_req_valid - && (st0_amo_alloc_any || persist_amo_r + && (st0_amo_alloc_any || st1_amo_alloc_r || persist_amo_r || (fresh_r ? mshr_any_amo : probe_amo_hit_r)); + `UNUSED_VAR (st1_ld_alloc_r) `UNUSED_VAR (amo_st0) // only amo_valid/amo_op are consumed at S0 `UNUSED_VAR (is_replay_st0) @@ -860,13 +902,13 @@ module VX_cache_amo import VX_gpu_pkg::*; #( // Age-ordering guards, off the registered probe (see the header // block): an incoming AMO holds behind pending line-filling requests, - // an incoming load holds behind pending AMO passthroughs. The S0 - // alloc pulses catch the window between admit and allocate. + // an incoming load holds behind pending AMO passthroughs. The S0/S1 + // alloc terms catch the window between admit and the probe. wire amo_input_defer = core_req_valid && core_req_is_amo - && (st0_ld_alloc_any || persist_ld_r + && (st0_ld_alloc_any || st1_ld_alloc_r || persist_ld_r || (fresh_r ? mshr_any_ld : probe_ld_hit_r)); wire load_input_defer = core_req_valid && ~core_req_is_amo && ~core_req_rw - && (st0_amo_alloc_any || persist_amo_r + && (st0_amo_alloc_any || st1_amo_alloc_r || persist_amo_r || (fresh_r ? mshr_any_amo : probe_amo_hit_r)); assign req_input_defer = amo_input_defer || load_input_defer; @@ -875,6 +917,7 @@ module VX_cache_amo import VX_gpu_pkg::*; #( assign commit_busy = 1'b0; assign chain_stall = 1'b0; assign wb_pending = 1'b0; + assign wbq_full = 1'b0; assign rsp_data = '0; // no local commit window: loads are held at the input while a // forwarded AMO is pending on the line, so no read forward exists. diff --git a/hw/rtl/cache/VX_cache_bank.sv b/hw/rtl/cache/VX_cache_bank.sv index f62c57214f..28576fcd39 100644 --- a/hw/rtl/cache/VX_cache_bank.sv +++ b/hw/rtl/cache/VX_cache_bank.sv @@ -188,7 +188,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( amo_req_t replay_amo; // AMO engine interconnect (tied to 0 when the bank carries no AMO logic). - wire amo_hit_st1, amo_commit_busy, amo_chain_stall, amo_wb_pending; + wire amo_hit_st1, amo_commit_busy, amo_chain_stall, amo_wb_pending, amo_wbq_full; wire [`CS_WORD_WIDTH-1:0] amo_rsp_data; wire [WORD_SIZE-1:0] amo_rd_fwd_mask; wire [`CS_WORD_WIDTH-1:0] amo_rd_fwd_data; @@ -282,7 +282,8 @@ module VX_cache_bank import VX_gpu_pkg::*; #( end assign pipe_inflight = st0.req.valid || st1.req.valid || (| commit_valid); end - wire no_pending_req = ~pipe_inflight && mreq_queue_empty; + // Queued AMO results are still writes to land in the array. + wire no_pending_req = ~pipe_inflight && mreq_queue_empty && ~amo_commit_busy; VX_cache_flush #( .BANK_ID (BANK_ID), @@ -350,8 +351,11 @@ module VX_cache_bank import VX_gpu_pkg::*; #( && (replay_rw || st0.req.rw) && (replay_wsel == st0.req.word_idx); + // A fill starts a replay chain, and a chain carries at most one AMO (no + // request enters behind a pending AMO), so a fill waits until the AMO + // writeback queue has room for it; the queue drains meanwhile. wire replay_mux = replay_valid && ~fwd_head && ~replay_link_hold; - wire fill_mux = mem_rsp_valid && ~fwd_pending && ~fill_inflight && ~replay_link_hold; + wire fill_mux = mem_rsp_valid && ~fwd_pending && ~fill_inflight && ~replay_link_hold && ~amo_wbq_full; wire replay_grant = ~init_valid; wire replay_enable = replay_grant && replay_mux; @@ -369,7 +373,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire creq_enable = creq_grant && (amo_creq_path || amo_wb_path); assign replay_ready = replay_grant && ~fwd_head && ~replay_link_hold && ~(!WRITEBACK && replay_rw && mreq_queue_alm_full) && ~pipe_stall; - assign mem_rsp_ready = fill_grant && ~fwd_pending && ~fill_inflight && ~replay_link_hold && ~(WRITEBACK && mreq_queue_alm_full) && ~pipe_stall; + assign mem_rsp_ready = fill_grant && ~fwd_pending && ~fill_inflight && ~replay_link_hold && ~amo_wbq_full && ~(WRITEBACK && mreq_queue_alm_full) && ~pipe_stall; assign flush_ready = flush_grant && ~(WRITEBACK && mreq_queue_alm_full) && ~pipe_stall; assign core_req_ready = creq_grant && ~mreq_queue_alm_full && ~mshr_alm_full && ~pipe_stall && ~amo_commit_busy && ~req_input_defer; @@ -997,6 +1001,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .commit_busy (amo_commit_busy), .chain_stall (amo_chain_stall), .wb_pending (amo_wb_pending), + .wbq_full (amo_wbq_full), .rsp_data (amo_rsp_data), .rd_fwd_mask (amo_rd_fwd_mask), .rd_fwd_data (amo_rd_fwd_data), @@ -1015,7 +1020,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .req_input_defer (req_input_defer) ); end else begin : g_no_amo - assign {amo_hit_st1, amo_commit_busy, amo_wb_pending, amo_chain_stall} = '0; + assign {amo_hit_st1, amo_commit_busy, amo_wb_pending, amo_chain_stall, amo_wbq_full} = '0; assign {amo_rd_fwd_mask, amo_rd_fwd_data} = '0; assign {amo_rsp_data, amo_wb_addr, amo_wb_word_idx, amo_wb_byteen} = '0; assign {amo_wb_data, amo_wb_tag, amo_wb_idx, amo_wb_attr} = '0; diff --git a/hw/rtl/cache/VX_cache_mshr.sv b/hw/rtl/cache/VX_cache_mshr.sv index fdbdcaa787..22605fd148 100644 --- a/hw/rtl/cache/VX_cache_mshr.sv +++ b/hw/rtl/cache/VX_cache_mshr.sv @@ -373,6 +373,11 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( // in arrival order like any request; the probe outputs alone provide // the same-line ordering guard. assign amo_mask = (AMO_PASSTHRU != 0) ? amo_table : '0; + // At the LLC no request links behind a pending AMO: the bank defers + // every request to a line while an AMO to it waits here. + `RUNTIME_ASSERT (~((AMO_PASSTHRU == 0) && finalize_valid && ~finalize_is_release && finalize_is_pending + && valid_table[finalize_previd] && persisted_table[finalize_previd] && amo_table[finalize_previd]), + ("%t: *** %s request linked behind a pending AMO: id=%0d", $time, INSTANCE_ID, finalize_previd)) wire [MSHR_SIZE-1:0] probe_ld, probe_amo; for (genvar i = 0; i < MSHR_SIZE; ++i) begin : g_probe_matches From 843536c48df669cba58332271ebfdc6bad9e52d4 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 10 Oct 2026 23:51:53 -0700 Subject: [PATCH 019/124] sim: redesign the DRAM model: independent channels, refresh, configurable type The shared Ramulator wrapper (sim/common/dram_sim) capped every simulator's memory far below the configured device: - one injection queue for all channels accepted one request per DRAM cycle in total, and one response was drained per core cycle device-wide, so the whole memory delivered at most about one line per cycle; - each 64 B line was split into four 16 B requests, four bursts' worth of command bandwidth for one line; - Ramulator's AllBank refresh manager iterates over a "rank" level, which HBM2 and GDDR6 do not have, so refresh never fired; - the DRAM clock was a raw -DMEM_CLOCK_RATIO define (default 1, the core clock), unrelated to the configured platform. DramSim now models one independent controller per platform memory bank, in simx, rtlsim, opaesim (which also gets per-bank queues) and xrtsim alike: the caller passes the bank as the channel and the address within it; a channel accepts one transaction per DRAM cycle and drains its own responses; a transaction is one burst (64 B HBM2, 32 B GDDR6) with the A100 HBM interleave or GPGPU-Sim's RTX 3070 GDDR layout and IPOLY bank hash; refresh is issued per channel. VX_config.toml selects it: VX_CFG_PLATFORM_MEMORY_TYPE HBM2 | HBM2_A100 | GDDR6 VX_CFG_PLATFORM_MEMORY_CLOCK_RATE DRAM clock (MHz); with PLATFORM_CLOCK_RATE it sets the DRAM cycles per core cycle VX_CFG_PLATFORM_MEMORY_LATENCY fixed read latency (core cycles) VX_CFG_PLATFORM_MEMORY_WINDOW requests a channel holds in flight The refresh manager and Ramulator's GDDR6 device (shipped but not built upstream) are C++20 like Ramulator, so they live in a companion library, third_party/ramulator_ext -> libramulator-vortex.so, built by third_party/Makefile and part of the CI third-party cache key. tests/unittest/dram_sim checks the properties the old model broke: bandwidth scales with the channel count (40 channels = 40x one), the fixed latency adds exactly, the window bounds requests in flight, a two-burst block answers once, refresh stalls reads (fails with the AllBank manager), and the A100 and RTX 5080 shapes land on their targets: A100 open-row/row-conflict reads take 216/257 core cycles (GPGPU-Sim SM80: ~215/~255) and stream 952 of 1,373 B/cycle; GDDR6 at the 5080 shape streams 393 of its 418 B/cycle peak (the card's 960 GB/s at the 2,295 MHz base clock). Every rtlsim/simx cycle count moves; the perf baselines are re-recorded in the same series. Verified: tests/unittest/dram_sim passes (12 checks); every simulator builds with the new model at XLEN 32 and 64; the CI matrix reruns on the series. Co-Authored-By: Claude Opus 5.5 --- .github/actions/setup-vortex/action.yml | 2 +- .gitignore | 4 + VX_config.toml | 9 + docs/codebase.md | 3 +- docs/designs/simx_simulator_architecture.md | 4 +- docs/simulation.md | 21 + sim/avedsim/Makefile | 2 +- sim/common/dram_sim.cpp | 495 +++++++++++++----- sim/common/dram_sim.h | 40 +- sim/opaesim/Makefile | 2 +- sim/opaesim/opae_sim.cpp | 34 +- sim/rtlsim/Makefile | 2 +- sim/rtlsim/processor.cpp | 12 +- sim/simx/Makefile | 2 +- sim/simx/constants.h | 4 - sim/simx/mem/memory.cpp | 16 +- sim/simx/mem/memory.h | 5 +- sim/simx/processor.cpp | 9 +- sim/xrtsim/Makefile | 2 +- sim/xrtsim/xrt_sim.cpp | 22 +- tests/unittest/Makefile | 2 +- tests/unittest/dram_sim/Makefile | 16 + tests/unittest/dram_sim/main.cpp | 228 ++++++++ third_party/Makefile | 20 +- third_party/ramulator_ext/channel_refresh.cpp | 65 +++ 25 files changed, 821 insertions(+), 200 deletions(-) create mode 100644 tests/unittest/dram_sim/Makefile create mode 100644 tests/unittest/dram_sim/main.cpp create mode 100644 third_party/ramulator_ext/channel_refresh.cpp diff --git a/.github/actions/setup-vortex/action.yml b/.github/actions/setup-vortex/action.yml index b27d3c1305..a73448a887 100644 --- a/.github/actions/setup-vortex/action.yml +++ b/.github/actions/setup-vortex/action.yml @@ -49,7 +49,7 @@ runs: uses: actions/cache@v4 with: path: third_party - key: ${{ runner.os }}-thirdparty-${{ hashFiles('third_party/Makefile', '.gitmodules') }} + key: ${{ runner.os }}-thirdparty-${{ hashFiles('third_party/Makefile', 'third_party/ramulator_ext/**', '.gitmodules') }} # Publish WHERE the toolchain cache above was restored to. The `tools` cache # path is relative to the workspace, but nothing downstream could know that: diff --git a/.gitignore b/.gitignore index dbcea3d6e3..136b7f2186 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,7 @@ hw/syn/libs/asap7/verilog/ hw/syn/libs/asap7/metadata/ hw/syn/libs/asap7/.installed_* hw/syn/libs/asap7/install.sh + +# Built by third_party/Makefile +third_party/ramulator_ext/*.o +third_party/ramulator_ext/*.so diff --git a/VX_config.toml b/VX_config.toml index a3336551a8..8d331ff67b 100644 --- a/VX_config.toml +++ b/VX_config.toml @@ -79,6 +79,14 @@ VX_CFG_PLATFORM_MEMORY_DATA_SIZE = 64 VX_CFG_PLATFORM_MEMORY_INTERLEAVE = 1 VX_CFG_PLATFORM_MEMORY_PEAK_BW = 460000 # 460 GB/s VX_CFG_PLATFORM_CLOCK_RATE = 400 +# DRAM the simulators model behind each memory bank (one channel per bank), +# its clock (MHz), the fixed read latency added for the PHY, controller and +# interconnect (core cycles), and the requests a channel holds in flight +# (0 = unbounded). +VX_CFG_PLATFORM_MEMORY_TYPE = "HBM2" +VX_CFG_PLATFORM_MEMORY_CLOCK_RATE = 1000 +VX_CFG_PLATFORM_MEMORY_LATENCY = 0 +VX_CFG_PLATFORM_MEMORY_WINDOW = 0 [alu] VX_CFG_NUM_ALU_LANES = "expr: $VX_CFG_SIMD_WIDTH" @@ -456,6 +464,7 @@ VX_CFG_XLEN = [32, 64] VX_CFG_FLEN = [32, 64] VX_CFG_FPU_TYPE = ["DPI", "DSP", "FPNEW", "STD"] VX_CFG_TCU_TYPE = ["DPI", "DSP", "BHF", "TFR", "FPNEW"] +VX_CFG_PLATFORM_MEMORY_TYPE = ["HBM2", "HBM2_A100", "GDDR6"] [[param]] VX_CFG_DCACHE_NUM_REQS = "int" diff --git a/docs/codebase.md b/docs/codebase.md index e355ad3045..257af03e13 100644 --- a/docs/codebase.md +++ b/docs/codebase.md @@ -74,6 +74,7 @@ The directory/file layout of the Vortex codebase is as follows: - `runtime`: driver API tests - `mpi`: multi-process tests - `third_party`: external library submodules (cvfpu, softfloat, hardfloat, - ramulator, cocogfx) + ramulator, cocogfx), plus `ramulator_ext` (Vortex's Ramulator extensions: + per-channel refresh and the GDDR6 device) - `perf`: performance analysis resources - `miscs`: miscellaneous resources diff --git a/docs/designs/simx_simulator_architecture.md b/docs/designs/simx_simulator_architecture.md index 9913211a13..9da381bc39 100644 --- a/docs/designs/simx_simulator_architecture.md +++ b/docs/designs/simx_simulator_architecture.md @@ -118,7 +118,9 @@ atomics, threads, locks, or deferred-call plumbing — enforced by - `cache.{cpp,h}` — `Cache`: tags + MSHRs + replacement **and line data**; embeds the AMO unit. - `cache_cluster` — shared L2/L3 wrapper. -- `memory.{cpp,h}` — `Memory`: bottom-of-hierarchy DRAM (ramulator2); +- `memory.{cpp,h}` — `Memory`: bottom-of-hierarchy DRAM, one channel per + platform memory bank timed by `sim/common/dram_sim` (ramulator2, shared with + rtlsim/opaesim/xrtsim; see [Configuring the DRAM Model](../simulation.md#configuring-the-dram-model)); `attach_ram(RAM*)` owns the image; carries data in `MemRsp`; exposes `set_pre_send_hook` for SST mirroring. - `amo/amo_unit.{cpp,h}` — AMO RMW in the cache hierarchy (see diff --git a/docs/simulation.md b/docs/simulation.md index 29c6d3a2de..4b9d67fb8e 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -138,6 +138,27 @@ PERF: instrs=363180, cycles=53108, IPC=6.838518 > to compare configurations relative to one another, not as fixed targets to > reproduce. +### Configuring the DRAM Model + +The simulators (simx, rtlsim, opaesim, xrtsim) all model the device memory +the same way: one independent DRAM channel per platform memory bank, timed by +[Ramulator](https://github.com/CMU-SAFARI/ramulator2), with its refresh. These +`VX_config.toml` keys select it, overridable through `CONFIGS` or a profile: + +| Key | Default | Meaning | +|---|---|---| +| `VX_CFG_PLATFORM_MEMORY_TYPE` | `HBM2` | `HBM2` (JEDEC 2 Gbps), `HBM2_A100` (GPGPU-Sim's A100 timing) or `GDDR6` (x16 channels, 32 B bursts) | +| `VX_CFG_PLATFORM_MEMORY_NUM_BANKS` | 2 | memory banks, and so DRAM channels | +| `VX_CFG_PLATFORM_MEMORY_CLOCK_RATE` | 1000 | DRAM clock in MHz (GDDR6: the data clock, half the transfer rate) | +| `VX_CFG_PLATFORM_CLOCK_RATE` | 400 | core clock in MHz; with the DRAM clock it sets how many DRAM cycles a core cycle spans | +| `VX_CFG_PLATFORM_MEMORY_LATENCY` | 0 | fixed read latency added for the PHY, controller and interconnect, in core cycles | +| `VX_CFG_PLATFORM_MEMORY_WINDOW` | 0 | requests a channel holds in flight (0 = unbounded) | + +```bash +$ CONFIGS="-DVX_CFG_PLATFORM_MEMORY_TYPE=GDDR6 -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=16 -DVX_CFG_PLATFORM_MEMORY_CLOCK_RATE=8000" \ + ./ci/blackbox.sh --driver=simx --app=vecadd +``` + ## Additional Quick Start Scenarios Running Vortex simulators with different configurations and drivers is supported. For example: diff --git a/sim/avedsim/Makefile b/sim/avedsim/Makefile index 6dd3b52bf2..dfe8ab9bb4 100644 --- a/sim/avedsim/Makefile +++ b/sim/avedsim/Makefile @@ -25,7 +25,7 @@ CXXFLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) CXXFLAGS += $(CONFIGS) LDFLAGS += -shared $(THIRD_PARTY_DIR)/softfloat/build/Linux-x86_64-GCC/softfloat.a -LDFLAGS += -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator -L$(THIRD_PARTY_DIR)/ramulator -lramulator -pthread +LDFLAGS += -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator -L$(THIRD_PARTY_DIR)/ramulator -lramulator -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator_ext -L$(THIRD_PARTY_DIR)/ramulator_ext -Wl,--push-state,--no-as-needed -lramulator-vortex -Wl,--pop-state -pthread # control RTL debug tracing states DBG_TRACE_FLAGS += -DDBG_TRACE_PIPELINE diff --git a/sim/common/dram_sim.cpp b/sim/common/dram_sim.cpp index cbb881e713..e6f7f1a0c5 100644 --- a/sim/common/dram_sim.cpp +++ b/sim/common/dram_sim.cpp @@ -13,7 +13,14 @@ #include "dram_sim.h" #include "util.h" +#include +#include #include +#include +#include +#include +#include +#include DISABLE_WARNING_PUSH DISABLE_WARNING_UNUSED_PARAMETER @@ -27,146 +34,360 @@ DISABLE_WARNING_POP using namespace vortex; +namespace { + +// IPOLY(5) bank hash over 16 banks (Rau, "Pseudo-randomly interleaved memory", +// ISCA 1991), GPGPU-Sim's -dram_bnk_indexing_policy 2: the bank XORed with a +// function of the row, so streams whose bases differ by a power of two land in +// different banks instead of closing each other's rows. +uint32_t ipoly16(uint64_t row, uint32_t bank) { + auto bit = [row](int i) { return uint32_t(row >> i) & 1u; }; + uint32_t i0 = bit(11) ^ bit(10) ^ bit(9) ^ bit(8) ^ bit(6) ^ bit(4) ^ bit(3) ^ bit(0); + uint32_t i1 = bit(12) ^ bit(8) ^ bit(7) ^ bit(6) ^ bit(5) ^ bit(3) ^ bit(1) ^ bit(0); + uint32_t i2 = bit(9) ^ bit(8) ^ bit(7) ^ bit(6) ^ bit(4) ^ bit(2) ^ bit(1); + uint32_t i3 = bit(10) ^ bit(9) ^ bit(8) ^ bit(7) ^ bit(5) ^ bit(3) ^ bit(2); + return bank ^ (i0 | (i1 << 1) | (i2 << 2) | (i3 << 3)); +} + +} + class DramSim::Impl { private: - struct mem_req_t { - uint64_t addr; - bool is_write; - ResponseCallback callback; - void* arg; - }; - - struct mem_rsp_t { - ResponseCallback callback; - void* arg; - }; - - Ramulator::IFrontEnd* ramulator_frontend_; - Ramulator::IMemorySystem* ramulator_memorysystem_; - uint32_t cpu_channel_size_; - uint64_t cpu_cycles_; - uint32_t scaled_dram_cycles_; - static const uint32_t tick_cycles_ = 1000; - static const uint32_t dram_channel_size_ = 16; // 128 bits - std::queue pending_reqs_; - std::queue pending_rsps_; - - void handle_pending_responses() { - if (pending_rsps_.empty()) - return; - auto& rsp = pending_rsps_.front(); - if (rsp.callback(rsp.arg)) { - pending_rsps_.pop(); - } - } - - void handle_pending_requests() { - if (pending_reqs_.empty()) - return; - auto& req = pending_reqs_.front(); - auto req_type = req.is_write ? Ramulator::Request::Type::Write : Ramulator::Request::Type::Read; - std::function callback = nullptr; - if (req.callback) { - callback = [this, req_callback = std::move(req.callback), req_arg = std::move(req.arg)](Ramulator::Request& /*dram_req*/) { - this->pending_rsps_.push({req_callback, req_arg}); - }; - } - if (ramulator_frontend_->receive_external_requests(req_type, req.addr, 0, callback)) { - // Ramulator does not handle write responses, so we handle the callback ourselves. - if (req.is_write && req.callback) { - pending_rsps_.push({req.callback, req.arg}); - } - pending_reqs_.pop(); - } - } + // One request (a block); it completes when all its transactions have. + struct request_t { + ResponseCallback callback; + void* arg; + uint32_t pending; + }; + + struct mem_tx_t { + uint64_t addr; + bool is_write; + std::shared_ptr request; + }; + + struct mem_rsp_t { + std::shared_ptr request; + uint64_t ready_cycle; + }; + + // GDDR6 holds a fixed 16 Gb per device, so its row count shrinks as the + // channel count grows (Ramulator only knows the refresh timing of 4/8/16 Gb + // devices); rows alias above that capacity, which only affects row hits. + static constexpr uint32_t gddr6_density_mb_ = 16384; + + Config config_; + uint32_t tx_size_; + uint32_t row_mask_; + Ramulator::IFrontEnd* ramulator_frontend_; + Ramulator::IMemorySystem* ramulator_memorysystem_; + uint32_t ch_bits_; + uint64_t cycle_; + // DRAM ticks owed, in units of 1/core_clock: each core cycle adds + // dram_clock, each DRAM tick takes core_clock. + uint64_t clock_credit_; + // A channel accepts one transaction per DRAM cycle and returns its + // responses independently of the others; write acknowledgements do not wait + // behind read data. + std::vector> pending_txs_; + std::vector> read_rsps_; + std::vector> write_rsps_; + std::vector inflight_; + + // Ramulator address of a transaction: the channel in the low bits, then the + // fields in the order the RoBaRaCoCh mapper slices them. + uint64_t dram_addr(uint32_t channel, uint64_t addr) const { + uint64_t t; + if (config_.type == Type::GDDR6) { + // GPGPU-Sim's RTX 3070 layout (-gpgpu_mem_addr_mapping): 32 B bursts, + // two column bits, one bank bit, four column bits, three bank bits, + // row; the low bank bit selects the bank group. + uint64_t col = ((addr >> 5) & 0x3) | (((addr >> 8) & 0xf) << 2); + uint64_t row = (addr >> 15) & row_mask_; + uint32_t bank = ipoly16(row, uint32_t(((addr >> 7) & 0x1) | (((addr >> 12) & 0x7) << 1))); + t = row; + t = (t << 2) | (bank >> 2); + t = (t << 2) | (bank & 0x3); + t = (t << 6) | col; + } else { + // The A100 HBM interleave: a stream alternates column, pseudo-channel + // and column again at 64 B granules, then changes bank group every 4 KB + // and bank every 16 KB, with 2 KB rows. HBM2_A100 uses 2 of the 4 banks + // of a group (16 per channel), as GPGPU-Sim models the A100; HBM2 all 4. + uint32_t bank_bits = (config_.type == Type::HBM2_A100) ? 1 : 2; + uint64_t col = (((addr >> 8) & 0xf) << 1) | ((addr >> 6) & 0x1); + uint64_t pc = (addr >> 7) & 0x1; + uint64_t bg = (addr >> 12) & 0x3; + uint64_t bank = (addr >> 14) & ((1u << bank_bits) - 1); + uint64_t row = (addr >> (14 + bank_bits)) & row_mask_; + t = row; + t = (t << 2) | bank; + t = (t << 2) | bg; + t = (t << 1) | pc; + t = (t << 5) | col; + } + t = (t << ch_bits_) | channel; + return t * tx_size_; + } + + static void set_hbm2_device(YAML::Node& dram, const Config& config, uint32_t pow2_channels) { + dram["impl"] = "HBM2"; + dram["org"]["preset"] = "HBM2_8Gb"; + dram["org"]["density"] = 8192; + dram["org"]["channel"] = pow2_channels; + // A transaction is one burst: 2 x 256-bit prefetch = 64 B. + dram["org"]["channel_width"] = 256; + dram["timing"]["preset"] = "HBM2_2Gbps"; + if (config.type == Type::HBM2_A100) { + // GPGPU-Sim SM80_A100 HBM timing in DRAM cycles. One 64 B transaction + // is two of its 32 B atoms, so the same-bank-group spacing is twice its + // tCCDL. + auto timing = dram["timing"]; + timing["nCL"] = 22; + timing["nRCDRD"] = 22; + timing["nRCDWR"] = 22; + timing["nRP"] = 22; + timing["nRAS"] = 50; + timing["nRC"] = 72; + timing["nWR"] = 19; + timing["nRTPS"] = 7; + timing["nRTPL"] = 7; + timing["nCWL"] = 4; + timing["nCCDS"] = 2; + timing["nCCDL"] = 8; + timing["nRRDS"] = 7; + timing["nRRDL"] = 7; + } + } + + static void set_gddr6_device(YAML::Node& dram, const Config& config, uint32_t pow2_channels) { + // x16 channels with 2 KB pages: 512 columns of 16 bits, 4 bank groups of 4 + // banks; a 32 B transaction is a 16-beat burst (8 prefetched columns over a + // 32-bit internal channel width). + dram["impl"] = "GDDR6"; + dram["org"]["dq"] = 16; + dram["org"]["channel"] = pow2_channels; + dram["org"]["bankgroup"] = 4; + dram["org"]["bank"] = 4; + dram["org"]["column"] = 512; + dram["org"]["row"] = uint32_t((uint64_t(gddr6_density_mb_) << 20) / (uint64_t(pow2_channels) * 4 * 4 * 512 * 16)); + dram["org"]["density"] = gddr6_density_mb_; + dram["org"]["channel_width"] = 32; + // The transfer rate is twice the data clock. Bus timings are in clocks of + // the 32 B burst; array timings are GPGPU-Sim's RTX 3070 values + // (-gpgpu_dram_timing_opt) held in nanoseconds; refresh is JEDEC GDDR6. + auto timing = dram["timing"]; + timing["rate"] = 2 * config.dram_clock; + timing["nBL"] = 8; + timing["nCCDS"] = 8; + timing["nCCDL"] = 12; + timing["tCL"] = 6.8f; + timing["tRCDRD"] = 6.8f; + timing["tRCDWD"] = 6.8f; + timing["tRP"] = 6.8f; + timing["tRAS"] = 15.7f; + timing["tRC"] = 22.3f; + timing["tWR"] = 6.8f; + timing["tRTP"] = 1.2f; + timing["tCWL"] = 2.3f; + timing["tRRDS"] = 3.5f; + timing["tRRDL"] = 3.5f; + timing["tWTRS"] = 2.8f; + timing["tWTRL"] = 2.8f; + timing["tFAW"] = 16.0f; + timing["tRFC"] = 120.0f; + timing["tRFCpb"] = 60.0f; + timing["tREFI"] = 1900.0f; + timing["tRREFD"] = 8.0f; + } + + // Retires the ready head responses of `rsps`; a request completes with its + // last transaction. + void drain(std::deque& rsps, uint32_t ch) { + while (!rsps.empty()) { + auto& rsp = rsps.front(); + if (rsp.ready_cycle > cycle_) { + break; + } + auto& request = *rsp.request; + if (request.pending == 1 && request.callback && !request.callback(request.arg)) { + break; + } + if (--request.pending == 0) { + --inflight_[ch]; + } + rsps.pop_front(); + } + } + + void handle_pending_responses() { + for (uint32_t ch = 0; ch < config_.num_channels; ++ch) { + this->drain(read_rsps_[ch], ch); + this->drain(write_rsps_[ch], ch); + } + } + + void handle_pending_requests() { + for (uint32_t ch = 0; ch < config_.num_channels; ++ch) { + auto& txs = pending_txs_[ch]; + if (txs.empty()) { + continue; + } + auto& tx = txs.front(); + auto req_type = tx.is_write ? Ramulator::Request::Type::Write : Ramulator::Request::Type::Read; + std::function callback = nullptr; + if (!tx.is_write) { + callback = [this, ch, request = tx.request](Ramulator::Request& /*dram_req*/) { + this->read_rsps_[ch].push_back({request, this->cycle_ + 1 + this->config_.latency}); + }; + } + if (ramulator_frontend_->receive_external_requests(req_type, tx.addr, 0, callback)) { + // Ramulator does not handle write responses, so we handle the callback ourselves. + if (tx.is_write) { + write_rsps_[ch].push_back({tx.request, cycle_}); + } + txs.pop(); + } + } + } public: - Impl(uint32_t num_channels, uint32_t channel_size, float clock_ratio) { - YAML::Node dram_config; - dram_config["Frontend"]["impl"] = "GEM5"; - dram_config["MemorySystem"]["impl"] = "GenericDRAM"; - dram_config["MemorySystem"]["clock_ratio"] = 1; - dram_config["MemorySystem"]["DRAM"]["impl"] = "HBM2"; - dram_config["MemorySystem"]["DRAM"]["org"]["preset"] = "HBM2_8Gb"; - dram_config["MemorySystem"]["DRAM"]["org"]["density"] = 8192; - dram_config["MemorySystem"]["DRAM"]["org"]["channel"] = num_channels; - dram_config["MemorySystem"]["DRAM"]["timing"]["preset"] = "HBM2_2Gbps"; - dram_config["MemorySystem"]["Controller"]["impl"] = "Generic"; - dram_config["MemorySystem"]["Controller"]["Scheduler"]["impl"] = "FRFCFS"; - dram_config["MemorySystem"]["Controller"]["RefreshManager"]["impl"] = "AllBank"; - dram_config["MemorySystem"]["Controller"]["RowPolicy"]["impl"] = "OpenRowPolicy"; - // Per-command DRAM trace recording is a debugging aid only: it writes an - // entry for every DRAM command and dominates simulation time (100s of MB - // of I/O for even tiny workloads). Keep it opt-in via the DRAM_TRACE env - // var so default simx/rtlsim runs stay fast. - if (const char* trace_path = getenv("DRAM_TRACE")) { - YAML::Node draw_plugin; - draw_plugin["ControllerPlugin"]["impl"] = "TraceRecorder"; - draw_plugin["ControllerPlugin"]["path"] = trace_path[0] ? trace_path : "./trace/ramulator.log"; - dram_config["MemorySystem"]["Controller"]["plugins"].push_back(draw_plugin); - } - dram_config["MemorySystem"]["AddrMapper"]["impl"] = "RoBaRaCoCh"; - - ramulator_frontend_ = Ramulator::Factory::create_frontend(dram_config); - ramulator_memorysystem_ = Ramulator::Factory::create_memory_system(dram_config); - ramulator_frontend_->connect_memory_system(ramulator_memorysystem_); - ramulator_memorysystem_->connect_frontend(ramulator_frontend_); - - cpu_channel_size_ = channel_size; - scaled_dram_cycles_ = static_cast(clock_ratio * tick_cycles_); - this->reset(); - } - - ~Impl() { - std::ofstream nullstream("ramulator.stats.log"); - auto original_buf = std::cout.rdbuf(); - std::cout.rdbuf(nullstream.rdbuf()); - ramulator_frontend_->finalize(); - ramulator_memorysystem_->finalize(); - std::cout.rdbuf(original_buf); - } - - void reset() { - cpu_cycles_ = 0; - pending_reqs_ = std::queue(); - pending_rsps_ = std::queue(); - } - - void tick() { - cpu_cycles_ += tick_cycles_; - this->handle_pending_responses(); - while (cpu_cycles_ >= scaled_dram_cycles_) { - this->handle_pending_requests(); - ramulator_memorysystem_->tick(); - cpu_cycles_ -= scaled_dram_cycles_; - } - } - - void send_request(uint64_t addr, bool is_write, ResponseCallback response_cb, void* arg) { - if (cpu_channel_size_ > dram_channel_size_) { - uint32_t n = cpu_channel_size_ / dram_channel_size_; - for (uint32_t i = 0; i < n; ++i) { - uint64_t dram_byte_addr = (addr / cpu_channel_size_) * dram_channel_size_ + (i * dram_channel_size_); - if (i == 0) { - pending_reqs_.push({dram_byte_addr, is_write, response_cb, arg}); - } else { - pending_reqs_.push({dram_byte_addr, is_write, nullptr, nullptr}); - } - } - } else if (cpu_channel_size_ < dram_channel_size_) { - uint64_t dram_byte_addr = (addr / cpu_channel_size_) * dram_channel_size_; - pending_reqs_.push({dram_byte_addr, is_write, response_cb, arg}); - } else { - uint64_t dram_byte_addr = addr; - pending_reqs_.push({dram_byte_addr, is_write, response_cb, arg}); - } - } + Impl(const Config& config) : config_(config) { + uint32_t pow2_channels = 1; + ch_bits_ = 0; + while (pow2_channels < config.num_channels) { + pow2_channels <<= 1; + ++ch_bits_; + } + + YAML::Node dram; + if (config.type == Type::GDDR6) { + set_gddr6_device(dram, config, pow2_channels); + tx_size_ = 32; + row_mask_ = dram["org"]["row"].as() - 1; + } else { + set_hbm2_device(dram, config, pow2_channels); + tx_size_ = 64; + row_mask_ = 0x7fff; + } + + YAML::Node dram_config; + dram_config["Frontend"]["impl"] = "GEM5"; + dram_config["MemorySystem"]["impl"] = "GenericDRAM"; + dram_config["MemorySystem"]["clock_ratio"] = 1; + dram_config["MemorySystem"]["DRAM"] = dram; + dram_config["MemorySystem"]["Controller"]["impl"] = "Generic"; + dram_config["MemorySystem"]["Controller"]["Scheduler"]["impl"] = "FRFCFS"; + // libramulator-vortex: Ramulator's AllBank refresh never fires on rank-less HBM2/GDDR6. + dram_config["MemorySystem"]["Controller"]["RefreshManager"]["impl"] = "ChannelRefresh"; + dram_config["MemorySystem"]["Controller"]["RowPolicy"]["impl"] = "OpenRowPolicy"; + // Per-command DRAM trace recording is a debugging aid only: it writes an + // entry for every DRAM command and dominates simulation time (100s of MB + // of I/O for even tiny workloads). Keep it opt-in via the DRAM_TRACE env + // var so default simx/rtlsim runs stay fast. + if (const char* trace_path = getenv("DRAM_TRACE")) { + YAML::Node draw_plugin; + draw_plugin["ControllerPlugin"]["impl"] = "TraceRecorder"; + draw_plugin["ControllerPlugin"]["path"] = trace_path[0] ? trace_path : "./trace/ramulator.log"; + dram_config["MemorySystem"]["Controller"]["plugins"].push_back(draw_plugin); + } + dram_config["MemorySystem"]["AddrMapper"]["impl"] = "RoBaRaCoCh"; + + ramulator_frontend_ = Ramulator::Factory::create_frontend(dram_config); + ramulator_memorysystem_ = Ramulator::Factory::create_memory_system(dram_config); + ramulator_frontend_->connect_memory_system(ramulator_memorysystem_); + ramulator_memorysystem_->connect_frontend(ramulator_frontend_); + + pending_txs_.resize(config.num_channels); + read_rsps_.resize(config.num_channels); + write_rsps_.resize(config.num_channels); + inflight_.resize(config.num_channels, 0); + this->reset(); + } + + ~Impl() { + std::ofstream nullstream("ramulator.stats.log"); + auto original_buf = std::cout.rdbuf(); + std::cout.rdbuf(nullstream.rdbuf()); + ramulator_frontend_->finalize(); + ramulator_memorysystem_->finalize(); + std::cout.rdbuf(original_buf); + } + + void reset() { + cycle_ = 0; + clock_credit_ = 0; + for (auto& q : pending_txs_) { + q = std::queue(); + } + for (auto& q : read_rsps_) { + q.clear(); + } + for (auto& q : write_rsps_) { + q.clear(); + } + std::fill(inflight_.begin(), inflight_.end(), 0); + } + + void tick() { + ++cycle_; + clock_credit_ += config_.dram_clock; + this->handle_pending_responses(); + while (clock_credit_ >= config_.core_clock) { + this->handle_pending_requests(); + ramulator_memorysystem_->tick(); + clock_credit_ -= config_.core_clock; + } + } + + bool can_accept(uint32_t channel) const { + return (config_.window == 0) || (inflight_.at(channel) < config_.window); + } + + void send_request(uint32_t channel, uint64_t addr, bool is_write, ResponseCallback response_cb, void* arg) { + // A block wider than a burst is several transactions. + uint32_t n = (config_.block_size > tx_size_) ? (config_.block_size / tx_size_) : 1; + uint64_t base = addr & ~uint64_t(config_.block_size - 1); + auto request = std::make_shared(request_t{response_cb, arg, n}); + auto& txs = pending_txs_.at(channel); + for (uint32_t i = 0; i < n; ++i) { + txs.push({dram_addr(channel, base + i * tx_size_), is_write, request}); + } + ++inflight_[channel]; + } }; /////////////////////////////////////////////////////////////////////////////// -DramSim::DramSim(uint32_t num_channels, uint32_t channel_size, float clock_ratio) - : impl_(new Impl(num_channels, channel_size, clock_ratio)) +#define DRAM_STR_(x) #x +#define DRAM_STR(x) DRAM_STR_(x) + +DramSim::Config DramSim::platform_config() { + // The enum's value macro, not its per-value companion: the simulators get + // raw CONFIGS, where -DVX_CFG_PLATFORM_MEMORY_TYPE= defines no + // companion. + const std::string type = DRAM_STR(VX_CFG_PLATFORM_MEMORY_TYPE); + Config config; + if (type == "HBM2") { + config.type = Type::HBM2; + } else if (type == "HBM2_A100") { + config.type = Type::HBM2_A100; + } else if (type == "GDDR6") { + config.type = Type::GDDR6; + } else { + throw std::invalid_argument("dram_sim: unknown VX_CFG_PLATFORM_MEMORY_TYPE " + type); + } + config.num_channels = VX_CFG_PLATFORM_MEMORY_NUM_BANKS; + config.block_size = VX_CFG_PLATFORM_MEMORY_DATA_SIZE; + config.core_clock = VX_CFG_PLATFORM_CLOCK_RATE; + config.dram_clock = VX_CFG_PLATFORM_MEMORY_CLOCK_RATE; + config.latency = VX_CFG_PLATFORM_MEMORY_LATENCY; + config.window = VX_CFG_PLATFORM_MEMORY_WINDOW; + return config; +} + +DramSim::DramSim(const Config& config) + : impl_(new Impl(config)) {} DramSim::~DramSim() { @@ -181,6 +402,10 @@ void DramSim::tick() { impl_->tick(); } -void DramSim::send_request(uint64_t addr, bool is_write, ResponseCallback callback, void* arg) { - impl_->send_request(addr, is_write, callback, arg); -} \ No newline at end of file +bool DramSim::can_accept(uint32_t channel) const { + return impl_->can_accept(channel); +} + +void DramSim::send_request(uint32_t channel, uint64_t addr, bool is_write, ResponseCallback callback, void* arg) { + impl_->send_request(channel, addr, is_write, callback, arg); +} diff --git a/sim/common/dram_sim.h b/sim/common/dram_sim.h index 7e8741791c..cc88ff5d78 100644 --- a/sim/common/dram_sim.h +++ b/sim/common/dram_sim.h @@ -11,27 +11,53 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#pragma once + +#include namespace vortex { +// Device memory behind the platform's memory banks: one DRAM channel (an +// independent controller) per bank, timed by Ramulator. class DramSim { public: typedef bool (*ResponseCallback)(void *arg); - DramSim(uint32_t num_channels, uint32_t channel_size, float clock_ratio); + enum class Type { + HBM2, // HBM2 at the 2 Gbps JEDEC preset, 64 B bursts + HBM2_A100, // HBM2 with the A100 timing of GPGPU-Sim's SM80 configuration + GDDR6, // GDDR6 x16 channels, 32 B bursts, 2 KB pages + }; + + struct Config { + Type type; + uint32_t num_channels; // need not be a power of two + uint32_t block_size; // bytes per request + uint32_t core_clock; // MHz + uint32_t dram_clock; // MHz (GDDR6: the data clock, half the transfer rate) + uint32_t latency; // fixed read latency added in core cycles (PHY, controller, NoC) + uint32_t window; // requests a channel holds in flight; 0 = unbounded + }; + + // The memory configured in VX_config.toml. + static Config platform_config(); + + DramSim(const Config& config); ~DramSim(); void reset(); void tick(); - // addr: per-channel block address - void send_request(uint64_t addr, bool is_write, ResponseCallback response_cb, void* arg); + // False while the channel holds its window of requests. + bool can_accept(uint32_t channel) const; + + // addr: byte address within the channel. + void send_request(uint32_t channel, uint64_t addr, bool is_write, ResponseCallback response_cb, void* arg); private: - class Impl; - Impl* impl_; + class Impl; + Impl* impl_; }; -} \ No newline at end of file +} diff --git a/sim/opaesim/Makefile b/sim/opaesim/Makefile index 61fc9aa0d8..f4dbfb723a 100644 --- a/sim/opaesim/Makefile +++ b/sim/opaesim/Makefile @@ -21,7 +21,7 @@ CXXFLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) CXXFLAGS += $(CONFIGS) LDFLAGS += -shared $(THIRD_PARTY_DIR)/softfloat/build/Linux-x86_64-GCC/softfloat.a -LDFLAGS += -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator -L$(THIRD_PARTY_DIR)/ramulator -lramulator -pthread +LDFLAGS += -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator -L$(THIRD_PARTY_DIR)/ramulator -lramulator -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator_ext -L$(THIRD_PARTY_DIR)/ramulator_ext -Wl,--push-state,--no-as-needed -lramulator-vortex -Wl,--pop-state -pthread # debug trace enable flags DBG_TRACE_FLAGS += -DDBG_TRACE_PIPELINE diff --git a/sim/opaesim/opae_sim.cpp b/sim/opaesim/opae_sim.cpp index 94e9adfbe5..7e37ee5dde 100644 --- a/sim/opaesim/opae_sim.cpp +++ b/sim/opaesim/opae_sim.cpp @@ -40,15 +40,12 @@ #include #include #include +#include #include #include #include #include -#ifndef MEM_CLOCK_RATIO -#define MEM_CLOCK_RATIO 1 -#endif - #define CACHE_BLOCK_SIZE 64 #define CCI_LATENCY 8 @@ -100,7 +97,7 @@ class opae_sim::Impl { Impl() : device_(nullptr) , ram_(nullptr) - , dram_sim_(VX_CFG_PLATFORM_MEMORY_NUM_BANKS, VX_CFG_PLATFORM_MEMORY_DATA_SIZE, MEM_CLOCK_RATIO) + , dram_sim_(DramSim::platform_config()) , stop_(false) , host_buffer_ids_(0) #ifdef VCD_OUTPUT @@ -297,9 +294,8 @@ class opae_sim::Impl { reqs.clear(); } - { - std::queue empty; - std::swap(dram_queue_, empty); + for (auto& queue : dram_queues_) { + queue = std::queue(); } device_->reset = 1; @@ -318,9 +314,15 @@ class opae_sim::Impl { this->cci_bus_eval(); this->avs_bus_eval(); - if (!dram_queue_.empty()) { - auto mem_req = dram_queue_.front(); - dram_sim_.send_request(mem_req->addr, mem_req->write, [](void* arg)->bool { + dram_sim_.tick(); + + for (uint32_t b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + auto& queue = dram_queues_[b]; + if (queue.empty() || !dram_sim_.can_accept(b)) { + continue; + } + auto mem_req = queue.front(); + dram_sim_.send_request(b, uint64_t(mem_req->addr) * VX_CFG_PLATFORM_MEMORY_DATA_SIZE, mem_req->write, [](void* arg)->bool { auto orig_req = reinterpret_cast(arg); if (orig_req->ready) { delete orig_req; @@ -329,11 +331,9 @@ class opae_sim::Impl { } return true; }, mem_req); - dram_queue_.pop(); + queue.pop(); } - dram_sim_.tick(); - device_->clk = 0; this->eval(); device_->clk = 1; @@ -499,7 +499,7 @@ class opae_sim::Impl { mem_req->write = true; mem_req->ready = true; - dram_queue_.push(mem_req); + dram_queues_[b].push(mem_req); } else if (device_->avs_read[b]) { // process read request @@ -512,7 +512,7 @@ class opae_sim::Impl { pending_mem_reqs_[b].emplace_back(mem_req); // send dram request - dram_queue_.push(mem_req); + dram_queues_[b].push(mem_req); } device_->avs_waitrequest[b] = false; @@ -580,7 +580,7 @@ class opae_sim::Impl { } }; - std::queue dram_queue_; + std::array, VX_CFG_PLATFORM_MEMORY_NUM_BANKS> dram_queues_; #ifdef VCD_OUTPUT VerilatedVcdC *tfp_; diff --git a/sim/rtlsim/Makefile b/sim/rtlsim/Makefile index f6b8ca3cf8..f15c7da8a7 100644 --- a/sim/rtlsim/Makefile +++ b/sim/rtlsim/Makefile @@ -20,7 +20,7 @@ CXXFLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) CXXFLAGS += $(CONFIGS) LDFLAGS += $(THIRD_PARTY_DIR)/softfloat/build/Linux-x86_64-GCC/softfloat.a -LDFLAGS += -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator -L$(THIRD_PARTY_DIR)/ramulator -lramulator +LDFLAGS += -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator -L$(THIRD_PARTY_DIR)/ramulator -lramulator -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator_ext -L$(THIRD_PARTY_DIR)/ramulator_ext -Wl,--push-state,--no-as-needed -lramulator-vortex -Wl,--pop-state # control RTL debug tracing states DBG_TRACE_FLAGS += -DDBG_TRACE_PIPELINE diff --git a/sim/rtlsim/processor.cpp b/sim/rtlsim/processor.cpp index a0e223f2e0..b383d556ff 100644 --- a/sim/rtlsim/processor.cpp +++ b/sim/rtlsim/processor.cpp @@ -48,9 +48,6 @@ #include #include -#ifndef MEM_CLOCK_RATIO -#define MEM_CLOCK_RATIO 1 -#endif #ifndef TRACE_START_TIME #define TRACE_START_TIME 0ull @@ -101,7 +98,7 @@ bool sim_trace_enabled() { class Processor::Impl { public: - Impl() : dram_sim_(VX_CFG_PLATFORM_MEMORY_NUM_BANKS, VX_CFG_PLATFORM_MEMORY_DATA_SIZE, MEM_CLOCK_RATIO) { + Impl() : dram_sim_(DramSim::platform_config()) { // restart sim time so a re-created model registers at time zero timestamp = 0; @@ -306,9 +303,9 @@ class Processor::Impl { dram_sim_.tick(); for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { - if (!dram_queue_[b].empty()) { + if (!dram_queue_[b].empty() && dram_sim_.can_accept(b)) { auto mem_req = dram_queue_[b].front(); - dram_sim_.send_request(mem_req->addr, mem_req->write, [](void* arg)->bool { + dram_sim_.send_request(b, mem_req->bank_addr, mem_req->write, [](void* arg)->bool { // mark completed request as ready auto orig_req = reinterpret_cast(arg); orig_req->ready = true; @@ -400,6 +397,7 @@ class Processor::Impl { auto mem_req = new mem_req_t(); mem_req->tag = device_->mem_req_tag[b]; mem_req->addr = byte_addr; + mem_req->bank_addr = uint64_t(device_->mem_req_addr[b]) * VX_CFG_PLATFORM_MEMORY_DATA_SIZE; mem_req->write = true; mem_req->ready = false; @@ -414,6 +412,7 @@ class Processor::Impl { auto mem_req = new mem_req_t(); mem_req->tag = device_->mem_req_tag[b]; mem_req->addr = byte_addr; + mem_req->bank_addr = uint64_t(device_->mem_req_addr[b]) * VX_CFG_PLATFORM_MEMORY_DATA_SIZE; mem_req->write = false; mem_req->ready = false; ram_->read(mem_req->data.data(), byte_addr, VX_CFG_PLATFORM_MEMORY_DATA_SIZE); @@ -445,6 +444,7 @@ class Processor::Impl { Vrtlsim_shim* device; std::array data; uint64_t addr; + uint64_t bank_addr; uint64_t tag; bool write; bool ready; diff --git a/sim/simx/Makefile b/sim/simx/Makefile index a9638e5ac7..746bd6f7f8 100644 --- a/sim/simx/Makefile +++ b/sim/simx/Makefile @@ -27,7 +27,7 @@ CXXFLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) CXXFLAGS += $(CONFIGS) LDFLAGS += $(THIRD_PARTY_DIR)/softfloat/build/Linux-x86_64-GCC/softfloat.a -LDFLAGS += -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator -L$(THIRD_PARTY_DIR)/ramulator -lramulator +LDFLAGS += -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator -L$(THIRD_PARTY_DIR)/ramulator -lramulator -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator_ext -L$(THIRD_PARTY_DIR)/ramulator_ext -Wl,--push-state,--no-as-needed -lramulator-vortex -Wl,--pop-state XCONFIGS := $(shell python3 $(ROOT_DIR)/ci/gen_config.py --config=$(VORTEX_HOME)/VX_config.toml --cflags='$(CONFIGS) -DVX_CFG_XLEN=$(XLEN)') diff --git a/sim/simx/constants.h b/sim/simx/constants.h index 1bdfce56a5..2ef860870a 100644 --- a/sim/simx/constants.h +++ b/sim/simx/constants.h @@ -20,10 +20,6 @@ #define RAM_PAGE_SIZE 4096 #endif -#ifndef MEM_CLOCK_RATIO -#define MEM_CLOCK_RATIO 1 -#endif - #ifndef __MAX #define __MAX(a,b) (((a) > (b)) ? (a) : (b)) #endif diff --git a/sim/simx/mem/memory.cpp b/sim/simx/mem/memory.cpp index 74c3782edc..50db7006ae 100644 --- a/sim/simx/mem/memory.cpp +++ b/sim/simx/mem/memory.cpp @@ -55,13 +55,13 @@ class Memory::Impl { Impl(Memory* simobject, const Config& config) : simobject_(simobject) , config_(config) - , dram_sim_(config.num_banks, config.block_size, config.clock_ratio) + , dram_sim_(config.dram) , ram_(nullptr) { char sname[100]; snprintf(sname, 100, "%s-xbar", simobject->name().c_str()); - mem_xbar_ = MemCrossBar::Create(sname, ArbiterType::RoundRobin, config.num_ports, config.num_banks, - [lg2_block_size = log2ceil(config.block_size), num_banks = config.num_banks](const MemCrossBar::ReqType& req) { + mem_xbar_ = MemCrossBar::Create(sname, ArbiterType::RoundRobin, config.num_ports, config.dram.num_channels, + [lg2_block_size = log2ceil(config.dram.block_size), num_banks = config.dram.num_channels](const MemCrossBar::ReqType& req) { // Custom logic to calculate the output index using bank interleaving return (uint32_t)((req.addr >> lg2_block_size) & (num_banks-1)); }); @@ -85,8 +85,10 @@ class Memory::Impl { void tick() { dram_sim_.tick(); - for (uint32_t i = 0; i < config_.num_banks; ++i) { - if (mem_xbar_->ReqOut.at(i).empty()) + uint32_t num_banks = config_.dram.num_channels; + uint32_t lg2_block_size = log2ceil(config_.dram.block_size); + for (uint32_t i = 0; i < num_banks; ++i) { + if (mem_xbar_->ReqOut.at(i).empty() || !dram_sim_.can_accept(i)) continue; auto& mem_req = mem_xbar_->ReqOut.at(i).peek(); @@ -126,8 +128,10 @@ class Memory::Impl { // enqueue the request to the memory system auto req_args = new DramCallbackArgs{this, mem_req, i, rsp_data}; + uint64_t bank_addr = ((mem_req.addr >> lg2_block_size) / num_banks) << lg2_block_size; dram_sim_.send_request( - mem_req.addr, + i, + bank_addr, mem_req.is_write(), [](void* arg)->bool { auto rsp_args = reinterpret_cast(arg); diff --git a/sim/simx/mem/memory.h b/sim/simx/mem/memory.h index ea060c0e05..dd4a95be39 100644 --- a/sim/simx/mem/memory.h +++ b/sim/simx/mem/memory.h @@ -15,6 +15,7 @@ #include "types.h" #include +#include #include namespace vortex { @@ -22,10 +23,8 @@ namespace vortex { class Memory : public SimObject{ public: struct Config { - uint32_t num_banks; uint32_t num_ports; - uint32_t block_size; - float clock_ratio; + DramSim::Config dram; // one DRAM channel per memory bank }; struct PerfStats { diff --git a/sim/simx/processor.cpp b/sim/simx/processor.cpp index 0aa304656f..52ad76ca1a 100644 --- a/sim/simx/processor.cpp +++ b/sim/simx/processor.cpp @@ -51,12 +51,9 @@ ProcessorImpl::ProcessorImpl() kmu_ = Kmu::Create("kmu", total_cores); // create memory simulator - memsim_ = Memory::Create("dram", Memory::Config{ - VX_CFG_PLATFORM_MEMORY_NUM_BANKS, - VX_CFG_L3_MEM_PORTS, - VX_CFG_MEM_BLOCK_SIZE, - MEM_CLOCK_RATIO - }); + auto dram_config = DramSim::platform_config(); + dram_config.block_size = VX_CFG_MEM_BLOCK_SIZE; + memsim_ = Memory::Create("dram", Memory::Config{VX_CFG_L3_MEM_PORTS, dram_config}); char sname[100]; diff --git a/sim/xrtsim/Makefile b/sim/xrtsim/Makefile index 231d745806..4e51883d19 100644 --- a/sim/xrtsim/Makefile +++ b/sim/xrtsim/Makefile @@ -22,7 +22,7 @@ CXXFLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) CXXFLAGS += $(CONFIGS) LDFLAGS += -shared $(THIRD_PARTY_DIR)/softfloat/build/Linux-x86_64-GCC/softfloat.a -LDFLAGS += -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator -L$(THIRD_PARTY_DIR)/ramulator -lramulator -pthread +LDFLAGS += -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator -L$(THIRD_PARTY_DIR)/ramulator -lramulator -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator_ext -L$(THIRD_PARTY_DIR)/ramulator_ext -Wl,--push-state,--no-as-needed -lramulator-vortex -Wl,--pop-state -pthread # control RTL debug tracing states DBG_TRACE_FLAGS += -DDBG_TRACE_PIPELINE diff --git a/sim/xrtsim/xrt_sim.cpp b/sim/xrtsim/xrt_sim.cpp index fcab150cfe..af948d92dc 100644 --- a/sim/xrtsim/xrt_sim.cpp +++ b/sim/xrtsim/xrt_sim.cpp @@ -50,9 +50,6 @@ #include -#ifndef MEM_CLOCK_RATIO -#define MEM_CLOCK_RATIO 1 -#endif #define CACHE_BLOCK_SIZE 64 @@ -144,7 +141,7 @@ class xrt_sim::Impl { , ram_{} , host_ram_(nullptr) , host_alloc_(nullptr) - , dram_sim_(VX_CFG_PLATFORM_MEMORY_NUM_BANKS, VX_CFG_PLATFORM_MEMORY_DATA_SIZE, MEM_CLOCK_RATIO) + , dram_sim_(DramSim::platform_config()) , stop_(false) #ifdef VCD_OUTPUT , tfp_(nullptr) @@ -536,9 +533,9 @@ class xrt_sim::Impl { dram_sim_.tick(); for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { - if (!dram_queues_[b].empty()) { + if (!dram_queues_[b].empty() && dram_sim_.can_accept(b)) { auto mem_req = dram_queues_[b].front(); - dram_sim_.send_request(mem_req->addr, mem_req->write, [](void* arg)->bool { + dram_sim_.send_request(b, bank_local_addr(mem_req->addr), mem_req->write, [](void* arg)->bool { auto orig_req = reinterpret_cast(arg); if (orig_req->ready) { delete orig_req; @@ -556,6 +553,19 @@ class xrt_sim::Impl { #endif } + // Byte address within its bank (DRAM channel) of an AXI bank address: + // VX_mem_to_axi places the bank select above the data-size bits when the + // banks are interleaved. + static uint64_t bank_local_addr(uint64_t axi_addr) { + #if VX_CFG_PLATFORM_MEMORY_INTERLEAVE == 1 + constexpr uint32_t lg2_data_size = log2ceil(VX_CFG_PLATFORM_MEMORY_DATA_SIZE); + constexpr uint32_t bank_sel_bits = log2ceil(VX_CFG_PLATFORM_MEMORY_NUM_BANKS); + return (axi_addr >> (bank_sel_bits + lg2_data_size)) << lg2_data_size; + #else + return axi_addr; + #endif + } + void eval() { device_->eval(); #ifdef VCD_OUTPUT diff --git a/tests/unittest/Makefile b/tests/unittest/Makefile index 0c561d642e..33554509cd 100644 --- a/tests/unittest/Makefile +++ b/tests/unittest/Makefile @@ -1,7 +1,7 @@ ROOT_DIR := $(realpath ../..) include $(ROOT_DIR)/config.mk -PROJECTS := vx_malloc linked_list dxa_ooo rvc_decompress gfx_binsort gfx_tex_sw gfx_rast_sw gfx_msaa gfx_om_mrt +PROJECTS := vx_malloc linked_list dram_sim dxa_ooo rvc_decompress gfx_binsort gfx_tex_sw gfx_rast_sw gfx_msaa gfx_om_mrt all: $(foreach p,$(PROJECTS),$(MAKE) -C $(p) &&) true diff --git a/tests/unittest/dram_sim/Makefile b/tests/unittest/dram_sim/Makefile new file mode 100644 index 0000000000..7994c7a91d --- /dev/null +++ b/tests/unittest/dram_sim/Makefile @@ -0,0 +1,16 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := dram_sim + +SRC_DIR := $(VORTEX_HOME)/tests/unittest/$(PROJECT) +RAMULATOR_DIR := $(THIRD_PARTY_DIR)/ramulator + +SRCS := $(SRC_DIR)/main.cpp $(SIM_COMMON_DIR)/dram_sim.cpp + +CXXFLAGS += -I$(SIM_COMMON_DIR) +CXXFLAGS += -isystem $(RAMULATOR_DIR)/ext/spdlog/include -isystem $(RAMULATOR_DIR)/ext/yaml-cpp/include -isystem $(RAMULATOR_DIR)/src +LDFLAGS += -Wl,-rpath,$(RAMULATOR_DIR) -L$(RAMULATOR_DIR) -lramulator +LDFLAGS += -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator_ext -L$(THIRD_PARTY_DIR)/ramulator_ext -Wl,--push-state,--no-as-needed -lramulator-vortex -Wl,--pop-state + +include ../common.mk diff --git a/tests/unittest/dram_sim/main.cpp b/tests/unittest/dram_sim/main.cpp new file mode 100644 index 0000000000..81e127e145 --- /dev/null +++ b/tests/unittest/dram_sim/main.cpp @@ -0,0 +1,228 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include + +using namespace vortex; + +static uint64_t g_responses = 0; + +static bool on_response(void*) { + ++g_responses; + return true; +} + +// A100 HBM as GPGPU-Sim's SM80 configuration models it: 40 channels at +// 1,512 MHz behind a 1,410 MHz core, 190 cycles of fixed latency, a 257-request +// window per channel. +static DramSim::Config a100_config() { + DramSim::Config config; + config.type = DramSim::Type::HBM2_A100; + config.num_channels = 40; + config.block_size = 64; + config.core_clock = 1410; + config.dram_clock = 1512; + config.latency = 190; + config.window = 257; + return config; +} + +// Median core cycles for one read on an idle channel (the median, so the few +// reads that wait behind a refresh do not count); `addr_of(i)` picks the i-th +// address, and the first read (cold) is not counted. +template +static double unloaded_latency(const DramSim::Config& config, F addr_of) { + DramSim dram(config); + dram.reset(); + const int n = 33; + std::vector samples; + for (int i = 0; i < n; ++i) { + g_responses = 0; + uint64_t cycles = 0; + dram.send_request(0, addr_of(i), false, on_response, nullptr); + while (g_responses == 0) { + dram.tick(); + ++cycles; + } + if (i != 0) { + samples.push_back(cycles); + } + for (int k = 0; k < 256; ++k) { + dram.tick(); + } + } + std::sort(samples.begin(), samples.end()); + return double(samples[samples.size() / 2]); +} + +// Bytes per core cycle when every channel streams `blocks` sequential blocks; +// `peak_inflight` receives the most reads outstanding at once. +static double stream_bandwidth(const DramSim::Config& config, uint32_t blocks, uint64_t* peak_inflight = nullptr) { + DramSim dram(config); + dram.reset(); + g_responses = 0; + std::vector sent(config.num_channels, 0); + uint64_t total = uint64_t(blocks) * config.num_channels; + uint64_t issued = 0; + uint64_t cycles = 0; + uint64_t peak = 0; + while (g_responses < total) { + for (uint32_t ch = 0; ch < config.num_channels; ++ch) { + if (sent[ch] < blocks && dram.can_accept(ch)) { + dram.send_request(ch, uint64_t(sent[ch]) * config.block_size, false, on_response, nullptr); + ++sent[ch]; + ++issued; + } + } + peak = std::max(peak, issued - g_responses); + dram.tick(); + ++cycles; + } + if (peak_inflight) { + *peak_inflight = peak; + } + return double(total) * config.block_size / cycles; +} + +static int g_errors = 0; + +static void check(bool cond, const char* what, double got) { + printf("%s %s (%.1f)\n", cond ? "PASS" : "FAIL", what, got); + if (!cond) { + ++g_errors; + } +} + +int main() { + auto a100 = a100_config(); + + // Unloaded latency: open-row hits, then row conflicts (precharge + activate). + double open = unloaded_latency(a100, [](int i) { return uint64_t(i % 8) * 128; }); + double conflict = unloaded_latency(a100, [](int i) { return uint64_t(i) << 20; }); + check(open > 205 && open < 225, "A100 open-row read latency ~215 core cycles", open); + check(conflict > 245 && conflict < 265, "A100 row-conflict read latency ~255 core cycles", conflict); + + // The fixed latency is added exactly once per read (measured at a 1:1 clock + // ratio, so the DRAM clock phase cannot shift the comparison). + auto unit_clock = a100; + unit_clock.core_clock = unit_clock.dram_clock; + auto no_latency = unit_clock; + no_latency.latency = 0; + auto open_row = [](int i) { return uint64_t(i % 8) * 128; }; + double added = unloaded_latency(unit_clock, open_row) - unloaded_latency(no_latency, open_row); + check(added == 190, "fixed latency adds 190 cycles", added); + + // Every channel is an independent controller: 40 channels stream ~40x one. + auto one = a100; + one.num_channels = 1; + double bw1 = stream_bandwidth(one, 4096); + double bw40 = stream_bandwidth(a100, 4096); + check(bw40 / bw1 > 36 && bw40 / bw1 < 41, "bandwidth scales with the channel count", bw40 / bw1); + + // Saturated stream efficiency against the A100's 1,373 B/cycle peak. + check(bw40 > 0.6 * 1373 && bw40 <= 1373, "A100 stream reaches 60-100% of peak", bw40); + + // The window bounds the requests in flight. + auto narrow = a100; + narrow.num_channels = 1; + narrow.window = 8; + uint64_t peak = 0; + stream_bandwidth(narrow, 1024, &peak); + check(peak <= 8, "window of 8 bounds requests in flight", peak); + + // A block of two bursts returns one response. + auto wide = a100; + wide.num_channels = 1; + wide.block_size = 128; + { + DramSim dram(wide); + dram.reset(); + g_responses = 0; + dram.send_request(0, 0, false, on_response, nullptr); + for (int k = 0; k < 2000; ++k) { + dram.tick(); + } + check(g_responses == 1, "a 128 B block returns one response", double(g_responses)); + } + + // Plain HBM2 at the platform default: still independent channels. + auto hbm2 = a100; + hbm2.type = DramSim::Type::HBM2; + hbm2.num_channels = 2; + hbm2.latency = 0; + hbm2.window = 0; + double bw2 = stream_bandwidth(hbm2, 4096); + hbm2.num_channels = 1; + double bw1h = stream_bandwidth(hbm2, 4096); + check(bw2 / bw1h > 1.8 && bw2 / bw1h < 2.05, "HBM2 2 channels stream ~2x one", bw2 / bw1h); + + // Every channel refreshes: over many refresh intervals some open-row reads + // wait behind a refresh (nRFC = 160 cycles at a 1:1 clock). + { + auto refresh = unit_clock; + refresh.type = DramSim::Type::HBM2; + refresh.num_channels = 1; + refresh.latency = 0; + DramSim dram(refresh); + dram.reset(); + uint64_t min_lat = ~0ull, max_lat = 0; + for (int i = 0; i < 400; ++i) { + g_responses = 0; + uint64_t cycles = 0; + dram.send_request(0, uint64_t(i % 8) * 128, false, on_response, nullptr); + while (g_responses == 0) { + dram.tick(); + ++cycles; + } + if (i != 0) { + min_lat = std::min(min_lat, cycles); + max_lat = std::max(max_lat, cycles); + } + for (int k = 0; k < 97; ++k) { + dram.tick(); + } + } + check(max_lat - min_lat > 100, "refresh stalls reads by ~nRFC", double(max_lat - min_lat)); + } + + // GDDR6 at the rtx5080 profile's shape: 32 x16 channels at 15 GT/s (7,500 + // MHz data clock) behind a 2,295 MHz core, 32 B per 8-clock burst: 418 B/cycle + // peak, the card's 960 GB/s. + auto gddr6 = a100; + gddr6.type = DramSim::Type::GDDR6; + gddr6.num_channels = 32; + gddr6.core_clock = 2295; + gddr6.dram_clock = 7500; + gddr6.latency = 0; + gddr6.window = 0; + double gpeak = 32.0 * 32 / 8 * 7500 / 2295; + double gbw32 = stream_bandwidth(gddr6, 4096); + check(gbw32 > 0.6 * gpeak && gbw32 <= gpeak, "GDDR6 stream reaches 60-100% of peak", gbw32); + auto gddr6_one = gddr6; + gddr6_one.num_channels = 1; + double gbw1 = stream_bandwidth(gddr6_one, 4096); + check(gbw32 / gbw1 > 29 && gbw32 / gbw1 < 33, "GDDR6 bandwidth scales with the channel count", gbw32 / gbw1); + double gopen = unloaded_latency(gddr6_one, [](int i) { return uint64_t(i % 8) * 32; }); + double gconflict = unloaded_latency(gddr6_one, [](int i) { return uint64_t(i) << 22; }); + check(gopen < gconflict, "GDDR6 row conflicts cost more than row hits", gconflict - gopen); + + if (g_errors != 0) { + printf("FAILED: %d check(s)\n", g_errors); + return 1; + } + printf("PASSED!\n"); + return 0; +} diff --git a/third_party/Makefile b/third_party/Makefile index a5df42e081..b9e5d9f37f 100644 --- a/third_party/Makefile +++ b/third_party/Makefile @@ -7,7 +7,24 @@ softfloat: ramulator/libramulator.so: cd ramulator && mkdir -p build && cd build && cmake .. && make -j4 -ramulator: ramulator/libramulator.so + +# Vortex's Ramulator extensions: a per-channel refresh manager, and the GDDR6 +# device model Ramulator ships but does not build. Both register with +# Ramulator's factory when the library loads, built as Ramulator builds itself. +RAMULATOR_EXT_DIR := ramulator_ext +RAMULATOR_EXT_INCS := -isystem ramulator/src -isystem ramulator/ext/spdlog/include -isystem ramulator/ext/yaml-cpp/include +RAMULATOR_EXT_CXXFLAGS := -std=c++20 -O3 -DNDEBUG -fPIC $(RAMULATOR_EXT_INCS) + +$(RAMULATOR_EXT_DIR)/channel_refresh.o: $(RAMULATOR_EXT_DIR)/channel_refresh.cpp + $(CXX) $(RAMULATOR_EXT_CXXFLAGS) -Wall -Wextra -Werror -c $< -o $@ + +$(RAMULATOR_EXT_DIR)/gddr6.o: ramulator/src/dram/impl/GDDR6.cpp + $(CXX) $(RAMULATOR_EXT_CXXFLAGS) -c $< -o $@ + +$(RAMULATOR_EXT_DIR)/libramulator-vortex.so: $(RAMULATOR_EXT_DIR)/channel_refresh.o $(RAMULATOR_EXT_DIR)/gddr6.o ramulator/libramulator.so + $(CXX) -shared $(filter %.o,$^) -Lramulator -lramulator -o $@ + +ramulator: ramulator/libramulator.so $(RAMULATOR_EXT_DIR)/libramulator-vortex.so cocogfx/libcocogfx.a: $(MAKE) -C cocogfx @@ -16,6 +33,7 @@ cocogfx: cocogfx/libcocogfx.a clean: $(MAKE) -C softfloat/build/Linux-x86_64-GCC clean rm -rf ramulator/build ramulator/libramulator.so + rm -f $(RAMULATOR_EXT_DIR)/*.o $(RAMULATOR_EXT_DIR)/*.so $(MAKE) -C cocogfx clean .PHONY: all cvfpu softfloat ramulator cocogfx \ No newline at end of file diff --git a/third_party/ramulator_ext/channel_refresh.cpp b/third_party/ramulator_ext/channel_refresh.cpp new file mode 100644 index 0000000000..3888669fe9 --- /dev/null +++ b/third_party/ramulator_ext/channel_refresh.cpp @@ -0,0 +1,65 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +#include "base/base.h" +#include "dram_controller/controller.h" +#include "dram_controller/refresh.h" + +namespace Ramulator { + +// All-bank refresh every nREFI, one REFab per channel, the channels staggered +// across the interval as independent controllers are. Ramulator's AllBank +// manager refreshes per rank, and HBM2 and GDDR6 have no rank level, so with it +// these devices never refresh. +class ChannelRefresh : public IRefreshManager, public Implementation { + RAMULATOR_REGISTER_IMPLEMENTATION(IRefreshManager, ChannelRefresh, "ChannelRefresh", "All-bank refresh of a rank-less channel.") +private: + Clk_t clk_ = 0; + IDRAMController* ctrl_ = nullptr; + int num_levels_ = -1; + int nrefi_ = -1; + int req_id_ = -1; + Clk_t next_refresh_ = -1; + +public: + void init() override { + ctrl_ = cast_parent(); + } + + void setup(IFrontEnd* /*frontend*/, IMemorySystem* /*memory_system*/) override { + auto dram = ctrl_->m_dram; + num_levels_ = int(dram->m_levels.size()); + nrefi_ = dram->m_timing_vals("nREFI"); + req_id_ = dram->m_requests("all-bank-refresh"); + int num_channels = dram->get_level_size("channel"); + next_refresh_ = nrefi_ + Clk_t(ctrl_->m_channel_id) * nrefi_ / num_channels; + } + + void tick() override { + if (++clk_ != next_refresh_) { + return; + } + next_refresh_ += nrefi_; + std::vector addr_vec(num_levels_, -1); + addr_vec[0] = ctrl_->m_channel_id; + Request req(addr_vec, req_id_); + if (!ctrl_->priority_send(req)) { + throw std::runtime_error("Failed to send refresh!"); + } + } +}; + +} From 0897cbd9af42b08e0371d13dae98adb60705033d Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 10 Oct 2026 23:52:22 -0700 Subject: [PATCH 020/124] profiles: add a100, h100, v100, rtx1080 and tinygpu; document profiles Adds the named machines next to rtx5080 (renamed from rtx_5080), each with a source for every value and sizes rounded to the powers of two the caches accept: - tinygpu: 1 core, 2 warps x 2 threads, shared memory at its default size, default ISA (so it runs the shipped RV32IMAF/RV64IMAFD runtime); the FPGA bring-up and fast simulation machine; - rtx1080 (GTX 1080), v100, a100, h100 and rtx5080 as SimX-scale study machines, now including their DRAM: HBM2 for V100, A100 HBM timing, HBM3 bandwidth on the HBM2 model for H100, GDDR5X/GDDR7 bandwidth on the GDDR6 model for the GeForce parts. Documents profiles and the DRAM keys in docs/simulation.md, configure --profile in docs/install_vortex.md and the profile layer in the build configuration design. Verified: every profile builds and passes vecadd on simx (XLEN 32); tinygpu passes sgemm on rtlsim at XLEN 32 and 64 with the same instruction count as simx, and sgemm2 (-t2) and dotproduct, which go through shared memory, on simx and rtlsim at XLEN 32 and 64. Co-Authored-By: Claude Opus 5.5 --- docs/codebase.md | 2 ++ docs/designs/build_configuration_system.md | 20 ++++++++++++ docs/install_vortex.md | 6 ++-- docs/simulation.md | 36 ++++++++++++++++++++++ profiles/a100.json | 35 +++++++++++++++++++++ profiles/h100.json | 32 +++++++++++++++++++ profiles/rtx1080.json | 32 +++++++++++++++++++ profiles/{rtx_5080.json => rtx5080.json} | 32 ++++++++++++++----- profiles/tinygpu.json | 15 +++++++++ profiles/v100.json | 32 +++++++++++++++++++ 10 files changed, 233 insertions(+), 9 deletions(-) create mode 100644 profiles/a100.json create mode 100644 profiles/h100.json create mode 100644 profiles/rtx1080.json rename profiles/{rtx_5080.json => rtx5080.json} (62%) create mode 100644 profiles/tinygpu.json create mode 100644 profiles/v100.json diff --git a/docs/codebase.md b/docs/codebase.md index 257af03e13..52d8951279 100644 --- a/docs/codebase.md +++ b/docs/codebase.md @@ -5,6 +5,8 @@ The directory/file layout of the Vortex codebase is as follows: - `VX_config.toml` / `VX_types.toml`: hardware configuration system — the single source of truth for all build-time parameters (see the [build configuration system](designs/build_configuration_system.md)). +- `profiles`: hardware profiles — named machines (`a100`, `rtx5080`, `tinygpu`, ...) + layered over `VX_config.toml` (see [Hardware Profiles](simulation.md#hardware-profiles)) - `ci`: continuous integration scripts - `testcases`: declarative test catalog (YAML), executed by the pytest harness - `baselines/perf`: golden performance baselines for the perf-regression gate diff --git a/docs/designs/build_configuration_system.md b/docs/designs/build_configuration_system.md index c231a03bbe..ccc4a9a3bb 100644 --- a/docs/designs/build_configuration_system.md +++ b/docs/designs/build_configuration_system.md @@ -241,6 +241,26 @@ Two details are load-bearing: against a driver built with another disagrees about the hardware shape with no diagnostic; rebuild the application with the same `CONFIGS` first. +### 6.1 Hardware profiles + +A hardware profile (`profiles/.json`) is a named machine layered between +the TOML defaults and the caller's flags. `PROFILE` — set by +`configure --profile=` or `blackbox.sh --profile=` — makes +`config.mk` fold the profile into `CONFIGS` once, so every consumer in the +table above sees one `CONFIGS`: + +```make +override CONFIGS := $(shell python3 $(VORTEX_HOME)/ci/gen_config.py --config=... \ + --profile=$(PROFILE) --merge-configs --cflags='$(CONFIGS)') +``` + +`--merge-configs` drops every profile entry for a knob the caller already sets +(recognising `X_DISABLE`/`X_ENABLE` and `X_`/`X=` as one knob), so +explicit flags win, then appends the caller's `CONFIGS` verbatim. The +generator validates a profile against the TOML: every key must be a public +knob, enum values must be declared, values are literals (expressions belong in +the TOML), and `sources` is required. + --- ## 7. The hardware/software boundary diff --git a/docs/install_vortex.md b/docs/install_vortex.md index d473aa34e3..d849384e75 100644 --- a/docs/install_vortex.md +++ b/docs/install_vortex.md @@ -40,8 +40,10 @@ sourcing is required and multiple Vortex trees can coexist on one machine without a global `~/.bashrc` clobber. - Useful configure options: `--xlen={32,64}` selects the ISA width and - `--prefix=` the install location. Two SimX build knobs are plain + Useful configure options: `--xlen={32,64}` selects the ISA width, + `--prefix=` the install location, and `--profile=` builds the + machine described by `profiles/.json` by default (see + [Hardware Profiles](simulation.md#hardware-profiles)). Two SimX build knobs are plain config defines, passed like any `VX_CFG_*` option through `CONFIGS`: `-DSIMX_MT=` runs the simulator on `T` host threads (default serial; cycle results are bit-identical for every thread count), and diff --git a/docs/simulation.md b/docs/simulation.md index 4b9d67fb8e..af69ae9a27 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -138,6 +138,42 @@ PERF: instrs=363180, cycles=53108, IPC=6.838518 > to compare configurations relative to one another, not as fixed targets to > reproduce. +### Hardware Profiles + +A hardware profile is a named machine in `profiles/.json`: a set of +`VX_CFG_*` values, grouped into `core`, `chip` and `platform` sections, with a +`sources` entry saying where each value comes from (optionally also a +`reference` block of measurements from the real part, and `extends` to build +on another profile). Every flow that includes `config.mk` builds the same +machine from it: SimX, rtlsim, the FPGA and ASIC flows, and the kernels. + +| Profile | Machine | +|---|---| +| `tinygpu` | 1 core, 2 warps x 2 threads, shared memory | +| `rtx1080` | GeForce GTX 1080 (Pascal): 20 cores x 64 warps x 32 threads, 2 MB L2, GDDR5X as 16 GDDR6 channels | +| `v100` | Tesla V100 (Volta): 80 cores, 4 MB L2, 32 HBM2 channels at 877 MHz | +| `a100` | A100 (Ampere): 108 cores, 32 MB L2, GPGPU-Sim's A100 HBM timing | +| `h100` | H100 SXM5 (Hopper): 132 cores, 32 MB L2, HBM3 bandwidth on the HBM2 model | +| `rtx5080` | GeForce RTX 5080 (Blackwell): 84 cores x 48 warps, 64 MB L2, GDDR7 bandwidth on the GDDR6 model | + +Select one per build with `../configure --profile=`, or per run: + +```bash +$ ./ci/blackbox.sh --driver=simx --profile=rtx5080 --app=sgemm --args="-n256" +``` + +`CONFIGS` and blackbox's own `--clusters/--cores/--warps/--threads/--l2cache/--l3cache` +flags win over the profile: the profile only supplies the knobs they leave +unset, and everything else falls back to `VX_config.toml`. + +```bash +$ CONFIGS="-DVX_CFG_L2_SIZE=8388608" ./ci/blackbox.sh --driver=simx --profile=a100 --warps=32 --app=sgemm +``` + +The other profiles (20 to 132 cores) are SimX-scale machines for architecture +studies, not FPGA builds; `tinygpu` is the small one, sized for FPGA +bring-up and fast simulation. + ### Configuring the DRAM Model The simulators (simx, rtlsim, opaesim, xrtsim) all model the device memory diff --git a/profiles/a100.json b/profiles/a100.json new file mode 100644 index 0000000000..1449b1fbf1 --- /dev/null +++ b/profiles/a100.json @@ -0,0 +1,35 @@ +{ + "name": "a100", + "description": "NVIDIA A100 SXM4 (GA100, Ampere, sm_80) at its 1410 MHz boost clock: one Vortex core per SM.", + "core": { + "VX_CFG_NUM_WARPS": 64, + "VX_CFG_NUM_THREADS": 32, + "VX_CFG_ISSUE_WIDTH": 4, + "VX_CFG_DCACHE_SIZE": 131072 + }, + "chip": { + "VX_CFG_NUM_CLUSTERS": 1, + "VX_CFG_NUM_CORES": 108, + "VX_CFG_L2_ENABLE": true, + "VX_CFG_L2_SIZE": 33554432, + "VX_CFG_L2_NUM_WAYS": 16, + "VX_CFG_L2_NUM_BANKS": 32 + }, + "platform": { + "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 32, + "VX_CFG_PLATFORM_CLOCK_RATE": 1410, + "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 1555000, + "VX_CFG_PLATFORM_MEMORY_TYPE": "HBM2_A100", + "VX_CFG_PLATFORM_MEMORY_CLOCK_RATE": 1512, + "VX_CFG_PLATFORM_MEMORY_LATENCY": 190, + "VX_CFG_PLATFORM_MEMORY_WINDOW": 257 + }, + "sources": { + "NUM_CORES, NUM_WARPS, NUM_THREADS, ISSUE_WIDTH": "108 SMs, 64 resident warps/SM, 4 schedulers x 32 lanes (NVIDIA A100 Tensor Core GPU Architecture whitepaper, Table 3)", + "DCACHE_SIZE": "192 KB combined L1/shared per SM (whitepaper), rounded down to the power of two the cache accepts", + "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "40 MB L2 in 80 slices (whitepaper), rounded down to 32 MB and 32 banks; 16 ways as in GPGPU-Sim's SM80_A100 dl2", + "PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_CLOCK_RATE": "HBM2 at GPGPU-Sim SM80_A100's 1512 MHz DRAM clock; 32 channels x 32 B per DRAM cycle = 1548 GB/s, the card's 1555 GB/s", + "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_LATENCY, PLATFORM_MEMORY_WINDOW": "HBM timing, fixed DRAM latency and per-channel request window of GPGPU-Sim's SM80_A100 configuration (accel-sim gpu-simulator/configs/tested-cfgs/SM80_A100/gpgpusim.config)", + "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1410 MHz boost clock, 1555 GB/s (whitepaper, Table 3)" + } +} diff --git a/profiles/h100.json b/profiles/h100.json new file mode 100644 index 0000000000..ce1a216401 --- /dev/null +++ b/profiles/h100.json @@ -0,0 +1,32 @@ +{ + "name": "h100", + "description": "NVIDIA H100 SXM5 (GH100, Hopper, sm_90) at its 1980 MHz boost clock: one Vortex core per SM.", + "core": { + "VX_CFG_NUM_WARPS": 64, + "VX_CFG_NUM_THREADS": 32, + "VX_CFG_ISSUE_WIDTH": 4, + "VX_CFG_DCACHE_SIZE": 262144 + }, + "chip": { + "VX_CFG_NUM_CLUSTERS": 1, + "VX_CFG_NUM_CORES": 132, + "VX_CFG_L2_ENABLE": true, + "VX_CFG_L2_SIZE": 33554432, + "VX_CFG_L2_NUM_WAYS": 16, + "VX_CFG_L2_NUM_BANKS": 32 + }, + "platform": { + "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 32, + "VX_CFG_PLATFORM_CLOCK_RATE": 1980, + "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 3350000, + "VX_CFG_PLATFORM_MEMORY_TYPE": "HBM2", + "VX_CFG_PLATFORM_MEMORY_CLOCK_RATE": 3272 + }, + "sources": { + "NUM_CORES, NUM_WARPS, NUM_THREADS, ISSUE_WIDTH": "132 SMs (SXM5), 64 resident warps/SM, 4 schedulers x 32 lanes (NVIDIA H100 Tensor Core GPU Architecture whitepaper)", + "DCACHE_SIZE": "256 KB combined L1/shared per SM (whitepaper)", + "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "50 MB L2 (whitepaper), rounded down to the power of two the cache accepts; 16 ways and 32 banks as in the a100 profile", + "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_CLOCK_RATE": "HBM3 modelled with the HBM2 timing: 32 channels x 32 B per DRAM cycle at 3272 MHz = 3.35 TB/s, the card's bandwidth; latency not calibrated", + "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1980 MHz boost clock, 3.35 TB/s (H100 SXM5 datasheet)" + } +} diff --git a/profiles/rtx1080.json b/profiles/rtx1080.json new file mode 100644 index 0000000000..7728591970 --- /dev/null +++ b/profiles/rtx1080.json @@ -0,0 +1,32 @@ +{ + "name": "rtx1080", + "description": "NVIDIA GeForce GTX 1080 (GP104, Pascal, sm_61) at its 1607 MHz base clock: one Vortex core per SM.", + "core": { + "VX_CFG_NUM_WARPS": 64, + "VX_CFG_NUM_THREADS": 32, + "VX_CFG_ISSUE_WIDTH": 4, + "VX_CFG_DCACHE_SIZE": 32768 + }, + "chip": { + "VX_CFG_NUM_CLUSTERS": 1, + "VX_CFG_NUM_CORES": 20, + "VX_CFG_L2_ENABLE": true, + "VX_CFG_L2_SIZE": 2097152, + "VX_CFG_L2_NUM_WAYS": 16, + "VX_CFG_L2_NUM_BANKS": 16 + }, + "platform": { + "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 16, + "VX_CFG_PLATFORM_CLOCK_RATE": 1607, + "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 320000, + "VX_CFG_PLATFORM_MEMORY_TYPE": "GDDR6", + "VX_CFG_PLATFORM_MEMORY_CLOCK_RATE": 5000 + }, + "sources": { + "NUM_CORES, NUM_WARPS, NUM_THREADS, ISSUE_WIDTH": "20 SMs, 64 resident warps/SM, 4 schedulers x 32 lanes (NVIDIA GeForce GTX 1080 whitepaper)", + "DCACHE_SIZE": "48 KB unified L1/texture per SM (whitepaper), rounded down to the power of two the cache accepts", + "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "2 MB L2 (whitepaper); 16 ways, 16 banks for the 16 memory channels", + "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_CLOCK_RATE": "256-bit GDDR5X at 10 Gbps modelled as 16 GDDR6 x16 channels at a 5000 MHz data clock: 16 x 2 B x 10 GT/s = 320 GB/s, the card's bandwidth; latency not calibrated", + "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1607 MHz base clock, 320 GB/s (whitepaper)" + } +} diff --git a/profiles/rtx_5080.json b/profiles/rtx5080.json similarity index 62% rename from profiles/rtx_5080.json rename to profiles/rtx5080.json index 8dc5906364..6aa20dcd91 100644 --- a/profiles/rtx_5080.json +++ b/profiles/rtx5080.json @@ -1,5 +1,5 @@ { - "name": "rtx_5080", + "name": "rtx5080", "description": "NVIDIA GeForce RTX 5080 (GB203, Blackwell, sm_120) at its 2295 MHz base clock: one Vortex core per SM.", "core": { "VX_CFG_NUM_WARPS": 48, @@ -20,7 +20,10 @@ "platform": { "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 32, "VX_CFG_PLATFORM_CLOCK_RATE": 2295, - "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 960000 + "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 960000, + "VX_CFG_PLATFORM_MEMORY_TYPE": "GDDR6", + "VX_CFG_PLATFORM_MEMORY_CLOCK_RATE": 7500, + "VX_CFG_PLATFORM_MEMORY_LATENCY": 407 }, "reference": { "clock_mhz": 2295, @@ -28,18 +31,33 @@ "l2_hit_cycles": 354.4, "dram_cycles": 802, "dram_peak_bytes_per_cycle": 423, - "stream_efficiency": { "4MiB": 0.795, "16MiB": 0.894, "64MiB": 0.919 }, - "sgemm_cycles": { "128": 24193, "256": 45476, "512": 270396, "1024": 1845287 }, - "stream_cycles": { "4MiB": 25569, "16MiB": 90741, "64MiB": 427878 } + "stream_efficiency": { + "4MiB": 0.795, + "16MiB": 0.894, + "64MiB": 0.919 + }, + "sgemm_cycles": { + "128": 24193, + "256": 45476, + "512": 270396, + "1024": 1845287 + }, + "stream_cycles": { + "4MiB": 25569, + "16MiB": 90741, + "64MiB": 427878 + } }, "sources": { - "NUM_CORES, NUM_WARPS, NUM_THREADS, ISSUE_WIDTH": "84 SMs, 48 resident warps/SM, 4 schedulers x 32 lanes (NVIDIA RTX Blackwell GPU Architecture whitepaper, App. B; cudaGetDeviceProperties on the user's RTX 5080)", + "NUM_CORES, NUM_WARPS, NUM_THREADS, ISSUE_WIDTH": "84 SMs, 48 resident warps/SM, 4 schedulers x 32 lanes (NVIDIA RTX Blackwell GPU Architecture whitepaper, App. B; cudaGetDeviceProperties on an RTX 5080)", "DCACHE_SIZE": "128 KB L1/shared per SM (whitepaper App. B)", "L2_SIZE, L2_NUM_WAYS": "64 MB L2 (whitepaper App. B); 16 ways as in GPGPU-Sim's Ampere/Hopper L2 configs", "L2_NUM_BANKS": "32 banks as the nearest power of two to the 48 L2 slices measured by arXiv 2507.10789", "DCACHE_MSHR_SIZE, L2_MSHR_SIZE": "384 L1 MSHR entries per SM (24 x 16 banks) and 192 per L2 slice, GPGPU-Sim SM86_RTX3070 (-gpgpu_cache:dl1 A:384:48, dl2 A:192:4)", "PLATFORM_MEMORY_NUM_BANKS": "8 x 32-bit GDDR7 devices = 32 independent channels (whitepaper App. B)", "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "2295 MHz base clock, 960 GB/s (whitepaper App. B)", - "reference": "Nsight Compute on the user's RTX 5080, clocks locked to base, cold L2, 2026-10-05 (docs/reports/simx_vs_rtx5080_mlp_report.md)" + "reference": "Nsight Compute on an RTX 5080 at its 2295 MHz base clock with a cold L2, 2026-10-05", + "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_CLOCK_RATE": "GDDR7 modelled as GDDR6 x16 channels: 32 channels x 2 B x 15 GT/s (7500 MHz data clock) = 960 GB/s, the card's bandwidth", + "PLATFORM_MEMORY_LATENCY": "407 core cycles of PHY/NoC latency, calibrated in SimX against the 802-cycle pointer-chase DRAM latency in 'reference' (recalibrate when the memory model changes)" } } diff --git a/profiles/tinygpu.json b/profiles/tinygpu.json new file mode 100644 index 0000000000..9c496a6237 --- /dev/null +++ b/profiles/tinygpu.json @@ -0,0 +1,15 @@ +{ + "name": "tinygpu", + "description": "The smallest Vortex: one core of 2 warps x 2 threads with shared memory.", + "core": { + "VX_CFG_NUM_WARPS": 2, + "VX_CFG_NUM_THREADS": 2 + }, + "chip": { + "VX_CFG_NUM_CLUSTERS": 1, + "VX_CFG_NUM_CORES": 1 + }, + "sources": { + "all": "A minimal reference machine for small FPGAs and fast simulation (1 core, NW=NT=2, shared memory at its default size); the ISA keeps the defaults (RV32IMAF/RV64IMAFD) so it runs the shipped toolchain runtime" + } +} diff --git a/profiles/v100.json b/profiles/v100.json new file mode 100644 index 0000000000..c58be16476 --- /dev/null +++ b/profiles/v100.json @@ -0,0 +1,32 @@ +{ + "name": "v100", + "description": "NVIDIA Tesla V100 SXM2 (GV100, Volta, sm_70) at its 1530 MHz boost clock: one Vortex core per SM.", + "core": { + "VX_CFG_NUM_WARPS": 64, + "VX_CFG_NUM_THREADS": 32, + "VX_CFG_ISSUE_WIDTH": 4, + "VX_CFG_DCACHE_SIZE": 131072 + }, + "chip": { + "VX_CFG_NUM_CLUSTERS": 1, + "VX_CFG_NUM_CORES": 80, + "VX_CFG_L2_ENABLE": true, + "VX_CFG_L2_SIZE": 4194304, + "VX_CFG_L2_NUM_WAYS": 16, + "VX_CFG_L2_NUM_BANKS": 32 + }, + "platform": { + "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 32, + "VX_CFG_PLATFORM_CLOCK_RATE": 1530, + "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 900000, + "VX_CFG_PLATFORM_MEMORY_TYPE": "HBM2", + "VX_CFG_PLATFORM_MEMORY_CLOCK_RATE": 877 + }, + "sources": { + "NUM_CORES, NUM_WARPS, NUM_THREADS, ISSUE_WIDTH": "80 SMs, 64 resident warps/SM, 4 processing blocks with one scheduler each (NVIDIA Tesla V100 GPU Architecture whitepaper, Table 1)", + "DCACHE_SIZE": "128 KB combined L1/shared per SM (whitepaper)", + "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "6 MB L2 (whitepaper), rounded down to the power of two the cache accepts; 16 ways, 32 banks for the 32 memory channels", + "PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_CLOCK_RATE": "4 HBM2 stacks x 8 channels = 32 channels at 877 MHz: 32 x 32 B x 877 MHz = 898 GB/s (whitepaper)", + "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1530 MHz boost clock, 900 GB/s (whitepaper, Table 1)" + } +} From 21861d671c662f15b06c6669f6730cfde0167615 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 10 Oct 2026 23:52:29 -0700 Subject: [PATCH 021/124] CHANGELOG: profiles, RV64 portability, zero warnings, DRAM model Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d7cb17ba7..3e15dd5077 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,23 @@ All notable changes to Vortex are documented here. The format is based on follows the version pins recorded in [VERSION](VERSION) (`VORTEX_VERSION`, `TOOLCHAIN_REV`, `GEM5_REV`). +## [Unreleased] + +### Added + +- **Hardware profiles.** `profiles/.json` names a machine (`tinygpu`, `rtx1080`, `v100`, `a100`, `h100`, `rtx5080`) that every flow + builds through `configure --profile=` or `blackbox.sh --profile=`; explicit `CONFIGS` win over the profile. + See [docs/simulation.md](docs/simulation.md#hardware-profiles). + +### Changed + +- **RV64 portability.** Device index math, DXA/OM addresses, pack-load strides, RTU scene/payload pointers and the ray log carry full + width on RV64 (`size_t`/`uintptr_t`, identical RV32 code); RTU slot reads into FP registers are NaN-boxed. +- **Zero warnings restored.** Every C/C++ build (simulators, runtime, kernel library, tests, OpenCL/HIP/Vulkan apps) is `-Werror` and + Verilator warnings are fatal again, with the RTL and host code fixed to build clean. +- **DRAM model redesign.** simx, rtlsim, opaesim and xrtsim model one independent Ramulator channel per memory bank (one burst per + transaction, per-channel responses, refresh); `VX_CFG_PLATFORM_MEMORY_{TYPE,CLOCK_RATE,LATENCY,WINDOW}` select HBM2, HBM2_A100 or GDDR6. + ## [3.0] — 2026-10-07 The 3.0 release introduces a fixed-function graphics stack (rasterizer, texture units, and output mergers), tensor core structured sparsity (2:4), warpgroup-level matrix multiplication (WGMMA), global-to-local data transfer acceleration (DXA), a new hardware kernel scheduler (KMU) and Command Processor (CP) architecture, a new asynchronous runtime API (`vortex2.h`), asynchronous barriers with arrive/wait/event semantics, compressed instruction set (RVC) support, hardware atomics, an MMU/SV32 virtual memory stack, a Mesa/lavapipe Vulkan backend (`vortexpipe`), HIP via chipStar, gem5 integration, a SimX v3 TLM architecture with fixed-size handshake channels, productized Synopsys and Yosys ASIC synthesis flows, and a refreshed toolchain (LLVM 20, POCL 7.0). Build and configuration infrastructure was reworked: TOML-driven HW configuration ([VX_config.toml](VX_config.toml) + [VX_types.toml](VX_types.toml)) decoupling SimX/runtime from the RTL source tree, a `VX_CFG_` macro namespace that resolves toolchain preprocessor collisions, retirement of the global `toolchain_env.sh` to enable parallel multi-version Vortex worktrees on the same shell, consolidation of `kernel/`/`runtime/` under a shared `sw/` root, a single-source [VERSION](VERSION) file driving CI toolchain pinning, Perfetto trace export ([ci/perfetto.py](ci/perfetto.py)), and new top-level [AGENTS.md](AGENTS.md) + [CONTRIBUTING.md](CONTRIBUTING.md) for AI-agent and contributor workflows. From 49613f892751d9ec388f4be38718d88e1b273811 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 11 Oct 2026 01:36:30 -0700 Subject: [PATCH 022/124] ci: default GEM5_HOME to the configured tool directory regression.sh fell back to $HOME/tools/gem5 when GEM5_HOME was unset, but ci/gem5_install.sh installs gem5 under $TOOLDIR/gem5, so the gem5 cell failed in any build configured with another --tooldir ("gem5.opt not found"). CI sets GEM5_HOME explicitly and is unaffected. The fallback now follows TOOLDIR, defaulting to the configure-time value. Verified: the gem5 cell passes at XLEN 64 in a build configured with a non-default --tooldir. Co-Authored-By: Claude Opus 5.5 --- ci/regression.sh.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/regression.sh.in b/ci/regression.sh.in index 70165bfa08..774434c5d3 100755 --- a/ci/regression.sh.in +++ b/ci/regression.sh.in @@ -118,7 +118,7 @@ gem5() echo "begin gem5 tests..." if [ -z "$GEM5_HOME" ]; then - GEM5_HOME=$HOME/tools/gem5 + GEM5_HOME=${TOOLDIR:-@TOOLDIR@}/gem5 fi if [ ! -x "$GEM5_HOME/build/X86/gem5.opt" ]; then echo "error: $GEM5_HOME/build/X86/gem5.opt not found — run ci/gem5_install.sh first" From 1c6ef7efa7314dbb78a33ad906c46aa24b1ee7d8 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 11 Oct 2026 02:22:39 -0700 Subject: [PATCH 023/124] ci: re-record the rtlsim perf-gate baselines Regenerated with `pytest ci -m perf_gate --update-baselines` (XLEN 32, rtlsim) on this series, with the toolchain the CI will download (llvm-vortex a81a5957f406). All 49 entries move, because the per-channel DRAM model changes every rtlsim cycle count: - dxa/wgmma cases: 13-20% fewer cycles (independent channels, no shared injection queue); - most core, graphics and tensor cases: within a few percent; - vecadd: +21% instructions and +24%/+32% cycles (nt4/nt16) from the global index guard the kernel now needs on a rounded-up grid (one argument load and a branch in a kernel of ~two dozen instructions); - nine entries change instruction counts, from the toolchain and the test kernel changes in this series. Verified: the 49 perf_gate cases pass on rtlsim while recording. Co-Authored-By: Claude Opus 5.5 --- ci/baselines/perf/core.json | 72 +++++++++++++++---------------- ci/baselines/perf/dxa.json | 44 +++++++++---------- ci/baselines/perf/graphics.json | 46 ++++++++++---------- ci/baselines/perf/raytracing.json | 12 +++--- ci/baselines/perf/tensor.json | 20 ++++----- ci/baselines/perf/tensor_mx.json | 4 +- ci/baselines/perf/tensor_sp.json | 4 +- ci/baselines/perf/tensor_wg.json | 12 +++--- 8 files changed, 107 insertions(+), 107 deletions(-) diff --git a/ci/baselines/perf/core.json b/ci/baselines/perf/core.json index 5e226eba16..b08f98ad35 100644 --- a/ci/baselines/perf/core.json +++ b/ci/baselines/perf/core.json @@ -1,177 +1,177 @@ { "core:raycast-nt16:rtlsim": { "32": { - "cycles": 568490, + "cycles": 562137, "instrs": 137486 }, "app": "raycast", "args": "", - "config_hash": "ee3a227211fa9942", + "config_hash": "314d6d30965b89c4", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:raycast-nt4:rtlsim": { "32": { - "cycles": 1495440, + "cycles": 1495556, "instrs": 259228 }, "app": "raycast", "args": "", - "config_hash": "c8b680eabeb18655", + "config_hash": "909d80cfbaadadc2", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:sgemm-mc-nt16:rtlsim": { "32": { - "cycles": 781696, + "cycles": 779835, "instrs": 623680 }, "app": "sgemm", "args": "-n128", - "config_hash": "58eb3cc06964207a", + "config_hash": "92050636b69caeb2", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=16 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "core:sgemm-mc-nt4:rtlsim": { "32": { - "cycles": 2211340, + "cycles": 2228982, "instrs": 2494528 }, "app": "sgemm", "args": "-n128", - "config_hash": "aa6f26c7306417a8", + "config_hash": "72b6506beadbe8a0", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "core:sgemm-nt16:rtlsim": { "32": { - "cycles": 1823521, + "cycles": 1909825, "instrs": 623632 }, "app": "sgemm", "args": "-n128", - "config_hash": "b390d957eec8220f", + "config_hash": "514f489ab80d25c0", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:sgemm-nt4:rtlsim": { "32": { - "cycles": 6055079, + "cycles": 6189932, "instrs": 2494480 }, "app": "sgemm", "args": "-n128", - "config_hash": "13d03df76d55105e", + "config_hash": "4af79268b2c7557f", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:sgemmx-nt16:rtlsim": { "32": { - "cycles": 1297311, + "cycles": 1261549, "instrs": 266896 }, "app": "sgemmx", "args": "-n128", - "config_hash": "68638b6685801699", + "config_hash": "d53587fae488c32b", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:sgemmx-nt4:rtlsim": { "32": { - "cycles": 4210203, + "cycles": 4181334, "instrs": 1067536 }, "app": "sgemmx", "args": "-n128", - "config_hash": "837240be3049d62e", + "config_hash": "572494ba2df6f58a", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:sgemv-nt16:rtlsim": { "32": { - "cycles": 633593, - "instrs": 79120 + "cycles": 583960, + "instrs": 66800 }, "app": "sgemv", "args": "-m512 -n512", - "config_hash": "c6a3720118661bf7", + "config_hash": "587ac34faae9f341", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:sgemv-nt4:rtlsim": { "32": { - "cycles": 1060219, - "instrs": 316432 + "cycles": 931381, + "instrs": 267152 }, "app": "sgemv", "args": "-m512 -n512", - "config_hash": "db6bb5af90ecc1a5", + "config_hash": "93f8522bd055dda2", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:softmax-nt16:rtlsim": { "32": { - "cycles": 1274656, + "cycles": 1273285, "instrs": 464089 }, "app": "softmax", "args": "-n64", - "config_hash": "1c0e7742deb14093", + "config_hash": "9c8c364608242127", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:softmax-nt4:rtlsim": { "32": { - "cycles": 4860631, + "cycles": 4861515, "instrs": 1789797 }, "app": "softmax", "args": "-n64", - "config_hash": "3797eecae3200315", + "config_hash": "b2c24fbda6abd30b", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:stencil3d-nt16:rtlsim": { "32": { - "cycles": 2388738, + "cycles": 2395091, "instrs": 892944 }, "app": "stencil3d", "args": "-n32", - "config_hash": "aa768c2a9c65b092", + "config_hash": "fec6d70d6e8e83c3", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:stencil3d-nt4:rtlsim": { "32": { - "cycles": 4177542, + "cycles": 4283200, "instrs": 1785872 }, "app": "stencil3d", "args": "-n32", - "config_hash": "5400d882a6c96ac1", + "config_hash": "bb64a7573d2c2365", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:vecadd-nt16:rtlsim": { "32": { - "cycles": 75714, - "instrs": 24592 + "cycles": 99638, + "instrs": 29712 }, "app": "vecadd", "args": "-n16384", - "config_hash": "98b8bdc5e546ea62", + "config_hash": "58a51468cf8f99e7", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:vecadd-nt4:rtlsim": { "32": { - "cycles": 293078, - "instrs": 98320 + "cycles": 363323, + "instrs": 118800 }, "app": "vecadd", "args": "-n16384", - "config_hash": "1510b8438164b252", + "config_hash": "4ca729a11fbf88a3", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/dxa.json b/ci/baselines/perf/dxa.json index d864f76b96..453b0775f5 100644 --- a/ci/baselines/perf/dxa.json +++ b/ci/baselines/perf/dxa.json @@ -1,122 +1,122 @@ { "dxa:perf_gate-copy-mcast:rtlsim": { "32": { - "cycles": 3411, + "cycles": 3313, "instrs": 961 }, "app": "dxa_copy_mcast", "args": "", - "config_hash": "cbcd10be884ceab7", + "config_hash": "2df97b749a0febe3", "configs": "-DVX_CFG_EXT_DXA_ENABLE", "driver": "rtlsim" }, "dxa:perf_gate-copy-nt16:rtlsim": { "32": { - "cycles": 2096, + "cycles": 1869, "instrs": 592 }, "app": "dxa_copy", "args": "-d1", - "config_hash": "3a1ec1b81b5b3baf", + "config_hash": "59a6c715ed820634", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "dxa:perf_gate-copy-nt4:rtlsim": { "32": { - "cycles": 6844, + "cycles": 6773, "instrs": 1792 }, "app": "dxa_copy", "args": "-d1", - "config_hash": "3a888067fcc449d6", + "config_hash": "534d84e7f493e414", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-mcast-nt16:rtlsim": { "32": { - "cycles": 14333410, + "cycles": 14056427, "instrs": 4962320 }, "app": "sgemm2_dxa_mcast", "args": "-n128", - "config_hash": "14050e4dcef65266", + "config_hash": "589585790379eee0", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-mcast-nt4:rtlsim": { "32": { - "cycles": 14345910, + "cycles": 14108810, "instrs": 4962320 }, "app": "sgemm2_dxa_mcast", "args": "-n128", - "config_hash": "a55b900d67aa3657", + "config_hash": "2879ae8c33e35bb8", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-nt16:rtlsim": { "32": { - "cycles": 3978026, + "cycles": 3895374, "instrs": 1263632 }, "app": "sgemm2_dxa", "args": "-n128 -t4 -m1", - "config_hash": "36f5886fdc97e040", + "config_hash": "f3759c20ec885c91", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-nt4:rtlsim": { "32": { - "cycles": 14632454, + "cycles": 14130928, "instrs": 5017616 }, "app": "sgemm2_dxa", "args": "-n128 -t4 -m1", - "config_hash": "9ccf890d5ac45a62", + "config_hash": "aa3ad14268d9b8b5", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-dxa-mcast-nt16:rtlsim": { "32": { - "cycles": 98435, + "cycles": 79024, "instrs": 32384 }, "app": "sgemm_tcu_wg_dxa_mcast", "args": "-m 128 -n 128 -k 64", - "config_hash": "ee7678a7e7e9b79b", + "config_hash": "7080653c22396eb6", "configs": "-DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-dxa-mcast-nt4:rtlsim": { "32": { - "cycles": 400679, + "cycles": 348356, "instrs": 234560 }, "app": "sgemm_tcu_wg_dxa_mcast", "args": "-m 128 -n 128 -k 64", - "config_hash": "c79015e4caec7b93", + "config_hash": "401d267a03887e99", "configs": "-DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-dxa:rtlsim": { "32": { - "cycles": 39761, + "cycles": 34417, "instrs": 9160 }, "app": "sgemm_tcu_wg_dxa", "args": "-m 128 -n 128 -k 64", - "config_hash": "5a35710ef3aec3a6", + "config_hash": "3117dd957603b2e8", "configs": "-DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DWGMMA_NRC=16 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_THREADS=32", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-sp-dxa:rtlsim": { "32": { - "cycles": 68208, + "cycles": 55358, "instrs": 9328 }, "app": "sgemm_tcu_wg_sp_dxa", "args": "-m 128 -n 128 -k 128", - "config_hash": "d12fad3fd43e61c6", + "config_hash": "5bec3cc54a0c63f1", "configs": "-DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_THREADS=32", "driver": "rtlsim" } diff --git a/ci/baselines/perf/graphics.json b/ci/baselines/perf/graphics.json index 32c723239d..f4f9eb603d 100644 --- a/ci/baselines/perf/graphics.json +++ b/ci/baselines/perf/graphics.json @@ -1,100 +1,100 @@ { "graphics:perf_gate-draw3d-mc:rtlsim": { "32": { - "cycles": 531740, - "instrs": 204102 + "cycles": 529292, + "instrs": 203136 }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "c8c2af44277e2cb0", + "config_hash": "ae1062062a26ef92", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "graphics:perf_gate-draw3d-nt16:rtlsim": { "32": { - "cycles": 145662, - "instrs": 52516 + "cycles": 142905, + "instrs": 51650 }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "ee0410ff09e7ea58", + "config_hash": "e397892c0cb72ec2", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "graphics:perf_gate-draw3d-nt4:rtlsim": { "32": { - "cycles": 531359, - "instrs": 204102 + "cycles": 528252, + "instrs": 203136 }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "00c7939a6d958e0e", + "config_hash": "15664f7fbe13f120", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "graphics:perf_gate-om-nt16:rtlsim": { "32": { - "cycles": 191808, - "instrs": 69648 + "cycles": 188898, + "instrs": 68624 }, "app": "gfx_om", "args": "-w 128 -h 128 -r whitebox_128.png", - "config_hash": "60103e7b6eb2c625", + "config_hash": "b2ccda0f4e3d0471", "configs": "-DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "graphics:perf_gate-om-nt4:rtlsim": { "32": { - "cycles": 754454, - "instrs": 278544 + "cycles": 740590, + "instrs": 274448 }, "app": "gfx_om", "args": "-w 128 -h 128 -r whitebox_128.png", - "config_hash": "ee88907a281d364a", + "config_hash": "e542501f5bf6269e", "configs": "-DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "graphics:perf_gate-raster-nt16:rtlsim": { "32": { - "cycles": 14959, + "cycles": 14865, "instrs": 4708 }, "app": "gfx_raster", "args": "-w 128 -h 128 -t triangle.cgltrace -r triangle_ref_128.png", - "config_hash": "3ba867af954e5cbf", + "config_hash": "7272f5969f7ee30d", "configs": "-DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "graphics:perf_gate-raster-nt4:rtlsim": { "32": { - "cycles": 55633, + "cycles": 55526, "instrs": 18640 }, "app": "gfx_raster", "args": "-w 128 -h 128 -t triangle.cgltrace -r triangle_ref_128.png", - "config_hash": "859b3c4bd2208b92", + "config_hash": "05e09632b1bbf2b7", "configs": "-DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "graphics:perf_gate-tex-nt16:rtlsim": { "32": { - "cycles": 34994, + "cycles": 35342, "instrs": 12304 }, "app": "gfx_tex", "args": "-i toad.png -r toad_ref_f0.png -f 0 -g 0", - "config_hash": "1c38a250b6b85273", + "config_hash": "9669b4d06963f72d", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "graphics:perf_gate-tex-nt4:rtlsim": { "32": { - "cycles": 136738, + "cycles": 138329, "instrs": 49168 }, "app": "gfx_tex", "args": "-i toad.png -r toad_ref_f0.png -f 0 -g 0", - "config_hash": "226df03058dfb436", + "config_hash": "aee6b6136e6a4575", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/raytracing.json b/ci/baselines/perf/raytracing.json index 86b562e3b1..bda30910c0 100644 --- a/ci/baselines/perf/raytracing.json +++ b/ci/baselines/perf/raytracing.json @@ -1,34 +1,34 @@ { "raytracing:perf_gate-rt_raycast-mc:rtlsim": { "32": { - "cycles": 84873, + "cycles": 84378, "instrs": 39346 }, "app": "rt_raycast", "args": "", - "config_hash": "c71b720ea7aa7e72", + "config_hash": "937f99317c42a59f", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "raytracing:perf_gate-rt_raycast-nt16:rtlsim": { "32": { - "cycles": 53310, + "cycles": 53283, "instrs": 11372 }, "app": "rt_raycast", "args": "", - "config_hash": "3049a9d721044cde", + "config_hash": "dfd33a99196e04ad", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "raytracing:perf_gate-rt_raycast-nt4:rtlsim": { "32": { - "cycles": 159542, + "cycles": 159558, "instrs": 39330 }, "app": "rt_raycast", "args": "", - "config_hash": "14ae0ed831d1cd4a", + "config_hash": "c17be24af5bd78b9", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor.json b/ci/baselines/perf/tensor.json index 3249cc1b16..fae07189e9 100644 --- a/ci/baselines/perf/tensor.json +++ b/ci/baselines/perf/tensor.json @@ -1,56 +1,56 @@ { "tensor:perf_gate-fp16-isw4:rtlsim": { "32": { - "cycles": 195794, + "cycles": 191642, "instrs": 130592 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "6a095ed165469a30", + "config_hash": "d21f5479c2459776", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" }, "tensor:perf_gate-fp16-mc-nt16:rtlsim": { "32": { - "cycles": 44583, + "cycles": 42941, "instrs": 47296 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "0fe2a8e2a75fcc10", + "config_hash": "23f4b1de5a361780", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=16 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "tensor:perf_gate-fp16-mc-nt4:rtlsim": { "32": { - "cycles": 221331, + "cycles": 221484, "instrs": 336448 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "b2ef81f2fa1e6a43", + "config_hash": "c4461f9b2a2b71d7", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "tensor:perf_gate-fp16:rtlsim": { "32": { - "cycles": 251208, + "cycles": 253809, "instrs": 130576 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "5534ddd0051a5c15", + "config_hash": "9c6dfe0c35a740a9", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32", "driver": "rtlsim" }, "tensor:perf_gate-sgemm2-fp16:rtlsim": { "32": { - "cycles": 592658, + "cycles": 594027, "instrs": 409888 }, "app": "sgemm2_tcu", "args": "-m 128 -n 128 -k 64", - "config_hash": "421a4ab52f0f072a", + "config_hash": "9c8bbf01434d9d8d", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_mx.json b/ci/baselines/perf/tensor_mx.json index c9b137227e..1fbd356b34 100644 --- a/ci/baselines/perf/tensor_mx.json +++ b/ci/baselines/perf/tensor_mx.json @@ -1,12 +1,12 @@ { "tensor_mx:perf_gate-mxfp8:rtlsim": { "32": { - "cycles": 77127, + "cycles": 76601, "instrs": 34832 }, "app": "sgemm_tcu_mx", "args": "-m 128 -n 128 -k32", - "config_hash": "719fe3a7ce81115d", + "config_hash": "46d80398194a01e4", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DITYPE=mxfp8 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_sp.json b/ci/baselines/perf/tensor_sp.json index e8e15a7ab2..47e3a2926a 100644 --- a/ci/baselines/perf/tensor_sp.json +++ b/ci/baselines/perf/tensor_sp.json @@ -1,12 +1,12 @@ { "tensor_sp:perf_gate-sparse-fp16:rtlsim": { "32": { - "cycles": 187290, + "cycles": 184985, "instrs": 100880 }, "app": "sgemm_tcu_sp", "args": "-m 128 -n 128 -k 128", - "config_hash": "acf4fa4675243f0f", + "config_hash": "cfe93fc4d3413031", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_wg.json b/ci/baselines/perf/tensor_wg.json index 6ea2c1d004..57e02b2b8f 100644 --- a/ci/baselines/perf/tensor_wg.json +++ b/ci/baselines/perf/tensor_wg.json @@ -1,34 +1,34 @@ { "tensor_wg:perf_gate-wgmma-fedp2k-rs:rtlsim": { "32": { - "cycles": 157272, + "cycles": 158687, "instrs": 108512 }, "app": "sgemm_tcu_wg", "args": "-m64 -n64 -k64", - "config_hash": "7e8b9759dc36dba0", + "config_hash": "f019a0cb823990c9", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_TCU_FEDP2K -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DWGMMA_RS", "driver": "rtlsim" }, "tensor_wg:perf_gate-wgmma-fp16-ss:rtlsim": { "32": { - "cycles": 841657, + "cycles": 836986, "instrs": 525536 }, "app": "sgemm_tcu_wg", "args": "-m 128 -n 128 -k 128", - "config_hash": "c5a71c297fc4dafe", + "config_hash": "7b6f63cb95491231", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=32 -DWGMMA_SS -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" }, "tensor_wg:perf_gate-wgmma-sparse:rtlsim": { "32": { - "cycles": 401322, + "cycles": 398222, "instrs": 258976 }, "app": "sgemm_tcu_wg_sp", "args": "-m 128 -n 128 -k 128", - "config_hash": "ab1451598dfbc872", + "config_hash": "f620d8a9b0396d1e", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" } From 2970e9da33bfa90ba61a51f47c0b9be2f41c1df2 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 11 Oct 2026 02:40:46 -0700 Subject: [PATCH 024/124] tests: compute dogfood's fused multiply-add references fused The kernels write a*b +/- b, which clang contracts into fmadd/fmsub/fnmadd/ fnmsub (single rounding), but the host references computed the same expressions unfused (a*b rounded first). Near a = 1, a*b - b cancels, and the reference's rounding of a*b becomes hundreds of ulp of the small result, so fmsub failed whenever the configuration's element count put a sample near that point (4 cores x 8 warps x 8 threads with an L2: up to 512 ulp). The references now use fmaf, matching the instructions the kernels exercise; the kernels are unchanged. Verified: dogfood passes 22/22 on simx at that configuration (failing before) and at the default, and on rtlsim at that configuration; the kernel still emits fmadd.s x2, fmsub.s, fnmadd.s x2 and fnmsub.s. Co-Authored-By: Claude Opus 5.5 --- tests/regression/dogfood/testcases.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/regression/dogfood/testcases.h b/tests/regression/dogfood/testcases.h index f3562bb179..29e8e819c1 100644 --- a/tests/regression/dogfood/testcases.h +++ b/tests/regression/dogfood/testcases.h @@ -351,7 +351,7 @@ class Test_FMADD : public ITestCase { auto b = (float*)src2; auto c = (float*)dst; for (uint32_t i = 0; i < n; ++i) { - auto ref = a[i] * b[i] + b[i]; + auto ref = fmaf(a[i], b[i], b[i]); if (!almost_equal(c[i], ref)) { std::cout << "error at result #" << std::dec << i << std::hex << ": expected=" << ref << ", actual=" << c[i] << ", a=" << a[i] << ", b=" << b[i] << std::endl; ++errors; @@ -381,7 +381,7 @@ class Test_FMSUB : public ITestCase { auto b = (float*)src2; auto c = (float*)dst; for (uint32_t i = 0; i < n; ++i) { - auto ref = a[i] * b[i] - b[i]; + auto ref = fmaf(a[i], b[i], -b[i]); if (!almost_equal(c[i], ref)) { std::cout << "error at result #" << std::dec << i << std::hex << ": expected=" << ref << ", actual=" << c[i] << ", a=" << a[i] << ", b=" << b[i] << std::endl; ++errors; @@ -411,7 +411,7 @@ class Test_FNMADD : public ITestCase { auto b = (float*)src2; auto c = (float*)dst; for (uint32_t i = 0; i < n; ++i) { - auto ref = -a[i] * b[i] - b[i]; + auto ref = fmaf(-a[i], b[i], -b[i]); if (!almost_equal(c[i], ref)) { std::cout << "error at result #" << std::dec << i << std::hex << ": expected=" << ref << ", actual=" << c[i] << ", a=" << a[i] << ", b=" << b[i] << std::endl; ++errors; @@ -441,7 +441,7 @@ class Test_FNMSUB : public ITestCase { auto b = (float*)src2; auto c = (float*)dst; for (uint32_t i = 0; i < n; ++i) { - auto ref = -a[i] * b[i] + b[i]; + auto ref = fmaf(-a[i], b[i], b[i]); if (!almost_equal(c[i], ref)) { std::cout << "error at result #" << std::dec << i << std::hex << ": expected=" << ref << ", actual=" << c[i] << ", a=" << a[i] << ", b=" << b[i] << std::endl; ++errors; @@ -471,8 +471,8 @@ class Test_FNMADD_MADD : public ITestCase { auto b = (float*)src2; auto c = (float*)dst; for (uint32_t i = 0; i < n; ++i) { - auto x = -a[i] * b[i] - b[i]; - auto y = a[i] * b[i] + b[i]; + auto x = fmaf(-a[i], b[i], -b[i]); + auto y = fmaf(a[i], b[i], b[i]); auto ref = x + y; if (!almost_equal(c[i], ref)) { std::cout << "error at result #" << std::dec << i << std::hex << ": expected=" << ref << ", actual=" << c[i] << ", a=" << a[i] << ", b=" << b[i] << std::endl; From 0a46eef3b56aaf77a273ac922bd759f92994ed94 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 11 Oct 2026 10:46:39 -0700 Subject: [PATCH 025/124] vm: share one VPN superpage mask function from the TLB package VX_tlb_cam and VX_mmu each defined an identical vpn_mask function. In the multi-core VM configurations (2-4 cores with an L2/L3) Verilator reported VARHIDDEN for it, which the zero-warnings policy makes fatal, so the vm:tlb-concurrent, vm:mc-l2-amo and vm:mc-l3 rtlsim builds failed. The function now lives once in VX_tlb_pkg as tlb_vpn_mask, next to tlb_perm_ok, and both modules call it. Verified: the vm cell on rtlsim passes 23/23 at XLEN 32 and 64. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/vm/VX_mmu.sv | 5 +---- hw/rtl/vm/VX_tlb_cam.sv | 9 ++------- hw/rtl/vm/VX_tlb_pkg.sv | 5 +++++ 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/hw/rtl/vm/VX_mmu.sv b/hw/rtl/vm/VX_mmu.sv index 06b4ea7f48..9166421400 100644 --- a/hw/rtl/vm/VX_mmu.sv +++ b/hw/rtl/vm/VX_mmu.sv @@ -67,9 +67,6 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( localparam FIELDS_W = F_RW_LO + 1; localparam PAYLOAD_W = LANE_W + FIELDS_W; - function automatic [TLB_VPN_WIDTH-1:0] vpn_mask (input [TLB_LEVEL_WIDTH-1:0] level); - vpn_mask = {TLB_VPN_WIDTH{1'b1}} << (level * TLB_LEVEL_BITS); - endfunction function automatic tlb_access_e access_of (input logic rw); access_of = (EXEC_SIDE != 0) ? TLB_ACC_EX : (rw ? TLB_ACC_WR : TLB_ACC_RD); @@ -255,7 +252,7 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( if (ADDR_WIDTH > PAGE_OFFSET_W + TLB_VPN_WIDTH) begin : g_replay_va_hi `UNUSED_VAR (replay_raddr[ADDR_WIDTH-1 : PAGE_OFFSET_W + TLB_VPN_WIDTH]) end - wire [TLB_VPN_WIDTH-1:0] replay_low = replay_vpn & ~vpn_mask(replay_level); + wire [TLB_VPN_WIDTH-1:0] replay_low = replay_vpn & ~tlb_vpn_mask(replay_level); wire [TLB_PPN_WIDTH-1:0] replay_ppn_sp = replay_ppn | TLB_PPN_WIDTH'(replay_low); wire [ADDR_WIDTH-1:0] replay_taddr = {replay_ppn_sp, replay_raddr[PAGE_OFFSET_W-1:0]}; diff --git a/hw/rtl/vm/VX_tlb_cam.sv b/hw/rtl/vm/VX_tlb_cam.sv index ec63559c33..950d974626 100644 --- a/hw/rtl/vm/VX_tlb_cam.sv +++ b/hw/rtl/vm/VX_tlb_cam.sv @@ -56,11 +56,6 @@ module VX_tlb_cam import VX_tlb_pkg::*; #( reg [TLB_SIZE-1:0] valid_r; reg [TLB_SIZE-1:0] mru_r; - // Mask that drops the low level*TLB_LEVEL_BITS VPN bits (superpages). - function automatic [TLB_VPN_WIDTH-1:0] vpn_mask (input [TLB_LEVEL_WIDTH-1:0] level); - vpn_mask = {TLB_VPN_WIDTH{1'b1}} << (level * TLB_LEVEL_BITS); - endfunction - // --------------------------------------------------------------------- // Per-lane lookup // --------------------------------------------------------------------- @@ -77,7 +72,7 @@ module VX_tlb_cam import VX_tlb_pkg::*; #( for (genvar l = 0; l < NUM_REQS; ++l) begin : g_lookup wire [TLB_SIZE-1:0] hit_vec; for (genvar i = 0; i < TLB_SIZE; ++i) begin : g_match - wire [TLB_VPN_WIDTH-1:0] mask = vpn_mask(entries_r[i].level); + wire [TLB_VPN_WIDTH-1:0] mask = tlb_vpn_mask(entries_r[i].level); assign hit_vec[i] = valid_r[i] && ((entries_r[i].vpn & mask) == (lookup_vpn[l] & mask)); end @@ -115,7 +110,7 @@ module VX_tlb_cam import VX_tlb_pkg::*; #( wire [TLB_FLAGS_WIDTH-1:0] sel_flags = sel[TLB_LEVEL_WIDTH +: TLB_FLAGS_WIDTH]; wire [TLB_LEVEL_WIDTH-1:0] sel_level = sel[0 +: TLB_LEVEL_WIDTH]; - wire [TLB_VPN_WIDTH-1:0] low = lookup_vpn[l] & ~vpn_mask(sel_level); + wire [TLB_VPN_WIDTH-1:0] low = lookup_vpn[l] & ~tlb_vpn_mask(sel_level); // Splice the superpage's intra-page index from the VPN into the PPN. assign lookup_ppn[l] = sel_ppn | TLB_PPN_WIDTH'(low); assign lookup_flags[l] = sel_flags; diff --git a/hw/rtl/vm/VX_tlb_pkg.sv b/hw/rtl/vm/VX_tlb_pkg.sv index 8392a7ffbe..bc9aa17da4 100644 --- a/hw/rtl/vm/VX_tlb_pkg.sv +++ b/hw/rtl/vm/VX_tlb_pkg.sv @@ -72,6 +72,11 @@ package VX_tlb_pkg; && ( want_w || want_x || flags[TLB_FLAG_R]); endfunction + // Mask that drops the low level*TLB_LEVEL_BITS VPN bits (superpages). + function automatic logic [TLB_VPN_WIDTH-1:0] tlb_vpn_mask (input logic [TLB_LEVEL_WIDTH-1:0] level); + tlb_vpn_mask = {TLB_VPN_WIDTH{1'b1}} << (level * TLB_LEVEL_BITS); + endfunction + endpackage `endif // VX_TLB_PKG_VH From f80af4c365a3d3c13398c62d60fccf4a6fed0c5e Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 12 Oct 2026 02:24:42 -0700 Subject: [PATCH 026/124] sim: configure the DRAM model from data-sheet values The DRAM keys added with the model redesign mixed device settings with wrapper internals: a DRAM clock, a request window, an A100-specific type, per-type timing tables in DRAM cycles, a forced 256-bit channel and custom address maps. Studying another GPU meant re-deriving all of them. A channel is now described the way a data sheet describes it: VX_CFG_PLATFORM_MEMORY_TYPE (HBM2, HBM3, GDDR6), _DATA_RATE (MT/s per pin, default twice the core clock), _NUM_BANKS (channels), _LATENCY (fixed PHY/controller/interconnect cycles) and _QUEUE_SIZE (requests in flight). - One Ramulator memory system per channel: independent controllers, any channel count. - JEDEC organization with 32 B bursts (HBM: BL4 on 64-bit pseudo-channels; GDDR6: x16), array timings in nanoseconds converted by Ramulator at the data rate, bus timings in clocks. HBM array timing is JEDEC HBM2 as GPGPU-Sim's QV100 configuration lists it (tRCD/tCL/tRP 14, tRAS 33, tRC 47 ns); GDDR6 is Ramulator's own x16 preset (the GPGPU-Sim RTX 3070 values used before were half of real GDDR6). - Address map: four bursts per row visit, then the next pseudo-channel, bank group and bank, with the bank permuted by the folded row, so streams a power of two apart do not share banks; Ramulator's MOP4CLXOR mapper XORs with column bits instead and sent such streams to the same banks. - No refresh: GPU controllers hide refresh with per-bank refresh, for which Ramulator has no manager (its HBM REFsb also lacks the tRFCsb busy time); all-bank refresh cost every channel ~9% of its time and added 8.5 cycles to sgemm's mean memory latency. ChannelRefresh is removed; ramulator_ext only builds GDDR6.cpp. The default machine clocks the DRAM with the core: 14/26 cycles idle (open row / row conflict; master 13/27), one 64 B block per cycle at peak. SimX sgemm -n128 mean memory latency at NT=NW=4/8/16: 17.9/18.4/18.5 cycles (master 16.4/17.6/19.0). Profiles: v100 HBM2 1752 MT/s, a100 HBM2 3038 and h100 HBM3 6545 (32 channels carrying the cards' 40-channel bandwidth), rtx1080 GDDR6 10000, rtx5080 GDDR6 15000 with LATENCY 339 (407 less the 68 cycles JEDEC GDDR6 adds to a row conflict; derived, not re-measured). Streams reach 93-98% of peak. The dram_sim unit test checks idle latency, channel scaling, peak efficiency, strided streams and the queue bound. Co-Authored-By: Claude Opus 5.5 --- .github/actions/setup-vortex/action.yml | 2 +- CHANGELOG.md | 6 +- VX_config.toml | 15 +- docs/codebase.md | 4 +- docs/simulation.md | 38 +- profiles/a100.json | 10 +- profiles/h100.json | 6 +- profiles/rtx1080.json | 4 +- profiles/rtx5080.json | 8 +- profiles/v100.json | 4 +- sim/common/dram_sim.cpp | 448 +++++++++--------- sim/common/dram_sim.h | 14 +- tests/unittest/dram_sim/main.cpp | 154 +++--- third_party/Makefile | 9 +- third_party/ramulator_ext/channel_refresh.cpp | 65 --- 15 files changed, 367 insertions(+), 420 deletions(-) delete mode 100644 third_party/ramulator_ext/channel_refresh.cpp diff --git a/.github/actions/setup-vortex/action.yml b/.github/actions/setup-vortex/action.yml index a73448a887..b27d3c1305 100644 --- a/.github/actions/setup-vortex/action.yml +++ b/.github/actions/setup-vortex/action.yml @@ -49,7 +49,7 @@ runs: uses: actions/cache@v4 with: path: third_party - key: ${{ runner.os }}-thirdparty-${{ hashFiles('third_party/Makefile', 'third_party/ramulator_ext/**', '.gitmodules') }} + key: ${{ runner.os }}-thirdparty-${{ hashFiles('third_party/Makefile', '.gitmodules') }} # Publish WHERE the toolchain cache above was restored to. The `tools` cache # path is relative to the workspace, but nothing downstream could know that: diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e15dd5077..b384f6a82b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,8 +19,10 @@ follows the version pins recorded in [VERSION](VERSION) (`VORTEX_VERSION`, width on RV64 (`size_t`/`uintptr_t`, identical RV32 code); RTU slot reads into FP registers are NaN-boxed. - **Zero warnings restored.** Every C/C++ build (simulators, runtime, kernel library, tests, OpenCL/HIP/Vulkan apps) is `-Werror` and Verilator warnings are fatal again, with the RTL and host code fixed to build clean. -- **DRAM model redesign.** simx, rtlsim, opaesim and xrtsim model one independent Ramulator channel per memory bank (one burst per - transaction, per-channel responses, refresh); `VX_CFG_PLATFORM_MEMORY_{TYPE,CLOCK_RATE,LATENCY,WINDOW}` select HBM2, HBM2_A100 or GDDR6. +- **DRAM model redesign.** simx, rtlsim, opaesim and xrtsim model one independent Ramulator controller and device per memory bank + (JEDEC HBM2/HBM3/GDDR6 organization and nanosecond timing, 32 B bursts, permuted open-page bank interleave, per-channel + responses), configured from data-sheet values: `VX_CFG_PLATFORM_MEMORY_{TYPE,DATA_RATE,LATENCY,QUEUE_SIZE}`. The default machine + keeps its idle read latency (14/26 cycles open-row/conflict, was 13/27) and one-block-per-cycle peak. ## [3.0] — 2026-10-07 diff --git a/VX_config.toml b/VX_config.toml index 8d331ff67b..f51186cdec 100644 --- a/VX_config.toml +++ b/VX_config.toml @@ -80,13 +80,16 @@ VX_CFG_PLATFORM_MEMORY_INTERLEAVE = 1 VX_CFG_PLATFORM_MEMORY_PEAK_BW = 460000 # 460 GB/s VX_CFG_PLATFORM_CLOCK_RATE = 400 # DRAM the simulators model behind each memory bank (one channel per bank), -# its clock (MHz), the fixed read latency added for the PHY, controller and -# interconnect (core cycles), and the requests a channel holds in flight -# (0 = unbounded). +# as a data sheet gives it: the JEDEC standard, its data rate (MT/s per pin), +# the fixed read latency added for the PHY, controller and interconnect (core +# cycles), and the requests a channel holds in flight (0 = unbounded). +# Simulation-only: the RTL never reads them. The default clocks the DRAM with +# the core: 14 cycles for an idle open-row read, 26 for a row conflict, one +# 64 B block per cycle at peak; profiles set their own. VX_CFG_PLATFORM_MEMORY_TYPE = "HBM2" -VX_CFG_PLATFORM_MEMORY_CLOCK_RATE = 1000 +VX_CFG_PLATFORM_MEMORY_DATA_RATE = "expr: 2 * $VX_CFG_PLATFORM_CLOCK_RATE" VX_CFG_PLATFORM_MEMORY_LATENCY = 0 -VX_CFG_PLATFORM_MEMORY_WINDOW = 0 +VX_CFG_PLATFORM_MEMORY_QUEUE_SIZE = 0 [alu] VX_CFG_NUM_ALU_LANES = "expr: $VX_CFG_SIMD_WIDTH" @@ -464,7 +467,7 @@ VX_CFG_XLEN = [32, 64] VX_CFG_FLEN = [32, 64] VX_CFG_FPU_TYPE = ["DPI", "DSP", "FPNEW", "STD"] VX_CFG_TCU_TYPE = ["DPI", "DSP", "BHF", "TFR", "FPNEW"] -VX_CFG_PLATFORM_MEMORY_TYPE = ["HBM2", "HBM2_A100", "GDDR6"] +VX_CFG_PLATFORM_MEMORY_TYPE = ["HBM2", "HBM3", "GDDR6"] [[param]] VX_CFG_DCACHE_NUM_REQS = "int" diff --git a/docs/codebase.md b/docs/codebase.md index 52d8951279..9e68135e0f 100644 --- a/docs/codebase.md +++ b/docs/codebase.md @@ -76,7 +76,7 @@ The directory/file layout of the Vortex codebase is as follows: - `runtime`: driver API tests - `mpi`: multi-process tests - `third_party`: external library submodules (cvfpu, softfloat, hardfloat, - ramulator, cocogfx), plus `ramulator_ext` (Vortex's Ramulator extensions: - per-channel refresh and the GDDR6 device) + ramulator, cocogfx); `ramulator_ext` holds the build of the GDDR6 device + model Ramulator ships but does not build - `perf`: performance analysis resources - `miscs`: miscellaneous resources diff --git a/docs/simulation.md b/docs/simulation.md index af69ae9a27..8fbc55f94d 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -151,9 +151,9 @@ machine from it: SimX, rtlsim, the FPGA and ASIC flows, and the kernels. |---|---| | `tinygpu` | 1 core, 2 warps x 2 threads, shared memory | | `rtx1080` | GeForce GTX 1080 (Pascal): 20 cores x 64 warps x 32 threads, 2 MB L2, GDDR5X as 16 GDDR6 channels | -| `v100` | Tesla V100 (Volta): 80 cores, 4 MB L2, 32 HBM2 channels at 877 MHz | -| `a100` | A100 (Ampere): 108 cores, 32 MB L2, GPGPU-Sim's A100 HBM timing | -| `h100` | H100 SXM5 (Hopper): 132 cores, 32 MB L2, HBM3 bandwidth on the HBM2 model | +| `v100` | Tesla V100 (Volta): 80 cores, 4 MB L2, 32 HBM2 channels at 1752 MT/s | +| `a100` | A100 (Ampere): 108 cores, 32 MB L2, HBM2 at the card's 1555 GB/s | +| `h100` | H100 SXM5 (Hopper): 132 cores, 32 MB L2, HBM3 at the card's 3.35 TB/s | | `rtx5080` | GeForce RTX 5080 (Blackwell): 84 cores x 48 warps, 64 MB L2, GDDR7 bandwidth on the GDDR6 model | Select one per build with `../configure --profile=`, or per run: @@ -177,21 +177,35 @@ bring-up and fast simulation. ### Configuring the DRAM Model The simulators (simx, rtlsim, opaesim, xrtsim) all model the device memory -the same way: one independent DRAM channel per platform memory bank, timed by -[Ramulator](https://github.com/CMU-SAFARI/ramulator2), with its refresh. These -`VX_config.toml` keys select it, overridable through `CONFIGS` or a profile: +the same way: one independent DRAM channel (controller and device) per +platform memory bank, timed by +[Ramulator](https://github.com/CMU-SAFARI/ramulator2) with its JEDEC device +models. Each channel maps addresses the way GPU memory controllers do: four +bursts per row visit, then the next bank, with the bank permuted by the row so +strided streams spread over the banks. Refresh is not modelled: GPU +controllers hide it with per-bank refresh, for which Ramulator has no manager. A channel is described the way a data sheet describes it; these +`VX_config.toml` keys select it, overridable through `CONFIGS` or a profile, +and the RTL never reads them: | Key | Default | Meaning | |---|---|---| -| `VX_CFG_PLATFORM_MEMORY_TYPE` | `HBM2` | `HBM2` (JEDEC 2 Gbps), `HBM2_A100` (GPGPU-Sim's A100 timing) or `GDDR6` (x16 channels, 32 B bursts) | -| `VX_CFG_PLATFORM_MEMORY_NUM_BANKS` | 2 | memory banks, and so DRAM channels | -| `VX_CFG_PLATFORM_MEMORY_CLOCK_RATE` | 1000 | DRAM clock in MHz (GDDR6: the data clock, half the transfer rate) | -| `VX_CFG_PLATFORM_CLOCK_RATE` | 400 | core clock in MHz; with the DRAM clock it sets how many DRAM cycles a core cycle spans | +| `VX_CFG_PLATFORM_MEMORY_TYPE` | `HBM2` | JEDEC standard: `HBM2` (also HBM2E), `HBM3` or `GDDR6` (x16 channels) | +| `VX_CFG_PLATFORM_MEMORY_NUM_BANKS` | 2 | memory banks, and so DRAM channels (HBM: 128-bit channels) | +| `VX_CFG_PLATFORM_MEMORY_DATA_RATE` | 2 x `PLATFORM_CLOCK_RATE` | transfer rate per pin in MT/s; the default clocks the DRAM with the core | +| `VX_CFG_PLATFORM_CLOCK_RATE` | 400 | core clock in MHz; with the data rate it sets how many DRAM cycles a core cycle spans | | `VX_CFG_PLATFORM_MEMORY_LATENCY` | 0 | fixed read latency added for the PHY, controller and interconnect, in core cycles | -| `VX_CFG_PLATFORM_MEMORY_WINDOW` | 0 | requests a channel holds in flight (0 = unbounded) | +| `VX_CFG_PLATFORM_MEMORY_QUEUE_SIZE` | 0 | requests a channel holds in flight (0 = unbounded) | + +The model's peak bandwidth follows from the keys: channels x channel width x +data rate (HBM 16 B, GDDR6 2 B per transfer); `VX_CFG_PLATFORM_MEMORY_PEAK_BW` +separately states the platform's, which the runtime reports. The device's +array timings are held in nanoseconds, so a different data rate changes the +bandwidth but not the latency. The default machine clocks the DRAM with the +core: an idle read takes 14 cycles on an open row and 26 on a row conflict, +and the memory delivers up to one 64 B block per cycle. ```bash -$ CONFIGS="-DVX_CFG_PLATFORM_MEMORY_TYPE=GDDR6 -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=16 -DVX_CFG_PLATFORM_MEMORY_CLOCK_RATE=8000" \ +$ CONFIGS="-DVX_CFG_PLATFORM_MEMORY_TYPE=GDDR6 -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=16 -DVX_CFG_PLATFORM_MEMORY_DATA_RATE=14000" \ ./ci/blackbox.sh --driver=simx --app=vecadd ``` diff --git a/profiles/a100.json b/profiles/a100.json index 1449b1fbf1..7a6003fe0b 100644 --- a/profiles/a100.json +++ b/profiles/a100.json @@ -19,17 +19,17 @@ "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 32, "VX_CFG_PLATFORM_CLOCK_RATE": 1410, "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 1555000, - "VX_CFG_PLATFORM_MEMORY_TYPE": "HBM2_A100", - "VX_CFG_PLATFORM_MEMORY_CLOCK_RATE": 1512, + "VX_CFG_PLATFORM_MEMORY_TYPE": "HBM2", + "VX_CFG_PLATFORM_MEMORY_DATA_RATE": 3038, "VX_CFG_PLATFORM_MEMORY_LATENCY": 190, - "VX_CFG_PLATFORM_MEMORY_WINDOW": 257 + "VX_CFG_PLATFORM_MEMORY_QUEUE_SIZE": 257 }, "sources": { "NUM_CORES, NUM_WARPS, NUM_THREADS, ISSUE_WIDTH": "108 SMs, 64 resident warps/SM, 4 schedulers x 32 lanes (NVIDIA A100 Tensor Core GPU Architecture whitepaper, Table 3)", "DCACHE_SIZE": "192 KB combined L1/shared per SM (whitepaper), rounded down to the power of two the cache accepts", "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "40 MB L2 in 80 slices (whitepaper), rounded down to 32 MB and 32 banks; 16 ways as in GPGPU-Sim's SM80_A100 dl2", - "PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_CLOCK_RATE": "HBM2 at GPGPU-Sim SM80_A100's 1512 MHz DRAM clock; 32 channels x 32 B per DRAM cycle = 1548 GB/s, the card's 1555 GB/s", - "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_LATENCY, PLATFORM_MEMORY_WINDOW": "HBM timing, fixed DRAM latency and per-channel request window of GPGPU-Sim's SM80_A100 configuration (accel-sim gpu-simulator/configs/tested-cfgs/SM80_A100/gpgpusim.config)", + "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_DATA_RATE": "5 HBM2 stacks x 8 128-bit channels = 40 channels at 2430 MT/s, modelled as 32 channels at 3038 MT/s for a power-of-two interleave: 32 x 16 B x 3038 MT/s = 1555 GB/s, the card's bandwidth; the array timing is in nanoseconds, so latency is unchanged", + "PLATFORM_MEMORY_LATENCY, PLATFORM_MEMORY_QUEUE_SIZE": "Fixed DRAM latency and per-channel request queue of GPGPU-Sim's SM80_A100 configuration (accel-sim gpu-simulator/configs/tested-cfgs/SM80_A100/gpgpusim.config)", "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1410 MHz boost clock, 1555 GB/s (whitepaper, Table 3)" } } diff --git a/profiles/h100.json b/profiles/h100.json index ce1a216401..d95d2fabda 100644 --- a/profiles/h100.json +++ b/profiles/h100.json @@ -19,14 +19,14 @@ "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 32, "VX_CFG_PLATFORM_CLOCK_RATE": 1980, "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 3350000, - "VX_CFG_PLATFORM_MEMORY_TYPE": "HBM2", - "VX_CFG_PLATFORM_MEMORY_CLOCK_RATE": 3272 + "VX_CFG_PLATFORM_MEMORY_TYPE": "HBM3", + "VX_CFG_PLATFORM_MEMORY_DATA_RATE": 6545 }, "sources": { "NUM_CORES, NUM_WARPS, NUM_THREADS, ISSUE_WIDTH": "132 SMs (SXM5), 64 resident warps/SM, 4 schedulers x 32 lanes (NVIDIA H100 Tensor Core GPU Architecture whitepaper)", "DCACHE_SIZE": "256 KB combined L1/shared per SM (whitepaper)", "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "50 MB L2 (whitepaper), rounded down to the power of two the cache accepts; 16 ways and 32 banks as in the a100 profile", - "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_CLOCK_RATE": "HBM3 modelled with the HBM2 timing: 32 channels x 32 B per DRAM cycle at 3272 MHz = 3.35 TB/s, the card's bandwidth; latency not calibrated", + "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_DATA_RATE": "5 HBM3 stacks x 16 64-bit channels at 5236 MT/s = 40 128-bit channel pairs, modelled as 32 at 6545 MT/s for a power-of-two interleave: 32 x 16 B x 6545 MT/s = 3.35 TB/s, the card's bandwidth; latency not calibrated", "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1980 MHz boost clock, 3.35 TB/s (H100 SXM5 datasheet)" } } diff --git a/profiles/rtx1080.json b/profiles/rtx1080.json index 7728591970..73908bb3d2 100644 --- a/profiles/rtx1080.json +++ b/profiles/rtx1080.json @@ -20,13 +20,13 @@ "VX_CFG_PLATFORM_CLOCK_RATE": 1607, "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 320000, "VX_CFG_PLATFORM_MEMORY_TYPE": "GDDR6", - "VX_CFG_PLATFORM_MEMORY_CLOCK_RATE": 5000 + "VX_CFG_PLATFORM_MEMORY_DATA_RATE": 10000 }, "sources": { "NUM_CORES, NUM_WARPS, NUM_THREADS, ISSUE_WIDTH": "20 SMs, 64 resident warps/SM, 4 schedulers x 32 lanes (NVIDIA GeForce GTX 1080 whitepaper)", "DCACHE_SIZE": "48 KB unified L1/texture per SM (whitepaper), rounded down to the power of two the cache accepts", "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "2 MB L2 (whitepaper); 16 ways, 16 banks for the 16 memory channels", - "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_CLOCK_RATE": "256-bit GDDR5X at 10 Gbps modelled as 16 GDDR6 x16 channels at a 5000 MHz data clock: 16 x 2 B x 10 GT/s = 320 GB/s, the card's bandwidth; latency not calibrated", + "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_DATA_RATE": "256-bit GDDR5X at 10 Gbps modelled as 16 GDDR6 x16 channels at 10000 MT/s: 16 x 2 B x 10 GT/s = 320 GB/s, the card's bandwidth; latency not calibrated", "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1607 MHz base clock, 320 GB/s (whitepaper)" } } diff --git a/profiles/rtx5080.json b/profiles/rtx5080.json index 6aa20dcd91..d90b047d99 100644 --- a/profiles/rtx5080.json +++ b/profiles/rtx5080.json @@ -22,8 +22,8 @@ "VX_CFG_PLATFORM_CLOCK_RATE": 2295, "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 960000, "VX_CFG_PLATFORM_MEMORY_TYPE": "GDDR6", - "VX_CFG_PLATFORM_MEMORY_CLOCK_RATE": 7500, - "VX_CFG_PLATFORM_MEMORY_LATENCY": 407 + "VX_CFG_PLATFORM_MEMORY_DATA_RATE": 15000, + "VX_CFG_PLATFORM_MEMORY_LATENCY": 339 }, "reference": { "clock_mhz": 2295, @@ -57,7 +57,7 @@ "PLATFORM_MEMORY_NUM_BANKS": "8 x 32-bit GDDR7 devices = 32 independent channels (whitepaper App. B)", "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "2295 MHz base clock, 960 GB/s (whitepaper App. B)", "reference": "Nsight Compute on an RTX 5080 at its 2295 MHz base clock with a cold L2, 2026-10-05", - "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_CLOCK_RATE": "GDDR7 modelled as GDDR6 x16 channels: 32 channels x 2 B x 15 GT/s (7500 MHz data clock) = 960 GB/s, the card's bandwidth", - "PLATFORM_MEMORY_LATENCY": "407 core cycles of PHY/NoC latency, calibrated in SimX against the 802-cycle pointer-chase DRAM latency in 'reference' (recalibrate when the memory model changes)" + "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_DATA_RATE": "30 Gbps GDDR7 x8 channels modelled as GDDR6 x16 channels at 15000 MT/s: the same channel count, 32 B burst time and 960 GB/s (32 x 2 B x 15 GT/s)", + "PLATFORM_MEMORY_LATENCY": "339 core cycles of PHY/NoC latency: the 407 calibrated in SimX against the 802-cycle pointer-chase DRAM latency in 'reference', less the 68 cycles JEDEC GDDR6 timing adds to the device's row-conflict latency (derived, not re-run; recalibrate with a pointer chase when the memory model changes)" } } diff --git a/profiles/v100.json b/profiles/v100.json index c58be16476..451551d910 100644 --- a/profiles/v100.json +++ b/profiles/v100.json @@ -20,13 +20,13 @@ "VX_CFG_PLATFORM_CLOCK_RATE": 1530, "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 900000, "VX_CFG_PLATFORM_MEMORY_TYPE": "HBM2", - "VX_CFG_PLATFORM_MEMORY_CLOCK_RATE": 877 + "VX_CFG_PLATFORM_MEMORY_DATA_RATE": 1752 }, "sources": { "NUM_CORES, NUM_WARPS, NUM_THREADS, ISSUE_WIDTH": "80 SMs, 64 resident warps/SM, 4 processing blocks with one scheduler each (NVIDIA Tesla V100 GPU Architecture whitepaper, Table 1)", "DCACHE_SIZE": "128 KB combined L1/shared per SM (whitepaper)", "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "6 MB L2 (whitepaper), rounded down to the power of two the cache accepts; 16 ways, 32 banks for the 32 memory channels", - "PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_CLOCK_RATE": "4 HBM2 stacks x 8 channels = 32 channels at 877 MHz: 32 x 32 B x 877 MHz = 898 GB/s (whitepaper)", + "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_DATA_RATE": "4 HBM2 stacks x 8 128-bit channels = 32 channels at 1752 MT/s: 32 x 16 B x 1752 MT/s = 897 GB/s (whitepaper)", "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1530 MHz boost clock, 900 GB/s (whitepaper, Table 1)" } } diff --git a/sim/common/dram_sim.cpp b/sim/common/dram_sim.cpp index e6f7f1a0c5..bc899b89fc 100644 --- a/sim/common/dram_sim.cpp +++ b/sim/common/dram_sim.cpp @@ -36,17 +36,64 @@ using namespace vortex; namespace { -// IPOLY(5) bank hash over 16 banks (Rau, "Pseudo-randomly interleaved memory", -// ISCA 1991), GPGPU-Sim's -dram_bnk_indexing_policy 2: the bank XORed with a -// function of the row, so streams whose bases differ by a power of two land in -// different banks instead of closing each other's rows. -uint32_t ipoly16(uint64_t row, uint32_t bank) { - auto bit = [row](int i) { return uint32_t(row >> i) & 1u; }; - uint32_t i0 = bit(11) ^ bit(10) ^ bit(9) ^ bit(8) ^ bit(6) ^ bit(4) ^ bit(3) ^ bit(0); - uint32_t i1 = bit(12) ^ bit(8) ^ bit(7) ^ bit(6) ^ bit(5) ^ bit(3) ^ bit(1) ^ bit(0); - uint32_t i2 = bit(9) ^ bit(8) ^ bit(7) ^ bit(6) ^ bit(4) ^ bit(2) ^ bit(1); - uint32_t i3 = bit(10) ^ bit(9) ^ bit(8) ^ bit(7) ^ bit(5) ^ bit(3) ^ bit(2); - return bank ^ (i0 | (i1 << 1) | (i2 << 2) | (i3 << 3)); +// Bus timings are DRAM clocks, fixed by the burst; array timings are +// nanoseconds, which Ramulator converts at the configured data rate. +struct clock_timing_t { + const char* name; + int clocks; +}; + +struct ns_timing_t { + const char* name; + float ns; +}; + +// A 32 B burst is BL4 on a 64-bit pseudo-channel: two clocks, and the same +// spacing between column commands to another bank group (four within one). +// nRTW only completes the set Ramulator requires. +const clock_timing_t hbm_bus_timing[] = { + {"nBL", 2}, {"nCCDS", 2}, {"nCCDL", 4}, {"nRTW", 3}, +}; + +// JEDEC HBM2 array timing as GPGPU-Sim's SM7_QV100 configuration lists it (its +// JEDEC line, at 1 GHz: RCD=RP=CL=14, RAS=33, RC=47, WR=12, RRD=4, RTPL=4); +// the rest from Ramulator's HBM2_2Gbps preset, whose 1 ns clock makes its +// cycles nanoseconds. The array does not speed up with the interface: HBM2E +// and HBM3 parts keep these timings (GPGPU-Sim's SM80_A100 and SM90_H100 HBM +// lines are the same 14/33/47 ns at their own clocks). Refresh timings complete +// the set Ramulator requires. +const ns_timing_t hbm_array_timing[] = { + {"tCL", 14}, {"tRCDRD", 14}, {"tRCDWR", 14}, {"tRP", 14}, {"tRAS", 33}, {"tRC", 47}, + {"tCWL", 2}, {"tWR", 12}, {"tRTPS", 2}, {"tRTPL", 4}, {"tRRDS", 4}, {"tRRDL", 4}, + {"tWTRS", 3}, {"tWTRL", 4}, {"tFAW", 15}, {"tRFCSB", 160}, {"tREFI", 3900}, {"tRREFD", 8}, +}; + +// Ramulator's GDDR6 clock is half the data rate: a 32 B burst, 16 beats on a +// x16 channel, is eight clocks, and column commands are two (three within a +// bank group) command clocks apart, a command clock being eight data beats. +const clock_timing_t gddr6_bus_timing[] = { + {"nBL", 8}, {"nCCDS", 8}, {"nCCDL", 12}, +}; + +// Ramulator's x16 GDDR6 timing at 2000 (its GDDR6_2000_1350mV_double preset +// and secondary tables) at its 570 ps command clock, in nanoseconds; refresh +// timings complete the set Ramulator requires. +const ns_timing_t gddr6_array_timing[] = { + {"tCL", 13.7f}, {"tRCDRD", 14.8f}, {"tRCDWD", 9.1f}, {"tRP", 14.8f}, {"tRAS", 30.2f}, {"tRC", 45.0f}, + {"tWR", 14.8f}, {"tRTP", 2.3f}, {"tCWL", 3.4f}, {"tRRDS", 2.9f}, {"tRRDL", 3.4f}, + {"tWTRS", 5.1f}, {"tWTRL", 6.3f}, {"tFAW", 16.0f}, + {"tRFC", 120.0f}, {"tRFCpb", 60.0f}, {"tREFI", 1900.0f}, {"tRREFD", 8.0f}, +}; + +template +void set_timing(YAML::Node timing, uint32_t data_rate, const clock_timing_t (&bus)[B], const ns_timing_t (&array)[A]) { + timing["rate"] = data_rate; + for (auto& t : bus) { + timing[t.name] = t.clocks; + } + for (auto& t : array) { + timing[t.name] = t.ns; + } } } @@ -71,139 +118,126 @@ class DramSim::Impl { uint64_t ready_cycle; }; - // GDDR6 holds a fixed 16 Gb per device, so its row count shrinks as the - // channel count grows (Ramulator only knows the refresh timing of 4/8/16 Gb - // devices); rows alias above that capacity, which only affects row hits. - static constexpr uint32_t gddr6_density_mb_ = 16384; + // An independent controller and its device. It accepts one transaction per + // DRAM cycle and returns its responses independently of the other channels; + // write acknowledgements do not wait behind read data. + struct channel_t { + Ramulator::IFrontEnd* frontend; + Ramulator::IMemorySystem* memory_system; + std::queue pending_txs; + std::deque read_rsps; + std::deque write_rsps; + uint32_t inflight; + }; + + // Each standard moves 32 B per column command (prefetch x channel width). + static constexpr uint32_t tx_size_ = 32; + + // Every device here holds 8 Gb per channel; addresses above alias, which + // only affects row hits. + static constexpr uint64_t channel_size_ = uint64_t(1) << 30; + + // Bursts per row visit (minimalist open page, 128 B). + static constexpr uint32_t row_visit_bits_ = 2; Config config_; - uint32_t tx_size_; - uint32_t row_mask_; - Ramulator::IFrontEnd* ramulator_frontend_; - Ramulator::IMemorySystem* ramulator_memorysystem_; - uint32_t ch_bits_; + uint32_t bank_bits_; // pseudo-channel, bank group and bank + uint32_t column_bits_; // bursts per row + std::vector channels_; uint64_t cycle_; - // DRAM ticks owed, in units of 1/core_clock: each core cycle adds - // dram_clock, each DRAM tick takes core_clock. + // DRAM ticks owed, in units of 1/(2 x core_clock): each core cycle adds the + // data rate, each DRAM tick (half the data rate) takes 2 x core_clock. uint64_t clock_credit_; - // A channel accepts one transaction per DRAM cycle and returns its - // responses independently of the others; write acknowledgements do not wait - // behind read data. - std::vector> pending_txs_; - std::vector> read_rsps_; - std::vector> write_rsps_; - std::vector inflight_; - - // Ramulator address of a transaction: the channel in the low bits, then the - // fields in the order the RoBaRaCoCh mapper slices them. - uint64_t dram_addr(uint32_t channel, uint64_t addr) const { - uint64_t t; - if (config_.type == Type::GDDR6) { - // GPGPU-Sim's RTX 3070 layout (-gpgpu_mem_addr_mapping): 32 B bursts, - // two column bits, one bank bit, four column bits, three bank bits, - // row; the low bank bit selects the bank group. - uint64_t col = ((addr >> 5) & 0x3) | (((addr >> 8) & 0xf) << 2); - uint64_t row = (addr >> 15) & row_mask_; - uint32_t bank = ipoly16(row, uint32_t(((addr >> 7) & 0x1) | (((addr >> 12) & 0x7) << 1))); - t = row; - t = (t << 2) | (bank >> 2); - t = (t << 2) | (bank & 0x3); - t = (t << 6) | col; - } else { - // The A100 HBM interleave: a stream alternates column, pseudo-channel - // and column again at 64 B granules, then changes bank group every 4 KB - // and bank every 16 KB, with 2 KB rows. HBM2_A100 uses 2 of the 4 banks - // of a group (16 per channel), as GPGPU-Sim models the A100; HBM2 all 4. - uint32_t bank_bits = (config_.type == Type::HBM2_A100) ? 1 : 2; - uint64_t col = (((addr >> 8) & 0xf) << 1) | ((addr >> 6) & 0x1); - uint64_t pc = (addr >> 7) & 0x1; - uint64_t bg = (addr >> 12) & 0x3; - uint64_t bank = (addr >> 14) & ((1u << bank_bits) - 1); - uint64_t row = (addr >> (14 + bank_bits)) & row_mask_; - t = row; - t = (t << 2) | bank; - t = (t << 2) | bg; - t = (t << 1) | pc; - t = (t << 5) | col; - } - t = (t << ch_bits_) | channel; - return t * tx_size_; - } - static void set_hbm2_device(YAML::Node& dram, const Config& config, uint32_t pow2_channels) { - dram["impl"] = "HBM2"; - dram["org"]["preset"] = "HBM2_8Gb"; + static void set_hbm_device(YAML::Node dram, const Config& config) { + // Ramulator's 8 Gb organization: 2 pseudo-channels x 4 bank groups x 4 + // banks, 1 KB rows; a 128-bit channel width with the 2n prefetch makes a + // transaction one 32 B pseudo-channel burst. + dram["impl"] = (config.type == Type::HBM3) ? "HBM3" : "HBM2"; + dram["org"]["preset"] = (config.type == Type::HBM3) ? "HBM3_8Gb" : "HBM2_8Gb"; dram["org"]["density"] = 8192; - dram["org"]["channel"] = pow2_channels; - // A transaction is one burst: 2 x 256-bit prefetch = 64 B. - dram["org"]["channel_width"] = 256; - dram["timing"]["preset"] = "HBM2_2Gbps"; - if (config.type == Type::HBM2_A100) { - // GPGPU-Sim SM80_A100 HBM timing in DRAM cycles. One 64 B transaction - // is two of its 32 B atoms, so the same-bank-group spacing is twice its - // tCCDL. - auto timing = dram["timing"]; - timing["nCL"] = 22; - timing["nRCDRD"] = 22; - timing["nRCDWR"] = 22; - timing["nRP"] = 22; - timing["nRAS"] = 50; - timing["nRC"] = 72; - timing["nWR"] = 19; - timing["nRTPS"] = 7; - timing["nRTPL"] = 7; - timing["nCWL"] = 4; - timing["nCCDS"] = 2; - timing["nCCDL"] = 8; - timing["nRRDS"] = 7; - timing["nRRDL"] = 7; - } + dram["org"]["channel"] = 1; + dram["org"]["channel_width"] = 128; + set_timing(dram["timing"], config.data_rate, hbm_bus_timing, hbm_array_timing); } - static void set_gddr6_device(YAML::Node& dram, const Config& config, uint32_t pow2_channels) { - // x16 channels with 2 KB pages: 512 columns of 16 bits, 4 bank groups of 4 - // banks; a 32 B transaction is a 16-beat burst (8 prefetched columns over a - // 32-bit internal channel width). + static void set_gddr6_device(YAML::Node dram, const Config& config) { + // One x16 channel of a 16 Gb device (8 Gb), 4 bank groups of 4 banks, + // 2 KB pages. A 32 B burst is 16n prefetch on 16 pins; Ramulator's GDDR6 + // prefetches 8 columns, so its columns are 32 bits. dram["impl"] = "GDDR6"; - dram["org"]["dq"] = 16; - dram["org"]["channel"] = pow2_channels; + dram["org"]["dq"] = 32; + dram["org"]["channel"] = 1; dram["org"]["bankgroup"] = 4; dram["org"]["bank"] = 4; + dram["org"]["row"] = 1 << 15; dram["org"]["column"] = 512; - dram["org"]["row"] = uint32_t((uint64_t(gddr6_density_mb_) << 20) / (uint64_t(pow2_channels) * 4 * 4 * 512 * 16)); - dram["org"]["density"] = gddr6_density_mb_; + dram["org"]["density"] = 8192; dram["org"]["channel_width"] = 32; - // The transfer rate is twice the data clock. Bus timings are in clocks of - // the 32 B burst; array timings are GPGPU-Sim's RTX 3070 values - // (-gpgpu_dram_timing_opt) held in nanoseconds; refresh is JEDEC GDDR6. - auto timing = dram["timing"]; - timing["rate"] = 2 * config.dram_clock; - timing["nBL"] = 8; - timing["nCCDS"] = 8; - timing["nCCDL"] = 12; - timing["tCL"] = 6.8f; - timing["tRCDRD"] = 6.8f; - timing["tRCDWD"] = 6.8f; - timing["tRP"] = 6.8f; - timing["tRAS"] = 15.7f; - timing["tRC"] = 22.3f; - timing["tWR"] = 6.8f; - timing["tRTP"] = 1.2f; - timing["tCWL"] = 2.3f; - timing["tRRDS"] = 3.5f; - timing["tRRDL"] = 3.5f; - timing["tWTRS"] = 2.8f; - timing["tWTRL"] = 2.8f; - timing["tFAW"] = 16.0f; - timing["tRFC"] = 120.0f; - timing["tRFCpb"] = 60.0f; - timing["tREFI"] = 1900.0f; - timing["tRREFD"] = 8.0f; + set_timing(dram["timing"], config.data_rate, gddr6_bus_timing, gddr6_array_timing); + } + + YAML::Node channel_config(uint32_t ch) const { + YAML::Node config; + config["Frontend"]["impl"] = "GEM5"; + auto memory_system = config["MemorySystem"]; + memory_system["impl"] = "GenericDRAM"; + memory_system["clock_ratio"] = 1; + if (config_.type == Type::GDDR6) { + set_gddr6_device(memory_system["DRAM"], config_); + } else { + set_hbm_device(memory_system["DRAM"], config_); + } + auto controller = memory_system["Controller"]; + controller["impl"] = "Generic"; + controller["Scheduler"]["impl"] = "FRFCFS"; + // No refresh is modelled: Ramulator's refresh manager refreshes per rank + // and these devices have no rank level. GPU controllers hide refresh with + // per-bank refresh, for which Ramulator has no manager (and its HBM REFsb + // lacks the bank's tRFCsb busy time); all-bank refresh would cost every + // channel ~9% of its time instead. + controller["RefreshManager"]["impl"] = "AllBank"; + controller["RowPolicy"]["impl"] = "OpenRowPolicy"; + // Per-command DRAM trace recording is a debugging aid only: it writes an + // entry for every DRAM command and dominates simulation time (100s of MB + // of I/O for even tiny workloads). Keep it opt-in via the DRAM_TRACE env + // var so default simx/rtlsim runs stay fast. + if (const char* trace_path = getenv("DRAM_TRACE")) { + std::string path = trace_path[0] ? trace_path : "./trace/ramulator.log"; + YAML::Node trace_plugin; + trace_plugin["ControllerPlugin"]["impl"] = "TraceRecorder"; + trace_plugin["ControllerPlugin"]["path"] = path + "." + std::to_string(ch); + controller["plugins"].push_back(trace_plugin); + } + // Fields in the order dram_addr lays them out. + memory_system["AddrMapper"]["impl"] = "RoBaRaCoCh"; + return config; + } + + // Ramulator address of a transaction. A row visit is four bursts, then the + // next pseudo-channel, bank group and bank (minimalist open page), and the + // bank is XORed with the row folded to the bank's width (permutation-based + // interleaving), so streams a power of two apart open rows in different + // banks. + uint64_t dram_addr(uint64_t addr) const { + uint64_t tx = (addr & (channel_size_ - 1)) / tx_size_; + uint64_t visit = tx & ((1u << row_visit_bits_) - 1); + tx >>= row_visit_bits_; + uint64_t bank = tx & ((1u << bank_bits_) - 1); + tx >>= bank_bits_; + uint64_t column = tx & ((1u << (column_bits_ - row_visit_bits_)) - 1); + uint64_t row = tx >> (column_bits_ - row_visit_bits_); + for (uint64_t r = row; r != 0; r >>= bank_bits_) { + bank ^= r & ((1u << bank_bits_) - 1); + } + uint64_t t = (row << bank_bits_) | bank; + t = (t << column_bits_) | (column << row_visit_bits_) | visit; + return t * tx_size_; } // Retires the ready head responses of `rsps`; a request completes with its // last transaction. - void drain(std::deque& rsps, uint32_t ch) { + void drain(std::deque& rsps, channel_t& channel) { while (!rsps.empty()) { auto& rsp = rsps.front(); if (rsp.ready_cycle > cycle_) { @@ -214,94 +248,49 @@ class DramSim::Impl { break; } if (--request.pending == 0) { - --inflight_[ch]; + --channel.inflight; } rsps.pop_front(); } } - void handle_pending_responses() { - for (uint32_t ch = 0; ch < config_.num_channels; ++ch) { - this->drain(read_rsps_[ch], ch); - this->drain(write_rsps_[ch], ch); + void issue(channel_t& channel) { + if (channel.pending_txs.empty()) { + return; } - } - - void handle_pending_requests() { - for (uint32_t ch = 0; ch < config_.num_channels; ++ch) { - auto& txs = pending_txs_[ch]; - if (txs.empty()) { - continue; - } - auto& tx = txs.front(); - auto req_type = tx.is_write ? Ramulator::Request::Type::Write : Ramulator::Request::Type::Read; - std::function callback = nullptr; - if (!tx.is_write) { - callback = [this, ch, request = tx.request](Ramulator::Request& /*dram_req*/) { - this->read_rsps_[ch].push_back({request, this->cycle_ + 1 + this->config_.latency}); - }; - } - if (ramulator_frontend_->receive_external_requests(req_type, tx.addr, 0, callback)) { - // Ramulator does not handle write responses, so we handle the callback ourselves. - if (tx.is_write) { - write_rsps_[ch].push_back({tx.request, cycle_}); - } - txs.pop(); + auto& tx = channel.pending_txs.front(); + auto req_type = tx.is_write ? Ramulator::Request::Type::Write : Ramulator::Request::Type::Read; + std::function callback = nullptr; + if (!tx.is_write) { + callback = [this, &channel, request = tx.request](Ramulator::Request& /*dram_req*/) { + channel.read_rsps.push_back({request, this->cycle_ + 1 + this->config_.latency}); + }; + } + if (channel.frontend->receive_external_requests(req_type, tx.addr, 0, callback)) { + // Ramulator does not handle write responses, so we handle the callback ourselves. + if (tx.is_write) { + channel.write_rsps.push_back({tx.request, cycle_}); } + channel.pending_txs.pop(); } } public: - Impl(const Config& config) : config_(config) { - uint32_t pow2_channels = 1; - ch_bits_ = 0; - while (pow2_channels < config.num_channels) { - pow2_channels <<= 1; - ++ch_bits_; + Impl(const Config& config) + : config_(config) + // HBM: 2 pseudo-channels x 4 bank groups x 4 banks, 32 bursts per 1 KB + // row; GDDR6: 4 x 4 banks, 64 bursts per 2 KB row. + , bank_bits_((config.type == Type::GDDR6) ? 4 : 5) + , column_bits_((config.type == Type::GDDR6) ? 6 : 5) + , channels_(config.num_channels) { + for (uint32_t ch = 0; ch < config.num_channels; ++ch) { + auto ramulator_config = this->channel_config(ch); + auto& channel = channels_[ch]; + channel.frontend = Ramulator::Factory::create_frontend(ramulator_config); + channel.memory_system = Ramulator::Factory::create_memory_system(ramulator_config); + channel.frontend->connect_memory_system(channel.memory_system); + channel.memory_system->connect_frontend(channel.frontend); } - - YAML::Node dram; - if (config.type == Type::GDDR6) { - set_gddr6_device(dram, config, pow2_channels); - tx_size_ = 32; - row_mask_ = dram["org"]["row"].as() - 1; - } else { - set_hbm2_device(dram, config, pow2_channels); - tx_size_ = 64; - row_mask_ = 0x7fff; - } - - YAML::Node dram_config; - dram_config["Frontend"]["impl"] = "GEM5"; - dram_config["MemorySystem"]["impl"] = "GenericDRAM"; - dram_config["MemorySystem"]["clock_ratio"] = 1; - dram_config["MemorySystem"]["DRAM"] = dram; - dram_config["MemorySystem"]["Controller"]["impl"] = "Generic"; - dram_config["MemorySystem"]["Controller"]["Scheduler"]["impl"] = "FRFCFS"; - // libramulator-vortex: Ramulator's AllBank refresh never fires on rank-less HBM2/GDDR6. - dram_config["MemorySystem"]["Controller"]["RefreshManager"]["impl"] = "ChannelRefresh"; - dram_config["MemorySystem"]["Controller"]["RowPolicy"]["impl"] = "OpenRowPolicy"; - // Per-command DRAM trace recording is a debugging aid only: it writes an - // entry for every DRAM command and dominates simulation time (100s of MB - // of I/O for even tiny workloads). Keep it opt-in via the DRAM_TRACE env - // var so default simx/rtlsim runs stay fast. - if (const char* trace_path = getenv("DRAM_TRACE")) { - YAML::Node draw_plugin; - draw_plugin["ControllerPlugin"]["impl"] = "TraceRecorder"; - draw_plugin["ControllerPlugin"]["path"] = trace_path[0] ? trace_path : "./trace/ramulator.log"; - dram_config["MemorySystem"]["Controller"]["plugins"].push_back(draw_plugin); - } - dram_config["MemorySystem"]["AddrMapper"]["impl"] = "RoBaRaCoCh"; - - ramulator_frontend_ = Ramulator::Factory::create_frontend(dram_config); - ramulator_memorysystem_ = Ramulator::Factory::create_memory_system(dram_config); - ramulator_frontend_->connect_memory_system(ramulator_memorysystem_); - ramulator_memorysystem_->connect_frontend(ramulator_frontend_); - - pending_txs_.resize(config.num_channels); - read_rsps_.resize(config.num_channels); - write_rsps_.resize(config.num_channels); - inflight_.resize(config.num_channels, 0); this->reset(); } @@ -309,39 +298,42 @@ class DramSim::Impl { std::ofstream nullstream("ramulator.stats.log"); auto original_buf = std::cout.rdbuf(); std::cout.rdbuf(nullstream.rdbuf()); - ramulator_frontend_->finalize(); - ramulator_memorysystem_->finalize(); + for (auto& channel : channels_) { + channel.frontend->finalize(); + channel.memory_system->finalize(); + } std::cout.rdbuf(original_buf); } void reset() { cycle_ = 0; clock_credit_ = 0; - for (auto& q : pending_txs_) { - q = std::queue(); - } - for (auto& q : read_rsps_) { - q.clear(); + for (auto& channel : channels_) { + channel.pending_txs = std::queue(); + channel.read_rsps.clear(); + channel.write_rsps.clear(); + channel.inflight = 0; } - for (auto& q : write_rsps_) { - q.clear(); - } - std::fill(inflight_.begin(), inflight_.end(), 0); } void tick() { ++cycle_; - clock_credit_ += config_.dram_clock; - this->handle_pending_responses(); - while (clock_credit_ >= config_.core_clock) { - this->handle_pending_requests(); - ramulator_memorysystem_->tick(); - clock_credit_ -= config_.core_clock; + clock_credit_ += config_.data_rate; + for (auto& channel : channels_) { + this->drain(channel.read_rsps, channel); + this->drain(channel.write_rsps, channel); + } + while (clock_credit_ >= 2 * config_.core_clock) { + for (auto& channel : channels_) { + this->issue(channel); + channel.memory_system->tick(); + } + clock_credit_ -= 2 * config_.core_clock; } } bool can_accept(uint32_t channel) const { - return (config_.window == 0) || (inflight_.at(channel) < config_.window); + return (config_.queue_size == 0) || (channels_.at(channel).inflight < config_.queue_size); } void send_request(uint32_t channel, uint64_t addr, bool is_write, ResponseCallback response_cb, void* arg) { @@ -349,11 +341,11 @@ class DramSim::Impl { uint32_t n = (config_.block_size > tx_size_) ? (config_.block_size / tx_size_) : 1; uint64_t base = addr & ~uint64_t(config_.block_size - 1); auto request = std::make_shared(request_t{response_cb, arg, n}); - auto& txs = pending_txs_.at(channel); + auto& ch = channels_.at(channel); for (uint32_t i = 0; i < n; ++i) { - txs.push({dram_addr(channel, base + i * tx_size_), is_write, request}); + ch.pending_txs.push({dram_addr(base + i * tx_size_), is_write, request}); } - ++inflight_[channel]; + ++ch.inflight; } }; @@ -370,8 +362,8 @@ DramSim::Config DramSim::platform_config() { Config config; if (type == "HBM2") { config.type = Type::HBM2; - } else if (type == "HBM2_A100") { - config.type = Type::HBM2_A100; + } else if (type == "HBM3") { + config.type = Type::HBM3; } else if (type == "GDDR6") { config.type = Type::GDDR6; } else { @@ -380,9 +372,9 @@ DramSim::Config DramSim::platform_config() { config.num_channels = VX_CFG_PLATFORM_MEMORY_NUM_BANKS; config.block_size = VX_CFG_PLATFORM_MEMORY_DATA_SIZE; config.core_clock = VX_CFG_PLATFORM_CLOCK_RATE; - config.dram_clock = VX_CFG_PLATFORM_MEMORY_CLOCK_RATE; + config.data_rate = VX_CFG_PLATFORM_MEMORY_DATA_RATE; config.latency = VX_CFG_PLATFORM_MEMORY_LATENCY; - config.window = VX_CFG_PLATFORM_MEMORY_WINDOW; + config.queue_size = VX_CFG_PLATFORM_MEMORY_QUEUE_SIZE; return config; } diff --git a/sim/common/dram_sim.h b/sim/common/dram_sim.h index cc88ff5d78..565dede377 100644 --- a/sim/common/dram_sim.h +++ b/sim/common/dram_sim.h @@ -23,10 +23,12 @@ class DramSim { public: typedef bool (*ResponseCallback)(void *arg); + // The JEDEC standards Ramulator models, each at its own organization and + // timing; the data rate picks the speed. enum class Type { - HBM2, // HBM2 at the 2 Gbps JEDEC preset, 64 B bursts - HBM2_A100, // HBM2 with the A100 timing of GPGPU-Sim's SM80 configuration - GDDR6, // GDDR6 x16 channels, 32 B bursts, 2 KB pages + HBM2, // HBM2/HBM2E: 128-bit channels of two 64-bit pseudo-channels, 32 B bursts + HBM3, // HBM3, a 128-bit channel per pair of 64-bit channels, 32 B bursts + GDDR6, // GDDR6 x16 channels, 32 B bursts, 2 KB pages }; struct Config { @@ -34,9 +36,9 @@ class DramSim { uint32_t num_channels; // need not be a power of two uint32_t block_size; // bytes per request uint32_t core_clock; // MHz - uint32_t dram_clock; // MHz (GDDR6: the data clock, half the transfer rate) + uint32_t data_rate; // MT/s per pin uint32_t latency; // fixed read latency added in core cycles (PHY, controller, NoC) - uint32_t window; // requests a channel holds in flight; 0 = unbounded + uint32_t queue_size; // requests a channel holds in flight; 0 = unbounded }; // The memory configured in VX_config.toml. @@ -49,7 +51,7 @@ class DramSim { void tick(); - // False while the channel holds its window of requests. + // False while the channel holds queue_size requests. bool can_accept(uint32_t channel) const; // addr: byte address within the channel. diff --git a/tests/unittest/dram_sim/main.cpp b/tests/unittest/dram_sim/main.cpp index 81e127e145..4ec135770c 100644 --- a/tests/unittest/dram_sim/main.cpp +++ b/tests/unittest/dram_sim/main.cpp @@ -25,24 +25,30 @@ static bool on_response(void*) { return true; } -// A100 HBM as GPGPU-Sim's SM80 configuration models it: 40 channels at -// 1,512 MHz behind a 1,410 MHz core, 190 cycles of fixed latency, a 257-request -// window per channel. +// The a100 profile: 32 HBM2 channels at 3,038 MT/s (the card's 40 at 2,430) +// behind a 1,410 MHz core, with the fixed latency (190 cycles) and request +// queue (257) of GPGPU-Sim's SM80 configuration. static DramSim::Config a100_config() { DramSim::Config config; - config.type = DramSim::Type::HBM2_A100; - config.num_channels = 40; + config.type = DramSim::Type::HBM2; + config.num_channels = 32; config.block_size = 64; config.core_clock = 1410; - config.dram_clock = 1512; + config.data_rate = 3038; config.latency = 190; - config.window = 257; + config.queue_size = 257; return config; } -// Median core cycles for one read on an idle channel (the median, so the few -// reads that wait behind a refresh do not count); `addr_of(i)` picks the i-th -// address, and the first read (cold) is not counted. +// Peak bytes per core cycle: a 32 B burst every two DRAM clocks per +// pseudo-channel (two per channel), or every eight clocks per GDDR6 channel. +static double peak_bandwidth(const DramSim::Config& config) { + double bytes_per_clock = (config.type == DramSim::Type::GDDR6) ? 4.0 : 32.0; + return config.num_channels * bytes_per_clock * (config.data_rate / 2.0) / config.core_clock; +} + +// Median core cycles for one read on an idle channel; `addr_of(i)` picks the +// i-th address, and the first read (cold) is not counted. template static double unloaded_latency(const DramSim::Config& config, F addr_of) { DramSim dram(config); @@ -68,9 +74,11 @@ static double unloaded_latency(const DramSim::Config& config, F addr_of) { return double(samples[samples.size() / 2]); } -// Bytes per core cycle when every channel streams `blocks` sequential blocks; +// Bytes per core cycle when every channel streams `blocks` blocks, taken in +// turn from `streams` sequential streams `stride` bytes apart; // `peak_inflight` receives the most reads outstanding at once. -static double stream_bandwidth(const DramSim::Config& config, uint32_t blocks, uint64_t* peak_inflight = nullptr) { +static double stream_bandwidth(const DramSim::Config& config, uint32_t blocks, uint64_t* peak_inflight = nullptr, + uint32_t streams = 1, uint64_t stride = 0) { DramSim dram(config); dram.reset(); g_responses = 0; @@ -82,7 +90,8 @@ static double stream_bandwidth(const DramSim::Config& config, uint32_t blocks, u while (g_responses < total) { for (uint32_t ch = 0; ch < config.num_channels; ++ch) { if (sent[ch] < blocks && dram.can_accept(ch)) { - dram.send_request(ch, uint64_t(sent[ch]) * config.block_size, false, on_response, nullptr); + uint64_t addr = uint64_t(sent[ch] % streams) * stride + uint64_t(sent[ch] / streams) * config.block_size; + dram.send_request(ch, addr, false, on_response, nullptr); ++sent[ch]; ++issued; } @@ -97,6 +106,12 @@ static double stream_bandwidth(const DramSim::Config& config, uint32_t blocks, u return double(total) * config.block_size / cycles; } +// Alternates between two rows of one bank: the rows from address bit 15 on +// fold to the same bank permutation (HBM 33 = 1|1 in 5-bit chunks, GDDR6 +// 17 = 1|1 in 4-bit chunks). +static uint64_t hbm_conflict(int i) { return uint64_t(i % 2) * 33 << 15; } +static uint64_t gddr6_conflict(int i) { return uint64_t(i % 2) * 17 << 15; } + static int g_errors = 0; static void check(bool cond, const char* what, double got) { @@ -111,37 +126,44 @@ int main() { // Unloaded latency: open-row hits, then row conflicts (precharge + activate). double open = unloaded_latency(a100, [](int i) { return uint64_t(i % 8) * 128; }); - double conflict = unloaded_latency(a100, [](int i) { return uint64_t(i) << 20; }); + double conflict = unloaded_latency(a100, hbm_conflict); check(open > 205 && open < 225, "A100 open-row read latency ~215 core cycles", open); check(conflict > 245 && conflict < 265, "A100 row-conflict read latency ~255 core cycles", conflict); // The fixed latency is added exactly once per read (measured at a 1:1 clock // ratio, so the DRAM clock phase cannot shift the comparison). auto unit_clock = a100; - unit_clock.core_clock = unit_clock.dram_clock; + unit_clock.core_clock = unit_clock.data_rate / 2; auto no_latency = unit_clock; no_latency.latency = 0; auto open_row = [](int i) { return uint64_t(i % 8) * 128; }; double added = unloaded_latency(unit_clock, open_row) - unloaded_latency(no_latency, open_row); check(added == 190, "fixed latency adds 190 cycles", added); - // Every channel is an independent controller: 40 channels stream ~40x one. + // Every channel is an independent controller: 32 channels stream ~32x one, + // and the count need not be a power of two. auto one = a100; one.num_channels = 1; double bw1 = stream_bandwidth(one, 4096); - double bw40 = stream_bandwidth(a100, 4096); - check(bw40 / bw1 > 36 && bw40 / bw1 < 41, "bandwidth scales with the channel count", bw40 / bw1); - - // Saturated stream efficiency against the A100's 1,373 B/cycle peak. - check(bw40 > 0.6 * 1373 && bw40 <= 1373, "A100 stream reaches 60-100% of peak", bw40); - - // The window bounds the requests in flight. + double bw32 = stream_bandwidth(a100, 4096); + check(bw32 / bw1 > 29 && bw32 / bw1 < 33, "bandwidth scales with the channel count", bw32 / bw1); + auto five = a100; + five.num_channels = 5; + double bw5 = stream_bandwidth(five, 4096); + check(bw5 / bw1 > 4.5 && bw5 / bw1 < 5.1, "5 channels stream ~5x one", bw5 / bw1); + + // Saturated stream efficiency against the computed peak (1,103 B/cycle, the + // card's 1,555 GB/s). + double a100_peak = peak_bandwidth(a100); + check(bw32 > 0.75 * a100_peak && bw32 <= a100_peak, "A100 stream reaches 75-100% of peak", bw32); + + // The controller queue bounds the requests in flight. auto narrow = a100; narrow.num_channels = 1; - narrow.window = 8; + narrow.queue_size = 8; uint64_t peak = 0; stream_bandwidth(narrow, 1024, &peak); - check(peak <= 8, "window of 8 bounds requests in flight", peak); + check(peak <= 8, "queue of 8 bounds requests in flight", peak); // A block of two bursts returns one response. auto wide = a100; @@ -158,65 +180,45 @@ int main() { check(g_responses == 1, "a 128 B block returns one response", double(g_responses)); } - // Plain HBM2 at the platform default: still independent channels. - auto hbm2 = a100; - hbm2.type = DramSim::Type::HBM2; - hbm2.num_channels = 2; - hbm2.latency = 0; - hbm2.window = 0; - double bw2 = stream_bandwidth(hbm2, 4096); - hbm2.num_channels = 1; - double bw1h = stream_bandwidth(hbm2, 4096); + // The platform default: 2 HBM2 channels clocked with a 400 MHz core. + DramSim::Config base{DramSim::Type::HBM2, 2, 64, 400, 800, 0, 0}; + double base_open = unloaded_latency(base, [](int i) { return uint64_t(i % 8) * 64; }); + double base_conflict = unloaded_latency(base, hbm_conflict); + check(base_open == 14, "default open-row read latency 14 cycles", base_open); + check(base_conflict == 26, "default row-conflict read latency 26 cycles", base_conflict); + + // Streams a power of two apart open their rows in different banks: four + // streams 1 MB apart on one channel keep over half a single stream's rate + // (an unpermuted map keeps a third). They still share banks over time, so + // each row visit pays an activation. + auto base_one = base; + base_one.num_channels = 1; + double seq = stream_bandwidth(base_one, 4096); + double strided = stream_bandwidth(base_one, 4096, nullptr, 4, 1 << 20); + check(strided > 0.55 * seq, "four streams 1 MB apart keep over half one stream's rate", strided / seq); + double bw2 = stream_bandwidth(base, 4096); + double bw1h = stream_bandwidth(base_one, 4096); check(bw2 / bw1h > 1.8 && bw2 / bw1h < 2.05, "HBM2 2 channels stream ~2x one", bw2 / bw1h); - // Every channel refreshes: over many refresh intervals some open-row reads - // wait behind a refresh (nRFC = 160 cycles at a 1:1 clock). - { - auto refresh = unit_clock; - refresh.type = DramSim::Type::HBM2; - refresh.num_channels = 1; - refresh.latency = 0; - DramSim dram(refresh); - dram.reset(); - uint64_t min_lat = ~0ull, max_lat = 0; - for (int i = 0; i < 400; ++i) { - g_responses = 0; - uint64_t cycles = 0; - dram.send_request(0, uint64_t(i % 8) * 128, false, on_response, nullptr); - while (g_responses == 0) { - dram.tick(); - ++cycles; - } - if (i != 0) { - min_lat = std::min(min_lat, cycles); - max_lat = std::max(max_lat, cycles); - } - for (int k = 0; k < 97; ++k) { - dram.tick(); - } - } - check(max_lat - min_lat > 100, "refresh stalls reads by ~nRFC", double(max_lat - min_lat)); - } - - // GDDR6 at the rtx5080 profile's shape: 32 x16 channels at 15 GT/s (7,500 - // MHz data clock) behind a 2,295 MHz core, 32 B per 8-clock burst: 418 B/cycle - // peak, the card's 960 GB/s. - auto gddr6 = a100; - gddr6.type = DramSim::Type::GDDR6; - gddr6.num_channels = 32; - gddr6.core_clock = 2295; - gddr6.dram_clock = 7500; - gddr6.latency = 0; - gddr6.window = 0; - double gpeak = 32.0 * 32 / 8 * 7500 / 2295; + // HBM3 at the h100 profile's shape: 32 channels at 6,545 MT/s behind a + // 1,980 MHz core, the card's 3.35 TB/s. + DramSim::Config h100{DramSim::Type::HBM3, 32, 64, 1980, 6545, 0, 0}; + double h100_peak = peak_bandwidth(h100); + double hbw = stream_bandwidth(h100, 4096); + check(hbw > 0.75 * h100_peak && hbw <= h100_peak, "H100 stream reaches 75-100% of peak", hbw); + + // GDDR6 at the rtx5080 profile's shape: 32 x16 channels at 15 GT/s behind a + // 2,295 MHz core: 418 B/cycle peak, the card's 960 GB/s. + DramSim::Config gddr6{DramSim::Type::GDDR6, 32, 64, 2295, 15000, 0, 0}; + double gpeak = peak_bandwidth(gddr6); double gbw32 = stream_bandwidth(gddr6, 4096); - check(gbw32 > 0.6 * gpeak && gbw32 <= gpeak, "GDDR6 stream reaches 60-100% of peak", gbw32); + check(gbw32 > 0.7 * gpeak && gbw32 <= gpeak, "GDDR6 stream reaches 70-100% of peak", gbw32); auto gddr6_one = gddr6; gddr6_one.num_channels = 1; double gbw1 = stream_bandwidth(gddr6_one, 4096); check(gbw32 / gbw1 > 29 && gbw32 / gbw1 < 33, "GDDR6 bandwidth scales with the channel count", gbw32 / gbw1); double gopen = unloaded_latency(gddr6_one, [](int i) { return uint64_t(i % 8) * 32; }); - double gconflict = unloaded_latency(gddr6_one, [](int i) { return uint64_t(i) << 22; }); + double gconflict = unloaded_latency(gddr6_one, gddr6_conflict); check(gopen < gconflict, "GDDR6 row conflicts cost more than row hits", gconflict - gopen); if (g_errors != 0) { diff --git a/third_party/Makefile b/third_party/Makefile index b9e5d9f37f..a32ff0ff30 100644 --- a/third_party/Makefile +++ b/third_party/Makefile @@ -8,20 +8,17 @@ softfloat: ramulator/libramulator.so: cd ramulator && mkdir -p build && cd build && cmake .. && make -j4 -# Vortex's Ramulator extensions: a per-channel refresh manager, and the GDDR6 -# device model Ramulator ships but does not build. Both register with +# The GDDR6 device model Ramulator ships but does not build. It registers with # Ramulator's factory when the library loads, built as Ramulator builds itself. RAMULATOR_EXT_DIR := ramulator_ext RAMULATOR_EXT_INCS := -isystem ramulator/src -isystem ramulator/ext/spdlog/include -isystem ramulator/ext/yaml-cpp/include RAMULATOR_EXT_CXXFLAGS := -std=c++20 -O3 -DNDEBUG -fPIC $(RAMULATOR_EXT_INCS) -$(RAMULATOR_EXT_DIR)/channel_refresh.o: $(RAMULATOR_EXT_DIR)/channel_refresh.cpp - $(CXX) $(RAMULATOR_EXT_CXXFLAGS) -Wall -Wextra -Werror -c $< -o $@ - $(RAMULATOR_EXT_DIR)/gddr6.o: ramulator/src/dram/impl/GDDR6.cpp + mkdir -p $(RAMULATOR_EXT_DIR) $(CXX) $(RAMULATOR_EXT_CXXFLAGS) -c $< -o $@ -$(RAMULATOR_EXT_DIR)/libramulator-vortex.so: $(RAMULATOR_EXT_DIR)/channel_refresh.o $(RAMULATOR_EXT_DIR)/gddr6.o ramulator/libramulator.so +$(RAMULATOR_EXT_DIR)/libramulator-vortex.so: $(RAMULATOR_EXT_DIR)/gddr6.o ramulator/libramulator.so $(CXX) -shared $(filter %.o,$^) -Lramulator -lramulator -o $@ ramulator: ramulator/libramulator.so $(RAMULATOR_EXT_DIR)/libramulator-vortex.so diff --git a/third_party/ramulator_ext/channel_refresh.cpp b/third_party/ramulator_ext/channel_refresh.cpp deleted file mode 100644 index 3888669fe9..0000000000 --- a/third_party/ramulator_ext/channel_refresh.cpp +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright © 2019-2023 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -#include "base/base.h" -#include "dram_controller/controller.h" -#include "dram_controller/refresh.h" - -namespace Ramulator { - -// All-bank refresh every nREFI, one REFab per channel, the channels staggered -// across the interval as independent controllers are. Ramulator's AllBank -// manager refreshes per rank, and HBM2 and GDDR6 have no rank level, so with it -// these devices never refresh. -class ChannelRefresh : public IRefreshManager, public Implementation { - RAMULATOR_REGISTER_IMPLEMENTATION(IRefreshManager, ChannelRefresh, "ChannelRefresh", "All-bank refresh of a rank-less channel.") -private: - Clk_t clk_ = 0; - IDRAMController* ctrl_ = nullptr; - int num_levels_ = -1; - int nrefi_ = -1; - int req_id_ = -1; - Clk_t next_refresh_ = -1; - -public: - void init() override { - ctrl_ = cast_parent(); - } - - void setup(IFrontEnd* /*frontend*/, IMemorySystem* /*memory_system*/) override { - auto dram = ctrl_->m_dram; - num_levels_ = int(dram->m_levels.size()); - nrefi_ = dram->m_timing_vals("nREFI"); - req_id_ = dram->m_requests("all-bank-refresh"); - int num_channels = dram->get_level_size("channel"); - next_refresh_ = nrefi_ + Clk_t(ctrl_->m_channel_id) * nrefi_ / num_channels; - } - - void tick() override { - if (++clk_ != next_refresh_) { - return; - } - next_refresh_ += nrefi_; - std::vector addr_vec(num_levels_, -1); - addr_vec[0] = ctrl_->m_channel_id; - Request req(addr_vec, req_id_); - if (!ctrl_->priority_send(req)) { - throw std::runtime_error("Failed to send refresh!"); - } - } -}; - -} From 03efecbaf3017cb53c95dbd74897dfa666415f15 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 12 Oct 2026 02:24:49 -0700 Subject: [PATCH 027/124] simx: issue warps with the RTL's greedy round-robin arbiter SimX picked the next warp greedy-then-oldest, with ages reset whenever a warp stopped requesting and ties going to the lowest warp id; the RTL's scoreboard arbiter (VX_scoreboard out_arb, round-robin, STICKY) keeps the current warp while it is ready and then passes to the next one in index order. On sgemm_tcu fp16 (NT=8) RTL warps run in lockstep, loading and waiting on memory together, while SimX warps drift apart (17/17/16/14 blocks per warp against 16 each) and overlap their memory waits. The gap grew with memory latency and reached 6.9% (tensor:model_parity-fp16, 5% tolerance) once the DRAM clock followed the core clock. SimX now uses a greedy round-robin arbiter matching VX_rr_arbiter with STICKY=1. model_parity-fp16: simx 39747 vs rtlsim 39221 cycles (+1.3%, was -6.9%); at 2000 MT/s +1.75% (was -3.8%); sgemm parity unchanged within 0.3%. Co-Authored-By: Claude Opus 5.5 --- sim/simx/core.cpp | 2 +- sim/simx/types.h | 48 ++++++++++++++--------------------------------- 2 files changed, 15 insertions(+), 35 deletions(-) diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index f67dba6e23..ec729d83e3 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -69,7 +69,7 @@ class Core::Impl { , fetch_latch_(ctx, "fetch_latch", 2, 2) , decode_latch_(ctx, "decode_latch", 1, 2) , pending_icache_(VX_CFG_NUM_WARPS) - , ibuffer_arbs_(VX_CFG_ISSUE_WIDTH, {ArbiterType::GTO, PER_ISSUE_WARPS}) + , ibuffer_arbs_(VX_CFG_ISSUE_WIDTH, {ArbiterType::GreedyRR, PER_ISSUE_WARPS}) , issue_lock_owner_(VX_CFG_ISSUE_WIDTH, kLockOpen) , fu_credits_(VX_CFG_ISSUE_WIDTH, std::vector((uint32_t)FUType::Count, 0)) , fu_full_(VX_CFG_ISSUE_WIDTH, BitVector<>((uint32_t)FUType::Count, 0)) diff --git a/sim/simx/types.h b/sim/simx/types.h index 6423d1a61a..1658771c6e 100644 --- a/sim/simx/types.h +++ b/sim/simx/types.h @@ -895,7 +895,7 @@ enum class ArbiterType { Priority, RoundRobin, Matrix, - GTO + GreedyRR }; inline std::ostream &operator<<(std::ostream &os, const ArbiterType& type) { @@ -903,7 +903,7 @@ inline std::ostream &operator<<(std::ostream &os, const ArbiterType& type) { case ArbiterType::Priority: os << "Priority"; break; case ArbiterType::RoundRobin: os << "RoundRobin"; break; case ArbiterType::Matrix: os << "Matrix"; break; - case ArbiterType::GTO: os << "GTO"; break; + case ArbiterType::GreedyRR: os << "GreedyRR"; break; default: assert(false); } return os; @@ -1017,56 +1017,36 @@ class MatrixArbiter : public IArbiterImpl { std::vector> priority_matrix_; }; -class GTOArbiter : public IArbiterImpl { +// Greedy round-robin: keeps granting the current owner while it requests, +// then hands over to the next requester after it in index order. +class GreedyRRArbiter : public IArbiterImpl { public: - GTOArbiter(uint32_t size) - : size_(size) - , age_(size, 0) - , last_grant_(-1u) { + GreedyRRArbiter(uint32_t size) : size_(size) { this->reset(); } uint32_t grant(const BitVector<>& requests) override { assert(requests.size() == size_); - // greedy: keep granting same requester if still active if (last_grant_ < size_ && requests.test(last_grant_)) { - this->update_ages(requests, last_grant_); return last_grant_; } - // Then-Oldest: find the requester with the highest age - uint32_t best = -1u; - uint32_t best_age = 0; + uint32_t start = (last_grant_ < size_) ? (last_grant_ + 1) % size_ : 0; for (uint32_t i = 0; i < size_; ++i) { - if (requests.test(i) && (best == -1u || age_[i] > best_age)) { - best = i; - best_age = age_[i]; + uint32_t idx = (start + i) % size_; + if (requests.test(idx)) { + last_grant_ = idx; + return idx; } } - if (best != -1u) { - last_grant_ = best; - this->update_ages(requests, best); - } - return best; + return -1u; } void reset() override { - std::fill(age_.begin(), age_.end(), 0); last_grant_ = -1u; } private: - void update_ages(const BitVector<>& requests, uint32_t granted) { - for (uint32_t i = 0; i < size_; ++i) { - if (i == granted || !requests.test(i)) { - age_[i] = 0; - } else { - ++age_[i]; - } - } - } - uint32_t size_; - std::vector age_; uint32_t last_grant_; }; @@ -1083,8 +1063,8 @@ class Arbiter { case ArbiterType::Matrix: impl_ = std::make_shared(size); break; - case ArbiterType::GTO: - impl_ = std::make_shared(size); + case ArbiterType::GreedyRR: + impl_ = std::make_shared(size); break; default: assert(false); From 72b574b0af7246823e9b679397361e4fa6555f50 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 12 Oct 2026 02:24:50 -0700 Subject: [PATCH 028/124] simx: let a cache bank pipeline hold one request per stage The bank pipeline is a timed FIFO whose delay is the configured latency, created with the default capacity of two, so a bank accepted only two requests per LATENCY cycles: 0.5 per cycle at the stock L2 latency of 4, far less with large caches. The RTL bank (PIPELINE_STAGES = LATENCY) accepts one request per cycle at any depth. Size the FIFO to the latency (at least two). Co-Authored-By: Claude Opus 5.5 --- sim/simx/mem/cache.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index 41ed3b411a..8c4dcfc0a8 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -18,6 +18,7 @@ #if VX_CFG_EXT_A_ENABLED #include "amo_unit.h" #endif +#include #include #include #include @@ -649,7 +650,7 @@ class CacheBank : public SimObject { const Cache::Config &config, const params_t ¶ms, uint32_t bank_id) - : SimObject(ctx, name), core_req_in(this), core_rsp_out(this), mem_req_out(this), mem_rsp_in(this, config.mshr_size + AMO_PASSTHRU_CAP), config_(config), params_(params), bank_id_(bank_id), sets_(params.sets_per_bank, set_t(params.lines_per_set, params.sectors_per_line)), mshr_(config.mshr_size), pipe_req_(TFifo::Create("", config.latency)), rand_ctr_(0), adm_seq_ctr_(0) + : SimObject(ctx, name), core_req_in(this), core_rsp_out(this), mem_req_out(this), mem_rsp_in(this, config.mshr_size + AMO_PASSTHRU_CAP), config_(config), params_(params), bank_id_(bank_id), sets_(params.sets_per_bank, set_t(params.lines_per_set, params.sectors_per_line)), mshr_(config.mshr_size), pipe_req_(TFifo::Create("", config.latency, std::max(2u, config.latency))), rand_ctr_(0), adm_seq_ctr_(0) #if VX_CFG_EXT_A_ENABLED , amo_unit_(__MAX(2u, (uint32_t)VX_CFG_AMO_RS_SIZE)) #endif From c068cea4db39199be4791bcdee63bbb607854fcd Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 12 Oct 2026 02:24:51 -0700 Subject: [PATCH 029/124] ci: re-record the rtlsim perf-gate baselines for the DRAM model Regenerated with pytest ci -m perf_gate --update-baselines (49/49) after the DRAM model's data-sheet configuration: the default DRAM is now clocked with the core instead of at 1 GHz, with JEDEC timing, 32 B bursts and permuted bank interleave, so memory-bound cases moved. Co-Authored-By: Claude Opus 5.5 --- ci/baselines/perf/core.json | 64 +++++++++++++++---------------- ci/baselines/perf/dxa.json | 44 ++++++++++----------- ci/baselines/perf/graphics.json | 38 +++++++++--------- ci/baselines/perf/raytracing.json | 12 +++--- ci/baselines/perf/tensor.json | 20 +++++----- ci/baselines/perf/tensor_mx.json | 4 +- ci/baselines/perf/tensor_sp.json | 4 +- ci/baselines/perf/tensor_wg.json | 12 +++--- 8 files changed, 99 insertions(+), 99 deletions(-) diff --git a/ci/baselines/perf/core.json b/ci/baselines/perf/core.json index b08f98ad35..77e8c4785a 100644 --- a/ci/baselines/perf/core.json +++ b/ci/baselines/perf/core.json @@ -1,177 +1,177 @@ { "core:raycast-nt16:rtlsim": { "32": { - "cycles": 562137, + "cycles": 563524, "instrs": 137486 }, "app": "raycast", "args": "", - "config_hash": "314d6d30965b89c4", + "config_hash": "396cee9df6025cdb", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:raycast-nt4:rtlsim": { "32": { - "cycles": 1495556, + "cycles": 1498642, "instrs": 259228 }, "app": "raycast", "args": "", - "config_hash": "909d80cfbaadadc2", + "config_hash": "262f24032ef14b58", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:sgemm-mc-nt16:rtlsim": { "32": { - "cycles": 779835, + "cycles": 787850, "instrs": 623680 }, "app": "sgemm", "args": "-n128", - "config_hash": "92050636b69caeb2", + "config_hash": "ad2b37492d761e8f", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=16 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "core:sgemm-mc-nt4:rtlsim": { "32": { - "cycles": 2228982, + "cycles": 2239426, "instrs": 2494528 }, "app": "sgemm", "args": "-n128", - "config_hash": "72b6506beadbe8a0", + "config_hash": "606d011dc6f612d3", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "core:sgemm-nt16:rtlsim": { "32": { - "cycles": 1909825, + "cycles": 1838654, "instrs": 623632 }, "app": "sgemm", "args": "-n128", - "config_hash": "514f489ab80d25c0", + "config_hash": "9390754d3719fca3", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:sgemm-nt4:rtlsim": { "32": { - "cycles": 6189932, + "cycles": 5939731, "instrs": 2494480 }, "app": "sgemm", "args": "-n128", - "config_hash": "4af79268b2c7557f", + "config_hash": "740fd84fa2938103", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:sgemmx-nt16:rtlsim": { "32": { - "cycles": 1261549, + "cycles": 1290130, "instrs": 266896 }, "app": "sgemmx", "args": "-n128", - "config_hash": "d53587fae488c32b", + "config_hash": "2e1c44e05f9ac34d", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:sgemmx-nt4:rtlsim": { "32": { - "cycles": 4181334, + "cycles": 4219466, "instrs": 1067536 }, "app": "sgemmx", "args": "-n128", - "config_hash": "572494ba2df6f58a", + "config_hash": "024adb49ed96bdd3", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:sgemv-nt16:rtlsim": { "32": { - "cycles": 583960, + "cycles": 685107, "instrs": 66800 }, "app": "sgemv", "args": "-m512 -n512", - "config_hash": "587ac34faae9f341", + "config_hash": "a3d9df9695acbb95", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:sgemv-nt4:rtlsim": { "32": { - "cycles": 931381, + "cycles": 872926, "instrs": 267152 }, "app": "sgemv", "args": "-m512 -n512", - "config_hash": "93f8522bd055dda2", + "config_hash": "0e8990b25e64ad7f", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:softmax-nt16:rtlsim": { "32": { - "cycles": 1273285, + "cycles": 1272929, "instrs": 464089 }, "app": "softmax", "args": "-n64", - "config_hash": "9c8c364608242127", + "config_hash": "9e0c1c9767c760ea", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:softmax-nt4:rtlsim": { "32": { - "cycles": 4861515, + "cycles": 4860309, "instrs": 1789797 }, "app": "softmax", "args": "-n64", - "config_hash": "b2c24fbda6abd30b", + "config_hash": "eb5fed4bfce19d76", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:stencil3d-nt16:rtlsim": { "32": { - "cycles": 2395091, + "cycles": 2306960, "instrs": 892944 }, "app": "stencil3d", "args": "-n32", - "config_hash": "fec6d70d6e8e83c3", + "config_hash": "6925a5138be930ea", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:stencil3d-nt4:rtlsim": { "32": { - "cycles": 4283200, + "cycles": 4180969, "instrs": 1785872 }, "app": "stencil3d", "args": "-n32", - "config_hash": "bb64a7573d2c2365", + "config_hash": "b83a1e3735f79d39", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:vecadd-nt16:rtlsim": { "32": { - "cycles": 99638, + "cycles": 100640, "instrs": 29712 }, "app": "vecadd", "args": "-n16384", - "config_hash": "58a51468cf8f99e7", + "config_hash": "ef0c2548e7c5bf86", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:vecadd-nt4:rtlsim": { "32": { - "cycles": 363323, + "cycles": 367765, "instrs": 118800 }, "app": "vecadd", "args": "-n16384", - "config_hash": "4ca729a11fbf88a3", + "config_hash": "ce1eb79984a98552", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/dxa.json b/ci/baselines/perf/dxa.json index 453b0775f5..bf1e1a010c 100644 --- a/ci/baselines/perf/dxa.json +++ b/ci/baselines/perf/dxa.json @@ -1,122 +1,122 @@ { "dxa:perf_gate-copy-mcast:rtlsim": { "32": { - "cycles": 3313, + "cycles": 3381, "instrs": 961 }, "app": "dxa_copy_mcast", "args": "", - "config_hash": "2df97b749a0febe3", + "config_hash": "6a815f855f6e0d05", "configs": "-DVX_CFG_EXT_DXA_ENABLE", "driver": "rtlsim" }, "dxa:perf_gate-copy-nt16:rtlsim": { "32": { - "cycles": 1869, + "cycles": 1994, "instrs": 592 }, "app": "dxa_copy", "args": "-d1", - "config_hash": "59a6c715ed820634", + "config_hash": "626c5f6244584b1f", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "dxa:perf_gate-copy-nt4:rtlsim": { "32": { - "cycles": 6773, + "cycles": 6827, "instrs": 1792 }, "app": "dxa_copy", "args": "-d1", - "config_hash": "534d84e7f493e414", + "config_hash": "7c32a11d123bfb13", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-mcast-nt16:rtlsim": { "32": { - "cycles": 14056427, + "cycles": 14022240, "instrs": 4962320 }, "app": "sgemm2_dxa_mcast", "args": "-n128", - "config_hash": "589585790379eee0", + "config_hash": "822d0ca194266a27", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-mcast-nt4:rtlsim": { "32": { - "cycles": 14108810, + "cycles": 14063502, "instrs": 4962320 }, "app": "sgemm2_dxa_mcast", "args": "-n128", - "config_hash": "2879ae8c33e35bb8", + "config_hash": "87ea544e77719254", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-nt16:rtlsim": { "32": { - "cycles": 3895374, + "cycles": 3941267, "instrs": 1263632 }, "app": "sgemm2_dxa", "args": "-n128 -t4 -m1", - "config_hash": "f3759c20ec885c91", + "config_hash": "e55b61598f5f10ea", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-nt4:rtlsim": { "32": { - "cycles": 14130928, + "cycles": 14352313, "instrs": 5017616 }, "app": "sgemm2_dxa", "args": "-n128 -t4 -m1", - "config_hash": "aa3ad14268d9b8b5", + "config_hash": "21da2f2737bf03dd", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-dxa-mcast-nt16:rtlsim": { "32": { - "cycles": 79024, + "cycles": 82160, "instrs": 32384 }, "app": "sgemm_tcu_wg_dxa_mcast", "args": "-m 128 -n 128 -k 64", - "config_hash": "7080653c22396eb6", + "config_hash": "20304652a1bfb94c", "configs": "-DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-dxa-mcast-nt4:rtlsim": { "32": { - "cycles": 348356, + "cycles": 360010, "instrs": 234560 }, "app": "sgemm_tcu_wg_dxa_mcast", "args": "-m 128 -n 128 -k 64", - "config_hash": "401d267a03887e99", + "config_hash": "bf4520d008c07dc9", "configs": "-DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-dxa:rtlsim": { "32": { - "cycles": 34417, + "cycles": 36021, "instrs": 9160 }, "app": "sgemm_tcu_wg_dxa", "args": "-m 128 -n 128 -k 64", - "config_hash": "3117dd957603b2e8", + "config_hash": "2a4e017d98740636", "configs": "-DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DWGMMA_NRC=16 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_THREADS=32", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-sp-dxa:rtlsim": { "32": { - "cycles": 55358, + "cycles": 59789, "instrs": 9328 }, "app": "sgemm_tcu_wg_sp_dxa", "args": "-m 128 -n 128 -k 128", - "config_hash": "5bec3cc54a0c63f1", + "config_hash": "f05069d7cf0ee3ff", "configs": "-DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_THREADS=32", "driver": "rtlsim" } diff --git a/ci/baselines/perf/graphics.json b/ci/baselines/perf/graphics.json index f4f9eb603d..b5ee86bfc2 100644 --- a/ci/baselines/perf/graphics.json +++ b/ci/baselines/perf/graphics.json @@ -1,100 +1,100 @@ { "graphics:perf_gate-draw3d-mc:rtlsim": { "32": { - "cycles": 529292, + "cycles": 530195, "instrs": 203136 }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "ae1062062a26ef92", + "config_hash": "1e2937b20579ad3f", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "graphics:perf_gate-draw3d-nt16:rtlsim": { "32": { - "cycles": 142905, - "instrs": 51650 + "cycles": 143620, + "instrs": 51652 }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "e397892c0cb72ec2", + "config_hash": "edc924bc4f246971", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "graphics:perf_gate-draw3d-nt4:rtlsim": { "32": { - "cycles": 528252, + "cycles": 529393, "instrs": 203136 }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "15664f7fbe13f120", + "config_hash": "d682cc8c6f99c06b", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "graphics:perf_gate-om-nt16:rtlsim": { "32": { - "cycles": 188898, + "cycles": 189231, "instrs": 68624 }, "app": "gfx_om", "args": "-w 128 -h 128 -r whitebox_128.png", - "config_hash": "b2ccda0f4e3d0471", + "config_hash": "46a66220c1d9956b", "configs": "-DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "graphics:perf_gate-om-nt4:rtlsim": { "32": { - "cycles": 740590, + "cycles": 740701, "instrs": 274448 }, "app": "gfx_om", "args": "-w 128 -h 128 -r whitebox_128.png", - "config_hash": "e542501f5bf6269e", + "config_hash": "a7554f6fa88474dc", "configs": "-DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "graphics:perf_gate-raster-nt16:rtlsim": { "32": { - "cycles": 14865, + "cycles": 14785, "instrs": 4708 }, "app": "gfx_raster", "args": "-w 128 -h 128 -t triangle.cgltrace -r triangle_ref_128.png", - "config_hash": "7272f5969f7ee30d", + "config_hash": "e2e1879d7d6f9aaf", "configs": "-DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "graphics:perf_gate-raster-nt4:rtlsim": { "32": { - "cycles": 55526, + "cycles": 55565, "instrs": 18640 }, "app": "gfx_raster", "args": "-w 128 -h 128 -t triangle.cgltrace -r triangle_ref_128.png", - "config_hash": "05e09632b1bbf2b7", + "config_hash": "6e48f6b249533cc5", "configs": "-DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "graphics:perf_gate-tex-nt16:rtlsim": { "32": { - "cycles": 35342, + "cycles": 35441, "instrs": 12304 }, "app": "gfx_tex", "args": "-i toad.png -r toad_ref_f0.png -f 0 -g 0", - "config_hash": "9669b4d06963f72d", + "config_hash": "c8c6b30537e93e2f", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "graphics:perf_gate-tex-nt4:rtlsim": { "32": { - "cycles": 138329, + "cycles": 136487, "instrs": 49168 }, "app": "gfx_tex", "args": "-i toad.png -r toad_ref_f0.png -f 0 -g 0", - "config_hash": "aee6b6136e6a4575", + "config_hash": "a4fc4d97e6e80586", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/raytracing.json b/ci/baselines/perf/raytracing.json index bda30910c0..1fb0ab7f7b 100644 --- a/ci/baselines/perf/raytracing.json +++ b/ci/baselines/perf/raytracing.json @@ -1,34 +1,34 @@ { "raytracing:perf_gate-rt_raycast-mc:rtlsim": { "32": { - "cycles": 84378, + "cycles": 85045, "instrs": 39346 }, "app": "rt_raycast", "args": "", - "config_hash": "937f99317c42a59f", + "config_hash": "9a4f28d2555289b5", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "raytracing:perf_gate-rt_raycast-nt16:rtlsim": { "32": { - "cycles": 53283, + "cycles": 53434, "instrs": 11372 }, "app": "rt_raycast", "args": "", - "config_hash": "dfd33a99196e04ad", + "config_hash": "aef578263250d585", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "raytracing:perf_gate-rt_raycast-nt4:rtlsim": { "32": { - "cycles": 159558, + "cycles": 159655, "instrs": 39330 }, "app": "rt_raycast", "args": "", - "config_hash": "c17be24af5bd78b9", + "config_hash": "c7b45fe5c1828bc5", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor.json b/ci/baselines/perf/tensor.json index fae07189e9..3552c4620b 100644 --- a/ci/baselines/perf/tensor.json +++ b/ci/baselines/perf/tensor.json @@ -1,56 +1,56 @@ { "tensor:perf_gate-fp16-isw4:rtlsim": { "32": { - "cycles": 191642, + "cycles": 197465, "instrs": 130592 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "d21f5479c2459776", + "config_hash": "da0a50e46103b7dc", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" }, "tensor:perf_gate-fp16-mc-nt16:rtlsim": { "32": { - "cycles": 42941, + "cycles": 42404, "instrs": 47296 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "23f4b1de5a361780", + "config_hash": "2d12f1a0058fcc3b", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=16 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "tensor:perf_gate-fp16-mc-nt4:rtlsim": { "32": { - "cycles": 221484, + "cycles": 221592, "instrs": 336448 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "c4461f9b2a2b71d7", + "config_hash": "fce886092e60e0eb", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "tensor:perf_gate-fp16:rtlsim": { "32": { - "cycles": 253809, + "cycles": 256743, "instrs": 130576 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "9c6dfe0c35a740a9", + "config_hash": "f0558a3b3dfde51f", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32", "driver": "rtlsim" }, "tensor:perf_gate-sgemm2-fp16:rtlsim": { "32": { - "cycles": 594027, + "cycles": 590798, "instrs": 409888 }, "app": "sgemm2_tcu", "args": "-m 128 -n 128 -k 64", - "config_hash": "9c8bbf01434d9d8d", + "config_hash": "24fd7a811fabb8c4", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_mx.json b/ci/baselines/perf/tensor_mx.json index 1fbd356b34..1dc1b78f41 100644 --- a/ci/baselines/perf/tensor_mx.json +++ b/ci/baselines/perf/tensor_mx.json @@ -1,12 +1,12 @@ { "tensor_mx:perf_gate-mxfp8:rtlsim": { "32": { - "cycles": 76601, + "cycles": 76393, "instrs": 34832 }, "app": "sgemm_tcu_mx", "args": "-m 128 -n 128 -k32", - "config_hash": "46d80398194a01e4", + "config_hash": "3cfc301023342a7d", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DITYPE=mxfp8 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_sp.json b/ci/baselines/perf/tensor_sp.json index 47e3a2926a..2c4f92004c 100644 --- a/ci/baselines/perf/tensor_sp.json +++ b/ci/baselines/perf/tensor_sp.json @@ -1,12 +1,12 @@ { "tensor_sp:perf_gate-sparse-fp16:rtlsim": { "32": { - "cycles": 184985, + "cycles": 183094, "instrs": 100880 }, "app": "sgemm_tcu_sp", "args": "-m 128 -n 128 -k 128", - "config_hash": "cfe93fc4d3413031", + "config_hash": "90defe3d20b6f739", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_wg.json b/ci/baselines/perf/tensor_wg.json index 57e02b2b8f..c7f771c72f 100644 --- a/ci/baselines/perf/tensor_wg.json +++ b/ci/baselines/perf/tensor_wg.json @@ -1,34 +1,34 @@ { "tensor_wg:perf_gate-wgmma-fedp2k-rs:rtlsim": { "32": { - "cycles": 158687, + "cycles": 158181, "instrs": 108512 }, "app": "sgemm_tcu_wg", "args": "-m64 -n64 -k64", - "config_hash": "f019a0cb823990c9", + "config_hash": "ba81956fe68e94b6", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_TCU_FEDP2K -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DWGMMA_RS", "driver": "rtlsim" }, "tensor_wg:perf_gate-wgmma-fp16-ss:rtlsim": { "32": { - "cycles": 836986, + "cycles": 834873, "instrs": 525536 }, "app": "sgemm_tcu_wg", "args": "-m 128 -n 128 -k 128", - "config_hash": "7b6f63cb95491231", + "config_hash": "6ee14f25ec573117", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=32 -DWGMMA_SS -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" }, "tensor_wg:perf_gate-wgmma-sparse:rtlsim": { "32": { - "cycles": 398222, + "cycles": 400023, "instrs": 258976 }, "app": "sgemm_tcu_wg_sp", "args": "-m 128 -n 128 -k 128", - "config_hash": "f620d8a9b0396d1e", + "config_hash": "a9ea3d8bd83a7eed", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" } From 046ff33b0fe44ff67d0901ed1089bfbcde84faea Mon Sep 17 00:00:00 2001 From: tinebp Date: Tue, 13 Oct 2026 16:46:30 -0700 Subject: [PATCH 030/124] Flush caches with one broadcast write and per-port flush tokens The kernel-boundary flush had three costs and one race: - the command processor flushed the cores one after another, each with a separate read that started and awaited that core's flush; - every cache forwarded its flush through the shared levels, including the read-only icache, so each L1 walked L2/L3 again; - a cache forwarded a single token on memory port 0, so writes queued on another port could reach the next level after its walk and stay dirty (reproduced with random input backpressure; the software command processor's long turnaround hides it in plain rtlsim). Now: - The CP starts every core with one broadcast DCR write to CACHE_FLUSH, then reads the cores in turn. VX_dcr_data tracks the flush with pending/complete/read state, so the flushes overlap and the responses, which have no back-pressure, never collide. A flush read with no write before it is no longer a way to start one. - VX_cache_init answers flush tokens locally: walk all banks, then (with FLUSH_FORWARD) send one token per memory port once the request crossbar has drained, wait for every port's answer, and answer the requesters. Token responses are recognized by phase; answers take bank 0's lane into the response crossbar. Boundary nets stay flop-driven. - Read-only caches (icache, tcache, rtcache, rcache) set FLUSH_FORWARD=0: they hold no dirty data, and the dcache's flush walks the shared levels. - Every independently buffered write path carries its own token: every dcache lane unless the core has a private dcache (store_drained covers that case), and every OM input of the ocache. - OM aperture stores are invisible to busy between the dcache and the OM ingress, so VX_om_steer now holds a flush token bound for L2 until a VX_graphics controller has drained the OM and flushed the ocache. A half-received colour+depth record is not waited for, so a fence in a fragment shader cannot deadlock on a beat queued behind the held token. SimX already flushes the whole hierarchy functionally and does not time the flush; this brings the RTL's kernel-boundary cost toward that model (model_parity and perf_gate unchanged). Tests: new tests/regression/flush_coherence (rotating writes plus a tail store burst, checked every round). CI adds flush_coherence-{1,2,3} and DBG_FLUSH_STALL stress cases (flush_coherence-stress-{1,2,3}, graphics:draw3d-flush-stress-{1,2}); DBG_FLUSH_STALL adds random input backpressure in VX_cache_init. Under stress, the single-token design, port-0-only tokens and lane-0-only tokens all fail. cp_dcr_proxy unittest covers the broadcast flush over 3 and 0 cores. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/graphics.yaml | 23 ++- ci/testcases/regression.yaml | 72 +++++++ docs/assets/img/cache_flush_sequence.svg | 175 ++++++++-------- docs/assets/img/cache_flush_tree.svg | 187 ++++++++---------- docs/designs/cache_subsystem.md | 158 +++++++++------ docs/designs/command_processor.md | 15 +- docs/designs/lsu_pipeline_design.md | 2 +- docs/designs/output_merger_architecture.md | 18 +- hw/rtl/VX_cluster.sv | 4 +- hw/rtl/VX_gpu_pkg.sv | 5 +- hw/rtl/VX_graphics.sv | 143 +++++++++++--- hw/rtl/VX_socket.sv | 6 + hw/rtl/cache/VX_cache.sv | 154 +++++++++++++-- hw/rtl/cache/VX_cache_bank.sv | 1 - hw/rtl/cache/VX_cache_cluster.sv | 2 + hw/rtl/cache/VX_cache_flush.sv | 8 +- hw/rtl/cache/VX_cache_init.sv | 209 ++++++++++++++------ hw/rtl/cache/VX_cache_wrap.sv | 2 + hw/rtl/core/VX_core.sv | 9 +- hw/rtl/core/VX_dcr_data.sv | 50 +++-- hw/rtl/core/VX_dcr_flush.sv | 2 +- hw/rtl/core/VX_mem_unit.sv | 35 +++- hw/rtl/cp/VX_cp_dcr_proxy.sv | 33 ++-- hw/rtl/om/VX_om_ingress.sv | 7 +- hw/rtl/om/VX_om_steer.sv | 46 ++++- hw/unittest/cp_dcr_proxy/main.cpp | 65 +++++- hw/unittest/gfx/VX_gfx_top.sv | 2 + hw/unittest/vm/VX_vm_top.sv | 1 + sim/common/cmd_processor.cpp | 33 ++-- sim/common/cmd_processor.h | 2 + sim/rtlsim/main.cpp | 1 + sim/rtlsim/processor.cpp | 4 +- sim/simx/main.cpp | 1 + sw/runtime/rtlsim/vortex.cpp | 5 + sw/runtime/simx/vortex.cpp | 4 + tests/regression/Makefile | 1 + tests/regression/flush_coherence/Makefile | 16 ++ tests/regression/flush_coherence/common.h | 21 ++ tests/regression/flush_coherence/kernel.cpp | 28 +++ tests/regression/flush_coherence/main.cpp | 190 ++++++++++++++++++ 40 files changed, 1301 insertions(+), 439 deletions(-) create mode 100644 tests/regression/flush_coherence/Makefile create mode 100644 tests/regression/flush_coherence/common.h create mode 100644 tests/regression/flush_coherence/kernel.cpp create mode 100644 tests/regression/flush_coherence/main.cpp diff --git a/ci/testcases/graphics.yaml b/ci/testcases/graphics.yaml index c337b03306..252736abaf 100644 --- a/ci/testcases/graphics.yaml +++ b/ci/testcases/graphics.yaml @@ -693,4 +693,25 @@ tests: app: gfx_draw3d args: -w 128 -h 128 -t box.cgltrace -r box_ref_128.png configs: -DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE - shape: {cores: 2, l2cache: true, threads: 4} \ No newline at end of file + shape: {cores: 2, l2cache: true, threads: 4} +# Flush ordering of OM aperture stores under random input backpressure +# (DBG_FLUSH_STALL): the last fragments' stores are often still in transit to +# the OM ingress when the kernel-boundary flush starts, so the flush must wait +# for them behind the dcache's flush tokens held at the OM steers. +- id: draw3d-flush-stress-1 + via: blackbox + drivers: + - rtlsim + app: gfx_draw3d + args: -w 128 -h 128 -t box.cgltrace -r box_ref_128.png + configs: -DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DDBG_FLUSH_STALL + shape: {threads: 16} +- id: draw3d-flush-stress-2 + via: blackbox + drivers: + - rtlsim + app: gfx_draw3d + args: -w 128 -h 128 -t box.cgltrace -r box_ref_128.png + configs: -DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DDBG_FLUSH_STALL + shape: {cores: 2, l2cache: true, threads: 4} + tier: full diff --git a/ci/testcases/regression.yaml b/ci/testcases/regression.yaml index 50e45420c0..b180c479b4 100644 --- a/ci/testcases/regression.yaml +++ b/ci/testcases/regression.yaml @@ -155,3 +155,75 @@ tests: drivers: - simx app: module_reload +# Kernel-boundary flush coherence: back-to-back launches with host rewrites, +# rotating line ownership across cores and cross-launch read-modify-writes. +# The shapes cover a write-back L1, a write-back L2 shared by 4 cores, and a +# write-back L3 shared by two clusters, where every core flushes at once. +- id: flush_coherence-1 + via: blackbox + drivers: + - simx + - rtlsim + app: flush_coherence +# The xrt leg runs the RTL command processor's flush (broadcast write, then +# per-core reads). +- id: flush_coherence-2 + via: blackbox + drivers: + - simx + - xrt + app: flush_coherence + shape: + cores: 4 + l2cache: true +- id: flush_coherence-3 + via: blackbox + drivers: + - simx + - rtlsim + app: flush_coherence + shape: + clusters: 2 + cores: 2 + l2cache: true + l3cache: true +# Flush ordering under random input backpressure (DBG_FLUSH_STALL): each round +# ends on a store burst, so a core's last writes are often still queued on one +# memory port or dcache lane while its token on another reaches the next level. +# Without a token per path these shapes lose those writes. +- id: flush_coherence-stress-1 + via: blackbox + drivers: + - rtlsim + app: flush_coherence + args: -n2048 -r8 -b16 + configs: -DDBG_FLUSH_STALL + shape: + cores: 4 + l2cache: true +- id: flush_coherence-stress-2 + via: blackbox + drivers: + - rtlsim + app: flush_coherence + args: -n2048 -r8 -b16 + configs: -DDBG_FLUSH_STALL + shape: + clusters: 2 + cores: 2 + l2cache: true + l3cache: true + tier: full +# Cores sharing a dcache: a lost lane store is only visible when it belongs to +# the core answered last, so this shape needs more rounds to expose it. +- id: flush_coherence-stress-3 + via: blackbox + drivers: + - rtlsim + app: flush_coherence + args: -n2048 -r24 -b16 + configs: -DDBG_FLUSH_STALL -DVX_CFG_SOCKET_SIZE=2 + shape: + cores: 4 + l2cache: true + tier: full diff --git a/docs/assets/img/cache_flush_sequence.svg b/docs/assets/img/cache_flush_sequence.svg index 780747378b..843f55ec28 100644 --- a/docs/assets/img/cache_flush_sequence.svg +++ b/docs/assets/img/cache_flush_sequence.svg @@ -1,4 +1,4 @@ - + @@ -22,6 +22,7 @@ .cm { font-family:"DejaVu Sans Mono", monospace; font-size:10px; fill:#1b2733; } .act { fill:#e8f0f7; stroke:#2e5e82; stroke-width:1.3; } .actg { fill:#f4faf4; stroke:#3a7d44; stroke-width:1.3; } + .acty { fill:#fdf6e3; stroke:#b58900; stroke-width:1.3; } .note { fill:#fbfcfd; stroke:#5a7184; stroke-width:1.2; } .loop { fill:none; stroke:#a03020; stroke-width:1.5; stroke-dasharray:7 4; } .loopl{ fill:#fbeeea; stroke:#a03020; stroke-width:1.3; } @@ -37,52 +38,51 @@ .lbr { fill:#a03020; font-size:11px; font-weight:bold; } - - Cache flush — DCR read to last writeback drained - One flush primitive exists: whole-cache invalidate-and-writeback. It is triggered by a DCR READ, carried as an ordinary MemReq with attr.is_flush, and it freezes the cache for its whole sweep. + + Cache flush — broadcast write to the last core's completion read + One flush primitive exists: whole-cache invalidate-and-writeback, requested by a flush token (a MemReq with attr.is_flush). Every ordered path carries its own token, and a cache answers a token only after flushing the next level. host / CP VX_dcr_data - VX_dcr_flush + VX_dcr_flush (per lane) VX_cache_init VX_cache_flush (per bank) next level - - - - - - + + + + + + - dcr_read(VX_DCR_BASE_CACHE_FLUSH, core_id) + dcr_write(VX_DCR_BASE_CACHE_FLUSH) - latch flush_pending_r - a DCR READ, not a write — the read's - response is what acknowledges the flush + every core: flush_pending_r = 1 + a write reaches all cores at once and has no + response, so all cores flush concurrently - dcr_flush_if.req = 1 (level-held) + dcr_flush_if.req = 1 (level-held, once stores drain) - - synthesize a degenerate MemReq: + a flush token per buffered dcache lane: rw=0, addr=0, data=0, byteen=0 attr = 1 << MEM_ATTR_FLUSH_OFFS - the flush rides port 0 as ordinary cache traffic + each token behind its own lane's stores - LOCK EVERY INPUT PORT + BLOCK EVERY INPUT input_enable = ~flush_req_enable - || lock_released[i] - valid=0 downstream, ready=0 upstream + while any token is visible at any input: + valid=0 downstream, ready=0 upstream STATE_WAIT1: drain the @@ -115,74 +115,89 @@ write-through: valid cleared, zero writebacks ever - writeback - - one dirty sector - per beat + writeback (on its bank's port) flush_ready gates the counter, so the walk pauses whenever the memory-request queue is near full. A write-through bank never gates on it — it emits no writebacks at all. - - STATE_WAIT2 — every bank EXCEPT bank 0 - Banks 1..N−1 wait for bank_empty. Bank 0 skips to - DONE, so its request — the one that propagates the - flush downstream — is the last to be issued. + + STATE_WAIT2 — every bank + waits for bank_empty: its write-backs must leave + the bank before the tokens go out behind them + + + flush_end — one pulse per bank + + + STATE_WAIT2 done: the tokens visible NOW are + this walk's; a later token starts the next walk + + + + STATE_FORWARD (FLUSH_FORWARD=1) + wait mem_req_empty: the crossbar has drained + + + one flush token on EVERY memory port, behind that port's write-backs - - flush_end — one pulse per bank + + same sequence one level down: one walk + for every core's tokens, then answers + each port's token - - STATE_WAIT2: accumulate until - flush_done == {NUM_BANKS{1'b1}} + + one response per port — recognized by phase (cache quiescent) - - release ONLY the originating port; its MemReq acks + + + STATE_RESPOND: each token answered locally - - dcr_flush_if.done — LEVEL-HELD until req drops + + done — all lanes answered; LEVEL-HELD - - DCR response — the flush read finally returns + + dcr_read(CACHE_FLUSH, cid), one core at a time + + answered when that core is done (at once if already) - - - - Core traffic has arbitration PRIORITY - The 2:1 arb puts core_bus_if on input 0, not the - flush. The flush waits until the LSU goes quiet — - which it does, because the host only issues a flush - after busy drops. STICKY hardens the 1-cycle gaps. - - - Why done is level-held rather than a pulse - One core's flush fans out to the dcache, the icache, and — with graphics enabled — the - socket and cluster gfx caches. Each leg finishes at a different time. Holding every - done high until req drops lets the parent simply AND them together, race-free. - A pulse would need a latch at every junction and would drop legs that finished early. - - - What the lock buys - Once locked, no core request can reach any bank pipeline. That single fact is what lets - STATE_FLUSH assume the only in-flight entries are its own walk, and it is why WAIT2's - bank_empty check need not re-test mshr_empty — nothing can refill the MSHR. - The cost: the cache is effectively OFF for the whole walk. Every other warp's loads and - stores stall. Flush is linear in cache size and is meant to be infrequent. - - - Walk length - write-through: LINES_PER_BANK - write-back: LINES_PER_BANK x NUM_WAYS - Write-through clears all ways in parallel (no way_en gate), so one pass over the lines - suffices. Write-back must visit each (way, line) separately because each dirty line - emits its own memory request. All banks walk concurrently. - - - Reset reuses the same machinery - The FSM's reset state is STATE_INIT, not STATE_IDLE: it walks every line clearing valid - bits, so the tag SRAM needs no async reset — which is what makes it map cleanly to BRAM. - A flush arriving during init is latched and answered immediately: init already invalidated - everything, so the init walk stands in for the flush walk. + + Why one token per memory port + Each port is a separately buffered path to the next level, and nothing orders one port's + writes against another port's token. A single token could reach the next level while another + port's last writes are still queued in front of it: that level's walk would release the token + and the writes would land after the walk, still dirty. A token behind each port's own data + is answered only once that data is in. The same rule gives one token per dcache lane. + The no-ack equivalent of waiting for store acknowledgements on every path. + + + Why done is level-held rather than a pulse + One core's flush fans out to every dcache lane, the icache, and — with graphics enabled — the + socket and cluster gfx caches. Each leg finishes at a different time. Holding every + done high until req drops lets the parent simply AND them together, race-free. + A pulse would need a latch at every junction and would drop legs that finished early. + + + What the block buys + While any token is visible, no core request can reach any bank pipeline. That is what lets + STATE_FLUSH assume the only in-flight entries are its own walk, makes every memory response + in STATE_FORWARD a token response, and makes a late token safe: it can only become visible + once all of its input's earlier requests were accepted, before the walk. + The cost: the cache is effectively OFF for the walk. Flush is linear in cache size. + + + Walk length + write-through: LINES_PER_BANK + write-back: LINES_PER_BANK x NUM_WAYS + Write-through clears all ways in parallel (no way_en gate), so one pass over the lines + suffices. Write-back must visit each (way, line) separately because each dirty line + emits its own memory request. All banks walk concurrently. + + + Reset reuses the same machinery + The FSM's reset state is STATE_INIT, not STATE_IDLE: it walks every line clearing valid + bits, so the tag SRAM needs no async reset — which is what makes it map cleanly to BRAM. + A flush arriving during init is latched and answered immediately: init already invalidated + everything, so the init walk stands in for the flush walk. diff --git a/docs/assets/img/cache_flush_tree.svg b/docs/assets/img/cache_flush_tree.svg index 8da61c02cd..aac49bd251 100644 --- a/docs/assets/img/cache_flush_tree.svg +++ b/docs/assets/img/cache_flush_tree.svg @@ -1,4 +1,4 @@ - + @@ -6,9 +6,6 @@ - - - - The flush fan-out tree — one DCR read, every cache + The flush fan-out tree — one broadcast write, every cache Solid blue = req fans DOWN. Dashed green = done ANDs back UP. Every done is level-held until req drops, which is what makes the AND race-free. - - VX_dcr_data - dcr_read(CACHE_FLUSH, cid) - holds req until done comes back + + VX_dcr_data (per core) + dcr_write(CACHE_FLUSH): all cores start + dcr_read(CACHE_FLUSH, cid): report one + holds req until done comes back - + - + VX_core — the fan-out point dcr_flush_if - - - req + + + req && store_drained - - VX_dcr_flush (dcache) - injects the synthetic MemReq onto - dcache port 0 only - VX_mem_unit.sv — i==0 && j==0 + + VX_dcr_flush per buffered lane + lane 0 of a private dcache, else every + lane. Lane dones AND together + VX_mem_unit.sv - - dcache - the lock freezes ALL its ports - - - - ORDERING CONSTRAINT - icache.req = dcr_flush_if.req - && dcache.done - The icache flush is SEQUENCED - BEHIND the dcache flush. - - - - - - - VX_dcr_flush (icache) - VX_core.sv — between fetch and iMMU - - - - icache - no dirty data — retires fast - - - Why the icache waits for the dcache - Both L1s forward their flush to the shared next level, and a cache that is - flushing locks out incoming core requests for its whole sweep. - The icache carries no dirty data, so it retires almost immediately. If it ran - concurrently, its forward would arrive at the next level while the dcache is - still evicting — and that level's lock would strand the dcache's writebacks - upstream of memory. - - - Why the icache is flushed at all - A kernel re-loaded to the same virtual address after a CACHE_FLUSH would - otherwise execute stale lines from the previous launch. The host issues - CMD_CACHE_FLUSH after every kernel, so this is the instruction-side half - of that contract. + + dcache + walks, then one token per + memory port to the next level + + + + req (concurrent with the dcache) + + + VX_dcr_flush (icache) + VX_core.sv — between fetch and iMMU + + + + icache (FLUSH_FORWARD=0) + invalidates locally; no forward + + + + Why a token per write path + A token only orders the requests ahead of it on its own path. Separate paths + into a cache can be buffered independently (a shared dcache's arbiter buffers + each lane), so a store on one lane could still be queued when another lane's + token arrives. The same rule gives one token per memory port between levels. + + + Why the icache flushes, and why only locally + A kernel re-loaded to the same virtual address would otherwise execute stale + lines from the previous launch. The icache holds no dirty data, and the dcache's + flush already walks and invalidates every shared level the icache reads through, + so the icache's flush ends at the icache. - - EXT_GFX_ANY_ENABLE only + + EXT_GFX_ANY_ENABLE only VX_socket — OR of core reqs socket_flush_req = |per_core_req - + tcache (TEX) - + rtcache (RTU) - + VX_cluster — OR of socket reqs cluster_flush_if.req - - rcache (raster) - - ocache (output merger) - - Sockets fire their reqs - sequentially; the shared - resource flushes once per - req. Later reqs see an - already-empty cache, so the - redundant sweeps are cheap, - not incorrect. - - rcache/ocache are cluster- - level, inside VX_graphics; - tcache/rtcache are socket- - level. + + rcache (raster, 1 token) + + ocache (1 token / OM input) + + tcache, rtcache and rcache + flush once for all cores; + the OR drops when the + last core finishes. + The ocache flushes once the + OM has drained, for the + cluster flush and for each + dcache token a steer holds + on its way to the L2. + tcache/rtcache: socket; + rcache/ocache: cluster. - - The completion AND tree + + The completion AND tree - VX_cluster: per_socket.done = cluster_flush_if.done - VX_socket: per_core.done = cluster_flush_if.done - & tcache.done - & rtcache.done - VX_core: dcr_flush_if.done = dcache.done - & icache.done - & cluster_flush_if.done + VX_cluster: per_socket.done = cluster_flush_if.done + VX_socket: per_core.done = cluster_flush_if.done + & tcache.done + & rtcache.done + VX_core: dcr_flush_if.done = dcache.done + & icache.done + & cluster_flush_if.done + VX_mem_unit: dcache.done = &lane_flush_done - Only when every leg reports done does VX_dcr_data return the DCR read - and drop req — which re-arms every VX_dcr_flush for the next flush. + When every leg reports done, VX_dcr_data drops req (re-arming every + VX_dcr_flush) and answers this core's flush read, at once if it came first. - - - done + + + done - - + + diff --git a/docs/designs/cache_subsystem.md b/docs/designs/cache_subsystem.md index 0ed3e50e98..73f7e308f1 100644 --- a/docs/designs/cache_subsystem.md +++ b/docs/designs/cache_subsystem.md @@ -511,11 +511,22 @@ The same machinery is reused for reset-time tag initialization. ![Flush fan-out tree](../assets/img/cache_flush_tree.svg) -The flush is triggered by a DCR **read** of `VX_DCR_BASE_CACHE_FLUSH` — a read, -not a write, because the read's *response* is what acknowledges completion. -[VX_dcr_data.sv](../../hw/rtl/core/VX_dcr_data.sv) latches `flush_pending_r`, -holds `dcr_flush_if.req` high, and returns the DCR response only once `done` -comes back. +The command processor flushes in two phases, because the DCR response bus has +no back-pressure and two cores answering in the same cycle would lose one: + +1. one DCR **write** to `VX_DCR_BASE_CACHE_FLUSH`. Writes reach every core, so + all cores start flushing together; a write has no response; +2. a DCR **read** of the same address per core, one at a time. The read's + *response* acknowledges that core's completion, at once if it has already + finished. + +[VX_dcr_data.sv](../../hw/rtl/core/VX_dcr_data.sv) keeps three bits per core: +`flush_pending_r` (running; holds `dcr_flush_if.req` high), `flush_complete_r` +(finished, not yet reported) and `flush_read_r` (a read waiting). A write opens +a new flush and clears `flush_complete_r`, so an older completion cannot answer +it. A read with no write before it starts the flush itself, so a requester that +only reads still works. All cores' tokens reach the shared levels together, +where one walk serves them all. From `VX_core` the request fans out to six possible destinations through six `VX_dcr_flush` instances — dcache, icache, tcache, rtcache, rcache, ocache — @@ -532,18 +543,17 @@ holding every `done` high until `req` drops is what lets each parent simply AND them together race-free. A pulse would need a latch at every junction and would drop legs that finished early. -**The icache flush is sequenced behind the dcache flush:** +**The icache flush stays local** (`FLUSH_FORWARD=0`): it holds no dirty data, +and the dcache's flush already walks and invalidates every shared level the +icache reads through. So the two L1s flush concurrently, and only the dcache's +tokens reach the next level. + +The dcache flush waits for the core's stores to leave the LSU: ```systemverilog -assign dcr_flush_icache_if.req = dcr_flush_if.req && dcr_flush_dcache_if.done; +assign dcr_flush_dcache_if.req = dcr_flush_if.req && store_drained; ``` -Both L1s forward their flush to the shared next level, and a cache that is -flushing locks out incoming core requests for its whole sweep. The icache carries -no dirty data and so retires almost immediately; if it ran concurrently, its -forward would arrive at the next level while the dcache is still evicting, and -that level's lock would strand the dcache's writebacks upstream of memory. - The icache is flushed at all because a kernel re-loaded to the same virtual address after a `CACHE_FLUSH` would otherwise execute stale lines from the previous launch. The host issues `CMD_CACHE_FLUSH` after every kernel launch — see @@ -552,8 +562,8 @@ previous launch. The host issues `CMD_CACHE_FLUSH` after every kernel launch — ### 6.2 VX_dcr_flush — injecting the request `VX_dcr_flush` synthesizes a degenerate `MemReq` (`rw=0, addr=0, data=0, -byteen=0, attr = 1 << MEM_ATTR_FLUSH_OFFS`) and merges it into the cache's port 0 -stream through a 2:1 `VX_mem_bus_arb`. +byteen=0, attr = 1 << MEM_ATTR_FLUSH_OFFS`), a **flush token**, and merges it +into one cache input stream through a 2:1 `VX_mem_bus_arb`. **Core traffic sits on the priority input, not the flush.** The flush waits until upstream is quiescent — which it reliably is, because the host only issues @@ -567,41 +577,81 @@ the first completion until the initiator drops `req`. Without that latch, a shar `req` across several instances would make the fast ones re-flush forever while waiting on the slow one. -Only **port 0** of the dcache is routed through `VX_dcr_flush` -([VX_mem_unit.sv:381](../../hw/rtl/core/VX_mem_unit.sv#L381), `i==0 && j==0`). The -input lock inside `VX_cache_init` is what propagates the freeze to every other -port. +**Every buffered write path gets its own token.** A token only orders the +requests ahead of it on its own path. A private dcache takes the lanes straight +from the LSU adapter, whose drain `store_drained` already waits for, so one +token on lane 0 suffices. Otherwise the lanes are buffered independently (a +shared dcache's socket arbiter, or a disabled dcache's passthrough), so each +lane gets its own `VX_dcr_flush` ([VX_mem_unit.sv](../../hw/rtl/core/VX_mem_unit.sv)) +and their level-held dones AND together. Each OM input of the ocache likewise has its own token +([VX_graphics.sv](../../hw/rtl/VX_graphics.sv)). The read-only rcache keeps a +single token. ### 6.3 End-to-end sequence ![Flush sequence](../assets/img/cache_flush_sequence.svg) -### 6.4 VX_cache_init — the input lock +### 6.4 VX_cache_init — the flush sequencer -`VX_cache_init` is a 5-state FSM (`IDLE → WAIT1 → FLUSH → WAIT2 → DONE`) sitting -between the cache's input ports and the request crossbar. +`VX_cache_init` sits between the cache's input ports and the request crossbar. +Its FSM is `IDLE → WAIT1 → FLUSH → WAIT2 → FORWARD → RESPOND`. ```systemverilog -wire input_enable = ~flush_req_enable || lock_released[i]; +wire input_enable = ~flush_req_enable; assign core_bus_out_if[i].req_valid = core_bus_in_if[i].req_valid && input_enable; -assign core_bus_in_if[i].req_ready = core_bus_out_if[i].req_ready && input_enable; +assign core_bus_in_if[i].req_ready = take_token || (core_bus_out_if[i].req_ready && input_enable); ``` -While a flush is in flight every input port presents `valid=0` downstream and -`ready=0` upstream, so upstream stalls. Only the port that originated the flush is -unlocked (`lock_released_n = flush_req_mask`), so its `MemReq` enters the cache and -generates the acknowledging response; the rest unlock on the return to IDLE. +While any token is visible at any input, every input presents `valid=0` +downstream and `ready=0` upstream, so upstream stalls. - `STATE_WAIT1` waits for `BANK_SEL_LATENCY × NUM_BANKS` outstanding crossbar requests to drain. When there is no crossbar latency this state is bypassed and IDLE goes straight to FLUSH. - `STATE_FLUSH` is one cycle: it pulses `flush_begin` to **all** banks at once. - `STATE_WAIT2` accumulates `flush_done |= flush_end` until every bank has - reported. - -That lock is the load-bearing piece for correctness. Once engaged, no core request -can reach any bank pipeline — which is what lets the per-bank walk assume the only -in-flight entries are its own. + reported. The tokens visible at that point are the ones this walk answers; + a token that arrives later starts the next walk. +- `STATE_FORWARD` (only with `FLUSH_FORWARD=1`) flushes the next level: once + the memory-request crossbar has drained (`mem_req_empty`), the cache sends + **one token on every memory port** and waits for every port's response. The + token for port *i* takes bank *i*'s lane into the request crossbar (bank *i* + maps to port *i*), so it leaves through the same registered outputs as the + write-backs ahead of it. +- `STATE_RESPOND` answers the tokens one per cycle. Each answer carries the + token's own tag, so upstream arbiters route it back, and takes bank 0's lane + into the response crossbar whenever bank 0 has no response of its own; the + token is consumed at its input as its answer is sent. Then the inputs reopen. + +**Why a token per memory port.** Each port is a separate, separately buffered +path to the next level, and nothing orders one port's data against another +port's token. A token sent on one port only could reach the next level while +another port's last writes are still queued in front of it; that level's walk +would release the token, and the writes would enter after the walk and stay +dirty. With one token per port, each token sits behind its own port's data in +that port's FIFO path, so the next level answers it only once that data is in. + +**Why a late token is safe.** A token becomes visible only at the head of its +input. Once any token is visible, every input is blocked, so a token that +appears during the walk had all of its input's earlier requests accepted before +the walk started. + +**Recognizing the token responses.** During `STATE_FORWARD` the cache is +quiescent: inputs blocked, MSHRs empty, no misses outstanding, and writes get no +responses. So every response on a memory port is a token response; they are +recognized by phase, with no extra tag bit. Non-cacheable requests are steered +around the cache by `VX_cache_bypass` and never reach this interface. + +**IO ordering.** `VX_cache_bypass` merges the cache's memory requests with +non-cacheable (IO) requests in a round-robin per-port arbiter, and the IO path +is unbuffered up to it. An IO request waiting there holds its own input, so the +token behind it on that input is not yet visible: a later walk answers it, and +that walk's tokens follow the IO request. A private dcache's lanes 1+ carry no +token, and `store_drained` waits for them before the core injects its token. + +**Passthrough levels** (a disabled cache) need nothing: `VX_mem_bus_arb` maps +input *i* to output *i* mod `MEM_PORTS`, so each port's stream, token included, +stays in order on one output. ### 6.5 VX_cache_flush — the per-bank walk @@ -613,7 +663,7 @@ A 6-state FSM per bank, about 14 flip-flops: | `STATE_IDLE` | wait for `flush_begin` | | `STATE_WAIT1` | wait for `mshr_empty` **and** `bank_empty` | | `STATE_FLUSH` | counter walk; drives `flush_valid`, `flush_line`, `flush_way`; increments on `flush_ready` | -| `STATE_WAIT2` | **every bank except bank 0** waits for `bank_empty` | +| `STATE_WAIT2` | wait for `bank_empty`: the write-backs must leave the bank before the tokens go out | | `STATE_DONE` | one-cycle `flush_end` pulse | **The counter width depends on the write policy:** @@ -627,18 +677,6 @@ since `do_flush = flush && (!WRITEBACK || way_en)` drops the way gate. Write-bac walks each `(way, line)` separately because each dirty line emits its own memory request. -**`STATE_WAIT2` is every bank *except* bank 0.** Bank 0 goes straight to DONE: - -```systemverilog -state_n = (BANK_ID == 0) ? STATE_DONE : STATE_WAIT2; -``` - -Bank 0 is the canonical egress that propagates the flush downstream to lower -levels, and that request must be issued **last**. The other banks therefore hold -their `flush_end` until their own writebacks have drained; since `VX_cache_init` -waits for the AND of all `flush_end`, bank 0's downstream forward cannot be -overtaken. - `flush_ready = flush_grant && ~(WRITEBACK && mreq_queue_alm_full) && ~pipe_stall` gates the counter, so under write-back with a near-full egress queue the walk pauses until it drains. @@ -676,8 +714,9 @@ tag/data write ports and the existing memory-request queue. | Component | Storage | |---|---| | `VX_cache_flush` | 3 bits state, 1 bit pending, ~10 bits counter — ~14 FF/bank | -| `VX_cache_init` | 3 bits state, `NUM_BANKS`-bit `flush_done`, `NUM_REQS`-bit `lock_released`, UUID register | -| `VX_dcr_flush` | 2 bits (`flush_inflight_r`, `flush_done_r`) + a 2:1 arbiter | +| `VX_cache_init` | 3 bits state, `NUM_BANKS`-bit `flush_done`, 2 × `MEM_PORTS`-bit token tracking, `NUM_REQS`-bit answer tracking, UUID register | +| `VX_dcr_flush` | 2 bits (`flush_inflight_r`, `flush_done_r`) + a 2:1 arbiter, per write path | +| `VX_cache` | a request-crossbar in-flight counter (forwarding caches only) and 2:1 muxes on bank 0's response lane and on the first `MEM_PORTS` request lanes | | tag / data RAM | **no extra storage** | | mreq queue | **no extra entries** | @@ -704,18 +743,20 @@ primitive. ### 6.8 Correctness invariants -1. **Atomicity against normal traffic.** Once a flush is in flight no core request - reaches a bank pipeline — enforced by the `VX_cache_init` input lock. +1. **Atomicity against normal traffic.** While any token is visible no core + request reaches a bank pipeline — enforced by the `VX_cache_init` input block. 2. **The bank is fully quiesced before the walk starts.** `STATE_WAIT1` waits for `mshr_empty && bank_empty` — both, not just the MSHR. Otherwise a fill could install a fresh line behind the walk pointer and survive the flush. 3. **Reset-time valid bits are zero** without async reset on the tag SRAM, via the `STATE_INIT` walk. This is what lets the tag array map cleanly to BRAM. The bank's highest-priority `init_valid` masks every other source while it runs. -4. **All banks finish before the cache acks** — `VX_cache_init` waits for the AND - of `flush_end` across all banks before unlocking the originating input. -5. **Bank 0's downstream forward goes out last** — banks 1..N−1 hold `flush_end` - until `bank_empty`. +4. **All banks finish before the next level is flushed** — `VX_cache_init` + waits for the AND of `flush_end` across all banks, and every bank holds + `flush_end` until `bank_empty`. +5. **A token is answered only after the next level has each port's data** — + with `FLUSH_FORWARD`, one token per memory port goes out once the request + crossbar has drained, and every port must answer before `STATE_RESPOND`. 6. **The init walk is one-shot per reset.** `flush_pending_r` only records a `flush_begin` arriving *during* init; it never re-enters `STATE_INIT`. @@ -724,14 +765,15 @@ Edge cases: - **Flush during init** → `flush_pending_r` records it, and `STATE_DONE` fires as soon as init completes. The init walk stands in for the flush walk, which is correct because init invalidated everything anyway. -- **Multiple ports racing the flush flag** → the loop in `STATE_IDLE` counts down, - so the last write wins and it latches the lowest-indexed one's UUID; all flagged - ports unlock together at the end. +- **Tokens on several inputs** → the loop in `STATE_IDLE` counts down, so it + latches the lowest-indexed one's UUID. Every token visible at the end of the + walk is answered, one per cycle through bank 0's response lane (behind any + bank response); a token that arrives later starts the next walk. - **Flush during a fill-forward drain** is asserted impossible (`RUNTIME_ASSERT(~(flush_fire && fwd_pending))`) — `STATE_WAIT1`'s `mshr_empty` precondition already excludes it. - **`STATE_WAIT2`'s `bank_empty` does not re-check `mshr_empty`.** It does not need - to: WAIT1 established it and the input lock guarantees nothing can refill. + to: WAIT1 established it and the input block guarantees nothing can refill. ### 6.9 Why flush is not the substrate for line-granular invalidation diff --git a/docs/designs/command_processor.md b/docs/designs/command_processor.md index 118c5dbdcb..0ad30a8011 100644 --- a/docs/designs/command_processor.md +++ b/docs/designs/command_processor.md @@ -129,7 +129,7 @@ come from `cmd_size_bytes` | `CMD_FENCE` | 0x07 | 8 | — | yes | yes | Ordering barrier (retires as a NOP today, §10) | | `CMD_EVENT_SIGNAL` | 0x08 | 20 | EVENT | yes | yes | Write a counter slot | | `CMD_EVENT_WAIT` | 0x09 | 28 | EVENT | yes | yes | Spin until a counter satisfies a compare | -| `CMD_CACHE_FLUSH` | 0x0A | 12 | DCR | yes | yes | Per-core cache flush sweep | +| `CMD_CACHE_FLUSH` | 0x0A | 12 | DCR | yes | yes | Cache flush: broadcast start, per-core completion | | `CMD_LAUNCH_QMD` | 0x0B | 12 | KMU | **no** | yes | Atomic launch: KMU descriptor read from memory (§2.3) | | `CMD_DRAW` | 0x0C | 12 | KMU | **no** | yes | Device-orchestrated graphics draw (§8.1) | @@ -470,10 +470,15 @@ unconditionally on every backend. barrier** — the property §8.1 depends on. - **DCR proxy** ([`VX_cp_dcr_proxy.sv`](../../hw/rtl/cp/VX_cp_dcr_proxy.sv)) — `IDLE→REQ→(WAIT_RSP)→DONE`. Writes complete at `REQ`; reads wait for - `dcr_rsp_valid`. `CMD_CACHE_FLUSH` re-enters `REQ` once per core, - sweeping a DCR *read* of `VX_DCR_BASE_CACHE_FLUSH` with the target core - index in `dcr_req_data[15:0]`, and retires only once the last core's - flush completes ([`:105-134`](../../hw/rtl/cp/VX_cp_dcr_proxy.sv#L105)). + `dcr_rsp_valid`. `CMD_CACHE_FLUSH` first passes through `START`: one DCR + *write* of `VX_DCR_BASE_CACHE_FLUSH`, which reaches every core and starts + all their flushes at once. It then re-enters `REQ` once per core, sweeping + a DCR *read* of the same address with the target core index in + `dcr_req_data[15:0]`; each read is answered when that core's flush has + completed, and the command retires after the last one. The DCR response + bus has no back-pressure, so the reads stay one at a time; the flushes + themselves overlap and share one walk of the shared caches + ([cache_subsystem.md §6](cache_subsystem.md#6-flush-architecture)). - **DMA** ([`VX_cp_dma.sv`](../../hw/rtl/cp/VX_cp_dma.sv)) — dual-port multi-beat bursts in ≤4 KB INCR chunks (`MAX_BURST` = 64 × 64 B beats), each chunk clipped so no burst crosses a 4 KB boundary on either diff --git a/docs/designs/lsu_pipeline_design.md b/docs/designs/lsu_pipeline_design.md index c1d19bbeea..7ba07a2124 100644 --- a/docs/designs/lsu_pipeline_design.md +++ b/docs/designs/lsu_pipeline_design.md @@ -318,7 +318,7 @@ the DXA writes and the tensor unit's reads. |---|---| | `VX_mem_coalescer` | merges word-sized lanes onto data-cache-word channels; lanes addressing the same cache word become one request | | `VX_lsu_adapter` | turns the lane bundle into one memory bus per channel; registers the request boundary | -| `VX_dcr_flush` | injects the host's cache-flush request, on channel 0 of block 0 | +| `VX_dcr_flush` | injects the host's cache-flush token on lane 0 of a private dcache, and on every lane otherwise (a shared dcache's socket arbiter, or a disabled dcache's passthrough, buffers each lane on its own) | The coalescer exists when there is more than one lane and the data-cache word is wider than an LSU word; otherwise the path is a pass-through that diff --git a/docs/designs/output_merger_architecture.md b/docs/designs/output_merger_architecture.md index 5c4d7adecf..69a4ecff43 100644 --- a/docs/designs/output_merger_architecture.md +++ b/docs/designs/output_merger_architecture.md @@ -183,9 +183,21 @@ bit arrives at the cluster untouched. One steer per socket→L2 trunk input (`L2_SOCKET_REQS = NUM_SOCKETS × L1_MEM_PORTS` of them, in [`VX_graphics.sv`](../../hw/rtl/VX_graphics.sv)). The steer demuxes on the single `is_addr_om` bit: aperture writes go to the -OM leg (registered), everything else passes **combinationally** to L2 (the -socket already registers its outgoing bus, and a skid here would sit on the -critical L1→L2 path). +OM leg, everything else to L2; each leg presents a registered master +interface (`OUT_BUF`). + +**Flush ordering: the steer is a fork.** A flush token from the socket (a +dcache's kernel-boundary flush or a fence) bound for L2 is held at the steer +until the OM side has been flushed. When the token reaches the steer, every +aperture store its trunk carried before it is already on the OM leg. A +controller in `VX_graphics` serves the requests present when a round starts +(the held tokens and the cluster flush), waits until no complete fragment is +left in the OM (steer OM legs, ingress record stages and OM cores idle), +flushes the ocache, and then releases those tokens to L2. A half-received +two-word record is not waited for: its depth beat can be queued behind a held +token, and the store it belongs to is not one the token orders. So a flush covers the OM writes of the aperture stores +issued before it, whatever `busy` reports while those stores are in transit +between the dcache and the ingress (a span no busy signal counts). **Placement is the deadlock argument.** The join is the last point where a request has left the socket but has not touched anything L2-owned. A full OM diff --git a/hw/rtl/VX_cluster.sv b/hw/rtl/VX_cluster.sv index 86fc979410..92c3cb4e23 100644 --- a/hw/rtl/VX_cluster.sv +++ b/hw/rtl/VX_cluster.sv @@ -292,8 +292,8 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; `ifdef EXT_GFX_ANY_ENABLE // OR all socket flush reqs together; broadcast .done to every socket. - // Sockets fire their reqs sequentially; the shared resource flushes - // once per req (subsequent reqs see an already-empty cache). + // The cores start their flushes together, so the read-only graphics caches + // flush once for all of them; the OR drops when the last core has finished. VX_dcr_flush_if cluster_flush_if(); VX_dcr_flush_if per_socket_cluster_flush_if [NUM_SOCKETS](); wire [NUM_SOCKETS-1:0] per_socket_cluster_flush_req; diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index d836c5a760..6a09fcfbea 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -1540,9 +1540,8 @@ package VX_gpu_pkg; localparam TCACHE_TAG_ID_BITS = (`CLOG2(`VX_CFG_TEX_MEM_QUEUE_SIZE) + TCACHE_BATCH_SEL_BITS); // Core-side (tex_core's cache_bus_if) tag width. localparam TCACHE_TAG_WIDTH = (UUID_WIDTH + TCACHE_TAG_ID_BITS); - // Cache-side tag width = core-side + 1 bit reserved by VX_dcr_flush so a - // host-issued CACHE_FLUSH DCR-read can inject a synthetic flush request - // onto port 0 without aliasing real requests. + // Cache-side tag width = core-side + 1 bit reserved by VX_dcr_flush, so + // a flush token can be injected without aliasing real requests. localparam TCACHE_BUS_TAG_WIDTH = (TCACHE_TAG_WIDTH + 1); localparam TCACHE_MEM_DATA_WIDTH = (TCACHE_LINE_SIZE * 8); localparam TCACHE_MEM_PORTS = 1; diff --git a/hw/rtl/VX_graphics.sv b/hw/rtl/VX_graphics.sv index 64510dba1b..a2a27fa7ed 100644 --- a/hw/rtl/VX_graphics.sv +++ b/hw/rtl/VX_graphics.sv @@ -263,6 +263,8 @@ module VX_graphics import VX_gpu_pkg::* .WRITEBACK (0), .DIRTY_BYTES (0), .NC_ENABLE (0), + // read-only: no dirty data, and the dcache's flush walks the shared levels + .FLUSH_FORWARD (0), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3) ) rcache ( @@ -297,6 +299,7 @@ module VX_graphics import VX_gpu_pkg::* om_dcrs_t om_core_dcrs [`VX_CFG_NUM_OM_CORES]; wire [L2_SOCKET_REQS-1:0] om_ingress_busy; + wire [L2_SOCKET_REQS-1:0] om_ingress_pending; // The DCRs are broadcast, so every OM core decodes identical values; the // ingresses read the aperture fields from core 0. om_dcrs_t om_dcrs; @@ -321,6 +324,11 @@ module VX_graphics import VX_gpu_pkg::* .TAG_WIDTH (L2_TAG_WIDTH) ) om_aperture_bus_if [L2_SOCKET_REQS](); + // Flush tokens are held at the steers until the OM side is flushed (below). + wire [L2_SOCKET_REQS-1:0] steer_flush_req; + reg [L2_SOCKET_REQS-1:0] steer_flush_ack_r; + wire [L2_SOCKET_REQS-1:0] steer_om_pending; + for (genvar i = 0; i < L2_SOCKET_REQS; ++i) begin : g_om_steer VX_om_steer #( .INSTANCE_ID (`SFORMATF(("cluster%0d-om-steer%0d", CLUSTER_ID, i))), @@ -328,11 +336,14 @@ module VX_graphics import VX_gpu_pkg::* .TAG_WIDTH (L2_TAG_WIDTH), .OUT_BUF (3) ) om_steer ( - .clk (clk), - .reset (reset), - .bus_in_if (socket_mem_bus_if[i]), - .l2_out_if (l2_out_bus_if[i]), - .om_out_if (om_aperture_bus_if[i]) + .clk (clk), + .reset (reset), + .bus_in_if (socket_mem_bus_if[i]), + .l2_out_if (l2_out_bus_if[i]), + .om_out_if (om_aperture_bus_if[i]), + .flush_req (steer_flush_req[i]), + .flush_ack (steer_flush_ack_r[i]), + .om_pending (steer_om_pending[i]) ); end @@ -344,12 +355,13 @@ module VX_graphics import VX_gpu_pkg::* .NUM_LANES (OM_CORE_LANES), .OUT_BUF (3) ) om_ingress ( - .clk (clk), - .reset (reset), - .dcrs (om_dcrs), - .mem_bus_if (om_aperture_bus_if[i]), - .om_bus_if (om_arb_in_if[i]), - .busy (om_ingress_busy[i]) + .clk (clk), + .reset (reset), + .dcrs (om_dcrs), + .mem_bus_if (om_aperture_bus_if[i]), + .om_bus_if (om_arb_in_if[i]), + .busy (om_ingress_busy[i]), + .frag_pending (om_ingress_pending[i]) ); end @@ -424,37 +436,106 @@ module VX_graphics import VX_gpu_pkg::* .TAG_WIDTH (OCACHE_BUS_TAG_WIDTH) ) ocache_flushable_bus_if [OCACHE_NUM_INPUTS * OCACHE_NUM_REQS] (); + // OM-side flush. Each round serves the requests present when it starts: the + // steers holding a flush token (every aperture store their trunk carried + // before it is past the steer) and the cluster flush. It waits until no + // complete fragment is left in the OM, so the ocache's flush tokens queue + // behind its writes, then flushes the ocache and releases those requesters. + // A half-received two-word record is not waited for: its depth beat can sit + // behind a held token, and it belongs to a store the token does not order. + localparam OMF_IDLE = 0; + localparam OMF_DRAIN = 1; + localparam OMF_FLUSH = 2; + localparam OMF_REARM = 3; + VX_dcr_flush_if ocache_flush_if(); - assign ocache_flush_if.req = cluster_flush_if.req; + + reg [1:0] omf_state_r; + reg [L2_SOCKET_REQS-1:0] omf_steers_r; // steers this round answers + reg omf_cluster_r; // this round answers the cluster flush + reg omf_cluster_done_r; + wire omf_cluster_req = cluster_flush_if.req && ~omf_cluster_done_r; + wire omf_drained = ~(| om_busy_w) && ~(| om_ingress_pending) && ~(| steer_om_pending); + + always @(posedge clk) begin + if (reset) begin + omf_state_r <= OMF_IDLE; + omf_steers_r <= '0; + omf_cluster_r <= 1'b0; + omf_cluster_done_r <= 1'b0; + steer_flush_ack_r <= '0; + end else begin + steer_flush_ack_r <= '0; + if (~cluster_flush_if.req) begin + omf_cluster_done_r <= 1'b0; + end + case (omf_state_r) + OMF_IDLE: begin + if ((| steer_flush_req) || omf_cluster_req) begin + omf_steers_r <= steer_flush_req; + omf_cluster_r <= omf_cluster_req; + omf_state_r <= OMF_DRAIN; + end + end + OMF_DRAIN: begin + if (omf_drained) begin + omf_state_r <= OMF_FLUSH; + end + end + OMF_FLUSH: begin + if (ocache_flush_if.done) begin + steer_flush_ack_r <= omf_steers_r; + if (omf_cluster_r) begin + omf_cluster_done_r <= 1'b1; + end + omf_state_r <= OMF_REARM; + end + end + default: begin // OMF_REARM: the injectors re-arm once req drops + if (~ocache_flush_if.done) begin + omf_state_r <= OMF_IDLE; + end + end + endcase + end + end + assign ocache_flush_if.req = (omf_state_r == OMF_FLUSH); // OM-core flush chain carries the OM tag + 1 flush bit. When early-Z widens // the shared bus tag (a wider early-Z requester), the flushed OM bus is - // zero-extended up to OCACHE_BUS_TAG_WIDTH below. + // zero-extended up to OCACHE_BUS_TAG_WIDTH below. Each OM input is a + // separately ordered write path into the ocache, so each carries its own + // flush token behind its writes. + localparam OCACHE_OM_INPUTS = `VX_CFG_NUM_OM_CORES * OCACHE_NUM_REQS; localparam OCACHE_OM_FLUSH_TAG_WIDTH = OCACHE_TAG_WIDTH + 1; VX_mem_bus_if #( .DATA_SIZE (OCACHE_WORD_SIZE), .TAG_WIDTH (OCACHE_OM_FLUSH_TAG_WIDTH) - ) ocache_om_flush_bus_if [1] (); + ) ocache_om_flush_bus_if [OCACHE_OM_INPUTS] (); - VX_dcr_flush #( - .WORD_SIZE (OCACHE_WORD_SIZE), - .TAG_WIDTH (OCACHE_TAG_WIDTH), - .REQ_OUT_BUF (3) // register cache-request master boundary; rsp registered by cache CORE_OUT_BUF - ) ocache_dcr_flush ( - .clk (clk), - .reset (reset), - .dcr_flush_if (ocache_flush_if), - .core_bus_if (ocache_bus_if[0]), - .cache_bus_if (ocache_om_flush_bus_if[0]) - ); + VX_dcr_flush_if ocache_om_flush_if [OCACHE_OM_INPUTS] (); + wire [OCACHE_OM_INPUTS-1:0] ocache_om_flush_done; + + for (genvar i = 0; i < OCACHE_OM_INPUTS; ++i) begin : g_ocache_om_flush + assign ocache_om_flush_if[i].req = ocache_flush_if.req; + assign ocache_om_flush_done[i] = ocache_om_flush_if[i].done; - `ASSIGN_VX_MEM_BUS_IF_EX (ocache_flushable_bus_if[0], ocache_om_flush_bus_if[0], - OCACHE_BUS_TAG_WIDTH, OCACHE_OM_FLUSH_TAG_WIDTH, 0); + VX_dcr_flush #( + .WORD_SIZE (OCACHE_WORD_SIZE), + .TAG_WIDTH (OCACHE_TAG_WIDTH), + .REQ_OUT_BUF (3) // register cache-request master boundary; rsp registered by cache CORE_OUT_BUF + ) ocache_dcr_flush ( + .clk (clk), + .reset (reset), + .dcr_flush_if (ocache_om_flush_if[i]), + .core_bus_if (ocache_bus_if[i]), + .cache_bus_if (ocache_om_flush_bus_if[i]) + ); - for (genvar i = 1; i < `VX_CFG_NUM_OM_CORES * OCACHE_NUM_REQS; ++i) begin : g_ocache_passthru - `ASSIGN_VX_MEM_BUS_IF_EX (ocache_flushable_bus_if[i], ocache_bus_if[i], - OCACHE_BUS_TAG_WIDTH, OCACHE_TAG_WIDTH, 0); + `ASSIGN_VX_MEM_BUS_IF_EX (ocache_flushable_bus_if[i], ocache_om_flush_bus_if[i], + OCACHE_BUS_TAG_WIDTH, OCACHE_OM_FLUSH_TAG_WIDTH, 0); end + assign ocache_flush_if.done = (& ocache_om_flush_done); `ifdef VX_CFG_RASTER_EARLYZ_ENABLE // Attach the early-Z depth readers as the trailing ocache inputs. Each reader @@ -516,7 +597,7 @@ module VX_graphics import VX_gpu_pkg::* assign rcache_flush_done = 1'b1; `endif `ifdef VX_CFG_EXT_OM_ENABLE - assign ocache_flush_done = ocache_flush_if.done; + assign ocache_flush_done = omf_cluster_done_r; `else assign ocache_flush_done = 1'b1; `endif diff --git a/hw/rtl/VX_socket.sv b/hw/rtl/VX_socket.sv index c14be8a0c4..652a4509db 100644 --- a/hw/rtl/VX_socket.sv +++ b/hw/rtl/VX_socket.sv @@ -160,6 +160,8 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .WRITE_ENABLE (0), .REPL_POLICY (`VX_CFG_ICACHE_REPL_POLICY), .NC_ENABLE (0), + // read-only: no dirty data, and the dcache's flush walks the shared levels + .FLUSH_FORWARD (0), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3) ) icache ( @@ -515,6 +517,8 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .WRITEBACK (0), .DIRTY_BYTES (0), .NC_ENABLE (0), + // read-only: no dirty data, and the dcache's flush walks the shared levels + .FLUSH_FORWARD (0), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3) ) tcache ( @@ -641,6 +645,8 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .WRITEBACK (0), .DIRTY_BYTES (0), .NC_ENABLE (0), + // read-only: no dirty data, and the dcache's flush walks the shared levels + .FLUSH_FORWARD (0), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3) ) rtcache ( diff --git a/hw/rtl/cache/VX_cache.sv b/hw/rtl/cache/VX_cache.sv index 7982a79cd4..46659a46b4 100644 --- a/hw/rtl/cache/VX_cache.sv +++ b/hw/rtl/cache/VX_cache.sv @@ -49,6 +49,8 @@ module VX_cache import VX_gpu_pkg::*; #( parameter IS_LLC = 0, // last-level cache: banks own AMO commit + RVA reservation tracking + parameter FLUSH_FORWARD = 1, // a flush also flushes the next level + parameter AMO_ENABLE = 0 // synthesize atomic-op logic in the banks ) ( `ifdef PERF_ENABLE @@ -98,20 +100,41 @@ module VX_cache import VX_gpu_pkg::*; #( wire [`UP(UUID_WIDTH)-1:0] flush_uuid; wire [NUM_BANKS-1:0] per_bank_flush_end; wire [NUM_BANKS-1:0] per_bank_core_req_fire; + wire mem_req_empty; + wire [MEM_PORTS-1:0] mem_flush_valid; + wire [MEM_PORTS-1:0] mem_flush_ready; + wire mem_flush_rsp_phase; + wire [MEM_PORTS-1:0] mem_flush_rsp_valid; + wire flush_rsp_valid; + wire [REQ_SEL_WIDTH-1:0] flush_rsp_idx; + wire [TAG_WIDTH-1:0] flush_rsp_tag; + wire flush_rsp_ready; VX_cache_init #( - .NUM_REQS (NUM_REQS), - .NUM_BANKS (NUM_BANKS), - .TAG_WIDTH (TAG_WIDTH), - .BANK_SEL_LATENCY (`TO_OUT_BUF_REG(REQ_XBAR_BUF)) // request xbar latency + .NUM_REQS (NUM_REQS), + .NUM_BANKS (NUM_BANKS), + .MEM_PORTS (MEM_PORTS), + .FLUSH_FORWARD (FLUSH_FORWARD), + .TAG_WIDTH (TAG_WIDTH), + .BANK_SEL_LATENCY (`TO_OUT_BUF_REG(REQ_XBAR_BUF)), // request xbar latency + .REQ_SEL_WIDTH (REQ_SEL_WIDTH) ) cache_init ( - .clk (clk), - .reset (reset), - .core_bus_in_if (core_bus_if), - .core_bus_out_if (core_bus2_if), - .bank_req_fire (per_bank_core_req_fire), - .flush_begin (per_bank_flush_begin), - .flush_uuid (flush_uuid), - .flush_end (per_bank_flush_end) + .clk (clk), + .reset (reset), + .core_bus_in_if (core_bus_if), + .core_bus_out_if (core_bus2_if), + .bank_req_fire (per_bank_core_req_fire), + .flush_begin (per_bank_flush_begin), + .flush_uuid (flush_uuid), + .flush_end (per_bank_flush_end), + .mem_req_empty (mem_req_empty), + .mem_flush_valid (mem_flush_valid), + .mem_flush_ready (mem_flush_ready), + .mem_flush_rsp_phase (mem_flush_rsp_phase), + .mem_flush_rsp_valid (mem_flush_rsp_valid), + .flush_rsp_valid (flush_rsp_valid), + .flush_rsp_idx (flush_rsp_idx), + .flush_rsp_tag (flush_rsp_tag), + .flush_rsp_ready (flush_rsp_ready) ); VX_mem_bus_if #( .DATA_SIZE (SECTOR_SIZE), @@ -161,6 +184,15 @@ module VX_cache import VX_gpu_pkg::*; #( assign mem_rsp_queue_sel[i] = 0; end end + // While the flush tokens are out the cache is quiescent (inputs blocked, + // MSHRs empty), so every memory response is a token's. + wire [MEM_PORTS-1:0] mem_rsp_xbar_valid; + wire [MEM_PORTS-1:0] mem_rsp_xbar_ready; + for (genvar i = 0; i < MEM_PORTS; ++i) begin : g_mem_flush_rsp + assign mem_flush_rsp_valid[i] = mem_flush_rsp_phase && mem_rsp_queue_valid[i]; + assign mem_rsp_xbar_valid[i] = mem_rsp_queue_valid[i] && ~mem_flush_rsp_phase; + assign mem_rsp_queue_ready[i] = mem_flush_rsp_phase || mem_rsp_xbar_ready[i]; + end wire [NUM_BANKS-1:0] per_bank_mem_rsp_valid; wire [NUM_BANKS-1:0][MEM_RSP_DATAW-MEM_ARB_SEL_BITS-1:0] per_bank_mem_rsp_pdata; wire [NUM_BANKS-1:0] per_bank_mem_rsp_ready; @@ -175,10 +207,10 @@ module VX_cache import VX_gpu_pkg::*; #( ) mem_rsp_xbar ( .clk (clk), .reset (reset), - .valid_in (mem_rsp_queue_valid), + .valid_in (mem_rsp_xbar_valid), .data_in (mem_rsp_queue_data_s), .sel_in (mem_rsp_queue_sel), - .ready_in (mem_rsp_queue_ready), + .ready_in (mem_rsp_xbar_ready), .valid_out (per_bank_mem_rsp_valid), .data_out (per_bank_mem_rsp_pdata), `UNUSED_PIN (sel_out), @@ -379,6 +411,28 @@ module VX_cache import VX_gpu_pkg::*; #( for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_per_bank_core_rsp_pdata assign per_bank_core_rsp_pdata[i] = {per_bank_core_rsp_data[i], per_bank_core_rsp_tag[i]}; end + // A flush token's answer takes bank 0's lane into the response crossbar + // when the bank has no response of its own, so it leaves through the same + // registered outputs as every other response. + wire [NUM_BANKS-1:0] core_rsp_xbar_valid; + wire [NUM_BANKS-1:0][CORE_RSP_DATAW-1:0] core_rsp_xbar_pdata; + wire [NUM_BANKS-1:0][REQ_SEL_WIDTH-1:0] core_rsp_xbar_idx; + wire [NUM_BANKS-1:0] core_rsp_xbar_ready; + for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_core_rsp_xbar_in + if (i == 0) begin : g_flush_rsp + wire flush_rsp_sel = flush_rsp_valid && ~per_bank_core_rsp_valid[0]; + assign core_rsp_xbar_valid[0] = per_bank_core_rsp_valid[0] || flush_rsp_valid; + assign core_rsp_xbar_pdata[0] = flush_rsp_sel ? {`CS_WORD_WIDTH'(0), flush_rsp_tag} : per_bank_core_rsp_pdata[0]; + assign core_rsp_xbar_idx[0] = flush_rsp_sel ? flush_rsp_idx : per_bank_core_rsp_idx[0]; + assign per_bank_core_rsp_ready[0] = core_rsp_xbar_ready[0]; + assign flush_rsp_ready = core_rsp_xbar_ready[0] && ~per_bank_core_rsp_valid[0]; + end else begin : g_bank_rsp + assign core_rsp_xbar_valid[i] = per_bank_core_rsp_valid[i]; + assign core_rsp_xbar_pdata[i] = per_bank_core_rsp_pdata[i]; + assign core_rsp_xbar_idx[i] = per_bank_core_rsp_idx[i]; + assign per_bank_core_rsp_ready[i] = core_rsp_xbar_ready[i]; + end + end VX_stream_omega #( .NUM_INPUTS (NUM_BANKS), .NUM_OUTPUTS (NUM_REQS), @@ -388,10 +442,10 @@ module VX_cache import VX_gpu_pkg::*; #( ) core_rsp_xbar ( .clk (clk), .reset (reset), - .valid_in (per_bank_core_rsp_valid), - .data_in (per_bank_core_rsp_pdata), - .sel_in (per_bank_core_rsp_idx), - .ready_in (per_bank_core_rsp_ready), + .valid_in (core_rsp_xbar_valid), + .data_in (core_rsp_xbar_pdata), + .sel_in (core_rsp_xbar_idx), + .ready_in (core_rsp_xbar_ready), .valid_out (core_rsp_valid), .data_out (core_rsp_pdata), `UNUSED_PIN (sel_out), @@ -426,6 +480,64 @@ module VX_cache import VX_gpu_pkg::*; #( wire [MEM_PORTS-1:0] mem_req_tmp_valid; wire [MEM_PORTS-1:0][`UP(`LOG2UP(NUM_BANKS))-1:0] mem_req_tmp_idx; wire [MEM_PORTS-1:0] mem_req_tmp_ready; + // The flush token for port i is a read of the port's first line with the + // flush attribute. It takes bank i's lane into the request crossbar (bank i + // maps to port i), so the crossbar forms its address and it leaves through + // the same registered outputs as the write-backs ahead of it. + wire [NUM_BANKS-1:0] mem_req_xbar_valid; + wire [NUM_BANKS-1:0][MEM_REQ_DATAW-1:0] mem_req_xbar_pdata; + wire [NUM_BANKS-1:0] mem_req_xbar_ready; + wire [MEM_REQ_DATAW-1:0] mem_flush_pdata = { + `CS_LINE_SECTOR_ADDR_WIDTH'(0), + 1'b0, + SECTOR_SIZE'(0), + `CS_SECTOR_WIDTH'(0), + BANK_MEM_TAG_WIDTH'(0), + `UP(MEM_ATTR_WIDTH)'(1 << MEM_ATTR_FLUSH_OFFS) + }; + for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_mem_req_xbar_in + if (i < MEM_PORTS) begin : g_flush_req + assign mem_req_xbar_valid[i] = per_bank_mem_req_valid[i] || mem_flush_valid[i]; + assign mem_req_xbar_pdata[i] = per_bank_mem_req_valid[i] ? per_bank_mem_req_pdata[i] : mem_flush_pdata; + assign mem_flush_ready[i] = mem_req_xbar_ready[i] && ~per_bank_mem_req_valid[i]; + `RUNTIME_ASSERT (~(mem_flush_valid[i] && per_bank_mem_req_valid[i]), ("%t: %s flush token collides with a bank %0d request", $time, INSTANCE_ID, i)) + end else begin : g_bank_req + assign mem_req_xbar_valid[i] = per_bank_mem_req_valid[i]; + assign mem_req_xbar_pdata[i] = per_bank_mem_req_pdata[i]; + end + assign per_bank_mem_req_ready[i] = mem_req_xbar_ready[i]; + end + // Write-backs from different banks are not ordered with each other inside + // the crossbar, so the tokens wait until it has drained. + if (FLUSH_FORWARD != 0) begin : g_mem_req_xbar_pending + localparam XBAR_IN_W = `CLOG2(NUM_BANKS+1); + localparam XBAR_OUT_W = `CLOG2(MEM_PORTS+1); + // generous bound on what the crossbar can hold: buffered stages plus outputs + localparam XBAR_CAPACITY = 4 * NUM_BANKS * (`CLOG2(NUM_BANKS) + 1); + wire [NUM_BANKS-1:0] xbar_in_fire = mem_req_xbar_valid & mem_req_xbar_ready; + wire [MEM_PORTS-1:0] xbar_out_fire = mem_req_tmp_valid & mem_req_tmp_ready; + wire [XBAR_IN_W-1:0] xbar_in_cnt; + wire [XBAR_OUT_W-1:0] xbar_out_cnt; + `POP_COUNT(xbar_in_cnt, xbar_in_fire); + `POP_COUNT(xbar_out_cnt, xbar_out_fire); + VX_pending_size #( + .SIZE (XBAR_CAPACITY), + .INCRW (XBAR_IN_W), + .DECRW (XBAR_OUT_W) + ) xbar_pending ( + .clk (clk), + .reset (reset), + .incr (xbar_in_cnt), + .decr (xbar_out_cnt), + .empty (mem_req_empty), + `UNUSED_PIN (alm_empty), + `UNUSED_PIN (full), + `UNUSED_PIN (alm_full), + `UNUSED_PIN (size) + ); + end else begin : g_no_mem_req_xbar_pending + assign mem_req_empty = 1'b1; + end VX_stream_omega #( .NUM_INPUTS (NUM_BANKS), .NUM_OUTPUTS (MEM_PORTS), @@ -435,10 +547,10 @@ module VX_cache import VX_gpu_pkg::*; #( ) mem_req_xbar ( .clk (clk), .reset (reset), - .valid_in (per_bank_mem_req_valid), - .data_in (per_bank_mem_req_pdata), + .valid_in (mem_req_xbar_valid), + .data_in (mem_req_xbar_pdata), .sel_in (per_bank_mem_req_bid), - .ready_in (per_bank_mem_req_ready), + .ready_in (mem_req_xbar_ready), .valid_out (mem_req_tmp_valid), .data_out (mem_req_pdata), .sel_out (mem_req_tmp_idx), diff --git a/hw/rtl/cache/VX_cache_bank.sv b/hw/rtl/cache/VX_cache_bank.sv index 28576fcd39..ee64fe7937 100644 --- a/hw/rtl/cache/VX_cache_bank.sv +++ b/hw/rtl/cache/VX_cache_bank.sv @@ -286,7 +286,6 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire no_pending_req = ~pipe_inflight && mreq_queue_empty && ~amo_commit_busy; VX_cache_flush #( - .BANK_ID (BANK_ID), .CACHE_SIZE (CACHE_SIZE), .LINE_SIZE (LINE_SIZE), .NUM_BANKS (NUM_BANKS), diff --git a/hw/rtl/cache/VX_cache_cluster.sv b/hw/rtl/cache/VX_cache_cluster.sv index 886f898d62..2efb8ab859 100644 --- a/hw/rtl/cache/VX_cache_cluster.sv +++ b/hw/rtl/cache/VX_cache_cluster.sv @@ -55,6 +55,7 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( parameter MEM_OUT_BUF = 3, // Memory request output buffer parameter IS_LLC = 0, // Indicates this cache cluster is the LLC (handles AMO completion). + parameter FLUSH_FORWARD = 1, // A flush also flushes the next level. parameter AMO_ENABLE = 0 // Enable atomic memory operations. ) ( @@ -145,6 +146,7 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( .NC_ENABLE (NC_ENABLE), .PASSTHRU (PASSTHRU), .IS_LLC (IS_LLC), + .FLUSH_FORWARD (FLUSH_FORWARD), .AMO_ENABLE (AMO_ENABLE) ) cache_wrap ( `ifdef PERF_ENABLE diff --git a/hw/rtl/cache/VX_cache_flush.sv b/hw/rtl/cache/VX_cache_flush.sv index 79e5aad266..c51a5aa2cc 100644 --- a/hw/rtl/cache/VX_cache_flush.sv +++ b/hw/rtl/cache/VX_cache_flush.sv @@ -14,7 +14,6 @@ `include "VX_cache_define.vh" module VX_cache_flush import VX_gpu_pkg::*; #( - parameter BANK_ID = 0, parameter CACHE_SIZE = 1024, // Size of cache in bytes parameter LINE_SIZE = 64, // Size of line inside a bank in bytes parameter NUM_BANKS = 1, // Number of banks @@ -80,13 +79,12 @@ module VX_cache_flush import VX_gpu_pkg::*; #( end STATE_FLUSH: begin if (counter == ((2 ** CTR_WIDTH)-1) && flush_ready) begin - state_n = (BANK_ID == 0) ? STATE_DONE : STATE_WAIT2; + state_n = STATE_WAIT2; end end STATE_WAIT2: begin - // ensure the bank is empty before notifying the cache flush unit, - // because the flush request to lower caches only goes through bank0 - // and it is important that request gets send out last. + // the write-backs must leave the bank before the cache sends + // its flush tokens behind them if (bank_empty) begin state_n = STATE_DONE; end diff --git a/hw/rtl/cache/VX_cache_init.sv b/hw/rtl/cache/VX_cache_init.sv index a27ba56dc6..8159a6ba32 100644 --- a/hw/rtl/cache/VX_cache_init.sv +++ b/hw/rtl/cache/VX_cache_init.sv @@ -13,28 +13,51 @@ `include "VX_cache_define.vh" +// Flush sequencer. Flush tokens (requests with the flush attribute) at any +// input start one walk: every input is blocked, in-flight requests drain and +// every bank writes back and invalidates its lines. With FLUSH_FORWARD, the +// cache then sends one token down each memory port, behind that port's +// write-backs, and waits for every port to answer, so the next level has +// each port's data before it releases that port's token. Finally each token +// visible at the end of the walk is answered locally, one at a time through +// the cache's own response path, and the inputs reopen. +// A token that arrives during the walk is visible only after all of its +// input's earlier requests were accepted, so the walk covered them. module VX_cache_init import VX_gpu_pkg::*; #( - parameter NUM_REQS = 4, // Number of Word requests per cycle - parameter NUM_BANKS = 1, // Number of banks - parameter TAG_WIDTH = UUID_WIDTH + 1, // core request tag size - parameter BANK_SEL_LATENCY = 1 // Bank select latency + parameter NUM_REQS = 4, // Number of Word requests per cycle + parameter NUM_BANKS = 1, // Number of banks + parameter MEM_PORTS = 1, // Number of memory ports + parameter FLUSH_FORWARD = 1, // Forward the flush to the next level + parameter TAG_WIDTH = UUID_WIDTH + 1, // core request tag size + parameter BANK_SEL_LATENCY = 1, // Bank select latency + parameter REQ_SEL_WIDTH = `UP(`CLOG2(NUM_REQS)) ) ( - input wire clk, - input wire reset, - VX_mem_bus_if.slave core_bus_in_if [NUM_REQS], - VX_mem_bus_if.master core_bus_out_if [NUM_REQS], - input wire [NUM_BANKS-1:0] bank_req_fire, - output wire [NUM_BANKS-1:0] flush_begin, + input wire clk, + input wire reset, + VX_mem_bus_if.slave core_bus_in_if [NUM_REQS], + VX_mem_bus_if.master core_bus_out_if [NUM_REQS], + input wire [NUM_BANKS-1:0] bank_req_fire, + output wire [NUM_BANKS-1:0] flush_begin, output wire [`UP(UUID_WIDTH)-1:0] flush_uuid, - input wire [NUM_BANKS-1:0] flush_end + input wire [NUM_BANKS-1:0] flush_end, + // flush tokens to the next level + input wire mem_req_empty, // no write-back left between the banks and the ports + output wire [MEM_PORTS-1:0] mem_flush_valid, + input wire [MEM_PORTS-1:0] mem_flush_ready, + output wire mem_flush_rsp_phase, + input wire [MEM_PORTS-1:0] mem_flush_rsp_valid, + // answers to the requesters' tokens + output wire flush_rsp_valid, + output wire [REQ_SEL_WIDTH-1:0] flush_rsp_idx, + output wire [TAG_WIDTH-1:0] flush_rsp_tag, + input wire flush_rsp_ready ); - `UNUSED_PARAM (TAG_WIDTH) - - localparam STATE_IDLE = 0; - localparam STATE_WAIT1 = 1; - localparam STATE_FLUSH = 2; - localparam STATE_WAIT2 = 3; - localparam STATE_DONE = 4; + localparam STATE_IDLE = 0; + localparam STATE_WAIT1 = 1; + localparam STATE_FLUSH = 2; + localparam STATE_WAIT2 = 3; + localparam STATE_FORWARD = 4; + localparam STATE_RESPOND = 5; reg [2:0] state, state_n; @@ -80,52 +103,106 @@ module VX_cache_init import VX_gpu_pkg::*; #( `UNUSED_VAR (bank_req_fire) end - reg [NUM_BANKS-1:0] flush_done, flush_done_n; - wire [NUM_REQS-1:0] flush_req_mask; for (genvar i = 0; i < NUM_REQS; ++i) begin : g_flush_req_mask assign flush_req_mask[i] = core_bus_in_if[i].req_valid && core_bus_in_if[i].req_data.attr[MEM_ATTR_FLUSH_OFFS]; end wire flush_req_enable = (| flush_req_mask); - reg [NUM_REQS-1:0] lock_released, lock_released_n; + reg [NUM_BANKS-1:0] flush_done, flush_done_n; + reg fwd_go, fwd_go_n; // the crossbar drained; only tokens enter it now + reg [MEM_PORTS-1:0] fwd_sent, fwd_sent_n; + reg [MEM_PORTS-1:0] fwd_done, fwd_done_n; + reg [NUM_REQS-1:0] rsp_pending, rsp_pending_n; // token still to be answered reg [`UP(UUID_WIDTH)-1:0] flush_uuid_r, flush_uuid_n; - for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_bus_out_req - // block core requests if a flush is active, - // unless the lock for this specific request has been released - wire input_enable = ~flush_req_enable || lock_released[i]; + // answer the lowest pending input; its token stays at the input until then + wire [NUM_REQS-1:0] rsp_sel_onehot; + wire [REQ_SEL_WIDTH-1:0] rsp_sel_idx; + wire rsp_sel_valid; + VX_priority_encoder #( + .N (NUM_REQS) + ) rsp_sel ( + .data_in (rsp_pending), + .onehot_out (rsp_sel_onehot), + .index_out (rsp_sel_idx), + .valid_out (rsp_sel_valid) + ); + assign flush_rsp_valid = (state == STATE_RESPOND) && rsp_sel_valid; + assign flush_rsp_idx = rsp_sel_idx; + wire flush_rsp_fire = flush_rsp_valid && flush_rsp_ready; + + wire [NUM_REQS-1:0][TAG_WIDTH-1:0] core_bus_in_tag; + for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_bus_in_tag + assign core_bus_in_tag[i] = core_bus_in_if[i].req_data.tag; + end + VX_onehot_mux #( + .DATAW (TAG_WIDTH), + .N (NUM_REQS) + ) rsp_tag_mux ( + .data_in (core_bus_in_tag), + .sel_in (rsp_sel_onehot), + .data_out (flush_rsp_tag) + ); + + wire [NUM_REQS-1:0] dbg_stall; +`ifdef DBG_FLUSH_STALL + // Random input backpressure, a new mask every 128 cycles (each input + // stalled half the time), to widen ordering windows under test. + reg [31:0] dbg_lfsr_r; + reg [6:0] dbg_cnt_r; + reg [NUM_REQS-1:0] dbg_stall_r; + always @(posedge clk) begin + if (reset) begin + dbg_lfsr_r <= 32'hACE1_2468; + dbg_cnt_r <= '0; + dbg_stall_r <= '0; + end else begin + dbg_lfsr_r <= {dbg_lfsr_r[30:0], dbg_lfsr_r[31] ^ dbg_lfsr_r[21] ^ dbg_lfsr_r[1] ^ dbg_lfsr_r[0]}; + dbg_cnt_r <= dbg_cnt_r + 7'd1; + if (dbg_cnt_r == '0) begin + dbg_stall_r <= (NUM_REQS > 1) ? NUM_REQS'(dbg_lfsr_r) : '0; + end + end + end + assign dbg_stall = dbg_stall_r; +`else + assign dbg_stall = '0; +`endif + for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_bus_out_req + // Normal requests stay blocked while any token is visible. A token is + // consumed here when its answer is sent and never reaches the banks. + wire take_token = flush_rsp_fire && rsp_sel_onehot[i]; + wire input_enable = ~flush_req_enable && ~dbg_stall[i]; assign core_bus_out_if[i].req_valid = core_bus_in_if[i].req_valid && input_enable; assign core_bus_out_if[i].req_data = core_bus_in_if[i].req_data; - assign core_bus_in_if[i].req_ready = core_bus_out_if[i].req_ready && input_enable; + assign core_bus_in_if[i].req_ready = take_token || (core_bus_out_if[i].req_ready && input_enable); end for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_bus_in_rsp - assign core_bus_in_if[i].rsp_valid = core_bus_out_if[i].rsp_valid; - assign core_bus_in_if[i].rsp_data = core_bus_out_if[i].rsp_data; + assign core_bus_in_if[i].rsp_valid = core_bus_out_if[i].rsp_valid; + assign core_bus_in_if[i].rsp_data = core_bus_out_if[i].rsp_data; assign core_bus_out_if[i].rsp_ready = core_bus_in_if[i].rsp_ready; end - reg [NUM_REQS-1:0][`UP(UUID_WIDTH)-1:0] core_bus_out_uuid; - wire [NUM_REQS-1:0] core_bus_out_ready; - for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_bus_out_uuid + reg [NUM_REQS-1:0][`UP(UUID_WIDTH)-1:0] core_bus_in_uuid; + for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_bus_in_uuid if (UUID_WIDTH != 0) begin : g_uuid - assign core_bus_out_uuid[i] = core_bus_in_if[i].req_data.tag.uuid; + assign core_bus_in_uuid[i] = core_bus_in_if[i].req_data.tag.uuid; end else begin : g_no_uuid - assign core_bus_out_uuid[i] = 0; + assign core_bus_in_uuid[i] = 0; end end - for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_bus_out_ready - assign core_bus_out_ready[i] = core_bus_out_if[i].req_ready; - end - always @(*) begin - state_n = state; - flush_done_n = flush_done; - lock_released_n = lock_released; - flush_uuid_n = flush_uuid_r; + state_n = state; + flush_done_n = flush_done; + fwd_go_n = fwd_go; + fwd_sent_n = fwd_sent; + fwd_done_n = fwd_done; + rsp_pending_n = rsp_pending & ~(rsp_sel_onehot & {NUM_REQS{flush_rsp_fire}}); + flush_uuid_n = flush_uuid_r; case (state) // STATE_IDLE: default: begin @@ -133,7 +210,7 @@ module VX_cache_init import VX_gpu_pkg::*; #( state_n = (BANK_SEL_LATENCY != 0) ? STATE_WAIT1 : STATE_FLUSH; for (integer i = NUM_REQS-1; i >= 0; --i) begin if (flush_req_mask[i]) begin - flush_uuid_n = core_bus_out_uuid[i]; + flush_uuid_n = core_bus_in_uuid[i]; end end end @@ -151,18 +228,25 @@ module VX_cache_init import VX_gpu_pkg::*; #( // wait for all banks to finish flushing flush_done_n = flush_done | flush_end; if (flush_done_n == {NUM_BANKS{1'b1}}) begin - state_n = STATE_DONE; flush_done_n = '0; - // only release current flush requests - // and keep normal requests locked - lock_released_n = flush_req_mask; + // answer the tokens visible now; later ones start a new walk + rsp_pending_n = flush_req_mask; + state_n = FLUSH_FORWARD ? STATE_FORWARD : STATE_RESPOND; end end - STATE_DONE: begin - // wait until released flush requests are issued - // when returning to IDLE state other requests will unlock - lock_released_n = lock_released & ~core_bus_out_ready; - if (lock_released_n == 0) begin + STATE_FORWARD: begin + fwd_go_n = fwd_go | mem_req_empty; + fwd_sent_n = fwd_sent | (mem_flush_valid & mem_flush_ready); + fwd_done_n = fwd_done | mem_flush_rsp_valid; + if (fwd_done_n == {MEM_PORTS{1'b1}}) begin + fwd_go_n = 1'b0; + fwd_sent_n = '0; + fwd_done_n = '0; + state_n = STATE_RESPOND; + end + end + STATE_RESPOND: begin + if (rsp_pending_n == '0) begin state_n = STATE_IDLE; end end @@ -171,13 +255,19 @@ module VX_cache_init import VX_gpu_pkg::*; #( always @(posedge clk) begin if (reset) begin - state <= STATE_IDLE; - flush_done <= '0; - lock_released <= '0; + state <= STATE_IDLE; + flush_done <= '0; + fwd_go <= 1'b0; + fwd_sent <= '0; + fwd_done <= '0; + rsp_pending <= '0; end else begin - state <= state_n; - flush_done <= flush_done_n; - lock_released <= lock_released_n; + state <= state_n; + flush_done <= flush_done_n; + fwd_go <= fwd_go_n; + fwd_sent <= fwd_sent_n; + fwd_done <= fwd_done_n; + rsp_pending <= rsp_pending_n; end flush_uuid_r <= flush_uuid_n; end @@ -185,4 +275,9 @@ module VX_cache_init import VX_gpu_pkg::*; #( assign flush_begin = {NUM_BANKS{state == STATE_FLUSH}}; assign flush_uuid = flush_uuid_r; + // The drain counter counts the tokens too, so the first one sent must not + // take the others' valid away. + assign mem_flush_valid = {MEM_PORTS{(state == STATE_FORWARD) && (fwd_go || mem_req_empty)}} & ~fwd_sent; + assign mem_flush_rsp_phase = (state == STATE_FORWARD); + endmodule diff --git a/hw/rtl/cache/VX_cache_wrap.sv b/hw/rtl/cache/VX_cache_wrap.sv index caff8c34d3..6aa1266e7a 100644 --- a/hw/rtl/cache/VX_cache_wrap.sv +++ b/hw/rtl/cache/VX_cache_wrap.sv @@ -55,6 +55,7 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( parameter MEM_OUT_BUF = 3, // Memory request output buffer parameter IS_LLC = 0, // Set when this cache is the LLC (enables AMO completion at this level). + parameter FLUSH_FORWARD = 1, // A flush also flushes the next level. parameter AMO_ENABLE = 0 // Enable atomic operations in cache banks. ) ( @@ -147,6 +148,7 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( .CORE_OUT_BUF (BYPASS_ENABLE ? 1 : CORE_OUT_BUF), .MEM_OUT_BUF (BYPASS_ENABLE ? 1 : MEM_OUT_BUF), .IS_LLC (IS_LLC), + .FLUSH_FORWARD (FLUSH_FORWARD), .AMO_ENABLE (AMO_ENABLE) ) cache ( .clk (clk), diff --git a/hw/rtl/core/VX_core.sv b/hw/rtl/core/VX_core.sv index 0de66a70f5..46b3918de4 100644 --- a/hw/rtl/core/VX_core.sv +++ b/hw/rtl/core/VX_core.sv @@ -185,12 +185,9 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( // flush could inject ahead of a store still in the LSU/coalescer and lose it. wire store_drained = (&lsu_sched_empty) && mem_unit_empty; assign dcr_flush_dcache_if.req = dcr_flush_if.req && store_drained; - // Both L1s forward their flush to the shared next level, and a cache that - // is flushing locks out incoming core requests for its whole sweep. The - // icache carries no dirty data and so retires almost immediately; gate it - // behind the dcache to keep that forward from arriving while the dcache is - // still evicting, which would strand the evictions upstream of memory. - assign dcr_flush_icache_if.req = dcr_flush_if.req && dcr_flush_dcache_if.done; + // Only the dcache flush walks the shared levels (the icache does not + // forward its flush), so the icache flushes alongside it. + assign dcr_flush_icache_if.req = dcr_flush_if.req; // Each VX_dcr_flush holds .done level-high until its req drops, so a // straight AND of the two dones reports the combined completion to // VX_dcr_data — which then drops req, re-arming both for the next flush. diff --git a/hw/rtl/core/VX_dcr_data.sv b/hw/rtl/core/VX_dcr_data.sv index 9c10d06f9b..746848286e 100644 --- a/hw/rtl/core/VX_dcr_data.sv +++ b/hw/rtl/core/VX_dcr_data.sv @@ -74,28 +74,40 @@ module VX_dcr_data import VX_gpu_pkg::*; #( // Cache flush via DCR interface ///////////////////////////////////////// - reg flush_pending_r; - - wire is_flush_read = dcr_bus_if.req_valid - && ~dcr_bus_if.req_data.rw - && (dcr_bus_if.req_data.addr == `VX_DCR_BASE_CACHE_FLUSH) - && (mpm_target_cid == 16'(CORE_ID)) - && ~flush_pending_r; + // A flush starts on a DCR write to CACHE_FLUSH, which reaches every core, + // and this core's flush read is answered once its flush has completed. The + // DCR response bus has no back-pressure, so the command processor starts + // all cores with one write and then reads them one at a time: the flushes + // overlap and the responses never collide. + reg flush_pending_r; // flush running + reg flush_complete_r; // flush finished, not yet reported + reg flush_read_r; // flush read waiting for completion + + wire is_flush_req = dcr_bus_if.req_valid + && (dcr_bus_if.req_data.addr == `VX_DCR_BASE_CACHE_FLUSH); + wire is_flush_write = is_flush_req && dcr_bus_if.req_data.rw; + wire is_flush_read = is_flush_req && ~dcr_bus_if.req_data.rw + && (mpm_target_cid == 16'(CORE_ID)); + + wire flush_done = flush_pending_r && dcr_flush_if.done; + wire flush_start = ~flush_pending_r && is_flush_write; + wire flush_rsp = flush_read_r && (flush_complete_r || flush_done); always @(posedge clk) begin if (reset) begin - flush_pending_r <= 1'b0; - end else if (is_flush_read) begin - flush_pending_r <= 1'b1; - end else if (dcr_flush_if.done) begin - flush_pending_r <= 1'b0; + flush_pending_r <= 1'b0; + flush_complete_r <= 1'b0; + flush_read_r <= 1'b0; + end else begin + flush_pending_r <= flush_start || (flush_pending_r && ~flush_done); + // a write opens a new flush, so an older completion cannot answer it + flush_complete_r <= ~is_flush_write && ~flush_rsp && (flush_complete_r || flush_done); + flush_read_r <= ~flush_rsp && (flush_read_r || is_flush_read); end end assign dcr_flush_if.req = flush_pending_r; - wire flush_done = flush_pending_r && dcr_flush_if.done; - // Return CSR data or flush-done on DCR response bus // Use intermediate registers — Vivado (Synth 8-453) forbids procedural // assignment to interface net signals; drive them with continuous assigns. @@ -107,10 +119,10 @@ module VX_dcr_data import VX_gpu_pkg::*; #( rsp_valid_r <= 1'b0; rsp_data_r <= '0; end else begin - rsp_valid_r <= dcr_csr_if_fire || flush_done; + rsp_valid_r <= dcr_csr_if_fire || flush_rsp; if (dcr_csr_if_fire) begin rsp_data_r <= dcr_csr_if.value; - end else if (flush_done) begin + end else if (flush_rsp) begin rsp_data_r <= '0; end end @@ -119,6 +131,12 @@ module VX_dcr_data import VX_gpu_pkg::*; #( assign dcr_bus_if.rsp_valid = rsp_valid_r; assign dcr_bus_if.rsp_data = rsp_data_r; + // the requester issues one read at a time, so the two answers never meet + `RUNTIME_ASSERT (~(dcr_csr_if_fire && flush_rsp), ("%t: %s CSR read and flush answered in the same cycle", $time, INSTANCE_ID)) + // a new flush starts only after every core reported the previous one + `RUNTIME_ASSERT (~(is_flush_write && flush_pending_r), ("%t: %s flush started while one is running", $time, INSTANCE_ID)) + `RUNTIME_ASSERT (~(is_flush_read && ~flush_pending_r && ~flush_complete_r), ("%t: %s flush read with no flush started", $time, INSTANCE_ID)) + `ifdef DBG_TRACE_PIPELINE always @(posedge clk) begin if (dcr_bus_if.req_valid && dcr_bus_if.req_data.rw) begin diff --git a/hw/rtl/core/VX_dcr_flush.sv b/hw/rtl/core/VX_dcr_flush.sv index 7617f23b46..051eaf85d2 100644 --- a/hw/rtl/core/VX_dcr_flush.sv +++ b/hw/rtl/core/VX_dcr_flush.sv @@ -87,7 +87,7 @@ module VX_dcr_flush import VX_gpu_pkg::*; #( assign dcr_flush_if.done = flush_done_r; assign flush_bus_if.rsp_ready = 1'b1; - // 2-to-1 arb: input 0 = flush, input 1 = real dcache port 0 + // 2-to-1 arb: input 0 = cache traffic, input 1 = the flush token VX_mem_bus_if #( .DATA_SIZE (WORD_SIZE), .TAG_WIDTH (TAG_WIDTH) diff --git a/hw/rtl/core/VX_mem_unit.sv b/hw/rtl/core/VX_mem_unit.sv index 28e7b51377..bcd4083306 100644 --- a/hw/rtl/core/VX_mem_unit.sv +++ b/hw/rtl/core/VX_mem_unit.sv @@ -366,6 +366,22 @@ module VX_mem_unit import VX_gpu_pkg::*; #( end + // A private dcache takes the lanes straight from the adapter, whose drain + // store_drained already waits for, so one token suffices. Otherwise (a + // shared dcache, or none) the lanes are buffered independently before the + // next flush point, so a store on one lane can still be queued when another + // lane's token arrives: every lane then carries its own token behind its + // stores, and the flush is done when every lane's is. + localparam DCACHE_PRIVATE = (`VX_CFG_SOCKET_SIZE == `VX_CFG_NUM_DCACHES); + localparam FLUSH_LANES = DCACHE_PRIVATE ? 1 : DCACHE_NUM_REQS; + VX_dcr_flush_if lane_flush_if [FLUSH_LANES] (); + wire [FLUSH_LANES-1:0] lane_flush_done; + for (genvar k = 0; k < FLUSH_LANES; ++k) begin : g_lane_flush + assign lane_flush_if[k].req = dcr_flush_if.req; + assign lane_flush_done[k] = lane_flush_if[k].done; + end + assign dcr_flush_if.done = (& lane_flush_done); + for (genvar i = 0; i < `VX_CFG_NUM_LSU_BLOCKS; ++i) begin : g_dcache_adapters VX_mem_bus_if #( @@ -398,8 +414,8 @@ module VX_mem_unit import VX_gpu_pkg::*; #( assign per_block_adapter_empty[i] = ~(| adapter_req_valid); for (genvar j = 0; j < DCACHE_CHANNELS; ++j) begin : g_dcache_bus_if - if (i == 0 && j == 0) begin : g_flush_port - // Port 0: route through VX_dcr_flush to inject flush requests + localparam LANE = i * DCACHE_CHANNELS + j; + if (LANE < FLUSH_LANES) begin : g_flush_lane VX_dcr_flush #( .WORD_SIZE (DCACHE_WORD_SIZE), .TAG_WIDTH (DCACHE_CORE_TAG_WIDTH), @@ -407,15 +423,14 @@ module VX_mem_unit import VX_gpu_pkg::*; #( ) dcr_flush ( .clk (clk), .reset (reset), - .dcr_flush_if (dcr_flush_if), - .core_bus_if (dcache_bus_tmp_if[0]), - .cache_bus_if (dcache_bus_if[0]) + .dcr_flush_if (lane_flush_if[LANE]), + .core_bus_if (dcache_bus_tmp_if[j]), + .cache_bus_if (dcache_bus_if[LANE]) ); - end else begin : g_passthru_port - // Ports 1+: pass through; tag is zero-extended from - // DCACHE_CORE_TAG_WIDTH to DCACHE_TAG_WIDTH on request, - // and MSB-stripped on response. - `ASSIGN_VX_MEM_BUS_IF_EX (dcache_bus_if[i * DCACHE_CHANNELS + j], dcache_bus_tmp_if[j], DCACHE_TAG_WIDTH_BASE, DCACHE_CORE_TAG_WIDTH, 0); + end else begin : g_passthru_lane + // tag is zero-extended from DCACHE_CORE_TAG_WIDTH to + // DCACHE_TAG_WIDTH on request, and MSB-stripped on response + `ASSIGN_VX_MEM_BUS_IF_EX (dcache_bus_if[LANE], dcache_bus_tmp_if[j], DCACHE_TAG_WIDTH_BASE, DCACHE_CORE_TAG_WIDTH, 0); end end diff --git a/hw/rtl/cp/VX_cp_dcr_proxy.sv b/hw/rtl/cp/VX_cp_dcr_proxy.sv index 94428c07ed..05a90fd6de 100644 --- a/hw/rtl/cp/VX_cp_dcr_proxy.sv +++ b/hw/rtl/cp/VX_cp_dcr_proxy.sv @@ -15,13 +15,16 @@ // when valid) → DONE → IDLE. // // For CMD_CACHE_FLUSH (cmd.arg0 = number of cores): -// A cache flush is a per-core DCR-read to VX_DCR_BASE_CACHE_FLUSH whose -// response is the per-core flush-complete signal (see VX_dcr_data). One -// CMD_CACHE_FLUSH sweeps that read across every core — REQ/WAIT_RSP per -// core — and retires only once the last core's flush completes. This is -// the ACQUIRE_MEM model: a single command in the ring, executed by -// the CP across all cores, that the host posts after CMD_LAUNCH so it -// sees coherent results. The host fills cmd.arg0 from VX_CAPS_NUM_CORES. +// IDLE → START (DCR write to VX_DCR_BASE_CACHE_FLUSH, which every core +// receives and starts flushing on) → then REQ/WAIT_RSP once per core: a +// DCR read of VX_DCR_BASE_CACHE_FLUSH answered when that core's flush has +// completed (see VX_dcr_data). The cores flush concurrently, so their dcache +// flushes reach the shared caches together and share one walk; the reads +// are serial because the DCR response bus cannot carry two at once. The +// command retires once the last core reports. This is the ACQUIRE_MEM +// model: a single command in the ring, executed by the CP across all cores, +// that the host posts after CMD_LAUNCH so it sees coherent results. The +// host fills cmd.arg0 from VX_CAPS_NUM_CORES. // // The most-recent read response is published on `last_rsp_data` and is // also exposed on the AXI-Lite regfile so the host can poll it after @@ -57,8 +60,9 @@ module VX_cp_dcr_proxy input wire [`VX_DCR_DATA_BITS-1:0] dcr_rsp_data ); - typedef enum logic [1:0] { + typedef enum logic [2:0] { S_IDLE, + S_START, // CMD_CACHE_FLUSH: broadcast write starting every core S_REQ, // hold dcr_req_valid until consumed (single cycle here) S_WAIT_RSP, // read commands only S_DONE @@ -106,12 +110,15 @@ module VX_cp_dcr_proxy flush_total <= cmd_ncores; flush_cid <= '0; // a 0-core flush (degenerate) retires immediately - state <= (cmd_ncores == '0) ? S_DONE : S_REQ; + state <= (cmd_ncores == '0) ? S_DONE : S_START; end else begin state <= S_REQ; end end end + S_START: begin + state <= S_REQ; + end S_REQ: begin // The Vortex DCR bus consumes the request in a single cycle // (req_valid handshakes combinationally; no req_ready backpressure). @@ -141,11 +148,11 @@ module VX_cp_dcr_proxy end always_comb begin - dcr_req_valid = (state == S_REQ); + dcr_req_valid = (state == S_START) || (state == S_REQ); if (pending_is_flush) begin - // a cache flush is a per-core DCR read to VX_DCR_BASE_CACHE_FLUSH; - // the target core index rides in the low 16 bits of dcr_req_data. - dcr_req_rw = 1'b0; + // the START write reaches every core; each per-core read carries the + // target core index in the low 16 bits of dcr_req_data. + dcr_req_rw = (state == S_START); dcr_req_addr = `VX_DCR_ADDR_BITS'(`VX_DCR_BASE_CACHE_FLUSH); dcr_req_data = `VX_DCR_DATA_BITS'(flush_cid); end else begin diff --git a/hw/rtl/om/VX_om_ingress.sv b/hw/rtl/om/VX_om_ingress.sv index c1344b4227..5aba9c0991 100644 --- a/hw/rtl/om/VX_om_ingress.sv +++ b/hw/rtl/om/VX_om_ingress.sv @@ -52,7 +52,8 @@ module VX_om_ingress import VX_gpu_pkg::*, VX_om_pkg::*; #( VX_mem_bus_if.slave mem_bus_if, // aperture writes, peeled by VX_om_steer VX_om_bus_if.master om_bus_if, // fragments -> VX_om_bus_arb -> VX_om_core - output wire busy + output wire busy, + output wire frag_pending ); `UNUSED_SPARAM (INSTANCE_ID) `UNUSED_PARAM (TAG_WIDTH) @@ -307,6 +308,10 @@ module VX_om_ingress import VX_gpu_pkg::*, VX_om_pkg::*; #( assign busy = (open_valid != '0) || rec_valid || om_bus_if.req_valid; + // Work that completes without another beat. An open record waits for its + // depth beat, which may be queued behind a flush token held at the steer. + assign frag_pending = rec_valid || om_bus_if.req_valid; + `RUNTIME_ASSERT(~(s2_fire && alloc && ~has_free), ("%t: *** %s: OM ingress hold table overflow — MAX_OPEN bound is wrong", $time, INSTANCE_ID)) `RUNTIME_ASSERT(~(s2_fire && s2_is_pair && s2_is_depth && ~hit), diff --git a/hw/rtl/om/VX_om_steer.sv b/hw/rtl/om/VX_om_steer.sv index cb94dce869..7aa803e5ee 100644 --- a/hw/rtl/om/VX_om_steer.sv +++ b/hw/rtl/om/VX_om_steer.sv @@ -25,10 +25,12 @@ // aperture store is presented to the caches as an ordinary IO store (they carry // attr through verbatim), so this bit arrives untouched. See VX_lsu_slice. // -// Buffering: only the OM leg is registered. The L2 pass-through stays -// combinational -- it is not a new driver (the socket already registers its -// outgoing bus), and a skid here would sit on the critical L1->L2 path and -// double the switch's cost for nothing. +// Flush ordering: the steer is a fork, so a flush token bound for L2 is held +// here until the OM side has been flushed. Every aperture store this trunk +// carried before the token is past the steer by then, so the OM flush, which +// waits for the OM to drain, covers them; then the token goes on to L2. +// +// Buffering: each leg presents a registered master interface (OUT_BUF). `include "VX_define.vh" @@ -43,7 +45,11 @@ module VX_om_steer import VX_gpu_pkg::*; #( VX_mem_bus_if.slave bus_in_if, // from the socket VX_mem_bus_if.master l2_out_if, // everything else -> L2 - VX_mem_bus_if.master om_out_if // aperture writes -> OM ingress (posted) + VX_mem_bus_if.master om_out_if, // aperture writes -> OM ingress (posted) + + output wire flush_req, // a flush token is held until the OM side is flushed + input wire flush_ack, // pulse: the OM side was flushed after flush_req rose + output wire om_pending // an aperture write is still buffered on the OM leg ); `UNUSED_SPARAM (INSTANCE_ID) @@ -53,11 +59,37 @@ module VX_om_steer import VX_gpu_pkg::*; #( wire is_om = bus_in_if.req_data.attr[MEM_ATTR_OM_OFFS]; + // ── flush tokens: held until the OM side is flushed ──────────────────── + wire is_flush = bus_in_if.req_valid && ~is_om && bus_in_if.req_data.attr[MEM_ATTR_FLUSH_OFFS]; + reg flush_req_r; // token held, waiting for the OM flush + reg flush_pass_r; // OM flushed: the token may go to L2 + wire flush_block = is_flush && ~flush_pass_r; + // ── request: demux by is_addr_om ─────────────────────────────────────── wire l2_ready; wire om_ready; - assign bus_in_if.req_ready = is_om ? om_ready : l2_ready; + assign bus_in_if.req_ready = is_om ? om_ready : (l2_ready && ~flush_block); + + wire flush_fire = is_flush && flush_pass_r && l2_ready; + always @(posedge clk) begin + if (reset) begin + flush_req_r <= 1'b0; + flush_pass_r <= 1'b0; + end else begin + if (flush_req_r && flush_ack) begin + flush_req_r <= 1'b0; + flush_pass_r <= 1'b1; + end else if (flush_block) begin + flush_req_r <= 1'b1; + end + if (flush_fire) begin + flush_pass_r <= 1'b0; + end + end + end + assign flush_req = flush_req_r; + assign om_pending = om_out_if.req_valid; // Both legs go through an OUT_BUF elastic buffer so the socket->L2 trunk and // the OM aperture path each present a fully-registered master interface. @@ -71,7 +103,7 @@ module VX_om_steer import VX_gpu_pkg::*; #( ) l2_req_buf ( .clk (clk), .reset (reset), - .valid_in (bus_in_if.req_valid && ~is_om), + .valid_in (bus_in_if.req_valid && ~is_om && ~flush_block), .ready_in (l2_ready), .data_in (l2_req_data_in), .data_out (l2_req_data_out), diff --git a/hw/unittest/cp_dcr_proxy/main.cpp b/hw/unittest/cp_dcr_proxy/main.cpp index 56f3e18cf3..861c0c2932 100644 --- a/hw/unittest/cp_dcr_proxy/main.cpp +++ b/hw/unittest/cp_dcr_proxy/main.cpp @@ -9,6 +9,8 @@ // S_REQ → write: S_DONE; read: S_WAIT_RSP // S_WAIT_RSP → dcr_rsp_valid ⇒ latch rsp_data_r, S_DONE // S_DONE → IDLE +// CMD_CACHE_FLUSH: IDLE → S_START (broadcast write) → per core +// S_REQ/S_WAIT_RSP (read with the core index) → S_DONE after the last. // // Coverage: // 1. Reset: no transitions, dcr_req_valid stays 0, done stays 0. @@ -19,11 +21,16 @@ // S_WAIT_RSP until dcr_rsp_valid arrives; rsp_data is latched // into last_rsp_data and visible while done pulses. // 4. Back-to-back write→read: FSM re-arms cleanly. -// 5. WAIT_RSP hangs if rsp_valid never arrives (no spurious done). +// 5. CMD_CACHE_FLUSH over 3 cores: one broadcast write to +// VX_DCR_BASE_CACHE_FLUSH, then one read per core in order with the +// core index as data, each waiting for its response; done only +// after the last core answers. +// 6. CMD_CACHE_FLUSH over 0 cores retires without any DCR request. // ============================================================================ #include "vl_simulator.h" #include "VVX_cp_dcr_proxy_top.h" +#include #include #include #include @@ -51,6 +58,7 @@ void sim_trace_enable(bool e) { trace_en = e; } enum CmdOp : uint8_t { OP_DCR_WRITE = 0x04, OP_DCR_READ = 0x05, + OP_CACHE_FLUSH = 0x0A, }; // Same packed-cmd layout as the cp_engine TB: hdr in the MSB word @@ -194,6 +202,61 @@ int main(int argc, char** argv) { EXPECT(sim->done == 1, "re-arm: done not asserted"); cycle(sim, tick); + // ----- Test 5: CMD_CACHE_FLUSH over 3 cores. arg0 = core count. ----- + constexpr uint32_t NCORES = 3; + set_cmd(sim.operator->(), OP_CACHE_FLUSH, NCORES); + sim->grant = 1; + cycle(sim, tick); // IDLE → S_START + sim->grant = 0; + + // S_START: one write that every core receives. + sim->eval(); + EXPECT(sim->dcr_req_valid == 1, "FLUSH: start write not issued"); + EXPECT(sim->dcr_req_rw == 1, "FLUSH: start must be a write"); + EXPECT(sim->dcr_req_addr == VX_DCR_BASE_CACHE_FLUSH, "FLUSH: start addr mismatch"); + EXPECT(sim->done == 0, "FLUSH: done premature in S_START"); + cycle(sim, tick); // S_START → S_REQ + + for (uint32_t cid = 0; cid < NCORES; ++cid) { + // S_REQ: this core's completion read. + sim->eval(); + EXPECT(sim->dcr_req_valid == 1, "FLUSH: per-core read not issued"); + EXPECT(sim->dcr_req_rw == 0, "FLUSH: per-core request must be a read"); + EXPECT(sim->dcr_req_addr == VX_DCR_BASE_CACHE_FLUSH, "FLUSH: read addr mismatch"); + EXPECT((sim->dcr_req_data & 0xffff) == cid, "FLUSH: read must target the cores in order"); + EXPECT(sim->done == 0, "FLUSH: done before every core answered"); + cycle(sim, tick); // S_REQ → S_WAIT_RSP + + // Nothing moves until this core answers. + for (int i = 0; i < 3; ++i) { + sim->eval(); + EXPECT(sim->dcr_req_valid == 0, "FLUSH: request while waiting for a core"); + EXPECT(sim->done == 0, "FLUSH: spurious done while waiting for a core"); + cycle(sim, tick); + } + sim->dcr_rsp_valid = 1; + sim->dcr_rsp_data = 0; + cycle(sim, tick); // → next core's S_REQ, or S_DONE + sim->dcr_rsp_valid = 0; + } + + sim->eval(); + EXPECT(sim->done == 1, "FLUSH: done not asserted after the last core"); + EXPECT(sim->dcr_req_valid == 0, "FLUSH: request after the last core"); + cycle(sim, tick); // S_DONE → IDLE + sim->eval(); + EXPECT(sim->done == 0, "FLUSH: done should pulse only one cycle"); + + // ----- Test 6: CMD_CACHE_FLUSH over 0 cores retires at once. ----- + set_cmd(sim.operator->(), OP_CACHE_FLUSH, 0); + sim->grant = 1; + cycle(sim, tick); // IDLE → S_DONE + sim->grant = 0; + sim->eval(); + EXPECT(sim->dcr_req_valid == 0, "FLUSH0: no request expected"); + EXPECT(sim->done == 1, "FLUSH0: done not asserted"); + cycle(sim, tick); + std::printf("PASSED\n"); return 0; } diff --git a/hw/unittest/gfx/VX_gfx_top.sv b/hw/unittest/gfx/VX_gfx_top.sv index 51aed0b6d5..c16aae9aaf 100644 --- a/hw/unittest/gfx/VX_gfx_top.sv +++ b/hw/unittest/gfx/VX_gfx_top.sv @@ -322,6 +322,7 @@ module VX_gfx_top import VX_gpu_pkg::* .WRITEBACK (0), .DIRTY_BYTES (0), .NC_ENABLE (0), + .FLUSH_FORWARD (0), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3) ) tcache ( @@ -473,6 +474,7 @@ module VX_gfx_top import VX_gpu_pkg::* .WRITEBACK (0), .DIRTY_BYTES (0), .NC_ENABLE (0), + .FLUSH_FORWARD (0), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3) ) rtcache ( diff --git a/hw/unittest/vm/VX_vm_top.sv b/hw/unittest/vm/VX_vm_top.sv index 7ac7aec7f0..6e2f7a32e0 100644 --- a/hw/unittest/vm/VX_vm_top.sv +++ b/hw/unittest/vm/VX_vm_top.sv @@ -410,6 +410,7 @@ module VX_vm_top import VX_gpu_pkg::*, VX_tlb_pkg::*; #( .WRITE_ENABLE (0), .REPL_POLICY (`VX_CFG_ICACHE_REPL_POLICY), .NC_ENABLE (0), + .FLUSH_FORWARD (0), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3) ) icache ( diff --git a/sim/common/cmd_processor.cpp b/sim/common/cmd_processor.cpp index 0972440954..c6e97fb46f 100644 --- a/sim/common/cmd_processor.cpp +++ b/sim/common/cmd_processor.cpp @@ -328,6 +328,22 @@ void CommandProcessor::apply_qmd_(uint64_t qmd_addr) { } } +// One DCR write to VX_DCR_BASE_CACHE_FLUSH reaches every core and starts its +// flush, so the cores flush concurrently and share one walk of the shared +// caches; the per-core reads, answered once that core's flush completes, are +// serial because the DCR response bus cannot carry two responses at once. +void CommandProcessor::flush_cores_(uint32_t num_cores) { + if (num_cores == 0 || !hooks_.vortex_dcr_read) { + return; + } + if (hooks_.vortex_dcr_write) { + hooks_.vortex_dcr_write(VX_DCR_BASE_CACHE_FLUSH, 0); + } + for (uint32_t cid = 0; cid < num_cores; ++cid) { + (void)hooks_.vortex_dcr_read(VX_DCR_BASE_CACHE_FLUSH, cid); + } +} + // Execute one draw-bundle step. Inline ops (DCR_WRITE/DCR_READ/CACHE_FLUSH) // complete immediately and return false; a launch step (LAUNCH/LAUNCH_QMD) // kicks the launch sub-FSM and returns true so the caller waits for the drain @@ -351,11 +367,7 @@ bool CommandProcessor::exec_inline_cmd_(const Cmd& c) { uint32_t(c.arg0 & 0xFFF), uint32_t(c.arg1 & 0xFFFFFFFF)); return false; case OP_CACHE_FLUSH: - if (hooks_.vortex_dcr_read) { - uint32_t n = uint32_t(c.arg0 & 0xFFFFFFFF); - for (uint32_t cid = 0; cid < n; ++cid) - (void)hooks_.vortex_dcr_read(VX_DCR_BASE_CACHE_FLUSH, cid); - } + this->flush_cores_(uint32_t(c.arg0 & 0xFFFFFFFF)); return false; default: // NOP / FENCE / unknown step — no-op (draws don't use MEM_*/EVENT_*). @@ -484,15 +496,8 @@ void CommandProcessor::tick_engine() { } eng_state_ = EngState::Retire; } else if (cur_cmd_.opcode == OP_CACHE_FLUSH) { - // Sweep a per-core DCR-read of VX_DCR_BASE_CACHE_FLUSH across - // [0, num_cores). arg0 carries num_cores; dcr_read routes each - // call to flush_caches() (drains dcache/L2/L3 to memsim). - if (hooks_.vortex_dcr_read) { - uint32_t n = uint32_t(cur_cmd_.arg0 & 0xFFFFFFFF); - for (uint32_t cid = 0; cid < n; ++cid) { - (void)hooks_.vortex_dcr_read(VX_DCR_BASE_CACHE_FLUSH, cid); - } - } + // arg0 carries num_cores. + this->flush_cores_(uint32_t(cur_cmd_.arg0 & 0xFFFFFFFF)); eng_state_ = EngState::Retire; } else if (cur_cmd_.opcode == OP_EVENT_SIG) { // CMD_EVENT_SIGNAL: write arg1 (8-byte value) to arg0 diff --git a/sim/common/cmd_processor.h b/sim/common/cmd_processor.h index 0ccd6fc379..38e0f80f4c 100644 --- a/sim/common/cmd_processor.h +++ b/sim/common/cmd_processor.h @@ -242,6 +242,8 @@ class CommandProcessor { // CMD_LAUNCH_QMD: read the in-memory KMU descriptor at `qmd_addr` and // replay its {dcr_addr,value} pairs through the DCR-write hook. void apply_qmd_(uint64_t qmd_addr); + // CMD_CACHE_FLUSH across cores [0, num_cores). + void flush_cores_(uint32_t num_cores); // Advance the launch FSM one step using cur_cmd_. void tick_launch(); // Advance the engine FSM one step. diff --git a/sim/rtlsim/main.cpp b/sim/rtlsim/main.cpp index ba92ba1a03..e6287b8488 100644 --- a/sim/rtlsim/main.cpp +++ b/sim/rtlsim/main.cpp @@ -139,6 +139,7 @@ int main(int argc, char **argv) { // flush GPU caches before reading back results { uint32_t dummy; + processor.dcr_write(VX_DCR_BASE_CACHE_FLUSH, 0); for (uint32_t cid = 0; cid < VX_CFG_NUM_CORES * VX_CFG_NUM_CLUSTERS; ++cid) { processor.dcr_read(VX_DCR_BASE_CACHE_FLUSH, cid, &dummy); } diff --git a/sim/rtlsim/processor.cpp b/sim/rtlsim/processor.cpp index b383d556ff..416c60208b 100644 --- a/sim/rtlsim/processor.cpp +++ b/sim/rtlsim/processor.cpp @@ -237,8 +237,8 @@ class Processor::Impl { device_->dcr_req_data = tag; this->tick(); device_->dcr_req_valid = 0; - this->tick(); - // READ response is returned when dcr_rsp_valid is high + // A core whose flush already finished answers on the next cycle, and the + // response is a one-cycle pulse, so start polling right away. while (!device_->dcr_rsp_valid) { this->tick(); } diff --git a/sim/simx/main.cpp b/sim/simx/main.cpp index ff1d406692..41f8bd473c 100644 --- a/sim/simx/main.cpp +++ b/sim/simx/main.cpp @@ -233,6 +233,7 @@ int main(int argc, char **argv) { // flush GPU caches before reading back results { uint32_t dummy; + processor.dcr_write(VX_DCR_BASE_CACHE_FLUSH, 0); for (uint32_t cid = 0; cid < VX_CFG_NUM_CORES * VX_CFG_NUM_CLUSTERS; ++cid) { processor.dcr_read(VX_DCR_BASE_CACHE_FLUSH, cid, &dummy); } diff --git a/sw/runtime/rtlsim/vortex.cpp b/sw/runtime/rtlsim/vortex.cpp index e0677b66c8..e5aa68d64f 100644 --- a/sw/runtime/rtlsim/vortex.cpp +++ b/sw/runtime/rtlsim/vortex.cpp @@ -153,6 +153,11 @@ class vx_device { ram_.write(static_cast(src), addr, bytes); }; h.vortex_dcr_write = [this](uint32_t addr, uint32_t value) { + // Wait for any background processor_.run() to finish so dcr_write + // does not race the Verilator state. + if (future_.valid()) { + future_.wait(); + } processor_.dcr_write(addr, value); }; h.vortex_dcr_read = [this](uint32_t addr, uint32_t tag) -> uint32_t { diff --git a/sw/runtime/simx/vortex.cpp b/sw/runtime/simx/vortex.cpp index 3d3088850f..d72ba8d60d 100644 --- a/sw/runtime/simx/vortex.cpp +++ b/sw/runtime/simx/vortex.cpp @@ -160,6 +160,10 @@ class vx_device { ram_.write(static_cast(src), addr, bytes); }; h.vortex_dcr_write = [this](uint32_t addr, uint32_t value) { + // Wait for any background processor_.run() to finish before writing DCR. + if (future_.valid()) { + future_.wait(); + } processor_.dcr_write(addr, value); }; h.vortex_dcr_read = [this](uint32_t addr, uint32_t tag) -> uint32_t { diff --git a/tests/regression/Makefile b/tests/regression/Makefile index e50ebfff69..3fd4c75708 100644 --- a/tests/regression/Makefile +++ b/tests/regression/Makefile @@ -15,6 +15,7 @@ TESTS := \ dxa_copy dxa_copy_mcast dxa_kmajor_check \ async_barrier async_gbarrier gbar_phase bar_slot_phase packld wgather wsync quad_lod \ occupancy multikernel reopen module_reload single_cta \ + flush_coherence \ vm_test vm_fault vm_stress # --- common exclude list --------------------------------------------- diff --git a/tests/regression/flush_coherence/Makefile b/tests/regression/flush_coherence/Makefile new file mode 100644 index 0000000000..5ceced321a --- /dev/null +++ b/tests/regression/flush_coherence/Makefile @@ -0,0 +1,16 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := flush_coherence + +SRC_DIR := $(VORTEX_HOME)/tests/regression/$(PROJECT) + +SRCS := $(SRC_DIR)/main.cpp + +VX_SRCS := $(SRC_DIR)/kernel.cpp + +OPTS ?= -n2048 -r16 + +KERNEL_LIB := vortex2 + +include ../common.mk diff --git a/tests/regression/flush_coherence/common.h b/tests/regression/flush_coherence/common.h new file mode 100644 index 0000000000..bbdabc90a1 --- /dev/null +++ b/tests/regression/flush_coherence/common.h @@ -0,0 +1,21 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include + +typedef struct { + uint32_t count; + uint32_t round; + uint32_t rotate; + uint32_t burst; + uint64_t src_addr; + uint64_t dst_addr; + uint64_t acc_addr; + uint64_t tail_addr; +} kernel_arg_t; + +inline uint32_t tail_value(uint32_t i, uint32_t k, uint32_t round) { + return (i * 0x01000193u) ^ (k << 20) ^ (round * 0x2545F491u); +} + +#endif diff --git a/tests/regression/flush_coherence/kernel.cpp b/tests/regression/flush_coherence/kernel.cpp new file mode 100644 index 0000000000..de656daa29 --- /dev/null +++ b/tests/regression/flush_coherence/kernel.cpp @@ -0,0 +1,28 @@ +#include +#include "common.h" + +__kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { + uint32_t tid = blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= arg->count) { + return; + } + // The rotation moves each element, and so each line, to another thread + // and core from one launch to the next. + uint32_t i = tid + arg->rotate; + if (i >= arg->count) { + i -= arg->count; + } + auto src = reinterpret_cast(arg->src_addr); + auto dst = reinterpret_cast(arg->dst_addr); + auto acc = reinterpret_cast(arg->acc_addr); + uint32_t v = src[i]; + acc[i] += v ^ arg->round; + // A store burst saturates the shared levels, so the final stores are + // still queued in front of them when the flushes start. + auto tail = reinterpret_cast(arg->tail_addr); + for (uint32_t k = 0; k < arg->burst; ++k) { + tail[k * arg->count + i] = tail_value(i, k, arg->round); + } + // Ending on a store leaves the kernel's last writes in flight at exit. + dst[i] = v * 3u + arg->round; +} diff --git a/tests/regression/flush_coherence/main.cpp b/tests/regression/flush_coherence/main.cpp new file mode 100644 index 0000000000..573dbe66ad --- /dev/null +++ b/tests/regression/flush_coherence/main.cpp @@ -0,0 +1,190 @@ +// Kernel-boundary flush coherence under back-to-back launches. +// +// Every round the host rewrites src, then launches a kernel that reads src, +// accumulates into acc and writes dst. The element-to-thread mapping is +// rotated on three rounds out of four, so a line usually moves to another +// thread and often to another core. The post-launch cache flush must: +// - publish each launch's writes before the host reads dst or acc; +// - invalidate every core's (and every shared level's) copy of src before +// the next launch, or a core re-reading a line it cached would see the +// previous round's value; +// - make one launch's acc writes visible to the next launch's reads on +// other cores, also when no host transfer separates them (odd rounds do +// not read dst back). +// With -b, each thread ends on a burst of stores to a tail buffer, so the +// last stores may still be queued in front of the shared levels when the +// flushes start, and every round is checked. + +#include +#include "common.h" + +#include +#include +#include +#include +#include + +#define CHECK(expr) do { \ + vx_result_t _r = (expr); \ + if (_r != VX_SUCCESS) { \ + std::fprintf(stderr, "FAIL %s:%d: '%s' returned %s\n", \ + __FILE__, __LINE__, #expr, vx_result_string(_r)); \ + std::exit(1); \ + } \ + } while (0) + +namespace { +const char* kernel_file = "kernel.vxbin"; +uint32_t count = 2048; +uint32_t rounds = 16; +uint32_t burst = 0; + +void parse_args(int argc, char** argv) { + int c; + while ((c = getopt(argc, argv, "n:r:b:k:h")) != -1) { + switch (c) { + case 'n': count = (uint32_t)std::atoi(optarg); break; + case 'r': rounds = (uint32_t)std::atoi(optarg); break; + case 'b': burst = (uint32_t)std::atoi(optarg); break; + case 'k': kernel_file = optarg; break; + default: + std::printf("Usage: [-k kernel] [-n words] [-r rounds] [-b burst] [-h]\n"); + std::exit(c == 'h' ? 0 : -1); + } + } +} + +uint32_t src_value(uint32_t i, uint32_t round) { + uint32_t x = i * 0x9E3779B1u + round * 0x85EBCA77u + 1; + x ^= x >> 15; + return x * 0x2C1B3C6Du; +} + +uint32_t rotation(uint32_t round) { + return (round % 4 == 0) ? 0 : (round * 0x9E3779B1u) % count; +} + +int check(const char* what, uint32_t round, const std::vector& got, + const std::vector& ref) { + int errors = 0; + for (size_t i = 0; i < ref.size(); ++i) { + if (got[i] != ref[i]) { + if (errors < 8) { + std::printf("*** round %u %s[%zu]: expected=0x%08x actual=0x%08x\n", + round, what, i, ref[i], got[i]); + } + ++errors; + } + } + return errors; +} +} // namespace + +int main(int argc, char** argv) { + parse_args(argc, argv); + const uint64_t bytes = (uint64_t)count * sizeof(uint32_t); + const uint64_t tail_bytes = bytes * burst; + std::printf("flush_coherence: n=%u rounds=%u burst=%u\n", count, rounds, burst); + + vx_device_h dev = nullptr; + CHECK(vx_device_open(0, &dev)); + + vx_queue_info_t qi = { sizeof(qi), nullptr, VX_QUEUE_PRIORITY_NORMAL, 0 }; + vx_queue_h q = nullptr; + CHECK(vx_queue_create(dev, &qi, &q)); + + vx_buffer_h src_buf = nullptr, dst_buf = nullptr, acc_buf = nullptr, tail_buf = nullptr; + CHECK(vx_buffer_create(dev, bytes, VX_MEM_READ, &src_buf)); + CHECK(vx_buffer_create(dev, bytes, VX_MEM_WRITE, &dst_buf)); + CHECK(vx_buffer_create(dev, bytes, VX_MEM_READ_WRITE, &acc_buf)); + if (burst) { + CHECK(vx_buffer_create(dev, tail_bytes, VX_MEM_WRITE, &tail_buf)); + } + + vx_module_h mod = nullptr; + vx_kernel_h kern = nullptr; + CHECK(vx_module_load_file(dev, kernel_file, &mod)); + CHECK(vx_module_get_kernel(mod, "main", &kern)); + + kernel_arg_t arg{}; + arg.count = count; + CHECK(vx_buffer_address(src_buf, &arg.src_addr)); + CHECK(vx_buffer_address(dst_buf, &arg.dst_addr)); + CHECK(vx_buffer_address(acc_buf, &arg.acc_addr)); + if (burst) { + CHECK(vx_buffer_address(tail_buf, &arg.tail_addr)); + } + arg.burst = burst; + + uint32_t grid[1], block[1]; + CHECK(vx_device_max_occupancy_grid(dev, 1, &count, grid, block)); + + // Enqueued writes and launches read their host data asynchronously, so + // each round gets its own source buffer and argument block. + std::vector> h_src(rounds, std::vector(count)); + std::vector args(rounds, arg); + std::vector h_dst(count), h_acc(count, 0); + std::vector ref_dst(count), ref_acc(count, 0); + std::vector h_tail(count * burst), ref_tail(count * burst); + CHECK(vx_enqueue_write(q, acc_buf, 0, h_acc.data(), bytes, 0, nullptr, nullptr)); + + int errors = 0; + for (uint32_t r = 0; r < rounds; ++r) { + auto& src = h_src[r]; + for (uint32_t i = 0; i < count; ++i) { + src[i] = src_value(i, r); + ref_dst[i] = src[i] * 3u + r; + ref_acc[i] += src[i] ^ r; + for (uint32_t k = 0; k < burst; ++k) { + ref_tail[k * count + i] = tail_value(i, k, r); + } + } + CHECK(vx_enqueue_write(q, src_buf, 0, src.data(), bytes, 0, nullptr, nullptr)); + + args[r].round = r; + args[r].rotate = rotation(r); + vx_launch_info_t li{}; + li.struct_size = sizeof(li); + li.kernel = kern; + li.args_host = &args[r]; + li.args_size = sizeof(args[r]); + li.ndim = 1; + li.grid_dim[0] = grid[0]; + li.block_dim[0] = block[0]; + CHECK(vx_enqueue_launch(q, &li, 0, nullptr, nullptr)); + + if (burst || r % 2 == 0 || r + 1 == rounds) { + CHECK(vx_enqueue_read(q, h_dst.data(), dst_buf, 0, bytes, 0, nullptr, nullptr)); + if (burst) { + CHECK(vx_enqueue_read(q, h_tail.data(), tail_buf, 0, tail_bytes, 0, nullptr, nullptr)); + } + CHECK(vx_queue_finish(q, VX_TIMEOUT_INFINITE)); + errors += check("dst", r, h_dst, ref_dst); + if (burst) { + errors += check("tail", r, h_tail, ref_tail); + } + } + } + + CHECK(vx_enqueue_read(q, h_acc.data(), acc_buf, 0, bytes, 0, nullptr, nullptr)); + CHECK(vx_queue_finish(q, VX_TIMEOUT_INFINITE)); + errors += check("acc", rounds, h_acc, ref_acc); + + if (tail_buf) { + vx_buffer_release(tail_buf); + } + vx_buffer_release(acc_buf); + vx_buffer_release(dst_buf); + vx_buffer_release(src_buf); + vx_kernel_release(kern); + vx_module_release(mod); + vx_queue_release(q); + vx_device_release(dev); + + if (errors) { + std::printf("Found %d errors!\nFAILED!\n", errors); + return 1; + } + std::printf("PASSED!\n"); + return 0; +} From ef75f8e624643b92550af0d42d92fdc920830fce Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 14 Oct 2026 19:05:09 -0700 Subject: [PATCH 031/124] Hash the cache bank select so power-of-two strides spread over the banks With the bank taken straight from the address bits above the word offset, any stride that is a multiple of NUM_BANKS x LINE_SIZE lands every access of a warp in one or two banks: a 1 KiB row stride on a 16-bank cache uses two banks while the rest idle. Matrix columns, transposed tiles and per-row reductions all produce such strides. GPUs select banks and L2 slices with an XOR hash of the higher address bits instead. The bank is now the raw bank bits XORed with a function of the line address above them: IPOLY(5) for 16 banks and IPOLY(37) for 32 (Rau, ISCA 1991), an XOR fold otherwise. The same XOR restores the raw bits, so it is applied once on the request side and once where the memory address is rebuilt; each bank forms its raw bits before the memory crossbar, so the outgoing address still leaves from the crossbar's output register. Banks keep storing the line address without the bank field, so tags, MSHR, flush tokens and sectoring are unchanged. CS_BANK_TO_FULL_ADDR applies the hash so trace and assert addresses are real. SimX implements the identical function. Defaults: VX_CFG_L2_BANK_HASH = 1, and VX_CFG_DCACHE_BANK_HASH on from 4 banks. With two banks the hash can only swap lines on parity; requests that no longer serialize at a hot bank reach DRAM together and the queuing outweighs the spreading (sgemv-nt4 +8.8%, mxfp8 +7.4% when forced on). A 2-bank L2 measured neutral. perf_gate (rtlsim, re-recorded with --update-baselines; retired instructions unchanged everywhere, cases without a hashed cache unchanged): sgemv-nt16 -32.8%, fp16-mc-nt16 -13.8%, wgmma-dxa -13.5%, sgemm-mc-nt16 -8.2%, wgmma-sp-dxa -7.2%, sgemmx-nt16 -4.9%, wgmma-dxa-mcast-nt16 -3.7%, sgemm-nt16 -3.1%; worst +0.09%. fpga_gate (U55C, 300 MHz target): cache DUT 281.4 MHz / 27.4K LUT with the hash on (278.8 MHz before); core DUT 279.4 MHz, identical to the parent commit, so the core's distance from its 298 MHz baseline predates this change (its worst paths are in the LSU and operand collectors). Tests: cache:bank_hash-1..4 (hash off; 16-thread IPOLY dcache; 4-core write-back L2; write-back dirty-byte dcache) on SimX and rtlsim. Co-Authored-By: Claude Opus 5.5 --- VX_config.toml | 7 +++++ ci/baselines/perf/core.json | 50 +++++++++++++++---------------- ci/baselines/perf/dxa.json | 34 ++++++++++----------- ci/baselines/perf/graphics.json | 30 +++++++++---------- ci/baselines/perf/raytracing.json | 10 +++---- ci/baselines/perf/tensor.json | 14 ++++----- ci/baselines/perf/tensor_mx.json | 2 +- ci/baselines/perf/tensor_sp.json | 2 +- ci/baselines/perf/tensor_wg.json | 6 ++-- ci/testcases/cache.yaml | 40 +++++++++++++++++++++++++ docs/designs/cache_subsystem.md | 20 +++++++++++-- hw/rtl/VX_cluster.sv | 1 + hw/rtl/VX_gpu_pkg.sv | 28 +++++++++++++++++ hw/rtl/VX_socket.sv | 1 + hw/rtl/cache/VX_cache.sv | 49 +++++++++++++++++++++--------- hw/rtl/cache/VX_cache_bank.sv | 6 +++- hw/rtl/cache/VX_cache_cluster.sv | 2 ++ hw/rtl/cache/VX_cache_define.vh | 3 +- hw/rtl/cache/VX_cache_mshr.sv | 2 ++ hw/rtl/cache/VX_cache_wrap.sv | 2 ++ hw/unittest/cache/VX_cache_top.sv | 4 +++ sim/simx/cluster.cpp | 1 + sim/simx/mem/addr_hash.h | 46 ++++++++++++++++++++++++++++ sim/simx/mem/cache.cpp | 37 ++++++++++++++++++++--- sim/simx/mem/cache.h | 1 + sim/simx/socket.cpp | 1 + 26 files changed, 303 insertions(+), 96 deletions(-) create mode 100644 sim/simx/mem/addr_hash.h diff --git a/VX_config.toml b/VX_config.toml index f51186cdec..b3b89c6833 100644 --- a/VX_config.toml +++ b/VX_config.toml @@ -235,6 +235,10 @@ VX_CFG_NUM_ICACHES = "expr: up($VX_CFG_SOCKET_SIZE / 4) if $VX_CFG_ICACHE_ENABLE VX_CFG_NUM_DCACHES = "expr: up($VX_CFG_SOCKET_SIZE / 4) if $VX_CFG_DCACHE_ENABLE else 0" VX_CFG_DCACHE_NUM_BANKS = "expr: pow(2, clog2(min($VX_CFG_DCACHE_NUM_REQS, 16)))" +# XOR/IPOLY bank-select hash, so power-of-two strides spread over the banks. +# With two banks it can only swap a line on parity, and the extra parallel +# misses cost more in memory queuing than the spreading saves. +VX_CFG_DCACHE_BANK_HASH = "expr: int($VX_CFG_DCACHE_NUM_BANKS >= 4)" VX_CFG_L1_MEM_PORTS = "expr: min($VX_CFG_DCACHE_NUM_BANKS, $VX_CFG_PLATFORM_MEMORY_NUM_BANKS) if ($VX_CFG_ICACHE_ENABLE or $VX_CFG_DCACHE_ENABLE) else min($VX_CFG_DCACHE_NUM_REQS, $VX_CFG_PLATFORM_MEMORY_NUM_BANKS)" @@ -251,6 +255,9 @@ VX_CFG_L2_MREQ_SIZE = 0 VX_CFG_L2_CRSQ_SIZE = 0 VX_CFG_L2_NUM_BANKS = "expr: pow(2, clog2(min($VX_CFG_L2_NUM_REQS, 16)))" +# On at every bank count: unlike the dcache, a 2-bank L2 measured neutral +# (sgemm-mc-nt4 -0.6%, fp16-mc-nt4 +0.1%), and it gains from 4 banks up. +VX_CFG_L2_BANK_HASH = 1 VX_CFG_L2_MEM_PORTS = "expr: min($VX_CFG_L2_NUM_BANKS, $VX_CFG_PLATFORM_MEMORY_NUM_BANKS) if $VX_CFG_L2_ENABLE else min($VX_CFG_L2_NUM_REQS, $VX_CFG_PLATFORM_MEMORY_NUM_BANKS)" [l3cache] diff --git a/ci/baselines/perf/core.json b/ci/baselines/perf/core.json index 77e8c4785a..a02138c6d0 100644 --- a/ci/baselines/perf/core.json +++ b/ci/baselines/perf/core.json @@ -1,12 +1,12 @@ { "core:raycast-nt16:rtlsim": { "32": { - "cycles": 563524, + "cycles": 564053, "instrs": 137486 }, "app": "raycast", "args": "", - "config_hash": "396cee9df6025cdb", + "config_hash": "7a2c92e634460e90", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -17,40 +17,40 @@ }, "app": "raycast", "args": "", - "config_hash": "262f24032ef14b58", + "config_hash": "2532cbe6c87ca233", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:sgemm-mc-nt16:rtlsim": { "32": { - "cycles": 787850, + "cycles": 723234, "instrs": 623680 }, "app": "sgemm", "args": "-n128", - "config_hash": "ad2b37492d761e8f", + "config_hash": "6106872241e50c60", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=16 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "core:sgemm-mc-nt4:rtlsim": { "32": { - "cycles": 2239426, + "cycles": 2227069, "instrs": 2494528 }, "app": "sgemm", "args": "-n128", - "config_hash": "606d011dc6f612d3", + "config_hash": "ecbd8e8a804188dd", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "core:sgemm-nt16:rtlsim": { "32": { - "cycles": 1838654, + "cycles": 1782470, "instrs": 623632 }, "app": "sgemm", "args": "-n128", - "config_hash": "9390754d3719fca3", + "config_hash": "6dcefd7e66ed0cc0", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -61,18 +61,18 @@ }, "app": "sgemm", "args": "-n128", - "config_hash": "740fd84fa2938103", + "config_hash": "74a554f940a83c4a", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:sgemmx-nt16:rtlsim": { "32": { - "cycles": 1290130, + "cycles": 1226398, "instrs": 266896 }, "app": "sgemmx", "args": "-n128", - "config_hash": "2e1c44e05f9ac34d", + "config_hash": "efc73b5e23a33b8f", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -83,18 +83,18 @@ }, "app": "sgemmx", "args": "-n128", - "config_hash": "024adb49ed96bdd3", + "config_hash": "0f2e76cd4d21f998", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:sgemv-nt16:rtlsim": { "32": { - "cycles": 685107, + "cycles": 460344, "instrs": 66800 }, "app": "sgemv", "args": "-m512 -n512", - "config_hash": "a3d9df9695acbb95", + "config_hash": "bc1d0a1801524d27", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -105,18 +105,18 @@ }, "app": "sgemv", "args": "-m512 -n512", - "config_hash": "0e8990b25e64ad7f", + "config_hash": "7fbf266181d2ec8b", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:softmax-nt16:rtlsim": { "32": { - "cycles": 1272929, + "cycles": 1271056, "instrs": 464089 }, "app": "softmax", "args": "-n64", - "config_hash": "9e0c1c9767c760ea", + "config_hash": "c53ddc9ba2cd96bd", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -127,18 +127,18 @@ }, "app": "softmax", "args": "-n64", - "config_hash": "eb5fed4bfce19d76", + "config_hash": "6dea102e1b20464c", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:stencil3d-nt16:rtlsim": { "32": { - "cycles": 2306960, + "cycles": 2272549, "instrs": 892944 }, "app": "stencil3d", "args": "-n32", - "config_hash": "6925a5138be930ea", + "config_hash": "542851d41daf9431", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -149,18 +149,18 @@ }, "app": "stencil3d", "args": "-n32", - "config_hash": "b83a1e3735f79d39", + "config_hash": "fa607fa9e1280813", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:vecadd-nt16:rtlsim": { "32": { - "cycles": 100640, + "cycles": 99953, "instrs": 29712 }, "app": "vecadd", "args": "-n16384", - "config_hash": "ef0c2548e7c5bf86", + "config_hash": "0f4e3087ce474e4b", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -171,7 +171,7 @@ }, "app": "vecadd", "args": "-n16384", - "config_hash": "ce1eb79984a98552", + "config_hash": "0169f39a14561223", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/dxa.json b/ci/baselines/perf/dxa.json index bf1e1a010c..a331465e29 100644 --- a/ci/baselines/perf/dxa.json +++ b/ci/baselines/perf/dxa.json @@ -6,18 +6,18 @@ }, "app": "dxa_copy_mcast", "args": "", - "config_hash": "6a815f855f6e0d05", + "config_hash": "2be8c5960886f23f", "configs": "-DVX_CFG_EXT_DXA_ENABLE", "driver": "rtlsim" }, "dxa:perf_gate-copy-nt16:rtlsim": { "32": { - "cycles": 1994, + "cycles": 1992, "instrs": 592 }, "app": "dxa_copy", "args": "-d1", - "config_hash": "626c5f6244584b1f", + "config_hash": "37f76aae77fd1afd", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -28,18 +28,18 @@ }, "app": "dxa_copy", "args": "-d1", - "config_hash": "7c32a11d123bfb13", + "config_hash": "c3b3447f163ec6c3", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-mcast-nt16:rtlsim": { "32": { - "cycles": 14022240, + "cycles": 14023305, "instrs": 4962320 }, "app": "sgemm2_dxa_mcast", "args": "-n128", - "config_hash": "822d0ca194266a27", + "config_hash": "62067f11f3ef5af4", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -50,18 +50,18 @@ }, "app": "sgemm2_dxa_mcast", "args": "-n128", - "config_hash": "87ea544e77719254", + "config_hash": "98c58db8047bceb9", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-nt16:rtlsim": { "32": { - "cycles": 3941267, + "cycles": 3921184, "instrs": 1263632 }, "app": "sgemm2_dxa", "args": "-n128 -t4 -m1", - "config_hash": "e55b61598f5f10ea", + "config_hash": "845bebf9fbc108b1", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -72,18 +72,18 @@ }, "app": "sgemm2_dxa", "args": "-n128 -t4 -m1", - "config_hash": "21da2f2737bf03dd", + "config_hash": "8cf4b0e4e0c7d144", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-dxa-mcast-nt16:rtlsim": { "32": { - "cycles": 82160, + "cycles": 79131, "instrs": 32384 }, "app": "sgemm_tcu_wg_dxa_mcast", "args": "-m 128 -n 128 -k 64", - "config_hash": "20304652a1bfb94c", + "config_hash": "d3c9c80b16cdf2c2", "configs": "-DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -94,29 +94,29 @@ }, "app": "sgemm_tcu_wg_dxa_mcast", "args": "-m 128 -n 128 -k 64", - "config_hash": "bf4520d008c07dc9", + "config_hash": "38993044a82f32e0", "configs": "-DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-dxa:rtlsim": { "32": { - "cycles": 36021, + "cycles": 31160, "instrs": 9160 }, "app": "sgemm_tcu_wg_dxa", "args": "-m 128 -n 128 -k 64", - "config_hash": "2a4e017d98740636", + "config_hash": "699f26ecbecc6bb2", "configs": "-DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DWGMMA_NRC=16 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_THREADS=32", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-sp-dxa:rtlsim": { "32": { - "cycles": 59789, + "cycles": 55512, "instrs": 9328 }, "app": "sgemm_tcu_wg_sp_dxa", "args": "-m 128 -n 128 -k 128", - "config_hash": "f05069d7cf0ee3ff", + "config_hash": "8e607400e57e51f5", "configs": "-DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_THREADS=32", "driver": "rtlsim" } diff --git a/ci/baselines/perf/graphics.json b/ci/baselines/perf/graphics.json index b5ee86bfc2..b402bd0b6f 100644 --- a/ci/baselines/perf/graphics.json +++ b/ci/baselines/perf/graphics.json @@ -1,45 +1,45 @@ { "graphics:perf_gate-draw3d-mc:rtlsim": { "32": { - "cycles": 530195, + "cycles": 529875, "instrs": 203136 }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "1e2937b20579ad3f", + "config_hash": "0777067f98f2730f", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "graphics:perf_gate-draw3d-nt16:rtlsim": { "32": { - "cycles": 143620, + "cycles": 143542, "instrs": 51652 }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "edc924bc4f246971", + "config_hash": "550e1e3d5ffe23d0", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "graphics:perf_gate-draw3d-nt4:rtlsim": { "32": { - "cycles": 529393, + "cycles": 529389, "instrs": 203136 }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "d682cc8c6f99c06b", + "config_hash": "5cc3a870f65d86fa", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "graphics:perf_gate-om-nt16:rtlsim": { "32": { - "cycles": 189231, + "cycles": 189356, "instrs": 68624 }, "app": "gfx_om", "args": "-w 128 -h 128 -r whitebox_128.png", - "config_hash": "46a66220c1d9956b", + "config_hash": "f74350c24c97b1f5", "configs": "-DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -50,18 +50,18 @@ }, "app": "gfx_om", "args": "-w 128 -h 128 -r whitebox_128.png", - "config_hash": "a7554f6fa88474dc", + "config_hash": "8a9749c1c0d8264d", "configs": "-DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "graphics:perf_gate-raster-nt16:rtlsim": { "32": { - "cycles": 14785, + "cycles": 14626, "instrs": 4708 }, "app": "gfx_raster", "args": "-w 128 -h 128 -t triangle.cgltrace -r triangle_ref_128.png", - "config_hash": "e2e1879d7d6f9aaf", + "config_hash": "967474845abde936", "configs": "-DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -72,18 +72,18 @@ }, "app": "gfx_raster", "args": "-w 128 -h 128 -t triangle.cgltrace -r triangle_ref_128.png", - "config_hash": "6e48f6b249533cc5", + "config_hash": "40f6d7ca7755b6ca", "configs": "-DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "graphics:perf_gate-tex-nt16:rtlsim": { "32": { - "cycles": 35441, + "cycles": 35297, "instrs": 12304 }, "app": "gfx_tex", "args": "-i toad.png -r toad_ref_f0.png -f 0 -g 0", - "config_hash": "c8c6b30537e93e2f", + "config_hash": "e44189b53e136920", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -94,7 +94,7 @@ }, "app": "gfx_tex", "args": "-i toad.png -r toad_ref_f0.png -f 0 -g 0", - "config_hash": "a4fc4d97e6e80586", + "config_hash": "f0cdf08354c9ae88", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/raytracing.json b/ci/baselines/perf/raytracing.json index 1fb0ab7f7b..e0ee3a7da4 100644 --- a/ci/baselines/perf/raytracing.json +++ b/ci/baselines/perf/raytracing.json @@ -1,23 +1,23 @@ { "raytracing:perf_gate-rt_raycast-mc:rtlsim": { "32": { - "cycles": 85045, + "cycles": 85040, "instrs": 39346 }, "app": "rt_raycast", "args": "", - "config_hash": "9a4f28d2555289b5", + "config_hash": "4f78e65dfd2497cb", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "raytracing:perf_gate-rt_raycast-nt16:rtlsim": { "32": { - "cycles": 53434, + "cycles": 53443, "instrs": 11372 }, "app": "rt_raycast", "args": "", - "config_hash": "aef578263250d585", + "config_hash": "c0af18c64430f778", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -28,7 +28,7 @@ }, "app": "rt_raycast", "args": "", - "config_hash": "c7b45fe5c1828bc5", + "config_hash": "e394fceca77fe6f1", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor.json b/ci/baselines/perf/tensor.json index 3552c4620b..112a272e80 100644 --- a/ci/baselines/perf/tensor.json +++ b/ci/baselines/perf/tensor.json @@ -6,29 +6,29 @@ }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "da0a50e46103b7dc", + "config_hash": "9cdc8bb95c856a38", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" }, "tensor:perf_gate-fp16-mc-nt16:rtlsim": { "32": { - "cycles": 42404, + "cycles": 36534, "instrs": 47296 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "2d12f1a0058fcc3b", + "config_hash": "153ac3d8001d2be6", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=16 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "tensor:perf_gate-fp16-mc-nt4:rtlsim": { "32": { - "cycles": 221592, + "cycles": 221752, "instrs": 336448 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "fce886092e60e0eb", + "config_hash": "21c6d64b10dab7cb", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, @@ -39,7 +39,7 @@ }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "f0558a3b3dfde51f", + "config_hash": "04953ed130a8c1a9", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32", "driver": "rtlsim" }, @@ -50,7 +50,7 @@ }, "app": "sgemm2_tcu", "args": "-m 128 -n 128 -k 64", - "config_hash": "24fd7a811fabb8c4", + "config_hash": "26d22a59f411cfbe", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_mx.json b/ci/baselines/perf/tensor_mx.json index 1dc1b78f41..a49403e311 100644 --- a/ci/baselines/perf/tensor_mx.json +++ b/ci/baselines/perf/tensor_mx.json @@ -6,7 +6,7 @@ }, "app": "sgemm_tcu_mx", "args": "-m 128 -n 128 -k32", - "config_hash": "3cfc301023342a7d", + "config_hash": "7c8765889acb4f32", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DITYPE=mxfp8 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_sp.json b/ci/baselines/perf/tensor_sp.json index 2c4f92004c..3ee3f24144 100644 --- a/ci/baselines/perf/tensor_sp.json +++ b/ci/baselines/perf/tensor_sp.json @@ -6,7 +6,7 @@ }, "app": "sgemm_tcu_sp", "args": "-m 128 -n 128 -k 128", - "config_hash": "90defe3d20b6f739", + "config_hash": "6e523b9d0f6fe4ee", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_wg.json b/ci/baselines/perf/tensor_wg.json index c7f771c72f..b2af51b8f2 100644 --- a/ci/baselines/perf/tensor_wg.json +++ b/ci/baselines/perf/tensor_wg.json @@ -6,7 +6,7 @@ }, "app": "sgemm_tcu_wg", "args": "-m64 -n64 -k64", - "config_hash": "ba81956fe68e94b6", + "config_hash": "c65e3b69950519cb", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_TCU_FEDP2K -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DWGMMA_RS", "driver": "rtlsim" }, @@ -17,7 +17,7 @@ }, "app": "sgemm_tcu_wg", "args": "-m 128 -n 128 -k 128", - "config_hash": "6ee14f25ec573117", + "config_hash": "0d41f1f51f46d61d", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=32 -DWGMMA_SS -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" }, @@ -28,7 +28,7 @@ }, "app": "sgemm_tcu_wg_sp", "args": "-m 128 -n 128 -k 128", - "config_hash": "a9ea3d8bd83a7eed", + "config_hash": "b4944cbd79928ac8", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" } diff --git a/ci/testcases/cache.yaml b/ci/testcases/cache.yaml index de3e6c289d..3a2c5b7088 100644 --- a/ci/testcases/cache.yaml +++ b/ci/testcases/cache.yaml @@ -264,3 +264,43 @@ tests: configs: -DVX_CFG_SOCKET_SIZE=3 -DVX_CFG_NUM_ICACHES=2 shape: cores: 3 +# Hashed bank select, on by default for the dcache and the L2. These cover the +# hash off at both levels, 16 threads, and a write-back L2, where every fill and +# write-back rebuilds the address. +- id: bank_hash-1 + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm + configs: -DVX_CFG_SOCKET_SIZE=1 -DVX_CFG_DCACHE_BANK_HASH=0 -DVX_CFG_L2_BANK_HASH=0 + shape: + cores: 2 + l2cache: true +- id: bank_hash-2 + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm + shape: + threads: 16 +- id: bank_hash-3 + via: blackbox + drivers: + - simx + - rtlsim + app: mstress + configs: -DVX_CFG_SOCKET_SIZE=1 -DVX_CFG_L2_WRITEBACK=1 + shape: + cores: 4 + l2cache: true +- id: bank_hash-4 + via: blackbox + drivers: + - simx + - rtlsim + app: mstress + configs: -DVX_CFG_DCACHE_WRITEBACK=1 -DVX_CFG_DCACHE_DIRTYBYTES=1 + shape: + threads: 16 diff --git a/docs/designs/cache_subsystem.md b/docs/designs/cache_subsystem.md index 73f7e308f1..8fce9737ef 100644 --- a/docs/designs/cache_subsystem.md +++ b/docs/designs/cache_subsystem.md @@ -224,6 +224,22 @@ rather than being a field of its own. Two consequences are worth internalizing: pipeline carrying only `word_idx` and still tell the tag and data stages which sector to mark and write. +**Hashed bank select (`BANK_HASH`).** With the bank taken straight from the +address bits, any power-of-two stride that is a multiple of `NUM_BANKS × LINE` +lands every access in one or two banks. With `BANK_HASH=1` the bank is the raw +bank bits XORed with `cache_bank_hash` of the line address above them +([VX_gpu_pkg.sv](../../hw/rtl/VX_gpu_pkg.sv)): IPOLY for 16 and 32 banks, which +spreads every power-of-two stride evenly, and an XOR fold of the higher bits for +other bank counts. The same mask XORed again restores the raw bits, so it is +applied once on the request side (`core_req_bid`) and once where the memory +address is rebuilt from `{line, bank}`. The banks keep storing the line address +without the bank field, so tags, MSHR, the flush walk and sectoring are +unchanged. `VX_CFG_DCACHE_BANK_HASH` and `VX_CFG_L2_BANK_HASH` set it for the +dcache and the L2; SimX uses the same functions. The L2 default is on; the dcache +default is on from 4 banks, because with two banks the hash can only swap lines on +parity and the extra parallel misses cost more in memory queuing than the +spreading saves. + The memory side transacts in sectors, so the memory address carries `CS_SECTOR_SEL_BITS` below the line address (`CS_MEM_SECTOR_ADDR_WIDTH`, `CS_LINE_SECTOR_ADDR_WIDTH`). With one sector per line these collapse back to the @@ -1001,8 +1017,8 @@ identically at all three levels — the dcache is not special. - **Miss latency hiding** insufficient (banks idle while misses pend) → raise `*_MSHR_SIZE` and check the LSU pending pool upstream; verify `MEM_PORTS` is not serializing the drain. -- **Bank conflicts** (hit-path stalls) → more banks via a smaller - `DCACHE_WORD_SIZE`; check address strides against the line-interleaved mapping. +- **Bank conflicts** (hit-path stalls) → check that the bank hash is on for that + level, then more banks via a smaller `DCACHE_WORD_SIZE`. - **Store bandwidth** bound → enable a shared level so `WRITEBACK` derives to 1 where legal; add `*_DIRTYBYTES` for partial-line store patterns. - **BRAM / timing** pressure → cut `*_NUM_WAYS` or `*_SIZE` before cutting banks; diff --git a/hw/rtl/VX_cluster.sv b/hw/rtl/VX_cluster.sv index 92c3cb4e23..f02c30047b 100644 --- a/hw/rtl/VX_cluster.sv +++ b/hw/rtl/VX_cluster.sv @@ -232,6 +232,7 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; .NC_ENABLE (1), .PASSTHRU (!`VX_CFG_L2_ENABLED), .IS_LLC (L2_IS_LLC), + .BANK_HASH (`VX_CFG_L2_BANK_HASH), .AMO_ENABLE (`VX_CFG_EXT_A_ENABLED) ) l2cache ( .clk (clk), diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index 6a09fcfbea..9ae6a9a2a9 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -224,6 +224,34 @@ package VX_gpu_pkg; // wire/reg sizing for any per-cache plumbing. localparam MEM_ATTR_WIDTH = $bits(mem_bus_attr_t); + // Cache bank-select hash: XORed into the raw bank bits, a function of the + // line address above them, so XORing it again restores the raw bits. IPOLY + // for 16 and 32 banks (Rau, ISCA 1991: conflict-free for power-of-two + // strides); an XOR fold of the bits above the bank field otherwise. + function automatic logic [7:0] cache_bank_hash(input logic [31:0] hi, input integer bits); + logic [7:0] f; + f = '0; + if (bits == 4) begin + f[0] = hi[11] ^ hi[10] ^ hi[9] ^ hi[8] ^ hi[6] ^ hi[4] ^ hi[3] ^ hi[0]; + f[1] = hi[12] ^ hi[8] ^ hi[7] ^ hi[6] ^ hi[5] ^ hi[3] ^ hi[1] ^ hi[0]; + f[2] = hi[9] ^ hi[8] ^ hi[7] ^ hi[6] ^ hi[4] ^ hi[2] ^ hi[1]; + f[3] = hi[10] ^ hi[9] ^ hi[8] ^ hi[7] ^ hi[5] ^ hi[3] ^ hi[2]; + end else if (bits == 5) begin + f[0] = hi[13] ^ hi[12] ^ hi[11] ^ hi[10] ^ hi[9] ^ hi[6] ^ hi[5] ^ hi[3] ^ hi[0]; + f[1] = hi[14] ^ hi[13] ^ hi[12] ^ hi[11] ^ hi[10] ^ hi[7] ^ hi[6] ^ hi[4] ^ hi[1]; + f[2] = hi[14] ^ hi[10] ^ hi[9] ^ hi[8] ^ hi[7] ^ hi[6] ^ hi[3] ^ hi[2] ^ hi[0]; + f[3] = hi[11] ^ hi[10] ^ hi[9] ^ hi[8] ^ hi[7] ^ hi[4] ^ hi[3] ^ hi[1]; + f[4] = hi[12] ^ hi[11] ^ hi[10] ^ hi[9] ^ hi[8] ^ hi[5] ^ hi[4] ^ hi[2]; + end else if (bits > 0 && bits <= 8) begin + for (integer i = 0; i < 15; ++i) begin + if ((i % bits) == 0) begin + f = f ^ (8'(hi >> i) & 8'((1 << bits) - 1)); + end + end + end + return f; + endfunction + localparam VX_DCR_ADDR_WIDTH = `VX_DCR_ADDR_BITS; localparam VX_DCR_DATA_WIDTH = `VX_DCR_DATA_BITS; diff --git a/hw/rtl/VX_socket.sv b/hw/rtl/VX_socket.sv index 652a4509db..315ccf713a 100644 --- a/hw/rtl/VX_socket.sv +++ b/hw/rtl/VX_socket.sv @@ -220,6 +220,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .CORE_OUT_BUF (3), .MEM_OUT_BUF (3), .IS_LLC (DCACHE_IS_LLC), + .BANK_HASH (`VX_CFG_DCACHE_BANK_HASH), .AMO_ENABLE (`VX_CFG_EXT_A_ENABLED) ) dcache ( `ifdef PERF_ENABLE diff --git a/hw/rtl/cache/VX_cache.sv b/hw/rtl/cache/VX_cache.sv index 46659a46b4..de35722db6 100644 --- a/hw/rtl/cache/VX_cache.sv +++ b/hw/rtl/cache/VX_cache.sv @@ -51,6 +51,8 @@ module VX_cache import VX_gpu_pkg::*; #( parameter FLUSH_FORWARD = 1, // a flush also flushes the next level + parameter BANK_HASH = 0, // hashed bank select (cache_bank_hash) + parameter AMO_ENABLE = 0 // synthesize atomic-op logic in the banks ) ( `ifdef PERF_ENABLE @@ -65,6 +67,7 @@ module VX_cache import VX_gpu_pkg::*; #( `STATIC_ASSERT(WRITE_ENABLE || !WRITEBACK, ("invalid parameter: writeback requires write enable")) `STATIC_ASSERT(WRITEBACK || !DIRTY_BYTES, ("invalid parameter: dirty bytes require writeback")) `STATIC_ASSERT(NUM_BANKS >= MEM_PORTS, ("invalid parameter: number of banks must be greater or equal to number of memory ports")) + `STATIC_ASSERT(!BANK_HASH || NUM_BANKS <= 256, ("invalid parameter: bank hash supports up to 256 banks")) localparam REQ_SEL_WIDTH = `UP(`CS_REQ_SEL_BITS); localparam WORD_SEL_WIDTH = `UP(`CS_WORD_SEL_BITS); localparam MSHR_ADDR_WIDTH = `LOG2UP(MSHR_SIZE); @@ -78,7 +81,7 @@ module VX_cache import VX_gpu_pkg::*; #( localparam CORE_REQ_DATAW = LINE_ADDR_WIDTH + 1 + WORD_SEL_WIDTH + WORD_SIZE + WORD_WIDTH + TAG_WIDTH + `UP(MEM_ATTR_WIDTH); localparam CORE_RSP_DATAW = WORD_WIDTH + TAG_WIDTH; localparam BANK_MEM_TAG_WIDTH = UUID_WIDTH + MSHR_ADDR_WIDTH; - localparam MEM_REQ_DATAW = (`CS_LINE_SECTOR_ADDR_WIDTH + 1 + SECTOR_SIZE + `CS_SECTOR_WIDTH + BANK_MEM_TAG_WIDTH + `UP(MEM_ATTR_WIDTH)); + localparam MEM_REQ_DATAW = (`UP(BANK_SEL_BITS) + `CS_LINE_SECTOR_ADDR_WIDTH + 1 + SECTOR_SIZE + `CS_SECTOR_WIDTH + BANK_MEM_TAG_WIDTH + `UP(MEM_ATTR_WIDTH)); localparam MEM_RSP_DATAW = `CS_SECTOR_WIDTH + MEM_TAG_WIDTH; localparam MEM_PORTS_SEL_BITS = `CLOG2(MEM_PORTS); localparam MEM_PORTS_SEL_WIDTH = `UP(MEM_PORTS_SEL_BITS); @@ -283,7 +286,12 @@ module VX_cache import VX_gpu_pkg::*; #( end for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_req_bid if (NUM_BANKS > 1) begin : g_multibanks - assign core_req_bid[i] = core_req_addr[i][WORD_SEL_BITS +: BANK_SEL_BITS]; + wire [BANK_SEL_BITS-1:0] raw_bid = core_req_addr[i][WORD_SEL_BITS +: BANK_SEL_BITS]; + if (BANK_HASH != 0) begin : g_hash + assign core_req_bid[i] = raw_bid ^ BANK_SEL_BITS'(cache_bank_hash(32'(core_req_line_addr[i]), BANK_SEL_BITS)); + end else begin : g_raw + assign core_req_bid[i] = raw_bid; + end end else begin : g_singlebank assign core_req_bid[i] = '0; end @@ -341,6 +349,7 @@ module VX_cache import VX_gpu_pkg::*; #( for (genvar bank_id = 0; bank_id < NUM_BANKS; ++bank_id) begin : g_banks VX_cache_bank #( .BANK_ID (bank_id), + .BANK_HASH (BANK_HASH), .INSTANCE_ID (`SFORMATF(("%s-bank%0d", INSTANCE_ID, bank_id))), .CACHE_SIZE (CACHE_SIZE), .LINE_SIZE (LINE_SIZE), @@ -459,7 +468,16 @@ module VX_cache import VX_gpu_pkg::*; #( end wire [NUM_BANKS-1:0][MEM_REQ_DATAW-1:0] per_bank_mem_req_pdata; for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_per_bank_mem_req_pdata + // the raw bank bits of the memory address, formed while the bank id is a constant + wire [`UP(BANK_SEL_BITS)-1:0] raw_bid; + if (NUM_BANKS > 1 && BANK_HASH != 0) begin : g_hash + wire [`CS_LINE_ADDR_WIDTH-1:0] line_addr = per_bank_mem_req_addr[i][`CS_LINE_SECTOR_ADDR_WIDTH-1 -: `CS_LINE_ADDR_WIDTH]; + assign raw_bid = BANK_SEL_BITS'(i) ^ BANK_SEL_BITS'(cache_bank_hash(32'(line_addr), BANK_SEL_BITS)); + end else begin : g_raw + assign raw_bid = `UP(BANK_SEL_BITS)'(i); + end assign per_bank_mem_req_pdata[i] = { + raw_bid, per_bank_mem_req_addr[i], per_bank_mem_req_rw[i], per_bank_mem_req_byteen[i], @@ -487,16 +505,18 @@ module VX_cache import VX_gpu_pkg::*; #( wire [NUM_BANKS-1:0] mem_req_xbar_valid; wire [NUM_BANKS-1:0][MEM_REQ_DATAW-1:0] mem_req_xbar_pdata; wire [NUM_BANKS-1:0] mem_req_xbar_ready; - wire [MEM_REQ_DATAW-1:0] mem_flush_pdata = { - `CS_LINE_SECTOR_ADDR_WIDTH'(0), - 1'b0, - SECTOR_SIZE'(0), - `CS_SECTOR_WIDTH'(0), - BANK_MEM_TAG_WIDTH'(0), - `UP(MEM_ATTR_WIDTH)'(1 << MEM_ATTR_FLUSH_OFFS) - }; for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_mem_req_xbar_in if (i < MEM_PORTS) begin : g_flush_req + // line 0 hashes to 0, so the raw bank bits are the lane's bank id + wire [MEM_REQ_DATAW-1:0] mem_flush_pdata = { + `UP(BANK_SEL_BITS)'(i), + `CS_LINE_SECTOR_ADDR_WIDTH'(0), + 1'b0, + SECTOR_SIZE'(0), + `CS_SECTOR_WIDTH'(0), + BANK_MEM_TAG_WIDTH'(0), + `UP(MEM_ATTR_WIDTH)'(1 << MEM_ATTR_FLUSH_OFFS) + }; assign mem_req_xbar_valid[i] = per_bank_mem_req_valid[i] || mem_flush_valid[i]; assign mem_req_xbar_pdata[i] = per_bank_mem_req_valid[i] ? per_bank_mem_req_pdata[i] : mem_flush_pdata; assign mem_flush_ready[i] = mem_req_xbar_ready[i] && ~per_bank_mem_req_valid[i]; @@ -558,6 +578,7 @@ module VX_cache import VX_gpu_pkg::*; #( `UNUSED_PIN (collisions) ); for (genvar i = 0; i < MEM_PORTS; ++i) begin : g_mem_bus_tmp_if + wire [`UP(BANK_SEL_BITS)-1:0] mem_req_raw_bid_w; wire mem_req_rw_w; wire [`CS_LINE_SECTOR_ADDR_WIDTH-1:0] mem_req_addr_w; wire [SECTOR_SIZE-1:0] mem_req_byteen_w; @@ -565,6 +586,7 @@ module VX_cache import VX_gpu_pkg::*; #( wire [BANK_MEM_TAG_WIDTH-1:0] mem_req_tag_w; wire [`UP(MEM_ATTR_WIDTH)-1:0] mem_req_attr_w; assign { + mem_req_raw_bid_w, mem_req_addr_w, mem_req_rw_w, mem_req_byteen_w, @@ -588,17 +610,16 @@ module VX_cache import VX_gpu_pkg::*; #( if (NUM_BANKS != MEM_PORTS) begin : g_arb_sel wire [MEM_ARB_SEL_BITS-1:0] mem_req_arb_sel; assign mem_req_arb_sel = mem_req_tmp_idx[i][`LOG2UP(NUM_BANKS)-1 -: MEM_ARB_SEL_BITS]; - assign mem_bus_tmp_if[i].req_data.addr = (`CS_MEM_SECTOR_ADDR_WIDTH'({mem_req_line_w, mem_req_tmp_idx[i]}) << `CS_SECTOR_SEL_BITS) - | `CS_MEM_SECTOR_ADDR_WIDTH'(mem_req_sec_w); assign mem_bus_tmp_if[i].req_data.tag = {mem_req_tag_w, mem_req_arb_sel}; end else begin : g_no_arb_sel `UNUSED_VAR (mem_req_tmp_idx) - assign mem_bus_tmp_if[i].req_data.addr = (`CS_MEM_SECTOR_ADDR_WIDTH'({mem_req_line_w, MEM_PORTS_SEL_WIDTH'(i)}) << `CS_SECTOR_SEL_BITS) - | `CS_MEM_SECTOR_ADDR_WIDTH'(mem_req_sec_w); assign mem_bus_tmp_if[i].req_data.tag = MEM_TAG_WIDTH'(mem_req_tag_w); end + assign mem_bus_tmp_if[i].req_data.addr = (`CS_MEM_SECTOR_ADDR_WIDTH'({mem_req_line_w, mem_req_raw_bid_w}) << `CS_SECTOR_SEL_BITS) + | `CS_MEM_SECTOR_ADDR_WIDTH'(mem_req_sec_w); end else begin : g_singlebank `UNUSED_VAR (mem_req_tmp_idx) + `UNUSED_VAR (mem_req_raw_bid_w) `UNUSED_VAR (mem_req_line_w) `UNUSED_VAR (mem_req_sec_w) assign mem_bus_tmp_if[i].req_data.addr = `CS_MEM_SECTOR_ADDR_WIDTH'(mem_req_addr_w); diff --git a/hw/rtl/cache/VX_cache_bank.sv b/hw/rtl/cache/VX_cache_bank.sv index ee64fe7937..410a61dcc6 100644 --- a/hw/rtl/cache/VX_cache_bank.sv +++ b/hw/rtl/cache/VX_cache_bank.sv @@ -16,6 +16,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( parameter `STRING INSTANCE_ID= "", parameter BANK_ID = 0, + parameter BANK_HASH = 0, // hashed bank select (cache_bank_hash) parameter NUM_REQS = 1, parameter CACHE_SIZE = 1024, // cache size in bytes parameter LINE_SIZE = 16, // line size in bytes @@ -834,6 +835,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( VX_cache_mshr #( .INSTANCE_ID (`SFORMATF(("%s-mshr", INSTANCE_ID))), .BANK_ID (BANK_ID), + .BANK_HASH (BANK_HASH), .LINE_SIZE (LINE_SIZE), .SECTOR_SIZE (SECTOR_SIZE), .NUM_BANKS (NUM_BANKS), @@ -1316,7 +1318,9 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire [`VX_CFG_XLEN-1:0] core_req_full_addr = `CS_BANK_TO_FULL_ADDR(core_req_addr, BANK_ID); wire [`VX_CFG_XLEN-1:0] full_addr_st0 = `CS_BANK_TO_FULL_ADDR(st0.req.addr, BANK_ID); wire [`VX_CFG_XLEN-1:0] full_addr_st1 = `CS_BANK_TO_FULL_ADDR(st1.req.addr, BANK_ID); - wire [`VX_CFG_XLEN-1:0] mreq_queue_full_addr = `CS_BANK_TO_FULL_ADDR(mreq_queue_addr, BANK_ID); + wire [`CS_LINE_ADDR_WIDTH-1:0] mreq_queue_line = mreq_queue_addr[`CS_LINE_SECTOR_ADDR_WIDTH-1 -: `CS_LINE_ADDR_WIDTH]; + wire [`VX_CFG_XLEN-1:0] mreq_queue_full_addr = `CS_BANK_TO_FULL_ADDR(mreq_queue_line, BANK_ID) + | ((`VX_CFG_XLEN'(mreq_queue_addr) & `VX_CFG_XLEN'((1 << `CS_SECTOR_SEL_BITS) - 1)) << `CLOG2(SECTOR_SIZE)); always @(posedge clk) begin if (input_stall || pipe_stall) begin diff --git a/hw/rtl/cache/VX_cache_cluster.sv b/hw/rtl/cache/VX_cache_cluster.sv index 2efb8ab859..31868bcd1b 100644 --- a/hw/rtl/cache/VX_cache_cluster.sv +++ b/hw/rtl/cache/VX_cache_cluster.sv @@ -56,6 +56,7 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( parameter IS_LLC = 0, // Indicates this cache cluster is the LLC (handles AMO completion). parameter FLUSH_FORWARD = 1, // A flush also flushes the next level. + parameter BANK_HASH = 0, // Hashed bank select (cache_bank_hash). parameter AMO_ENABLE = 0 // Enable atomic memory operations. ) ( @@ -147,6 +148,7 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( .PASSTHRU (PASSTHRU), .IS_LLC (IS_LLC), .FLUSH_FORWARD (FLUSH_FORWARD), + .BANK_HASH (BANK_HASH), .AMO_ENABLE (AMO_ENABLE) ) cache_wrap ( `ifdef PERF_ENABLE diff --git a/hw/rtl/cache/VX_cache_define.vh b/hw/rtl/cache/VX_cache_define.vh index 3f904816fc..c3ccbb49b0 100644 --- a/hw/rtl/cache/VX_cache_define.vh +++ b/hw/rtl/cache/VX_cache_define.vh @@ -74,7 +74,8 @@ /////////////////////////////////////////////////////////////////////////////// -`define CS_BANK_TO_FULL_ADDR(x, b) {x, (`VX_CFG_XLEN-$bits(x))'(b << (`VX_CFG_XLEN-$bits(x)-`CS_BANK_SEL_BITS))} +// b is the bank id; with BANK_HASH the raw bank bits are b XOR the line's hash +`define CS_BANK_TO_FULL_ADDR(x, b) {x, (`VX_CFG_XLEN-$bits(x))'(((b) ^ (BANK_HASH ? int'(cache_bank_hash(32'(x), `CS_BANK_SEL_BITS)) : 0)) << (`VX_CFG_XLEN-$bits(x)-`CS_BANK_SEL_BITS))} `define CS_MEM_TO_FULL_ADDR(x) {x, (`VX_CFG_XLEN-$bits(x))'(0)} /////////////////////////////////////////////////////////////////////////////// diff --git a/hw/rtl/cache/VX_cache_mshr.sv b/hw/rtl/cache/VX_cache_mshr.sv index 22605fd148..e088fe95fc 100644 --- a/hw/rtl/cache/VX_cache_mshr.sv +++ b/hw/rtl/cache/VX_cache_mshr.sv @@ -45,6 +45,7 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( parameter `STRING INSTANCE_ID= "", parameter BANK_ID = 0, + parameter BANK_HASH = 0, // hashed bank select (cache_bank_hash) parameter LINE_SIZE = 16, // Size of line inside a bank in bytes parameter SECTOR_SIZE = LINE_SIZE, // Size of a sector in bytes (coalescing/fill granule); = LINE_SIZE => 1 sector parameter NUM_BANKS = 1, // Number of banks @@ -128,6 +129,7 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( output wire empty ); `UNUSED_PARAM (BANK_ID) + `UNUSED_PARAM (BANK_HASH) reg [`CS_LINE_ADDR_WIDTH-1:0] addr_table [0:MSHR_SIZE-1]; // Per-entry sector. Coalescing matches on {line, sector} so same-line diff --git a/hw/rtl/cache/VX_cache_wrap.sv b/hw/rtl/cache/VX_cache_wrap.sv index 6aa1266e7a..24ca0f7a62 100644 --- a/hw/rtl/cache/VX_cache_wrap.sv +++ b/hw/rtl/cache/VX_cache_wrap.sv @@ -56,6 +56,7 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( parameter IS_LLC = 0, // Set when this cache is the LLC (enables AMO completion at this level). parameter FLUSH_FORWARD = 1, // A flush also flushes the next level. + parameter BANK_HASH = 0, // Hashed bank select (cache_bank_hash). parameter AMO_ENABLE = 0 // Enable atomic operations in cache banks. ) ( @@ -149,6 +150,7 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( .MEM_OUT_BUF (BYPASS_ENABLE ? 1 : MEM_OUT_BUF), .IS_LLC (IS_LLC), .FLUSH_FORWARD (FLUSH_FORWARD), + .BANK_HASH (BANK_HASH), .AMO_ENABLE (AMO_ENABLE) ) cache ( .clk (clk), diff --git a/hw/unittest/cache/VX_cache_top.sv b/hw/unittest/cache/VX_cache_top.sv index b58bd48dd5..0e25d21b96 100644 --- a/hw/unittest/cache/VX_cache_top.sv +++ b/hw/unittest/cache/VX_cache_top.sv @@ -71,6 +71,9 @@ module VX_cache_top import VX_gpu_pkg::*; #( // Memory request output buffer parameter MEM_OUT_BUF = 3, + // Hashed bank select + parameter BANK_HASH = (NUM_BANKS >= 4), + parameter MEM_TAG_WIDTH = `CACHE_MEM_TAG_WIDTH(MSHR_SIZE, NUM_BANKS, MEM_PORTS, UUID_WIDTH) ) ( input wire clk, @@ -182,6 +185,7 @@ module VX_cache_top import VX_gpu_pkg::*; #( .DIRTY_BYTES (DIRTY_BYTES), .AMO_ENABLE (AMO_ENABLE), .IS_LLC (IS_LLC), + .BANK_HASH (BANK_HASH), .CORE_OUT_BUF (CORE_OUT_BUF), .MEM_OUT_BUF (MEM_OUT_BUF) ) cache ( diff --git a/sim/simx/cluster.cpp b/sim/simx/cluster.cpp index 937c2ae131..d8451f2618 100644 --- a/sim/simx/cluster.cpp +++ b/sim/simx/cluster.cpp @@ -84,6 +84,7 @@ class Cluster::Impl { VX_CFG_L2_LATENCY, // pipeline latency VX_CFG_L2_REPL_POLICY, // replacement policy (VX_CFG_L2_ENABLED != 0) && (VX_CFG_L3_ENABLED == 0), // is_llc + (VX_CFG_L2_BANK_HASH != 0) && (VX_CFG_L2_NUM_BANKS > 1), // bank_hash }); diff --git a/sim/simx/mem/addr_hash.h b/sim/simx/mem/addr_hash.h new file mode 100644 index 0000000000..e9117edd33 --- /dev/null +++ b/sim/simx/mem/addr_hash.h @@ -0,0 +1,46 @@ +// Copyright © 2019-2026 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +namespace vortex { + +// IPOLY index hashing (Rau, "Pseudo-randomly interleaved memory", ISCA 1991), +// with the GF(2) equations GPGPU-Sim uses for its memory partitions and DRAM +// banks. Each returns the mask f(hi) that is XORed into the index; f depends +// only on the higher bits, so the same XOR restores the original index. + +inline uint32_t ipoly_bit(uint64_t v, int i) { return uint32_t(v >> i) & 1u; } + +// IPOLY(37), 32 sets. +inline uint32_t ipoly32(uint64_t a) { + uint32_t i0 = ipoly_bit(a,13)^ipoly_bit(a,12)^ipoly_bit(a,11)^ipoly_bit(a,10)^ipoly_bit(a,9)^ipoly_bit(a,6)^ipoly_bit(a,5)^ipoly_bit(a,3)^ipoly_bit(a,0); + uint32_t i1 = ipoly_bit(a,14)^ipoly_bit(a,13)^ipoly_bit(a,12)^ipoly_bit(a,11)^ipoly_bit(a,10)^ipoly_bit(a,7)^ipoly_bit(a,6)^ipoly_bit(a,4)^ipoly_bit(a,1); + uint32_t i2 = ipoly_bit(a,14)^ipoly_bit(a,10)^ipoly_bit(a,9)^ipoly_bit(a,8)^ipoly_bit(a,7)^ipoly_bit(a,6)^ipoly_bit(a,3)^ipoly_bit(a,2)^ipoly_bit(a,0); + uint32_t i3 = ipoly_bit(a,11)^ipoly_bit(a,10)^ipoly_bit(a,9)^ipoly_bit(a,8)^ipoly_bit(a,7)^ipoly_bit(a,4)^ipoly_bit(a,3)^ipoly_bit(a,1); + uint32_t i4 = ipoly_bit(a,12)^ipoly_bit(a,11)^ipoly_bit(a,10)^ipoly_bit(a,9)^ipoly_bit(a,8)^ipoly_bit(a,5)^ipoly_bit(a,4)^ipoly_bit(a,2); + return i0 | (i1 << 1) | (i2 << 2) | (i3 << 3) | (i4 << 4); +} + +// IPOLY(5), 16 sets. +inline uint32_t ipoly16(uint64_t a) { + uint32_t i0 = ipoly_bit(a,11)^ipoly_bit(a,10)^ipoly_bit(a,9)^ipoly_bit(a,8)^ipoly_bit(a,6)^ipoly_bit(a,4)^ipoly_bit(a,3)^ipoly_bit(a,0); + uint32_t i1 = ipoly_bit(a,12)^ipoly_bit(a,8)^ipoly_bit(a,7)^ipoly_bit(a,6)^ipoly_bit(a,5)^ipoly_bit(a,3)^ipoly_bit(a,1)^ipoly_bit(a,0); + uint32_t i2 = ipoly_bit(a,9)^ipoly_bit(a,8)^ipoly_bit(a,7)^ipoly_bit(a,6)^ipoly_bit(a,4)^ipoly_bit(a,2)^ipoly_bit(a,1); + uint32_t i3 = ipoly_bit(a,10)^ipoly_bit(a,9)^ipoly_bit(a,8)^ipoly_bit(a,7)^ipoly_bit(a,5)^ipoly_bit(a,3)^ipoly_bit(a,2); + return i0 | (i1 << 1) | (i2 << 2) | (i3 << 3); +} + +} diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index 8c4dcfc0a8..81b10a3692 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -15,6 +15,7 @@ #include "mem_block_pool.h" #include "debug.h" #include "types.h" +#include "addr_hash.h" #if VX_CFG_EXT_A_ENABLED #include "amo_unit.h" #endif @@ -50,7 +51,9 @@ struct params_t { int32_t tag_select_addr_start; int32_t tag_select_addr_end; - params_t(const Cache::Config &config) { + bool bank_hash; + + params_t(const Cache::Config &config) : bank_hash(config.bank_hash) { int32_t offset_bits = config.L - config.W; int32_t index_bits = config.C - (config.L + config.A + config.B); assert(offset_bits >= 0); @@ -85,13 +88,35 @@ struct params_t { // Tag select this->tag_select_addr_start = (1 + this->set_select_addr_end); this->tag_select_addr_end = (config.addr_width - 1); + assert(!bank_hash || (config.B != 0 && config.B <= 8)); + } + + // The bank is the raw bank bits XORed with a function of the bits above + // them, so power-of-two strides spread over the banks; applying the same + // XOR again restores the raw bits. IPOLY for 16/32 banks; other bank counts + // XOR-fold the higher bits (Frailong, Jalby, Lenfant, "XOR-schemes", ICPP 1985). + uint32_t bank_xor(uint64_t addr, uint32_t raw) const { + uint32_t bits = bank_select_addr_end - bank_select_addr_start + 1; + uint64_t hi = addr >> (bank_select_addr_end + 1); + if (bits == 4) { + return raw ^ ipoly16(hi); + } + if (bits == 5) { + return raw ^ ipoly32(hi); + } + uint32_t mask = (1u << bits) - 1, f = 0; + for (uint32_t i = 0; i < 15; i += bits) { + f ^= uint32_t(hi >> i) & mask; + } + return raw ^ f; } uint32_t addr_bank_id(uint64_t addr) const { - if (bank_select_addr_end >= bank_select_addr_start) - return (uint32_t)bit_getw(addr, bank_select_addr_start, bank_select_addr_end); - else + if (bank_select_addr_end < bank_select_addr_start) { return 0; + } + uint32_t raw = (uint32_t)bit_getw(addr, bank_select_addr_start, bank_select_addr_end); + return bank_hash ? bank_xor(addr, raw) : raw; } uint32_t addr_set_id(uint64_t addr) const { @@ -123,6 +148,10 @@ struct params_t { addr = bit_setw(addr, set_select_addr_start, set_select_addr_end, set_id); if (tag_select_addr_end >= tag_select_addr_start) addr = bit_setw(addr, tag_select_addr_start, tag_select_addr_end, tag); + if (bank_hash && bank_select_addr_end >= bank_select_addr_start) { + uint32_t raw = bank_xor(addr, bank_id); + addr = bit_setw(addr, bank_select_addr_start, bank_select_addr_end, raw); + } return addr; } diff --git a/sim/simx/mem/cache.h b/sim/simx/mem/cache.h index 70ad9e6e3f..8729a52e88 100644 --- a/sim/simx/mem/cache.h +++ b/sim/simx/mem/cache.h @@ -43,6 +43,7 @@ class Cache : public SimObject { uint16_t latency; // pipeline latency uint8_t repl_policy; // replacement policy (ReplPolicy) bool is_llc; // is last-level cache + bool bank_hash = false; // hashed bank select }; struct PerfStats { diff --git a/sim/simx/socket.cpp b/sim/simx/socket.cpp index 73f8bd0625..38ea84f880 100644 --- a/sim/simx/socket.cpp +++ b/sim/simx/socket.cpp @@ -91,6 +91,7 @@ class Socket::Impl { VX_CFG_DCACHE_LATENCY, // pipeline latency (capacity-scaled, matches hardware) VX_CFG_DCACHE_REPL_POLICY, // replacement policy (VX_CFG_DCACHE_ENABLED != 0) && (VX_CFG_L2_ENABLED == 0) && (VX_CFG_L3_ENABLED == 0), // is_llc + (VX_CFG_DCACHE_BANK_HASH != 0) && (VX_CFG_DCACHE_NUM_BANKS > 1), // bank_hash }); #ifdef VX_CFG_EXT_TEX_ENABLE From bdf39536d2c146452225ff1b4b084bd3cca64b23 Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 14 Oct 2026 19:05:33 -0700 Subject: [PATCH 032/124] Keep the other sectors' dirty bytes when a sector refills a line With DIRTY_BYTES=1 and more than one sector per line, every fill cleared the whole line's byte mask, but a refill (a sector installed into a line that is already resident) leaves the other sectors resident and the tags keep their dirty bits. A dirty sibling sector then wrote back with byteen 0 and DRAM dropped its data: mstress on 4 cores with a write-back, sectored L2 and L2_DIRTYBYTES=1 read back never-written memory in 2048 results, every line losing its first sector. A refill now clears only the installed sector's mask; a fresh install still clears the whole line. This matches the tags and SimX, which already reset one sector on a refill. One-sector caches are unchanged, since a refill cannot happen there. The commit-stage check compared line-wide ORs, which a dirty sibling satisfies; a per-sector check now asserts that every dirty sector carries dirty bytes. With the fix reverted it fires on the first lost beat. Tests: cache:mstress-6 (the failing config) on SimX and rtlsim. Also run on rtlsim: mstress-1/2/4, bank_hash-4, L2 dirty bytes off, and L2+L3 write-back with dirty bytes. Cycle counts are unchanged. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/cache.yaml | 12 ++++++++++++ hw/rtl/cache/VX_cache_bank.sv | 9 +++++++++ hw/rtl/cache/VX_cache_data.sv | 13 ++++++++++++- 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/ci/testcases/cache.yaml b/ci/testcases/cache.yaml index 3a2c5b7088..d78b7f904d 100644 --- a/ci/testcases/cache.yaml +++ b/ci/testcases/cache.yaml @@ -196,6 +196,18 @@ tests: clusters: 2 l2cache: true l3cache: true +# Dirty-byte write-back on a sectored L2: a sector refill keeps the other +# sectors' dirty bytes. +- id: mstress-6 + via: blackbox + drivers: + - simx + - rtlsim + app: mstress + configs: -DVX_CFG_SOCKET_SIZE=1 -DVX_CFG_L2_WRITEBACK=1 -DVX_CFG_L2_DIRTYBYTES=1 + shape: + cores: 4 + l2cache: true - id: sgemm-20 via: blackbox drivers: diff --git a/hw/rtl/cache/VX_cache_bank.sv b/hw/rtl/cache/VX_cache_bank.sv index 410a61dcc6..cb3f748a3b 100644 --- a/hw/rtl/cache/VX_cache_bank.sv +++ b/hw/rtl/cache/VX_cache_bank.sv @@ -144,6 +144,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( req_t req; logic [`CS_WORD_WIDTH-1:0] wdata; logic [NUM_WAYS-1:0] tag_matches; + logic is_refill; // fill into an already-resident line } data_t; typedef struct packed { // response + memory request (S0 -> S1 -> stC) @@ -664,6 +665,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( dat_in = st0; dat_in.req.way_idx = evict_way_st0; dat_in.tag_matches = tag_matches_st0 & {NUM_WAYS{~creq_hit_order_hazard_st0}}; + dat_in.is_refill = lk_st0.is_refill; end // commit path: the request (with the resolved hit/victim way and MSHR id) @@ -752,6 +754,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .reset (reset), .init (do_init_std), .fill (do_fill_std && ~stD.req.is_passthru_fill && ~pipe_stall), + .refill (stD.is_refill), .flush (do_flush_std && ~pipe_stall), .read (do_read_std && ~pipe_stall), .write (do_write_std && ~pipe_stall), @@ -1226,6 +1229,12 @@ module VX_cache_bank import VX_gpu_pkg::*; #( if (DIRTY_BYTES) begin : g_dirty_bytes wire has_dirty_bytes = (| evict_byteen_stc); `RUNTIME_ASSERT (~do_fill_or_flush_stc || (stC.lk.is_dirty == has_dirty_bytes), ("missmatch dirty bytes: dirty_line=%b, dirty_bytes=%b, addr=0x%0h", stC.lk.is_dirty, has_dirty_bytes, `CS_BANK_TO_FULL_ADDR(addr_stc, BANK_ID))) + // per-sector: a dirty sector must carry dirty bytes, or its writeback beat is dropped. + wire [SEC-1:0] sec_has_dirty_bytes; + for (genvar s = 0; s < SEC; ++s) begin : g_sec_has_dirty_bytes + assign sec_has_dirty_bytes[s] = (| evict_byteen_stc[s*SECTOR_SIZE +: SECTOR_SIZE]); + end + `RUNTIME_ASSERT (~do_fill_or_flush_stc || ((stC.lk.evict_dirty_mask & ~sec_has_dirty_bytes) == '0), ("dirty sector without dirty bytes: dirty_mask=%b, dirty_bytes=%b, addr=0x%0h", stC.lk.evict_dirty_mask, sec_has_dirty_bytes, `CS_BANK_TO_FULL_ADDR(addr_stc, BANK_ID))) end // fill request on a read/write miss (one sector); multi-beat writeback // on a dirty eviction. The two are mutually exclusive (a writeback is a diff --git a/hw/rtl/cache/VX_cache_data.sv b/hw/rtl/cache/VX_cache_data.sv index abc3472ba3..d30d289f13 100644 --- a/hw/rtl/cache/VX_cache_data.sv +++ b/hw/rtl/cache/VX_cache_data.sv @@ -46,6 +46,7 @@ module VX_cache_data import VX_gpu_pkg::*; #( // inputs input wire init, input wire fill, + input wire refill, // fill into an already-resident line input wire flush, input wire read, input wire write, @@ -108,10 +109,19 @@ module VX_cache_data import VX_gpu_pkg::*; #( assign write_mask[i] = write_byteen & {WORD_SIZE{word_en}}; end + // A sector refill must clear only the installed sector's mask: the other + // resident sectors may hold dirty bytes not yet written back. + wire [`CS_SECTORS_PER_LINE-1:0][SECTOR_SIZE-1:0] fill_clr_mask; + for (genvar i = 0; i < `CS_SECTORS_PER_LINE; ++i) begin : g_fill_clr_mask + wire sec_en = ~refill || (`CS_SECTORS_PER_LINE == 1) + || (sector_idx == `UP(`CS_SECTOR_SEL_BITS)'(i)); + assign fill_clr_mask[i] = {SECTOR_SIZE{fill && sec_en}}; + end + wire [NUM_WAYS-1:0][LINE_SIZE-1:0] byteen_rdata; for (genvar i = 0; i < NUM_WAYS; ++i) begin : g_byteen_store wire [LINE_SIZE-1:0] byteen_wdata = {LINE_SIZE{write}}; // only asserted on writes - wire [LINE_SIZE-1:0] byteen_wren = {LINE_SIZE{init || fill || flush}} | write_mask; + wire [LINE_SIZE-1:0] byteen_wren = {LINE_SIZE{init || flush}} | fill_clr_mask | write_mask; wire byteen_write = ((fill || flush) && ((NUM_WAYS == 1) || (evict_way == i))) || (write && tag_matches[i]) || init; @@ -141,6 +151,7 @@ module VX_cache_data import VX_gpu_pkg::*; #( assign evict_byteen = byteen_rdata[way_idx_r]; end else begin : g_no_dirty_bytes `UNUSED_VAR (init) + `UNUSED_VAR (refill) `UNUSED_VAR (flush) `UNUSED_VAR (way_idx_r) assign evict_byteen = '1; // update whole line From f9dfa6a13be344f3694d03a3105906cb0a86b31a Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 14 Oct 2026 23:37:18 -0700 Subject: [PATCH 033/124] Let fills follow each other into a cache bank pipe back to back A bank with more than two pipeline stages (PIPE_EX > 0: the L2, the L3, and L1s above 16 KB) accepted a memory fill only when no other fill was in its pipe. The fill sector waited in one staging register until its data-array write at stD, so a second fill would have overwritten it. That capped a bank at one fill every 1 + max(chain, PIPE_EX) cycles. GPU caches take a fill every cycle while earlier fills are still writing (line-fill buffers). The staged sector now rides a resetless delay line beside the request to its array write, clock-enabled with the pipeline, so the next fill can restage the buffer right behind it. Each stage has one source, so it costs flops but no LUTs, and deep lines map onto shift registers. A simulation-only twin of the line asserts that the staged sector's address matches the fill writing the array. SimX woke a fill's MSHR chain at install while the RTL wakes it at accept, and gated fills on an empty pipe at every latency. It now wakes the chain and arms the forward window at accept, forwarding from the staged sector like the RTL. Because SimX links a core request to its chain at pipe exit rather than at S0, a fill waits while a same-line core request is still in the pipe, and further requests to that line wait with it, so a forwarded load can never overtake an older store. SimX now matches rtlsim on 4-core NT=16 sgemv to 0.03% (it was 3.5% optimistic). fpga_gate cache DUT at DCACHE_LATENCY=4 (PIPE_EX=2, 4 banks): 300.4 -> 309.6 MHz, LUT 27,534 -> 27,403, FF 26,589 -> 30,708 (the 4 x 2 x 512-bit delay line). perf_gate (rtlsim, re-recorded with --update-baselines; retired instructions unchanged): only the multi-core L2 cases move, sgemm-mc-nt16 -0.40%, fp16-mc-nt4 -0.08%, draw3d-mc 0.00%, fp16-mc-nt16 +0.52%. The stock L2 is shallow; on the rtx5080 profile (L2 latency 10) a 4 MiB stream gains 6.2%. A trace diff of fp16-mc-nt16 shows no L2 service time getting worse (hit latency 4.08 -> 4.02, the in-bank miss path identical, same DRAM traffic): a fill accepted one cycle earlier reorders the two cores' L2 traffic and their warps line up differently. The same case moves by up to 1.8% from a one-cycle change of the simulated DRAM latency alone, and over 11 problem sizes the change has a geomean of 0.985. Co-Authored-By: Claude Opus 5.5 --- ci/baselines/perf/core.json | 2 +- ci/baselines/perf/graphics.json | 2 +- ci/baselines/perf/tensor.json | 4 +- hw/rtl/cache/VX_cache_bank.sv | 71 ++++++++++++--------- sim/simx/mem/cache.cpp | 109 +++++++++++++++++++------------- 5 files changed, 109 insertions(+), 79 deletions(-) diff --git a/ci/baselines/perf/core.json b/ci/baselines/perf/core.json index a02138c6d0..01cd65540c 100644 --- a/ci/baselines/perf/core.json +++ b/ci/baselines/perf/core.json @@ -23,7 +23,7 @@ }, "core:sgemm-mc-nt16:rtlsim": { "32": { - "cycles": 723234, + "cycles": 720350, "instrs": 623680 }, "app": "sgemm", diff --git a/ci/baselines/perf/graphics.json b/ci/baselines/perf/graphics.json index b402bd0b6f..489e16a09b 100644 --- a/ci/baselines/perf/graphics.json +++ b/ci/baselines/perf/graphics.json @@ -1,7 +1,7 @@ { "graphics:perf_gate-draw3d-mc:rtlsim": { "32": { - "cycles": 529875, + "cycles": 529868, "instrs": 203136 }, "app": "gfx_draw3d", diff --git a/ci/baselines/perf/tensor.json b/ci/baselines/perf/tensor.json index 112a272e80..975aa8d248 100644 --- a/ci/baselines/perf/tensor.json +++ b/ci/baselines/perf/tensor.json @@ -12,7 +12,7 @@ }, "tensor:perf_gate-fp16-mc-nt16:rtlsim": { "32": { - "cycles": 36534, + "cycles": 36723, "instrs": 47296 }, "app": "sgemm_tcu", @@ -23,7 +23,7 @@ }, "tensor:perf_gate-fp16-mc-nt4:rtlsim": { "32": { - "cycles": 221752, + "cycles": 221581, "instrs": 336448 }, "app": "sgemm_tcu", diff --git a/hw/rtl/cache/VX_cache_bank.sv b/hw/rtl/cache/VX_cache_bank.sv index cb3f748a3b..4b95432384 100644 --- a/hw/rtl/cache/VX_cache_bank.sv +++ b/hw/rtl/cache/VX_cache_bank.sv @@ -160,7 +160,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // Shared signals // ------------------------------------------------------------------------ wire crsp_queue_stall, mshr_alm_full; - wire mshr_allocate_st0, fill_inflight; + wire mshr_allocate_st0; (* max_fanout = 64 *) wire pipe_stall; // Authoritative "no entry held", straight off the MSHR's valid mask. The // pending-size counter below is a proxy: it is fed by separate increment and @@ -217,8 +217,8 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire fwd_head, fwd_fire, fwd_pending; wire [`CS_WORD_WIDTH-1:0] fwd_word; - // Fill buffer: the sector staged at fill accept owns all in-flight fill - // data; it feeds the data-array fill port and the forward-response word. + // Fill buffer: the sector staged at fill accept; it feeds the forward- + // response word and enters the fill-data delay line to the array write. reg [`CS_SECTOR_WIDTH-1:0] fbuf_data_r; reg [`CS_LINE_ADDR_WIDTH-1:0] fbuf_addr_r; @@ -356,7 +356,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // request enters behind a pending AMO), so a fill waits until the AMO // writeback queue has room for it; the queue drains meanwhile. wire replay_mux = replay_valid && ~fwd_head && ~replay_link_hold; - wire fill_mux = mem_rsp_valid && ~fwd_pending && ~fill_inflight && ~replay_link_hold && ~amo_wbq_full; + wire fill_mux = mem_rsp_valid && ~fwd_pending && ~replay_link_hold && ~amo_wbq_full; wire replay_grant = ~init_valid; wire replay_enable = replay_grant && replay_mux; @@ -374,7 +374,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire creq_enable = creq_grant && (amo_creq_path || amo_wb_path); assign replay_ready = replay_grant && ~fwd_head && ~replay_link_hold && ~(!WRITEBACK && replay_rw && mreq_queue_alm_full) && ~pipe_stall; - assign mem_rsp_ready = fill_grant && ~fwd_pending && ~fill_inflight && ~replay_link_hold && ~amo_wbq_full && ~(WRITEBACK && mreq_queue_alm_full) && ~pipe_stall; + assign mem_rsp_ready = fill_grant && ~fwd_pending && ~replay_link_hold && ~amo_wbq_full && ~(WRITEBACK && mreq_queue_alm_full) && ~pipe_stall; assign flush_ready = flush_grant && ~(WRITEBACK && mreq_queue_alm_full) && ~pipe_stall; assign core_req_ready = creq_grant && ~mreq_queue_alm_full && ~mshr_alm_full && ~pipe_stall && ~amo_commit_busy && ~req_input_defer; @@ -391,25 +391,6 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire [MSHR_ADDR_WIDTH-1:0] mem_rsp_id = mem_rsp_tag[MSHR_ADDR_WIDTH-1:0]; - // A fill's sector lives only in the staging buffer until its data-array - // write at stD, so with deferred stages (PIPE_EX>0) a back-to-back fill - // must be held off while one is in flight. At PIPE_EX=0 the array write - // samples the buffer on the same edge a new fill re-stages it (old value - // read), so no interlock is needed. - if (PIPE_EX > 0) begin : g_fill_inflight - reg [PIPE_EX-1:0] fill_busy; - always @(posedge clk) begin - if (reset) begin - fill_busy <= '0; - end else if (~pipe_stall) begin - fill_busy <= PIPE_EX'({fill_busy, (mem_rsp_fire && ~is_passthru_fill_sel)}); - end - end - assign fill_inflight = (| fill_busy); - end else begin : g_no_fill_inflight - assign fill_inflight = 1'b0; - end - // Fill word_idx: place the installed sector in the high (sector) bits of the // in-line word offset; low (word-in-sector) bits are don't-care for a fill. wire [WORD_SEL_WIDTH-1:0] fill_word_idx; @@ -442,8 +423,8 @@ module VX_cache_bank import VX_gpu_pkg::*; #( assign flush_tag = '0; end - // Word payload mux (store / AMO-writeback data). Fill data does not ride - // the pipeline: the staged fill buffer feeds the data array directly. + // Word payload mux (store / AMO-writeback data). Fill data takes its own + // delay line from the staged fill buffer to the data array. wire [`CS_WORD_WIDTH-1:0] wdata_sel; if (WRITE_ENABLE) begin : g_wdata_sel assign wdata_sel = replay_mux ? replay_data @@ -734,10 +715,40 @@ module VX_cache_bank import VX_gpu_pkg::*; #( assign sector_idx_std = '0; end - // The staged fill sector, replicated across the line so each sector's - // slices see their words; the data array writes only the installed sector - // (per-slice gate). One copy == the full line when 1 sector/line. - wire [`CS_LINE_WIDTH-1:0] fbuf_line = {`CS_SECTORS_PER_LINE{fbuf_data_r}}; + // A fill's sector is staged when the fill enters S0 and then rides beside + // it to its data-array write at stD, so the next fill may restage the + // buffer right behind it. Each stage has one source and holds on the + // pipeline enable; without a reset, deep stages map onto shift registers. + wire [`CS_SECTOR_WIDTH-1:0] fill_data_std; + VX_pipe_register #( + .DATAW (`CS_SECTOR_WIDTH), + .DEPTH (PIPE_EX) + ) reg_fill_data ( + .clk (clk), + .reset (reset), + .enable (~pipe_stall), + .data_in (fbuf_data_r), + .data_out (fill_data_std) + ); +`ifndef SYNTHESIS + wire [`CS_LINE_ADDR_WIDTH-1:0] fill_addr_std; + VX_pipe_register #( + .DATAW (`CS_LINE_ADDR_WIDTH), + .DEPTH (PIPE_EX) + ) reg_fill_addr ( + .clk (clk), + .reset (reset), + .enable (~pipe_stall), + .data_in (fbuf_addr_r), + .data_out (fill_addr_std) + ); + `RUNTIME_ASSERT (~(do_fill_std && ~stD.req.is_passthru_fill) || (stD.req.addr == fill_addr_std), ("%t: %s fill sector misaligned with its array write: addr=0x%0h, staged=0x%0h", $time, INSTANCE_ID, `CS_BANK_TO_FULL_ADDR(stD.req.addr, BANK_ID), `CS_BANK_TO_FULL_ADDR(fill_addr_std, BANK_ID))) +`endif + + // The fill sector, replicated across the line so each sector's slices see + // their words; the data array writes only the installed sector (per-slice + // gate). One copy == the full line when 1 sector/line. + wire [`CS_LINE_WIDTH-1:0] fbuf_line = {`CS_SECTORS_PER_LINE{fill_data_std}}; VX_cache_data #( .CACHE_SIZE (CACHE_SIZE), diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index 81b10a3692..0bcd46007d 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -720,7 +720,6 @@ class CacheBank : public SimObject { perf_stats_ = Cache::PerfStats(); pending_mshr_size_ = 0; pending_amo_probes_ = 0; - pipe_fill_count_ = 0; pipe_core_lines_.clear(); pending_read_reqs_ = 0; pending_write_reqs_ = 0; @@ -735,6 +734,7 @@ class CacheBank : public SimObject { fwd_set_ = 0; fwd_tag_ = 0; fwd_sector_ = 0; + fwd_data_.reset(); crsp_sent_ = false; flushing_ = false; flush_set_idx_ = 0; @@ -816,11 +816,16 @@ class CacheBank : public SimObject { } // 2) fill only when no replay is pending and no forward drain is in - // progress (a new fill would re-arm the window mid-chain). A fill still - // riding the pipe also blocks the next one: its chain is only marked for - // replay at install, and the replay priority above must get a chance to - // drain it before another fill is accepted. - if (!this->mem_rsp_in.empty() && !fwd_active_ && pipe_fill_count_ == 0) { + // progress (a new fill would re-arm the window mid-chain). Each fill + // carries its own sector down the pipe, so fills may follow each other + // back to back. The fill wakes its chain on accept, so a same-line core + // request still in the pipe must reach its chain first (the hardware + // links it at S0): the fill waits, and so does any further request to + // that line, which keeps a stream of them from starving the fill. + bool fill_held = false; + uint32_t held_set_id = 0; + uint64_t held_addr_tag = 0; + if (!this->mem_rsp_in.empty() && !fwd_active_) { auto &mem_rsp = this->mem_rsp_in.peek(); #if VX_CFG_EXT_A_ENABLED // Non-LLC AMO passthrough response: forward straight to core @@ -848,25 +853,47 @@ class CacheBank : public SimObject { #endif uint32_t mshr_id = mem_rsp.tag; const auto &root_peek = mshr_.peek(mshr_id); - bank_req_t bank_req; - bank_req.reset(); - bank_req.type = bank_req_t::Fill; - bank_req.addr = params_.mem_addr_sector(bank_id_, root_peek.set_id, root_peek.addr_tag, root_peek.sector_id); - bank_req.hart_id = root_peek.bank_req.hart_id; - bank_req.uuid = root_peek.bank_req.uuid; - bank_req.mshr_id = mshr_id; - bank_req.data = mem_rsp.data; - pipe_req_->push(bank_req); - ++pipe_fill_count_; - DT(3, this->name() << " fill-rsp: " << mem_rsp); - this->mem_rsp_in.pop(); - --pending_fill_reqs_; - return; + const uint32_t set_id = root_peek.set_id; + const uint64_t addr_tag = root_peek.addr_tag; + const uint32_t sector_id = root_peek.sector_id; + if (this->pipeLineBusy(set_id, addr_tag)) { + fill_held = true; + held_set_id = set_id; + held_addr_tag = addr_tag; + } else { + bank_req_t bank_req; + bank_req.reset(); + bank_req.type = bank_req_t::Fill; + bank_req.addr = params_.mem_addr_sector(bank_id_, set_id, addr_tag, sector_id); + bank_req.hart_id = root_peek.bank_req.hart_id; + bank_req.uuid = root_peek.bank_req.uuid; + bank_req.mshr_id = mshr_id; + bank_req.data = mem_rsp.data; + pipe_req_->push(bank_req); + // The chain wakes at accept: its replays enter the pipe behind the fill, + // so they reach the array after the install. The forward window answers + // the chain's leading reads from the staged sector meanwhile. + mshr_.replay(mshr_id); + fwd_active_ = true; + fwd_set_ = set_id; + fwd_tag_ = addr_tag; + fwd_sector_ = sector_id; + fwd_data_ = mem_rsp.data; + DT(3, this->name() << " fill-rsp: " << mem_rsp); + this->mem_rsp_in.pop(); + --pending_fill_reqs_; + return; + } } // 3) core request if (!this->core_req_in.empty()) { auto &core_req = this->core_req_in.peek(); + if (fill_held + && params_.addr_set_id(core_req.addr) == held_set_id + && params_.addr_tag(core_req.addr) == held_addr_tag) { + return; + } // Conservative MSHR occupancy check: any request that may miss must // reserve a slot. Counts both currently-allocated entries and // in-flight pipe requests that haven't reached MSHR allocation yet. @@ -904,11 +931,9 @@ class CacheBank : public SimObject { // MSHR entry only at pipe exit; defer on those too, or the AMO // overtakes it and a younger same-address load can chain onto the // pre-AMO fill (and the fill installs a pre-AMO line). - for (const auto &kv : pipe_core_lines_) { - if (kv.first == amo_set_id && kv.second == amo_addr_tag) { - ++perf_stats_.mshr_stalls; - return; - } + if (this->pipeLineBusy(amo_set_id, amo_addr_tag)) { + ++perf_stats_.mshr_stalls; + return; } } #else @@ -1107,12 +1132,6 @@ class CacheBank : public SimObject { sec.dirty = false; sec.dirty_mask = 0; sec.data = bank_req.data; - mshr_.replay(bank_req.mshr_id); - // arm the fill-forward window for this chain - fwd_active_ = true; - fwd_set_ = set_id; - fwd_tag_ = addr_tag; - fwd_sector_ = sector_id; return true; } @@ -1127,6 +1146,7 @@ class CacheBank : public SimObject { bool processForward() { if (!mshr_.has_ready_reqs()) { fwd_active_ = false; // chain drained + fwd_data_.reset(); return false; } const auto *head = mshr_.peek_next_replay(); @@ -1139,15 +1159,7 @@ class CacheBank : public SimObject { || head->bank_req.write || memop_is_atomic(head->bank_req.op)) { fwd_active_ = false; - return false; - } - // The armed sector is normally resident (the fill just installed it); a - // concurrent invalidate falls back to the replay path, which re-fetches. - auto &set = sets_.at(head->set_id); - int present_id = set.find_resident(head->addr_tag); - uint32_t sector_id = head->sector_id; - if (present_id == -1 || !set.lines.at(present_id).sectors.at(sector_id).valid) { - fwd_active_ = false; + fwd_data_.reset(); return false; } if (crsp_sent_ || this->core_rsp_out.full()) { @@ -1156,7 +1168,7 @@ class CacheBank : public SimObject { bank_req_t req; mshr_.dequeue(&req); MemRsp rsp{req.req_tag, req.hart_id, req.uuid}; - rsp.data = set.lines.at(present_id).sectors.at(sector_id).data; + rsp.data = fwd_data_; this->core_rsp_out.send(rsp); DT(3, this->name() << " fwd-rsp: " << rsp); return true; @@ -1397,8 +1409,6 @@ class CacheBank : public SimObject { if (!this->processFill(bank_req)) { return; // stall } - assert(pipe_fill_count_ != 0); - --pipe_fill_count_; pipe_req_->pop(); } break; @@ -1736,12 +1746,20 @@ class CacheBank : public SimObject { MSHR mshr_; uint32_t pending_mshr_size_; uint32_t pending_amo_probes_; // AmoProbe requests in the pipe, each holding a table reservation - uint32_t pipe_fill_count_; // fills riding the pipe; gates the next fill's admission // (set, tag) of Core requests inside the pipe: their MSHR entries exist // only from pipe exit, so ordering checks against in-flight lines (the // AMO admission defer) must consult this mirror as well. std::vector> pipe_core_lines_; + bool pipeLineBusy(uint32_t set_id, uint64_t addr_tag) const { + for (const auto &kv : pipe_core_lines_) { + if (kv.first == set_id && kv.second == addr_tag) { + return true; + } + } + return false; + } + void pipeLineErase(uint32_t set_id, uint64_t addr_tag) { for (auto it = pipe_core_lines_.begin(); it != pipe_core_lines_.end(); ++it) { if (it->first == set_id && it->second == addr_tag) { @@ -1761,11 +1779,12 @@ class CacheBank : public SimObject { uint32_t rand_ctr_; uint64_t adm_seq_ctr_; // bank admission-order stamp (see bank_req_t::adm_seq) - // Fill-forward window: chain key armed by the last installed fill. + // Fill-forward window: chain key and sector staged by the last accepted fill. bool fwd_active_; uint32_t fwd_set_; uint64_t fwd_tag_; uint32_t fwd_sector_; + std::shared_ptr fwd_data_; bool crsp_sent_; // core-response port used this tick (pipeline priority) // Flush walk state. From 7ce8a7483bfb50c3c208502c058a03c91cec9f50 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 17 Oct 2026 13:05:22 -0700 Subject: [PATCH 034/124] Flush only the L1 caches between launches when the LLC is the coherence point A launch boundary used to walk every cache level. When a single last-level cache serves every memory client, it is already the point of coherence, so back-to-back launches only need the L1s cleaned. CMD_CACHE_FLUSH gains an L1-only flag that reaches the caches as a flush-token kind; a shared LLC answers such a token without walking, while replicated LLCs and caches with peer memory clients beside them (TEX, RTU, RASTER, OM, DXA) still walk and invalidate. Whether an LLC is shared is derived from topology, and static asserts reject a write-back LLC with more than one instance. The runtime posts flushes lazily and issues a full flush before any device-memory transfer. Skipping the per-launch full flush exposed that hardware busy dropped while uncached IO stores were still inside the hierarchy, losing console output. busy now stays high until every write issued by the cores is accepted at the next level, through the memory ports, and until the platform makes it visible (bank acceptance in rtlsim, the AXI B response in Vortex_axi). The command processor retires a full flush only after busy drops, so its next DMA cannot overtake LLC write-backs on a shared memory bank. SimX keeps a shared LLC's contents across an L1-only flush, matching the RTL. Flush cycles per launch: 1 core 422 -> 152, 4 cores + L2 2,750 -> 180 for L1-only launches; full flushes and sgemm/raycast cycles unchanged. Tests: flush_coherence chain-1..6 (incl. replicated-LLC and DXA-peer cases that fail without the fix), chain-stress-1, model_parity-flush-chain. Co-Authored-By: Claude Opus 5.5 --- VX_types.toml | 5 + ci/testcases/regression.yaml | 87 +++++++++++++ docs/designs/cache_subsystem.md | 68 +++++++++- docs/designs/command_processor.md | 40 ++++-- hw/rtl/VX_cluster.sv | 9 +- hw/rtl/VX_gpu_pkg.sv | 10 ++ hw/rtl/VX_graphics.sv | 54 +++++++- hw/rtl/VX_socket.sv | 49 +++++++- hw/rtl/Vortex.sv | 15 ++- hw/rtl/Vortex_axi.sv | 29 ++++- hw/rtl/cache/VX_cache.sv | 13 +- hw/rtl/cache/VX_cache_bypass.sv | 34 ++++- hw/rtl/cache/VX_cache_cluster.sv | 54 +++++++- hw/rtl/cache/VX_cache_init.sv | 38 +++++- hw/rtl/cache/VX_cache_wrap.sv | 11 +- hw/rtl/core/VX_core.sv | 14 ++- hw/rtl/core/VX_dcr_data.sv | 9 +- hw/rtl/core/VX_dcr_flush.sv | 8 +- hw/rtl/core/VX_lsu_slice.sv | 3 +- hw/rtl/core/VX_mem_unit.sv | 2 + hw/rtl/cp/VX_cp_core.sv | 3 +- hw/rtl/cp/VX_cp_dcr_proxy.sv | 30 +++-- hw/rtl/cp/VX_cp_pkg.sv | 2 + hw/rtl/mem/VX_mem_req_pending.sv | 53 ++++++++ hw/rtl/om/VX_om_steer.sv | 2 + hw/unittest/cache/VX_cache_top.sv | 3 +- .../cp_dcr_proxy/VX_cp_dcr_proxy_top.sv | 6 +- hw/unittest/cp_dcr_proxy/main.cpp | 70 ++++++++++- hw/unittest/gfx/VX_gfx_top.sv | 11 +- hw/unittest/mem_unit/VX_mem_unit_top.sv | 1 + hw/unittest/tensor/VX_tensor_top.sv | 7 +- hw/unittest/vm/VX_vm_top.sv | 9 +- sim/common/cmd_processor.cpp | 14 ++- sim/common/cmd_processor.h | 14 ++- sim/rtlsim/processor.cpp | 10 ++ sim/rtlsim/processor.h | 4 + sim/rtlsim/rtlsim_shim.sv | 28 ++++- sim/simx/cluster.cpp | 17 +-- sim/simx/cluster.h | 4 +- sim/simx/constants.h | 21 ++++ sim/simx/mem/cache.cpp | 28 +++-- sim/simx/mem/cache.h | 7 +- sim/simx/mem/cache_cluster.cpp | 4 +- sim/simx/mem/cache_cluster.h | 2 +- sim/simx/processor.cpp | 23 ++-- sim/simx/processor_impl.h | 3 +- sim/simx/socket.cpp | 7 +- sim/simx/socket.h | 2 +- sw/runtime/common/device.cpp | 107 +++++++++++----- sw/runtime/common/queue.cpp | 13 +- sw/runtime/common/vortex2_internal.h | 28 +++-- sw/runtime/firesim/vortex.cpp | 4 + sw/runtime/rtlsim/vortex.cpp | 4 + tests/regression/flush_coherence/common.h | 1 + tests/regression/flush_coherence/kernel.cpp | 24 +++- tests/regression/flush_coherence/main.cpp | 118 +++++++++++++++++- 56 files changed, 1077 insertions(+), 159 deletions(-) create mode 100644 hw/rtl/mem/VX_mem_req_pending.sv diff --git a/VX_types.toml b/VX_types.toml index 7143523976..2e00ed2573 100644 --- a/VX_types.toml +++ b/VX_types.toml @@ -73,6 +73,11 @@ VX_DCR_BASE_CACHE_FLUSH = 0x000 VX_DCR_BASE_MPM_VALUE = 0x001 VX_DCR_BASE_STATE_END = 0x002 +[cache_flush] +# VX_DCR_BASE_CACHE_FLUSH write value: flush only the levels above the +# last-level cache, which stays the point of coherence for the next kernel. +VX_CACHE_FLUSH_L1_ONLY = 0x10000 + [dcr_mmu] VX_DCR_MMU_STATE_BEGIN = 0x004 VX_DCR_MMU_SATP_LO = 0x004 diff --git a/ci/testcases/regression.yaml b/ci/testcases/regression.yaml index b180c479b4..0d4ad00450 100644 --- a/ci/testcases/regression.yaml +++ b/ci/testcases/regression.yaml @@ -187,6 +187,83 @@ tests: cores: 2 l2cache: true l3cache: true +# Kernel to kernel with no host transfer between (one command batch): the +# launch flushes are L1-only, so the last-level cache alone carries each +# round's writes to the next round, and the full flush runs before readback. +- id: flush_coherence-chain-1 + via: blackbox + drivers: + - simx + - rtlsim + app: flush_coherence + args: -n2048 -r16 -c +- id: flush_coherence-chain-2 + via: blackbox + drivers: + - simx + - rtlsim + app: flush_coherence + args: -n2048 -r16 -c + configs: -DVX_CFG_SOCKET_SIZE=1 + shape: + cores: 4 + l2cache: true +- id: flush_coherence-chain-3 + via: blackbox + drivers: + - simx + - rtlsim + app: flush_coherence + args: -n2048 -r16 -c + shape: + clusters: 2 + cores: 2 + l2cache: true + l3cache: true +# Replicated last-level caches (per-core dcaches without an L2, per-cluster L2s +# without an L3) are not a point of coherence, so an L1-only flush must still +# invalidate them. +- id: flush_coherence-chain-4 + via: blackbox + drivers: + - simx + - rtlsim + app: flush_coherence + args: -n2048 -r16 -c + shape: + cores: 2 +- id: flush_coherence-chain-5 + via: blackbox + drivers: + - simx + - rtlsim + app: flush_coherence + args: -n2048 -r16 -c + shape: + clusters: 2 + cores: 2 + l2cache: true +# Each round loads its input with a DXA tile copy, which reaches memory beside +# the single dcache, so that dcache is not a point of coherence and must still +# be written back on an L1-only flush. +- id: flush_coherence-chain-6 + via: blackbox + drivers: + - simx + - rtlsim + app: flush_coherence + args: -n2048 -r16 -c -x + configs: -DVX_CFG_EXT_DXA_ENABLE +# The shared L2 keeps its lines across the L1-only launch flushes. +- id: model_parity-flush-chain + check: model_parity + via: blackbox + app: flush_coherence + args: -n2048 -r16 -c + configs: -DVX_CFG_SOCKET_SIZE=1 + shape: + cores: 4 + l2cache: true # Flush ordering under random input backpressure (DBG_FLUSH_STALL): each round # ends on a store burst, so a core's last writes are often still queued on one # memory port or dcache lane while its token on another reaches the next level. @@ -227,3 +304,13 @@ tests: cores: 4 l2cache: true tier: full +- id: flush_coherence-chain-stress-1 + via: blackbox + drivers: + - rtlsim + app: flush_coherence + args: -n2048 -r8 -b16 -c + configs: -DDBG_FLUSH_STALL -DVX_CFG_SOCKET_SIZE=1 + shape: + cores: 4 + l2cache: true diff --git a/docs/designs/cache_subsystem.md b/docs/designs/cache_subsystem.md index 8fce9737ef..7f7c26e390 100644 --- a/docs/designs/cache_subsystem.md +++ b/docs/designs/cache_subsystem.md @@ -465,6 +465,35 @@ Replay is the mirror image: it gates only when **not** `WRITEBACK` and the replayed request is a write (`replay_rw`), because a write-through bank forwards every write downstream and so does need a slot for it. +### 4.9 Requests in flight and device `busy` + +Stores are ack-less, so a core goes idle while its last requests are still +on their way out, and the console rings (`VX_MEM_IO_COUT`) are read right +after a launch retires, with no flush in between. Device `busy` therefore +covers every request the cores issued until memory has completed it, the way +a GPU's outstanding-write counters do: + +- the core stays busy while a request waits at its dcache port, past the flush + injector's buffer; +- every place outside the banks where a request can wait reports it through + `req_empty`: `VX_cache_bypass` counts from its uncached inputs and the + cache's memory side to its memory ports, `VX_cache_cluster` adds its + core-side and memory-side arbiters, and the socket memory arbiter and the OM + steers do the same. Each counter is a `VX_mem_req_pending` over the + section's input and output handshakes, which holds because these sections + neither create nor drop requests; the socket, cluster and device fold them + into `busy`; +- the platform wrapper extends `busy` past the device's memory ports to the + point where a write is visible to the host's DMA: acceptance by the bank in + `rtlsim_shim`, the AXI B response in `Vortex_axi`. The OPAE AFU needs + neither, since its DMA queues behind Vortex's writes on the same in-order + Avalon path. + +Cacheable requests left in the banks are covered by the next flush instead +(§6). A request's next holding point reaches `busy` through no more registers +than the one it leaves, so the coverage handed from one counter to the next has +no gap. + --- ## 5. Parallelism and scaling @@ -578,8 +607,16 @@ previous launch. The host issues `CMD_CACHE_FLUSH` after every kernel launch — ### 6.2 VX_dcr_flush — injecting the request `VX_dcr_flush` synthesizes a degenerate `MemReq` (`rw=0, addr=0, data=0, -byteen=0, attr = 1 << MEM_ATTR_FLUSH_OFFS`), a **flush token**, and merges it -into one cache input stream through a 2:1 `VX_mem_bus_arb`. +byteen=l1_only, attr = 1 << MEM_ATTR_FLUSH_OFFS`), a **flush token**, and merges it +into one cache input stream through a 2:1 `VX_mem_bus_arb`. Byteen bit 0 +marks an **L1-only** token (§6.4; a receiving cache tests for any byteen bit, +since a level with wider words sees the bit at the token's line offset); `VX_dcr_data` latches it from the START +write's `VX_CACHE_FLUSH_L1_ONLY` bit, and the dcache and icache injectors carry +it. The ocache, whose tokens reach the L2, sends an L1-only token only when +every requester of its flush round is L1-only: the cluster flush (kind latched +from the same broadcast write) and the dcache tokens held at the OM steers. +The other graphics caches do not forward and send full tokens. A fence +carries no byte enables, so a fence token is always a full one. **Core traffic sits on the priority input, not the flush.** The flush waits until upstream is quiescent — which it reliably is, because the host only issues @@ -639,6 +676,33 @@ downstream and `ready=0` upstream, so upstream stalls. into the response crossbar whenever bank 0 has no response of its own; the token is consumed at its input as its answer is sent. Then the inputs reopen. +**L1-only tokens.** Between kernels nothing outside the GPU reads memory, so +the last-level cache, the point of coherence, need not be written back. A +token with a byteen bit set asks only the levels above the LLC to flush: + +- a non-LLC level walks as usual (write-back levels write back, write-through + levels invalidate, so the next kernel sees other cores' and clusters' + updates) and forwards the token on every port **with the same kind** + (`mem_flush_l1_only`, byteen bit 0 of the forwarded token). Forwarding is what + lands a write-through level's last stores in the next level before the + token is answered; +- a last-level cache that alone serves every memory client (`SHARED_LLC`: + the L3; the L2 of a single cluster; the dcache of a single core with no + texture, ray-tracing, DXA, raster or OM unit, since those reach memory below + or beside it) goes from IDLE (or WAIT1) straight to RESPOND, with no walk and + no forward: once the token is visible, its input's earlier requests are + inside the cache, ahead of anything the next kernel sends. Any other LLC + (per-core dcaches without an L2, per-cluster L2s without an L3, a dcache + beside fixed-function or DXA ports) walks like any other level: its lines + may be stale copies of another instance's writes, or dirty data a client + beside it must read from memory. `Vortex.sv` asserts that a write-back LLC + has a single instance. + +The kind of a walk is L1-only only if every token it answers is, so a full +token is never answered by an L1-only walk. A token without the bit behaves +exactly as before. The runtime issues a full flush before any transfer to or +from device memory, because the host DMA path bypasses the LLC. + **Why a token per memory port.** Each port is a separate, separately buffered path to the next level, and nothing orders one port's data against another port's token. A token sent on one port only could reach the next level while diff --git a/docs/designs/command_processor.md b/docs/designs/command_processor.md index 0ad30a8011..356ddb000b 100644 --- a/docs/designs/command_processor.md +++ b/docs/designs/command_processor.md @@ -168,7 +168,13 @@ come from `cmd_size_bytes` (`WAIT_OP_EQ/GE/GT/NE`, [`VX_cp_pkg.sv:109-114`](../../hw/rtl/cp/VX_cp_pkg.sv#L109)). - **`CMD_CACHE_FLUSH`** — `arg0` = number of cores. Provides AMD `ACQUIRE_MEM`-style cache maintenance; the host fills `arg0` from - `VX_CAPS_NUM_CORES`. + `VX_CAPS_NUM_CORES`. Header flag bit 2 (`F_FLUSH_L1_ONLY`) asks for an + L1-only flush: the START write carries `VX_CACHE_FLUSH_L1_ONLY`, and the + last-level cache answers without its walk + ([cache_subsystem.md §6.4](cache_subsystem.md#64-vx_cache_init--the-flush-sequencer)). + The runtime flushes lazily: an L1-only flush ahead of the next launch, a + full flush ahead of the next device-memory transfer (see the launch + encoding in §9). --- @@ -317,10 +323,10 @@ is a single-engine CP. Three things gate raising it: 3. **The host serializes anyway.** All host queues funnel into one ring behind one mutex (§9). -The **host-side** scaling limit is the launch path, not the ring: 22 +The **host-side** scaling limit is the launch path, not the ring: 21 commands per launch, unbatched, each with its own doorbell and its own busy-poll loop (§9). `CMD_LAUNCH_QMD` collapses the 20 DCR writes to one -command; batching collapses 22 doorbells to one. Both are implemented in +command; batching collapses 21 doorbells to one. Both are implemented in the Emulation CP and the runtime; the RTL CP needs the QMD mirror. ### 4.5 The submit handshake end to end @@ -475,7 +481,10 @@ unconditionally on every backend. all their flushes at once. It then re-enters `REQ` once per core, sweeping a DCR *read* of the same address with the target core index in `dcr_req_data[15:0]`; each read is answered when that core's flush has - completed, and the command retires after the last one. The DCR response + completed, and the command retires after the last one; a full flush then + holds `DONE` until Vortex `busy` drops, which covers every write until its + AXI B response, so a DMA after the flush cannot overtake the last-level + cache's write-backs on bank 0. The DCR response bus has no back-pressure, so the reads stay one at a time; the flushes themselves overlap and share one walk of the shared caches ([cache_subsystem.md §6](cache_subsystem.md#6-flush-architecture)). @@ -649,13 +658,24 @@ whole run costs one doorbell and one poll. **Launch encoding.** A kernel launch emits **20** `CMD_DCR_WRITE`s ([`queue.cpp:401-427`](../../sw/runtime/common/queue.cpp#L401)) — 4 for the startup/entry PCs, 2 for the args pointer, and 14 for block/grid/cluster -dims, lmem size, block size and warp steps — then `CMD_LAUNCH`, then -`CMD_CACHE_FLUSH` with `arg0 = num_cores`, then a COUT drain +dims, lmem size, block size and warp steps — then `CMD_LAUNCH`, then a COUT drain ([`device.cpp:502-520`](../../sw/runtime/common/device.cpp#L502)). Each group is independently skippable through the legacy escape hatches (no kernel, empty args, `ndim == 0`), so the real count ranges from 20 down to -0. This path does **not** batch: all 22 lines take the non-batch -`cp_submit_cl_`, so one launch costs 22 doorbells and 22 poll loops. Only +0. The kernel-boundary `CMD_CACHE_FLUSH` (`arg0 = num_cores`) is posted lazily +by `cp_ring_append_`, under the same ring lock, from two flags: `l1_dirty_` (a +kernel ran since the last flush) puts an L1-only flush ahead of the next +launch; `l1_dirty_` or `llc_dirty_` (a kernel ran since the last full flush) +puts a full flush ahead of the next `CMD_MEM_*` that touches device memory +outside the IO aperture, because the CP DMA reaches device memory beneath +the last-level cache. A launch followed by a transfer therefore costs one +full flush, and back-to-back launches one L1-only flush each. The COUT drain +needs no flush: the console rings are uncached, and the device `busy` the +launch retires on stays high until memory has completed every write the cores +issued ([cache_subsystem.md §4.9](cache_subsystem.md#49-requests-in-flight-and-device-busy)). The drain's +own transfers stay in the IO aperture and never trigger a flush. +This path does **not** batch: all 21 lines take the non-batch +`cp_submit_cl_`, so one launch costs 21 doorbells and 21 poll loops. Only `enqueue_commands` and the `enqueue_draw` fallback batch. **Queues and serialization.** `vx_queue_create` makes an unbounded number @@ -752,8 +772,8 @@ The following gaps remain **open**. defaults to 1; the Emulation CP models only `q0_`; the runtime programs only queue 0 and funnels every host queue through one ring lock (§9). Real concurrency depends on items 5 and 6. -9. **Batch the launch path.** `enqueue_launch` submits its 22 commands - unbatched — 22 doorbells, 22 busy-polls — while `cp_batch_begin/end` +9. **Batch the launch path.** `enqueue_launch` submits its 21 commands + unbatched — 21 doorbells, 21 busy-polls — while `cp_batch_begin/end` already exists and is used by `enqueue_commands`. Cheap win independent of item 6. 10. **EVENT_WAIT fairness + backoff.** `EVENT_WAIT` holds its arbiter diff --git a/hw/rtl/VX_cluster.sv b/hw/rtl/VX_cluster.sv index f02c30047b..9b817df196 100644 --- a/hw/rtl/VX_cluster.sv +++ b/hw/rtl/VX_cluster.sv @@ -206,6 +206,7 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; .DATA_SIZE (L2_SECTOR_SIZE), .TAG_WIDTH (L2_MEM_TAG_WIDTH) ) l2_mem_bus_if[L2_MEM_PORTS](); + wire l2_req_empty; VX_cache_wrap #( .INSTANCE_ID (`SFORMATF(("%s-l2cache", INSTANCE_ID))), @@ -232,6 +233,7 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; .NC_ENABLE (1), .PASSTHRU (!`VX_CFG_L2_ENABLED), .IS_LLC (L2_IS_LLC), + .SHARED_LLC (L2_SHARED_LLC), .BANK_HASH (`VX_CFG_L2_BANK_HASH), .AMO_ENABLE (`VX_CFG_EXT_A_ENABLED) ) l2cache ( @@ -241,7 +243,8 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; .cache_perf (l2_perf), `endif .core_bus_if (per_socket_mem_bus_if), - .mem_bus_if (l2_mem_bus_if) + .mem_bus_if (l2_mem_bus_if), + .req_empty (l2_req_empty) ); // Cluster DCR distribution — declared before its consumers (sockets/gfx). @@ -485,9 +488,9 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; wire busy_r; `ifdef EXT_GFX_ANY_ENABLE - `BUFFER_EX(busy_r, dcr_bus_if.req_valid | (|per_socket_busy) | gfx_busy | mmu_busy, 1'b1, 1, (NUM_SOCKETS > 1)); + `BUFFER_EX(busy_r, dcr_bus_if.req_valid | (|per_socket_busy) | gfx_busy | mmu_busy | ~l2_req_empty, 1'b1, 1, (NUM_SOCKETS > 1)); `else - `BUFFER_EX(busy_r, dcr_bus_if.req_valid | (|per_socket_busy) | mmu_busy, 1'b1, 1, (NUM_SOCKETS > 1)); + `BUFFER_EX(busy_r, dcr_bus_if.req_valid | (|per_socket_busy) | mmu_busy | ~l2_req_empty, 1'b1, 1, (NUM_SOCKETS > 1)); `endif assign busy = busy_r | dcr_bus_if.req_valid | launch_link_busy; diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index 9ae6a9a2a9..39e4140148 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -1544,6 +1544,13 @@ package VX_gpu_pkg; // L1 dcache is the LLC iff neither L2 nor L3 is enabled. localparam DCACHE_IS_LLC = !`VX_CFG_L2_ENABLED && !`VX_CFG_L3_ENABLED; + // It is the point of coherence between kernels only as the one dcache with + // no other memory client beside it (the texture, ray-tracing and DXA ports + // join below it, the raster and OM caches beside it). + localparam DCACHE_SHARED_LLC = DCACHE_IS_LLC && (`VX_CFG_NUM_CORES == 1) && (`VX_CFG_NUM_CLUSTERS == 1) + && !`VX_CFG_EXT_TEX_ENABLED && !`VX_CFG_EXT_RTU_ENABLED + && !`VX_CFG_EXT_RASTER_ENABLED && !`VX_CFG_EXT_OM_ENABLED + && !`VX_CFG_EXT_DXA_ENABLED; ////////////////////////// Tex / Tcache Parameters //////////////////////// `ifdef VX_CFG_EXT_TEX_ENABLE @@ -1773,6 +1780,8 @@ package VX_gpu_pkg; // L2 is the LLC iff L2 is enabled and L3 is not. localparam L2_IS_LLC = `VX_CFG_L2_ENABLED && !`VX_CFG_L3_ENABLED; + // Every memory client of a cluster reaches memory through its L2. + localparam L2_SHARED_LLC = L2_IS_LLC && (`VX_CFG_NUM_CLUSTERS == 1); /////////////////////////////// L3 Parameters ///////////////////////////// @@ -1806,6 +1815,7 @@ package VX_gpu_pkg; `endif // L3 is the LLC whenever it is enabled. localparam L3_IS_LLC = `VX_CFG_L3_ENABLED; + localparam L3_SHARED_LLC = L3_IS_LLC; /////////////////////////////////////////////////////////////////////////// diff --git a/hw/rtl/VX_graphics.sv b/hw/rtl/VX_graphics.sv index a2a27fa7ed..a2ef22cb37 100644 --- a/hw/rtl/VX_graphics.sv +++ b/hw/rtl/VX_graphics.sv @@ -67,6 +67,9 @@ module VX_graphics import VX_gpu_pkg::* ); `UNUSED_PARAM (CLUSTER_ID) + wire rcache_req_empty; + wire ocache_req_empty; + // Fan one DCR slave input out to one master per consumer unit. Each // unit's internal case-statement filters by DCR address range. The // VX_dcr_arb owns the rsp_valid/rsp_data signaling on dcr_bus_if so @@ -233,6 +236,7 @@ module VX_graphics import VX_gpu_pkg::* .clk (clk), .reset (reset), .dcr_flush_if (rcache_flush_if), + .l1_only (1'b0), .core_bus_if (rcache_bus_if[0]), .cache_bus_if (rcache_flushable_bus_if[0]) ); @@ -274,7 +278,8 @@ module VX_graphics import VX_gpu_pkg::* .cache_perf (rcache_perf), `endif .core_bus_if (rcache_flushable_bus_if), - .mem_bus_if (rcache_mem_bus_tmp_if) + .mem_bus_if (rcache_mem_bus_tmp_if), + .req_empty (rcache_req_empty) ); `ASSIGN_VX_MEM_BUS_IF_EX (rcache_mem_bus_if, rcache_mem_bus_tmp_if[0], @@ -326,8 +331,10 @@ module VX_graphics import VX_gpu_pkg::* // Flush tokens are held at the steers until the OM side is flushed (below). wire [L2_SOCKET_REQS-1:0] steer_flush_req; + wire [L2_SOCKET_REQS-1:0] steer_flush_l1_only; reg [L2_SOCKET_REQS-1:0] steer_flush_ack_r; wire [L2_SOCKET_REQS-1:0] steer_om_pending; + wire [L2_SOCKET_REQS-1:0] steer_req_empty; for (genvar i = 0; i < L2_SOCKET_REQS; ++i) begin : g_om_steer VX_om_steer #( @@ -342,9 +349,24 @@ module VX_graphics import VX_gpu_pkg::* .l2_out_if (l2_out_bus_if[i]), .om_out_if (om_aperture_bus_if[i]), .flush_req (steer_flush_req[i]), + .flush_l1_only (steer_flush_l1_only[i]), .flush_ack (steer_flush_ack_r[i]), .om_pending (steer_om_pending[i]) ); + wire [1:0] steer_out_fire = { + om_aperture_bus_if[i].req_valid && om_aperture_bus_if[i].req_ready, + l2_out_bus_if[i].req_valid && l2_out_bus_if[i].req_ready + }; + VX_mem_req_pending #( + .NUM_INPUTS (1), + .NUM_OUTPUTS (2) + ) steer_pending ( + .clk (clk), + .reset (reset), + .in_fire (socket_mem_bus_if[i].req_valid && socket_mem_bus_if[i].req_ready), + .out_fire (steer_out_fire), + .empty (steer_req_empty[i]) + ); end for (genvar i = 0; i < L2_SOCKET_REQS; ++i) begin : g_om_ingress @@ -454,7 +476,21 @@ module VX_graphics import VX_gpu_pkg::* reg [L2_SOCKET_REQS-1:0] omf_steers_r; // steers this round answers reg omf_cluster_r; // this round answers the cluster flush reg omf_cluster_done_r; + reg omf_l1_only_r; // every requester of this round is L1-only wire omf_cluster_req = cluster_flush_if.req && ~omf_cluster_done_r; + + // The cluster flush takes its kind from the broadcast write that starts it, + // which every core receives too. The cores raise the cluster flush request + // only after taking that write, so the latch is current by then. + reg cluster_flush_l1_only_r; + always @(posedge clk) begin + if (reset) begin + cluster_flush_l1_only_r <= 1'b0; + end else if (dcr_bus_if.req_valid && dcr_bus_if.req_data.rw + && (dcr_bus_if.req_data.addr == `VX_DCR_BASE_CACHE_FLUSH)) begin + cluster_flush_l1_only_r <= (dcr_bus_if.req_data.data & VX_DCR_DATA_WIDTH'(`VX_CACHE_FLUSH_L1_ONLY)) != 0; + end + end wire omf_drained = ~(| om_busy_w) && ~(| om_ingress_pending) && ~(| steer_om_pending); always @(posedge clk) begin @@ -463,6 +499,7 @@ module VX_graphics import VX_gpu_pkg::* omf_steers_r <= '0; omf_cluster_r <= 1'b0; omf_cluster_done_r <= 1'b0; + omf_l1_only_r <= 1'b0; steer_flush_ack_r <= '0; end else begin steer_flush_ack_r <= '0; @@ -474,6 +511,8 @@ module VX_graphics import VX_gpu_pkg::* if ((| steer_flush_req) || omf_cluster_req) begin omf_steers_r <= steer_flush_req; omf_cluster_r <= omf_cluster_req; + omf_l1_only_r <= (& (~steer_flush_req | steer_flush_l1_only)) + && (~omf_cluster_req || cluster_flush_l1_only_r); omf_state_r <= OMF_DRAIN; end end @@ -528,6 +567,7 @@ module VX_graphics import VX_gpu_pkg::* .clk (clk), .reset (reset), .dcr_flush_if (ocache_om_flush_if[i]), + .l1_only (omf_l1_only_r), .core_bus_if (ocache_bus_if[i]), .cache_bus_if (ocache_om_flush_bus_if[i]) ); @@ -577,7 +617,8 @@ module VX_graphics import VX_gpu_pkg::* .cache_perf (ocache_perf), `endif .core_bus_if (ocache_flushable_bus_if), - .mem_bus_if (ocache_mem_bus_tmp_if) + .mem_bus_if (ocache_mem_bus_tmp_if), + .req_empty (ocache_req_empty) ); `ASSIGN_VX_MEM_BUS_IF_EX (ocache_mem_bus_if, ocache_mem_bus_tmp_if[0], @@ -615,9 +656,16 @@ module VX_graphics import VX_gpu_pkg::* `endif `ifdef VX_CFG_EXT_OM_ENABLE wire om_busy_any = (| om_busy_w) || (| om_ingress_busy); + // requests held in a steer's buffers on the way to L2 or the OM + wire om_req_busy = ~(& steer_req_empty); `else wire om_busy_any = 1'b0; + wire om_req_busy = 1'b0; + assign ocache_req_empty = 1'b1; +`endif +`ifndef VX_CFG_EXT_RASTER_ENABLE + assign rcache_req_empty = 1'b1; `endif - assign busy = raster_busy_any || om_busy_any; + assign busy = raster_busy_any || om_busy_any || om_req_busy || ~rcache_req_empty || ~ocache_req_empty; endmodule diff --git a/hw/rtl/VX_socket.sv b/hw/rtl/VX_socket.sv index 315ccf713a..3b04931533 100644 --- a/hw/rtl/VX_socket.sv +++ b/hw/rtl/VX_socket.sv @@ -138,6 +138,9 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .DATA_SIZE (ICACHE_LINE_SIZE), .TAG_WIDTH (ICACHE_MEM_TAG_WIDTH) ) icache_mem_bus_if[1](); + wire icache_req_empty; + wire tcache_req_empty; + wire rtcache_req_empty; VX_cache_cluster #( .INSTANCE_ID (`SFORMATF(("%s-icache", INSTANCE_ID))), @@ -171,7 +174,8 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .clk (clk), .reset (reset), .core_bus_if (icache_phys_bus_if), - .mem_bus_if (icache_mem_bus_if) + .mem_bus_if (icache_mem_bus_if), + .req_empty (icache_req_empty) ); /////////////////////////////////////////////////////////////////////////// @@ -192,6 +196,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .DATA_SIZE (DCACHE_SECTOR_SIZE), .TAG_WIDTH (DCACHE_MEM_TAG_WIDTH) ) dcache_mem_bus_if[L1_MEM_PORTS](); + wire dcache_req_empty; VX_cache_cluster #( .INSTANCE_ID (`SFORMATF(("%s-dcache", INSTANCE_ID))), @@ -220,6 +225,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .CORE_OUT_BUF (3), .MEM_OUT_BUF (3), .IS_LLC (DCACHE_IS_LLC), + .SHARED_LLC (DCACHE_SHARED_LLC), .BANK_HASH (`VX_CFG_DCACHE_BANK_HASH), .AMO_ENABLE (`VX_CFG_EXT_A_ENABLED) ) dcache ( @@ -229,7 +235,8 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .clk (clk), .reset (reset), .core_bus_if (dcache_phys_bus_if), - .mem_bus_if (dcache_mem_bus_if) + .mem_bus_if (dcache_mem_bus_if), + .req_empty (dcache_req_empty) ); /////////////////////////////////////////////////////////////////////////// @@ -488,6 +495,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .clk (clk), .reset (reset), .dcr_flush_if (tcache_flush_if), + .l1_only (1'b0), .core_bus_if (tcache_bus_if[0]), .cache_bus_if (tcache_flushable_bus_if[0]) ); @@ -529,7 +537,8 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .cache_perf (socket_tcache_perf), `endif .core_bus_if (tcache_flushable_bus_if), - .mem_bus_if (tcache_mem_bus_tmp_if) + .mem_bus_if (tcache_mem_bus_tmp_if), + .req_empty (tcache_req_empty) ); `endif // VX_CFG_EXT_TEX_ENABLE @@ -616,6 +625,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .clk (clk), .reset (reset), .dcr_flush_if (rtcache_flush_if), + .l1_only (1'b0), .core_bus_if (rtcache_bus_if[0]), .cache_bus_if (rtcache_flushable_bus_if[0]) ); @@ -657,7 +667,8 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; `UNUSED_PIN (cache_perf), `endif .core_bus_if (rtcache_flushable_bus_if), - .mem_bus_if (rtcache_mem_bus_tmp_if) + .mem_bus_if (rtcache_mem_bus_tmp_if), + .req_empty (rtcache_req_empty) ); `endif // VX_CFG_EXT_RTU_ENABLE @@ -744,6 +755,8 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; /////////////////////////////////////////////////////////////////////////// + wire mem_arb_empty; + // L2-facing memory ports. Port 0 arbitrates icache and dcache with the // socket-resident units' memory ports (tcache, rtcache, DXA gmem) as // peers; priority order keeps icache first and DXA bulk traffic last so @@ -789,6 +802,21 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .bus_out_if (socket_arb_out_if) ); + wire [SOCKET_MEM_ARB_REQS-1:0] arb_in_fire; + for (genvar j = 0; j < SOCKET_MEM_ARB_REQS; ++j) begin : g_arb_in_fire + assign arb_in_fire[j] = socket_arb_in_if[j].req_valid && socket_arb_in_if[j].req_ready; + end + VX_mem_req_pending #( + .NUM_INPUTS (SOCKET_MEM_ARB_REQS), + .NUM_OUTPUTS (1) + ) mem_arb_pending ( + .clk (clk), + .reset (reset), + .in_fire (arb_in_fire), + .out_fire (socket_arb_out_if[0].req_valid && socket_arb_out_if[0].req_ready), + .empty (mem_arb_empty) + ); + `ASSIGN_VX_MEM_BUS_IF (mem_bus_if[0], socket_arb_out_if[0]); end else begin : g_i VX_mem_bus_if #( @@ -910,8 +938,19 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; for (genvar i = 0; i < `VX_CFG_SOCKET_SIZE; ++i) begin : g_kmu_link_valid assign per_core_kmu_valid[i] = per_core_kmu_bus_if[i].valid; end + // Stores are ack-less, so the cores go idle with their last requests still + // on the way out; the socket stays busy until none is held in its caches' + // uncached paths or arbiters. +`ifndef VX_CFG_EXT_TEX_ENABLE + assign tcache_req_empty = 1'b1; +`endif +`ifndef VX_CFG_EXT_RTU_ENABLE + assign rtcache_req_empty = 1'b1; +`endif + wire mem_req_empty = icache_req_empty && dcache_req_empty && tcache_req_empty + && rtcache_req_empty && mem_arb_empty; wire busy_r; - `BUFFER_EX(busy_r, dcr_bus_if.req_valid | (|per_core_busy), 1'b1, 1, (`VX_CFG_SOCKET_SIZE > 1)); + `BUFFER_EX(busy_r, dcr_bus_if.req_valid | (|per_core_busy) | ~mem_req_empty, 1'b1, 1, (`VX_CFG_SOCKET_SIZE > 1)); assign busy = busy_r | dcr_bus_if.req_valid | kmu_bus_if[0].valid | (|per_core_kmu_valid); endmodule diff --git a/hw/rtl/Vortex.sv b/hw/rtl/Vortex.sv index 2b4b49df89..98b5fb6c8e 100644 --- a/hw/rtl/Vortex.sv +++ b/hw/rtl/Vortex.sv @@ -54,6 +54,13 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( `STATIC_ASSERT(`IS_POW2(`VX_CFG_NUM_CORES), ("NUM_CORES must be a power of 2")); `STATIC_ASSERT(`IS_POW2(`VX_CFG_SOCKET_SIZE), ("SOCKET_SIZE must be a power of 2")); + // A write-back LLC must be the only instance: replicated ones never make + // their dirty lines visible to each other. + `STATIC_ASSERT(!(DCACHE_IS_LLC && `VX_CFG_DCACHE_WRITEBACK) || ((`VX_CFG_NUM_CORES == 1) && (`VX_CFG_NUM_CLUSTERS == 1)), + ("a write-back dcache that is the last-level cache must be the only dcache")); + `STATIC_ASSERT(!(L2_IS_LLC && `VX_CFG_L2_WRITEBACK) || (`VX_CFG_NUM_CLUSTERS == 1), + ("a write-back L2 that is the last-level cache must be the only L2")); + // Every cache strictly above the LLC must be write-through. // A WB intermediate could absorb a hart-B store without the LLC // seeing it; a later SC from hart-A on the same line would @@ -123,6 +130,8 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( .DATA_SIZE (L3_SECTOR_SIZE), .TAG_WIDTH (L3_MEM_TAG_WIDTH) ) mem_bus_if[L3_MEM_PORTS](); + // no request held in the L3 bypass path short of the memory ports + wire l3_req_empty; VX_cache_wrap #( .INSTANCE_ID ("l3cache"), @@ -149,6 +158,7 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( .NC_ENABLE (1), .PASSTHRU (!`VX_CFG_L3_ENABLED), .IS_LLC (L3_IS_LLC), + .SHARED_LLC (L3_SHARED_LLC), .AMO_ENABLE (`VX_CFG_EXT_A_ENABLED) ) l3cache ( .clk (clk), @@ -159,7 +169,8 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( `endif .core_bus_if (per_cluster_mem_bus_if), - .mem_bus_if (mem_bus_if) + .mem_bus_if (mem_bus_if), + .req_empty (l3_req_empty) ); for (genvar i = 0; i < L3_MEM_PORTS; ++i) begin : g_mem_bus_if @@ -364,7 +375,7 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( assign per_cluster_kmu_valid[c] = per_cluster_kmu_bus_if[c].valid; end wire busy_r; - `BUFFER_EX(busy_r, kmu_busy | dcr_bus_if.req_valid | (|per_cluster_busy) | dev_mmu_busy, 1'b1, 1, (`VX_CFG_NUM_CLUSTERS > 1)); + `BUFFER_EX(busy_r, kmu_busy | dcr_bus_if.req_valid | (|per_cluster_busy) | dev_mmu_busy | ~l3_req_empty, 1'b1, 1, (`VX_CFG_NUM_CLUSTERS > 1)); assign busy = busy_r | kmu_busy | dcr_bus_if.req_valid | (|per_cluster_kmu_valid); `ifdef PERF_ENABLE diff --git a/hw/rtl/Vortex_axi.sv b/hw/rtl/Vortex_axi.sv index 4566016c42..5a159c9d2a 100644 --- a/hw/rtl/Vortex_axi.sv +++ b/hw/rtl/Vortex_axi.sv @@ -107,6 +107,8 @@ module Vortex_axi import VX_gpu_pkg::*; #( wire [VX_MEM_TAG_WIDTH-1:0] mem_rsp_tag [VX_MEM_PORTS]; wire mem_rsp_ready [VX_MEM_PORTS]; + wire vx_busy; + `SCOPE_IO_SWITCH (1); Vortex vortex ( @@ -137,7 +139,7 @@ module Vortex_axi import VX_gpu_pkg::*; #( .dcr_rsp_data (dcr_rsp_data), .start (start), - .busy (busy) + .busy (vx_busy) ); wire mem_req_valid_a [VX_MEM_PORTS]; @@ -269,4 +271,29 @@ module Vortex_axi import VX_gpu_pkg::*; #( .m_axi_rresp (m_axi_rresp) ); + // AXI orders a write against another master's access only once its B + // response is back, so the device stays busy until every write is acked. + wire [VX_MEM_PORTS-1:0] vx_wr_fire; + wire [AXI_NUM_BANKS-1:0] axi_b_fire; + wire wr_empty; + for (genvar i = 0; i < VX_MEM_PORTS; ++i) begin : g_vx_wr_fire + assign vx_wr_fire[i] = mem_req_valid_a[i] && mem_req_ready_a[i] && mem_req_rw_a[i]; + end + for (genvar i = 0; i < AXI_NUM_BANKS; ++i) begin : g_axi_b_fire + assign axi_b_fire[i] = m_axi_bvalid[i] && m_axi_bready[i]; + end + VX_mem_req_pending #( + .NUM_INPUTS (VX_MEM_PORTS), + .NUM_OUTPUTS (AXI_NUM_BANKS), + .SIZE (32 * AXI_NUM_BANKS) + ) wr_pending ( + .clk (clk), + .reset (reset), + .in_fire (vx_wr_fire), + .out_fire (axi_b_fire), + .empty (wr_empty) + ); + + assign busy = vx_busy || ~wr_empty; + endmodule diff --git a/hw/rtl/cache/VX_cache.sv b/hw/rtl/cache/VX_cache.sv index de35722db6..a682b600b1 100644 --- a/hw/rtl/cache/VX_cache.sv +++ b/hw/rtl/cache/VX_cache.sv @@ -48,6 +48,7 @@ module VX_cache import VX_gpu_pkg::*; #( parameter MEM_OUT_BUF = 3, parameter IS_LLC = 0, // last-level cache: banks own AMO commit + RVA reservation tracking + parameter SHARED_LLC = 0, // the LLC serves every memory client alone (point of coherence) parameter FLUSH_FORWARD = 1, // a flush also flushes the next level @@ -105,6 +106,7 @@ module VX_cache import VX_gpu_pkg::*; #( wire [NUM_BANKS-1:0] per_bank_core_req_fire; wire mem_req_empty; wire [MEM_PORTS-1:0] mem_flush_valid; + wire mem_flush_l1_only; wire [MEM_PORTS-1:0] mem_flush_ready; wire mem_flush_rsp_phase; wire [MEM_PORTS-1:0] mem_flush_rsp_valid; @@ -117,6 +119,7 @@ module VX_cache import VX_gpu_pkg::*; #( .NUM_BANKS (NUM_BANKS), .MEM_PORTS (MEM_PORTS), .FLUSH_FORWARD (FLUSH_FORWARD), + .SHARED_LLC (SHARED_LLC), .TAG_WIDTH (TAG_WIDTH), .BANK_SEL_LATENCY (`TO_OUT_BUF_REG(REQ_XBAR_BUF)), // request xbar latency .REQ_SEL_WIDTH (REQ_SEL_WIDTH) @@ -131,6 +134,7 @@ module VX_cache import VX_gpu_pkg::*; #( .flush_end (per_bank_flush_end), .mem_req_empty (mem_req_empty), .mem_flush_valid (mem_flush_valid), + .mem_flush_l1_only (mem_flush_l1_only), .mem_flush_ready (mem_flush_ready), .mem_flush_rsp_phase (mem_flush_rsp_phase), .mem_flush_rsp_valid (mem_flush_rsp_valid), @@ -499,9 +503,10 @@ module VX_cache import VX_gpu_pkg::*; #( wire [MEM_PORTS-1:0][`UP(`LOG2UP(NUM_BANKS))-1:0] mem_req_tmp_idx; wire [MEM_PORTS-1:0] mem_req_tmp_ready; // The flush token for port i is a read of the port's first line with the - // flush attribute. It takes bank i's lane into the request crossbar (bank i - // maps to port i), so the crossbar forms its address and it leaves through - // the same registered outputs as the write-backs ahead of it. + // flush attribute; byteen bit 0 carries the L1-only kind. It takes bank + // i's lane into the request crossbar (bank i maps to port i), so the + // crossbar forms its address and it leaves through the same registered + // outputs as the write-backs ahead of it. wire [NUM_BANKS-1:0] mem_req_xbar_valid; wire [NUM_BANKS-1:0][MEM_REQ_DATAW-1:0] mem_req_xbar_pdata; wire [NUM_BANKS-1:0] mem_req_xbar_ready; @@ -512,7 +517,7 @@ module VX_cache import VX_gpu_pkg::*; #( `UP(BANK_SEL_BITS)'(i), `CS_LINE_SECTOR_ADDR_WIDTH'(0), 1'b0, - SECTOR_SIZE'(0), + SECTOR_SIZE'(mem_flush_l1_only), `CS_SECTOR_WIDTH'(0), BANK_MEM_TAG_WIDTH'(0), `UP(MEM_ATTR_WIDTH)'(1 << MEM_ATTR_FLUSH_OFFS) diff --git a/hw/rtl/cache/VX_cache_bypass.sv b/hw/rtl/cache/VX_cache_bypass.sv index 931cf0ebab..29232bc2fa 100644 --- a/hw/rtl/cache/VX_cache_bypass.sv +++ b/hw/rtl/cache/VX_cache_bypass.sv @@ -39,7 +39,10 @@ module VX_cache_bypass import VX_gpu_pkg::*; #( VX_mem_bus_if.slave core_bus_in_if [NUM_REQS], VX_mem_bus_if.master core_bus_out_if [NUM_REQS], VX_mem_bus_if.slave mem_bus_in_if [MEM_PORTS], - VX_mem_bus_if.master mem_bus_out_if [MEM_PORTS] + VX_mem_bus_if.master mem_bus_out_if [MEM_PORTS], + + // no uncached or outgoing request held short of the memory side + output wire req_empty ); localparam DIRECT_PASSTHRU = !CACHE_ENABLE && (`CS_WORD_SEL_BITS == 0) && (NUM_REQS == MEM_PORTS); localparam CORE_DATA_WIDTH = WORD_SIZE * 8; @@ -224,4 +227,33 @@ module VX_cache_bypass import VX_gpu_pkg::*; #( .bus_in_if (mem_bus_out_src_if), .bus_out_if (mem_bus_out_if) ); + + if (!DIRECT_PASSTHRU) begin : g_req_pending + localparam NUM_SRCS = NUM_REQS + (CACHE_ENABLE ? MEM_PORTS : 0); + wire [NUM_SRCS-1:0] src_fire; + wire [MEM_PORTS-1:0] dst_fire; + for (genvar i = 0; i < NUM_REQS; ++i) begin : g_nc_fire + assign src_fire[i] = core_bus_in_nc_if[i].req_valid && core_bus_in_nc_if[i].req_ready; + end + if (CACHE_ENABLE) begin : g_cache_fire + for (genvar i = 0; i < MEM_PORTS; ++i) begin : g_mem_in_fire + assign src_fire[NUM_REQS + i] = mem_bus_in_if[i].req_valid && mem_bus_in_if[i].req_ready; + end + end + for (genvar i = 0; i < MEM_PORTS; ++i) begin : g_dst_fire + assign dst_fire[i] = mem_bus_out_if[i].req_valid && mem_bus_out_if[i].req_ready; + end + VX_mem_req_pending #( + .NUM_INPUTS (NUM_SRCS), + .NUM_OUTPUTS (MEM_PORTS) + ) req_pending ( + .clk (clk), + .reset (reset), + .in_fire (src_fire), + .out_fire (dst_fire), + .empty (req_empty) + ); + end else begin : g_no_req_pending + assign req_empty = 1'b1; + end endmodule diff --git a/hw/rtl/cache/VX_cache_cluster.sv b/hw/rtl/cache/VX_cache_cluster.sv index 31868bcd1b..0dfc14eca1 100644 --- a/hw/rtl/cache/VX_cache_cluster.sv +++ b/hw/rtl/cache/VX_cache_cluster.sv @@ -55,6 +55,7 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( parameter MEM_OUT_BUF = 3, // Memory request output buffer parameter IS_LLC = 0, // Indicates this cache cluster is the LLC (handles AMO completion). + parameter SHARED_LLC = 0, // The LLC serves every memory client alone (point of coherence). parameter FLUSH_FORWARD = 1, // A flush also flushes the next level. parameter BANK_HASH = 0, // Hashed bank select (cache_bank_hash). @@ -67,7 +68,10 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( output cache_perf_t cache_perf, `endif VX_mem_bus_if.slave core_bus_if [NUM_INPUTS * NUM_REQS], - VX_mem_bus_if.master mem_bus_if [MEM_PORTS] + VX_mem_bus_if.master mem_bus_if [MEM_PORTS], + + // no uncached or outgoing request waiting in the arbiters or bypass paths + output wire req_empty ); localparam NUM_CACHES = `UP(NUM_UNITS); localparam PASSTHRU = (NUM_UNITS == 0); @@ -89,6 +93,10 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( .DATA_SIZE (WORD_SIZE), .TAG_WIDTH (ARB_TAG_WIDTH) ) arb_core_bus_if[NUM_CACHES * NUM_REQS](); + wire [NUM_REQS-1:0] core_arb_empty; + wire [NUM_CACHES-1:0] cache_req_empty; + wire [MEM_PORTS-1:0] mem_arb_empty; + assign req_empty = (& core_arb_empty) && (& cache_req_empty) && (& mem_arb_empty); for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_arb VX_mem_bus_if #( .DATA_SIZE (WORD_SIZE), @@ -116,6 +124,28 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( .bus_in_if (core_bus_tmp_if), .bus_out_if (arb_core_bus_tmp_if) ); + if (NUM_INPUTS != NUM_CACHES) begin : g_core_arb_pending + wire [NUM_INPUTS-1:0] in_fire; + wire [NUM_CACHES-1:0] out_fire; + for (genvar j = 0; j < NUM_INPUTS; ++j) begin : g_in_fire + assign in_fire[j] = core_bus_tmp_if[j].req_valid && core_bus_tmp_if[j].req_ready; + end + for (genvar k = 0; k < NUM_CACHES; ++k) begin : g_out_fire + assign out_fire[k] = arb_core_bus_tmp_if[k].req_valid && arb_core_bus_tmp_if[k].req_ready; + end + VX_mem_req_pending #( + .NUM_INPUTS (NUM_INPUTS), + .NUM_OUTPUTS (NUM_CACHES) + ) core_arb_pending ( + .clk (clk), + .reset (reset), + .in_fire (in_fire), + .out_fire (out_fire), + .empty (core_arb_empty[i]) + ); + end else begin : g_no_core_arb_pending + assign core_arb_empty[i] = 1'b1; + end for (genvar k = 0; k < NUM_CACHES; ++k) begin : g_arb_core_bus_if `ASSIGN_VX_MEM_BUS_IF (arb_core_bus_if[k * NUM_REQS + i], arb_core_bus_tmp_if[k]); end @@ -147,6 +177,7 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( .NC_ENABLE (NC_ENABLE), .PASSTHRU (PASSTHRU), .IS_LLC (IS_LLC), + .SHARED_LLC (SHARED_LLC), .FLUSH_FORWARD (FLUSH_FORWARD), .BANK_HASH (BANK_HASH), .AMO_ENABLE (AMO_ENABLE) @@ -157,7 +188,8 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( .clk (clk), .reset (reset), .core_bus_if (arb_core_bus_if[i * NUM_REQS +: NUM_REQS]), - .mem_bus_if (cache_mem_bus_if[i * MEM_PORTS +: MEM_PORTS]) + .mem_bus_if (cache_mem_bus_if[i * MEM_PORTS +: MEM_PORTS]), + .req_empty (cache_req_empty[i]) ); end for (genvar i = 0; i < MEM_PORTS; ++i) begin : g_mem_bus_if @@ -187,6 +219,24 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( .bus_in_if (arb_core_bus_tmp_if), .bus_out_if (mem_bus_tmp_if) ); + if (NUM_CACHES > 1) begin : g_mem_arb_pending + wire [NUM_CACHES-1:0] in_fire; + for (genvar j = 0; j < NUM_CACHES; ++j) begin : g_in_fire + assign in_fire[j] = arb_core_bus_tmp_if[j].req_valid && arb_core_bus_tmp_if[j].req_ready; + end + VX_mem_req_pending #( + .NUM_INPUTS (NUM_CACHES), + .NUM_OUTPUTS (1) + ) mem_arb_pending ( + .clk (clk), + .reset (reset), + .in_fire (in_fire), + .out_fire (mem_bus_tmp_if[0].req_valid && mem_bus_tmp_if[0].req_ready), + .empty (mem_arb_empty[i]) + ); + end else begin : g_no_mem_arb_pending + assign mem_arb_empty[i] = 1'b1; + end if (WRITE_ENABLE) begin : g_we `ASSIGN_VX_MEM_BUS_IF (mem_bus_if[i], mem_bus_tmp_if[0]); end else begin : g_ro diff --git a/hw/rtl/cache/VX_cache_init.sv b/hw/rtl/cache/VX_cache_init.sv index 8159a6ba32..9dc988ff3a 100644 --- a/hw/rtl/cache/VX_cache_init.sv +++ b/hw/rtl/cache/VX_cache_init.sv @@ -23,11 +23,20 @@ // the cache's own response path, and the inputs reopen. // A token that arrives during the walk is visible only after all of its // input's earlier requests were accepted, so the walk covered them. +// An L1-only token (any byteen bit set: a next level with wider words sees +// the bit at the token's line offset) is forwarded with its kind. A +// last-level cache that alone serves every memory client is the point of +// coherence for it: once the token is visible, its input's earlier requests +// are inside the cache, ahead of anything the next kernel sends, so it answers +// the token without a walk. Any other last-level cache still walks, since a +// replicated instance or a client reaching memory beside it may hold or need +// lines it has not written back. module VX_cache_init import VX_gpu_pkg::*; #( parameter NUM_REQS = 4, // Number of Word requests per cycle parameter NUM_BANKS = 1, // Number of banks parameter MEM_PORTS = 1, // Number of memory ports parameter FLUSH_FORWARD = 1, // Forward the flush to the next level + parameter SHARED_LLC = 0, // The LLC serving every memory client alone: answers L1-only tokens without a walk parameter TAG_WIDTH = UUID_WIDTH + 1, // core request tag size parameter BANK_SEL_LATENCY = 1, // Bank select latency parameter REQ_SEL_WIDTH = `UP(`CLOG2(NUM_REQS)) @@ -43,6 +52,7 @@ module VX_cache_init import VX_gpu_pkg::*; #( // flush tokens to the next level input wire mem_req_empty, // no write-back left between the banks and the ports output wire [MEM_PORTS-1:0] mem_flush_valid, + output wire mem_flush_l1_only, input wire [MEM_PORTS-1:0] mem_flush_ready, output wire mem_flush_rsp_phase, input wire [MEM_PORTS-1:0] mem_flush_rsp_valid, @@ -109,12 +119,20 @@ module VX_cache_init import VX_gpu_pkg::*; #( end wire flush_req_enable = (| flush_req_mask); + wire [NUM_REQS-1:0] flush_l1_mask; + for (genvar i = 0; i < NUM_REQS; ++i) begin : g_flush_l1_mask + assign flush_l1_mask[i] = flush_req_mask[i] && (| core_bus_in_if[i].req_data.byteen); + end + // every visible token is L1-only + wire flush_l1_only = (flush_l1_mask == flush_req_mask); + reg [NUM_BANKS-1:0] flush_done, flush_done_n; reg fwd_go, fwd_go_n; // the crossbar drained; only tokens enter it now reg [MEM_PORTS-1:0] fwd_sent, fwd_sent_n; reg [MEM_PORTS-1:0] fwd_done, fwd_done_n; reg [NUM_REQS-1:0] rsp_pending, rsp_pending_n; // token still to be answered reg [`UP(UUID_WIDTH)-1:0] flush_uuid_r, flush_uuid_n; + reg l1_only_r, l1_only_n; // answer the lowest pending input; its token stays at the input until then wire [NUM_REQS-1:0] rsp_sel_onehot; @@ -203,11 +221,19 @@ module VX_cache_init import VX_gpu_pkg::*; #( fwd_done_n = fwd_done; rsp_pending_n = rsp_pending & ~(rsp_sel_onehot & {NUM_REQS{flush_rsp_fire}}); flush_uuid_n = flush_uuid_r; + l1_only_n = l1_only_r; case (state) // STATE_IDLE: default: begin if (flush_req_enable) begin - state_n = (BANK_SEL_LATENCY != 0) ? STATE_WAIT1 : STATE_FLUSH; + if (BANK_SEL_LATENCY != 0) begin + state_n = STATE_WAIT1; + end else if (SHARED_LLC && flush_l1_only) begin + rsp_pending_n = flush_req_mask; + state_n = STATE_RESPOND; + end else begin + state_n = STATE_FLUSH; + end for (integer i = NUM_REQS-1; i >= 0; --i) begin if (flush_req_mask[i]) begin flush_uuid_n = core_bus_in_uuid[i]; @@ -217,7 +243,12 @@ module VX_cache_init import VX_gpu_pkg::*; #( end STATE_WAIT1: begin if (no_inflight_reqs) begin - state_n = STATE_FLUSH; + if (SHARED_LLC && flush_l1_only) begin + rsp_pending_n = flush_req_mask; + state_n = STATE_RESPOND; + end else begin + state_n = STATE_FLUSH; + end end end STATE_FLUSH: begin @@ -231,6 +262,7 @@ module VX_cache_init import VX_gpu_pkg::*; #( flush_done_n = '0; // answer the tokens visible now; later ones start a new walk rsp_pending_n = flush_req_mask; + l1_only_n = flush_l1_only; state_n = FLUSH_FORWARD ? STATE_FORWARD : STATE_RESPOND; end end @@ -270,6 +302,7 @@ module VX_cache_init import VX_gpu_pkg::*; #( rsp_pending <= rsp_pending_n; end flush_uuid_r <= flush_uuid_n; + l1_only_r <= l1_only_n; end assign flush_begin = {NUM_BANKS{state == STATE_FLUSH}}; @@ -279,5 +312,6 @@ module VX_cache_init import VX_gpu_pkg::*; #( // take the others' valid away. assign mem_flush_valid = {MEM_PORTS{(state == STATE_FORWARD) && (fwd_go || mem_req_empty)}} & ~fwd_sent; assign mem_flush_rsp_phase = (state == STATE_FORWARD); + assign mem_flush_l1_only = l1_only_r; endmodule diff --git a/hw/rtl/cache/VX_cache_wrap.sv b/hw/rtl/cache/VX_cache_wrap.sv index 24ca0f7a62..e9f34d17bc 100644 --- a/hw/rtl/cache/VX_cache_wrap.sv +++ b/hw/rtl/cache/VX_cache_wrap.sv @@ -55,6 +55,7 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( parameter MEM_OUT_BUF = 3, // Memory request output buffer parameter IS_LLC = 0, // Set when this cache is the LLC (enables AMO completion at this level). + parameter SHARED_LLC = 0, // The LLC serves every memory client alone (point of coherence). parameter FLUSH_FORWARD = 1, // A flush also flushes the next level. parameter BANK_HASH = 0, // Hashed bank select (cache_bank_hash). @@ -68,7 +69,10 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( output cache_perf_t cache_perf, `endif VX_mem_bus_if.slave core_bus_if [NUM_REQS], - VX_mem_bus_if.master mem_bus_if [MEM_PORTS] + VX_mem_bus_if.master mem_bus_if [MEM_PORTS], + + // no uncached or outgoing request waiting in the bypass path + output wire req_empty ); `STATIC_ASSERT(NUM_BANKS == (1 << `CLOG2(NUM_BANKS)), ("invalid parameter")) localparam CACHE_MEM_TAG_WIDTH = `CACHE_MEM_TAG_WIDTH(MSHR_SIZE, NUM_BANKS, MEM_PORTS, UUID_WIDTH); @@ -108,9 +112,11 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( .core_bus_in_if (core_bus_if), .core_bus_out_if(core_bus_cache_if), .mem_bus_in_if (mem_bus_cache_if), - .mem_bus_out_if (mem_bus_tmp_if) + .mem_bus_out_if (mem_bus_tmp_if), + .req_empty (req_empty) ); end else begin : g_no_bypass + assign req_empty = 1'b1; for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_bus_cache_if `ASSIGN_VX_MEM_BUS_IF (core_bus_cache_if[i], core_bus_if[i]); end @@ -149,6 +155,7 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( .CORE_OUT_BUF (BYPASS_ENABLE ? 1 : CORE_OUT_BUF), .MEM_OUT_BUF (BYPASS_ENABLE ? 1 : MEM_OUT_BUF), .IS_LLC (IS_LLC), + .SHARED_LLC (SHARED_LLC), .FLUSH_FORWARD (FLUSH_FORWARD), .BANK_HASH (BANK_HASH), .AMO_ENABLE (AMO_ENABLE) diff --git a/hw/rtl/core/VX_core.sv b/hw/rtl/core/VX_core.sv index 46b3918de4..618fabade9 100644 --- a/hw/rtl/core/VX_core.sv +++ b/hw/rtl/core/VX_core.sv @@ -174,6 +174,7 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( VX_dcr_flush_if dcr_flush_if(); VX_dcr_flush_if dcr_flush_dcache_if(); VX_dcr_flush_if dcr_flush_icache_if(); + wire dcr_flush_l1_only; // High when the in-core global-store path is empty (no store still in // flight toward the dcache). Gates `busy` so the flush waits for the tail. @@ -209,7 +210,8 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( .reset (reset), .dcr_bus_if (dcr_bus_if), .dcr_csr_if (dcr_csr_if), - .dcr_flush_if(dcr_flush_if) + .dcr_flush_if(dcr_flush_if), + .flush_l1_only(dcr_flush_l1_only) ); `SCOPE_IO_SWITCH (3); @@ -271,6 +273,7 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( .clk (clk), .reset (reset), .dcr_flush_if (dcr_flush_icache_if), + .l1_only (dcr_flush_l1_only), .core_bus_if (fetch_icache_if[0]), .cache_bus_if (mmu_icache_if[0]) ); @@ -445,6 +448,7 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( .empty (mem_unit_empty), .lsu_mem_if (lsu_mem_if), .dcr_flush_if (dcr_flush_dcache_if), + .flush_l1_only (dcr_flush_l1_only), .dcache_bus_if (mmu_dcache_if) ); @@ -477,7 +481,13 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( // busy tracks only kernel execution: warps resident or stores not yet at L1. // ~mmu_empty also keeps the core busy while a translation or parked store is // still in flight, so kernel completion cannot race un-drained accesses. - assign busy = sched_busy || ~(&lsu_sched_empty) || ~mem_unit_empty || ~mmu_empty; + // A request waiting at the dcache port sits past the flush injector's + // buffer, which mem_unit_empty does not cover. + wire [DCACHE_NUM_REQS-1:0] dcache_req_valid; + for (genvar i = 0; i < DCACHE_NUM_REQS; ++i) begin : g_dcache_req_valid + assign dcache_req_valid[i] = mmu_dcache_if[i].req_valid; + end + assign busy = sched_busy || ~(&lsu_sched_empty) || ~mem_unit_empty || (| dcache_req_valid) || ~mmu_empty; // BAR (vx_barrier / vx_barrier_arrive) drains LSU + MMU before suspending or registering arrival. assign warp_ctl_if.lsu_sched_drained = (&lsu_sched_empty) && mmu_empty; diff --git a/hw/rtl/core/VX_dcr_data.sv b/hw/rtl/core/VX_dcr_data.sv index 746848286e..44f5624b82 100644 --- a/hw/rtl/core/VX_dcr_data.sv +++ b/hw/rtl/core/VX_dcr_data.sv @@ -27,7 +27,8 @@ module VX_dcr_data import VX_gpu_pkg::*; #( VX_dcr_csr_if.master dcr_csr_if, // DCR-triggered cache flush - VX_dcr_flush_if.master dcr_flush_if + VX_dcr_flush_if.master dcr_flush_if, + output wire flush_l1_only ); `UNUSED_SPARAM (INSTANCE_ID) @@ -82,6 +83,7 @@ module VX_dcr_data import VX_gpu_pkg::*; #( reg flush_pending_r; // flush running reg flush_complete_r; // flush finished, not yet reported reg flush_read_r; // flush read waiting for completion + reg flush_l1_only_r; // the last-level cache keeps its lines wire is_flush_req = dcr_bus_if.req_valid && (dcr_bus_if.req_data.addr == `VX_DCR_BASE_CACHE_FLUSH); @@ -98,15 +100,20 @@ module VX_dcr_data import VX_gpu_pkg::*; #( flush_pending_r <= 1'b0; flush_complete_r <= 1'b0; flush_read_r <= 1'b0; + flush_l1_only_r <= 1'b0; end else begin flush_pending_r <= flush_start || (flush_pending_r && ~flush_done); // a write opens a new flush, so an older completion cannot answer it flush_complete_r <= ~is_flush_write && ~flush_rsp && (flush_complete_r || flush_done); flush_read_r <= ~flush_rsp && (flush_read_r || is_flush_read); + if (flush_start) begin + flush_l1_only_r <= (dcr_bus_if.req_data.data & VX_DCR_DATA_WIDTH'(`VX_CACHE_FLUSH_L1_ONLY)) != 0; + end end end assign dcr_flush_if.req = flush_pending_r; + assign flush_l1_only = flush_l1_only_r; // Return CSR data or flush-done on DCR response bus // Use intermediate registers — Vivado (Synth 8-453) forbids procedural diff --git a/hw/rtl/core/VX_dcr_flush.sv b/hw/rtl/core/VX_dcr_flush.sv index 051eaf85d2..8bf7ec4883 100644 --- a/hw/rtl/core/VX_dcr_flush.sv +++ b/hw/rtl/core/VX_dcr_flush.sv @@ -33,6 +33,7 @@ module VX_dcr_flush import VX_gpu_pkg::*; #( input wire reset, VX_dcr_flush_if.slave dcr_flush_if, + input wire l1_only, // the last-level cache need not walk VX_mem_bus_if.slave core_bus_if, // TAG_WIDTH VX_mem_bus_if.master cache_bus_if // TAG_WIDTH+1 ); @@ -72,12 +73,13 @@ module VX_dcr_flush import VX_gpu_pkg::*; #( rw: 1'b0, addr: '0, data: '0, - byteen: '0, + byteen: WORD_SIZE'(l1_only), attr: MEM_ATTR_WIDTH'(1 << MEM_ATTR_FLUSH_OFFS), tag: '0 }; - // The flush request only consumes MEM_ATTR_FLUSH_OFFS downstream; the - // remaining req_data fields are tied off and intentionally unused. + // The flush request only consumes MEM_ATTR_FLUSH_OFFS and byteen bit 0 + // (the L1-only kind) downstream; the remaining req_data fields are tied + // off and intentionally unused. `UNUSED_VAR (flush_bus_if.req_data.rw) `UNUSED_VAR (flush_bus_if.req_data.addr) `UNUSED_VAR (flush_bus_if.req_data.data) diff --git a/hw/rtl/core/VX_lsu_slice.sv b/hw/rtl/core/VX_lsu_slice.sv index affb932298..066df5073e 100644 --- a/hw/rtl/core/VX_lsu_slice.sv +++ b/hw/rtl/core/VX_lsu_slice.sv @@ -261,7 +261,8 @@ module VX_lsu_slice import VX_gpu_pkg::*; #( default : mem_req_byteen_w = {LSU_WORD_SIZE{1'b1}}; endcase end - assign mem_req_byteen[i] = mem_req_byteen_w; + // a fence is a flush token: it carries no data, so no byte enables + assign mem_req_byteen[i] = req_is_fence ? '0 : mem_req_byteen_w; end // memory misalignment not supported! diff --git a/hw/rtl/core/VX_mem_unit.sv b/hw/rtl/core/VX_mem_unit.sv index bcd4083306..a766858777 100644 --- a/hw/rtl/core/VX_mem_unit.sv +++ b/hw/rtl/core/VX_mem_unit.sv @@ -38,6 +38,7 @@ module VX_mem_unit import VX_gpu_pkg::*; #( VX_lsu_mem_if.slave lsu_mem_if [`VX_CFG_NUM_LSU_BLOCKS], VX_dcr_flush_if.slave dcr_flush_if, + input wire flush_l1_only, VX_mem_bus_if.master dcache_bus_if [DCACHE_NUM_REQS] ); // Per-block empty: high when the global-store path holds no in-flight @@ -424,6 +425,7 @@ module VX_mem_unit import VX_gpu_pkg::*; #( .clk (clk), .reset (reset), .dcr_flush_if (lane_flush_if[LANE]), + .l1_only (flush_l1_only), .core_bus_if (dcache_bus_tmp_if[j]), .cache_bus_if (dcache_bus_if[LANE]) ); diff --git a/hw/rtl/cp/VX_cp_core.sv b/hw/rtl/cp/VX_cp_core.sv index 9ff16e575a..ee0849b4e7 100644 --- a/hw/rtl/cp/VX_cp_core.sv +++ b/hw/rtl/cp/VX_cp_core.sv @@ -358,7 +358,8 @@ module VX_cp_core .dcr_req_addr (gpu_if_int.dcr_req_addr), .dcr_req_data (gpu_if_int.dcr_req_data), .dcr_rsp_valid (gpu_if_int.dcr_rsp_valid), - .dcr_rsp_data (gpu_if_int.dcr_rsp_data) + .dcr_rsp_data (gpu_if_int.dcr_rsp_data), + .gpu_busy (gpu_if_int.busy) ); `UNUSED_VAR (gpu_if_int.dcr_req_ready) diff --git a/hw/rtl/cp/VX_cp_dcr_proxy.sv b/hw/rtl/cp/VX_cp_dcr_proxy.sv index 05a90fd6de..3ae9a3d583 100644 --- a/hw/rtl/cp/VX_cp_dcr_proxy.sv +++ b/hw/rtl/cp/VX_cp_dcr_proxy.sv @@ -16,12 +16,15 @@ // // For CMD_CACHE_FLUSH (cmd.arg0 = number of cores): // IDLE → START (DCR write to VX_DCR_BASE_CACHE_FLUSH, which every core -// receives and starts flushing on) → then REQ/WAIT_RSP once per core: a +// receives and starts flushing on; F_FLUSH_L1_ONLY sets +// VX_CACHE_FLUSH_L1_ONLY in its value) → then REQ/WAIT_RSP once per core: a // DCR read of VX_DCR_BASE_CACHE_FLUSH answered when that core's flush has // completed (see VX_dcr_data). The cores flush concurrently, so their dcache // flushes reach the shared caches together and share one walk; the reads // are serial because the DCR response bus cannot carry two at once. The -// command retires once the last core reports. This is the ACQUIRE_MEM +// command retires once the last core reports and, for a full flush, once +// Vortex busy drops: busy covers every write until memory acknowledges it, +// so a DMA after the flush cannot overtake the write-backs. This is the ACQUIRE_MEM // model: a single command in the ring, executed by the CP across all cores, // that the host posts after CMD_LAUNCH so it sees coherent results. The // host fills cmd.arg0 from VX_CAPS_NUM_CORES. @@ -39,7 +42,7 @@ module VX_cp_dcr_proxy input wire grant, // verilator lint_off UNUSED - // Only cmd.hdr.opcode, cmd.arg0, and cmd.arg1 are read here. arg2 and + // Only cmd.hdr.opcode/flags, cmd.arg0, and cmd.arg1 are read here. arg2 and // profile_slot pass through untouched on the way to the engine; the // top-level instantiation hands us the full struct. input cmd_t cmd, @@ -57,7 +60,9 @@ module VX_cp_dcr_proxy output logic [`VX_DCR_ADDR_BITS-1:0] dcr_req_addr, output logic [`VX_DCR_DATA_BITS-1:0] dcr_req_data, input wire dcr_rsp_valid, - input wire [`VX_DCR_DATA_BITS-1:0] dcr_rsp_data + input wire [`VX_DCR_DATA_BITS-1:0] dcr_rsp_data, + + input wire gpu_busy ); typedef enum logic [2:0] { @@ -75,6 +80,7 @@ module VX_cp_dcr_proxy state_e state; logic pending_is_read; logic pending_is_flush; + logic pending_l1_only; // cmd fields are only valid for one cycle (grant pulse); capture on IDLE → REQ. logic [`VX_DCR_ADDR_BITS-1:0] pending_addr; logic [`VX_DCR_DATA_BITS-1:0] pending_data; @@ -88,11 +94,17 @@ module VX_cp_dcr_proxy wire [`VX_DCR_DATA_BITS-1:0] cmd_data = cmd.arg1[`VX_DCR_DATA_BITS-1:0]; wire [CIDW-1:0] cmd_ncores = cmd.arg0[CIDW-1:0]; + // The last-level cache forwards a full token behind its write-backs, so they + // are already counted in busy when the last core reports. An L1-only flush + // writes nothing back past the last-level cache. + wire flush_drain = pending_is_flush && !pending_l1_only && gpu_busy; + always_ff @(posedge clk) begin if (reset) begin state <= S_IDLE; pending_is_read <= 1'b0; pending_is_flush <= 1'b0; + pending_l1_only <= 1'b0; pending_addr <= '0; pending_data <= '0; rsp_data_r <= '0; @@ -104,6 +116,7 @@ module VX_cp_dcr_proxy if (grant) begin pending_is_read <= is_read; pending_is_flush <= is_flush; + pending_l1_only <= cmd.hdr.flags[F_FLUSH_L1_ONLY]; pending_addr <= cmd_addr; pending_data <= cmd_data; if (is_flush) begin @@ -140,7 +153,9 @@ module VX_cp_dcr_proxy end end S_DONE: begin - state <= S_IDLE; + if (!flush_drain) begin + state <= S_IDLE; + end end default: state <= S_IDLE; endcase @@ -154,13 +169,14 @@ module VX_cp_dcr_proxy // target core index in the low 16 bits of dcr_req_data. dcr_req_rw = (state == S_START); dcr_req_addr = `VX_DCR_ADDR_BITS'(`VX_DCR_BASE_CACHE_FLUSH); - dcr_req_data = `VX_DCR_DATA_BITS'(flush_cid); + dcr_req_data = (state == S_START) ? (pending_l1_only ? `VX_DCR_DATA_BITS'(`VX_CACHE_FLUSH_L1_ONLY) : '0) + : `VX_DCR_DATA_BITS'(flush_cid); end else begin dcr_req_rw = !pending_is_read; dcr_req_addr = pending_addr; dcr_req_data = pending_data; end - done = (state == S_DONE); + done = (state == S_DONE) && !flush_drain; last_rsp_data = rsp_data_r; end diff --git a/hw/rtl/cp/VX_cp_pkg.sv b/hw/rtl/cp/VX_cp_pkg.sv index a7f1d1ae70..8a8f1d9452 100644 --- a/hw/rtl/cp/VX_cp_pkg.sv +++ b/hw/rtl/cp/VX_cp_pkg.sv @@ -80,6 +80,8 @@ package VX_cp_pkg; localparam int F_PROFILE = 0; localparam int F_FENCE_PRE = 1; + // CMD_CACHE_FLUSH: flush only the levels above the last-level cache + localparam int F_FLUSH_L1_ONLY = 2; typedef struct packed { logic [15:0] reserved; diff --git a/hw/rtl/mem/VX_mem_req_pending.sv b/hw/rtl/mem/VX_mem_req_pending.sv new file mode 100644 index 0000000000..b92e287920 --- /dev/null +++ b/hw/rtl/mem/VX_mem_req_pending.sv @@ -0,0 +1,53 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// Tracks the requests held between the input and output handshakes of a +// section that neither creates nor drops requests (arbiters, buffers). +module VX_mem_req_pending #( + parameter NUM_INPUTS = 1, + parameter NUM_OUTPUTS = 1, + parameter SIZE = 8 * NUM_OUTPUTS + 2 * NUM_INPUTS // bound on the requests the section can hold +) ( + input wire clk, + input wire reset, + input wire [NUM_INPUTS-1:0] in_fire, + input wire [NUM_OUTPUTS-1:0] out_fire, + output wire empty +); + localparam INCRW = `CLOG2(NUM_INPUTS+1); + localparam DECRW = `CLOG2(NUM_OUTPUTS+1); + + wire [INCRW-1:0] in_cnt; + wire [DECRW-1:0] out_cnt; + `POP_COUNT(in_cnt, in_fire); + `POP_COUNT(out_cnt, out_fire); + + VX_pending_size #( + .SIZE (SIZE), + .INCRW (INCRW), + .DECRW (DECRW) + ) pending ( + .clk (clk), + .reset (reset), + .incr (in_cnt), + .decr (out_cnt), + .empty (empty), + `UNUSED_PIN (alm_empty), + `UNUSED_PIN (full), + `UNUSED_PIN (alm_full), + `UNUSED_PIN (size) + ); + +endmodule diff --git a/hw/rtl/om/VX_om_steer.sv b/hw/rtl/om/VX_om_steer.sv index 7aa803e5ee..0fd8ce9c8e 100644 --- a/hw/rtl/om/VX_om_steer.sv +++ b/hw/rtl/om/VX_om_steer.sv @@ -48,6 +48,7 @@ module VX_om_steer import VX_gpu_pkg::*; #( VX_mem_bus_if.master om_out_if, // aperture writes -> OM ingress (posted) output wire flush_req, // a flush token is held until the OM side is flushed + output wire flush_l1_only, // the held token is an L1-only one input wire flush_ack, // pulse: the OM side was flushed after flush_req rose output wire om_pending // an aperture write is still buffered on the OM leg ); @@ -89,6 +90,7 @@ module VX_om_steer import VX_gpu_pkg::*; #( end end assign flush_req = flush_req_r; + assign flush_l1_only = (| bus_in_if.req_data.byteen); assign om_pending = om_out_if.req_valid; // Both legs go through an OUT_BUF elastic buffer so the socket->L2 trunk and diff --git a/hw/unittest/cache/VX_cache_top.sv b/hw/unittest/cache/VX_cache_top.sv index 0e25d21b96..b75215a485 100644 --- a/hw/unittest/cache/VX_cache_top.sv +++ b/hw/unittest/cache/VX_cache_top.sv @@ -195,7 +195,8 @@ module VX_cache_top import VX_gpu_pkg::*; #( .clk (clk), .reset (reset), .core_bus_if (core_bus_if), - .mem_bus_if (mem_bus_if) + .mem_bus_if (mem_bus_if), + `UNUSED_PIN (req_empty) ); endmodule diff --git a/hw/unittest/cp_dcr_proxy/VX_cp_dcr_proxy_top.sv b/hw/unittest/cp_dcr_proxy/VX_cp_dcr_proxy_top.sv index 060b56a28d..1de85c8b61 100644 --- a/hw/unittest/cp_dcr_proxy/VX_cp_dcr_proxy_top.sv +++ b/hw/unittest/cp_dcr_proxy/VX_cp_dcr_proxy_top.sv @@ -28,7 +28,8 @@ module VX_cp_dcr_proxy_top output wire [`VX_DCR_ADDR_BITS-1:0] dcr_req_addr, output wire [`VX_DCR_DATA_BITS-1:0] dcr_req_data, input wire dcr_rsp_valid, - input wire [`VX_DCR_DATA_BITS-1:0] dcr_rsp_data + input wire [`VX_DCR_DATA_BITS-1:0] dcr_rsp_data, + input wire gpu_busy ); cmd_t cmd_typed; @@ -46,7 +47,8 @@ module VX_cp_dcr_proxy_top .dcr_req_addr (dcr_req_addr), .dcr_req_data (dcr_req_data), .dcr_rsp_valid (dcr_rsp_valid), - .dcr_rsp_data (dcr_rsp_data) + .dcr_rsp_data (dcr_rsp_data), + .gpu_busy (gpu_busy) ); endmodule : VX_cp_dcr_proxy_top diff --git a/hw/unittest/cp_dcr_proxy/main.cpp b/hw/unittest/cp_dcr_proxy/main.cpp index 861c0c2932..171acef369 100644 --- a/hw/unittest/cp_dcr_proxy/main.cpp +++ b/hw/unittest/cp_dcr_proxy/main.cpp @@ -26,6 +26,11 @@ // core index as data, each waiting for its response; done only // after the last core answers. // 6. CMD_CACHE_FLUSH over 0 cores retires without any DCR request. +// 7. CMD_CACHE_FLUSH with the L1-only flag: the broadcast write carries +// VX_CACHE_FLUSH_L1_ONLY (test 5 checks a full flush writes 0), the +// per-core read still carries the core index. +// 8. A full CMD_CACHE_FLUSH holds done while Vortex is busy; an L1-only +// one retires regardless. // ============================================================================ #include "vl_simulator.h" @@ -80,11 +85,14 @@ static void pack_cmd(uint32_t out_words[9], (static_cast(flags) << 8); } +// CMD_CACHE_FLUSH header flag bit 2 (F_FLUSH_L1_ONLY). +constexpr uint8_t FLUSH_FLAG_L1_ONLY = 0x04; + template static void set_cmd(T* top, uint8_t opcode, - uint64_t arg0 = 0, uint64_t arg1 = 0) { + uint64_t arg0 = 0, uint64_t arg1 = 0, uint8_t flags = 0) { uint32_t words[9]; - pack_cmd(words, opcode, 0, arg0, arg1, /*arg2=*/0, /*profile_slot=*/0); + pack_cmd(words, opcode, flags, arg0, arg1, /*arg2=*/0, /*profile_slot=*/0); for (int i = 0; i < 9; ++i) top->cmd_packed[i] = words[i]; } @@ -105,6 +113,7 @@ int main(int argc, char** argv) { sim->grant = 0; sim->dcr_rsp_valid = 0; sim->dcr_rsp_data = 0; + sim->gpu_busy = 0; set_cmd(sim.operator->(), 0); tick = sim.reset(tick); @@ -214,6 +223,7 @@ int main(int argc, char** argv) { EXPECT(sim->dcr_req_valid == 1, "FLUSH: start write not issued"); EXPECT(sim->dcr_req_rw == 1, "FLUSH: start must be a write"); EXPECT(sim->dcr_req_addr == VX_DCR_BASE_CACHE_FLUSH, "FLUSH: start addr mismatch"); + EXPECT(sim->dcr_req_data == 0, "FLUSH: a full flush starts with value 0"); EXPECT(sim->done == 0, "FLUSH: done premature in S_START"); cycle(sim, tick); // S_START → S_REQ @@ -257,6 +267,62 @@ int main(int argc, char** argv) { EXPECT(sim->done == 1, "FLUSH0: done not asserted"); cycle(sim, tick); + // ----- Test 7: L1-only CMD_CACHE_FLUSH over 1 core. ----- + set_cmd(sim.operator->(), OP_CACHE_FLUSH, 1, 0, FLUSH_FLAG_L1_ONLY); + sim->grant = 1; + cycle(sim, tick); // IDLE → S_START + sim->grant = 0; + sim->eval(); + EXPECT(sim->dcr_req_valid == 1, "FLUSHL1: start write not issued"); + EXPECT(sim->dcr_req_rw == 1, "FLUSHL1: start must be a write"); + EXPECT(sim->dcr_req_data == VX_CACHE_FLUSH_L1_ONLY, "FLUSHL1: start value must carry the L1-only flag"); + cycle(sim, tick); // S_START → S_REQ + sim->eval(); + EXPECT(sim->dcr_req_valid == 1, "FLUSHL1: per-core read not issued"); + EXPECT(sim->dcr_req_rw == 0, "FLUSHL1: per-core request must be a read"); + EXPECT(sim->dcr_req_data == 0, "FLUSHL1: read must carry the core index only"); + cycle(sim, tick); // S_REQ → S_WAIT_RSP + sim->dcr_rsp_valid = 1; + sim->dcr_rsp_data = 0; + cycle(sim, tick); // → S_DONE + sim->dcr_rsp_valid = 0; + sim->eval(); + EXPECT(sim->done == 1, "FLUSHL1: done not asserted"); + cycle(sim, tick); + + // ----- Test 8: a full flush waits for busy; an L1-only one does not. ----- + for (int l1_only = 0; l1_only < 2; ++l1_only) { + set_cmd(sim.operator->(), OP_CACHE_FLUSH, 1, 0, l1_only ? FLUSH_FLAG_L1_ONLY : 0); + sim->grant = 1; + cycle(sim, tick); // IDLE → S_START + sim->grant = 0; + cycle(sim, tick); // S_START → S_REQ + cycle(sim, tick); // S_REQ → S_WAIT_RSP + sim->gpu_busy = 1; + sim->dcr_rsp_valid = 1; + cycle(sim, tick); // → S_DONE + sim->dcr_rsp_valid = 0; + if (l1_only) { + sim->eval(); + EXPECT(sim->done == 1, "DRAIN: an L1-only flush must not wait for busy"); + cycle(sim, tick); + sim->gpu_busy = 0; + } else { + for (int i = 0; i < 4; ++i) { + sim->eval(); + EXPECT(sim->done == 0, "DRAIN: a full flush retired while busy"); + EXPECT(sim->dcr_req_valid == 0, "DRAIN: request while draining"); + cycle(sim, tick); + } + sim->gpu_busy = 0; + sim->eval(); + EXPECT(sim->done == 1, "DRAIN: done not asserted once idle"); + cycle(sim, tick); + sim->eval(); + EXPECT(sim->done == 0, "DRAIN: done should pulse only one cycle"); + } + } + std::printf("PASSED\n"); return 0; } diff --git a/hw/unittest/gfx/VX_gfx_top.sv b/hw/unittest/gfx/VX_gfx_top.sv index c16aae9aaf..aae201f6e1 100644 --- a/hw/unittest/gfx/VX_gfx_top.sv +++ b/hw/unittest/gfx/VX_gfx_top.sv @@ -292,6 +292,7 @@ module VX_gfx_top import VX_gpu_pkg::* .clk (clk), .reset (reset), .dcr_flush_if (tcache_flush_if), + .l1_only (1'b0), .core_bus_if (tcache_bus_if[0]), .cache_bus_if (tcache_flushable_bus_if[0]) ); @@ -329,7 +330,8 @@ module VX_gfx_top import VX_gpu_pkg::* .clk (clk), .reset (reset), .core_bus_if (tcache_flushable_bus_if), - .mem_bus_if (tcache_mem_bus_tmp_if) + .mem_bus_if (tcache_mem_bus_tmp_if), + `UNUSED_PIN (req_empty) ); `else assign flush_done_w[0] = 1'b1; @@ -444,6 +446,7 @@ module VX_gfx_top import VX_gpu_pkg::* .clk (clk), .reset (reset), .dcr_flush_if (rtcache_flush_if), + .l1_only (1'b0), .core_bus_if (rtcache_bus_if[0]), .cache_bus_if (rtcache_flushable_bus_if[0]) ); @@ -481,7 +484,8 @@ module VX_gfx_top import VX_gpu_pkg::* .clk (clk), .reset (reset), .core_bus_if (rtcache_flushable_bus_if), - .mem_bus_if (rtcache_mem_bus_tmp_if) + .mem_bus_if (rtcache_mem_bus_tmp_if), + `UNUSED_PIN (req_empty) ); `else assign flush_done_w[1] = 1'b1; @@ -765,7 +769,8 @@ module VX_gfx_top import VX_gpu_pkg::* .clk (clk), .reset (reset), .core_bus_if (l2_core_if), - .mem_bus_if (l2_mem_if) + .mem_bus_if (l2_mem_if), + `UNUSED_PIN (req_empty) ); endmodule diff --git a/hw/unittest/mem_unit/VX_mem_unit_top.sv b/hw/unittest/mem_unit/VX_mem_unit_top.sv index e88c3d87b9..ad2c75034e 100644 --- a/hw/unittest/mem_unit/VX_mem_unit_top.sv +++ b/hw/unittest/mem_unit/VX_mem_unit_top.sv @@ -130,6 +130,7 @@ module VX_mem_unit_top import VX_gpu_pkg::*; #( .empty (mem_unit_empty), .lsu_mem_if (lsu_mem_if), .dcr_flush_if (dcr_flush_if), + .flush_l1_only (1'b0), .dcache_bus_if (mem_bus_if) ); diff --git a/hw/unittest/tensor/VX_tensor_top.sv b/hw/unittest/tensor/VX_tensor_top.sv index 70244c6cc7..629ab576c4 100644 --- a/hw/unittest/tensor/VX_tensor_top.sv +++ b/hw/unittest/tensor/VX_tensor_top.sv @@ -278,6 +278,7 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( .empty (mem_unit_empty[c]), .lsu_mem_if (lsu_mem_if), .dcr_flush_if (dcr_flush_dcache_if), + .flush_l1_only (1'b0), .dcache_bus_if (dcache_bus_if) ); @@ -394,7 +395,8 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( .clk (clk), .reset (reset), .core_bus_if (all_dc_core_if), - .mem_bus_if (dcache_mem_bus_if) + .mem_bus_if (dcache_mem_bus_if), + `UNUSED_PIN (req_empty) ); // ======================================================================== @@ -513,7 +515,8 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( .clk (clk), .reset (reset), .core_bus_if (l2_core_if), - .mem_bus_if (l2_mem_if) + .mem_bus_if (l2_mem_if), + `UNUSED_PIN (req_empty) ); assign empty = (& mem_unit_empty) && (& lsu_sched_empty) && !dxa_busy; diff --git a/hw/unittest/vm/VX_vm_top.sv b/hw/unittest/vm/VX_vm_top.sv index 6e2f7a32e0..54e4c29b15 100644 --- a/hw/unittest/vm/VX_vm_top.sv +++ b/hw/unittest/vm/VX_vm_top.sv @@ -315,7 +315,8 @@ module VX_vm_top import VX_gpu_pkg::*, VX_tlb_pkg::*; #( .clk (clk), .reset (reset), .core_bus_if (dmmu_phys_if), - .mem_bus_if (all_dc_mem_if[s * L1_MEM_PORTS +: L1_MEM_PORTS]) + .mem_bus_if (all_dc_mem_if[s * L1_MEM_PORTS +: L1_MEM_PORTS]), + `UNUSED_PIN (req_empty) ); if (INCLUDE_IMMU != 0) begin : g_immu @@ -417,7 +418,8 @@ module VX_vm_top import VX_gpu_pkg::*, VX_tlb_pkg::*; #( .clk (clk), .reset (reset), .core_bus_if (immu_phys_if), - .mem_bus_if (ic_mem_if) + .mem_bus_if (ic_mem_if), + `UNUSED_PIN (req_empty) ); end else begin : g_no_immu // Keep the socket arbiter 2-wide (so TLB_SOCKET_ID_WIDTH matches the @@ -625,7 +627,8 @@ module VX_vm_top import VX_gpu_pkg::*, VX_tlb_pkg::*; #( .clk (clk), .reset (reset), .core_bus_if (l2_core_if), - .mem_bus_if (l2_mem_if) + .mem_bus_if (l2_mem_if), + `UNUSED_PIN (req_empty) ); assign empty = (& dmmu_empty) && (& immu_empty) && l2tlb_empty && ptw_empty; diff --git a/sim/common/cmd_processor.cpp b/sim/common/cmd_processor.cpp index c6e97fb46f..2ace286517 100644 --- a/sim/common/cmd_processor.cpp +++ b/sim/common/cmd_processor.cpp @@ -332,16 +332,20 @@ void CommandProcessor::apply_qmd_(uint64_t qmd_addr) { // flush, so the cores flush concurrently and share one walk of the shared // caches; the per-core reads, answered once that core's flush completes, are // serial because the DCR response bus cannot carry two responses at once. -void CommandProcessor::flush_cores_(uint32_t num_cores) { +void CommandProcessor::flush_cores_(uint32_t num_cores, bool l1_only) { if (num_cores == 0 || !hooks_.vortex_dcr_read) { return; } if (hooks_.vortex_dcr_write) { - hooks_.vortex_dcr_write(VX_DCR_BASE_CACHE_FLUSH, 0); + hooks_.vortex_dcr_write(VX_DCR_BASE_CACHE_FLUSH, + l1_only ? VX_CACHE_FLUSH_L1_ONLY : 0); } for (uint32_t cid = 0; cid < num_cores; ++cid) { (void)hooks_.vortex_dcr_read(VX_DCR_BASE_CACHE_FLUSH, cid); } + if (!l1_only && hooks_.vortex_drain) { + hooks_.vortex_drain(); + } } // Execute one draw-bundle step. Inline ops (DCR_WRITE/DCR_READ/CACHE_FLUSH) @@ -367,7 +371,8 @@ bool CommandProcessor::exec_inline_cmd_(const Cmd& c) { uint32_t(c.arg0 & 0xFFF), uint32_t(c.arg1 & 0xFFFFFFFF)); return false; case OP_CACHE_FLUSH: - this->flush_cores_(uint32_t(c.arg0 & 0xFFFFFFFF)); + this->flush_cores_(uint32_t(c.arg0 & 0xFFFFFFFF), + (c.flags & FLUSH_FLAG_L1_ONLY) != 0); return false; default: // NOP / FENCE / unknown step — no-op (draws don't use MEM_*/EVENT_*). @@ -497,7 +502,8 @@ void CommandProcessor::tick_engine() { eng_state_ = EngState::Retire; } else if (cur_cmd_.opcode == OP_CACHE_FLUSH) { // arg0 carries num_cores. - this->flush_cores_(uint32_t(cur_cmd_.arg0 & 0xFFFFFFFF)); + this->flush_cores_(uint32_t(cur_cmd_.arg0 & 0xFFFFFFFF), + (cur_cmd_.flags & FLUSH_FLAG_L1_ONLY) != 0); eng_state_ = EngState::Retire; } else if (cur_cmd_.opcode == OP_EVENT_SIG) { // CMD_EVENT_SIGNAL: write arg1 (8-byte value) to arg0 diff --git a/sim/common/cmd_processor.h b/sim/common/cmd_processor.h index 38e0f80f4c..956d786e03 100644 --- a/sim/common/cmd_processor.h +++ b/sim/common/cmd_processor.h @@ -98,6 +98,11 @@ class CommandProcessor { // before retiring the CMD_LAUNCH. std::function vortex_busy; + // Block until memory has acknowledged every write Vortex issued. A + // full CMD_CACHE_FLUSH retires only then. Unset for a model whose + // writes land as they are issued. + std::function vortex_drain; + // Backend decodes the MMU fault-report DCRs. Published in DEV_CAPS // so the runtime does not read them on a target that would leave // the DCR bus waiting for a response. @@ -164,6 +169,10 @@ class CommandProcessor { // mode is its own descriptor field (modern-GPU convention). static constexpr uint8_t MEM_FLAG_PHYSICAL = 0x04; + // CMD_CACHE_FLUSH header flag: flush only the levels above the last-level + // cache. + static constexpr uint8_t FLUSH_FLAG_L1_ONLY = 0x04; + // Decoded cmd record (matches cmd_t struct layout on-wire). struct Cmd { uint8_t opcode; @@ -242,8 +251,9 @@ class CommandProcessor { // CMD_LAUNCH_QMD: read the in-memory KMU descriptor at `qmd_addr` and // replay its {dcr_addr,value} pairs through the DCR-write hook. void apply_qmd_(uint64_t qmd_addr); - // CMD_CACHE_FLUSH across cores [0, num_cores). - void flush_cores_(uint32_t num_cores); + // CMD_CACHE_FLUSH across cores [0, num_cores); l1_only leaves the + // last-level cache unwalked. + void flush_cores_(uint32_t num_cores, bool l1_only); // Advance the launch FSM one step using cur_cmd_. void tick_launch(); // Advance the engine FSM one step. diff --git a/sim/rtlsim/processor.cpp b/sim/rtlsim/processor.cpp index 416c60208b..98e1eaa40d 100644 --- a/sim/rtlsim/processor.cpp +++ b/sim/rtlsim/processor.cpp @@ -246,6 +246,12 @@ class Processor::Impl { return 0; } + void drain() { + while (device_->busy) { + this->tick(); + } + } + private: void reset() { @@ -497,4 +503,8 @@ int Processor::dcr_write(uint32_t addr, uint32_t value) { int Processor::dcr_read(uint32_t addr, uint32_t tag, uint32_t* value) { return impl_->dcr_read(addr, tag, value); +} + +void Processor::drain() { + impl_->drain(); } \ No newline at end of file diff --git a/sim/rtlsim/processor.h b/sim/rtlsim/processor.h index 3293fe1823..c6ae7ae464 100644 --- a/sim/rtlsim/processor.h +++ b/sim/rtlsim/processor.h @@ -40,6 +40,10 @@ class Processor { int dcr_read(uint32_t addr, uint32_t tag, uint32_t* value); + // Clock the device until busy drops, i.e. until memory has taken every + // write it issued. + void drain(); + private: class Impl; diff --git a/sim/rtlsim/rtlsim_shim.sv b/sim/rtlsim/rtlsim_shim.sv index f5159027b9..bde1d6501f 100644 --- a/sim/rtlsim/rtlsim_shim.sv +++ b/sim/rtlsim/rtlsim_shim.sv @@ -73,6 +73,8 @@ module rtlsim_shim import VX_gpu_pkg::*; #( wire [VX_MEM_TAG_WIDTH-1:0] vx_mem_rsp_tag [VX_MEM_PORTS]; wire vx_mem_rsp_ready [VX_MEM_PORTS]; + wire vx_busy; + `SCOPE_IO_SWITCH (1); Vortex vortex ( @@ -103,7 +105,7 @@ module rtlsim_shim import VX_gpu_pkg::*; #( .dcr_rsp_data (dcr_rsp_data), .start (start), - .busy (busy) + .busy (vx_busy) ); wire mem_req_valid_a [VX_MEM_PORTS]; @@ -204,4 +206,28 @@ module rtlsim_shim import VX_gpu_pkg::*; #( .mem_rsp_ready_out (mem_rsp_ready) ); + // Memory applies a write once it takes it, so the device stays busy until + // every write Vortex issued has crossed the bank adapter. + wire [VX_MEM_PORTS-1:0] vx_wr_fire; + wire [MEM_NUM_BANKS-1:0] mem_wr_fire; + wire wr_empty; + for (genvar i = 0; i < VX_MEM_PORTS; ++i) begin : g_vx_wr_fire + assign vx_wr_fire[i] = mem_req_valid_a[i] && mem_req_ready_a[i] && mem_req_rw_a[i]; + end + for (genvar i = 0; i < MEM_NUM_BANKS; ++i) begin : g_mem_wr_fire + assign mem_wr_fire[i] = mem_req_valid[i] && mem_req_ready[i] && mem_req_rw[i]; + end + VX_mem_req_pending #( + .NUM_INPUTS (VX_MEM_PORTS), + .NUM_OUTPUTS (MEM_NUM_BANKS) + ) wr_pending ( + .clk (clk), + .reset (reset), + .in_fire (vx_wr_fire), + .out_fire (mem_wr_fire), + .empty (wr_empty) + ); + + assign busy = vx_busy || ~wr_empty; + endmodule diff --git a/sim/simx/cluster.cpp b/sim/simx/cluster.cpp index d8451f2618..0978067d9e 100644 --- a/sim/simx/cluster.cpp +++ b/sim/simx/cluster.cpp @@ -85,6 +85,7 @@ class Cluster::Impl { VX_CFG_L2_REPL_POLICY, // replacement policy (VX_CFG_L2_ENABLED != 0) && (VX_CFG_L3_ENABLED == 0), // is_llc (VX_CFG_L2_BANK_HASH != 0) && (VX_CFG_L2_NUM_BANKS > 1), // bank_hash + L2_SHARED_LLC, // shared_llc }); @@ -476,9 +477,9 @@ class Cluster::Impl { return 0; } - void dcache_flush_begin() { + void dcache_flush_begin(bool l1_only) { for (auto& socket : sockets_) { - socket->dcache_flush_begin(); + socket->dcache_flush_begin(l1_only); } } @@ -540,8 +541,8 @@ class Cluster::Impl { } #endif - void l2_flush_begin() { - l2cache_->flush_begin(); + void l2_flush_begin(bool l1_only) { + l2cache_->flush_begin(l1_only); } bool l2_flush_done() const { @@ -659,8 +660,8 @@ Core* Cluster::get_core(uint32_t idx) const { return impl_->get_core(idx); } -void Cluster::dcache_flush_begin() { - impl_->dcache_flush_begin(); +void Cluster::dcache_flush_begin(bool l1_only) { + impl_->dcache_flush_begin(l1_only); } bool Cluster::dcache_flush_done() const { @@ -695,8 +696,8 @@ void Cluster::rtcache_flush_begin() { impl_->rtcache_flush_begin(); } bool Cluster::rtcache_flush_done() const { return impl_->rtcache_flush_done(); } #endif -void Cluster::l2_flush_begin() { - impl_->l2_flush_begin(); +void Cluster::l2_flush_begin(bool l1_only) { + impl_->l2_flush_begin(l1_only); } bool Cluster::l2_flush_done() const { diff --git a/sim/simx/cluster.h b/sim/simx/cluster.h index 2ff60836b4..bc7fa210a4 100644 --- a/sim/simx/cluster.h +++ b/sim/simx/cluster.h @@ -117,7 +117,7 @@ class Cluster : public SimObject { // → L2 → L3) to avoid downstream evictions racing the next-level walk. // L1 fanout: dcache + icache + socket-local {tcache, rtcache} (forwarded // to the sockets) + cluster-local {rcache, ocache}. - void dcache_flush_begin(); + void dcache_flush_begin(bool l1_only); bool dcache_flush_done() const; void icache_flush_begin(); bool icache_flush_done() const; @@ -137,7 +137,7 @@ class Cluster : public SimObject { void rtcache_flush_begin(); bool rtcache_flush_done() const; #endif - void l2_flush_begin(); + void l2_flush_begin(bool l1_only); bool l2_flush_done() const; #ifdef VX_CFG_EXT_RASTER_ENABLE diff --git a/sim/simx/constants.h b/sim/simx/constants.h index 2ef860870a..9b3703bda0 100644 --- a/sim/simx/constants.h +++ b/sim/simx/constants.h @@ -39,6 +39,27 @@ namespace vortex { inline constexpr uint32_t XLENB = (VX_CFG_XLEN / 8); +// A last-level cache that alone serves every memory client is the point of +// coherence between kernels. The texture, ray-tracing and DXA ports join below +// the dcache and the raster and OM caches beside it, so a dcache with any of +// them is not one. +inline constexpr bool DCACHE_SHARED_LLC = (VX_CFG_DCACHE_ENABLED != 0) && (VX_CFG_L2_ENABLED == 0) && (VX_CFG_L3_ENABLED == 0) + && (VX_CFG_NUM_CORES == 1) && (VX_CFG_NUM_CLUSTERS == 1) + && !VX_CFG_EXT_TEX_ENABLED && !VX_CFG_EXT_RTU_ENABLED + && !VX_CFG_EXT_RASTER_ENABLED && !VX_CFG_EXT_OM_ENABLED + && !VX_CFG_EXT_DXA_ENABLED; +inline constexpr bool L2_SHARED_LLC = (VX_CFG_L2_ENABLED != 0) && (VX_CFG_L3_ENABLED == 0) && (VX_CFG_NUM_CLUSTERS == 1); +inline constexpr bool L3_SHARED_LLC = (VX_CFG_L3_ENABLED != 0); + +// A write-back LLC must be the only instance: replicated ones never make their +// dirty lines visible to each other. +static_assert(!((VX_CFG_L2_ENABLED == 0) && (VX_CFG_L3_ENABLED == 0) && VX_CFG_DCACHE_WRITEBACK) + || ((VX_CFG_NUM_CORES == 1) && (VX_CFG_NUM_CLUSTERS == 1)), + "a write-back dcache that is the last-level cache must be the only dcache"); +static_assert(!((VX_CFG_L2_ENABLED != 0) && (VX_CFG_L3_ENABLED == 0) && VX_CFG_L2_WRITEBACK) + || (VX_CFG_NUM_CLUSTERS == 1), + "a write-back L2 that is the last-level cache must be the only L2"); + inline constexpr uint32_t MAX_NUM_CORES = 1024; inline constexpr uint32_t MAX_NUM_WARPS = 64; inline constexpr uint32_t MAX_NUM_REGS = 32; diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index 0bcd46007d..1a647e06ee 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -696,8 +696,9 @@ class CacheBank : public SimObject { // a writeback request for every dirty line via mem_req_out (write-back only; // write-through caches have nothing to evict). flush_done() reports when the // walk finishes AND the cache is otherwise idle. - void flush_begin() { - if (!config_.write_back) { + void flush_begin(bool keep_lines) { + keep_lines_ = keep_lines; + if (!config_.write_back || keep_lines) { flushing_ = false; flush_set_idx_ = 0; flush_way_idx_ = 0; @@ -726,9 +727,12 @@ class CacheBank : public SimObject { pending_fill_reqs_ = 0; rand_ctr_ = 0; adm_seq_ctr_ = 0; - for (auto &set : sets_) { - set.reset(); + if (!keep_lines_) { + for (auto &set : sets_) { + set.reset(); + } } + keep_lines_ = false; mshr_.reset(); fwd_active_ = false; fwd_set_ = 0; @@ -1789,6 +1793,7 @@ class CacheBank : public SimObject { // Flush walk state. bool flushing_; + bool keep_lines_ = false; // the next launch reset keeps the lines uint32_t flush_set_idx_; uint32_t flush_way_idx_; uint32_t flush_sector_idx_; @@ -1887,7 +1892,8 @@ class Cache::Impl { if (config_.bypass) return; // calculate cache initialization cycles - init_cycles_ = params_.sets_per_bank; + init_cycles_ = keep_lines_ ? 0 : params_.sets_per_bank; + keep_lines_ = false; } void tick() { @@ -1954,10 +1960,13 @@ class Cache::Impl { return perf_stats; } - void flush_begin() { + void flush_begin(bool l1_only) { if (config_.bypass) return; + // The shared last-level cache stays the point of coherence across an + // L1-only flush and keeps its lines into the next launch. + keep_lines_ = l1_only && config_.shared_llc; for (auto &bank : banks_) { - bank->flush_begin(); + bank->flush_begin(keep_lines_); } } @@ -2004,6 +2013,7 @@ class Cache::Impl { std::vector nc_mem_arbs_; MemCrossBar::Ptr bank_core_xbar_; uint32_t init_cycles_; + bool keep_lines_ = false; }; /////////////////////////////////////////////////////////////////////////////// @@ -2027,8 +2037,8 @@ Cache::PerfStats Cache::perf_stats() const { return impl_->perf_stats(); } -void Cache::flush_begin() { - impl_->flush_begin(); +void Cache::flush_begin(bool l1_only) { + impl_->flush_begin(l1_only); } bool Cache::flush_done() const { diff --git a/sim/simx/mem/cache.h b/sim/simx/mem/cache.h index 8729a52e88..4e7e8c5654 100644 --- a/sim/simx/mem/cache.h +++ b/sim/simx/mem/cache.h @@ -44,6 +44,7 @@ class Cache : public SimObject { uint8_t repl_policy; // replacement policy (ReplPolicy) bool is_llc; // is last-level cache bool bank_hash = false; // hashed bank select + bool shared_llc = false; // the LLC serves every memory client alone }; struct PerfStats { @@ -81,8 +82,10 @@ class Cache : public SimObject { // Flush API: arm a write-back walk that emits dirty lines via mem_req_out. // flush_done() returns true once the walk completes; the caller is expected - // to tick the simulator until then. Write-through caches no-op. - void flush_begin(); + // to tick the simulator until then. Write-through caches no-op. On an + // L1-only flush the write-back last-level cache skips the walk and keeps + // its lines across the next launch reset. + void flush_begin(bool l1_only = false); bool flush_done() const; protected: diff --git a/sim/simx/mem/cache_cluster.cpp b/sim/simx/mem/cache_cluster.cpp index ff52d56a5f..ee88f524d9 100644 --- a/sim/simx/mem/cache_cluster.cpp +++ b/sim/simx/mem/cache_cluster.cpp @@ -82,9 +82,9 @@ Cache::PerfStats CacheCluster::perf_stats() const { return perf; } -void CacheCluster::flush_begin() { +void CacheCluster::flush_begin(bool l1_only) { for (auto cache : caches_) { - cache->flush_begin(); + cache->flush_begin(l1_only); } } diff --git a/sim/simx/mem/cache_cluster.h b/sim/simx/mem/cache_cluster.h index edbe5958c4..174c355fcd 100644 --- a/sim/simx/mem/cache_cluster.h +++ b/sim/simx/mem/cache_cluster.h @@ -33,7 +33,7 @@ class CacheCluster : public SimObject { Cache::PerfStats perf_stats() const; - void flush_begin(); + void flush_begin(bool l1_only = false); bool flush_done() const; private: diff --git a/sim/simx/processor.cpp b/sim/simx/processor.cpp index 52ad76ca1a..bdcf97023d 100644 --- a/sim/simx/processor.cpp +++ b/sim/simx/processor.cpp @@ -104,6 +104,8 @@ ProcessorImpl::ProcessorImpl() VX_CFG_L3_LATENCY, // pipeline latency VX_CFG_L3_REPL_POLICY, // replacement policy VX_CFG_L3_ENABLED != 0, // is_llc when L3 is the LLC + false, // bank_hash + L3_SHARED_LLC, // shared_llc } ); @@ -204,7 +206,7 @@ void ProcessorImpl::attach_ram(RAM* ram) { #endif } -void ProcessorImpl::flush_caches() { +void ProcessorImpl::flush_caches(bool l1_only) { // Cache hierarchy is drained inside-out: issue all L1 flush_begin() calls // up-front so icache, dcache, and graphics caches flush in parallel, then // tick until all surfaces report flush_done(). @@ -212,7 +214,7 @@ void ProcessorImpl::flush_caches() { // L1 surfaces: dcache + icache + graphics caches. // Write-through surfaces early-exit in Cache::flush_begin(). for (auto& cluster : clusters_) { - cluster->dcache_flush_begin(); + cluster->dcache_flush_begin(l1_only); cluster->icache_flush_begin(); #ifdef VX_CFG_EXT_TEX_ENABLE cluster->tcache_flush_begin(); @@ -252,7 +254,7 @@ void ProcessorImpl::flush_caches() { // L2 caches. for (auto& cluster : clusters_) { - cluster->l2_flush_begin(); + cluster->l2_flush_begin(l1_only); } while (true) { bool all_done = true; @@ -265,7 +267,7 @@ void ProcessorImpl::flush_caches() { } // L3 cache (single instance at processor level). - l3cache_->flush_begin(); + l3cache_->flush_begin(l1_only); while (!l3cache_->flush_done() || !SimPlatform::instance().idle()) { SimPlatform::instance().tick(); } @@ -361,6 +363,11 @@ int ProcessorImpl::dcr_write(uint32_t addr, uint32_t value) { kmu_->dcr_write(addr, value); return 0; } + if (addr == VX_DCR_BASE_CACHE_FLUSH) { + // The flush itself runs on the per-core reads. + flush_l1_only_ = (value & VX_CACHE_FLUSH_L1_ONLY) != 0; + return 0; + } #ifdef VX_CFG_VM_ENABLE // Device SATP for the shared walker complex, assembled from two // 32-bit halves and fanned out to every cluster on the high write. @@ -393,10 +400,10 @@ int ProcessorImpl::dcr_write(uint32_t addr, uint32_t value) { int ProcessorImpl::dcr_read(uint32_t addr, uint32_t tag, uint32_t* value) { if (addr == VX_DCR_BASE_CACHE_FLUSH) { - // Drain dirty cache lines to DRAM before the host reads back results. - // After flush_caches() returns every dirty line has reached memsim_'s - // backing RAM. - this->flush_caches(); + // After flush_caches() returns every dirty line of the flushed levels has + // reached memsim_'s backing RAM; an L1-only flush leaves the shared + // last-level cache in place. + this->flush_caches(flush_l1_only_); *value = 0; return 0; } diff --git a/sim/simx/processor_impl.h b/sim/simx/processor_impl.h index c9c7c7692d..3e2f697b87 100644 --- a/sim/simx/processor_impl.h +++ b/sim/simx/processor_impl.h @@ -72,7 +72,7 @@ class ProcessorImpl { // Drain dirty data from caches (write-back path) all the way to DRAM. // Walks L1 dcaches → L2 → L3, ticking the simulator between phases so // each level's evictions reach the next before that level itself flushes. - void flush_caches(); + void flush_caches(bool l1_only); private: @@ -103,6 +103,7 @@ class ProcessorImpl { // kmu_->start()). reset() clears it so a back-to-back kernel launch // via cycle() re-dispatches the KMU. bool is_cycle_initialized_; + bool flush_l1_only_ = false; // kind of the flush the last start write opened }; } diff --git a/sim/simx/socket.cpp b/sim/simx/socket.cpp index 38ea84f880..640f263776 100644 --- a/sim/simx/socket.cpp +++ b/sim/simx/socket.cpp @@ -92,6 +92,7 @@ class Socket::Impl { VX_CFG_DCACHE_REPL_POLICY, // replacement policy (VX_CFG_DCACHE_ENABLED != 0) && (VX_CFG_L2_ENABLED == 0) && (VX_CFG_L3_ENABLED == 0), // is_llc (VX_CFG_DCACHE_BANK_HASH != 0) && (VX_CFG_DCACHE_NUM_BANKS > 1), // bank_hash + DCACHE_SHARED_LLC, // shared_llc }); #ifdef VX_CFG_EXT_TEX_ENABLE @@ -417,7 +418,7 @@ class Socket::Impl { return cores_.at(idx); } - void dcache_flush_begin() { dcaches_->flush_begin(); } + void dcache_flush_begin(bool l1_only) { dcaches_->flush_begin(l1_only); } bool dcache_flush_done() const { return dcaches_->flush_done(); } void icache_flush_begin() { icaches_->flush_begin(); } bool icache_flush_done() const { return icaches_->flush_done(); } @@ -505,8 +506,8 @@ Core::Ptr& Socket::core(uint32_t idx) { return impl_->core(idx); } -void Socket::dcache_flush_begin() { - impl_->dcache_flush_begin(); +void Socket::dcache_flush_begin(bool l1_only) { + impl_->dcache_flush_begin(l1_only); } bool Socket::dcache_flush_done() const { diff --git a/sim/simx/socket.h b/sim/simx/socket.h index f0d91a0605..15127a24e9 100644 --- a/sim/simx/socket.h +++ b/sim/simx/socket.h @@ -77,7 +77,7 @@ class Socket : public SimObject { // Forwarded cache flush (write-back eviction walk). The walk is a no-op // on write-through caches (`Cache::flush_begin` early-exits); forwarding // ensures a future write-back config exercises the same code path. - void dcache_flush_begin(); + void dcache_flush_begin(bool l1_only); bool dcache_flush_done() const; void icache_flush_begin(); bool icache_flush_done() const; diff --git a/sw/runtime/common/device.cpp b/sw/runtime/common/device.cpp index 0eee737e92..c8d25999af 100644 --- a/sw/runtime/common/device.cpp +++ b/sw/runtime/common/device.cpp @@ -84,6 +84,9 @@ constexpr uint32_t CP_STATUS_BUSY = 0x1; // is physical — the MMU-aware CP DMA skips translation. constexpr uint8_t CP_MEM_FLAG_PHYSICAL = 0x04; +// CMD_CACHE_FLUSH header flag: flush only the levels above the last-level cache. +constexpr uint8_t CP_FLUSH_FLAG_L1_ONLY = 0x04; + constexpr uint32_t CP_RING_SIZE_LOG2 = 16; // 64 KiB constexpr uint32_t CP_RING_SIZE = 1u << CP_RING_SIZE_LOG2; constexpr uint8_t CP_OPCODE_MEM_WRITE = 0x01; @@ -363,7 +366,7 @@ vx_result_t Device::cp_init() { } // Resolve the core count now, while nothing holds cp_mu_. Deferring it to - // the first cp_submit_cache_flush would reach query_caps from inside an + // the first cache flush would reach query_caps from inside an // open batch, which already owns cp_mu_ -- and it is not recursive. { auto rc = this->query_caps(VX_CAPS_NUM_CORES, &cp_num_cores_); @@ -513,7 +516,73 @@ const char* cp_opcode_name(uint8_t op) { } } // namespace +namespace { +// CMD_CACHE_FLUSH on-wire layout (cmd_size=12): +// bytes 0..3 header { opcode=0x0A, flags=CP_FLUSH_FLAG_L1_ONLY?, reserved=0 } +// bytes 4..11 arg0 number of cores to flush +// The CP sweeps a per-core flush DCR-read across [0, num_cores) and +// retires the command only when the last core's flush completes. +void build_cache_flush(uint8_t cl[CP_CL_BYTES], uint64_t num_cores, bool l1_only) { + std::memset(cl, 0, CP_CL_BYTES); + cl[0] = CP_OPCODE_CACHE_FLUSH; + cl[1] = l1_only ? CP_FLUSH_FLAG_L1_ONLY : 0; + std::memcpy(cl + 4, &num_cores, sizeof(num_cores)); +} + +// True when a CMD_MEM_* transfer reaches device memory outside the IO +// aperture. IO accesses bypass the caches, so they need no flush. +bool cp_dma_touches_device_mem(const uint8_t* cl) { + uint64_t dst = 0, src = 0, size = 0; + std::memcpy(&dst, cl + 4, sizeof(dst)); + std::memcpy(&src, cl + 12, sizeof(src)); + std::memcpy(&size, cl + 20, sizeof(size)); + auto outside_io = [size](uint64_t addr) { + return addr < VX_MEM_IO_BASE_ADDR || addr + size > VX_MEM_IO_END_ADDR; + }; + switch (cl[0]) { + case CP_OPCODE_MEM_WRITE: return outside_io(dst); + case CP_OPCODE_MEM_READ: return outside_io(src); + case CP_OPCODE_MEM_COPY: return outside_io(dst) || outside_io(src); + default: return false; + } +} +} // namespace + vx_result_t Device::cp_ring_append_(const void* cl) { + // Caller holds cp_mu_. A kernel's results are flushed lazily, by the + // next command that needs them: the next launch gets an L1-only flush + // (the last-level cache is the point of coherence between kernels), and + // the next transfer to or from device memory a full one, since the CP + // DMA reaches device memory beneath the last-level cache. Appending the + // flush under the same lock keeps any other command from slipping between. + const uint8_t* b = static_cast(cl); + const bool dma = cp_dma_touches_device_mem(b); + const bool launch = (b[0] == CP_OPCODE_LAUNCH) + || (b[0] == CP_OPCODE_LAUNCH_QMD) + || (b[0] == CP_OPCODE_DRAW); + if ((dma && (l1_dirty_ || llc_dirty_)) || (launch && l1_dirty_)) { + uint8_t flush[CP_CL_BYTES]; + build_cache_flush(flush, cp_num_cores_, /*l1_only=*/!dma); + auto r = cp_ring_write_(flush); + if (r != VX_SUCCESS) return r; + l1_dirty_ = false; + llc_dirty_ = llc_dirty_ && !dma; + } + assert(!(dma && (l1_dirty_ || llc_dirty_))); + auto r = cp_ring_write_(cl); + if (r != VX_SUCCESS) return r; + if (launch) { + // a draw descriptor ends on its own L1-only flush step + l1_dirty_ = (b[0] != CP_OPCODE_DRAW); + llc_dirty_ = true; + } else if (b[0] == CP_OPCODE_CACHE_FLUSH) { + l1_dirty_ = false; + llc_dirty_ = llc_dirty_ && (b[1] & CP_FLUSH_FLAG_L1_ONLY); + } + return VX_SUCCESS; +} + +vx_result_t Device::cp_ring_write_(const void* cl) { // Caller holds cp_mu_. Write one CL into the ring at the current tail — // a plain memcpy through the ring's CP-visible host pointer — then bump // tail + reserve the seqnum slot. No doorbell, no poll. @@ -576,9 +645,8 @@ vx_result_t Device::cp_batch_end() { if (r != VX_SUCCESS) { return r; } - // The batch's trailing CMD_CACHE_FLUSH(es) have retired, so every kernel's - // writes are coherent: drain the console rings once for the whole batch - // (deferred from each in-batch cp_submit_launch). + // Drain the console rings once for the whole batch (deferred from each + // in-batch cp_submit_launch). return drain_cout(); } @@ -811,15 +879,13 @@ vx_result_t Device::cp_submit_launch() { if (r != VX_SUCCESS) { return r; } - // Cache coherence: post an explicit cache flush right after the launch - // (ACQUIRE_MEM model) so the host observes coherent kernel results. - r = cp_submit_cache_flush(); - if (r != VX_SUCCESS) return r; - // In a batch the flush has only been appended, not retired — defer the - // COUT drain to cp_batch_end (one drain for the whole sequence). + // The kernel-boundary cache flush (ACQUIRE_MEM model) is posted by the + // next command that needs it (see cp_ring_append_). In a batch the + // launch has only been appended, not retired — defer the COUT drain to + // cp_batch_end (one drain for the whole sequence). if (tls_batch_owner == this) return VX_SUCCESS; - // Final COUT drain: the flush has made the kernel's writes coherent, so - // the tail-end console output left in the rings is now safe to read. + // Final COUT drain: the console rings live in the uncached IO aperture, + // and a launch completes only once its stores have left the device. return drain_cout(); } @@ -860,7 +926,7 @@ vx_result_t Device::cp_submit_launch_qmd(uint64_t qmd_addr) { // The CP reads the in-memory KMU descriptor (a {count,(addr,value)...} // list the caller staged) and replays it before pulsing start — one ring // command in place of the ~18 CMD_DCR_WRITEs a plain launch costs. Same - // trailing CMD_CACHE_FLUSH / COUT-drain discipline as cp_submit_launch. + // COUT-drain discipline as cp_submit_launch. uint8_t cl[CP_CL_BYTES] = {0}; auto r = ensure_mmu_satp(); if (r != VX_SUCCESS) { @@ -874,8 +940,6 @@ vx_result_t Device::cp_submit_launch_qmd(uint64_t qmd_addr) { if (r != VX_SUCCESS) { return r; } - r = cp_submit_cache_flush(); - if (r != VX_SUCCESS) return r; if (tls_batch_owner == this) return VX_SUCCESS; return drain_cout(); } @@ -905,19 +969,6 @@ vx_result_t Device::cp_submit_draw(uint64_t desc_addr) { return drain_cout(); } -vx_result_t Device::cp_submit_cache_flush() { - // CMD_CACHE_FLUSH on-wire layout (cmd_size=12): - // bytes 0..3 header { opcode=0x0A, flags=0, reserved=0 } - // bytes 4..11 arg0 number of cores to flush - // The CP sweeps a per-core flush DCR-read across [0, num_cores) and - // retires the command only when the last core's flush completes. - // No-op on write-through cache configs (the Vortex default). - uint8_t cl[CP_CL_BYTES] = {0}; - cl[0] = CP_OPCODE_CACHE_FLUSH; - std::memcpy(cl + 4, &cp_num_cores_, sizeof(cp_num_cores_)); - return cp_submit_cl_(cl); -} - vx_result_t Device::cp_submit_dcr_read(uint32_t addr, uint32_t tag, uint32_t* out_value) { if (!out_value) return VX_ERR_INVALID_VALUE; diff --git a/sw/runtime/common/queue.cpp b/sw/runtime/common/queue.cpp index df63f7a9dc..68543c0f60 100644 --- a/sw/runtime/common/queue.cpp +++ b/sw/runtime/common/queue.cpp @@ -505,6 +505,8 @@ namespace { constexpr uint8_t CP_OP_DCR_WRITE = 0x04; constexpr uint8_t CP_OP_CACHE_FLUSH = 0x0A; constexpr uint8_t CP_OP_LAUNCH_QMD = 0x0B; +// CMD_CACHE_FLUSH header flag: flush only the levels above the last-level cache. +constexpr uint8_t CP_FLUSH_FLAG_L1_ONLY = 0x04; // Fixed per-step stride in a draw descriptor (28-byte cmd-record prefix). // Must equal CommandProcessor::DRAW_STEP_BYTES. constexpr int CP_DRAW_STEP_BYTES = 28; @@ -682,7 +684,7 @@ vx_result_t cmd_stage_qmds(Device* device_, const std::vector& recs, // Submit one staged launch: CMD_LAUNCH_QMD when the CP decodes it, else // replay the descriptor pairs as CMD_DCR_WRITEs followed by a plain -// CMD_LAUNCH (same trailing cache-flush discipline in cp_submit_launch). +// CMD_LAUNCH. vx_result_t cmd_submit_launch(Device* device_, const CmdStaged& st) { if (device_->cp_supports_qmd()) { return device_->cp_submit_launch_qmd(st.qmd_addr); @@ -820,9 +822,10 @@ vx_result_t Queue::enqueue_draw(const vx_command_t* commands, if (r == VX_SUCCESS && use_op_draw) { std::vector desc(4, 0); // header placeholder uint32_t num_steps = 0; - auto put_step = [&](uint8_t opcode, uint64_t a0, uint64_t a1) { + auto put_step = [&](uint8_t opcode, uint8_t flags, uint64_t a0, uint64_t a1) { uint8_t step[CP_DRAW_STEP_BYTES] = {0}; step[0] = opcode; + step[1] = flags; std::memcpy(step + 4, &a0, sizeof(a0)); std::memcpy(step + 12, &a1, sizeof(a1)); desc.insert(desc.end(), step, step + CP_DRAW_STEP_BYTES); @@ -831,10 +834,10 @@ vx_result_t Queue::enqueue_draw(const vx_command_t* commands, for (size_t i = 0; i < recs.size(); ++i) { const CmdRec& rec = recs[i]; if (rec.is_launch) { - put_step(CP_OP_LAUNCH_QMD, staged[i].qmd_addr, 0); - put_step(CP_OP_CACHE_FLUSH, num_cores, 0); + put_step(CP_OP_LAUNCH_QMD, 0, staged[i].qmd_addr, 0); + put_step(CP_OP_CACHE_FLUSH, CP_FLUSH_FLAG_L1_ONLY, num_cores, 0); } else { - put_step(CP_OP_DCR_WRITE, rec.dcr_addr, rec.dcr_value); + put_step(CP_OP_DCR_WRITE, 0, rec.dcr_addr, rec.dcr_value); } } std::memcpy(desc.data(), &num_steps, sizeof(num_steps)); diff --git a/sw/runtime/common/vortex2_internal.h b/sw/runtime/common/vortex2_internal.h index ef803eefc7..23c688ad63 100644 --- a/sw/runtime/common/vortex2_internal.h +++ b/sw/runtime/common/vortex2_internal.h @@ -233,15 +233,15 @@ class Device : public RefCounted { vx_result_t cp_submit_dcr_write(uint32_t addr, uint32_t value); // Post one CMD_LAUNCH to the ring, commit Q_TAIL, and wait for - // Q_SEQNUM. Synchronous. Followed by an implicit CMD_CACHE_FLUSH so - // the host observes coherent results (see cp_submit_cache_flush). + // Q_SEQNUM. Synchronous. The kernel's results are flushed by the next + // command that needs them (see cp_ring_append_). vx_result_t cp_submit_launch(); // Post one CMD_LAUNCH_QMD: the CP reads the KMU descriptor (a // {count,(dcr_addr,value)...} list the caller staged to device memory at // `qmd_addr`) and replays it, then pulses start — one ring command in - // place of a launch's ~18 CMD_DCR_WRITEs (NVIDIA QMD model). Followed by an - // implicit CMD_CACHE_FLUSH, like cp_submit_launch. Batch-aware. + // place of a launch's ~18 CMD_DCR_WRITEs (NVIDIA QMD model). Flushed like + // cp_submit_launch. Batch-aware. vx_result_t cp_submit_launch_qmd(uint64_t qmd_addr); // Post one CMD_DRAW (OP_DRAW): arg0 = device address of a resident draw @@ -255,12 +255,6 @@ class Device : public RefCounted { bool cp_supports_draw() const { return cp_supports_draw_; } bool cp_supports_qmd() const { return cp_supports_qmd_; } - // Post one CMD_CACHE_FLUSH to the ring (AMD ACQUIRE_MEM model): the CP - // sweeps a per-core cache flush across all cores and retires the - // command only when the last core's flush completes. A no-op on - // write-through cache configs. Posted after every CMD_LAUNCH. - vx_result_t cp_submit_cache_flush(); - // Program the device MMU's SATP once, ahead of the first launch that // could translate. Called from every launch path. vx_result_t ensure_mmu_satp(); @@ -273,7 +267,7 @@ class Device : public RefCounted { // ----- Batched CP submission (one draw = one ring batch) ----- // cp_batch_begin holds the ring lock and switches cp_submit_* into // append-only mode: each subsequent cp_submit_dcr_write / cp_submit_launch - // / cp_submit_cache_flush writes its CL into the ring and reserves a + // writes its CL into the ring and reserves a // seqnum but does NOT ring the doorbell or poll. cp_batch_end commits // Q_TAIL once, polls Q_SEQNUM once for the last appended command, drains // COUT once, and releases the lock. The CP then retires the whole @@ -391,7 +385,12 @@ class Device : public RefCounted { // Append one CL at the current tail and reserve its seqnum (bump // cp_tail_ + cp_expected_seqnum_). Caller must hold cp_mu_ (directly in // the non-batch path, or via cp_batch_begin). No doorbell, no poll. + // Inserts the lazy kernel-boundary CMD_CACHE_FLUSH (AMD ACQUIRE_MEM + // model) under the same lock: L1-only ahead of a launch while + // l1_dirty_, full ahead of a CMD_MEM_* touching device memory while + // either flag is set. vx_result_t cp_ring_append_(const void* cl); + vx_result_t cp_ring_write_(const void* cl); // Build + submit a CMD_MEM_* descriptor (opcode, dst, src, size). // `physical` sets the CMD_MEM header flag so the CP DMA skips VM @@ -457,6 +456,13 @@ class Device : public RefCounted { uint64_t cp_tail_ = 0; uint64_t cp_expected_seqnum_ = 0; uint64_t cp_num_cores_ = 0; // cached VX_CAPS_NUM_CORES, used for CMD_CACHE_FLUSH + // A kernel ran since the last flush, so the L1s may hold its writes or + // stale lines. + bool l1_dirty_ = false; + // A kernel ran since the last full cache flush, so the last-level cache + // may hold lines the CP DMA (which reaches device memory beneath it) + // would miss or later overwrite. Both flags are guarded by cp_mu_. + bool llc_dirty_ = false; std::mutex cp_mu_; // serialize ring writes // Seqnum the last command appended to the open batch will reach; // cp_batch_end polls for it once. Touched only by the batch owner, which diff --git a/sw/runtime/firesim/vortex.cpp b/sw/runtime/firesim/vortex.cpp index 973c79bc49..6c31af7458 100644 --- a/sw/runtime/firesim/vortex.cpp +++ b/sw/runtime/firesim/vortex.cpp @@ -158,6 +158,10 @@ class vx_device { return false; return busy; }; + h.vortex_drain = [this]() { + bool busy = true; + while (busy && sim_.is_busy(&busy) == 0) {} + }; #ifdef VX_CFG_VM_ENABLE // The device MMU control surface answers the fault-report DCRs. h.mmu_fault_report = true; diff --git a/sw/runtime/rtlsim/vortex.cpp b/sw/runtime/rtlsim/vortex.cpp index e5aa68d64f..8d7f24e96f 100644 --- a/sw/runtime/rtlsim/vortex.cpp +++ b/sw/runtime/rtlsim/vortex.cpp @@ -175,6 +175,10 @@ class vx_device { if (!future_.valid()) return false; return future_.wait_for(std::chrono::seconds(0)) != std::future_status::ready; }; + h.vortex_drain = [this]() { + if (future_.valid()) future_.wait(); + processor_.drain(); + }; #ifdef VX_CFG_VM_ENABLE // The device MMU control surface answers the fault-report DCRs. h.mmu_fault_report = true; diff --git a/tests/regression/flush_coherence/common.h b/tests/regression/flush_coherence/common.h index bbdabc90a1..6b04d30d2d 100644 --- a/tests/regression/flush_coherence/common.h +++ b/tests/regression/flush_coherence/common.h @@ -8,6 +8,7 @@ typedef struct { uint32_t round; uint32_t rotate; uint32_t burst; + uint32_t dxa_slot; // 1 + DXA descriptor slot of src, 0 to load with the LSU uint64_t src_addr; uint64_t dst_addr; uint64_t acc_addr; diff --git a/tests/regression/flush_coherence/kernel.cpp b/tests/regression/flush_coherence/kernel.cpp index de656daa29..1c3e14ed9f 100644 --- a/tests/regression/flush_coherence/kernel.cpp +++ b/tests/regression/flush_coherence/kernel.cpp @@ -1,4 +1,8 @@ #include +#ifdef VX_CFG_EXT_DXA_ENABLE +#include +#include +#endif #include "common.h" __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { @@ -15,7 +19,25 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto src = reinterpret_cast(arg->src_addr); auto dst = reinterpret_cast(arg->dst_addr); auto acc = reinterpret_cast(arg->acc_addr); - uint32_t v = src[i]; + uint32_t v; +#ifdef VX_CFG_EXT_DXA_ENABLE + if (arg->dxa_slot != 0) { + // The tile copy reads memory beside the dcache, not through it, so it + // sees the previous launch's stores only once a flush has written them + // back. The host sets no rotation, so this block's tile holds element i. + auto shmem = reinterpret_cast(__local_mem()); + vortex::barrier bar(0); + if (get_sub_group_id() == 0) { + bar.expect_tx(1); + vx_dxa_issue_1d_wg(arg->dxa_slot - 1, bar.id(), shmem, blockIdx.x * blockDim.x); + } + bar.arrive_and_wait(); + v = shmem[threadIdx.x]; + } else +#endif + { + v = src[i]; + } acc[i] += v ^ arg->round; // A store burst saturates the shared levels, so the final stores are // still queued in front of them when the flushes start. diff --git a/tests/regression/flush_coherence/main.cpp b/tests/regression/flush_coherence/main.cpp index 573dbe66ad..9f45ff84ba 100644 --- a/tests/regression/flush_coherence/main.cpp +++ b/tests/regression/flush_coherence/main.cpp @@ -14,8 +14,18 @@ // With -b, each thread ends on a burst of stores to a tail buffer, so the // last stores may still be queued in front of the shared levels when the // flushes start, and every round is checked. +// With -x, each launch loads src with a DXA tile copy, which reaches memory +// beside the dcache, instead of with the LSU. +// With -c, the rounds form one command batch with no host transfer between +// kernels: each round reads the buffer the previous round wrote (ping-pong), +// and the host only reads back after the last round. Kernel-boundary flushes +// alone must then make each round's writes visible to the next, including to +// a core whose cache still holds the lines from two rounds earlier. #include +#ifdef VX_CFG_EXT_DXA_ENABLE +#include +#endif #include "common.h" #include @@ -38,17 +48,21 @@ const char* kernel_file = "kernel.vxbin"; uint32_t count = 2048; uint32_t rounds = 16; uint32_t burst = 0; +bool chain = false; +bool use_dxa = false; void parse_args(int argc, char** argv) { int c; - while ((c = getopt(argc, argv, "n:r:b:k:h")) != -1) { + while ((c = getopt(argc, argv, "n:r:b:k:cxh")) != -1) { switch (c) { case 'n': count = (uint32_t)std::atoi(optarg); break; case 'r': rounds = (uint32_t)std::atoi(optarg); break; case 'b': burst = (uint32_t)std::atoi(optarg); break; case 'k': kernel_file = optarg; break; + case 'c': chain = true; break; + case 'x': use_dxa = true; break; default: - std::printf("Usage: [-k kernel] [-n words] [-r rounds] [-b burst] [-h]\n"); + std::printf("Usage: [-k kernel] [-n words] [-r rounds] [-b burst] [-c] [-x] [-h]\n"); std::exit(c == 'h' ? 0 : -1); } } @@ -60,6 +74,15 @@ uint32_t src_value(uint32_t i, uint32_t round) { return x * 0x2C1B3C6Du; } +#ifdef VX_CFG_EXT_DXA_ENABLE +void program_dxa(vx_device_h dev, uint32_t slot, uint64_t addr, uint32_t tile) { + if (vortex::dxa::program_1d(dev, slot, addr, count, tile, sizeof(uint32_t)) != 0) { + std::fprintf(stderr, "FAIL: DXA descriptor %u\n", slot); + std::exit(1); + } +} +#endif + uint32_t rotation(uint32_t round) { return (round % 4 == 0) ? 0 : (round * 0x9E3779B1u) % count; } @@ -84,7 +107,7 @@ int main(int argc, char** argv) { parse_args(argc, argv); const uint64_t bytes = (uint64_t)count * sizeof(uint32_t); const uint64_t tail_bytes = bytes * burst; - std::printf("flush_coherence: n=%u rounds=%u burst=%u\n", count, rounds, burst); + std::printf("flush_coherence: n=%u rounds=%u burst=%u chain=%d\n", count, rounds, burst, chain); vx_device_h dev = nullptr; CHECK(vx_device_open(0, &dev)); @@ -119,6 +142,23 @@ int main(int argc, char** argv) { uint32_t grid[1], block[1]; CHECK(vx_device_max_occupancy_grid(dev, 1, &count, grid, block)); + // DXA descriptor slot 0 covers src, 1 dst and 2 the chain's middle buffer. + uint32_t lmem_size = 0; + if (use_dxa) { +#ifdef VX_CFG_EXT_DXA_ENABLE + if (count % block[0] != 0) { + std::printf("Error: -x needs a word count that is a multiple of the block size (%u)\n", block[0]); + return -1; + } + lmem_size = block[0] * sizeof(uint32_t); + program_dxa(dev, 0, arg.src_addr, block[0]); + program_dxa(dev, 1, arg.dst_addr, block[0]); +#else + std::printf("Error: -x needs a build with VX_CFG_EXT_DXA_ENABLE\n"); + return -1; +#endif + } + // Enqueued writes and launches read their host data asynchronously, so // each round gets its own source buffer and argument block. std::vector> h_src(rounds, std::vector(count)); @@ -129,7 +169,69 @@ int main(int argc, char** argv) { CHECK(vx_enqueue_write(q, acc_buf, 0, h_acc.data(), bytes, 0, nullptr, nullptr)); int errors = 0; - for (uint32_t r = 0; r < rounds; ++r) { + vx_buffer_h mid_buf = nullptr; + if (chain) { + // round r reads the buffer round r-1 wrote: src, then dst and mid in turn + CHECK(vx_buffer_create(dev, bytes, VX_MEM_READ_WRITE, &mid_buf)); + uint64_t pp_addr[2]; + CHECK(vx_buffer_address(dst_buf, &pp_addr[0])); + CHECK(vx_buffer_address(mid_buf, &pp_addr[1])); +#ifdef VX_CFG_EXT_DXA_ENABLE + if (use_dxa) { + program_dxa(dev, 2, pp_addr[1], block[0]); + } +#endif + auto& src = h_src[0]; + for (uint32_t i = 0; i < count; ++i) { + src[i] = src_value(i, 0); + ref_dst[i] = src[i]; + } + CHECK(vx_enqueue_write(q, src_buf, 0, src.data(), bytes, 0, nullptr, nullptr)); + + std::vector lis(rounds); + std::vector cmds(rounds); + for (uint32_t r = 0; r < rounds; ++r) { + for (uint32_t i = 0; i < count; ++i) { + ref_acc[i] += ref_dst[i] ^ r; + ref_dst[i] = ref_dst[i] * 3u + r; + } + args[r].round = r; + args[r].rotate = use_dxa ? 0 : rotation(r); + args[r].src_addr = (r == 0) ? arg.src_addr : pp_addr[(r - 1) % 2]; + args[r].dxa_slot = !use_dxa ? 0 : ((r == 0) ? 1 : 2 + (r - 1) % 2); + args[r].dst_addr = pp_addr[r % 2]; + vx_launch_info_t& li = lis[r]; + li = vx_launch_info_t{}; + li.struct_size = sizeof(li); + li.kernel = kern; + li.args_host = &args[r]; + li.args_size = sizeof(args[r]); + li.ndim = 1; + li.grid_dim[0] = grid[0]; + li.block_dim[0] = block[0]; + li.lmem_size = lmem_size; + cmds[r].type = VX_COMMAND_LAUNCH; + cmds[r].data.launch = &li; + } + for (uint32_t i = 0; i < count; ++i) { + for (uint32_t k = 0; k < burst; ++k) { + ref_tail[k * count + i] = tail_value(i, k, rounds - 1); + } + } + CHECK(vx_enqueue_commands(q, cmds.data(), rounds, 0, nullptr, nullptr)); + + vx_buffer_h last_buf = ((rounds - 1) % 2 == 0) ? dst_buf : mid_buf; + CHECK(vx_enqueue_read(q, h_dst.data(), last_buf, 0, bytes, 0, nullptr, nullptr)); + if (burst) { + CHECK(vx_enqueue_read(q, h_tail.data(), tail_buf, 0, tail_bytes, 0, nullptr, nullptr)); + } + CHECK(vx_queue_finish(q, VX_TIMEOUT_INFINITE)); + errors += check("dst", rounds - 1, h_dst, ref_dst); + if (burst) { + errors += check("tail", rounds - 1, h_tail, ref_tail); + } + } + for (uint32_t r = 0; !chain && r < rounds; ++r) { auto& src = h_src[r]; for (uint32_t i = 0; i < count; ++i) { src[i] = src_value(i, r); @@ -142,7 +244,8 @@ int main(int argc, char** argv) { CHECK(vx_enqueue_write(q, src_buf, 0, src.data(), bytes, 0, nullptr, nullptr)); args[r].round = r; - args[r].rotate = rotation(r); + args[r].rotate = use_dxa ? 0 : rotation(r); + args[r].dxa_slot = use_dxa ? 1 : 0; vx_launch_info_t li{}; li.struct_size = sizeof(li); li.kernel = kern; @@ -151,6 +254,7 @@ int main(int argc, char** argv) { li.ndim = 1; li.grid_dim[0] = grid[0]; li.block_dim[0] = block[0]; + li.lmem_size = lmem_size; CHECK(vx_enqueue_launch(q, &li, 0, nullptr, nullptr)); if (burst || r % 2 == 0 || r + 1 == rounds) { @@ -170,6 +274,9 @@ int main(int argc, char** argv) { CHECK(vx_queue_finish(q, VX_TIMEOUT_INFINITE)); errors += check("acc", rounds, h_acc, ref_acc); + if (mid_buf) { + vx_buffer_release(mid_buf); + } if (tail_buf) { vx_buffer_release(tail_buf); } @@ -179,6 +286,7 @@ int main(int argc, char** argv) { vx_kernel_release(kern); vx_module_release(mod); vx_queue_release(q); + vx_device_dump_perf(dev, stdout); vx_device_release(dev); if (errors) { From 4f173e313f78c88f2d6ea79cc07451cfe8983df5 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 18 Oct 2026 03:25:36 -0700 Subject: [PATCH 035/124] Let a write-back LLC allocate on a store miss without fetching the line A store miss in a write-back last-level cache used to fetch the whole line from memory only to overwrite part of it, so streaming writes cost a full read per line. With VX_CFG_{DCACHE,L2,L3}_LAZY_FETCH the cache instead installs the sector with its dirty-byte mask and a partial bit and issues no fill. A load or AMO that reaches a partial sector misses; its fill merges the memory data under the bytes the stores have not written. Evictions and flushes write back only the dirty bytes. Fill responses with no data use a loopback queue, and an MSHR chain that meets a partial sector parks, refetches and retries. This is the write-allocate-without-fetch policy GPU L2s use for partial writes. The dirty-byte mask is now one way-folded single-port distributed RAM that is read and written only at the commit stage, as a whole-line read-modify-write. It replaces the per-way dual-port per-byte RAMs, is cleared when a line is installed, and removes the old dirty-byte path from the critical cone. The MSHR exposes a registered dequeue_reads so the park decision does not wait on the payload read. The feature is off by default; enabling it turns on the dirty-byte mask. With it off the cache is logically identical to before and cycles are unchanged. SimX models the same mechanism behind the same knobs. 1-core vecadd with lazy fetch: DRAM reads -33%, memory requests -25%, cycles unchanged (latency-bound). Cache DUT at 300 MHz target (U55C, median of a placer-directive sweep): feature on 288.3 MHz vs 276.9 MHz for the previous dirty-byte implementation, with 1.4k fewer LUTs. Tests: lazy_fetch app, cache:lazy_fetch-1/2, parity-lazy-fetch, parity-vecadd-lazy-l2. Co-Authored-By: Claude Opus 5.5 --- VX_config.toml | 13 +- ci/testcases/cache.yaml | 21 ++ ci/testcases/core.yaml | 16 ++ hw/rtl/VX_cluster.sv | 1 + hw/rtl/VX_socket.sv | 1 + hw/rtl/Vortex.sv | 1 + hw/rtl/cache/VX_cache.sv | 4 + hw/rtl/cache/VX_cache_bank.sv | 343 +++++++++++++++++++++---- hw/rtl/cache/VX_cache_cluster.sv | 3 + hw/rtl/cache/VX_cache_data.sv | 144 ++++++----- hw/rtl/cache/VX_cache_mshr.sv | 56 +++- hw/rtl/cache/VX_cache_tags.sv | 69 ++++- hw/rtl/cache/VX_cache_wrap.sv | 3 + hw/unittest/cache/VX_cache_top.sv | 4 + hw/unittest/gfx/VX_gfx_top.sv | 1 + hw/unittest/tensor/VX_tensor_top.sv | 2 + hw/unittest/vm/VX_vm_top.sv | 2 + sim/simx/cluster.cpp | 1 + sim/simx/mem/cache.cpp | 266 +++++++++++++++++-- sim/simx/mem/cache.h | 1 + sim/simx/processor.cpp | 1 + sim/simx/socket.cpp | 1 + tests/regression/Makefile | 2 +- tests/regression/lazy_fetch/Makefile | 16 ++ tests/regression/lazy_fetch/common.h | 134 ++++++++++ tests/regression/lazy_fetch/kernel.cpp | 174 +++++++++++++ tests/regression/lazy_fetch/main.cpp | 217 ++++++++++++++++ 27 files changed, 1355 insertions(+), 142 deletions(-) create mode 100644 tests/regression/lazy_fetch/Makefile create mode 100644 tests/regression/lazy_fetch/common.h create mode 100644 tests/regression/lazy_fetch/kernel.cpp create mode 100644 tests/regression/lazy_fetch/main.cpp diff --git a/VX_config.toml b/VX_config.toml index b3b89c6833..9ccfef0cc5 100644 --- a/VX_config.toml +++ b/VX_config.toml @@ -216,7 +216,12 @@ VX_CFG_DCACHE_NUM_WAYS = 4 # stays moderate: NT 1/2/4/8/16/32/64 -> word 4/8/8/16/16/32/32, banks 1/1/2/2/4/4/8. VX_CFG_DCACHE_WORD_SIZE = "expr: min($VX_CFG_L1_LINE_SIZE, min($VX_CFG_MEM_BLOCK_SIZE, ($VX_CFG_XLEN / 8) * pow(2, (clog2($VX_CFG_NUM_LSU_LANES) + 1) / 2)))" VX_CFG_DCACHE_WRITEBACK = "expr: int($dcache_is_llc and $single_core)" -VX_CFG_DCACHE_DIRTYBYTES = 0 +# LAZY_FETCH (write-back LLC only): a store miss allocates its line without a +# fill and a later read fetches and merges under the dirty bytes. It needs the +# per-byte dirty masks, so it turns DIRTYBYTES on; DIRTYBYTES alone keeps +# fetch-on-write. The same pair exists for L2 and L3. +VX_CFG_DCACHE_DIRTYBYTES = "expr: $VX_CFG_DCACHE_LAZY_FETCH" +VX_CFG_DCACHE_LAZY_FETCH = 0 VX_CFG_DCACHE_REPL_POLICY = "expr: $__cache_repl_fifo" VX_CFG_DCACHE_MSHR_SIZE = 16 # dcache is unsectored: line = sector = mem transaction granule = L1_LINE (which @@ -246,7 +251,8 @@ VX_CFG_L1_MEM_PORTS = "expr: min($VX_CFG_DCACHE_NUM_BANKS, $VX_CFG_PLATFORM_MEMO VX_CFG_L2_SIZE = 1048576 VX_CFG_L2_NUM_WAYS = 8 VX_CFG_L2_WRITEBACK = "expr: int($l2_is_llc and $single_cluster)" -VX_CFG_L2_DIRTYBYTES = 0 +VX_CFG_L2_DIRTYBYTES = "expr: $VX_CFG_L2_LAZY_FETCH" +VX_CFG_L2_LAZY_FETCH = 0 VX_CFG_L2_REPL_POLICY = "expr: $__cache_repl_fifo" VX_CFG_L2_MSHR_SIZE = 16 VX_CFG_L2_LATENCY = "expr: 4 + max(0, clog2($VX_CFG_L2_SIZE) - clog2(1048576))" @@ -264,7 +270,8 @@ VX_CFG_L2_MEM_PORTS = "expr: min($VX_CFG_L2_NUM_BANKS, $VX_CFG_PLATFORM_MEMORY_N VX_CFG_L3_SIZE = 2097152 VX_CFG_L3_NUM_WAYS = 8 VX_CFG_L3_WRITEBACK = "expr: int($l3_is_llc)" -VX_CFG_L3_DIRTYBYTES = 0 +VX_CFG_L3_DIRTYBYTES = "expr: $VX_CFG_L3_LAZY_FETCH" +VX_CFG_L3_LAZY_FETCH = 0 VX_CFG_L3_REPL_POLICY = "expr: $__cache_repl_fifo" VX_CFG_L3_MSHR_SIZE = 16 VX_CFG_L3_LATENCY = "expr: 4 + max(0, clog2($VX_CFG_L3_SIZE) - clog2(2097152))" diff --git a/ci/testcases/cache.yaml b/ci/testcases/cache.yaml index d78b7f904d..ceb79a2788 100644 --- a/ci/testcases/cache.yaml +++ b/ci/testcases/cache.yaml @@ -316,3 +316,24 @@ tests: configs: -DVX_CFG_DCACHE_WRITEBACK=1 -DVX_CFG_DCACHE_DIRTYBYTES=1 shape: threads: 16 +# Lazy fetch-on-read on a write-back LLC: a store miss allocates its line +# without a fill, and a load or AMO of a partially written line fetches it and +# merges under the dirty bytes. The default 1-core dcache is such an LLC. +- id: lazy_fetch-1 + via: blackbox + drivers: + - simx + - rtlsim + app: lazy_fetch + configs: -DVX_CFG_EXT_A_ENABLE -DVX_CFG_DCACHE_LAZY_FETCH=1 +# The same on a write-back L2 shared by 4 cores behind write-through L1s. +- id: lazy_fetch-2 + via: blackbox + drivers: + - simx + - rtlsim + app: lazy_fetch + configs: -DVX_CFG_EXT_A_ENABLE -DVX_CFG_SOCKET_SIZE=1 -DVX_CFG_L2_WRITEBACK=1 -DVX_CFG_L2_LAZY_FETCH=1 + shape: + cores: 4 + l2cache: true diff --git a/ci/testcases/core.yaml b/ci/testcases/core.yaml index b09ebb949e..4f3f4c657e 100644 --- a/ci/testcases/core.yaml +++ b/ci/testcases/core.yaml @@ -131,6 +131,22 @@ tests: via: blackbox app: vecadd args: -n16384 +# Streaming stores into a lazy-fetch write-back L2: store misses allocate +# through the bank loopback instead of a memory fill. +- id: parity-vecadd-lazy-l2 + check: model_parity + via: blackbox + app: vecadd + args: -n16384 + configs: -DVX_CFG_L2_LAZY_FETCH=1 + shape: {l2cache: true} +# Lazy-fetch refetch paths on the default write-back dcache: partial-sector +# loads park and refetch, and same-set thrash evicts lines mid-refetch. +- id: parity-lazy-fetch + check: model_parity + via: blackbox + app: lazy_fetch + configs: -DVX_CFG_DCACHE_LAZY_FETCH=1 # FPU + LSU locality, multi-warp reuse (~1.29M cycles on the default shape). - id: parity-sgemm check: model_parity diff --git a/hw/rtl/VX_cluster.sv b/hw/rtl/VX_cluster.sv index 9b817df196..be46c3f4e4 100644 --- a/hw/rtl/VX_cluster.sv +++ b/hw/rtl/VX_cluster.sv @@ -227,6 +227,7 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; .WRITE_ENABLE (1), .WRITEBACK (`VX_CFG_L2_WRITEBACK), .DIRTY_BYTES (`VX_CFG_L2_DIRTYBYTES), + .LAZY_FETCH (`VX_CFG_L2_LAZY_FETCH), .REPL_POLICY (`VX_CFG_L2_REPL_POLICY), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3), diff --git a/hw/rtl/VX_socket.sv b/hw/rtl/VX_socket.sv index 3b04931533..d65f86df73 100644 --- a/hw/rtl/VX_socket.sv +++ b/hw/rtl/VX_socket.sv @@ -220,6 +220,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .WRITE_ENABLE (1), .WRITEBACK (`VX_CFG_DCACHE_WRITEBACK), .DIRTY_BYTES (`VX_CFG_DCACHE_DIRTYBYTES), + .LAZY_FETCH (`VX_CFG_DCACHE_LAZY_FETCH), .REPL_POLICY (`VX_CFG_DCACHE_REPL_POLICY), .NC_ENABLE (1), .CORE_OUT_BUF (3), diff --git a/hw/rtl/Vortex.sv b/hw/rtl/Vortex.sv index 98b5fb6c8e..a605dedd04 100644 --- a/hw/rtl/Vortex.sv +++ b/hw/rtl/Vortex.sv @@ -152,6 +152,7 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( .WRITE_ENABLE (1), .WRITEBACK (`VX_CFG_L3_WRITEBACK), .DIRTY_BYTES (`VX_CFG_L3_DIRTYBYTES), + .LAZY_FETCH (`VX_CFG_L3_LAZY_FETCH), .REPL_POLICY (`VX_CFG_L3_REPL_POLICY), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3), diff --git a/hw/rtl/cache/VX_cache.sv b/hw/rtl/cache/VX_cache.sv index a682b600b1..3ee2bae7aa 100644 --- a/hw/rtl/cache/VX_cache.sv +++ b/hw/rtl/cache/VX_cache.sv @@ -40,6 +40,8 @@ module VX_cache import VX_gpu_pkg::*; #( parameter DIRTY_BYTES = 0, // Enable dirty bytes on writeback + parameter LAZY_FETCH = 0, // Write misses allocate without a fill (write-validate) + parameter REPL_POLICY = `CS_REPL_FIFO, // Replacement policy parameter TAG_WIDTH = UUID_WIDTH + 1, // core request tag size @@ -67,6 +69,7 @@ module VX_cache import VX_gpu_pkg::*; #( `STATIC_ASSERT(NUM_BANKS == (1 << `CLOG2(NUM_BANKS)), ("invalid parameter: number of banks must be power of 2")) `STATIC_ASSERT(WRITE_ENABLE || !WRITEBACK, ("invalid parameter: writeback requires write enable")) `STATIC_ASSERT(WRITEBACK || !DIRTY_BYTES, ("invalid parameter: dirty bytes require writeback")) + `STATIC_ASSERT(!LAZY_FETCH || (WRITEBACK && DIRTY_BYTES && IS_LLC), ("invalid parameter: lazy fetch requires a write-back LLC with dirty bytes")) `STATIC_ASSERT(NUM_BANKS >= MEM_PORTS, ("invalid parameter: number of banks must be greater or equal to number of memory ports")) `STATIC_ASSERT(!BANK_HASH || NUM_BANKS <= 256, ("invalid parameter: bank hash supports up to 256 banks")) localparam REQ_SEL_WIDTH = `UP(`CS_REQ_SEL_BITS); @@ -365,6 +368,7 @@ module VX_cache import VX_gpu_pkg::*; #( .WRITE_ENABLE (WRITE_ENABLE), .WRITEBACK (WRITEBACK), .DIRTY_BYTES (DIRTY_BYTES), + .LAZY_FETCH (LAZY_FETCH), .REPL_POLICY (REPL_POLICY), .CRSQ_SIZE (CRSQ_SIZE), .MSHR_SIZE (MSHR_SIZE), diff --git a/hw/rtl/cache/VX_cache_bank.sv b/hw/rtl/cache/VX_cache_bank.sv index 4b95432384..90d8f1ddcb 100644 --- a/hw/rtl/cache/VX_cache_bank.sv +++ b/hw/rtl/cache/VX_cache_bank.sv @@ -31,6 +31,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( parameter WRITE_ENABLE = 1, parameter WRITEBACK = 0, parameter DIRTY_BYTES = 0, + parameter LAZY_FETCH = 0, // write miss allocates without a fill; a load to a partial sector refetches parameter REPL_POLICY = `CS_REPL_FIFO, parameter TAG_WIDTH = UUID_WIDTH + 1, parameter CORE_OUT_BUF = 0, @@ -102,6 +103,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // core-rsp queue depth (pow2): registered-skid minimum (2) plus CRSQ_SIZE extra slots localparam CRSQ_QUEUE_SIZE = 1 << `CLOG2(2 + CRSQ_SIZE); `STATIC_ASSERT(LATENCY >= 2, ("invalid parameter: cache bank LATENCY must be >= 2")) + `STATIC_ASSERT(!LAZY_FETCH || (WRITEBACK && DIRTY_BYTES && IS_LLC), ("invalid parameter: lazy fetch requires a write-back LLC with dirty bytes")) `UNUSED_PARAM (MRSQ_SIZE) // ======================================================================== @@ -144,7 +146,6 @@ module VX_cache_bank import VX_gpu_pkg::*; #( req_t req; logic [`CS_WORD_WIDTH-1:0] wdata; logic [NUM_WAYS-1:0] tag_matches; - logic is_refill; // fill into an already-resident line } data_t; typedef struct packed { // response + memory request (S0 -> S1 -> stC) @@ -152,6 +153,15 @@ module VX_cache_bank import VX_gpu_pkg::*; #( lookup_t lk; } commit_t; + // Lazy-fetch request flavors ride a side pipeline beside the request; it + // exists only when the feature is enabled. + typedef struct packed { + logic is_lazy; // data-less fill of a write-miss root + logic is_refetch; // fill whose root refetches a partial sector + logic is_retry; // refetch re-issued after its sector was evicted + logic is_park; // replay parked on a partial sector: issues its refetch + } lazy_req_t; + data_t sel_req, st0, dat_in, stD; // request + fill line: sel -> S0 -> stD commit_t cmt_in, st1, stC; // request + lookup delta: S0 -> S1 -> stC lookup_t lk_st0; // S0 lookup results @@ -174,6 +184,8 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire [`CS_LINE_ADDR_WIDTH-1:0] mem_rsp_addr; wire [`UP(`CS_SECTOR_SEL_BITS)-1:0] mem_rsp_sector; // sector this fill installs wire [MSHR_ADDR_WIDTH-1:0] mshr_alloc_id, mshr_previd; + wire mshr_fill_refetch, mshr_fill_retry, mshr_dequeue_reads; + wire lazyq_empty, lazyq_alm_full; wire mshr_pending_raw; wire mshr_pending_wr_raw; wire mshr_pending_ovl_raw; @@ -244,7 +256,6 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire do_read_stc = stC.req.valid && stC.req.is_creq && ~stC.req.rw; wire do_write_stc = stC.req.valid && stC.req.is_creq && stC.req.rw; - wire do_init_std = stD.req.valid && stD.req.is_init; wire do_fill_std = stD.req.valid && stD.req.is_fill; wire do_flush_std = stD.req.valid && stD.req.is_flush; wire do_read_std = stD.req.valid && stD.req.is_creq && ~stD.req.rw; @@ -285,7 +296,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( assign pipe_inflight = st0.req.valid || st1.req.valid || (| commit_valid); end // Queued AMO results are still writes to land in the array. - wire no_pending_req = ~pipe_inflight && mreq_queue_empty && ~amo_commit_busy; + wire no_pending_req = ~pipe_inflight && mreq_queue_empty && lazyq_empty && ~amo_commit_busy; VX_cache_flush #( .CACHE_SIZE (CACHE_SIZE), @@ -352,11 +363,39 @@ module VX_cache_bank import VX_gpu_pkg::*; #( && (replay_rw || st0.req.rw) && (replay_wsel == st0.req.word_idx); + // Lazy fetch: a write-miss root's fill loops back (lazyq) as a data-less + // fill below memory fills. A load/AMO head of a partial chain parks and + // refetches. A refetch merges into its resident sector one stage late, at + // stC; one whose sector was evicted installs nothing and retries as a + // plain fill (nothing can dirty an absent sector, so its data is current). + wire lazyq_valid = ~lazyq_empty; + wire [MSHR_ADDR_WIDTH-1:0] lazyq_id; + wire lazy_sel = ~mem_rsp_valid && lazyq_valid; + wire chain_partial, chain_retry; + // Lazy-fetch request flavors, folded to constants when the feature is off. + lazy_req_t lz_sel, lz_st0, lz_st1, lz_stc; + wire lazy_st0 = lz_st0.is_lazy; + wire refetch_st0 = lz_st0.is_refetch; + wire retry_st0 = lz_st0.is_retry; + wire park_st0 = lz_st0.is_park; + wire park_stc = lz_stc.is_park; + wire refetch_stc = lz_stc.is_refetch; + wire retry_stc = lz_stc.is_retry; + `UNUSED_VAR ({lz_st1, lz_stc.is_lazy}) + wire park_sel = (LAZY_FETCH != 0) && chain_partial && mshr_dequeue_reads; + wire refetch_hold = st0.req.valid && st0.req.is_fill && refetch_st0; + // The chain's head waits out a refetch at S0, and a head that would park + // waits for memory-request room; nothing else is selected meanwhile. Built + // from registers only and applied at the MSHR output, off the dequeue cone. + wire replay_block = refetch_hold || (park_sel && mreq_queue_alm_full); + // A fill starts a replay chain, and a chain carries at most one AMO (no // request enters behind a pending AMO), so a fill waits until the AMO // writeback queue has room for it; the queue drains meanwhile. wire replay_mux = replay_valid && ~fwd_head && ~replay_link_hold; - wire fill_mux = mem_rsp_valid && ~fwd_pending && ~replay_link_hold && ~amo_wbq_full; + wire refetch_inflight; + wire fill_mux = (mem_rsp_valid || lazyq_valid) && ~fwd_pending && ~replay_link_hold && ~amo_wbq_full + && ~refetch_inflight && ~replay_block; wire replay_grant = ~init_valid; wire replay_enable = replay_grant && replay_mux; @@ -374,22 +413,29 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire creq_enable = creq_grant && (amo_creq_path || amo_wb_path); assign replay_ready = replay_grant && ~fwd_head && ~replay_link_hold && ~(!WRITEBACK && replay_rw && mreq_queue_alm_full) && ~pipe_stall; - assign mem_rsp_ready = fill_grant && ~fwd_pending && ~replay_link_hold && ~amo_wbq_full && ~(WRITEBACK && mreq_queue_alm_full) && ~pipe_stall; + assign mem_rsp_ready = fill_grant && ~fwd_pending && ~replay_link_hold && ~amo_wbq_full && ~refetch_inflight + && ~replay_block && ~(WRITEBACK && mreq_queue_alm_full) && ~pipe_stall; assign flush_ready = flush_grant && ~(WRITEBACK && mreq_queue_alm_full) && ~pipe_stall; - assign core_req_ready = creq_grant && ~mreq_queue_alm_full && ~mshr_alm_full && ~pipe_stall + // a core request may become a write-miss root, which pushes to lazyq. + wire creq_alm_full = mreq_queue_alm_full || lazyq_alm_full || replay_block; + assign core_req_ready = creq_grant && ~creq_alm_full && ~mshr_alm_full && ~pipe_stall && ~amo_commit_busy && ~req_input_defer; wire init_fire = init_valid; wire replay_fire = replay_valid && replay_ready; wire mem_rsp_fire = mem_rsp_valid && mem_rsp_ready; + wire lazy_fire = lazy_sel && mem_rsp_ready; + wire fill_fire = mem_rsp_fire || lazy_fire; + wire park_fire = replay_fire && park_sel; wire flush_fire = flush_valid && flush_ready; // amo_wb_path already excludes the cycle a fresh AMO commits at S1, so the // writeback never races the chain update. - wire amo_wb_fire = amo_wb_path && creq_grant && ~mreq_queue_alm_full && ~mshr_alm_full && ~pipe_stall; + wire amo_wb_fire = amo_wb_path && creq_grant && ~creq_alm_full && ~mshr_alm_full && ~pipe_stall; wire core_req_fire = (amo_creq_path || amo_wb_path) && creq_grant - && ~mreq_queue_alm_full && ~mshr_alm_full && ~pipe_stall; + && ~creq_alm_full && ~mshr_alm_full && ~pipe_stall; wire [MSHR_ADDR_WIDTH-1:0] mem_rsp_id = mem_rsp_tag[MSHR_ADDR_WIDTH-1:0]; + wire [MSHR_ADDR_WIDTH-1:0] fill_id = lazy_sel ? lazyq_id : mem_rsp_id; // Fill word_idx: place the installed sector in the high (sector) bits of the // in-line word offset; low (word-in-sector) bits are don't-care for a fill. @@ -441,7 +487,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // it never re-commits at S1. always @(*) begin sel_req = '0; - sel_req.req.valid = init_fire || replay_fire || mem_rsp_fire || flush_fire || core_req_fire; + sel_req.req.valid = init_fire || replay_fire || fill_fire || flush_fire || core_req_fire; sel_req.req.is_init = init_valid; sel_req.req.is_fill = fill_enable; sel_req.req.is_flush = flush_enable; @@ -462,7 +508,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( : (amo_wb_pending ? amo_wb_word_idx : core_req_wsel)); sel_req.req.req_idx = replay_mux ? replay_idx : (amo_wb_pending ? amo_wb_idx : core_req_idx); sel_req.req.tag = (init_valid | flush_valid) ? (flush_valid ? flush_tag : '0) - : (replay_mux ? replay_tag : (fill_mux ? mem_rsp_tag_s + : (replay_mux ? replay_tag : (fill_mux ? (lazy_sel ? '0 : mem_rsp_tag_s) : (amo_wb_pending ? amo_wb_tag : core_req_tag))); sel_req.req.mshr_id = replay_id; sel_req.req.amo = replay_mux ? replay_amo : (amo_wb_pending ? amo_req_t'('0) @@ -471,6 +517,11 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // tag_matches is computed at S0; left 0 here (overridden at the data bubble). end + assign lz_sel.is_lazy = fill_enable && lazy_sel; + assign lz_sel.is_refetch = fill_enable && ~lazy_sel && mshr_fill_refetch; + assign lz_sel.is_retry = fill_enable && ~lazy_sel && mshr_fill_retry; + assign lz_sel.is_park = replay_enable && park_sel; + // UUID extraction (debug + MSHR ordering): per stage, from the carried tag. wire [`UP(UUID_WIDTH)-1:0] req_uuid_sel, req_uuid_st0, req_uuid_st1, req_uuid_stc; if (UUID_WIDTH != 0) begin : g_req_uuid @@ -521,6 +572,11 @@ module VX_cache_bank import VX_gpu_pkg::*; #( ); wire [`CS_WAY_SEL_WIDTH-1:0] fill_way_st0 = line_present_any_st0 ? present_way_st0 : victim_way; wire [`CS_WAY_SEL_WIDTH-1:0] evict_way_st0 = st0.req.is_fill ? fill_way_st0 : st0.req.way_idx; + // A refetch (first attempt) uses no victim: it merges into the way that + // holds its sector, carried apart from evict_way so that the tag compare + // stays off the array write enables. + wire refetch_first_st0 = do_fill_st0 && refetch_st0 && ~retry_st0; + wire [`CS_WAY_SEL_WIDTH-1:0] merge_way_st0 = refetch_first_st0 ? hit_idx_st0 : evict_way_st0; // Fill way as one-hot, straight from the per-way presence vector: routing // it through the binary encode -> evict_way -> per-way compare round trip @@ -546,12 +602,27 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .lookup_hit (st1.lk.is_hit), .lookup_line (st1.req.addr[`CS_LINE_SEL_BITS-1:0]), .lookup_way (st1.req.way_idx), - .repl_valid (do_fill_st0 && ~st0.req.is_passthru_fill && ~pipe_stall), + // a refetch merges into its resident sector and uses no victim + .repl_valid (do_fill_st0 && ~st0.req.is_passthru_fill && ~refetch_first_st0 && ~pipe_stall), .repl_line (line_idx_st0), .repl_line_n (sel_req.req.addr[`CS_LINE_SEL_BITS-1:0]), .repl_way (victim_way) ); + wire creq_hit_order_hazard_st0; + + // Loads and AMOs read the sector, so a partial one is a miss for them. The + // array enables use the raw tag match (a partial sector is still resident); + // only the commit decisions, registered at S1, see the partial check. + wire needs_data_st0 = st0.req.is_creq && ~st0.req.rw; + wire partial_match_st0; + wire partial_miss_st0 = (LAZY_FETCH != 0) && needs_data_st0 && partial_match_st0; + // A refetch finds its partial sector resident (it merges) unless the + // sector was evicted meanwhile: its data may then predate the eviction's + // writeback, so it installs nothing and its chain retries. + wire fill_resident_st0 = (| tag_matches_st0); + wire fill_evicted_st0 = refetch_first_st0 && ~fill_resident_st0; + VX_cache_tags #( .CACHE_SIZE (CACHE_SIZE), .LINE_SIZE (LINE_SIZE), @@ -560,6 +631,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .WORD_SIZE (WORD_SIZE), .SECTOR_SIZE(SECTOR_SIZE), .WRITEBACK (WRITEBACK), + .LAZY_FETCH (LAZY_FETCH), .AMO_ENABLE ((AMO_ENABLE != 0) && (IS_LLC == 0)) ) cache_tags ( .clk (clk), @@ -567,12 +639,14 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .stall (pipe_stall), .init (do_init_st0), .flush (do_flush_st0 && ~pipe_stall), - .fill (do_fill_st0 && ~st0.req.is_passthru_fill && ~pipe_stall), + .fill (do_fill_st0 && ~st0.req.is_passthru_fill && ~refetch_first_st0 && ~pipe_stall), + .refetch (refetch_first_st0 && ~pipe_stall), .read (do_read_st0 && ~pipe_stall), .write (do_write_st0 && ~pipe_stall), // non-LLC AMO forwards downstream and invalidates its own copy so the // issuer's later plain load refetches the new value. .invalidate (is_amo_fwd_st0 && lk_st0.is_hit && ~pipe_stall), + .fill_nodata (lazy_st0), .line_idx (line_idx_st0), .line_idx_n (sel_req.req.addr[`CS_LINE_SEL_BITS-1:0]), .line_tag (line_tag_st0), @@ -583,7 +657,8 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .line_present (line_present_st0), .evict_dirty (evict_dirty_st0), .evict_dirty_mask (evict_dirty_mask_st0), - .evict_tag (evict_tag_st0) + .evict_tag (evict_tag_st0), + .partial_match (partial_match_st0) ); VX_onehot_encoder #( @@ -606,23 +681,24 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // disjoint words — the common case in streaming, where the whole warp's // beats land on one sector — keep the fast release path, as does // read-after-read on the same word. - wire creq_hit_order_hazard_st0 = st0.req.is_creq && ~st0.req.is_replay - && mshr_pending_raw - && (mshr_pending_wr_raw || (st0.req.rw && mshr_pending_ovl_raw)) - && ~is_amo_fwd_st0 - && ~((AMO_ENABLE != 0) && st0.req.amo.amo_valid); + assign creq_hit_order_hazard_st0 = st0.req.is_creq && ~st0.req.is_replay + && mshr_pending_raw + && (mshr_pending_wr_raw || (st0.req.rw && mshr_pending_ovl_raw)) + && ~is_amo_fwd_st0 + && ~((AMO_ENABLE != 0) && st0.req.amo.amo_valid); // S0 lookup delta (single combinational driver). The AMO requester is forced // non-pending so it never coalesces onto a prior same-line entry. always @(*) begin lk_st0 = '0; - lk_st0.is_hit = (| tag_matches_st0) && ~creq_hit_order_hazard_st0; + lk_st0.is_hit = (| tag_matches_st0) && ~partial_miss_st0 && ~creq_hit_order_hazard_st0; lk_st0.is_dirty = evict_dirty_st0; lk_st0.is_refill = st0.req.is_fill && line_present_any_st0; lk_st0.evict_dirty_mask = evict_dirty_mask_st0; lk_st0.evict_tag = evict_tag_st0; lk_st0.mshr_previd = mshr_previd; - lk_st0.mshr_pending = mshr_pending_raw && ~is_amo_fwd_st0; + // a parked replay is its own chain's root: it issues the refetch. + lk_st0.mshr_pending = mshr_pending_raw && ~is_amo_fwd_st0 && ~park_st0; end // ======================================================================== @@ -646,14 +722,13 @@ module VX_cache_bank import VX_gpu_pkg::*; #( dat_in = st0; dat_in.req.way_idx = evict_way_st0; dat_in.tag_matches = tag_matches_st0 & {NUM_WAYS{~creq_hit_order_hazard_st0}}; - dat_in.is_refill = lk_st0.is_refill; end // commit path: the request (with the resolved hit/victim way and MSHR id) // plus the lookup delta. The word payload is dropped here. always @(*) begin cmt_in.req = st0.req; - cmt_in.req.way_idx = st0.req.is_creq ? hit_idx_st0 : evict_way_st0; + cmt_in.req.way_idx = st0.req.is_creq ? hit_idx_st0 : merge_way_st0; cmt_in.req.mshr_id = st0.req.is_replay ? st0.req.mshr_id : mshr_alloc_id; cmt_in.lk = lk_st0; end @@ -693,12 +768,71 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .data_out (stC) ); + // A refetch whose sector is still resident merges at stC; a data-less or + // refetch fill writes no data at stD. + wire fill_merge_st0 = refetch_first_st0 && fill_resident_st0; + wire fill_nodata_st0 = lazy_st0 || refetch_first_st0; + wire fill_merge_stc, fill_nodata_std, lazy_std; + if (LAZY_FETCH != 0) begin : g_lazy_pipe + wire fill_merge_st1; + VX_pipe_register #( + .DATAW ($bits(lazy_req_t)), + .RESETW ($bits(lazy_req_t)) + ) reg_lz_s0 ( + .clk (clk), + .reset (reset), + .enable (~pipe_stall), + .data_in (lz_sel), + .data_out (lz_st0) + ); + VX_pipe_register #( + .DATAW ($bits(lazy_req_t) + 1), + .RESETW ($bits(lazy_req_t) + 1) + ) reg_lz_s1 ( + .clk (clk), + .reset (reset), + .enable (~pipe_stall), + .data_in ({lz_st0, fill_merge_st0}), + .data_out ({lz_st1, fill_merge_st1}) + ); + VX_pipe_register #( + .DATAW ($bits(lazy_req_t) + 1), + .RESETW ($bits(lazy_req_t) + 1), + .DEPTH (PIPE_EX) + ) reg_lz_cmt ( + .clk (clk), + .reset (reset), + .enable (~pipe_stall), + .data_in ({lz_st1, fill_merge_st1}), + .data_out ({lz_stc, fill_merge_stc}) + ); + VX_pipe_register #( + .DATAW (2), + .RESETW (2), + .DEPTH (PIPE_EX) + ) reg_lz_dat ( + .clk (clk), + .reset (reset), + .enable (~pipe_stall), + .data_in ({lazy_st0, fill_nodata_st0}), + .data_out ({lazy_std, fill_nodata_std}) + ); + end else begin : g_no_lazy_pipe + assign lz_st0 = '0; + assign lz_st1 = '0; + assign lz_stc = '0; + assign fill_merge_stc = 1'b0; + assign fill_nodata_std = 1'b0; + assign lazy_std = 1'b0; + `UNUSED_VAR ({lz_sel, fill_merge_st0, fill_nodata_st0}) + end + // a passthru-AMO replay carries its result word instead of an installed // line, so it counts as a hit at the commit stage. The AMO commit signals // (is_amo_replay_st1) resolve at stC, so the replay-must-hit check is taken // there; with PIPE_EX>0 an st1-staged check would sample them a stage early. wire eff_hit_stc = stC.lk.is_hit || is_amo_replay_st1; - `RUNTIME_ASSERT (~(stC.req.valid && stC.req.is_replay && ~eff_hit_stc), ("missed mshr replay")) + `RUNTIME_ASSERT (~(stC.req.valid && stC.req.is_replay && ~park_stc && ~eff_hit_stc), ("missed mshr replay")) // ======================================================================== // Data array (driven at stD; outputs land at stC) @@ -742,7 +876,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .data_in (fbuf_addr_r), .data_out (fill_addr_std) ); - `RUNTIME_ASSERT (~(do_fill_std && ~stD.req.is_passthru_fill) || (stD.req.addr == fill_addr_std), ("%t: %s fill sector misaligned with its array write: addr=0x%0h, staged=0x%0h", $time, INSTANCE_ID, `CS_BANK_TO_FULL_ADDR(stD.req.addr, BANK_ID), `CS_BANK_TO_FULL_ADDR(fill_addr_std, BANK_ID))) + `RUNTIME_ASSERT (~(do_fill_std && ~stD.req.is_passthru_fill && ~lazy_std) || (stD.req.addr == fill_addr_std), ("%t: %s fill sector misaligned with its array write: addr=0x%0h, staged=0x%0h", $time, INSTANCE_ID, `CS_BANK_TO_FULL_ADDR(stD.req.addr, BANK_ID), `CS_BANK_TO_FULL_ADDR(fill_addr_std, BANK_ID))) `endif // The fill sector, replicated across the line so each sector's slices see @@ -750,6 +884,18 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // gate). One copy == the full line when 1 sector/line. wire [`CS_LINE_WIDTH-1:0] fbuf_line = {`CS_SECTORS_PER_LINE{fill_data_std}}; + // sector of the request at commit (the missed sector of a fill request). + wire [`UP(`CS_SECTOR_SEL_BITS)-1:0] sector_idx_stc; + // a refetch merges into its resident sector at stC (see refetch_hold) + wire do_merge_stc = stC.req.valid && stC.req.is_fill && fill_merge_stc; + // Dirty-byte mask updates land at stC in pipeline order, once per request: + // a fill held at stC by its writeback beats still reads the evicted mask. + wire refetch_first_stc = stC.req.is_fill && refetch_stc && ~retry_stc; + wire mask_set_stc = do_write_stc && stC.lk.is_hit; + wire mask_fill_stc = stC.req.valid && stC.req.is_fill && ~stC.req.is_passthru_fill && ~refetch_first_stc; + wire mask_clr_line_stc = (mask_fill_stc && ~stC.lk.is_refill) || (stC.req.valid && stC.req.is_flush); + wire mask_clr_sec_stc = mask_fill_stc && stC.lk.is_refill; + VX_cache_data #( .CACHE_SIZE (CACHE_SIZE), .LINE_SIZE (LINE_SIZE), @@ -759,13 +905,23 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .SECTOR_SIZE (SECTOR_SIZE), .WRITE_ENABLE (WRITE_ENABLE), .WRITEBACK (WRITEBACK), - .DIRTY_BYTES (DIRTY_BYTES) + .DIRTY_BYTES (DIRTY_BYTES), + .LAZY_FETCH (LAZY_FETCH) ) cache_data ( .clk (clk), .reset (reset), - .init (do_init_std), .fill (do_fill_std && ~stD.req.is_passthru_fill && ~pipe_stall), - .refill (stD.is_refill), + .fill_nodata (fill_nodata_std), + .mask_write ((mask_set_stc || mask_clr_line_stc || mask_clr_sec_stc) && ~pipe_stall), + .mask_set (mask_set_stc), + .mask_clr_line (mask_clr_line_stc), + .mask_clr_sector (mask_clr_sec_stc), + .mask_way (stC.req.way_idx), + .mask_line (stC.req.addr[`CS_LINE_SEL_BITS-1:0]), + .mask_word (stC.req.word_idx), + .mask_byteen (stC.req.byteen), + .mask_sector (sector_idx_stc), + .merge_write (do_merge_stc && ~pipe_stall), .flush (do_flush_std && ~pipe_stall), .read (do_read_std && ~pipe_stall), .write (do_write_std && ~pipe_stall), @@ -776,7 +932,6 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .write_word (stD.wdata), .word_idx (stD.req.word_idx), .sector_idx (sector_idx_std), - .way_idx_r (stC.req.way_idx), .write_byteen (stD.req.byteen), .read_data (read_data_stc), .evict_byteen (evict_byteen_stc) @@ -829,7 +984,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire mshr_release_fire = mshr_finalize_st1 && mshr_release_st1 && ~pipe_stall; wire [1:0] mshr_dequeue; - `POP_COUNT(mshr_dequeue, {(replay_fire || fwd_fire), mshr_release_fire}); + `POP_COUNT(mshr_dequeue, {((replay_fire && ~park_fire) || fwd_fire), mshr_release_fire}); VX_pending_size #( .SIZE (MSHR_SIZE), @@ -858,6 +1013,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .AMO_ENABLE (AMO_ENABLE != 0), .AMO_PASSTHRU ((AMO_ENABLE != 0) && (IS_LLC == 0)), .WORD_SEL_WIDTH (WORD_SEL_WIDTH), + .LAZY_FETCH (LAZY_FETCH), .DATA_WIDTH (WORD_SEL_WIDTH + WORD_SIZE + `CS_WORD_WIDTH + TAG_WIDTH + REQ_SEL_WIDTH + AMO_REQ_BITS) ) cache_mshr ( .clk (clk), @@ -865,10 +1021,12 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .deq_req_uuid (req_uuid_sel), .alc_req_uuid (req_uuid_st0), .fin_req_uuid (req_uuid_st1), - .fill_valid (mem_rsp_fire), - .fill_id (mem_rsp_id), + .fill_valid (fill_fire), + .fill_id (fill_id), .fill_addr (mem_rsp_addr), .fill_sector (mem_rsp_sector), + .fill_refetch (mshr_fill_refetch), + .fill_retry (mshr_fill_retry), .probe_addr (core_req_addr), .probe_pending_ld (mshr_probe_pending_ld), .probe_pending_amo (mshr_probe_pending_amo), @@ -879,9 +1037,13 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .dequeue_valid (replay_valid), .dequeue_addr (replay_addr), .dequeue_rw (replay_rw), + .dequeue_reads (mshr_dequeue_reads), .dequeue_data ({replay_wsel, replay_byteen, replay_data, replay_tag, replay_idx, replay_amo}), .dequeue_id (replay_id), .dequeue_ready (replay_ready || fwd_fire), + .dequeue_park (park_sel), + .dequeue_park_retry (chain_retry), + .dequeue_block (replay_block), .allocate_valid (mshr_allocate_st0 && ~pipe_stall), .allocate_addr (st0.req.addr), .allocate_sector (sector_idx_st0), @@ -890,6 +1052,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // Only non-LLC AMOs must not coalesce; at the LLC same-line AMOs coalesce // and serialize their commits on the single filled line. .allocate_is_amo (AMO_ENABLE ? st0.req.amo.amo_valid : 1'b0), + .allocate_refetch (needs_data_st0 && partial_match_st0), .allocate_data ({st0.req.word_idx, st0.req.byteen, write_word_st0, st0.req.tag, st0.req.req_idx, st0.req.amo}), .allocate_id (mshr_alloc_id), .allocate_pending (mshr_pending_raw), @@ -969,7 +1132,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .is_creq_st0 (st0.req.is_creq), // Atomics are exempt from the hit-order hazard, so their S0 hit is // the raw tag match; the MSHR probe stays off the commit_busy cone. - .is_hit_st0 ((| tag_matches_st0)), + .is_hit_st0 ((| tag_matches_st0) && ~partial_miss_st0), .is_replay_st0 (st0.req.is_replay), // Commit ports are fed from stC (the deferred data-output stage), so // the AMO RMW operands and the read word align at PIPE_EX>0. At @@ -1084,7 +1247,10 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // ======================================================================== reg fwd_active_r; - wire fwd_stage = mem_rsp_fire && ~is_passthru_fill_sel; + // A refetch fill stages its sector for the array write but cannot answer + // the chain: the staged copy lacks the dirty bytes the array merges in. + wire fbuf_stage = mem_rsp_fire && ~is_passthru_fill_sel; + wire fwd_stage = fbuf_stage && ~mshr_fill_refetch; wire fwd_close = replay_valid && fwd_active_r && (replay_rw || replay_amo.amo_valid); @@ -1108,11 +1274,11 @@ module VX_cache_bank import VX_gpu_pkg::*; #( end else begin if (fwd_stage) begin fwd_active_r <= 1'b1; - end else if (fwd_close) begin + end else if (fwd_close || ((LAZY_FETCH != 0) && fill_fire)) begin fwd_active_r <= 1'b0; end end - if (fwd_stage) begin + if (fbuf_stage) begin fbuf_data_r <= mem_rsp_data; fbuf_addr_r <= mem_rsp_addr; end @@ -1163,13 +1329,13 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire mreq_queue_rw; wire is_fill_or_flush_stc = stC.req.is_fill || (stC.req.is_flush && WRITEBACK); + // a write-miss root under lazy fetch loops its fill back instead. + wire lazy_alloc_stc = (LAZY_FETCH != 0) && do_write_stc && ~stC.lk.is_hit && ~stC.lk.mshr_pending; wire do_fill_or_flush_stc = stC.req.valid && is_fill_or_flush_stc; // a sector refill keeps the resident line (no eviction), so never writes back. - wire do_writeback_stc = do_fill_or_flush_stc && stC.lk.is_dirty && ~stC.lk.is_refill; + wire do_writeback_stc = do_fill_or_flush_stc && stC.lk.is_dirty && ~stC.lk.is_refill && ~refetch_first_stc; wire [`CS_LINE_ADDR_WIDTH-1:0] evict_addr_stc = {stC.lk.evict_tag, stC.req.addr[`CS_LINE_SEL_BITS-1:0]}; - // sector of the request at commit (the missed sector of a fill request). - wire [`UP(`CS_SECTOR_SEL_BITS)-1:0] sector_idx_stc; if (`CS_SECTOR_SEL_BITS != 0) begin : g_sector_idx_stc assign sector_idx_stc = stC.req.word_idx[`CS_WORD_SEL_BITS-1 -: `CS_SECTOR_SEL_BITS]; end else begin : g_sector_idx_stc0 @@ -1239,18 +1405,21 @@ module VX_cache_bank import VX_gpu_pkg::*; #( if (WRITEBACK) begin : g_wb if (DIRTY_BYTES) begin : g_dirty_bytes wire has_dirty_bytes = (| evict_byteen_stc); - `RUNTIME_ASSERT (~do_fill_or_flush_stc || (stC.lk.is_dirty == has_dirty_bytes), ("missmatch dirty bytes: dirty_line=%b, dirty_bytes=%b, addr=0x%0h", stC.lk.is_dirty, has_dirty_bytes, `CS_BANK_TO_FULL_ADDR(addr_stc, BANK_ID))) + // a refetch evicts nothing: the mask it reads is its resident way's + wire check_evict_bytes = do_fill_or_flush_stc && ~refetch_first_stc; + `RUNTIME_ASSERT (~check_evict_bytes || (stC.lk.is_dirty == has_dirty_bytes), ("missmatch dirty bytes: dirty_line=%b, dirty_bytes=%b, addr=0x%0h", stC.lk.is_dirty, has_dirty_bytes, `CS_BANK_TO_FULL_ADDR(addr_stc, BANK_ID))) // per-sector: a dirty sector must carry dirty bytes, or its writeback beat is dropped. wire [SEC-1:0] sec_has_dirty_bytes; for (genvar s = 0; s < SEC; ++s) begin : g_sec_has_dirty_bytes assign sec_has_dirty_bytes[s] = (| evict_byteen_stc[s*SECTOR_SIZE +: SECTOR_SIZE]); end - `RUNTIME_ASSERT (~do_fill_or_flush_stc || ((stC.lk.evict_dirty_mask & ~sec_has_dirty_bytes) == '0), ("dirty sector without dirty bytes: dirty_mask=%b, dirty_bytes=%b, addr=0x%0h", stC.lk.evict_dirty_mask, sec_has_dirty_bytes, `CS_BANK_TO_FULL_ADDR(addr_stc, BANK_ID))) + `RUNTIME_ASSERT (~check_evict_bytes || ((stC.lk.evict_dirty_mask & ~sec_has_dirty_bytes) == '0), ("dirty sector without dirty bytes: dirty_mask=%b, dirty_bytes=%b, addr=0x%0h", stC.lk.evict_dirty_mask, sec_has_dirty_bytes, `CS_BANK_TO_FULL_ADDR(addr_stc, BANK_ID))) end // fill request on a read/write miss (one sector); multi-beat writeback // on a dirty eviction. The two are mutually exclusive (a writeback is a // fill/flush commit; a fill request is a creq commit). - wire fill_req_push = (do_read_stc || do_write_stc) && ~stC.lk.is_hit && ~stC.lk.mshr_pending && ~pipe_stall; + wire fill_req_push = (do_read_stc || do_write_stc) && ~stC.lk.is_hit && ~stC.lk.mshr_pending + && ~lazy_alloc_stc && ~pipe_stall; assign mreq_queue_push = fill_req_push || wb_beat_accept; assign mreq_queue_addr = is_wb_beat ? wb_mreq_addr : rd_mreq_addr; assign mreq_queue_rw = is_wb_beat; @@ -1317,12 +1486,90 @@ module VX_cache_bank import VX_gpu_pkg::*; #( ); assign mem_req_valid = ~mreq_queue_empty; + // Lazy-fetch loopback: write-miss roots awaiting their data-less fill. + // Sized like the memory request queue's reservation: a core request is + // admitted only while every request already in the pipe still fits. + if (LAZY_FETCH != 0) begin : g_lazyq + wire lazyq_full; + localparam LAZYQ_SIZE = 1 << `CLOG2(PIPELINE_STAGES + 1); + VX_fifo_queue #( + .DATAW (MSHR_ADDR_WIDTH), + .DEPTH (LAZYQ_SIZE), + .ALM_FULL (LAZYQ_SIZE - PIPELINE_STAGES) + ) lazy_queue ( + .clk (clk), + .reset (reset), + .push (lazy_alloc_stc && ~pipe_stall), + .pop (lazy_fire), + .data_in (stC.req.mshr_id), + .data_out (lazyq_id), + .empty (lazyq_empty), + .alm_full (lazyq_alm_full), + .full (lazyq_full), + `UNUSED_PIN (alm_empty), + `UNUSED_PIN (size) + ); + + // The partial state of the draining chain's sector: set by a data-less + // fill, cleared by a data fill, and corrected at S0 when a refetch + // finds its sector evicted (the replay waits out that cycle). + reg chain_partial_r, chain_retry_r; + always @(posedge clk) begin + if (reset) begin + chain_partial_r <= 1'b0; + chain_retry_r <= 1'b0; + end else if (fill_fire) begin + chain_partial_r <= lazy_fire; + chain_retry_r <= 1'b0; + end else if (refetch_hold && ~pipe_stall) begin + chain_partial_r <= fill_evicted_st0; + chain_retry_r <= fill_evicted_st0; + end + end + assign chain_partial = chain_partial_r; + assign chain_retry = chain_retry_r; + + // A refetch's fill sector must stay staged until its stC merge. + reg refetch_inflight_r; + always @(posedge clk) begin + if (reset) begin + refetch_inflight_r <= 1'b0; + end else if (mem_rsp_fire && mshr_fill_refetch) begin + refetch_inflight_r <= 1'b1; + end else if (stC.req.valid && stC.req.is_fill && refetch_stc && ~pipe_stall) begin + refetch_inflight_r <= 1'b0; + end + end + assign refetch_inflight = refetch_inflight_r; + + `RUNTIME_ASSERT (~(lazy_alloc_stc && ~pipe_stall && lazyq_full && ~lazy_fire), + ("%t: %s lazy queue overflow", $time, INSTANCE_ID)) + `RUNTIME_ASSERT (~(park_fire && fwd_fire), ("%t: %s park and forward in one cycle", $time, INSTANCE_ID)) + `RUNTIME_ASSERT (~replay_valid || (mshr_dequeue_reads == (~replay_rw || replay_amo.amo_valid)), + ("%t: %s chain head read flag disagrees with its payload", $time, INSTANCE_ID)) + // Nothing is selected behind a refetch, so its stC merge has the array + // to itself and the next lookup reads the cleared partial bit. + `RUNTIME_ASSERT (~(refetch_hold && sel_req.req.valid), ("%t: %s request selected behind a refetch", $time, INSTANCE_ID)) + `RUNTIME_ASSERT (~(st0.req.valid && park_st0 && (| tag_matches_st0) && ~partial_miss_st0), ("%t: %s parked replay hit", $time, INSTANCE_ID)) + // Only a refetch can find its sector resident. + `RUNTIME_ASSERT (~(do_fill_st0 && ~refetch_first_st0 && ~st0.req.is_passthru_fill && fill_resident_st0), + ("%t: %s fill into a resident sector: addr=0x%0h", $time, INSTANCE_ID, `CS_BANK_TO_FULL_ADDR(st0.req.addr, BANK_ID))) + end else begin : g_no_lazyq + assign lazyq_id = '0; + assign lazyq_empty = 1'b1; + assign lazyq_alm_full = 1'b0; + assign chain_partial = 1'b0; + assign chain_retry = 1'b0; + assign refetch_inflight = 1'b0; + `UNUSED_VAR ({lazy_alloc_stc, lazy_fire, refetch_hold, fill_resident_st0, fill_evicted_st0, partial_match_st0}) + end + `UNUSED_VAR (do_lookup_st0) /////////////////////////////////////////////////////////////////////////////// `ifdef PERF_ENABLE - assign perf_read_miss = do_read_st1 && ~st1.lk.is_hit; + assign perf_read_miss = do_read_st1 && ~st1.lk.is_hit && ~lz_st1.is_park; assign perf_write_miss = do_write_st1 && ~st1.lk.is_hit; assign perf_evictions = do_writeback_stc; assign perf_mshr_stall = mshr_alm_full; @@ -1348,8 +1595,16 @@ module VX_cache_bank import VX_gpu_pkg::*; #( crsp_queue_stall, mreq_queue_alm_full, mshr_alm_full)) end if (mem_rsp_fire) begin - `TRACE(2, ("%t: %s fill-rsp: addr=0x%0h, mshr_id=%0d, data=0x%h (#%0d)\n", $time, INSTANCE_ID, - mem_rsp_full_addr, mem_rsp_id, mem_rsp_data, req_uuid_sel)) + `TRACE(2, ("%t: %s fill-rsp: addr=0x%0h, mshr_id=%0d, refetch=%b, retry=%b, data=0x%h (#%0d)\n", $time, INSTANCE_ID, + mem_rsp_full_addr, mem_rsp_id, mshr_fill_refetch, mshr_fill_retry, mem_rsp_data, req_uuid_sel)) + end + if (lazy_fire) begin + `TRACE(2, ("%t: %s lazy-fill: addr=0x%0h, mshr_id=%0d\n", $time, INSTANCE_ID, + mem_rsp_full_addr, lazyq_id)) + end + if (park_fire) begin + `TRACE(2, ("%t: %s mshr-park: addr=0x%0h, tag=0x%0h, req_idx=%0d (#%0d)\n", $time, INSTANCE_ID, + replay_full_addr, replay_tag, replay_idx, req_uuid_sel)) end if (replay_fire) begin `TRACE(2, ("%t: %s mshr-pop: addr=0x%0h, tag=0x%0h, req_idx=%0d (#%0d)\n", $time, INSTANCE_ID, diff --git a/hw/rtl/cache/VX_cache_cluster.sv b/hw/rtl/cache/VX_cache_cluster.sv index 0dfc14eca1..88dc19d40d 100644 --- a/hw/rtl/cache/VX_cache_cluster.sv +++ b/hw/rtl/cache/VX_cache_cluster.sv @@ -44,6 +44,8 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( parameter DIRTY_BYTES = 0, // Enable dirty bytes on writeback + parameter LAZY_FETCH = 0, // Write misses allocate without a fill (write-validate) + parameter REPL_POLICY = `CS_REPL_FIFO, // Replacement policy parameter TAG_WIDTH = UUID_WIDTH + 1, // core request tag size @@ -164,6 +166,7 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( .WRITE_ENABLE (WRITE_ENABLE), .WRITEBACK (WRITEBACK), .DIRTY_BYTES (DIRTY_BYTES), + .LAZY_FETCH (LAZY_FETCH), .REPL_POLICY (REPL_POLICY), .CRSQ_SIZE (CRSQ_SIZE), .MSHR_SIZE (MSHR_SIZE), diff --git a/hw/rtl/cache/VX_cache_data.sv b/hw/rtl/cache/VX_cache_data.sv index d30d289f13..bc5bf12385 100644 --- a/hw/rtl/cache/VX_cache_data.sv +++ b/hw/rtl/cache/VX_cache_data.sv @@ -39,14 +39,28 @@ module VX_cache_data import VX_gpu_pkg::*; #( parameter SECTOR_SIZE = LINE_SIZE, // Size of a sector in bytes (fill granule); = LINE_SIZE => 1 sector parameter WRITE_ENABLE = 1, // Enable cache writeable parameter WRITEBACK = 0, // Enable cache writeback - parameter DIRTY_BYTES = 0 // Enable dirty bytes on writeback + parameter DIRTY_BYTES = 0, // Enable dirty bytes on writeback + parameter LAZY_FETCH = 0 // Fills may be data-less or merge under the dirty bytes ) ( input wire clk, input wire reset, // inputs - input wire init, input wire fill, - input wire refill, // fill into an already-resident line + input wire fill_nodata, // data-less fill: allocate only, no array write + // dirty-byte mask access at the commit stage (stC), one stage after the + // array access: a store sets its bytes, a fill/flush reads and clears + input wire mask_write, + input wire mask_set, + input wire mask_clr_line, + input wire mask_clr_sector, + input wire [`CS_WAY_SEL_WIDTH-1:0] mask_way, + input wire [`CS_LINE_SEL_BITS-1:0] mask_line, + input wire [`UP(`CS_WORD_SEL_BITS)-1:0] mask_word, + input wire [WORD_SIZE-1:0] mask_byteen, + input wire [`UP(`CS_SECTOR_SEL_BITS)-1:0] mask_sector, + // merge of a refetched sector into its resident copy at the commit stage + // (the stage behind it is empty): only the clean bytes + input wire merge_write, input wire flush, input wire read, input wire write, @@ -58,14 +72,11 @@ module VX_cache_data import VX_gpu_pkg::*; #( input wire [WORD_SIZE-1:0] write_byteen, input wire [`UP(`CS_WORD_SEL_BITS)-1:0] word_idx, input wire [`UP(`CS_SECTOR_SEL_BITS)-1:0] sector_idx, // sector being filled - input wire [`CS_WAY_SEL_WIDTH-1:0] way_idx_r, // outputs output wire [`CS_LINE_WIDTH-1:0] read_data, output wire [LINE_SIZE-1:0] evict_byteen ); - // The main data array resolves the way at read-issue and folds it into the - // array address, so the S1 output-way register is only used by the (narrow, - // off-path) dirty-byte mask below. + `STATIC_ASSERT(!LAZY_FETCH || (DIRTY_BYTES && WRITEBACK), ("invalid parameter: lazy fetch requires write-back dirty bytes")) localparam WAY_SEL_BITS = `CS_WAY_SEL_BITS; localparam DATA_RAM_DEPTH = `CS_LINES_PER_BANK * NUM_WAYS; @@ -84,6 +95,7 @@ module VX_cache_data import VX_gpu_pkg::*; #( `UNUSED_PIN (valid_out) ); + wire fill_write = fill && ~((LAZY_FETCH != 0) && fill_nodata); wire is_evict = fill || flush; wire hit_any = (| tag_matches); wire [`CS_WAY_SEL_WIDTH-1:0] way_sel = is_evict ? evict_way : hit_way; @@ -91,69 +103,66 @@ module VX_cache_data import VX_gpu_pkg::*; #( // {way, line} -> physical address wire [DATA_ADDR_WIDTH-1:0] data_addr; if (NUM_WAYS > 1) begin : g_way_addr - assign data_addr = DATA_ADDR_WIDTH'({way_sel[WAY_SEL_BITS-1:0], line_idx}); + assign data_addr = merge_write ? DATA_ADDR_WIDTH'({mask_way[WAY_SEL_BITS-1:0], mask_line}) + : DATA_ADDR_WIDTH'({way_sel[WAY_SEL_BITS-1:0], line_idx}); end else begin : g_no_way_addr - `UNUSED_VAR (way_sel) - assign data_addr = line_idx; + `UNUSED_VAR ({way_sel, mask_way}) + assign data_addr = merge_write ? mask_line : line_idx; end - // Per-byte dirty mask (writeback only). Unlike the main data array this is - // kept as one line-indexed array per way (evict way selected at S1): it is - // narrow and off the load path, and -- crucially -- it must be cleared for - // every way during the line-only init walk, which the way-folded data - // layout cannot do in a single pass. + // Per-byte dirty mask (writeback only), one mask word per {way, line}. It + // is read and updated only at the commit stage, so each access sees every + // older update with no bypass. A line's mask is cleared when it is + // installed, which leaves nothing for the reset walk to clear. if (DIRTY_BYTES != 0) begin : g_dirty_bytes - wire [`CS_WORDS_PER_LINE-1:0][WORD_SIZE-1:0] write_mask; - for (genvar i = 0; i < `CS_WORDS_PER_LINE; ++i) begin : g_write_mask - wire word_en = (`CS_WORDS_PER_LINE == 1) || (word_idx == i); - assign write_mask[i] = write_byteen & {WORD_SIZE{word_en}}; + wire [`CS_WORDS_PER_LINE-1:0][WORD_SIZE-1:0] set_word; + for (genvar i = 0; i < `CS_WORDS_PER_LINE; ++i) begin : g_set_word + wire word_en = (`CS_WORDS_PER_LINE == 1) || (mask_word == i); + assign set_word[i] = mask_byteen & {WORD_SIZE{mask_set && word_en}}; end - - // A sector refill must clear only the installed sector's mask: the other - // resident sectors may hold dirty bytes not yet written back. - wire [`CS_SECTORS_PER_LINE-1:0][SECTOR_SIZE-1:0] fill_clr_mask; - for (genvar i = 0; i < `CS_SECTORS_PER_LINE; ++i) begin : g_fill_clr_mask - wire sec_en = ~refill || (`CS_SECTORS_PER_LINE == 1) - || (sector_idx == `UP(`CS_SECTOR_SEL_BITS)'(i)); - assign fill_clr_mask[i] = {SECTOR_SIZE{fill && sec_en}}; + // A sector refill clears only the installed sector: the other resident + // sectors may hold dirty bytes not yet written back. + wire [`CS_SECTORS_PER_LINE-1:0][SECTOR_SIZE-1:0] clr_sector; + for (genvar i = 0; i < `CS_SECTORS_PER_LINE; ++i) begin : g_clr_sector + wire sec_en = (`CS_SECTORS_PER_LINE == 1) || (mask_sector == `UP(`CS_SECTOR_SEL_BITS)'(i)); + assign clr_sector[i] = {SECTOR_SIZE{mask_clr_line || (mask_clr_sector && sec_en)}}; end - - wire [NUM_WAYS-1:0][LINE_SIZE-1:0] byteen_rdata; - for (genvar i = 0; i < NUM_WAYS; ++i) begin : g_byteen_store - wire [LINE_SIZE-1:0] byteen_wdata = {LINE_SIZE{write}}; // only asserted on writes - wire [LINE_SIZE-1:0] byteen_wren = {LINE_SIZE{init || flush}} | fill_clr_mask | write_mask; - wire byteen_write = ((fill || flush) && ((NUM_WAYS == 1) || (evict_way == i))) - || (write && tag_matches[i]) - || init; - wire byteen_read = fill || flush; - - // The dirty mask has 1-bit (per-byte) write granularity, which - // block RAM cannot do -- inferring BRAM shatters it into dozens of - // tiny RAMB18 per way. It is small, so map it to distributed RAM. - VX_sp_ram #( - .DATAW (LINE_SIZE), - .WRENW (LINE_SIZE), - .SIZE (`CS_LINES_PER_BANK), - .OUT_REG (1), - .LUTRAM (1), - .RDW_MODE ("R") - ) byteen_store ( - .clk (clk), - .reset (reset), - .read (byteen_read), - .write (byteen_write), - .wren (byteen_wren), - .addr (line_idx), - .wdata (byteen_wdata), - .rdata (byteen_rdata[i]) - ); + wire [LINE_SIZE-1:0] set_mask = set_word; + wire [LINE_SIZE-1:0] clr_mask = clr_sector; + + wire [DATA_ADDR_WIDTH-1:0] mask_addr; + if (NUM_WAYS > 1) begin : g_mask_way_addr + assign mask_addr = DATA_ADDR_WIDTH'({mask_way[WAY_SEL_BITS-1:0], mask_line}); + end else begin : g_mask_line_addr + assign mask_addr = mask_line; end - assign evict_byteen = byteen_rdata[way_idx_r]; + + // A whole-word read-modify-write shares one write enable across the + // mask, so it maps onto wide distributed RAM rather than one cell per + // byte (or a shattered block RAM). + wire [LINE_SIZE-1:0] mask_rdata; + VX_sp_ram #( + .DATAW (LINE_SIZE), + .SIZE (DATA_RAM_DEPTH), + .OUT_REG (0), + .LUTRAM (1), + .RDW_MODE ("W"), + .INIT_ENABLE (1), + .INIT_VALUE (0) + ) byteen_store ( + .clk (clk), + .reset (reset), + .read (1'b1), + .write (mask_write), + .wren (1'b1), + .addr (mask_addr), + .wdata ((mask_rdata & ~clr_mask) | set_mask), + .rdata (mask_rdata) + ); + assign evict_byteen = mask_rdata; end else begin : g_no_dirty_bytes - `UNUSED_VAR (init) - `UNUSED_VAR (refill) - `UNUSED_VAR (flush) - `UNUSED_VAR (way_idx_r) + `UNUSED_VAR ({mask_write, mask_set, mask_clr_line, mask_clr_sector}) + `UNUSED_VAR ({mask_line, mask_word, mask_byteen, mask_sector}) assign evict_byteen = '1; // update whole line end @@ -180,14 +189,19 @@ module VX_cache_data import VX_gpu_pkg::*; #( if (WRITE_ENABLE) begin : g_wren // fill writes the fetched sector's slices; a store writes only the hit slice. - assign slice_write = (fill && fill_sec_en) || (write && hit_any && word_en); - assign slice_wren = fill ? {WORD_SIZE{1'b1}} : write_byteen; - assign slice_wdata = fill ? fill_data[s] : write_word; + wire merge_sec_en = (`CS_SECTORS_PER_LINE == 1) + || (mask_sector == `UP(`CS_SECTOR_SEL_BITS)'(SLICE_SECTOR)); + assign slice_write = (fill_write && fill_sec_en) || (write && hit_any && word_en) + || (merge_write && merge_sec_en); + assign slice_wren = merge_write ? ~evict_byteen[s * WORD_SIZE +: WORD_SIZE] + : (fill ? {WORD_SIZE{1'b1}} : write_byteen); + assign slice_wdata = (fill || merge_write) ? fill_data[s] : write_word; end else begin : g_no_wren `UNUSED_VAR (write) `UNUSED_VAR (write_word) `UNUSED_VAR (write_byteen) `UNUSED_VAR (hit_any) + `UNUSED_VAR ({fill_write, merge_write}) assign slice_write = fill && fill_sec_en; assign slice_wren = 1'b1; assign slice_wdata = fill_data[s]; diff --git a/hw/rtl/cache/VX_cache_mshr.sv b/hw/rtl/cache/VX_cache_mshr.sv index e088fe95fc..c8e66e0c10 100644 --- a/hw/rtl/cache/VX_cache_mshr.sv +++ b/hw/rtl/cache/VX_cache_mshr.sv @@ -55,6 +55,7 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( parameter AMO_ENABLE = 0, // Per-entry AMO tracking (probe outputs) parameter AMO_PASSTHRU = 0, // Non-LLC passthrough: AMO entries never coalesce parameter WORD_SEL_WIDTH = 1, // word-in-line select width (order-hazard granularity) + parameter LAZY_FETCH = 0, // Chains may park and refetch a partial sector parameter MSHR_ADDR_WIDTH = `LOG2UP(MSHR_SIZE) ) ( @@ -73,6 +74,10 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( output wire [`CS_LINE_ADDR_WIDTH-1:0] fill_addr, // sector of the entry that initiated this fill (which sector to install) output wire [`UP(`CS_SECTOR_SEL_BITS)-1:0] fill_sector, + // the entry that initiated this fill refetches a resident partial sector + output wire fill_refetch, + // ... after a refetch found it evicted: the sector it re-reads stayed clean + output wire fill_retry, // probe: pending requests for `probe_addr`'s line, split by type. // probe_pending_ld : a non-AMO (line-filling) request is pending — @@ -96,9 +101,16 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( output wire dequeue_valid, output wire [`CS_LINE_ADDR_WIDTH-1:0] dequeue_addr, output wire dequeue_rw, + output wire dequeue_reads, // the head reads its sector (load or AMO) output wire [DATA_WIDTH-1:0] dequeue_data, output wire [MSHR_ADDR_WIDTH-1:0] dequeue_id, input wire dequeue_ready, + // park the chain at its head instead of consuming it: the entry stays + // allocated (it becomes the root of a refetch) and the walk stops. + input wire dequeue_park, + input wire dequeue_park_retry, + // withhold the head this cycle (registered-only inputs) + input wire dequeue_block, // allocate input wire allocate_valid, @@ -107,6 +119,7 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( input wire [WORD_SEL_WIDTH-1:0] allocate_word_idx, input wire allocate_rw, input wire allocate_is_amo, // AMO: never coalesce + input wire allocate_refetch, // reads a resident partial sector input wire [DATA_WIDTH-1:0] allocate_data, output wire [MSHR_ADDR_WIDTH-1:0] allocate_id, output wire allocate_pending, @@ -154,6 +167,7 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( // amo_mask is the per-entry AMO flag, zero when AMO_ENABLE=0 so the // mask folds out of addr_matches and the probe logic disappears. wire [MSHR_SIZE-1:0] amo_mask; + wire [MSHR_SIZE-1:0] amo_entries; reg allocate_rdy, allocate_rdy_n; reg [MSHR_ADDR_WIDTH-1:0] allocate_id_r, allocate_id_n; @@ -165,6 +179,8 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( wire allocate_fire = allocate_valid && allocate_ready; wire dequeue_fire = dequeue_valid && dequeue_ready; + wire dequeue_park_fire = (LAZY_FETCH != 0) && dequeue_fire && dequeue_park; + wire dequeue_free = dequeue_fire && ~dequeue_park_fire; wire [MSHR_SIZE-1:0] addr_matches; for (genvar i = 0; i < MSHR_SIZE; ++i) begin : g_addr_matches @@ -179,7 +195,7 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( // and hits the line the releasing entry just installed. assign addr_matches[i] = valid_table[i] && (addr_table[i] == allocate_addr) && (sector_table[i] == allocate_sector) && ~amo_mask[i] - && ~(dequeue_fire && (dequeue_id == MSHR_ADDR_WIDTH'(i))) + && ~(dequeue_free && (dequeue_id == MSHR_ADDR_WIDTH'(i))) && ~(finalize_valid && finalize_is_release && (finalize_id == MSHR_ADDR_WIDTH'(i))); end @@ -193,7 +209,7 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( // the next id never re-selects the slot being allocated this cycle. wire [MSHR_SIZE-1:0] dequeue_free_oh, finalize_free_oh, allocate_set_oh; for (genvar i = 0; i < MSHR_SIZE; ++i) begin : g_free_delta - assign dequeue_free_oh[i] = dequeue_fire && (dequeue_id == MSHR_ADDR_WIDTH'(i)); + assign dequeue_free_oh[i] = dequeue_free && (dequeue_id == MSHR_ADDR_WIDTH'(i)); assign finalize_free_oh[i] = finalize_valid && finalize_is_release && (finalize_id == MSHR_ADDR_WIDTH'(i)); assign allocate_set_oh[i] = allocate_fire && (allocate_id == MSHR_ADDR_WIDTH'(i)); end @@ -229,7 +245,9 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( dequeue_id_n = fill_id; end - if (dequeue_fire) begin + if (dequeue_park_fire) begin + dequeue_val_n = 0; + end else if (dequeue_fire) begin valid_table_n[dequeue_id] = 0; if (next_table[dequeue_id]) begin dequeue_id_n = next_index[dequeue_id]; @@ -343,6 +361,34 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( assign fill_addr = addr_table[fill_id]; assign fill_sector = sector_table[fill_id]; + if (LAZY_FETCH != 0) begin : g_refetch + reg [MSHR_SIZE-1:0] refetch_table, retry_table; + always @(posedge clk) begin + if (allocate_fire) begin + refetch_table[allocate_id] <= allocate_refetch; + retry_table[allocate_id] <= 1'b0; + end + if (dequeue_park_fire) begin + refetch_table[dequeue_id] <= 1'b1; + retry_table[dequeue_id] <= dequeue_park_retry; + end + end + assign fill_refetch = refetch_table[fill_id]; + assign fill_retry = retry_table[fill_id]; + // registered with the pointer so the park decision stays off the + // payload read + reg dequeue_reads_r; + always @(posedge clk) begin + dequeue_reads_r <= ~write_table[dequeue_id_n] || amo_entries[dequeue_id_n]; + end + assign dequeue_reads = dequeue_reads_r; + end else begin : g_no_refetch + assign fill_refetch = 1'b0; + assign fill_retry = 1'b0; + assign dequeue_reads = 1'b0; + `UNUSED_VAR ({allocate_refetch, dequeue_park, dequeue_park_retry, dequeue_park_fire, amo_entries}) + end + if (AMO_ENABLE != 0) begin : g_amo reg [MSHR_SIZE-1:0] amo_table; // An entry only participates in the pending-AMO probe once it has @@ -375,6 +421,7 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( // in arrival order like any request; the probe outputs alone provide // the same-line ordering guard. assign amo_mask = (AMO_PASSTHRU != 0) ? amo_table : '0; + assign amo_entries = amo_table; // At the LLC no request links behind a pending AMO: the bank defers // every request to a line while an AMO to it waits here. `RUNTIME_ASSERT (~((AMO_PASSTHRU == 0) && finalize_valid && ~finalize_is_release && finalize_is_pending @@ -414,6 +461,7 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( assign persist_amo = finalize_valid && ~finalize_is_release && amo_table[finalize_id]; end else begin : g_no_amo assign amo_mask = '0; + assign amo_entries = '0; assign probe_pending_ld = 1'b0; assign probe_pending_amo = 1'b0; assign {probe_any_ld, probe_any_amo, persist_ld, persist_amo} = '0; @@ -449,7 +497,7 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( // of an older chained read of the same word. assign allocate_pending_ovl = |(addr_matches & word_matches); - assign dequeue_valid = dequeue_val; + assign dequeue_valid = dequeue_val && ~((LAZY_FETCH != 0) && dequeue_block); // Sampled with dequeue_id_n, alongside the payload in mshr_store, rather // than read live at dequeue_id_r a cycle later. The entry's slot can be // freed and reallocated while the pointer still names it -- the free set diff --git a/hw/rtl/cache/VX_cache_tags.sv b/hw/rtl/cache/VX_cache_tags.sv index 8f208a219a..5ecfc5c0be 100644 --- a/hw/rtl/cache/VX_cache_tags.sv +++ b/hw/rtl/cache/VX_cache_tags.sv @@ -35,6 +35,7 @@ module VX_cache_tags import VX_gpu_pkg::*; #( parameter WORD_SIZE = 1, // Size of a word in bytes parameter SECTOR_SIZE = LINE_SIZE, // Size of a sector in bytes (fill/eviction granule); = LINE_SIZE => 1 sector parameter WRITEBACK = 0, // Enable cache writeback + parameter LAZY_FETCH = 0, // Track per-sector partial (allocated-by-store) state parameter AMO_ENABLE = 0 // Enable the AMO-passthrough line invalidate (non-LLC banks only) ) ( input wire clk, @@ -48,6 +49,10 @@ module VX_cache_tags import VX_gpu_pkg::*; #( input wire read, input wire write, input wire invalidate, // clear valid on the hit way + // the fill installs no data: the sector becomes partial (cleared otherwise) + input wire fill_nodata, + // a refetch completes its resident partial sector: clear its partial bit + input wire refetch, input wire [`CS_LINE_SEL_BITS-1:0] line_idx, input wire [`CS_LINE_SEL_BITS-1:0] line_idx_n, input wire [`CS_TAG_SEL_BITS-1:0] line_tag, @@ -70,7 +75,9 @@ module VX_cache_tags import VX_gpu_pkg::*; #( // per-sector dirty vector of the evict way (drives the multi-beat per-sector // writeback); a single bit == evict_dirty when 1 sector/line. output wire [`CS_SECTORS_PER_LINE-1:0] evict_dirty_mask, - output wire [`CS_TAG_SEL_BITS-1:0] evict_tag + output wire [`CS_TAG_SEL_BITS-1:0] evict_tag, + // the requested sector is resident but partial (a load/AMO to it refetches) + output wire partial_match ); // tag store: tag only (valid + dirty decoupled into side LUTRAMs) localparam SEC = `CS_SECTORS_PER_LINE; @@ -82,6 +89,8 @@ module VX_cache_tags import VX_gpu_pkg::*; #( wire [NUM_WAYS-1:0][`CS_TAG_SEL_BITS-1:0] read_tag; wire [NUM_WAYS-1:0][SEC-1:0] read_valid; wire [NUM_WAYS-1:0][SEC-1:0] read_dirty; + wire [NUM_WAYS-1:0][SEC-1:0] read_partial; + wire [NUM_WAYS-1:0] partial_hit; if (WRITEBACK) begin : g_evict_tag_wb assign evict_dirty = (| read_dirty[evict_way]); // dirty if any sector dirty @@ -117,6 +126,13 @@ module VX_cache_tags import VX_gpu_pkg::*; #( wire [NUM_WAYS-1:0][SEC-1:0] dirty_wdata; wire [NUM_WAYS-1:0][SEC-1:0] dirty_rdata; + // Per-sector partial store (lazy fetch only): a fill writes its sector's + // bit, set by a data-less fill and cleared by a data fill. The bit is only + // meaningful while the sector is valid, so no other operation touches it. + wire [NUM_WAYS-1:0][SEC-1:0] partial_wren; + wire [NUM_WAYS-1:0][SEC-1:0] partial_wdata; + wire [NUM_WAYS-1:0][SEC-1:0] partial_rdata; + for (genvar i = 0; i < NUM_WAYS; ++i) begin : g_way_decode wire way_en = (NUM_WAYS == 1) || (evict_way == i); wire do_init = init; // init all ways @@ -163,8 +179,12 @@ module VX_cache_tags import VX_gpu_pkg::*; #( // hit: tag match AND the requested sector is valid. read_valid folds in // a same-set fill from the previous cycle (see the bypass above), so - // this compare is exact even on the fill's write edge. - wire raw_hit = read_valid[i][sector_idx] && tag_eq; + // this compare is exact even on the fill's write edge. Whether a partial + // sector serves the lookup is decided outside, off the array enables. + wire sector_valid = read_valid[i][sector_idx] && tag_eq; + wire sector_partial = (LAZY_FETCH != 0) && read_partial[i][sector_idx]; + wire raw_hit = sector_valid; + assign partial_hit[i] = sector_valid && sector_partial; // AMO passthrough invalidate: clear the requested sector's valid. wire do_inval = (AMO_ENABLE != 0) && invalidate && raw_hit; @@ -224,6 +244,23 @@ module VX_cache_tags import VX_gpu_pkg::*; #( assign read_dirty[i] = {SEC{1'b0}}; end + if (LAZY_FETCH != 0) begin : g_partial + // Same access pattern as the valid store, so the way_filled bypass + // corrects a same-set fill from the previous cycle the same way. A + // refetch's clear needs none: nothing is looked up right behind it. + wire rdw_nodata; + `BUFFER_EX(rdw_nodata, fill_nodata, ~stall, 1, 1); + wire do_refetch = refetch && sector_valid; + assign partial_wren[i] = (do_fill || do_refetch) ? sec_oh : {SEC{1'b0}}; + assign partial_wdata[i] = {SEC{do_fill && fill_nodata}}; + assign read_partial[i] = way_filled ? ((partial_rdata[i] & ~rdw_sec_oh) | (rdw_sec_oh & {SEC{rdw_nodata}})) + : partial_rdata[i]; + end else begin : g_no_partial + assign partial_wren[i] = '0; + assign partial_wdata[i] = '0; + assign read_partial[i] = {SEC{1'b0}}; + end + assign tag_matches[i] = raw_hit; // line resident in this way: tag matches and at least one sector valid // (a same-cycle-prior fill is folded into tag_eq/read_valid). @@ -304,4 +341,30 @@ module VX_cache_tags import VX_gpu_pkg::*; #( `UNUSED_VAR ({dirty_wren, dirty_wdata, dirty_rdata}) end + if (LAZY_FETCH != 0) begin : g_partial_store + VX_dp_ram #( + .DATAW (NUM_WAYS * SEC), + .WRENW (NUM_WAYS * SEC), + .SIZE (`CS_LINES_PER_BANK), + .OUT_REG (1), + .LUTRAM (1), + .RDW_MODE ("R") + ) partial_store ( + .clk (clk), + .reset (reset), + .read (~stall), + .write (| partial_wren), + .wren (partial_wren), + .waddr (line_idx), + .raddr (line_idx_n), + .wdata (partial_wdata), + .rdata (partial_rdata) + ); + assign partial_match = (| partial_hit); + end else begin : g_no_partial_store + assign partial_rdata = '0; + assign partial_match = 1'b0; + `UNUSED_VAR ({partial_wren, partial_wdata, partial_rdata, partial_hit, read_partial, fill_nodata, refetch}) + end + endmodule diff --git a/hw/rtl/cache/VX_cache_wrap.sv b/hw/rtl/cache/VX_cache_wrap.sv index e9f34d17bc..0f87a80e45 100644 --- a/hw/rtl/cache/VX_cache_wrap.sv +++ b/hw/rtl/cache/VX_cache_wrap.sv @@ -42,6 +42,8 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( parameter DIRTY_BYTES = 0, // Enable dirty bytes on writeback + parameter LAZY_FETCH = 0, // Write misses allocate without a fill (write-validate) + parameter REPL_POLICY = `CS_REPL_FIFO, // Replacement policy parameter TAG_WIDTH = UUID_WIDTH + 1, // core request tag size @@ -145,6 +147,7 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( .WRITE_ENABLE (WRITE_ENABLE), .WRITEBACK (WRITEBACK), .DIRTY_BYTES (DIRTY_BYTES), + .LAZY_FETCH (LAZY_FETCH), .REPL_POLICY (REPL_POLICY), .CRSQ_SIZE (CRSQ_SIZE), .MSHR_SIZE (MSHR_SIZE), diff --git a/hw/unittest/cache/VX_cache_top.sv b/hw/unittest/cache/VX_cache_top.sv index b75215a485..5d797998c9 100644 --- a/hw/unittest/cache/VX_cache_top.sv +++ b/hw/unittest/cache/VX_cache_top.sv @@ -52,6 +52,9 @@ module VX_cache_top import VX_gpu_pkg::*; #( // Enable dirty bytes on writeback parameter DIRTY_BYTES = `VX_CFG_DCACHE_DIRTYBYTES, + // Write misses allocate without a fill (write-validate) + parameter LAZY_FETCH = `VX_CFG_DCACHE_LAZY_FETCH, + // Bank pipeline depth (capacity-scaled) parameter LATENCY = `VX_CFG_DCACHE_LATENCY, @@ -183,6 +186,7 @@ module VX_cache_top import VX_gpu_pkg::*; #( .WRITE_ENABLE (WRITE_ENABLE), .WRITEBACK (WRITEBACK), .DIRTY_BYTES (DIRTY_BYTES), + .LAZY_FETCH (LAZY_FETCH), .AMO_ENABLE (AMO_ENABLE), .IS_LLC (IS_LLC), .BANK_HASH (BANK_HASH), diff --git a/hw/unittest/gfx/VX_gfx_top.sv b/hw/unittest/gfx/VX_gfx_top.sv index aae201f6e1..ce4ef6876d 100644 --- a/hw/unittest/gfx/VX_gfx_top.sv +++ b/hw/unittest/gfx/VX_gfx_top.sv @@ -758,6 +758,7 @@ module VX_gfx_top import VX_gpu_pkg::* .WRITE_ENABLE (1), .WRITEBACK (`VX_CFG_L2_WRITEBACK), .DIRTY_BYTES (`VX_CFG_L2_DIRTYBYTES), + .LAZY_FETCH (`VX_CFG_L2_LAZY_FETCH), .REPL_POLICY (`VX_CFG_L2_REPL_POLICY), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3), diff --git a/hw/unittest/tensor/VX_tensor_top.sv b/hw/unittest/tensor/VX_tensor_top.sv index 629ab576c4..abd8b0b526 100644 --- a/hw/unittest/tensor/VX_tensor_top.sv +++ b/hw/unittest/tensor/VX_tensor_top.sv @@ -385,6 +385,7 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( .WRITE_ENABLE (1), .WRITEBACK (`VX_CFG_DCACHE_WRITEBACK), .DIRTY_BYTES (`VX_CFG_DCACHE_DIRTYBYTES), + .LAZY_FETCH (`VX_CFG_DCACHE_LAZY_FETCH), .REPL_POLICY (`VX_CFG_DCACHE_REPL_POLICY), .NC_ENABLE (1), .CORE_OUT_BUF (3), @@ -504,6 +505,7 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( .WRITE_ENABLE (1), .WRITEBACK (`VX_CFG_L2_WRITEBACK), .DIRTY_BYTES (`VX_CFG_L2_DIRTYBYTES), + .LAZY_FETCH (`VX_CFG_L2_LAZY_FETCH), .REPL_POLICY (`VX_CFG_L2_REPL_POLICY), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3), diff --git a/hw/unittest/vm/VX_vm_top.sv b/hw/unittest/vm/VX_vm_top.sv index 54e4c29b15..056711d448 100644 --- a/hw/unittest/vm/VX_vm_top.sv +++ b/hw/unittest/vm/VX_vm_top.sv @@ -305,6 +305,7 @@ module VX_vm_top import VX_gpu_pkg::*, VX_tlb_pkg::*; #( .WRITE_ENABLE (1), .WRITEBACK (`VX_CFG_DCACHE_WRITEBACK), .DIRTY_BYTES (`VX_CFG_DCACHE_DIRTYBYTES), + .LAZY_FETCH (`VX_CFG_DCACHE_LAZY_FETCH), .REPL_POLICY (`VX_CFG_DCACHE_REPL_POLICY), .NC_ENABLE (1), .CORE_OUT_BUF (3), @@ -616,6 +617,7 @@ module VX_vm_top import VX_gpu_pkg::*, VX_tlb_pkg::*; #( .WRITE_ENABLE (1), .WRITEBACK (`VX_CFG_L2_WRITEBACK), .DIRTY_BYTES (`VX_CFG_L2_DIRTYBYTES), + .LAZY_FETCH (`VX_CFG_L2_LAZY_FETCH), .REPL_POLICY (`VX_CFG_L2_REPL_POLICY), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3), diff --git a/sim/simx/cluster.cpp b/sim/simx/cluster.cpp index 0978067d9e..95b3669a3d 100644 --- a/sim/simx/cluster.cpp +++ b/sim/simx/cluster.cpp @@ -86,6 +86,7 @@ class Cluster::Impl { (VX_CFG_L2_ENABLED != 0) && (VX_CFG_L3_ENABLED == 0), // is_llc (VX_CFG_L2_BANK_HASH != 0) && (VX_CFG_L2_NUM_BANKS > 1), // bank_hash L2_SHARED_LLC, // shared_llc + VX_CFG_L2_LAZY_FETCH != 0, // lazy_fetch }); diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index 1a647e06ee..72124b8e3d 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -169,17 +169,24 @@ struct params_t { struct sector_t { bool valid; bool dirty; + bool partial; // allocated by a store: only dirty_mask bytes are known uint64_t dirty_mask; // per-byte dirty bits; the writeback byte-enable std::shared_ptr data; // sector bytes (= one mem_block) void reset() { valid = false; dirty = false; + partial = false; dirty_mask = 0; data.reset(); } }; +// A store may hit a partial sector; a load or AMO needs every byte. +static inline bool sector_hit(const sector_t& sec, bool needs_data) { + return sec.valid && !(needs_data && sec.partial); +} + struct line_t { uint64_t tag; uint32_t lru_ctr; @@ -369,7 +376,8 @@ struct bank_req_t { Fill = 1, Replay = 2, Core = 3, - AmoProbe = 4 // non-LLC AMO passthrough: probe-and-invalidate then forward + AmoProbe = 4, // non-LLC AMO passthrough: probe-and-invalidate then forward + Park = 5 // chain head parked on a partial sector: issues its refetch }; uint64_t addr; @@ -386,6 +394,13 @@ struct bank_req_t { // emit another response — only run sector_merge. bool skip_core_rsp; + // Fill flavors under lazy fetch: a data-less allocation for a write-miss + // root, or a refetch of a resident partial sector (merges under its dirty + // bytes). + bool nodata; + bool refetch; + bool refetch_retry; // refetch re-issued after its sector was evicted + // TLM data: // For Core writes: incoming write data + byteen. // For Fill: captured fill data from below (mem_rsp.data). @@ -425,6 +440,9 @@ struct bank_req_t { write = false; op = MemOp::LD; skip_core_rsp = false; + nodata = false; + refetch = false; + refetch_retry = false; data.reset(); byteen = 0; amo_cmp = 0; @@ -451,6 +469,8 @@ struct mshr_entry_t { uint32_t line_id; uint64_t seq; // enqueue order; replays drain oldest-first to preserve // program order among coalesced same-line accesses. + bool refetch; // root of a fill that refetches a resident partial sector + bool refetch_retry; // ... re-issued after the previous refetch found it evicted mshr_entry_t() { this->reset(); @@ -463,6 +483,8 @@ struct mshr_entry_t { sector_id = 0; line_id = 0; seq = 0; + refetch = false; + refetch_retry = false; } }; @@ -549,7 +571,8 @@ class MSHR { } // Enqueue a new core request and return the allocated entry id. - int enqueue(const bank_req_t &bank_req, uint32_t set_id, uint64_t addr_tag, uint32_t sector_id) { + int enqueue(const bank_req_t &bank_req, uint32_t set_id, uint64_t addr_tag, uint32_t sector_id, + bool refetch = false) { assert(bank_req.type == bank_req_t::Core); for (uint32_t i = 0, n = entries_.size(); i < n; ++i) { auto &entry = entries_.at(i); @@ -560,6 +583,8 @@ class MSHR { entry.sector_id = sector_id; entry.line_id = 0; // victim is selected at Fill time entry.seq = bank_req.adm_seq; + entry.refetch = refetch; + entry.refetch_retry = false; ++size_; return i; } @@ -600,6 +625,34 @@ class MSHR { ++ready_reqs_; } + // Park the chain at its head: the oldest ready replay and the rest of its + // (set,tag,sector) chain return to waiting for a fill, which the head now + // roots as a refetch. Returns the head's id. + uint32_t park_next_replay(bool retry) { + int head = -1; + for (int i = 0, n = (int)entries_.size(); i < n; ++i) { + const auto &entry = entries_.at(i); + if (entry.bank_req.type != bank_req_t::Replay) { + continue; + } + if (head == -1 || entry.seq < entries_.at(head).seq) { + head = i; + } + } + assert(head != -1); + auto &root = entries_.at(head); + for (auto &entry : entries_) { + if (entry.bank_req.type == bank_req_t::Replay && entry.set_id == root.set_id + && entry.addr_tag == root.addr_tag && entry.sector_id == root.sector_id) { + entry.bank_req.type = bank_req_t::Core; + --ready_reqs_; + } + } + root.refetch = true; + root.refetch_retry = retry; + return head; + } + // Peek the entry dequeue() would return next (oldest ready replay), or // nullptr. Lets the fill-forward drain inspect the chain head without // consuming it. @@ -679,11 +732,14 @@ class CacheBank : public SimObject { const Cache::Config &config, const params_t ¶ms, uint32_t bank_id) - : SimObject(ctx, name), core_req_in(this), core_rsp_out(this), mem_req_out(this), mem_rsp_in(this, config.mshr_size + AMO_PASSTHRU_CAP), config_(config), params_(params), bank_id_(bank_id), sets_(params.sets_per_bank, set_t(params.lines_per_set, params.sectors_per_line)), mshr_(config.mshr_size), pipe_req_(TFifo::Create("", config.latency, std::max(2u, config.latency))), rand_ctr_(0), adm_seq_ctr_(0) + : SimObject(ctx, name), core_req_in(this), core_rsp_out(this), mem_req_out(this), mem_rsp_in(this, config.mshr_size + AMO_PASSTHRU_CAP), config_(config), params_(params), bank_id_(bank_id), sets_(params.sets_per_bank, set_t(params.lines_per_set, params.sectors_per_line)), mshr_(config.mshr_size), pipe_req_(TFifo::Create("", config.latency, std::max(2u, config.latency))), lazy_q_(TFifo::Create("", 1, lazyq_size(config))), rand_ctr_(0), adm_seq_ctr_(0) #if VX_CFG_EXT_A_ENABLED , amo_unit_(__MAX(2u, (uint32_t)VX_CFG_AMO_RS_SIZE)) #endif { + // Lazy allocation leaves unfetched bytes in the line: only a write-back + // LLC can hold them until a read merges or a writeback masks them out. + assert(!config.lazy_fetch || (config.write_back && config.is_llc)); this->on_reset(); } @@ -740,6 +796,12 @@ class CacheBank : public SimObject { fwd_sector_ = 0; fwd_data_.reset(); crsp_sent_ = false; + chain_partial_ = false; + chain_retry_ = false; + chain_set_ = 0; + chain_tag_ = 0; + chain_sector_ = 0; + refetch_hold_ = false; flushing_ = false; flush_set_idx_ = 0; flush_way_idx_ = 0; @@ -788,7 +850,7 @@ class CacheBank : public SimObject { // the mem_latency accumulation exact. flush_begin() re-arms explicitly; // any packet reserved toward core_req_in/mem_rsp_in re-arms on arrival. if (this->core_req_in.size() == 0 && this->mem_rsp_in.size() == 0 - && pipe_req_->size() == 0 && !mshr_.has_ready_reqs() + && pipe_req_->size() == 0 && lazy_q_->size() == 0 && !mshr_.has_ready_reqs() && !fwd_active_ && !flushing_ && pending_fill_reqs_ == 0) { this->tick_sleep(); } @@ -812,6 +874,26 @@ class CacheBank : public SimObject { // drain completed the head this tick, this slot goes to new core // requests instead if (mshr_.has_ready_reqs() && !fwd_consumed) { + // A refetch fill learns whether its sector is still resident when it + // installs; its chain waits for that answer. + if (refetch_hold_) { + return; + } + // A chain whose sector is partial cannot serve a load or AMO: park at + // that head and refetch the sector; the rest of the chain waits. + if (chain_partial_) { + const auto *head = mshr_.peek_next_replay(); + if (head != nullptr + && head->set_id == chain_set_ && head->addr_tag == chain_tag_ && head->sector_id == chain_sector_ + && !head->bank_req.write) { + bank_req_t park_req = head->bank_req; + park_req.mshr_id = mshr_.park_next_replay(chain_retry_); + park_req.type = bank_req_t::Park; + pipe_req_->push(park_req); + DT(3, this->name() << " mshr-park: " << park_req); + return; + } + } bank_req_t bank_req; mshr_.dequeue(&bank_req); pipe_req_->push(bank_req); @@ -873,21 +955,57 @@ class CacheBank : public SimObject { bank_req.uuid = root_peek.bank_req.uuid; bank_req.mshr_id = mshr_id; bank_req.data = mem_rsp.data; + bank_req.refetch = root_peek.refetch; + bank_req.refetch_retry = root_peek.refetch_retry; pipe_req_->push(bank_req); // The chain wakes at accept: its replays enter the pipe behind the fill, // so they reach the array after the install. The forward window answers - // the chain's leading reads from the staged sector meanwhile. + // the chain's leading reads from the staged sector meanwhile; a refetch + // cannot, its staged sector lacks the dirty bytes the install merges in. mshr_.replay(mshr_id); - fwd_active_ = true; - fwd_set_ = set_id; - fwd_tag_ = addr_tag; - fwd_sector_ = sector_id; - fwd_data_ = mem_rsp.data; + this->setChain(set_id, addr_tag, sector_id, false); + if (bank_req.refetch) { + refetch_hold_ = true; + } else { + fwd_active_ = true; + fwd_set_ = set_id; + fwd_tag_ = addr_tag; + fwd_sector_ = sector_id; + fwd_data_ = mem_rsp.data; + } DT(3, this->name() << " fill-rsp: " << mem_rsp); this->mem_rsp_in.pop(); --pending_fill_reqs_; return; } + } else if (!lazy_q_->empty() && !fwd_active_) { + // Lazy allocation of a write-miss root: a data-less fill through the + // same path, below memory fills in priority. + uint32_t mshr_id = lazy_q_->peek(); + const auto &root_peek = mshr_.peek(mshr_id); + const uint32_t set_id = root_peek.set_id; + const uint64_t addr_tag = root_peek.addr_tag; + const uint32_t sector_id = root_peek.sector_id; + if (this->pipeLineBusy(set_id, addr_tag)) { + fill_held = true; + held_set_id = set_id; + held_addr_tag = addr_tag; + } else { + bank_req_t bank_req; + bank_req.reset(); + bank_req.type = bank_req_t::Fill; + bank_req.addr = params_.mem_addr_sector(bank_id_, set_id, addr_tag, sector_id); + bank_req.hart_id = root_peek.bank_req.hart_id; + bank_req.uuid = root_peek.bank_req.uuid; + bank_req.mshr_id = mshr_id; + bank_req.nodata = true; + pipe_req_->push(bank_req); + mshr_.replay(mshr_id); + this->setChain(set_id, addr_tag, sector_id, true); + DT(3, this->name() << " lazy-fill: " << bank_req); + lazy_q_->pop(); + return; + } } // 3) core request @@ -954,6 +1072,11 @@ class CacheBank : public SimObject { ++perf_stats_.mshr_stalls; return; } + // Any admitted request may become a write-miss root: the lazy queue + // keeps room for every request already in the pipe. + if (config_.lazy_fetch && lazy_q_->size() >= lazyq_size(config_) - config_.latency) { + return; + } // Admission-time coalescing: a request whose (set,tag,sector) chain is // still awaiting its fill must join it here, not enter the pipe as an @@ -1076,6 +1199,28 @@ class CacheBank : public SimObject { int resident_id = set.find_resident(addr_tag); const bool is_refill = (resident_id != -1); + // Only a refetch finds its sector resident (partial): it merges. A refetch + // whose sector was evicted meanwhile may carry data older than that + // eviction's writeback, so it installs nothing and its chain parks again; + // the retry then fills like any miss (an absent sector cannot be dirtied). + const bool sector_resident = is_refill && set.lines.at(resident_id).sectors.at(sector_id).valid; + const bool refetch_first = bank_req.refetch && !bank_req.refetch_retry; + assert(!sector_resident || refetch_first); + if (refetch_first) { + chain_partial_ = !sector_resident; + chain_retry_ = !sector_resident; + refetch_hold_ = false; + if (sector_resident) { + // Memory supplies the bytes no store wrote; the sector stays dirty + // in exactly the stored bytes, which are not in memory yet. + auto &sec = set.lines.at(resident_id).sectors.at(sector_id); + auto written = sec.data; + sec.data = bank_req.data; + sector_merge(sec, written, sec.dirty_mask); + sec.partial = false; + } + return true; + } int line_id; if (is_refill) { @@ -1132,13 +1277,41 @@ class CacheBank : public SimObject { line.tag = addr_tag; line.lru_ctr = 0; auto &sec = line.sectors.at(sector_id); - sec.valid = true; - sec.dirty = false; - sec.dirty_mask = 0; - sec.data = bank_req.data; + if (bank_req.nodata) { + sec.valid = true; + sec.dirty = false; + sec.partial = true; + sec.dirty_mask = 0; + sec.data.reset(); + } else { + sec.valid = true; + sec.dirty = false; + sec.partial = false; + sec.dirty_mask = 0; + sec.data = bank_req.data; + } + if (bank_req.refetch) { + chain_partial_ = false; + chain_retry_ = false; + refetch_hold_ = false; + } return true; } + // Track the partial state of the chain a fill just woke. + void setChain(uint32_t set_id, uint64_t addr_tag, uint32_t sector_id, bool partial) { + chain_partial_ = partial; + chain_retry_ = false; + chain_set_ = set_id; + chain_tag_ = addr_tag; + chain_sector_ = sector_id; + } + + // Lazy-fetch loopback depth: room for every request in the pipe plus one. + static uint32_t lazyq_size(const Cache::Config &config) { + return 1u << log2ceil(config.latency + 1); + } + // Fill-forward drain: complete the oldest ready replay directly when it is // a plain read on the armed chain — no pipeline traversal, and the input // slot stays free for new requests. The first write/AMO head closes the @@ -1416,6 +1589,24 @@ class CacheBank : public SimObject { pipe_req_->pop(); } break; + case bank_req_t::Park: { + if (this->mem_req_out.full()) { + return; // stall + } + uint32_t set_id = params_.addr_set_id(bank_req.addr); + uint64_t addr_tag = params_.addr_tag(bank_req.addr); + uint32_t sector_id = params_.addr_sector_id(bank_req.addr); + MemReq fill; + fill.addr = params_.mem_addr_sector(bank_id_, set_id, addr_tag, sector_id); + fill.tag = bank_req.mshr_id; + fill.hart_id = bank_req.hart_id; + fill.uuid = bank_req.uuid; + this->mem_req_out.send(fill, MEM_REQ_DELAY); + DT(3, this->name() << " refetch-req: " << fill); + ++pending_fill_reqs_; + pipe_req_->pop(); + } break; + case bank_req_t::Replay: { uint32_t set_id = params_.addr_set_id(bank_req.addr); uint64_t addr_tag = params_.addr_tag(bank_req.addr); @@ -1423,8 +1614,10 @@ class CacheBank : public SimObject { auto &set = sets_.at(set_id); int32_t free_id = -1, repl_id = 0; int present_id = set.tag_match(addr_tag, config_.repl_policy, rand_ctr_, &free_id, &repl_id); - // Sector hit: line resident AND the requested sector valid. - int hit_id = (present_id != -1 && set.lines.at(present_id).sectors.at(sector_id).valid) ? present_id : -1; + // Sector hit: line resident AND the requested sector valid (and complete + // when the access reads it). + const bool needs_data = !bank_req.write; + int hit_id = (present_id != -1 && sector_hit(set.lines.at(present_id).sectors.at(sector_id), needs_data)) ? present_id : -1; // The replayed sector is normally present — the fill that woke this // replay just installed it. It can still be gone when a concurrent @@ -1452,7 +1645,8 @@ class CacheBank : public SimObject { return; // stall: need a slot to re-track the miss bank_req_t refill = bank_req; refill.type = bank_req_t::Core; - int mshr_id = mshr_.enqueue(refill, set_id, addr_tag, sector_id); + const bool refetch = (present_id != -1) && set.lines.at(present_id).sectors.at(sector_id).valid; + int mshr_id = mshr_.enqueue(refill, set_id, addr_tag, sector_id, refetch); if (!fill_pending) { MemReq fill; fill.addr = params_.mem_addr_sector(bank_id_, set_id, addr_tag, sector_id); @@ -1530,7 +1724,11 @@ class CacheBank : public SimObject { int present_id = set.tag_match(addr_tag, config_.repl_policy, rand_ctr_, &free_id, &repl_id); // Sector hit: line resident AND the requested sector valid (a sector miss // on a resident line takes the miss path and refills just that sector). - int hit_id = (present_id != -1 && set.lines.at(present_id).sectors.at(sector_id).valid) ? present_id : -1; + // A load or AMO to a partial sector misses and refetches it. + const bool needs_data = !bank_req.write; + int hit_id = (present_id != -1 && sector_hit(set.lines.at(present_id).sectors.at(sector_id), needs_data)) ? present_id : -1; + const bool partial_resident = (present_id != -1) && (hit_id == -1) + && set.lines.at(present_id).sectors.at(sector_id).valid; if (hit_id != -1) { // Hit while a same-(set,tag,sector) chain still carries a write or @@ -1560,7 +1758,12 @@ class CacheBank : public SimObject { } else { assert(!mshr_.full()); int id = mshr_.enqueue(bank_req, set_id, addr_tag, sector_id); - mshr_.defer_to_replay(id); + // A chain still awaiting its fill (a refetch of this partial + // sector) wakes this entry itself; only a draining chain needs it + // marked ready here. + if (!mshr_.has_pending_fill(set_id, addr_tag, sector_id, id)) { + mshr_.defer_to_replay(id); + } DT(3, this->name() << " mshr-defer: " << bank_req); } --pending_mshr_size_; @@ -1663,13 +1866,23 @@ class CacheBank : public SimObject { // entries never mark new waiters, so they must not suppress the // fill for this fresh miss. bool fill_pending = mshr_.has_pending_fill(set_id, addr_tag, sector_id, -1); - if (!fill_pending && this->mem_req_out.full()) + // Lazy fetch: a write-miss root allocates through the loopback + // instead of fetching the sector. + const bool lazy_alloc = config_.lazy_fetch && bank_req.write && !fill_pending; + if (lazy_alloc && lazy_q_->full()) { return; + } + if (!fill_pending && !lazy_alloc && this->mem_req_out.full()) { + return; + } assert(!mshr_.full()); - int mshr_id = mshr_.enqueue(bank_req, set_id, addr_tag, sector_id); + int mshr_id = mshr_.enqueue(bank_req, set_id, addr_tag, sector_id, partial_resident); DT(3, this->name() << " mshr-enqueue: " << bank_req); - if (!fill_pending) { + if (lazy_alloc) { + lazy_q_->push(mshr_id); + DT(3, this->name() << " lazy-alloc: " << bank_req); + } else if (!fill_pending) { MemReq fill; fill.addr = params_.mem_addr_sector(bank_id_, set_id, addr_tag, sector_id); // op defaults to MemOp::LD — fill is a load. @@ -1774,6 +1987,7 @@ class CacheBank : public SimObject { assert(false && "in-pipe line mirror underflow"); } TFifo::Ptr pipe_req_; + TFifo::Ptr lazy_q_; // write-miss roots awaiting their data-less fill Cache::PerfStats perf_stats_; @@ -1791,6 +2005,14 @@ class CacheBank : public SimObject { std::shared_ptr fwd_data_; bool crsp_sent_; // core-response port used this tick (pipeline priority) + // Partial state of the chain the last fill woke (lazy fetch). + bool chain_partial_; + bool chain_retry_; // the partial state came from an evicted refetch + uint32_t chain_set_; + uint64_t chain_tag_; + uint32_t chain_sector_; + bool refetch_hold_; // a refetch fill has not installed yet: its chain waits + // Flush walk state. bool flushing_; bool keep_lines_ = false; // the next launch reset keeps the lines diff --git a/sim/simx/mem/cache.h b/sim/simx/mem/cache.h index 4e7e8c5654..5dd4c9bd08 100644 --- a/sim/simx/mem/cache.h +++ b/sim/simx/mem/cache.h @@ -45,6 +45,7 @@ class Cache : public SimObject { bool is_llc; // is last-level cache bool bank_hash = false; // hashed bank select bool shared_llc = false; // the LLC serves every memory client alone + bool lazy_fetch = false; // write miss allocates without a fill (write-back LLC only) }; struct PerfStats { diff --git a/sim/simx/processor.cpp b/sim/simx/processor.cpp index bdcf97023d..aef1822cf6 100644 --- a/sim/simx/processor.cpp +++ b/sim/simx/processor.cpp @@ -106,6 +106,7 @@ ProcessorImpl::ProcessorImpl() VX_CFG_L3_ENABLED != 0, // is_llc when L3 is the LLC false, // bank_hash L3_SHARED_LLC, // shared_llc + VX_CFG_L3_LAZY_FETCH != 0, // lazy_fetch } ); diff --git a/sim/simx/socket.cpp b/sim/simx/socket.cpp index 640f263776..5a1861155a 100644 --- a/sim/simx/socket.cpp +++ b/sim/simx/socket.cpp @@ -93,6 +93,7 @@ class Socket::Impl { (VX_CFG_DCACHE_ENABLED != 0) && (VX_CFG_L2_ENABLED == 0) && (VX_CFG_L3_ENABLED == 0), // is_llc (VX_CFG_DCACHE_BANK_HASH != 0) && (VX_CFG_DCACHE_NUM_BANKS > 1), // bank_hash DCACHE_SHARED_LLC, // shared_llc + VX_CFG_DCACHE_LAZY_FETCH != 0, // lazy_fetch }); #ifdef VX_CFG_EXT_TEX_ENABLE diff --git a/tests/regression/Makefile b/tests/regression/Makefile index 3fd4c75708..bb82626739 100644 --- a/tests/regression/Makefile +++ b/tests/regression/Makefile @@ -15,7 +15,7 @@ TESTS := \ dxa_copy dxa_copy_mcast dxa_kmajor_check \ async_barrier async_gbarrier gbar_phase bar_slot_phase packld wgather wsync quad_lod \ occupancy multikernel reopen module_reload single_cta \ - flush_coherence \ + flush_coherence lazy_fetch \ vm_test vm_fault vm_stress # --- common exclude list --------------------------------------------- diff --git a/tests/regression/lazy_fetch/Makefile b/tests/regression/lazy_fetch/Makefile new file mode 100644 index 0000000000..7a39f6c48c --- /dev/null +++ b/tests/regression/lazy_fetch/Makefile @@ -0,0 +1,16 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := lazy_fetch + +SRC_DIR := $(VORTEX_HOME)/tests/regression/$(PROJECT) + +SRCS := $(SRC_DIR)/main.cpp + +VX_SRCS := $(SRC_DIR)/kernel.cpp + +OPTS ?= -n16 + +KERNEL_LIB := vortex2 + +include ../common.mk diff --git a/tests/regression/lazy_fetch/common.h b/tests/regression/lazy_fetch/common.h new file mode 100644 index 0000000000..308f768331 --- /dev/null +++ b/tests/regression/lazy_fetch/common.h @@ -0,0 +1,134 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include + +// Each thread owns NUM_CASES regions per round, one cache line (REGION_WORDS +// words) each, laid out round-major so neighbouring lines belong to different +// threads and the buffer outgrows the cache across rounds. +#define REGION_WORDS 16 +#define NUM_CASES 6 +#define LANE_WORDS 8 +// Same-set lines: one per thread and round, a page apart, so they compete for +// one cache set and a line being refetched is often evicted before its fill. +#define SET_STRIDE_WORDS 1024 + +enum { + CASE_HALF = 0, // store half the line, then load all of it (same thread) + CASE_BYTES = 1, // byte/halfword stores, then load the words holding them + CASE_AMO = 2, // stores, then AMO add and LR/SC on written and unwritten words + CASE_STORE = 3, // byte stores only: checked after the kernel + CASE_WARP = 4, // one warp stores half the line, the next warp loads all of it + CASE_LANE = 5 // a warp's lanes store into one line, then load its unwritten half +}; + +typedef struct { + uint64_t buf_addr; + uint64_t set_addr; + uint64_t err_addr; + uint32_t rounds; + uint32_t num_threads; // threads in the grid + uint32_t warp_size; + uint32_t cross_warp; // blocks hold an even number of warps + uint32_t amo; // the device implements the A extension + uint32_t phase; // 0: exercise the cases, 1: re-check every region + uint32_t rotate; // phase 1: check the regions of thread (gid + rotate) +} kernel_arg_t; + +// Pre-pattern the host writes over the whole buffer. +static inline uint8_t pattern_byte(uint32_t i) { + return (uint8_t)(i * 37u + (i >> 9) + 0x5Au); +} + +static inline uint32_t pattern_word(uint32_t word_idx) { + uint32_t b = word_idx * 4; + return (uint32_t)pattern_byte(b) + | ((uint32_t)pattern_byte(b + 1) << 8) + | ((uint32_t)pattern_byte(b + 2) << 16) + | ((uint32_t)pattern_byte(b + 3) << 24); +} + +static inline uint32_t store_value(uint32_t region, uint32_t w) { + return 0xC0000000u | (region << 4) | w; +} + +static inline uint32_t region_index(uint32_t round, uint32_t num_threads, uint32_t gid, uint32_t c) { + return (round * num_threads + gid) * NUM_CASES + c; +} + +static inline uint32_t set_line(uint32_t round, uint32_t num_threads, uint32_t gid) { + return round * num_threads + gid; +} + +// Expected word `word_idx` of the same-set buffer once the kernel has run: +// each line's word 0 is stored, the rest keeps the pre-pattern. +static inline uint32_t expected_set_word(uint32_t word_idx) { + if ((word_idx % SET_STRIDE_WORDS) == 0) { + return 0xA0000000u | ((word_idx / SET_STRIDE_WORDS) << 4); + } + return pattern_word(word_idx); +} + +// Owner thread of a CASE_WARP region writes it when it sits in an even warp +// of its block; the thread one warp up reads it. +static inline bool warp_producer(uint32_t gid, uint32_t warp_size) { + return ((gid / warp_size) & 1) == 0; +} + +// Expected content of word w of region `region` once the kernel has run. +static inline uint32_t expected_word(uint32_t region, uint32_t w, const kernel_arg_t* arg) { + uint32_t p = pattern_word(region * REGION_WORDS + w); + uint32_t v = store_value(region, w); + switch (region % NUM_CASES) { + case CASE_HALF: + return (w < REGION_WORDS / 2) ? v : p; + case CASE_BYTES: + if (w == 1) { + return (p & ~0x0000ff00u) | 0x0000ee00u; + } + if (w == 10) { + return (p & ~0x0000ffffu) | 0x0000beefu; + } + return p; + case CASE_AMO: + if (w < 4 && !arg->amo) { + return v; + } + if (!arg->amo) { + return p; + } + if (w == 1) { + return v + 3; + } + if (w < 4) { + return v; + } + if (w == 8) { + return p + 5; + } + if (w == 12) { + return p + 1; + } + return p; + case CASE_STORE: + return ((w & 1) == 0) ? ((p & ~0xffu) | (0x30u + w)) : p; + case CASE_WARP: { + uint32_t gid = (region / NUM_CASES) % arg->num_threads; + if (arg->cross_warp && warp_producer(gid, arg->warp_size) && w >= REGION_WORDS / 2) { + return v; + } + return p; + } + default: { + // the region of each warp's lane 0 is shared by the warp's first lanes + uint32_t gid = (region / NUM_CASES) % arg->num_threads; + uint32_t lanes = (arg->warp_size < LANE_WORDS) ? arg->warp_size : LANE_WORDS; + if ((gid % arg->warp_size) == 0 && w < lanes) { + return v; + } + return p; + } + } +} + +#endif diff --git a/tests/regression/lazy_fetch/kernel.cpp b/tests/regression/lazy_fetch/kernel.cpp new file mode 100644 index 0000000000..5f0dc5e4cd --- /dev/null +++ b/tests/regression/lazy_fetch/kernel.cpp @@ -0,0 +1,174 @@ +#include +#include +#include "common.h" + +// Every access goes through volatile pointers: the point is to send each load +// to the cache right behind the stores it must observe, not to let the +// compiler forward the stored value. +static inline volatile uint32_t* region_ptr(const kernel_arg_t* arg, uint32_t region) { + return reinterpret_cast(arg->buf_addr) + region * REGION_WORDS; +} + +static inline uint32_t check_word(volatile uint32_t* r, uint32_t region, uint32_t w, + const kernel_arg_t* arg) { + uint32_t ref = expected_word(region, w, arg); + return (r[w] != ref) ? 1 : 0; +} + +static uint32_t run_half(const kernel_arg_t* arg, uint32_t region) { + auto r = region_ptr(arg, region); + for (uint32_t w = 0; w < REGION_WORDS / 2; ++w) { + r[w] = store_value(region, w); + } + uint32_t errors = 0; + for (uint32_t w = 0; w < REGION_WORDS; ++w) { + errors += check_word(r, region, w, arg); + } + return errors; +} + +static uint32_t run_bytes(const kernel_arg_t* arg, uint32_t region) { + auto r = region_ptr(arg, region); + reinterpret_cast(r)[5] = 0xee; + uint32_t errors = check_word(r, region, 1, arg); + reinterpret_cast(r)[20] = 0xbeef; + errors += check_word(r, region, 10, arg); + errors += check_word(r, region, 0, arg); + errors += check_word(r, region, 15, arg); + return errors; +} + +static uint32_t run_amo(const kernel_arg_t* arg, uint32_t region) { + auto r = region_ptr(arg, region); + for (uint32_t w = 0; w < 4; ++w) { + r[w] = store_value(region, w); + } + if (!arg->amo) { + return check_word(r, region, 8, arg) + check_word(r, region, 1, arg); + } + uint32_t errors = 0; + uint32_t* word8 = const_cast(&r[8]); + uint32_t* word1 = const_cast(&r[1]); + uint32_t* word12 = const_cast(&r[12]); + uint32_t old8 = __atomic_fetch_add(word8, 5u, __ATOMIC_SEQ_CST); + errors += (old8 != pattern_word(region * REGION_WORDS + 8)) ? 1 : 0; + uint32_t old1 = __atomic_fetch_add(word1, 3u, __ATOMIC_SEQ_CST); + errors += (old1 != store_value(region, 1)) ? 1 : 0; + // Only this thread touches the word, so the SC fails only spuriously. + uint32_t first = 0; + bool done = false; + for (uint32_t attempt = 0; attempt < 64 && !done; ++attempt) { + uint32_t old, sc_fail; + asm volatile ("lr.w %0, (%1)" : "=r"(old) : "r"(word12) : "memory"); + if (attempt == 0) { + first = old; + } + uint32_t newv = old + 1; + asm volatile ("sc.w %0, %2, (%1)" : "=r"(sc_fail) : "r"(word12), "r"(newv) : "memory"); + done = (sc_fail == 0); + } + errors += (!done || first != pattern_word(region * REGION_WORDS + 12)) ? 1 : 0; + return errors; +} + +static void run_store(const kernel_arg_t* arg, uint32_t region) { + auto b = reinterpret_cast(region_ptr(arg, region)); + for (uint32_t w = 0; w < REGION_WORDS; w += 2) { + b[w * 4] = (uint8_t)(0x30 + w); + } +} + +// Each lane's load follows the warp's coalesced store into the same line +// right away, so it reaches the cache while the store's allocation is still +// pending and must wait for the unwritten half to be fetched. +static uint32_t run_lane(const kernel_arg_t* arg, uint32_t round, uint32_t gid) { + uint32_t lane = gid % arg->warp_size; + uint32_t lanes = (arg->warp_size < LANE_WORDS) ? arg->warp_size : LANE_WORDS; + if (lane >= lanes) { + return 0; + } + uint32_t region = region_index(round, arg->num_threads, gid - lane, CASE_LANE); + auto r = region_ptr(arg, region); + uint32_t value = store_value(region, lane); + uint32_t loaded; + asm volatile ("sw %2, 0(%1)\n\tlw %0, %3(%1)" + : "=&r"(loaded) + : "r"(&r[lane]), "r"(value), "i"(LANE_WORDS * 4) + : "memory"); + uint32_t errors = (loaded != expected_word(region, lane + LANE_WORDS, arg)) ? 1 : 0; + errors += check_word(r, region, (lane + 1) % lanes, arg); + return errors; +} + +// Store word 0 of this thread's same-set line, then load word 8 right away. +static uint32_t run_set(const kernel_arg_t* arg, uint32_t round, uint32_t gid) { + uint32_t line = set_line(round, arg->num_threads, gid); + auto r = reinterpret_cast(arg->set_addr) + line * SET_STRIDE_WORDS; + uint32_t value = expected_set_word(line * SET_STRIDE_WORDS); + uint32_t loaded; + asm volatile ("sw %2, 0(%1)\n\tlw %0, 32(%1)" + : "=&r"(loaded) + : "r"(r), "r"(value) + : "memory"); + uint32_t errors = (loaded != expected_set_word(line * SET_STRIDE_WORDS + 8)) ? 1 : 0; + errors += (r[0] != value) ? 1 : 0; + return errors; +} + +static uint32_t check_region(const kernel_arg_t* arg, uint32_t region) { + auto r = region_ptr(arg, region); + uint32_t errors = 0; + for (uint32_t w = 0; w < REGION_WORDS; ++w) { + errors += check_word(r, region, w, arg); + } + return errors; +} + +__kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { + uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; + uint32_t errors = 0; + auto err = reinterpret_cast(arg->err_addr); + + if (arg->phase != 0) { + // Every region again, from another thread (another core when there are + // several), after the launch boundary flushed the partial lines. + uint32_t owner = (gid + arg->rotate) % arg->num_threads; + for (uint32_t round = 0; round < arg->rounds; ++round) { + for (uint32_t c = 0; c < NUM_CASES; ++c) { + errors += check_region(arg, region_index(round, arg->num_threads, owner, c)); + } + auto r = reinterpret_cast(arg->set_addr); + uint32_t base = set_line(round, arg->num_threads, owner) * SET_STRIDE_WORDS; + for (uint32_t w = 0; w < REGION_WORDS; ++w) { + errors += (r[base + w] != expected_set_word(base + w)) ? 1 : 0; + } + } + err[arg->num_threads + gid] = errors; + return; + } + + for (uint32_t round = 0; round < arg->rounds; ++round) { + errors += run_half(arg, region_index(round, arg->num_threads, gid, CASE_HALF)); + errors += run_bytes(arg, region_index(round, arg->num_threads, gid, CASE_BYTES)); + errors += run_amo(arg, region_index(round, arg->num_threads, gid, CASE_AMO)); + run_store(arg, region_index(round, arg->num_threads, gid, CASE_STORE)); + errors += run_lane(arg, round, gid); + errors += run_set(arg, round, gid); + if (arg->cross_warp) { + bool producer = warp_producer(gid, arg->warp_size); + if (producer) { + uint32_t region = region_index(round, arg->num_threads, gid, CASE_WARP); + auto r = region_ptr(arg, region); + for (uint32_t w = REGION_WORDS / 2; w < REGION_WORDS; ++w) { + r[w] = store_value(region, w); + } + } + vx_fence(); + __syncthreads(); + if (!producer) { + errors += check_region(arg, region_index(round, arg->num_threads, gid - arg->warp_size, CASE_WARP)); + } + } + } + err[gid] = errors; +} diff --git a/tests/regression/lazy_fetch/main.cpp b/tests/regression/lazy_fetch/main.cpp new file mode 100644 index 0000000000..60734b7711 --- /dev/null +++ b/tests/regression/lazy_fetch/main.cpp @@ -0,0 +1,217 @@ +#include +#include +#include +#include +#include +#include "common.h" + +#define RT_CHECK(_expr) \ + do { \ + int _ret = _expr; \ + if (0 == _ret) \ + break; \ + printf("Error: '%s' returned %d!\n", #_expr, (int)_ret); \ + cleanup(); \ + exit(-1); \ + } while (false) + +/////////////////////////////////////////////////////////////////////////////// + +const char* kernel_file = "kernel.vxbin"; +uint32_t rounds = 16; + +vx_device_h device = nullptr; +vx_buffer_h buf_buffer = nullptr; +vx_buffer_h err_buffer = nullptr; +vx_buffer_h set_buffer = nullptr; +vx_queue_h queue = nullptr; +vx_module_h module_ = nullptr; +vx_kernel_h kernel = nullptr; + +static void show_usage() { + std::cout << "Vortex Test." << std::endl; + std::cout << "Usage: [-k: kernel] [-n rounds] [-h: help]" << std::endl; +} + +static void parse_args(int argc, char **argv) { + int c; + while ((c = getopt(argc, argv, "n:k:h")) != -1) { + switch (c) { + case 'n': + rounds = atoi(optarg); + break; + case 'k': + kernel_file = optarg; + break; + case 'h': + show_usage(); + exit(0); + break; + default: + show_usage(); + exit(-1); + } + } +} + +void cleanup() { + if (device) { + if (buf_buffer) vx_buffer_release(buf_buffer); + if (err_buffer) vx_buffer_release(err_buffer); + if (set_buffer) vx_buffer_release(set_buffer); + if (kernel) vx_kernel_release(kernel); + if (module_) vx_module_release(module_); + if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); + vx_device_release(device); + } +} + +static int launch(const kernel_arg_t& arg, uint32_t grid, uint32_t block) { + vx_event_h ev = nullptr; + vx_launch_info_t li = {}; + li.struct_size = sizeof(li); + li.kernel = kernel; + li.args_host = &arg; + li.args_size = sizeof(arg); + li.ndim = 1; + li.grid_dim[0] = grid; + li.block_dim[0] = block; + int ret = vx_enqueue_launch(queue, &li, 0, nullptr, &ev); + if (ret != 0) { + return ret; + } + ret = vx_event_wait_value(ev, 1, VX_TIMEOUT_INFINITE); + vx_event_release(ev); + return ret; +} + +int main(int argc, char *argv[]) { + parse_args(argc, argv); + if (rounds == 0) { + rounds = 1; + } + + std::cout << "open device connection" << std::endl; + RT_CHECK(vx_device_open(0, &device)); + + vx_queue_info_t qi = { sizeof(qi), nullptr, VX_QUEUE_PRIORITY_NORMAL, 0 }; + RT_CHECK(vx_queue_create(device, &qi, &queue)); + + uint64_t num_cores, num_warps, num_threads, isa_flags; + RT_CHECK(vx_device_query(device, VX_CAPS_NUM_CORES, &num_cores)); + RT_CHECK(vx_device_query(device, VX_CAPS_NUM_WARPS, &num_warps)); + RT_CHECK(vx_device_query(device, VX_CAPS_NUM_THREADS, &num_threads)); + RT_CHECK(vx_device_query(device, VX_CAPS_ISA_FLAGS, &isa_flags)); + + // One block per core, holding all of its warps. + uint32_t block = num_warps * num_threads; + uint32_t grid = num_cores; + uint32_t total_threads = grid * block; + uint32_t num_regions = rounds * total_threads * NUM_CASES; + uint32_t num_words = num_regions * REGION_WORDS; + uint32_t buf_size = num_words * sizeof(uint32_t); + uint32_t err_size = 2 * total_threads * sizeof(uint32_t); + uint32_t set_words = rounds * total_threads * SET_STRIDE_WORDS; + uint32_t set_size = set_words * sizeof(uint32_t); + + kernel_arg_t arg = {}; + arg.rounds = rounds; + arg.num_threads = total_threads; + arg.warp_size = num_threads; + arg.cross_warp = (num_warps % 2) == 0; + arg.amo = (isa_flags & VX_ISA_STD_A) != 0; + + std::cout << "threads: " << total_threads << ", amo: " << arg.amo << ", regions: " << num_regions + << ", buffer size: " << buf_size << " bytes" << std::endl; + + RT_CHECK(vx_buffer_create(device, buf_size, VX_MEM_READ_WRITE, &buf_buffer)); + RT_CHECK(vx_buffer_address(buf_buffer, &arg.buf_addr)); + RT_CHECK(vx_buffer_create(device, err_size, VX_MEM_READ_WRITE, &err_buffer)); + RT_CHECK(vx_buffer_address(err_buffer, &arg.err_addr)); + RT_CHECK(vx_buffer_create(device, set_size, VX_MEM_READ_WRITE, &set_buffer)); + RT_CHECK(vx_buffer_address(set_buffer, &arg.set_addr)); + + std::vector h_buf(buf_size); + for (uint32_t i = 0; i < buf_size; ++i) { + h_buf[i] = pattern_byte(i); + } + std::vector h_err(2 * total_threads, 0xdeadbeef); + std::vector h_set(set_words); + for (uint32_t i = 0; i < set_words; ++i) { + h_set[i] = pattern_word(i); + } + + std::cout << "upload buffers" << std::endl; + RT_CHECK(vx_enqueue_write(queue, buf_buffer, 0, h_buf.data(), buf_size, 0, nullptr, nullptr)); + RT_CHECK(vx_enqueue_write(queue, err_buffer, 0, h_err.data(), err_size, 0, nullptr, nullptr)); + RT_CHECK(vx_enqueue_write(queue, set_buffer, 0, h_set.data(), set_size, 0, nullptr, nullptr)); + + RT_CHECK(vx_module_load_file(device, kernel_file, &module_)); + RT_CHECK(vx_module_get_kernel(module_, "main", &kernel)); + + std::cout << "launch kernel (exercise)" << std::endl; + arg.phase = 0; + RT_CHECK(launch(arg, grid, block)); + + std::cout << "launch kernel (re-check)" << std::endl; + arg.phase = 1; + arg.rotate = (num_cores > 1) ? block : num_threads; + RT_CHECK(launch(arg, grid, block)); + + std::cout << "download buffers" << std::endl; + vx_event_h read_ev = nullptr; + RT_CHECK(vx_enqueue_read(queue, h_buf.data(), buf_buffer, 0, buf_size, 0, nullptr, nullptr)); + RT_CHECK(vx_enqueue_read(queue, h_set.data(), set_buffer, 0, set_size, 0, nullptr, nullptr)); + RT_CHECK(vx_enqueue_read(queue, h_err.data(), err_buffer, 0, err_size, 0, nullptr, &read_ev)); + RT_CHECK(vx_event_wait_value(read_ev, 1, VX_TIMEOUT_INFINITE)); + vx_event_release(read_ev); + + std::cout << "verify result" << std::endl; + int errors = 0; + for (uint32_t t = 0; t < 2 * total_threads; ++t) { + if (h_err[t] != 0) { + if (errors < 16) { + std::cout << "error: thread " << (t % total_threads) << (t < total_threads ? " (exercise)" : " (re-check)") + << " reported 0x" << std::hex << h_err[t] << std::dec << " mismatches" << std::endl; + } + ++errors; + } + } + auto words = reinterpret_cast(h_buf.data()); + for (uint32_t region = 0; region < num_regions; ++region) { + for (uint32_t w = 0; w < REGION_WORDS; ++w) { + uint32_t ref = expected_word(region, w, &arg); + uint32_t cur = words[region * REGION_WORDS + w]; + if (cur != ref) { + if (errors < 16) { + std::cout << "error: region " << region << " (case " << (region % NUM_CASES) << ") word " << w + << std::hex << ": actual 0x" << cur << ", expected 0x" << ref << std::dec << std::endl; + } + ++errors; + } + } + } + + for (uint32_t i = 0; i < set_words; ++i) { + uint32_t ref = expected_set_word(i); + if (h_set[i] != ref) { + if (errors < 16) { + std::cout << "error: same-set word " << i << std::hex << ": actual 0x" << h_set[i] + << ", expected 0x" << ref << std::dec << std::endl; + } + ++errors; + } + } + + cleanup(); + + if (errors != 0) { + std::cout << "Found " << std::dec << errors << " errors!" << std::endl; + std::cout << "FAILED!" << std::endl; + return 1; + } + + std::cout << "PASSED!" << std::endl; + return 0; +} From e5872047882d0b81850af10a7cd8816d04f1e589 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 18 Oct 2026 14:46:03 -0700 Subject: [PATCH 036/124] core: decouple fetch width from issue width (VX_CFG_FETCH_WIDTH) The core fetched for every issue slice through one scheduler and one icache port, so front-end bandwidth and the scheduler's size grew with ISSUE_WIDTH as a single block. FETCH_WIDTH (FW, a power of two dividing ISSUE_WIDTH) splits the front end into FW fetch slices. Slice f owns the warps with wid % FW == f and has its own warp scheduler, fetch, decode and icache port. RTL structure: - VX_fetch_slice = VX_warp_sched -> VX_fetch -> VX_decode. - VX_scheduler stays core-wide. It holds CTA dispatch, with one launch lane per fetch slice, VX_warp_shared_unit for barriers, wspawn and warp retirement, and the per-core state. - VX_fetch_xbar is purely combinational and routes decoded instructions, ibuffer pops, warp control, CSRs and drain status between fetch and compute partitions. - When every execution unit has one block per issue slice, the core is built from VX_compute_slice (issue + execute + commit). Otherwise it keeps the stage-sliced structure. The unit blocks (ALU/LSU/SFU/FPU/TCU) are separate modules so both structures share them. SimX mirrors the structure: FW per-slice fetch and decode latches, FW icache ports and banks, and FW CTA launch lanes. The init pause on a block-shape change is modelled when FW > 1. Where the split put a register on a path master kept combinational, master's timing is restored: - all_ibuf_full is built from next-state ibuffer flags. - Warp control registers each shared-unit request once, and the unit acts on the granted request directly. Exit and wspawn therefore reach the scheduler in the same cycle as on master. - Exits, wspawns, sync barriers and barrier waits are posted. Only a barrier arrive, which writes back the phase, waits for the unit's response. Previously every barrier held warp control for about five cycles, which spread warps' barrier release five cycles apart. - An exit retires its warp on warp_done, and the CTA dispatcher frees the slot through the slice's registered active bit, as on master. - Core busy also counts cta_fire. This closes a one-cycle idle hole that let a CTA of at most FW warps end the kernel early. Warp control acts on a granted barrier request in the same cycle (SH_DIRECT), so with FW = 1 a barrier releases on the same cycle as before the split. Fixes that come with the split: - The global-barrier core index is taken modulo the cluster's core count at full width. The truncated modulo divided by zero at 2 and 4 cores and deadlocked gbar_phase at 1 cluster x 4 cores (CI: gbar-phase-3). - SATP is a live CSR, updated by a read-modify-write in the fire cycle, and the per-block CSR requests are ordered. With two SFU blocks, a page-table switch could otherwise race a translation (CI: vm-sfu2). - The per-CTA context RAMs live in VX_scheduler, with one read port per SFU block. - Static asserts on FETCH_WIDTH, the lane dispatcher shape, and the DXA in-flight limit. CI: several config1/dxa/tensor/tensor_wg rtlsim cases now also run with FETCH_WIDTH = 2 or 4. RTL at ISSUE_WIDTH = 4, 16 warps x 16 threads, FW = 1 -> 2: sgemm 103,793 -> 70,957 cycles (-31.6%), raycast 307,307 -> 299,006 (-2.7%). At the default config (FW = 1), rtlsim moves by at most +132 cycles (printf); sgemm +9, raycast +55. Co-Authored-By: Claude Opus 5.5 --- VX_config.toml | 6 +- ci/testcases/amo.yaml | 2 +- ci/testcases/config1.yaml | 14 +- ci/testcases/dxa.yaml | 4 +- ci/testcases/regression.yaml | 8 + ci/testcases/tensor.yaml | 6 +- ci/testcases/tensor_wg.yaml | 13 +- ci/testcases/vm.yaml | 9 + docs/assets/img/amo_architecture.svg | 2 +- docs/assets/img/lsu_architecture.svg | 6 +- docs/designs/atomic_memory_operations.md | 2 +- docs/designs/lsu_pipeline_design.md | 6 +- docs/designs/output_merger_architecture.md | 4 +- hw/rtl/VX_cluster.sv | 10 +- hw/rtl/VX_gpu_pkg.sv | 207 ++++- hw/rtl/VX_graphics.sv | 24 +- hw/rtl/VX_socket.sv | 27 +- hw/rtl/Vortex.sv | 2 +- hw/rtl/core/VX_alu_block.sv | 93 +++ hw/rtl/core/VX_alu_int.sv | 8 +- hw/rtl/core/VX_alu_unit.sv | 70 +- hw/rtl/core/VX_bar_unit.sv | 9 +- hw/rtl/core/VX_commit.sv | 131 +-- hw/rtl/core/VX_commit_slice.sv | 124 +++ hw/rtl/core/VX_compute_slice.sv | 192 +++++ hw/rtl/core/VX_core.sv | 767 ++++++++++++------ hw/rtl/core/VX_csr_cta.sv | 75 ++ hw/rtl/core/VX_csr_data.sv | 317 +------- hw/rtl/core/VX_csr_fpu.sv | 103 +++ hw/rtl/core/VX_csr_mpm.sv | 257 ++++++ hw/rtl/core/VX_csr_unit.sv | 49 +- hw/rtl/core/VX_cta_dispatch.sv | 616 ++++++++------ hw/rtl/core/VX_dcr_data.sv | 9 +- hw/rtl/core/VX_decompressor.sv | 66 +- hw/rtl/core/VX_dispatcher.sv | 13 +- hw/rtl/core/VX_execute.sv | 43 +- hw/rtl/core/VX_execute_slice.sv | 401 +++++++++ hw/rtl/core/VX_fetch.sv | 48 +- hw/rtl/core/VX_fetch_slice.sv | 149 ++++ hw/rtl/core/VX_fetch_xbar.sv | 229 ++++++ hw/rtl/core/VX_ibuffer.sv | 4 +- hw/rtl/core/VX_ipdom_stack.sv | 24 +- hw/rtl/core/VX_issue.sv | 22 +- hw/rtl/core/VX_issue_slice.sv | 18 +- hw/rtl/core/VX_lane_dispatch.sv | 32 +- hw/rtl/core/VX_lane_gather.sv | 32 +- hw/rtl/core/VX_lsu_agu.sv | 2 +- .../core/{VX_lsu_slice.sv => VX_lsu_block.sv} | 13 +- hw/rtl/core/VX_lsu_unit.sv | 14 +- hw/rtl/core/VX_operands.sv | 4 +- hw/rtl/core/VX_scheduler.sv | 751 ++++++----------- hw/rtl/core/VX_scoreboard.sv | 4 +- hw/rtl/core/VX_sfu_block.sv | 231 ++++++ hw/rtl/core/VX_sfu_shared.sv | 156 ++++ hw/rtl/core/VX_sfu_unit.sv | 305 ++----- hw/rtl/core/VX_split_join.sv | 20 +- hw/rtl/core/VX_uop_packld.sv | 2 +- hw/rtl/core/VX_uuid_gen.sv | 28 +- hw/rtl/core/VX_warp_sched.sv | 514 ++++++++++++ hw/rtl/core/VX_warp_shared_unit.sv | 268 ++++++ hw/rtl/core/VX_wctl_unit.sv | 150 +++- hw/rtl/dxa/VX_dxa_unit.sv | 10 +- hw/rtl/dxa/VX_dxa_worker.sv | 2 + hw/rtl/fpu/VX_fpu_block.sv | 261 ++++++ hw/rtl/fpu/VX_fpu_unit.sv | 231 +----- hw/rtl/interfaces/VX_sched_csr_if.sv | 16 +- hw/rtl/interfaces/VX_slice_sched_if.sv | 151 ++++ hw/rtl/interfaces/VX_warp_ctl_if.sv | 63 +- hw/rtl/interfaces/VX_warp_shared_if.sv | 44 + hw/rtl/om/VX_om_core.sv | 8 +- hw/rtl/om/VX_om_mem.sv | 2 +- hw/rtl/om/VX_om_steer.sv | 2 +- hw/rtl/raster/VX_raster_be.sv | 2 +- hw/rtl/raster/VX_raster_core.sv | 15 +- hw/rtl/raster/VX_raster_earlyz.sv | 2 +- hw/rtl/raster/VX_raster_mem.sv | 9 +- hw/rtl/raster/VX_raster_slice.sv | 4 +- hw/rtl/rtu/VX_rtu_unit.sv | 65 +- hw/rtl/tcu/VX_tcu_block.sv | 121 +++ hw/rtl/tcu/VX_tcu_core.sv | 4 +- hw/rtl/tcu/VX_tcu_meta.sv | 16 +- hw/rtl/tcu/VX_tcu_shared.sv | 118 +++ hw/rtl/tcu/VX_tcu_unit.sv | 241 ++---- hw/rtl/tex/VX_tex_core.sv | 8 +- hw/rtl/tex/VX_tex_mem.sv | 2 +- hw/rtl/tex/VX_tex_unit.sv | 2 - hw/syn/firesim/Makefile | 2 +- hw/syn/xilinx/aved/platforms.mk | 2 +- hw/unittest/core/VX_core_top.sv | 60 +- .../cta_dispatcher/VX_cta_dispatch_top.sv | 23 +- hw/unittest/gfx/VX_gfx_top.sv | 3 +- hw/unittest/issue/VX_issue_top.sv | 18 +- hw/unittest/raster_core/VX_raster_core_top.sv | 2 +- sim/firesim/firesim_sim.cpp | 2 +- sim/simx/constants.h | 9 + sim/simx/core.cpp | 166 ++-- sim/simx/cta_dispatcher.cpp | 96 ++- sim/simx/cta_dispatcher.h | 20 +- sim/simx/dxa/dxa_core.cpp | 4 + sim/simx/dxa/dxa_unit.cpp | 2 +- sim/simx/dxa/dxa_unit.h | 7 +- sim/simx/lsu_unit.cpp | 2 +- sim/simx/rtu/rtu_unit.cpp | 7 +- sim/simx/rtu/rtu_unit.h | 5 +- sim/simx/scheduler.cpp | 54 +- sim/simx/scheduler.h | 21 +- sim/simx/sfu_unit.cpp | 83 +- sim/simx/sfu_unit.h | 7 + sim/simx/socket.cpp | 10 +- sim/simx/tex/tex_unit.cpp | 2 +- sim/simx/tex/tex_unit.h | 7 +- sim/simx/wctl_unit.cpp | 37 + sim/simx/wctl_unit.h | 5 + 113 files changed, 6040 insertions(+), 2730 deletions(-) create mode 100644 hw/rtl/core/VX_alu_block.sv create mode 100644 hw/rtl/core/VX_commit_slice.sv create mode 100644 hw/rtl/core/VX_compute_slice.sv create mode 100644 hw/rtl/core/VX_csr_cta.sv create mode 100644 hw/rtl/core/VX_csr_fpu.sv create mode 100644 hw/rtl/core/VX_csr_mpm.sv create mode 100644 hw/rtl/core/VX_execute_slice.sv create mode 100644 hw/rtl/core/VX_fetch_slice.sv create mode 100644 hw/rtl/core/VX_fetch_xbar.sv rename hw/rtl/core/{VX_lsu_slice.sv => VX_lsu_block.sv} (98%) create mode 100644 hw/rtl/core/VX_sfu_block.sv create mode 100644 hw/rtl/core/VX_sfu_shared.sv create mode 100644 hw/rtl/core/VX_warp_sched.sv create mode 100644 hw/rtl/core/VX_warp_shared_unit.sv create mode 100644 hw/rtl/fpu/VX_fpu_block.sv create mode 100644 hw/rtl/interfaces/VX_slice_sched_if.sv create mode 100644 hw/rtl/interfaces/VX_warp_shared_if.sv create mode 100644 hw/rtl/tcu/VX_tcu_block.sv create mode 100644 hw/rtl/tcu/VX_tcu_shared.sv diff --git a/VX_config.toml b/VX_config.toml index 9ccfef0cc5..2646ef048a 100644 --- a/VX_config.toml +++ b/VX_config.toml @@ -52,9 +52,11 @@ VX_CFG_MAX_BAR_EVENTS = 32 VX_CFG_IBUF_SIZE = 4 VX_CFG_DISPATCH_QUEUE_SIZE = 4 VX_CFG_ISSUE_WIDTH = "expr: up($VX_CFG_NUM_WARPS / 16)" +# Fetch slices, each scheduling and fetching its own warps on its own icache +# port: a power of two that divides ISSUE_WIDTH. +VX_CFG_FETCH_WIDTH = 1 VX_CFG_SIMD_WIDTH = "expr: $VX_CFG_NUM_THREADS" VX_CFG_NUM_OPCS = "expr: up($VX_CFG_NUM_WARPS / (4 * $VX_CFG_ISSUE_WIDTH))" - VX_CFG_NUM_GPR_BANKS = 4 VX_CFG_NUM_VGPR_BANKS = 2 @@ -207,6 +209,8 @@ VX_CFG_ICACHE_MRSQ_SIZE = 0 VX_CFG_ICACHE_MREQ_SIZE = 0 VX_CFG_ICACHE_CRSQ_SIZE = 0 VX_CFG_ICACHE_MEM_PORTS = 1 +# banks behind the fetch-slice request ports +VX_CFG_ICACHE_NUM_BANKS = "expr: pow(2, clog2($VX_CFG_FETCH_WIDTH))" VX_CFG_ICACHE_LATENCY = "expr: 2 + max(0, clog2($VX_CFG_ICACHE_SIZE) - clog2(16384))" VX_CFG_DCACHE_SIZE = 16384 diff --git a/ci/testcases/amo.yaml b/ci/testcases/amo.yaml index de2b638690..8ce2da3fbd 100644 --- a/ci/testcases/amo.yaml +++ b/ci/testcases/amo.yaml @@ -7,7 +7,7 @@ defaults: configs: "-DVX_CFG_EXT_A_ENABLE" xlen: [32, 64] tier: smoke - touches: [hw/rtl/cache, hw/rtl/mem, hw/rtl/core/VX_decode.sv, hw/rtl/core/VX_lsu_slice.sv, sim/simx/amo, sim/simx/mem] + touches: [hw/rtl/cache, hw/rtl/mem, hw/rtl/core/VX_decode.sv, hw/rtl/core/VX_lsu_block.sv, sim/simx/amo, sim/simx/mem] tests: # RISC-V 'A' ISA conformance (rv{XLEN}ua-p-{amo*,lrsc}); shares the A-built sim. - id: isa-conformance diff --git a/ci/testcases/config1.yaml b/ci/testcases/config1.yaml index feab45fba1..cb2d49aa71 100644 --- a/ci/testcases/config1.yaml +++ b/ci/testcases/config1.yaml @@ -179,13 +179,13 @@ tests: drivers: - rtlsim app: diverge - configs: -DVX_CFG_ISSUE_WIDTH=2 + configs: -DVX_CFG_ISSUE_WIDTH=2 -DVX_CFG_FETCH_WIDTH=2 -DVX_CFG_NUM_SFU_BLOCKS=2 - id: diverge-18 via: blackbox drivers: - rtlsim app: diverge - configs: -DVX_CFG_ISSUE_WIDTH=4 + configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_FETCH_WIDTH=4 -DVX_CFG_NUM_SFU_BLOCKS=4 - id: diverge-19 via: blackbox drivers: @@ -254,13 +254,13 @@ tests: drivers: - rtlsim app: diverge - configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_NUM_ALU_BLOCKS=2 -DVX_CFG_NUM_ALU_LANES=2 + configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_NUM_ALU_BLOCKS=2 -DVX_CFG_NUM_ALU_LANES=2 -DVX_CFG_FETCH_WIDTH=2 - id: diverge-23 via: blackbox drivers: - rtlsim app: diverge - configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_NUM_ALU_BLOCKS=4 -DVX_CFG_NUM_ALU_LANES=4 + configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_NUM_ALU_BLOCKS=4 -DVX_CFG_NUM_ALU_LANES=4 -DVX_CFG_FETCH_WIDTH=4 -DVX_CFG_NUM_SFU_BLOCKS=4 - id: diverge-24 via: blackbox drivers: @@ -284,13 +284,13 @@ tests: drivers: - rtlsim app: regression/vecadd - configs: -DVX_CFG_ISSUE_WIDTH=2 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_FPU_LANES=2 + configs: -DVX_CFG_ISSUE_WIDTH=2 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_FPU_LANES=2 -DVX_CFG_FETCH_WIDTH=2 - id: regression/vecadd-2 via: blackbox drivers: - rtlsim app: regression/vecadd - configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_NUM_FPU_BLOCKS=4 -DVX_CFG_NUM_FPU_LANES=4 + configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_NUM_FPU_BLOCKS=4 -DVX_CFG_NUM_FPU_LANES=4 -DVX_CFG_FETCH_WIDTH=2 -DVX_CFG_NUM_SFU_BLOCKS=4 - id: regression/vecadd-3 via: blackbox drivers: @@ -349,7 +349,7 @@ tests: drivers: - rtlsim app: regression/vecadd - configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_NUM_LSU_BLOCKS=4 -DVX_CFG_NUM_LSU_LANES=4 + configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_NUM_LSU_BLOCKS=4 -DVX_CFG_NUM_LSU_LANES=4 -DVX_CFG_FETCH_WIDTH=4 -DVX_CFG_NUM_SFU_BLOCKS=4 - id: regression/vecadd-7 via: blackbox drivers: diff --git a/ci/testcases/dxa.yaml b/ci/testcases/dxa.yaml index 8cad41a2b3..913c71cb57 100644 --- a/ci/testcases/dxa.yaml +++ b/ci/testcases/dxa.yaml @@ -388,7 +388,7 @@ tests: drivers: - rtlsim app: sgemm_tcu_wg_dxa - configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DWGMMA_NRC=16 -DVX_CFG_TCU_TYPE=DPI -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 + configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DWGMMA_NRC=16 -DVX_CFG_TCU_TYPE=DPI -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_FETCH_WIDTH=2 shape: threads: 32 warps: 8 @@ -419,7 +419,7 @@ tests: drivers: - rtlsim app: sgemm_tcu_wg_sp_dxa - configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_TCU_TYPE=DPI + configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_TCU_TYPE=DPI -DVX_CFG_FETCH_WIDTH=4 -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 shape: threads: 32 diff --git a/ci/testcases/regression.yaml b/ci/testcases/regression.yaml index 0d4ad00450..af35816d28 100644 --- a/ci/testcases/regression.yaml +++ b/ci/testcases/regression.yaml @@ -90,6 +90,14 @@ tests: app: gbar_phase shape: cores: 2 +# Four cores in one cluster: every core must arrive under its own index. +- id: gbar-phase-3 + via: blackbox + drivers: + - xrt + app: gbar_phase + shape: + cores: 4 # SimX<->rtlsim cycle parity on the multi-core global barrier. Until this case # existed the gate could not observe the gbar path at all: every other parity # case is single-core, so USE_GBAR is 0 and the request queue is not built. diff --git a/ci/testcases/tensor.yaml b/ci/testcases/tensor.yaml index 062a8dd331..246774db01 100644 --- a/ci/testcases/tensor.yaml +++ b/ci/testcases/tensor.yaml @@ -123,13 +123,13 @@ tests: drivers: - rtlsim app: sgemm_tcu - configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_DPI -DVX_CFG_ISSUE_WIDTH=2 -DITYPE=fp16 -DOTYPE=fp32 + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_DPI -DVX_CFG_ISSUE_WIDTH=2 -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_FETCH_WIDTH=2 -DVX_CFG_NUM_SFU_BLOCKS=2 - id: sgemm_tcu-19 via: blackbox drivers: - rtlsim app: sgemm_tcu - configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_DPI -DVX_CFG_ISSUE_WIDTH=4 -DITYPE=fp16 -DOTYPE=fp32 + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_DPI -DVX_CFG_ISSUE_WIDTH=4 -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_FETCH_WIDTH=2 -DVX_CFG_NUM_SFU_BLOCKS=4 -DPROFILE_ENABLE - id: sgemm_tcu-20 via: blackbox @@ -148,7 +148,7 @@ tests: drivers: - rtlsim app: sgemm2_tcu - configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_DPI + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_DPI -DVX_CFG_FETCH_WIDTH=4 -DVX_CFG_NUM_SFU_BLOCKS=4 -DITYPE=fp16 -DOTYPE=fp32 args: -w 4 diff --git a/ci/testcases/tensor_wg.yaml b/ci/testcases/tensor_wg.yaml index a08f6cc835..b9bb956479 100644 --- a/ci/testcases/tensor_wg.yaml +++ b/ci/testcases/tensor_wg.yaml @@ -44,14 +44,14 @@ tests: drivers: - rtlsim app: sgemm_tcu_wg - configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_FETCH_WIDTH=2 -DVX_CFG_TCU_INT8_ENABLE -DITYPE=int8 -DOTYPE=int32 -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 - id: sgemm_tcu_wg-6 via: blackbox drivers: - rtlsim app: sgemm_tcu_wg - configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_FETCH_WIDTH=4 -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 - id: sgemm_tcu_wg-7 via: blackbox @@ -72,7 +72,7 @@ tests: drivers: - rtlsim app: sgemm_tcu_wg - configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_FETCH_WIDTH=2 -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DWGMMA_SS -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 - id: sgemm_tcu_wg-10 via: blackbox @@ -116,7 +116,7 @@ tests: drivers: - rtlsim app: sgemm_tcu_wg_sp - configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_FETCH_WIDTH=4 -DVX_CFG_TCU_SPARSE_ENABLE -DWGMMA_NRC=8 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 - id: sgemm_tcu_wg_sp-5 via: blackbox @@ -130,7 +130,7 @@ tests: drivers: - rtlsim app: sgemm_tcu_wg_sp - configs: -DVX_CFG_NUM_THREADS=32 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE + configs: -DVX_CFG_NUM_THREADS=32 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_FETCH_WIDTH=2 -DVX_CFG_TCU_SPARSE_ENABLE -DWGMMA_NRC=8 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 - id: sgemm_tcu_wg_sp-7 via: blackbox @@ -232,8 +232,9 @@ tests: args: -m64 -n64 -k64 # One execution block per unit: the four-block model ISSUE_WIDTH=4 implies is # several times the size, and a hosted CI runner cannot build it in a job. + # Two fetch slices: covers the fetch-slice front end on both models. configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 - -DVX_CFG_TCU_FEDP2K -DVX_CFG_TCU_TYPE_DPI -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DWGMMA_RS + -DVX_CFG_TCU_FEDP2K -DVX_CFG_TCU_TYPE_DPI -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DWGMMA_RS -DVX_CFG_FETCH_WIDTH=2 # Cooperative sparse WGMMA (default fp16 operands, 2:4 structured sparsity). # Not warp-width swept: the tensor-core fragment layout is defined over a fixed diff --git a/ci/testcases/vm.yaml b/ci/testcases/vm.yaml index 2c4d2b7c2b..6f98894b00 100644 --- a/ci/testcases/vm.yaml +++ b/ci/testcases/vm.yaml @@ -235,6 +235,15 @@ tests: # above) are reported, via the shared walker. Re-enable when L1 permission-fault # reporting is added. +# Two SFU blocks: every warp writes SATP at entry, from either block. +- id: vm-sfu2 + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm + configs: -DVX_CFG_VM_ENABLE -DVX_CFG_ISSUE_WIDTH=2 -DVX_CFG_NUM_SFU_BLOCKS=2 + # --------------------------------------------------------------------------- # check: model_parity — same VM-enabled app on both drivers (xlen 32); exact # instret + cycles within tolerance. SimX is the timing model of the RTL. diff --git a/docs/assets/img/amo_architecture.svg b/docs/assets/img/amo_architecture.svg index d95de9c2cf..6751a3d332 100644 --- a/docs/assets/img/amo_architecture.svg +++ b/docs/assets/img/amo_architecture.svg @@ -44,7 +44,7 @@ aq / rl are captured and carry no behavior - VX_lsu_slice — per-lane sideband + VX_lsu_block — per-lane sideband packs amo_req_t into each lane's mem_bus_attr_t: hart_id {core, warp, thread} amo_cmp comparand (Zacas only) diff --git a/docs/assets/img/lsu_architecture.svg b/docs/assets/img/lsu_architecture.svg index c173c657b1..ce3db01776 100644 --- a/docs/assets/img/lsu_architecture.svg +++ b/docs/assets/img/lsu_architecture.svg @@ -30,7 +30,7 @@ VX_lsu_unit execute stage - one VX_lsu_slice per + one VX_lsu_block per LSU block (the issue width) @@ -42,7 +42,7 @@ becomes packets: pid, sop, eop - VX_lsu_slice — request side + VX_lsu_block — request side VX_lsu_agu base + offset, or @@ -68,7 +68,7 @@ tag = header, operation, alignment, slot, fence - VX_lsu_slice — response side + VX_lsu_block — response side load formatter sub-word extract by the saved diff --git a/docs/designs/atomic_memory_operations.md b/docs/designs/atomic_memory_operations.md index 065ff80f0c..3388132790 100644 --- a/docs/designs/atomic_memory_operations.md +++ b/docs/designs/atomic_memory_operations.md @@ -90,7 +90,7 @@ LSU as `rs3_data` with no new operand path. ## 3. The LSU sideband -[`VX_lsu_slice.sv`](../../hw/rtl/core/VX_lsu_slice.sv) packs one `amo_req_t` +[`VX_lsu_block.sv`](../../hw/rtl/core/VX_lsu_block.sv) packs one `amo_req_t` per lane into the lane's memory attribute ([`VX_gpu_pkg.sv`](../../hw/rtl/VX_gpu_pkg.sv)): diff --git a/docs/designs/lsu_pipeline_design.md b/docs/designs/lsu_pipeline_design.md index 7ba07a2124..025c834d9f 100644 --- a/docs/designs/lsu_pipeline_design.md +++ b/docs/designs/lsu_pipeline_design.md @@ -6,7 +6,7 @@ interleave, per-lane classification and formatting, fence ordering, multi-packet tracking, the shared memory scheduler and its outstanding pool, address-class steering, coalescing, and the response path. Covers the RTL ([`VX_lsu_unit.sv`](../../hw/rtl/core/VX_lsu_unit.sv), -[`VX_lsu_slice.sv`](../../hw/rtl/core/VX_lsu_slice.sv), +[`VX_lsu_block.sv`](../../hw/rtl/core/VX_lsu_block.sv), [`VX_lsu_agu.sv`](../../hw/rtl/core/VX_lsu_agu.sv), [`VX_lsu_scheduler.sv`](../../hw/rtl/core/VX_lsu_scheduler.sv), [`VX_mem_scheduler.sv`](../../hw/rtl/libs/VX_mem_scheduler.sv), @@ -104,7 +104,7 @@ that is provably equivalent. ## 3. The slice — request side -[`VX_lsu_slice.sv`](../../hw/rtl/core/VX_lsu_slice.sv). +[`VX_lsu_block.sv`](../../hw/rtl/core/VX_lsu_block.sv). ### 3.1 Classification @@ -208,7 +208,7 @@ library block also used by the caches' other clients. | client | source | present | |---|---|---| -| 0 | the block's `VX_lsu_slice` | always | +| 0 | the block's `VX_lsu_block` | always | | 1 | the tensor unit's warp-level address generator | on block 0, when the tensor metadata path is enabled | Arbitration is round-robin, and the granted client's index is prepended to diff --git a/docs/designs/output_merger_architecture.md b/docs/designs/output_merger_architecture.md index 69a4ecff43..a8aeacb10d 100644 --- a/docs/designs/output_merger_architecture.md +++ b/docs/designs/output_merger_architecture.md @@ -8,7 +8,7 @@ and deadlock arguments, and the parallelism/scaling model. Covers the RTL ([`hw/rtl/om/`](../../hw/rtl/om/)), the SIMT-side plumbing ([`VX_decode.sv`](../../hw/rtl/core/VX_decode.sv), [`VX_gfx_uops.sv`](../../hw/rtl/core/VX_gfx_uops.sv), -[`VX_lsu_slice.sv`](../../hw/rtl/core/VX_lsu_slice.sv)), the cluster +[`VX_lsu_block.sv`](../../hw/rtl/core/VX_lsu_block.sv)), the cluster integration ([`VX_graphics.sv`](../../hw/rtl/VX_graphics.sv)), the SimX model ([`sim/simx/om/`](../../sim/simx/om/)), and the software contract ([`sw/kernel/include/vx_graphics.h`](../../sw/kernel/include/vx_graphics.h), @@ -170,7 +170,7 @@ byte-enables splitting the write (`byteen[2:0]` = depth writemask, ### 4.1 LSU attribute tagging -[`VX_lsu_slice.sv`](../../hw/rtl/core/VX_lsu_slice.sv) range-checks every +[`VX_lsu_block.sv`](../../hw/rtl/core/VX_lsu_block.sv) range-checks every lane address against the aperture and sets **two** attribute bits: `is_addr_om` (the steer's select) and `is_addr_io` (alongside it). The IO bit is not a hack — *uncached, bypassed, posted* is exactly the behaviour diff --git a/hw/rtl/VX_cluster.sv b/hw/rtl/VX_cluster.sv index be46c3f4e4..6204d4ee88 100644 --- a/hw/rtl/VX_cluster.sv +++ b/hw/rtl/VX_cluster.sv @@ -15,7 +15,6 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; #( - parameter CLUSTER_ID = 0, parameter `STRING INSTANCE_ID = "" ) ( `SCOPE_IO_DECL @@ -24,6 +23,8 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; input wire clk, input wire reset, + input wire [CLUSTER_ID_WIDTH-1:0] cluster_id, + `ifdef PERF_ENABLE input sysmem_perf_t sysmem_perf, `endif @@ -169,7 +170,7 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; ); VX_gbar_unit #( - .INSTANCE_ID (`SFORMATF(("gbar%0d", CLUSTER_ID))) + .INSTANCE_ID (`SFORMATF(("%s-gbar", INSTANCE_ID))) ) gbar_unit ( .clk (clk), .reset (reset), @@ -373,13 +374,13 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; for (genvar socket_id = 0; socket_id < NUM_SOCKETS; ++socket_id) begin : g_sockets VX_socket #( - .SOCKET_ID ((CLUSTER_ID * NUM_SOCKETS) + socket_id), .INSTANCE_ID (`SFORMATF(("%s-socket%0d", INSTANCE_ID, socket_id))) ) socket ( `SCOPE_IO_BIND (scope_socket+socket_id) .clk (clk), .reset (reset), + .socket_id (CORE_ID_WIDTH'(cluster_id) * CORE_ID_WIDTH'(NUM_SOCKETS) + CORE_ID_WIDTH'(socket_id)), `ifdef PERF_ENABLE .sysmem_perf (sysmem_perf_tmp), @@ -426,10 +427,11 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; assign per_socket_dcr_bus_if[DCR_GFX_IDX].rsp_data = gfx_dcr_bus_if.rsp_data; VX_graphics #( - .CLUSTER_ID (CLUSTER_ID) + .INSTANCE_ID (INSTANCE_ID) ) graphics ( .clk (clk), .reset (reset), + .cluster_id (cluster_id), `ifdef PERF_ENABLE `ifdef VX_CFG_EXT_RASTER_ENABLE .raster_perf (gfx_raster_perf), diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index 39e4140148..114b2ef153 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -30,6 +30,11 @@ package VX_gpu_pkg; localparam NT_WIDTH = `UP(NT_BITS); localparam NB_WIDTH = `UP(NB_BITS); + // core index across all clusters + localparam CORE_ID_BITS = `CLOG2(`VX_CFG_NUM_CLUSTERS * `VX_CFG_NUM_CORES); + localparam CORE_ID_WIDTH = `UP(CORE_ID_BITS); + localparam CLUSTER_ID_WIDTH = `LOG2UP(`VX_CFG_NUM_CLUSTERS); + localparam NUM_CTA_MAX = `VX_CFG_NUM_WARPS; localparam NCTA_BITS = `CLOG2(NUM_CTA_MAX); localparam NCTA_WIDTH = `UP(NCTA_BITS); @@ -625,6 +630,104 @@ package VX_gpu_pkg; end endfunction + // Fetch slice f schedules and fetches the warps with wid mod FETCH_WIDTH == f, + // naming them by their local id wfs = wid / FETCH_WIDTH. + localparam FETCH_FSW_BITS = `CLOG2(`VX_CFG_FETCH_WIDTH); + localparam FETCH_FSW_W = `UP(FETCH_FSW_BITS); + localparam PER_FETCH_WARPS = `VX_CFG_NUM_WARPS / `VX_CFG_FETCH_WIDTH; + localparam FETCH_WFS_BITS = `CLOG2(PER_FETCH_WARPS); + localparam FETCH_WFS_W = `UP(FETCH_WFS_BITS); + + function automatic logic [NW_WIDTH-1:0] wfs_to_wid( + input logic [FETCH_WFS_W-1:0] wfs, + input logic [FETCH_FSW_W-1:0] fsw + ); + if (FETCH_WFS_BITS == 0) begin + wfs_to_wid = NW_WIDTH'(fsw); + end else if (FETCH_FSW_BITS == 0) begin + wfs_to_wid = NW_WIDTH'(wfs); + end else begin + wfs_to_wid = NW_WIDTH'({wfs, fsw}); + end + endfunction + + function automatic logic [FETCH_FSW_W-1:0] wid_to_fsw( + input logic [NW_WIDTH-1:0] wid + ); + if (FETCH_FSW_BITS != 0) begin + wid_to_fsw = wid[FETCH_FSW_W-1:0]; + end else begin + wid_to_fsw = 0; + end + endfunction + + // FU block ports per fetch slice: a unit with B blocks serves warps with + // wid mod B == b, so B >= FETCH_WIDTH gives each fetch slice B / FETCH_WIDTH + // blocks and B < FETCH_WIDTH shares each block among FETCH_WIDTH / B slices. + localparam FETCH_ALU_PORTS = (`VX_CFG_NUM_ALU_BLOCKS > `VX_CFG_FETCH_WIDTH) ? (`VX_CFG_NUM_ALU_BLOCKS / `VX_CFG_FETCH_WIDTH) : 1; + localparam FETCH_SFU_PORTS = (`VX_CFG_NUM_SFU_BLOCKS > `VX_CFG_FETCH_WIDTH) ? (`VX_CFG_NUM_SFU_BLOCKS / `VX_CFG_FETCH_WIDTH) : 1; + + function automatic logic [FETCH_WFS_W-1:0] wid_to_wfs( + input logic [NW_WIDTH-1:0] wid + ); + if (FETCH_WFS_BITS != 0) begin + wid_to_wfs = FETCH_WFS_W'(wid >> FETCH_FSW_BITS); + end else begin + wid_to_wfs = 0; + end + endfunction + + // A unit with B blocks serves, in block b, exactly the warps with wid mod B == b + // (lane dispatch maps issue slot {batch, block} and isw = wid mod ISSUE_WIDTH), + // so a block's warp state is indexed by the remaining high wid bits. + localparam SFU_BLOCK_BITS = `CLOG2(`VX_CFG_NUM_SFU_BLOCKS); + localparam SFU_BLOCK_W = `UP(SFU_BLOCK_BITS); + localparam SFU_BLOCK_WARPS = `VX_CFG_NUM_WARPS / `VX_CFG_NUM_SFU_BLOCKS; + localparam SFU_BLOCK_NW_BITS = `CLOG2(SFU_BLOCK_WARPS); + localparam SFU_BLOCK_NW_W = `UP(SFU_BLOCK_NW_BITS); + + function automatic logic [SFU_BLOCK_W-1:0] wid_to_sfu_block( + input logic [NW_WIDTH-1:0] wid + ); + if (SFU_BLOCK_BITS != 0) begin + wid_to_sfu_block = wid[SFU_BLOCK_W-1:0]; + end else begin + wid_to_sfu_block = 0; + end + endfunction + + function automatic logic [SFU_BLOCK_NW_W-1:0] wid_to_sfu_local( + input logic [NW_WIDTH-1:0] wid + ); + if (SFU_BLOCK_NW_BITS != 0) begin + wid_to_sfu_local = SFU_BLOCK_NW_W'(wid >> SFU_BLOCK_BITS); + end else begin + wid_to_sfu_local = 0; + end + endfunction + + function automatic logic [NW_WIDTH-1:0] sfu_local_to_wid( + input logic [SFU_BLOCK_NW_W-1:0] lid, + input logic [SFU_BLOCK_W-1:0] block + ); + if (SFU_BLOCK_NW_BITS == 0) begin + sfu_local_to_wid = NW_WIDTH'(block); + end else if (SFU_BLOCK_BITS == 0) begin + sfu_local_to_wid = NW_WIDTH'(lid); + end else begin + sfu_local_to_wid = NW_WIDTH'({lid, block}); + end + endfunction + + // The core is built from ISSUE_WIDTH compute slices (VX_compute_slice) when + // every execution unit has one block per issue slice; otherwise it keeps the + // stage-sliced issue / execute / commit structure. + localparam COMPUTE_SLICES_ENABLE = (`VX_CFG_NUM_ALU_BLOCKS == `VX_CFG_ISSUE_WIDTH) + && (`VX_CFG_NUM_LSU_BLOCKS == `VX_CFG_ISSUE_WIDTH) + && (`VX_CFG_NUM_SFU_BLOCKS == `VX_CFG_ISSUE_WIDTH) + && (!`VX_CFG_EXT_F_ENABLED || (`VX_CFG_NUM_FPU_BLOCKS == `VX_CFG_ISSUE_WIDTH)) + && (!`VX_CFG_EXT_TCU_ENABLED || (`VX_CFG_NUM_TCU_BLOCKS == `VX_CFG_ISSUE_WIDTH)); + /////////////////////////////// TENSOR UNIT /////////////////////////////// `ifdef VX_CFG_EXT_TCU_ENABLE @@ -678,6 +781,22 @@ package VX_gpu_pkg; logic [BAR_SIZE_W-1:0] size_m1; } barrier_t; + // Core-wide warp-control requests (barrier, wspawn, warp exit), one per + // SFU block, served by VX_warp_shared_unit. + localparam WSH_BAR = 2'd0; + localparam WSH_WSPAWN = 2'd1; + localparam WSH_EXIT = 2'd2; + + typedef struct packed { + logic [1:0] kind; + logic has_rsp; // the requester waits for its completion + logic [NW_WIDTH-1:0] wid; + barrier_t bar; + logic [BAR_ADDR_W-1:0] bar_addr; + wspawn_t wspawn; + logic [`VX_CFG_MEM_ADDR_WIDTH-1:0] mscratch; // spawning warp's, inherited by the spawned warps + } warp_shared_req_t; + typedef struct packed { logic [NB_WIDTH-1:0] id; logic [NC_WIDTH-1:0] size_m1; @@ -885,6 +1004,72 @@ package VX_gpu_pkg; cta_lane_t [`VX_CFG_NUM_THREADS-1:0] lane_launch; } cta_warp_t; + // CTA dispatcher state broadcast to the CTA-CSR stores that serve CSR reads: + // the per-CTA context, shared by every launch lane. + typedef struct packed { + logic ctx_write; + logic [NCTA_WIDTH-1:0] ctx_waddr; + cta_ctx_t ctx_wdata; + } cta_bcast_t; + + // One launch lane's per-warp state: the warp's rank/lane record and CTA id. + typedef struct packed { + logic warp_write; + logic [NW_WIDTH-1:0] warp_waddr; + cta_warp_t warp_wdata; + logic id_write; + logic [NW_WIDTH-1:0] id_wid; + logic [NCTA_WIDTH-1:0] id_cta; + } cta_lane_bcast_t; + + // Scheduler traffic between an FU block and the fetch slice owning the warp, + // addressed by core-wide wid. + + // ALU branch resolution + typedef struct packed { + logic valid; + logic [NW_WIDTH-1:0] wid; + logic taken; + logic [PC_BITS-1:0] dest; + logic is_trap; + logic is_mret; + logic [3:0] trap_cause; + } sched_branch_t; + + // SFU warp control + typedef struct packed { + logic tmc_valid; + logic split_valid; + logic sjoin_valid; + logic wsync_valid; + logic [NW_WIDTH-1:0] wid; + tmc_t tmc; + split_t split; + join_t sjoin; + } sched_wctl_t; + + // SFU CSR writes to per-warp state + typedef struct packed { + logic valid; + logic [NW_WIDTH-1:0] wid; + logic [`VX_CFG_MEM_ADDR_WIDTH-1:0] data; + logic [NUM_TRAP_CSRS-1:0] trap_valid; + logic [`VX_CFG_XLEN-1:0] trap_data; + } sched_csr_wr_t; + + // SFU CSR reads of per-warp state, selected by the read wid and CTA id + typedef struct packed { + logic [`VX_CFG_NUM_THREADS-1:0] thread_mask; + logic [`VX_CFG_MEM_ADDR_WIDTH-1:0] mscratch; + logic [`VX_CFG_XLEN-1:0] mstatus; + logic [`VX_CFG_XLEN-1:0] mtvec; + logic [`VX_CFG_XLEN-1:0] mepc; + logic [`VX_CFG_XLEN-1:0] mcause; + logic [`VX_CFG_XLEN-1:0] mtval; + logic [NW_WIDTH-1:0] cta_rank; + cta_lane_t [`VX_CFG_NUM_THREADS-1:0] cta_lane; + } sched_csr_rd_t; + //////////////////////// instruction arguments //////////////////////////// // tcu_args_t at 27 bits = 1+1+1+2+5+5+4+4+4. Sparse variants are distinct opcodes. @@ -1402,7 +1587,7 @@ package VX_gpu_pkg; // Width of the tag carried over VX_lsu_sched_if: lsu_header + op_type // + per-lane align + packet allocator addr + fence flag (matches the - // local TAG_WIDTH built inside VX_lsu_slice). + // local TAG_WIDTH built inside VX_lsu_block). localparam LSU_CLIENT_TAG_WIDTH = $bits(lsu_header_t) + INST_LSU_BITS + (`VX_CFG_NUM_LSU_LANES * `CLOG2(LSU_WORD_SIZE)) + `LOG2UP(`VX_CFG_LSU_PENDING_SIZE) + 1; @@ -1463,8 +1648,12 @@ package VX_gpu_pkg; // Block size in bytes localparam ICACHE_LINE_SIZE = `VX_CFG_L1_LINE_SIZE; - // Core request tag Id bits - localparam ICACHE_TAG_ID_BITS = NW_WIDTH; + // Request ports per core: one per fetch slice, each fetching its own warps + localparam ICACHE_NUM_REQS = `VX_CFG_FETCH_WIDTH; + localparam ICACHE_NUM_BANKS = `VX_CFG_ICACHE_NUM_BANKS; + + // Core request tag Id bits: the warp id local to the requesting fetch slice + localparam ICACHE_TAG_ID_BITS = FETCH_WFS_W; // Core request tag bits as VX_fetch writes them (UUID + warp id). localparam ICACHE_FETCH_TAG_WIDTH = (UUID_WIDTH + ICACHE_TAG_ID_BITS); @@ -1485,9 +1674,9 @@ package VX_gpu_pkg; // Memory request tag bits `ifdef VX_CFG_ICACHE_ENABLE - localparam ICACHE_MEM_TAG_WIDTH = `CACHE_CLUSTER_MEM_TAG_WIDTH(`VX_CFG_ICACHE_MSHR_SIZE, 1, 1, `VX_CFG_NUM_ICACHES, UUID_WIDTH); + localparam ICACHE_MEM_TAG_WIDTH = `CACHE_CLUSTER_MEM_TAG_WIDTH(`VX_CFG_ICACHE_MSHR_SIZE, ICACHE_NUM_BANKS, 1, `VX_CFG_NUM_ICACHES, UUID_WIDTH); `else - localparam ICACHE_MEM_TAG_WIDTH = `CACHE_CLUSTER_BYPASS_MEM_TAG_WIDTH(1, 1, ICACHE_LINE_SIZE, ICACHE_WORD_SIZE, ICACHE_TAG_WIDTH, `VX_CFG_SOCKET_SIZE, `VX_CFG_NUM_ICACHES); + localparam ICACHE_MEM_TAG_WIDTH = `CACHE_CLUSTER_BYPASS_MEM_TAG_WIDTH(ICACHE_NUM_REQS, 1, ICACHE_LINE_SIZE, ICACHE_WORD_SIZE, ICACHE_TAG_WIDTH, `VX_CFG_SOCKET_SIZE, `VX_CFG_NUM_ICACHES); `endif ////////////////////////// Dcache Parameters ////////////////////////////// @@ -1556,8 +1745,10 @@ package VX_gpu_pkg; `ifdef VX_CFG_EXT_TEX_ENABLE // tex unit: per-request queue + per-request tag localparam TEX_REQ_TAG_WIDTH = (UUID_WIDTH + `CLOG2(`VX_CFG_TEX_REQ_QUEUE_SIZE)); + // core-level arb: SFU blocks -> the core's texture bus + localparam TEX_CORE_TAG_WIDTH = (TEX_REQ_TAG_WIDTH + `ARB_SEL_BITS(`VX_CFG_NUM_SFU_BLOCKS, 1)); // Single socket-level arb: socket cores -> per-socket TEX units - localparam TEX_REQ_ARB1_TAG_WIDTH = (TEX_REQ_TAG_WIDTH + `ARB_SEL_BITS(`VX_CFG_SOCKET_SIZE, `VX_CFG_NUM_TEX_CORES)); + localparam TEX_REQ_ARB1_TAG_WIDTH = (TEX_CORE_TAG_WIDTH + `ARB_SEL_BITS(`VX_CFG_SOCKET_SIZE, `VX_CFG_NUM_TEX_CORES)); localparam TCACHE_WORD_SIZE = 4; localparam TCACHE_ADDR_WIDTH = (`VX_CFG_MEM_ADDR_WIDTH - `CLOG2(TCACHE_WORD_SIZE)); @@ -1679,8 +1870,10 @@ package VX_gpu_pkg; `ifdef VX_CFG_EXT_RTU_ENABLE localparam RTU_REQ_QUEUE_SIZE = 4; localparam RTU_REQ_TAG_WIDTH = (UUID_WIDTH + `CLOG2(RTU_REQ_QUEUE_SIZE)); + // core-level arb: SFU blocks -> the core's RTU bus + localparam RTU_CORE_TAG_WIDTH = (RTU_REQ_TAG_WIDTH + `ARB_SEL_BITS(`VX_CFG_NUM_SFU_BLOCKS, 1)); // Single socket-level arb: socket cores -> per-socket RTU units - localparam RTU_REQ_ARB1_TAG_WIDTH = (RTU_REQ_TAG_WIDTH + `ARB_SEL_BITS(`VX_CFG_SOCKET_SIZE, `VX_CFG_NUM_RTU_CORES)); + localparam RTU_REQ_ARB1_TAG_WIDTH = (RTU_CORE_TAG_WIDTH + `ARB_SEL_BITS(`VX_CFG_SOCKET_SIZE, `VX_CFG_NUM_RTU_CORES)); // The RTU fetches a whole CW-BVH node (one 64 B cache line) per request, // so its cache is line-granular: word size == line size, one port/core. diff --git a/hw/rtl/VX_graphics.sv b/hw/rtl/VX_graphics.sv index a2ef22cb37..329dd7fc67 100644 --- a/hw/rtl/VX_graphics.sv +++ b/hw/rtl/VX_graphics.sv @@ -21,11 +21,13 @@ module VX_graphics import VX_gpu_pkg::* , VX_om_pkg::* `endif ; #( - parameter CLUSTER_ID = 0 + parameter `STRING INSTANCE_ID = "" ) ( input wire clk, input wire reset, + input wire [CLUSTER_ID_WIDTH-1:0] cluster_id, + `ifdef PERF_ENABLE `ifdef VX_CFG_EXT_RASTER_ENABLE output raster_perf_t raster_perf, @@ -65,7 +67,10 @@ module VX_graphics import VX_gpu_pkg::* // no in-band `done`); the cluster ORs this into the device busy aggregation. output wire busy ); - `UNUSED_PARAM (CLUSTER_ID) + `UNUSED_SPARAM (INSTANCE_ID) +`ifndef VX_CFG_EXT_RASTER_ENABLE + `UNUSED_VAR (cluster_id) +`endif wire rcache_req_empty; wire ocache_req_empty; @@ -137,8 +142,7 @@ module VX_graphics import VX_gpu_pkg::* for (genvar i = 0; i < `VX_CFG_NUM_RASTER_CORES; ++i) begin : g_raster_core VX_raster_core #( - .INSTANCE_ID (`SFORMATF(("cluster%0d-raster%0d", CLUSTER_ID, i))), - .INSTANCE_IDX (CLUSTER_ID * `VX_CFG_NUM_RASTER_CORES + i), + .INSTANCE_ID (`SFORMATF(("%s-raster%0d", INSTANCE_ID, i))), .NUM_INSTANCES (`VX_CFG_NUM_CLUSTERS * `VX_CFG_NUM_RASTER_CORES), .NUM_SLICES (`VX_CFG_RASTER_NUM_SLICES), // The front end's top-level walk unit is a coarse bin: bin_x/bin_y @@ -154,6 +158,8 @@ module VX_graphics import VX_gpu_pkg::* ) raster_core ( .clk (clk), .reset (reset), + .instance_idx (`LOG2UP(`VX_CFG_NUM_CLUSTERS * `VX_CFG_NUM_RASTER_CORES)'(cluster_id) * `LOG2UP(`VX_CFG_NUM_CLUSTERS * `VX_CFG_NUM_RASTER_CORES)'(`VX_CFG_NUM_RASTER_CORES) + + `LOG2UP(`VX_CFG_NUM_CLUSTERS * `VX_CFG_NUM_RASTER_CORES)'(i)), `ifdef PERF_ENABLE .perf_raster_if (per_core_raster_perf_if[i]), `endif @@ -247,7 +253,7 @@ module VX_graphics import VX_gpu_pkg::* end VX_cache_cluster #( - .INSTANCE_ID (`SFORMATF(("cluster%0d-rcache", CLUSTER_ID))), + .INSTANCE_ID (`SFORMATF(("%s-rcache", INSTANCE_ID))), .NUM_UNITS (`VX_CFG_NUM_RCACHES), .NUM_INPUTS (`VX_CFG_NUM_RASTER_CORES), .TAG_SEL_IDX (0), @@ -338,7 +344,7 @@ module VX_graphics import VX_gpu_pkg::* for (genvar i = 0; i < L2_SOCKET_REQS; ++i) begin : g_om_steer VX_om_steer #( - .INSTANCE_ID (`SFORMATF(("cluster%0d-om-steer%0d", CLUSTER_ID, i))), + .INSTANCE_ID (`SFORMATF(("%s-om-steer%0d", INSTANCE_ID, i))), .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), .TAG_WIDTH (L2_TAG_WIDTH), .OUT_BUF (3) @@ -371,7 +377,7 @@ module VX_graphics import VX_gpu_pkg::* for (genvar i = 0; i < L2_SOCKET_REQS; ++i) begin : g_om_ingress VX_om_ingress #( - .INSTANCE_ID (`SFORMATF(("cluster%0d-om-ingress%0d", CLUSTER_ID, i))), + .INSTANCE_ID (`SFORMATF(("%s-om-ingress%0d", INSTANCE_ID, i))), .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), .TAG_WIDTH (L2_TAG_WIDTH), .NUM_LANES (OM_CORE_LANES), @@ -408,7 +414,7 @@ module VX_graphics import VX_gpu_pkg::* for (genvar i = 0; i < `VX_CFG_NUM_OM_CORES; ++i) begin : g_om_core VX_om_core #( - .INSTANCE_ID (`SFORMATF(("cluster%0d-om%0d", CLUSTER_ID, i))), + .INSTANCE_ID (`SFORMATF(("%s-om%0d", INSTANCE_ID, i))), .NUM_LANES (OM_CORE_LANES) ) om_core ( .clk (clk), @@ -588,7 +594,7 @@ module VX_graphics import VX_gpu_pkg::* `endif VX_cache_cluster #( - .INSTANCE_ID (`SFORMATF(("cluster%0d-ocache", CLUSTER_ID))), + .INSTANCE_ID (`SFORMATF(("%s-ocache", INSTANCE_ID))), .NUM_UNITS (`VX_CFG_NUM_OCACHES), .NUM_INPUTS (OCACHE_NUM_INPUTS), .TAG_SEL_IDX (0), diff --git a/hw/rtl/VX_socket.sv b/hw/rtl/VX_socket.sv index d65f86df73..e8dd0c8248 100644 --- a/hw/rtl/VX_socket.sv +++ b/hw/rtl/VX_socket.sv @@ -18,7 +18,6 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; import VX_dxa_pkg::*; `endif #( - parameter SOCKET_ID = 0, parameter `STRING INSTANCE_ID = "" ) ( `SCOPE_IO_DECL @@ -27,6 +26,8 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; input wire clk, input wire reset, + input wire [CORE_ID_WIDTH-1:0] socket_id, + `ifdef PERF_ENABLE input sysmem_perf_t sysmem_perf, `endif @@ -127,12 +128,12 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; VX_mem_bus_if #( .DATA_SIZE (ICACHE_WORD_SIZE), .TAG_WIDTH (ICACHE_TAG_WIDTH) - ) per_core_icache_bus_if[`VX_CFG_SOCKET_SIZE](); + ) per_core_icache_bus_if[`VX_CFG_SOCKET_SIZE * ICACHE_NUM_REQS](); VX_mem_bus_if #( .DATA_SIZE (ICACHE_WORD_SIZE), .TAG_WIDTH (ICACHE_TAG_WIDTH) - ) icache_phys_bus_if[`VX_CFG_SOCKET_SIZE](); + ) icache_phys_bus_if[`VX_CFG_SOCKET_SIZE * ICACHE_NUM_REQS](); VX_mem_bus_if #( .DATA_SIZE (ICACHE_LINE_SIZE), @@ -149,10 +150,10 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .TAG_SEL_IDX (0), .CACHE_SIZE (`VX_CFG_ICACHE_SIZE), .LINE_SIZE (ICACHE_LINE_SIZE), - .NUM_BANKS (1), + .NUM_BANKS (ICACHE_NUM_BANKS), .NUM_WAYS (`VX_CFG_ICACHE_NUM_WAYS), .WORD_SIZE (ICACHE_WORD_SIZE), - .NUM_REQS (1), + .NUM_REQS (ICACHE_NUM_REQS), .MEM_PORTS (1), .CRSQ_SIZE (`VX_CFG_ICACHE_CRSQ_SIZE), .MSHR_SIZE (`VX_CFG_ICACHE_MSHR_SIZE), @@ -302,7 +303,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; VX_mmu #( .INSTANCE_ID ("immu"), - .NUM_REQS (`VX_CFG_SOCKET_SIZE), + .NUM_REQS (`VX_CFG_SOCKET_SIZE * ICACHE_NUM_REQS), .TLB_SIZE (`VX_CFG_ITLB_SIZE), .EXEC_SIDE (1), .DATA_SIZE (ICACHE_WORD_SIZE), @@ -362,7 +363,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; for (genvar i = 0; i < `VX_CFG_SOCKET_SIZE * DCACHE_NUM_REQS; ++i) begin : g_dcache_phys_passthru `ASSIGN_VX_MEM_BUS_IF (dcache_phys_bus_if[i], per_core_dcache_bus_if[i]); end - for (genvar i = 0; i < `VX_CFG_SOCKET_SIZE; ++i) begin : g_icache_phys_passthru + for (genvar i = 0; i < `VX_CFG_SOCKET_SIZE * ICACHE_NUM_REQS; ++i) begin : g_icache_phys_passthru `ASSIGN_VX_MEM_BUS_IF (icache_phys_bus_if[i], per_core_icache_bus_if[i]); end `endif @@ -403,7 +404,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; `ifdef VX_CFG_EXT_TEX_ENABLE VX_tex_bus_if #( .NUM_LANES (`VX_CFG_NUM_SFU_LANES), - .TAG_WIDTH (TEX_REQ_TAG_WIDTH) + .TAG_WIDTH (TEX_CORE_TAG_WIDTH) ) per_core_tex_bus_if[`VX_CFG_SOCKET_SIZE](); VX_tex_bus_if #( @@ -415,7 +416,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .NUM_INPUTS (`VX_CFG_SOCKET_SIZE), .NUM_LANES (`VX_CFG_NUM_SFU_LANES), .NUM_OUTPUTS (`VX_CFG_NUM_TEX_CORES), - .TAG_WIDTH (TEX_REQ_TAG_WIDTH), + .TAG_WIDTH (TEX_CORE_TAG_WIDTH), .ARBITER ("R"), .OUT_BUF_REQ ((`VX_CFG_SOCKET_SIZE != `VX_CFG_NUM_TEX_CORES) ? 3 : 0), // register request fan-in .OUT_BUF_RSP ((`VX_CFG_SOCKET_SIZE != `VX_CFG_NUM_TEX_CORES) ? 3 : 0) // register response distribution (skid registers the core-facing rsp and its backward ready) @@ -556,7 +557,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; VX_rtu_bus_if #( .NUM_LANES (`VX_CFG_NUM_SFU_LANES), - .TAG_WIDTH (RTU_REQ_TAG_WIDTH), + .TAG_WIDTH (RTU_CORE_TAG_WIDTH), .SRC_WIDTH (1) // a core does not know its own index; the arbiter fills it in ) per_core_rtu_bus_if[`VX_CFG_SOCKET_SIZE](); @@ -570,7 +571,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .NUM_INPUTS (`VX_CFG_SOCKET_SIZE), .NUM_LANES (`VX_CFG_NUM_SFU_LANES), .NUM_OUTPUTS (`VX_CFG_NUM_RTU_CORES), - .TAG_WIDTH (RTU_REQ_TAG_WIDTH), + .TAG_WIDTH (RTU_CORE_TAG_WIDTH), .ARBITER ("R"), // The arm may be buffered: it does not mean "the RTU has taken your ray" // (the RTU stages a ray per {src, wid}), so registering it cannot let a @@ -878,13 +879,13 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; );*/ VX_core #( - .CORE_ID ((SOCKET_ID * `VX_CFG_SOCKET_SIZE) + core_id), .INSTANCE_ID (`SFORMATF(("%s-core%0d", INSTANCE_ID, core_id))) ) core ( `SCOPE_IO_BIND (scope_core + core_id) .clk (clk), .reset (reset), + .core_id (CORE_ID_WIDTH'(socket_id * `VX_CFG_SOCKET_SIZE) + CORE_ID_WIDTH'(core_id)), `ifdef PERF_ENABLE .sysmem_perf (sysmem_perf_tmp), @@ -894,7 +895,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .dcache_bus_if (per_core_dcache_bus_if[core_id * DCACHE_NUM_REQS +: DCACHE_NUM_REQS]), - .icache_bus_if (per_core_icache_bus_if[core_id]), + .icache_bus_if (per_core_icache_bus_if[core_id * ICACHE_NUM_REQS +: ICACHE_NUM_REQS]), `ifdef VX_CFG_EXT_DXA_ENABLE .dxa_req_bus_if (per_core_dxa_req_bus_if[core_id]), diff --git a/hw/rtl/Vortex.sv b/hw/rtl/Vortex.sv index a605dedd04..e536ba88b8 100644 --- a/hw/rtl/Vortex.sv +++ b/hw/rtl/Vortex.sv @@ -335,13 +335,13 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( for (genvar cluster_id = 0; cluster_id < `VX_CFG_NUM_CLUSTERS; ++cluster_id) begin : g_clusters VX_cluster #( - .CLUSTER_ID (cluster_id), .INSTANCE_ID (`SFORMATF(("cluster%0d", cluster_id))) ) cluster ( `SCOPE_IO_BIND (scope_cluster + cluster_id) .clk (clk), .reset (reset), + .cluster_id (CLUSTER_ID_WIDTH'(cluster_id)), `ifdef PERF_ENABLE .sysmem_perf (sysmem_perf), diff --git a/hw/rtl/core/VX_alu_block.sv b/hw/rtl/core/VX_alu_block.sv new file mode 100644 index 0000000000..b07a14a074 --- /dev/null +++ b/hw/rtl/core/VX_alu_block.sv @@ -0,0 +1,93 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// One ALU block: the integer and mul/div datapaths behind a PE switch. +module VX_alu_block import VX_gpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "", + parameter `STRING BLOCK_STR = "", + parameter NUM_LANES = 1, + parameter PARTIAL_BW = 0 +) ( + input wire clk, + input wire reset, + + input wire [`LOG2UP(`VX_CFG_NUM_ALU_BLOCKS)-1:0] block_idx, + + VX_execute_if.slave execute_if, + VX_result_if.master result_if, + VX_branch_ctl_if.master branch_ctl_if +); + localparam PE_COUNT = 1 + `VX_CFG_EXT_M_ENABLED; + localparam PE_SEL_BITS = `CLOG2(PE_COUNT); + localparam PE_IDX_INT = 0; + localparam PE_IDX_MDV = PE_IDX_INT + `VX_CFG_EXT_M_ENABLED; + + VX_execute_if #( + .data_t (alu_execute_t) + ) pe_execute_if[PE_COUNT](); + + VX_result_if#( + .data_t (alu_result_t) + ) pe_result_if[PE_COUNT](); + + reg [`UP(PE_SEL_BITS)-1:0] pe_select; + always @(*) begin + pe_select = PE_IDX_INT; + if (`VX_CFG_EXT_M_ENABLED && (execute_if.data.op_args.alu.xtype == ALU_TYPE_MULDIV)) begin + pe_select = PE_IDX_MDV; + end + end + + VX_pe_switch #( + .PE_COUNT (PE_COUNT), + .NUM_LANES (NUM_LANES), + .ARBITER ("R"), + .REQ_OUT_BUF (0), + .RSP_OUT_BUF (PARTIAL_BW ? 1 : 3) + ) pe_switch ( + .clk (clk), + .reset (reset), + .pe_sel (pe_select), + .execute_in_if (execute_if), + .result_out_if (result_if), + .execute_out_if (pe_execute_if), + .result_in_if (pe_result_if) + ); + + VX_alu_int #( + .INSTANCE_ID (`SFORMATF(("%s-int%s", INSTANCE_ID, BLOCK_STR))), + .NUM_LANES (NUM_LANES) + ) alu_int ( + .clk (clk), + .reset (reset), + .block_idx (block_idx), + .execute_if (pe_execute_if[PE_IDX_INT]), + .branch_ctl_if (branch_ctl_if), + .result_if (pe_result_if[PE_IDX_INT]) + ); + +`ifdef VX_CFG_EXT_M_ENABLE + VX_alu_muldiv #( + .INSTANCE_ID (`SFORMATF(("%s-muldiv%s", INSTANCE_ID, BLOCK_STR))), + .NUM_LANES (NUM_LANES) + ) muldiv_unit ( + .clk (clk), + .reset (reset), + .execute_if (pe_execute_if[PE_IDX_MDV]), + .result_if (pe_result_if[PE_IDX_MDV]) + ); +`endif + +endmodule diff --git a/hw/rtl/core/VX_alu_int.sv b/hw/rtl/core/VX_alu_int.sv index 3a680055bf..0c081cc07f 100644 --- a/hw/rtl/core/VX_alu_int.sv +++ b/hw/rtl/core/VX_alu_int.sv @@ -15,12 +15,13 @@ module VX_alu_int import VX_gpu_pkg::*; #( parameter `STRING INSTANCE_ID = "", - parameter BLOCK_IDX = 0, parameter NUM_LANES = 1 ) ( input wire clk, input wire reset, + input wire [`LOG2UP(`VX_CFG_NUM_ALU_BLOCKS)-1:0] block_idx, + // Inputs VX_execute_if.slave execute_if, @@ -339,7 +340,10 @@ module VX_alu_int import VX_gpu_pkg::*; #( : is_br_static ? from_fullPC(br_result) : cbr_dest_r; wire [NW_WIDTH-1:0] br_wid; - `ASSIGN_BLOCKED_WID (br_wid, result_if.data.header.wid, BLOCK_IDX, `VX_CFG_NUM_ALU_BLOCKS) + `ASSIGN_BLOCKED_WID (br_wid, result_if.data.header.wid, block_idx, `VX_CFG_NUM_ALU_BLOCKS) + if (`VX_CFG_NUM_ALU_BLOCKS == 1) begin : g_block_idx_unused + `UNUSED_VAR (block_idx) + end VX_pipe_register #( .DATAW (1 + NW_WIDTH + 1 + PC_BITS + 1 + 1 + 4), diff --git a/hw/rtl/core/VX_alu_unit.sv b/hw/rtl/core/VX_alu_unit.sv index 3128187c54..e91603251c 100644 --- a/hw/rtl/core/VX_alu_unit.sv +++ b/hw/rtl/core/VX_alu_unit.sv @@ -27,14 +27,9 @@ module VX_alu_unit import VX_gpu_pkg::*; #( VX_branch_ctl_if.master branch_ctl_if [`VX_CFG_NUM_ALU_BLOCKS] ); - `UNUSED_SPARAM (INSTANCE_ID) localparam BLOCK_SIZE = `VX_CFG_NUM_ALU_BLOCKS; localparam NUM_LANES = `VX_CFG_NUM_ALU_LANES; localparam PARTIAL_BW = (BLOCK_SIZE != `VX_CFG_ISSUE_WIDTH) || (NUM_LANES != `VX_CFG_SIMD_WIDTH); - localparam PE_COUNT = 1 + `VX_CFG_EXT_M_ENABLED; - localparam PE_SEL_BITS = `CLOG2(PE_COUNT); - localparam PE_IDX_INT = 0; - localparam PE_IDX_MDV = PE_IDX_INT + `VX_CFG_EXT_M_ENABLED; VX_execute_if #( .data_t (alu_execute_t) @@ -51,66 +46,25 @@ module VX_alu_unit import VX_gpu_pkg::*; #( ) lane_dispatch ( .clk (clk), .reset (reset), + .port_base ('0), .dispatch_if(dispatch_if), .execute_if (per_block_execute_if) ); for (genvar block_idx = 0; block_idx < BLOCK_SIZE; ++block_idx) begin : g_blocks - - VX_execute_if #( - .data_t (alu_execute_t) - ) pe_execute_if[PE_COUNT](); - - VX_result_if#( - .data_t (alu_result_t) - ) pe_result_if[PE_COUNT](); - - reg [`UP(PE_SEL_BITS)-1:0] pe_select; - always @(*) begin - pe_select = PE_IDX_INT; - if (`VX_CFG_EXT_M_ENABLED && (per_block_execute_if[block_idx].data.op_args.alu.xtype == ALU_TYPE_MULDIV)) - pe_select = PE_IDX_MDV; - end - - VX_pe_switch #( - .PE_COUNT (PE_COUNT), + VX_alu_block #( + .INSTANCE_ID (INSTANCE_ID), + .BLOCK_STR (`SFORMATF(("%0d", block_idx))), .NUM_LANES (NUM_LANES), - .ARBITER ("R"), - .REQ_OUT_BUF (0), - .RSP_OUT_BUF (PARTIAL_BW ? 1 : 3) - ) pe_switch ( - .clk (clk), - .reset (reset), - .pe_sel (pe_select), - .execute_in_if (per_block_execute_if[block_idx]), - .result_out_if (per_block_result_if[block_idx]), - .execute_out_if (pe_execute_if), - .result_in_if (pe_result_if) - ); - - VX_alu_int #( - .INSTANCE_ID (`SFORMATF(("%s-int%0d", INSTANCE_ID, block_idx))), - .BLOCK_IDX (block_idx), - .NUM_LANES (NUM_LANES) - ) alu_int ( - .clk (clk), - .reset (reset), - .execute_if (pe_execute_if[PE_IDX_INT]), - .branch_ctl_if (branch_ctl_if[block_idx]), - .result_if (pe_result_if[PE_IDX_INT]) - ); - - `ifdef VX_CFG_EXT_M_ENABLE - VX_alu_muldiv #( - .INSTANCE_ID (`SFORMATF(("%s-muldiv%0d", INSTANCE_ID, block_idx))), - .NUM_LANES (NUM_LANES) - ) muldiv_unit ( - .clk (clk), - .reset (reset), - .execute_if (pe_execute_if[PE_IDX_MDV]), - .result_if (pe_result_if[PE_IDX_MDV]) + .PARTIAL_BW (PARTIAL_BW) + ) alu_block ( + .clk (clk), + .reset (reset), + .block_idx (`LOG2UP(BLOCK_SIZE)'(block_idx)), + .execute_if (per_block_execute_if[block_idx]), + .result_if (per_block_result_if[block_idx]), + .branch_ctl_if (branch_ctl_if[block_idx]) ); - `endif end VX_lane_gather #( diff --git a/hw/rtl/core/VX_bar_unit.sv b/hw/rtl/core/VX_bar_unit.sv index f3f7313e92..f608b71e79 100644 --- a/hw/rtl/core/VX_bar_unit.sv +++ b/hw/rtl/core/VX_bar_unit.sv @@ -14,12 +14,13 @@ `include "VX_define.vh" module VX_bar_unit import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - parameter CORE_ID = 0 + parameter `STRING INSTANCE_ID = "" ) ( input wire clk, input wire reset, + input wire [CORE_ID_WIDTH-1:0] core_id, + // local barrier interface input wire req_valid, input wire [NW_WIDTH-1:0] req_wid, @@ -36,7 +37,6 @@ module VX_bar_unit import VX_gpu_pkg::*; #( output wire [`VX_CFG_NUM_WARPS-1:0] unlock_mask // warps to unlock ); `UNUSED_SPARAM (INSTANCE_ID) - `UNUSED_PARAM (CORE_ID) // warp mask + warp count + event count localparam EVENT_WIDTH = `CLOG2(`VX_CFG_MAX_BAR_EVENTS + 1); @@ -329,12 +329,13 @@ module VX_bar_unit import VX_gpu_pkg::*; #( assign gbar_bus_if.req_valid = gbar_req_valid_r; assign {gbar_bus_if.req_data.id, gbar_bus_if.req_data.size_m1} = req_queue_data; - assign gbar_bus_if.req_data.core_id = NC_WIDTH'(CORE_ID % `VX_CFG_NUM_CORES); + assign gbar_bus_if.req_data.core_id = NC_WIDTH'(32'(core_id) % `VX_CFG_NUM_CORES); assign gbar_bus_if.rsp_ready = gbar_rsp_ready; `RUNTIME_ASSERT(~gbar_enqueue || ~gbar_pending_r[gbar_enqueue_id], ("%s duplicate global barrier generation: id=%0d", INSTANCE_ID, gbar_enqueue_id)) `RUNTIME_ASSERT(~gbar_enqueue || (store_waddr == BAR_ADDR_W'(gbar_enqueue_id)), ("%s invalid global barrier slot: id=%0d, slot=%0d", INSTANCE_ID, gbar_enqueue_id, store_waddr)) end else begin : g_nogbar + `UNUSED_VAR (core_id) assign gbar_bus_if.req_valid = 0; assign gbar_bus_if.req_data = 'x; diff --git a/hw/rtl/core/VX_commit.sv b/hw/rtl/core/VX_commit.sv index 44a5437129..4cc0927c18 100644 --- a/hw/rtl/core/VX_commit.sv +++ b/hw/rtl/core/VX_commit.sv @@ -26,125 +26,28 @@ module VX_commit import VX_gpu_pkg::*; #( VX_writeback_if.master writeback_if [`VX_CFG_ISSUE_WIDTH], VX_commit_sched_if.master commit_sched_if ); - `UNUSED_SPARAM (INSTANCE_ID) - localparam OUT_DATAW = $bits(commit_t); - - // commit arbitration - - VX_commit_if commit_arb_if[`VX_CFG_ISSUE_WIDTH](); - wire [`VX_CFG_ISSUE_WIDTH-1:0] committed_warps; - wire [`VX_CFG_ISSUE_WIDTH-1:0][PER_ISSUE_WARPS-1:0] commit_eop_wis; - - for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_commit_arbs - - wire [NUM_EX_UNITS-1:0] valid_in; - wire [NUM_EX_UNITS-1:0][PER_ISSUE_WARPS+OUT_DATAW-1:0] data_in; - wire [NUM_EX_UNITS-1:0] ready_in; - - // Decoded ahead of the output register so each warp's release reads - // its own flop. - for (genvar j = 0; j < NUM_EX_UNITS; ++j) begin : g_data_in - wire [PER_ISSUE_WARPS-1:0] eop_wis = PER_ISSUE_WARPS'(commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data.eop) - << wid_to_wis(commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data.wid); - assign valid_in[j] = commit_if[j * `VX_CFG_ISSUE_WIDTH + i].valid; - assign data_in[j] = {eop_wis, commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data}; - assign commit_if[j * `VX_CFG_ISSUE_WIDTH + i].ready = ready_in[j]; - end - - VX_stream_arb #( - .NUM_INPUTS (NUM_EX_UNITS), - .DATAW (PER_ISSUE_WARPS + OUT_DATAW), - .ARBITER ("P"), - .OUT_BUF (1) - ) commit_arb ( - .clk (clk), - .reset (reset), - .valid_in (valid_in), - .ready_in (ready_in), - .data_in (data_in), - .data_out ({commit_eop_wis[i], commit_arb_if[i].data}), - .valid_out (commit_arb_if[i].valid), - .ready_out (commit_arb_if[i].ready), - `UNUSED_PIN (sel_out) - ); - - wire commit_arb_fire = commit_arb_if[i].valid && commit_arb_if[i].ready; - assign committed_warps[i] = commit_arb_fire && commit_arb_if[i].data.eop; - end - - // notify scheduler: build per-warp committed mask from per-slot signals - wire [`VX_CFG_ISSUE_WIDTH-1:0][NW_WIDTH-1:0] committed_slot_wid; - for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_committed_wid - assign committed_slot_wid[i] = commit_arb_if[i].data.wid; - end - - logic [`VX_CFG_NUM_WARPS-1:0] committed_warp_mask; - wire [`VX_CFG_NUM_WARPS-1:0] committed_warp_mask_r; - always_comb begin - committed_warp_mask = '0; - for (integer i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin - if (committed_warps[i]) begin - committed_warp_mask[committed_slot_wid[i]] = 1'b1; - end + for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_slices + VX_commit_if slice_commit_if[NUM_EX_UNITS](); + for (genvar j = 0; j < NUM_EX_UNITS; ++j) begin : g_commit_if + `ASSIGN_VX_IF(slice_commit_if[j], commit_if[j * `VX_CFG_ISSUE_WIDTH + i]); end - end - `BUFFER(committed_warp_mask_r, committed_warp_mask); - assign commit_sched_if.committed_warps = committed_warp_mask_r; - - // Writeback - for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_writeback - wire [XLENB_W-1:0] bytesel_size = commit_arb_if[i].data.bytesel[BYTESEL_BITS-1 -: XLENB_W]; - wire [XLENB_W-1:0] bytesel_off = commit_arb_if[i].data.bytesel[0 +: XLENB_W]; - wire [`VX_CFG_SIMD_WIDTH-1:0][`VX_CFG_XLEN-1:0] writeback_data; - wire [`VX_CFG_SIMD_WIDTH-1:0][XLENB-1:0] writeback_byteen; + wire [PER_ISSUE_WARPS-1:0] committed_wis; - wire [XLENB-1:0] size_mask = (bytesel_size == XLENB_W'(7)) ? XLENB'(255) : - (bytesel_size == XLENB_W'(6)) ? XLENB'(127) : - (bytesel_size == XLENB_W'(5)) ? XLENB'(63) : - (bytesel_size == XLENB_W'(4)) ? XLENB'(31) : - (bytesel_size == XLENB_W'(3)) ? XLENB'(15) : - (bytesel_size == XLENB_W'(2)) ? XLENB'(7) : - (bytesel_size == XLENB_W'(1)) ? XLENB'(3) : XLENB'(1); - wire [XLENB-1:0] base_byteen = size_mask << bytesel_off; - - for (genvar lane = 0; lane < `VX_CFG_SIMD_WIDTH; ++lane) begin : g_bytesel - assign writeback_data[lane] = commit_arb_if[i].data.data[lane] << (8 * bytesel_off); - assign writeback_byteen[lane] = commit_arb_if[i].data.tmask[lane] ? base_byteen : '0; - end - - assign writeback_if[i].valid = commit_arb_if[i].valid; - assign writeback_if[i].data.uuid = commit_arb_if[i].data.uuid; - assign writeback_if[i].data.wis = wid_to_wis(commit_arb_if[i].data.wid); - assign writeback_if[i].data.eop_wis = commit_eop_wis[i]; - assign writeback_if[i].data.cta_id = commit_arb_if[i].data.cta_id; - assign writeback_if[i].data.sid = commit_arb_if[i].data.sid; - assign writeback_if[i].data.PC = commit_arb_if[i].data.PC; - assign writeback_if[i].data.tmask= commit_arb_if[i].data.tmask; - assign writeback_if[i].data.wb = commit_arb_if[i].data.wb; - assign writeback_if[i].data.wr_xregs = commit_arb_if[i].data.wr_xregs; - assign writeback_if[i].data.rd = commit_arb_if[i].data.rd; - assign writeback_if[i].data.byteen = writeback_byteen; - assign writeback_if[i].data.data = writeback_data; - assign writeback_if[i].data.sop = commit_arb_if[i].data.sop; - assign writeback_if[i].data.eop = commit_arb_if[i].data.eop; - assign commit_arb_if[i].ready = 1; - end + VX_commit_slice #( + .INSTANCE_ID (INSTANCE_ID) + ) commit_slice ( + .clk (clk), + .reset (reset), + .commit_if (slice_commit_if), + .writeback_if (writeback_if[i]), + .committed_wis (committed_wis) + ); -`ifdef DBG_TRACE_PIPELINE - for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_trace - for (genvar j = 0; j < NUM_EX_UNITS; ++j) begin : g_j - always @(posedge clk) begin - if (commit_if[j * `VX_CFG_ISSUE_WIDTH + i].valid && commit_if[j * `VX_CFG_ISSUE_WIDTH + i].ready) begin - `TRACE(1, ("%t: %s commit: wid=%0d, cta_id=%0d, sid=%0d, PC=0x%0h, ex=", $time, INSTANCE_ID, commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data.wid, commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data.cta_id, commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data.sid, to_fullPC(commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data.PC))) - VX_trace_pkg::trace_ex_type(1, j); - `TRACE(1, (", tmask=%b, wb=%b, wr_xregs=%b, rd=%0d, sop=%b, eop=%b, data=", commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data.tmask, commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data.wb, commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data.wr_xregs, commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data.rd, commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data.sop, commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data.eop)) - `TRACE_ARRAY1D(1, "0x%0h", commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data.data, `VX_CFG_SIMD_WIDTH) - `TRACE(1, (" (#%0d)\n", commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data.uuid)) - end - end + // wid = {wis, isw}: slice i owns every ISSUE_WIDTH-th warp starting at i + for (genvar w = 0; w < PER_ISSUE_WARPS; ++w) begin : g_committed + assign commit_sched_if.committed_warps[w * `VX_CFG_ISSUE_WIDTH + i] = committed_wis[w]; end end -`endif endmodule diff --git a/hw/rtl/core/VX_commit_slice.sv b/hw/rtl/core/VX_commit_slice.sv new file mode 100644 index 0000000000..b2a1133d24 --- /dev/null +++ b/hw/rtl/core/VX_commit_slice.sv @@ -0,0 +1,124 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// Commit for one issue slice: arbitrates its execution units' results onto the +// slice's writeback port and reports the warps whose instruction retired. +module VX_commit_slice import VX_gpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "" +) ( + input wire clk, + input wire reset, + + // inputs + VX_commit_if.slave commit_if [NUM_EX_UNITS], + + // outputs + VX_writeback_if.master writeback_if, + output wire [PER_ISSUE_WARPS-1:0] committed_wis // registered +); + `UNUSED_SPARAM (INSTANCE_ID) + localparam OUT_DATAW = $bits(commit_t); + + VX_commit_if commit_arb_if(); + wire [PER_ISSUE_WARPS-1:0] commit_eop_wis; + + wire [NUM_EX_UNITS-1:0] valid_in; + wire [NUM_EX_UNITS-1:0][PER_ISSUE_WARPS+OUT_DATAW-1:0] data_in; + wire [NUM_EX_UNITS-1:0] ready_in; + + // Decoded ahead of the output register so each warp's release reads + // its own flop. + for (genvar j = 0; j < NUM_EX_UNITS; ++j) begin : g_data_in + wire [PER_ISSUE_WARPS-1:0] eop_wis = PER_ISSUE_WARPS'(commit_if[j].data.eop) + << wid_to_wis(commit_if[j].data.wid); + assign valid_in[j] = commit_if[j].valid; + assign data_in[j] = {eop_wis, commit_if[j].data}; + assign commit_if[j].ready = ready_in[j]; + end + + VX_stream_arb #( + .NUM_INPUTS (NUM_EX_UNITS), + .DATAW (PER_ISSUE_WARPS + OUT_DATAW), + .ARBITER ("P"), + .OUT_BUF (1) + ) commit_arb ( + .clk (clk), + .reset (reset), + .valid_in (valid_in), + .ready_in (ready_in), + .data_in (data_in), + .data_out ({commit_eop_wis, commit_arb_if.data}), + .valid_out (commit_arb_if.valid), + .ready_out (commit_arb_if.ready), + `UNUSED_PIN (sel_out) + ); + + // notify scheduler + wire [PER_ISSUE_WARPS-1:0] committed_wis_n = commit_arb_if.valid ? commit_eop_wis : '0; + `BUFFER(committed_wis, committed_wis_n); + + // Writeback + + wire [XLENB_W-1:0] bytesel_size = commit_arb_if.data.bytesel[BYTESEL_BITS-1 -: XLENB_W]; + wire [XLENB_W-1:0] bytesel_off = commit_arb_if.data.bytesel[0 +: XLENB_W]; + wire [`VX_CFG_SIMD_WIDTH-1:0][`VX_CFG_XLEN-1:0] writeback_data; + wire [`VX_CFG_SIMD_WIDTH-1:0][XLENB-1:0] writeback_byteen; + + wire [XLENB-1:0] size_mask = (bytesel_size == XLENB_W'(7)) ? XLENB'(255) : + (bytesel_size == XLENB_W'(6)) ? XLENB'(127) : + (bytesel_size == XLENB_W'(5)) ? XLENB'(63) : + (bytesel_size == XLENB_W'(4)) ? XLENB'(31) : + (bytesel_size == XLENB_W'(3)) ? XLENB'(15) : + (bytesel_size == XLENB_W'(2)) ? XLENB'(7) : + (bytesel_size == XLENB_W'(1)) ? XLENB'(3) : XLENB'(1); + wire [XLENB-1:0] base_byteen = size_mask << bytesel_off; + + for (genvar lane = 0; lane < `VX_CFG_SIMD_WIDTH; ++lane) begin : g_bytesel + assign writeback_data[lane] = commit_arb_if.data.data[lane] << (8 * bytesel_off); + assign writeback_byteen[lane] = commit_arb_if.data.tmask[lane] ? base_byteen : '0; + end + + assign writeback_if.valid = commit_arb_if.valid; + assign writeback_if.data.uuid = commit_arb_if.data.uuid; + assign writeback_if.data.wis = wid_to_wis(commit_arb_if.data.wid); + assign writeback_if.data.eop_wis = commit_eop_wis; + assign writeback_if.data.cta_id = commit_arb_if.data.cta_id; + assign writeback_if.data.sid = commit_arb_if.data.sid; + assign writeback_if.data.PC = commit_arb_if.data.PC; + assign writeback_if.data.tmask= commit_arb_if.data.tmask; + assign writeback_if.data.wb = commit_arb_if.data.wb; + assign writeback_if.data.wr_xregs = commit_arb_if.data.wr_xregs; + assign writeback_if.data.rd = commit_arb_if.data.rd; + assign writeback_if.data.byteen = writeback_byteen; + assign writeback_if.data.data = writeback_data; + assign writeback_if.data.sop = commit_arb_if.data.sop; + assign writeback_if.data.eop = commit_arb_if.data.eop; + assign commit_arb_if.ready = 1; + +`ifdef DBG_TRACE_PIPELINE + for (genvar j = 0; j < NUM_EX_UNITS; ++j) begin : g_trace + always @(posedge clk) begin + if (commit_if[j].valid && commit_if[j].ready) begin + `TRACE(1, ("%t: %s commit: wid=%0d, cta_id=%0d, sid=%0d, PC=0x%0h, ex=", $time, INSTANCE_ID, commit_if[j].data.wid, commit_if[j].data.cta_id, commit_if[j].data.sid, to_fullPC(commit_if[j].data.PC))) + VX_trace_pkg::trace_ex_type(1, j); + `TRACE(1, (", tmask=%b, wb=%b, wr_xregs=%b, rd=%0d, sop=%b, eop=%b, data=", commit_if[j].data.tmask, commit_if[j].data.wb, commit_if[j].data.wr_xregs, commit_if[j].data.rd, commit_if[j].data.sop, commit_if[j].data.eop)) + `TRACE_ARRAY1D(1, "0x%0h", commit_if[j].data.data, `VX_CFG_SIMD_WIDTH) + `TRACE(1, (" (#%0d)\n", commit_if[j].data.uuid)) + end + end + end +`endif + +endmodule diff --git a/hw/rtl/core/VX_compute_slice.sv b/hw/rtl/core/VX_compute_slice.sv new file mode 100644 index 0000000000..33c771fdc1 --- /dev/null +++ b/hw/rtl/core/VX_compute_slice.sv @@ -0,0 +1,192 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// One issue slice carried through issue, execute and commit, serving the warps +// with wid mod ISSUE_WIDTH == slice_id with one block of every execution unit. +// Every slice is the same module: its identity arrives on slice_id/core_id, and +// only the core-wide resources (fetch slices, warp-control resources, memory +// system, SFU buses, WGMMA tile buffers and the metadata AGU) are reached through +// its ports. +module VX_compute_slice import VX_gpu_pkg::*; +`ifdef VX_CFG_EXT_TCU_ENABLE +import VX_tcu_pkg::*; +`endif +#( + parameter `STRING INSTANCE_ID = "", + parameter `STRING SLICE_STR = "" +) ( + `SCOPE_IO_DECL + + input wire clk, + input wire reset, + + input wire [CORE_ID_WIDTH-1:0] core_id, + input wire [ISSUE_ISW_W-1:0] slice_id, + +`ifdef PERF_ENABLE + output issue_perf_t issue_perf, +`endif + + // decoded instructions of the slice's warps + VX_decode_if.slave decode_if, + + // issued and retired warps (registered), indexed by wid / ISSUE_WIDTH + output wire warp_issued, + output wire [ISSUE_WIS_W-1:0] warp_issued_wis, + output wire [PER_ISSUE_WARPS-1:0] committed_wis, + + // scheduler traffic of the slice's ALU and SFU blocks + VX_branch_ctl_if.master branch_ctl_if, + VX_warp_ctl_if.master warp_ctl_if, + VX_sched_csr_if.slave sched_csr_if, +`ifdef VX_CFG_EXT_RTU_ENABLE + VX_sched_unlock_if.master sched_unlock_if, +`endif + + // LSU + VX_lsu_sched_if.master lsu_client_if, + + // SFU +`ifdef VX_CFG_EXT_DXA_ENABLE + VX_dxa_req_bus_if.master dxa_req_bus_if, +`endif +`ifdef VX_CFG_EXT_TEX_ENABLE + VX_tex_bus_if.master tex_bus_if, +`endif +`ifdef VX_CFG_EXT_RTU_ENABLE + VX_rtu_bus_if.master rtu_bus_if, +`endif + + // TCU +`ifdef VX_CFG_TCU_WGMMA_ENABLE + output wire tcu_exec_valid, + output wire tcu_exec_ready, + output tcu_execute_t tcu_exec_data, + input wire [TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs1_data, + input wire [TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs2_data, + input wire tcu_tbuf_ready, +`endif +`ifdef TCU_META_ENABLE + output wire tcu_agu_ld_valid, + output tcu_execute_t tcu_agu_ld_data, + input wire tcu_agu_ld_ready, + input wire tcu_agu_result_valid, + input tcu_result_t tcu_agu_result_data, + output wire tcu_agu_result_ready, + input wire tcu_meta_wr_en, + input wire [NW_WIDTH-1:0] tcu_meta_wr_wid, + input wire [4:0] tcu_meta_wr_idx, + input wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] tcu_meta_wr_data, +`endif + + // core-wide warp control + VX_warp_shared_if.master warp_shared_if +); + `STATIC_ASSERT (COMPUTE_SLICES_ENABLE, ("invalid parameter")) + + `SCOPE_IO_SWITCH (2); + + VX_dispatch_if dispatch_if [NUM_EX_UNITS](); + VX_commit_if commit_if [NUM_EX_UNITS](); + VX_writeback_if writeback_if(); + + // issue + + wire issue_fire; + wire [ISSUE_WIS_W-1:0] issue_wis; + + VX_issue_slice #( + .INSTANCE_ID (`SFORMATF(("%s-issue%s", INSTANCE_ID, SLICE_STR))) + ) issue_slice ( + `SCOPE_IO_BIND (0) + .clk (clk), + .reset (reset), + .slice_id (slice_id), + `ifdef PERF_ENABLE + .issue_perf (issue_perf), + `endif + .decode_if (decode_if), + .writeback_if (writeback_if), + .dispatch_if (dispatch_if), + .warp_issued (issue_fire), + .warp_issued_wis (issue_wis) + ); + + `BUFFER(warp_issued, issue_fire); + `BUFFER(warp_issued_wis, issue_wis); + + // execute + + VX_execute_slice #( + .INSTANCE_ID (`SFORMATF(("%s-execute", INSTANCE_ID))), + .SLICE_STR (SLICE_STR) + ) execute_slice ( + `SCOPE_IO_BIND (1) + .clk (clk), + .reset (reset), + .core_id (core_id), + .slice_id (slice_id), + .dispatch_if (dispatch_if), + .commit_if (commit_if), + .branch_ctl_if (branch_ctl_if), + .lsu_client_if (lsu_client_if), + `ifdef VX_CFG_EXT_DXA_ENABLE + .dxa_req_bus_if (dxa_req_bus_if), + `endif + `ifdef VX_CFG_EXT_TEX_ENABLE + .tex_bus_if (tex_bus_if), + `endif + `ifdef VX_CFG_EXT_RTU_ENABLE + .rtu_bus_if (rtu_bus_if), + .sched_unlock_if (sched_unlock_if), + `endif + `ifdef VX_CFG_TCU_WGMMA_ENABLE + .tcu_exec_valid (tcu_exec_valid), + .tcu_exec_ready (tcu_exec_ready), + .tcu_exec_data (tcu_exec_data), + .tcu_tbuf_rs1_data (tcu_tbuf_rs1_data), + .tcu_tbuf_rs2_data (tcu_tbuf_rs2_data), + .tcu_tbuf_ready (tcu_tbuf_ready), + `endif + `ifdef TCU_META_ENABLE + .tcu_agu_ld_valid (tcu_agu_ld_valid), + .tcu_agu_ld_data (tcu_agu_ld_data), + .tcu_agu_ld_ready (tcu_agu_ld_ready), + .tcu_agu_result_valid (tcu_agu_result_valid), + .tcu_agu_result_data (tcu_agu_result_data), + .tcu_agu_result_ready (tcu_agu_result_ready), + .tcu_meta_wr_en (tcu_meta_wr_en), + .tcu_meta_wr_wid (tcu_meta_wr_wid), + .tcu_meta_wr_idx (tcu_meta_wr_idx), + .tcu_meta_wr_data (tcu_meta_wr_data), + `endif + .warp_shared_if (warp_shared_if), + .warp_ctl_if (warp_ctl_if), + .sched_csr_if (sched_csr_if) + ); + + // commit + + VX_commit_slice #( + .INSTANCE_ID (`SFORMATF(("%s-commit", INSTANCE_ID))) + ) commit_slice ( + .clk (clk), + .reset (reset), + .commit_if (commit_if), + .writeback_if (writeback_if), + .committed_wis (committed_wis) + ); + +endmodule diff --git a/hw/rtl/core/VX_core.sv b/hw/rtl/core/VX_core.sv index 618fabade9..b8aaf1d528 100644 --- a/hw/rtl/core/VX_core.sv +++ b/hw/rtl/core/VX_core.sv @@ -17,8 +17,11 @@ `include "VX_fpu_define.vh" `endif -module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( - parameter CORE_ID = 0, +module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; +`ifdef VX_CFG_EXT_TCU_ENABLE +import VX_tcu_pkg::*; +`endif +#( parameter `STRING INSTANCE_ID = "" ) ( `SCOPE_IO_DECL @@ -27,6 +30,8 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( input wire clk, input wire reset, + input wire [CORE_ID_WIDTH-1:0] core_id, + `ifdef PERF_ENABLE input sysmem_perf_t sysmem_perf, `endif @@ -35,13 +40,11 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( VX_mem_bus_if.master dcache_bus_if [DCACHE_NUM_REQS], - VX_mem_bus_if.master icache_bus_if, + VX_mem_bus_if.master icache_bus_if [ICACHE_NUM_REQS], `ifdef VX_CFG_VM_ENABLE - // Address translation is relocated to the socket (one MMU per L1 cache); the - // core emits virtual addresses on its cache buses. The socket returns its - // MMU drain state so the core's busy/barrier logic still waits for in-flight - // translations to retire before completion/suspend. + // Translation happens at the socket (one MMU per L1 cache): the core emits + // virtual addresses, and busy/barrier logic waits on the socket MMU draining. input wire mmu_drained, `endif @@ -54,12 +57,6 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( VX_tex_bus_if.master tex_bus_if, `endif -`ifdef VX_CFG_EXT_OM_ENABLE -`endif - -`ifdef VX_CFG_EXT_RASTER_ENABLE -`endif - `ifdef VX_CFG_EXT_RTU_ENABLE VX_rtu_bus_if.master rtu_bus_if, `endif @@ -77,40 +74,19 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( // Status output wire busy ); - VX_schedule_if schedule_if(); - VX_fetch_if fetch_if(); - VX_decode_if decode_if(); - VX_sched_csr_if sched_csr_if(); - VX_decode_sched_if decode_sched_if(); - VX_issue_sched_if issue_sched_if[`VX_CFG_ISSUE_WIDTH](); - VX_commit_sched_if commit_sched_if(); - VX_branch_ctl_if branch_ctl_if[`VX_CFG_NUM_ALU_BLOCKS](); - VX_warp_ctl_if warp_ctl_if(); -`ifdef VX_CFG_EXT_RTU_ENABLE - VX_sched_unlock_if sched_unlock_if(); // RTU TRACE wstall release -> scheduler -`endif - - VX_dispatch_if dispatch_if[NUM_EX_UNITS * `VX_CFG_ISSUE_WIDTH](); - VX_commit_if commit_if[NUM_EX_UNITS * `VX_CFG_ISSUE_WIDTH](); - VX_writeback_if writeback_if[`VX_CFG_ISSUE_WIDTH](); + VX_warp_shared_if warp_shared_if[`VX_CFG_NUM_SFU_BLOCKS](); `ifdef VX_CFG_EXT_DXA_ENABLE VX_txbar_bus_if dxa_txbar_bus_if(); `endif - // cta_table_if removed: cluster-contiguous LMEM placement lets the - // DXA multicast writer compute receiver addresses as - // `issuer_base + r × smem_stride`, eliminating the per-slot lookup table. - VX_lsu_mem_if #( .NUM_LANES (`VX_CFG_NUM_LSU_LANES), .DATA_SIZE (LSU_WORD_SIZE), .TAG_WIDTH (LSU_TAG_WIDTH) ) lsu_mem_if[`VX_CFG_NUM_LSU_BLOCKS](); - // VX_lsu_scheduler instantiated per-LSU-block; all blocks have NUM_CLIENTS=2. - // Block 0 wires client 1 to the warp-level TCU AGU; other blocks tie it off. - // LSU client interfaces flow from execute as client 0. + // per-LSU-block memory scheduler clients from execute VX_lsu_sched_if lsu_client_if [`VX_CFG_NUM_LSU_BLOCKS](); wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_sched_empty; @@ -118,24 +94,11 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( VX_lsu_sched_if tcu_mem_if(); `endif - VX_mem_bus_if #( - .DATA_SIZE (DCACHE_WORD_SIZE), - .TAG_WIDTH (DCACHE_TAG_WIDTH_BASE) - ) mmu_dcache_if[DCACHE_NUM_REQS](); - - // VX_fetch -> VX_dcr_flush -> mmu_icache_if -> (i)MMU -> icache_bus_if. - // The flush wrapper consumes the dcr-triggered cache-flush request and - // injects a synthetic flush request into the icache stream, mirroring - // the dcache path in VX_mem_unit. + // fetch-slice icache requests; port 0 also carries the injected flush VX_mem_bus_if #( .DATA_SIZE (ICACHE_WORD_SIZE), .TAG_WIDTH (ICACHE_FETCH_TAG_WIDTH) - ) fetch_icache_if[1](); - - VX_mem_bus_if #( - .DATA_SIZE (ICACHE_WORD_SIZE), - .TAG_WIDTH (ICACHE_TAG_WIDTH_BASE) - ) mmu_icache_if[1](); + ) fetch_icache_if[ICACHE_NUM_REQS](); `ifdef VX_CFG_TCU_WGMMA_ENABLE localparam TCU_LMEM_BANK_ADDR_W = `VX_CFG_LMEM_LOG_SIZE - `CLOG2(LSU_WORD_SIZE) - `CLOG2(`VX_CFG_LMEM_NUM_BANKS); @@ -147,7 +110,6 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( ) tcu_lmem_if(); `endif - `ifdef PERF_ENABLE lmem_perf_t lmem_perf; coalescer_perf_t coalescer_perf; @@ -166,35 +128,25 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( VX_dcr_csr_if dcr_csr_if(); - // Single DCR-flush trigger from VX_dcr_data, fanned out below to BOTH - // the dcache (via VX_mem_unit) and the icache (via the wrapper inserted - // between VX_fetch and the iMMU). Without invalidating the icache too, - // a kernel re-loaded to the same VMA after a CACHE_FLUSH would execute - // stale lines from the previous launch. + // One DCR flush request invalidates both L1s: a kernel re-loaded at the same + // address must not execute stale icache lines. VX_dcr_flush_if dcr_flush_if(); VX_dcr_flush_if dcr_flush_dcache_if(); VX_dcr_flush_if dcr_flush_icache_if(); wire dcr_flush_l1_only; - // High when the in-core global-store path is empty (no store still in - // flight toward the dcache). Gates `busy` so the flush waits for the tail. + // the in-core global-store path is empty wire mem_unit_empty; - // Hold the dcache flush request until this core's stores have all reached L1. - // Stores are ack-less and lag warp-exit by many cycles, and the flush arbiter - // sits at the adapter output with no upstream visibility, so without this gate a - // flush could inject ahead of a store still in the LSU/coalescer and lose it. + // Stores are ack-less and lag warp exit, so the dcache flush waits until every + // store of this core has reached L1. wire store_drained = (&lsu_sched_empty) && mem_unit_empty; assign dcr_flush_dcache_if.req = dcr_flush_if.req && store_drained; // Only the dcache flush walks the shared levels (the icache does not // forward its flush), so the icache flushes alongside it. assign dcr_flush_icache_if.req = dcr_flush_if.req; - // Each VX_dcr_flush holds .done level-high until its req drops, so a - // straight AND of the two dones reports the combined completion to - // VX_dcr_data — which then drops req, re-arming both for the next flush. - // When gfx extensions are enabled, also gate on the cluster-shared - // gfx-cache flush done so that VX_dcr_data doesn't retire the flush DCR - // before tcache/rcache/ocache have actually invalidated. + // Each flush holds done until its req drops, so the dones AND into the combined + // completion (with the cluster graphics caches when present). `ifdef EXT_GFX_ANY_ENABLE assign cluster_flush_if.req = dcr_flush_if.req; assign dcr_flush_if.done = dcr_flush_dcache_if.done & dcr_flush_icache_if.done & cluster_flush_if.done; @@ -203,179 +155,531 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( `endif VX_dcr_data #( - .INSTANCE_ID (`SFORMATF(("%s-dcr_data", INSTANCE_ID))), - .CORE_ID (CORE_ID) + .INSTANCE_ID (`SFORMATF(("%s-dcr_data", INSTANCE_ID))) ) dcr_data ( .clk (clk), .reset (reset), + .core_id (core_id), .dcr_bus_if (dcr_bus_if), .dcr_csr_if (dcr_csr_if), .dcr_flush_if(dcr_flush_if), .flush_l1_only(dcr_flush_l1_only) ); - `SCOPE_IO_SWITCH (3); - - wire sched_busy; - VX_scheduler #( - .INSTANCE_ID (`SFORMATF(("%s-scheduler", INSTANCE_ID))), - .CORE_ID (CORE_ID) - ) scheduler ( - .clk (clk), - .reset (reset), + wire [PERF_CTR_BITS-1:0] sched_cycles; + wire [PERF_CTR_BITS-1:0] sched_instret; + wire lsu_drained; + VX_csr_mpm #( + .INSTANCE_ID (`SFORMATF(("%s-mpm", INSTANCE_ID))) + ) csr_mpm ( `ifdef PERF_ENABLE - .sched_perf (pipeline_perf.sched), - `endif - - .warp_ctl_if (warp_ctl_if), - .branch_ctl_if (branch_ctl_if), - `ifdef VX_CFG_EXT_RTU_ENABLE - .sched_unlock_if (sched_unlock_if), + .sysmem_perf (sysmem_perf_tmp), + .pipeline_perf (pipeline_perf), `endif - - .decode_sched_if(decode_sched_if), - .issue_sched_if (issue_sched_if), - .commit_sched_if(commit_sched_if), - - .kmu_bus_if (kmu_bus_if), - - .schedule_if (schedule_if), - .sched_csr_if (sched_csr_if), - .gbar_bus_if (gbar_bus_if), - - .busy (sched_busy) + .cycles (sched_cycles), + .instret (sched_instret), + .dcr_csr_if (dcr_csr_if) ); - VX_fetch #( - .INSTANCE_ID (`SFORMATF(("%s-fetch", INSTANCE_ID))) - ) fetch ( - `SCOPE_IO_BIND (0) - .clk (clk), - .reset (reset), - `ifdef PERF_ENABLE - .fetch_perf (pipeline_perf.fetch), - `endif - .icache_bus_if (fetch_icache_if[0]), - .schedule_if (schedule_if), - .fetch_if (fetch_if) - ); + `SCOPE_IO_SWITCH (3); - // Inject CACHE_FLUSH onto the icache stream so a host-side CMD_CACHE_FLUSH - // (issued after every kernel launch) invalidates instruction-cache lines - // belonging to the previous kernel image. + // Port 0 injects the cache flush into its request stream; the cache flushes + // every bank. The merged request leaves the core registered. VX_dcr_flush #( - .WORD_SIZE (ICACHE_WORD_SIZE), - .TAG_WIDTH (ICACHE_FETCH_TAG_WIDTH), - .REQ_OUT_BUF (3) // register core icache-request boundary; rsp already registered by L1 CORE_OUT_BUF + .WORD_SIZE (ICACHE_WORD_SIZE), + .TAG_WIDTH (ICACHE_FETCH_TAG_WIDTH), + .REQ_OUT_BUF (3) ) icache_dcr_flush ( .clk (clk), .reset (reset), .dcr_flush_if (dcr_flush_icache_if), .l1_only (dcr_flush_l1_only), .core_bus_if (fetch_icache_if[0]), - .cache_bus_if (mmu_icache_if[0]) + .cache_bus_if (icache_bus_if[0]) ); - VX_decode #( - .INSTANCE_ID (`SFORMATF(("%s-decode", INSTANCE_ID))) - ) decode ( - .clk (clk), - .reset (reset), - .fetch_if (fetch_if), - .decode_if (decode_if), - .decode_sched_if(decode_sched_if) - ); + // The other ports carry their fetch slice's registered requests, with the + // flush arbiter's select bit, the tag's LSB, clear. + for (genvar i = 1; i < ICACHE_NUM_REQS; ++i) begin : g_icache_ports + assign icache_bus_if[i].req_valid = fetch_icache_if[i].req_valid; + assign icache_bus_if[i].req_data.rw = fetch_icache_if[i].req_data.rw; + assign icache_bus_if[i].req_data.addr = fetch_icache_if[i].req_data.addr; + assign icache_bus_if[i].req_data.data = fetch_icache_if[i].req_data.data; + assign icache_bus_if[i].req_data.byteen = fetch_icache_if[i].req_data.byteen; + assign icache_bus_if[i].req_data.attr = fetch_icache_if[i].req_data.attr; + assign icache_bus_if[i].req_data.tag = {fetch_icache_if[i].req_data.tag, 1'b0}; + assign fetch_icache_if[i].req_ready = icache_bus_if[i].req_ready; + + assign fetch_icache_if[i].rsp_valid = icache_bus_if[i].rsp_valid; + assign fetch_icache_if[i].rsp_data.data = icache_bus_if[i].rsp_data.data; + assign fetch_icache_if[i].rsp_data.tag = icache_bus_if[i].rsp_data.tag[ICACHE_TAG_WIDTH-1:1]; + assign icache_bus_if[i].rsp_ready = fetch_icache_if[i].rsp_ready; + `UNUSED_VAR (icache_bus_if[i].rsp_data.tag[0]) + end - VX_issue #( - .INSTANCE_ID (`SFORMATF(("%s-issue", INSTANCE_ID))) - ) issue ( - `SCOPE_IO_BIND (1) + // scheduling and the front end: the core scheduler, FETCH_WIDTH fetch slices + // and the crossbar to the issue slices and ALU/SFU blocks - .clk (clk), - .reset (reset), + localparam FW = `VX_CFG_FETCH_WIDTH; + localparam NPA = FETCH_ALU_PORTS; + localparam NPS = FETCH_SFU_PORTS; - `ifdef PERF_ENABLE - .issue_perf (pipeline_perf.issue), - `endif + VX_slice_sched_if slice_sched_if[FW](); + VX_decode_if fs_decode_if[FW](); + VX_decode_if is_decode_if[`VX_CFG_ISSUE_WIDTH](); + VX_branch_ctl_if branch_ctl_if[`VX_CFG_NUM_ALU_BLOCKS](); + VX_warp_ctl_if warp_ctl_if[`VX_CFG_NUM_SFU_BLOCKS](); + VX_sched_csr_if sched_csr_if[`VX_CFG_NUM_SFU_BLOCKS](); +`ifdef VX_CFG_EXT_RTU_ENABLE + VX_sched_unlock_if sched_unlock_if[`VX_CFG_NUM_SFU_BLOCKS](); +`endif - .decode_if (decode_if), - .writeback_if (writeback_if), - .dispatch_if (dispatch_if), - .issue_sched_if (issue_sched_if) - ); + wire [`VX_CFG_ISSUE_WIDTH-1:0] issue_valid; + wire [`VX_CFG_ISSUE_WIDTH-1:0][ISSUE_WIS_W-1:0] issue_wis; + wire [`VX_CFG_NUM_WARPS-1:0] committed_warps; + wire [`VX_CFG_NUM_WARPS-1:0] sched_active_warps; +`ifdef VX_CFG_VM_ENABLE + wire [`VX_CFG_NUM_SFU_BLOCKS-1:0][`VX_CFG_XLEN-1:0] sched_satp; + wire [`VX_CFG_NUM_SFU_BLOCKS-1:0] satp_wr_valid; + wire [`VX_CFG_NUM_SFU_BLOCKS-1:0][`VX_CFG_XLEN-1:0] satp_wr_data; +`endif - VX_execute #( - .INSTANCE_ID (`SFORMATF(("%s-execute", INSTANCE_ID))), - .CORE_ID (CORE_ID) - ) execute ( - `SCOPE_IO_BIND (2) + sched_branch_t [FW-1:0][NPA-1:0] fs_branch; + sched_wctl_t [FW-1:0][NPS-1:0] fs_wctl; + wire [FW-1:0][NPS-1:0][NW_WIDTH-1:0] fs_dvstack_wid; + wire [FW-1:0][NPS-1:0][DV_STACK_SIZEW-1:0] fs_dvstack_ptr; + wire [FW-1:0][NPS-1:0][`VX_CFG_MEM_ADDR_WIDTH-1:0] fs_dvstack_mscratch; + wire [FW-1:0][NPS-1:0][NW_WIDTH-1:0] fs_csr_rd_wid; + sched_csr_rd_t [FW-1:0][NPS-1:0] fs_csr_rd; + sched_csr_wr_t [FW-1:0][NPS-1:0] fs_csr_wr; +`ifdef VX_CFG_EXT_RTU_ENABLE + wire [FW-1:0][NPS-1:0] fs_rtu_unlock_valid; + wire [FW-1:0][NPS-1:0][NW_WIDTH-1:0] fs_rtu_unlock_wid; +`endif + wire [FW-1:0][PER_FETCH_WARPS-1:0] fs_issued; + wire [FW-1:0][PER_FETCH_WARPS-1:0] fs_committed; + wire [FW-1:0][PER_FETCH_WARPS-1:0] fs_pending_alm_empty; + + wire [`VX_CFG_NUM_SFU_BLOCKS-1:0][NCTA_WIDTH-1:0] csr_rd_cta_id; + cta_ctx_t [`VX_CFG_NUM_SFU_BLOCKS-1:0] csr_rd_ctx; + for (genvar b = 0; b < `VX_CFG_NUM_SFU_BLOCKS; ++b) begin : g_csr_rd_cta_id + assign csr_rd_cta_id[b] = sched_csr_if[b].csr_rd_cta_id; + end + VX_scheduler #( + .INSTANCE_ID (`SFORMATF(("%s-scheduler", INSTANCE_ID))) + ) scheduler ( .clk (clk), .reset (reset), - + .core_id (core_id), `ifdef PERF_ENABLE - .sysmem_perf (sysmem_perf_tmp), - .pipeline_perf (pipeline_perf), - `ifdef VX_CFG_EXT_TCU_ENABLE - .tcu_perf (tcu_perf), + .sched_perf (pipeline_perf.sched), `endif + .slice_if (slice_sched_if), + .warp_shared_if (warp_shared_if), + `ifdef VX_CFG_EXT_DXA_ENABLE + .txbar_bus_if (dxa_txbar_bus_if), `endif + .committed_warps(committed_warps), + .csr_rd_cta_id (csr_rd_cta_id), + .csr_rd_ctx (csr_rd_ctx), + .kmu_bus_if (kmu_bus_if), + .gbar_bus_if (gbar_bus_if), + .cycles (sched_cycles), + .instret (sched_instret), + .active_warps (sched_active_warps), + `ifdef VX_CFG_VM_ENABLE + .satp (sched_satp), + .satp_wr_valid (satp_wr_valid), + .satp_wr_data (satp_wr_data), + `endif + .busy (sched_busy) + ); - // execute exposes LSU client interfaces; lsu_scheduler sits between - // execute and lsu_mem_if (which connects to mem_unit). - .lsu_client_if (lsu_client_if), - - `ifdef TCU_META_ENABLE - .tcu_mem_if (tcu_mem_if), + VX_fetch_xbar fetch_xbar ( + .fs_decode_if (fs_decode_if), + .is_decode_if (is_decode_if), + .issue_valid (issue_valid), + .issue_wis (issue_wis), + .committed_warps (committed_warps), + .fs_issued (fs_issued), + .fs_committed (fs_committed), + .fs_pending_alm_empty (fs_pending_alm_empty), + .branch_ctl_if (branch_ctl_if), + .warp_ctl_if (warp_ctl_if), + .sched_csr_if (sched_csr_if), + `ifdef VX_CFG_EXT_RTU_ENABLE + .sched_unlock_if (sched_unlock_if), `endif + .cycles (sched_cycles), + .instret (sched_instret), + .active_warps (sched_active_warps), + .lsu_sched_drained (lsu_drained), + .csr_rd_ctx (csr_rd_ctx), + `ifdef VX_CFG_VM_ENABLE + .satp (sched_satp), + .satp_wr_valid (satp_wr_valid), + .satp_wr_data (satp_wr_data), + `endif + .fs_branch (fs_branch), + .fs_wctl (fs_wctl), + .fs_dvstack_wid (fs_dvstack_wid), + .fs_dvstack_ptr (fs_dvstack_ptr), + .fs_dvstack_mscratch (fs_dvstack_mscratch), + .fs_csr_rd_wid (fs_csr_rd_wid), + .fs_csr_rd (fs_csr_rd), + `ifdef VX_CFG_EXT_RTU_ENABLE + .fs_rtu_unlock_valid (fs_rtu_unlock_valid), + .fs_rtu_unlock_wid (fs_rtu_unlock_wid), + `endif + .fs_csr_wr (fs_csr_wr) + ); + +`ifdef PERF_ENABLE + fetch_perf_t per_slice_fetch_perf [FW]; + `PERF_COUNTER_ADD (pipeline_perf.fetch, per_slice_fetch_perf, stalls, PERF_CTR_BITS, FW, (FW > 2)) +`endif + +`ifdef SCOPE + wire [FW-1:0] fetch_scope_bus_in, fetch_scope_bus_out; + VX_scope_switch #( + .N (FW) + ) fetch_scope_switch ( + .clk (clk), + .reset (scope_reset_w[0]), + .req_in (scope_bus_in_w[0]), + .rsp_out (scope_bus_out_w[0]), + .req_out (fetch_scope_bus_in), + .rsp_in (fetch_scope_bus_out) + ); +`endif - .dispatch_if (dispatch_if), - .commit_if (commit_if), + for (genvar f = 0; f < FW; ++f) begin : g_fetch_slices + VX_fetch_slice #( + .INSTANCE_ID (INSTANCE_ID), + .SLICE_STR ((FW > 1) ? `SFORMATF(("%0d", f)) : "") + ) fetch_slice ( + `ifdef SCOPE + .scope_reset (scope_reset_w[0]), + .scope_bus_in (fetch_scope_bus_in[f]), + .scope_bus_out (fetch_scope_bus_out[f]), + `endif + .clk (clk), + .reset (reset), + .core_id (core_id), + .slice_id (FETCH_FSW_W'(f)), + `ifdef PERF_ENABLE + .fetch_perf (per_slice_fetch_perf[f]), + `endif + .slice_sched_if (slice_sched_if[f]), + .icache_bus_if (fetch_icache_if[f]), + .decode_if (fs_decode_if[f]), + .branch (fs_branch[f]), + .wctl (fs_wctl[f]), + .dvstack_wid (fs_dvstack_wid[f]), + .dvstack_ptr (fs_dvstack_ptr[f]), + .dvstack_mscratch (fs_dvstack_mscratch[f]), + .csr_rd_wid (fs_csr_rd_wid[f]), + .csr_rd (fs_csr_rd[f]), + .csr_wr (fs_csr_wr[f]), + `ifdef VX_CFG_EXT_RTU_ENABLE + .rtu_unlock_valid (fs_rtu_unlock_valid[f]), + .rtu_unlock_wid (fs_rtu_unlock_wid[f]), + `endif + .issued (fs_issued[f]), + .committed (fs_committed[f]), + .pending_alm_empty (fs_pending_alm_empty[f]) + ); + end - .sched_csr_if (sched_csr_if), + // issue, execute and commit - .dcr_csr_if (dcr_csr_if), + if (COMPUTE_SLICES_ENABLE) begin : g_compute_slices - `ifdef VX_CFG_TCU_WGMMA_ENABLE - .tcu_lmem_if (tcu_lmem_if), + `ifdef PERF_ENABLE + issue_perf_t per_slice_issue_perf [`VX_CFG_ISSUE_WIDTH]; + `PERF_COUNTER_ADD (pipeline_perf.issue, per_slice_issue_perf, ibf_stalls, PERF_CTR_BITS, `VX_CFG_ISSUE_WIDTH, (`VX_CFG_ISSUE_WIDTH > 2)) + `PERF_COUNTER_ADD (pipeline_perf.issue, per_slice_issue_perf, scb_stalls, PERF_CTR_BITS, `VX_CFG_ISSUE_WIDTH, (`VX_CFG_ISSUE_WIDTH > 2)) + `PERF_COUNTER_ADD (pipeline_perf.issue, per_slice_issue_perf, opd_stalls, PERF_CTR_BITS, `VX_CFG_ISSUE_WIDTH, (`VX_CFG_ISSUE_WIDTH > 2)) + for (genvar i = 0; i < NUM_EX_UNITS; ++i) begin : g_issue_perf_units_uses + `PERF_COUNTER_ADD (pipeline_perf.issue, per_slice_issue_perf, dispatch_stalls[i], PERF_CTR_BITS, `VX_CFG_ISSUE_WIDTH, (`VX_CFG_ISSUE_WIDTH > 2)) + `PERF_COUNTER_ADD (pipeline_perf.issue, per_slice_issue_perf, dispatch_instrs[i], PERF_CTR_BITS, `VX_CFG_ISSUE_WIDTH, (`VX_CFG_ISSUE_WIDTH > 2)) + end `endif + `ifdef VX_CFG_EXT_DXA_ENABLE - .dxa_req_bus_if (dxa_req_bus_if), - .dxa_txbar_bus_if(dxa_txbar_bus_if), + VX_dxa_req_bus_if slice_dxa_req_bus_if[`VX_CFG_ISSUE_WIDTH](); `endif `ifdef VX_CFG_EXT_TEX_ENABLE - .tex_bus_if (tex_bus_if), - `endif - `ifdef VX_CFG_EXT_OM_ENABLE + VX_tex_bus_if #( + .NUM_LANES (`VX_CFG_NUM_SFU_LANES), + .TAG_WIDTH (TEX_REQ_TAG_WIDTH) + ) slice_tex_bus_if[`VX_CFG_ISSUE_WIDTH](); `endif `ifdef VX_CFG_EXT_RTU_ENABLE - .rtu_bus_if (rtu_bus_if), - .sched_unlock_if (sched_unlock_if), + VX_rtu_bus_if #( + .NUM_LANES (`VX_CFG_NUM_SFU_LANES), + .TAG_WIDTH (RTU_REQ_TAG_WIDTH) + ) slice_rtu_bus_if[`VX_CFG_ISSUE_WIDTH](); `endif - .warp_ctl_if (warp_ctl_if), - .branch_ctl_if (branch_ctl_if) - ); + `ifdef VX_CFG_TCU_WGMMA_ENABLE + wire [`VX_CFG_ISSUE_WIDTH-1:0] tcu_exec_valid, tcu_exec_ready; + tcu_execute_t tcu_exec_data [`VX_CFG_ISSUE_WIDTH]; + wire [`VX_CFG_ISSUE_WIDTH-1:0][TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs1_data; + wire [`VX_CFG_ISSUE_WIDTH-1:0][TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs2_data; + wire [`VX_CFG_ISSUE_WIDTH-1:0] tcu_tbuf_ready; + `endif + `ifdef TCU_META_ENABLE + wire [`VX_CFG_ISSUE_WIDTH-1:0] tcu_agu_ld_valid, tcu_agu_ld_ready; + tcu_execute_t tcu_agu_ld_data [`VX_CFG_ISSUE_WIDTH]; + wire [`VX_CFG_ISSUE_WIDTH-1:0] tcu_agu_result_valid, tcu_agu_result_ready; + tcu_result_t tcu_agu_result_data [`VX_CFG_ISSUE_WIDTH]; + wire tcu_meta_wr_en; + wire [NW_WIDTH-1:0] tcu_meta_wr_wid; + wire [4:0] tcu_meta_wr_idx; + wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] tcu_meta_wr_data; + `endif - VX_commit #( - .INSTANCE_ID (`SFORMATF(("%s-commit", INSTANCE_ID))) - ) commit ( - .clk (clk), - .reset (reset), + `ifdef SCOPE + wire [`VX_CFG_ISSUE_WIDTH-1:0] slice_scope_bus_in, slice_scope_bus_out; + VX_scope_switch #( + .N (`VX_CFG_ISSUE_WIDTH) + ) slice_scope_switch ( + .clk (clk), + .reset (scope_reset_w[1]), + .req_in (scope_bus_in_w[1]), + .rsp_out (scope_bus_out_w[1]), + .req_out (slice_scope_bus_in), + .rsp_in (slice_scope_bus_out) + ); + `endif + `SCOPE_IO_UNUSED (2) + + for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_slices + wire [PER_ISSUE_WARPS-1:0] committed_wis; + + VX_compute_slice #( + .INSTANCE_ID (INSTANCE_ID), + .SLICE_STR (`SFORMATF(("%0d", i))) + ) compute_slice ( + `ifdef SCOPE + .scope_reset (scope_reset_w[1]), + .scope_bus_in (slice_scope_bus_in[i]), + .scope_bus_out (slice_scope_bus_out[i]), + `endif + .clk (clk), + .reset (reset), + .core_id (core_id), + .slice_id (ISSUE_ISW_W'(i)), + `ifdef PERF_ENABLE + .issue_perf (per_slice_issue_perf[i]), + `endif + .decode_if (is_decode_if[i]), + .warp_issued (issue_valid[i]), + .warp_issued_wis (issue_wis[i]), + .committed_wis (committed_wis), + .branch_ctl_if (branch_ctl_if[i]), + .warp_ctl_if (warp_ctl_if[i]), + .sched_csr_if (sched_csr_if[i]), + `ifdef VX_CFG_EXT_RTU_ENABLE + .sched_unlock_if (sched_unlock_if[i]), + `endif + .lsu_client_if (lsu_client_if[i]), + `ifdef VX_CFG_EXT_DXA_ENABLE + .dxa_req_bus_if (slice_dxa_req_bus_if[i]), + `endif + `ifdef VX_CFG_EXT_TEX_ENABLE + .tex_bus_if (slice_tex_bus_if[i]), + `endif + `ifdef VX_CFG_EXT_RTU_ENABLE + .rtu_bus_if (slice_rtu_bus_if[i]), + `endif + `ifdef VX_CFG_TCU_WGMMA_ENABLE + .tcu_exec_valid (tcu_exec_valid[i]), + .tcu_exec_ready (tcu_exec_ready[i]), + .tcu_exec_data (tcu_exec_data[i]), + .tcu_tbuf_rs1_data (tcu_tbuf_rs1_data[i]), + .tcu_tbuf_rs2_data (tcu_tbuf_rs2_data[i]), + .tcu_tbuf_ready (tcu_tbuf_ready[i]), + `endif + `ifdef TCU_META_ENABLE + .tcu_agu_ld_valid (tcu_agu_ld_valid[i]), + .tcu_agu_ld_data (tcu_agu_ld_data[i]), + .tcu_agu_ld_ready (tcu_agu_ld_ready[i]), + .tcu_agu_result_valid (tcu_agu_result_valid[i]), + .tcu_agu_result_data (tcu_agu_result_data[i]), + .tcu_agu_result_ready (tcu_agu_result_ready[i]), + .tcu_meta_wr_en (tcu_meta_wr_en && (NW_WIDTH'(tcu_meta_wr_wid % `VX_CFG_NUM_TCU_BLOCKS) == NW_WIDTH'(i))), + .tcu_meta_wr_wid (tcu_meta_wr_wid), + .tcu_meta_wr_idx (tcu_meta_wr_idx), + .tcu_meta_wr_data (tcu_meta_wr_data), + `endif + .warp_shared_if (warp_shared_if[i]) + ); + + // wid = {wis, isw}: slice i owns every ISSUE_WIDTH-th warp starting at i + for (genvar w = 0; w < PER_ISSUE_WARPS; ++w) begin : g_committed + assign committed_warps[w * `VX_CFG_ISSUE_WIDTH + i] = committed_wis[w]; + end + end + + VX_sfu_shared #( + .INSTANCE_ID (`SFORMATF(("%s-execute-sfu-shared", INSTANCE_ID))) + ) sfu_shared ( + `ifdef VX_CFG_EXT_DXA_ENABLE + .block_dxa_req_bus_if (slice_dxa_req_bus_if), + .dxa_req_bus_if (dxa_req_bus_if), + `endif + `ifdef VX_CFG_EXT_TEX_ENABLE + .block_tex_bus_if (slice_tex_bus_if), + .tex_bus_if (tex_bus_if), + `endif + `ifdef VX_CFG_EXT_RTU_ENABLE + .block_rtu_bus_if (slice_rtu_bus_if), + .rtu_bus_if (rtu_bus_if), + `endif + .clk (clk), + .reset (reset) + ); + + `ifdef VX_CFG_EXT_TCU_ENABLE + VX_tcu_shared #( + .INSTANCE_ID (`SFORMATF(("%s-execute-tcu", INSTANCE_ID))), + .BLOCK_SIZE (`VX_CFG_ISSUE_WIDTH) + ) tcu_shared ( + `ifdef PERF_ENABLE + .tcu_perf (tcu_perf), + `endif + `ifdef VX_CFG_TCU_WGMMA_ENABLE + .exec_valid (tcu_exec_valid), + .exec_ready (tcu_exec_ready), + .exec_data (tcu_exec_data), + .tbuf_rs1_data (tcu_tbuf_rs1_data), + .tbuf_rs2_data (tcu_tbuf_rs2_data), + .tbuf_ready_eff (tcu_tbuf_ready), + .tcu_lmem_if (tcu_lmem_if), + `endif + `ifdef TCU_META_ENABLE + .agu_ld_valid (tcu_agu_ld_valid), + .agu_ld_data (tcu_agu_ld_data), + .agu_ld_ready (tcu_agu_ld_ready), + .agu_result_valid (tcu_agu_result_valid), + .agu_result_data (tcu_agu_result_data), + .agu_result_ready (tcu_agu_result_ready), + .meta_wr_en (tcu_meta_wr_en), + .meta_wr_wid (tcu_meta_wr_wid), + .meta_wr_idx (tcu_meta_wr_idx), + .meta_wr_data (tcu_meta_wr_data), + .tcu_mem_if (tcu_mem_if), + `endif + .clk (clk), + .reset (reset) + ); + `endif - .commit_if (commit_if), + end else begin : g_stage_sliced - .writeback_if (writeback_if), + VX_issue_sched_if issue_sched_if[`VX_CFG_ISSUE_WIDTH](); + VX_commit_sched_if commit_sched_if(); - .commit_sched_if(commit_sched_if) - ); + VX_dispatch_if dispatch_if[NUM_EX_UNITS * `VX_CFG_ISSUE_WIDTH](); + VX_commit_if commit_if[NUM_EX_UNITS * `VX_CFG_ISSUE_WIDTH](); + VX_writeback_if writeback_if[`VX_CFG_ISSUE_WIDTH](); + + VX_issue #( + .INSTANCE_ID (`SFORMATF(("%s-issue", INSTANCE_ID))) + ) issue ( + `SCOPE_IO_BIND (1) + + .clk (clk), + .reset (reset), + + `ifdef PERF_ENABLE + .issue_perf (pipeline_perf.issue), + `endif + + .decode_if (is_decode_if), + .writeback_if (writeback_if), + .dispatch_if (dispatch_if), + .issue_sched_if (issue_sched_if) + ); - // Per-block VX_lsu_scheduler instances: all parameterized NUM_CLIENTS=2. - // Block 0 wires client 1 to the warp-level TCU AGU; other blocks tie it off. - // Symmetric NUM_CLIENTS keeps module generation uniform — tied-off clients - // cost only a few muxes inside the round-robin arbiter. + for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_issue_sched + assign issue_valid[i] = issue_sched_if[i].valid; + assign issue_wis[i] = issue_sched_if[i].wis; + end + + VX_execute #( + .INSTANCE_ID (`SFORMATF(("%s-execute", INSTANCE_ID))) + ) execute ( + `SCOPE_IO_BIND (2) + + .clk (clk), + .reset (reset), + .core_id (core_id), + + `ifdef PERF_ENABLE + `ifdef VX_CFG_EXT_TCU_ENABLE + .tcu_perf (tcu_perf), + `endif + `endif + + // execute exposes LSU client interfaces; lsu_scheduler sits between + // execute and lsu_mem_if (which connects to mem_unit). + .lsu_client_if (lsu_client_if), + + `ifdef TCU_META_ENABLE + .tcu_mem_if (tcu_mem_if), + `endif + + .dispatch_if (dispatch_if), + .commit_if (commit_if), + + .sched_csr_if (sched_csr_if), + + `ifdef VX_CFG_TCU_WGMMA_ENABLE + .tcu_lmem_if (tcu_lmem_if), + `endif + `ifdef VX_CFG_EXT_DXA_ENABLE + .dxa_req_bus_if (dxa_req_bus_if), + `endif + `ifdef VX_CFG_EXT_TEX_ENABLE + .tex_bus_if (tex_bus_if), + `endif + `ifdef VX_CFG_EXT_RTU_ENABLE + .rtu_bus_if (rtu_bus_if), + .sched_unlock_if(sched_unlock_if), + `endif + + .warp_shared_if (warp_shared_if), + + .warp_ctl_if (warp_ctl_if), + .branch_ctl_if (branch_ctl_if) + ); + + VX_commit #( + .INSTANCE_ID (`SFORMATF(("%s-commit", INSTANCE_ID))) + ) commit ( + .clk (clk), + .reset (reset), + + .commit_if (commit_if), + + .writeback_if (writeback_if), + + .commit_sched_if(commit_sched_if) + ); + + assign committed_warps = commit_sched_if.committed_warps; + + end + + // One memory scheduler per LSU block. With the TCU metadata AGU, every block + // has a second client: block 0 serves the AGU and the others tie it off, which + // keeps the blocks identical at the cost of a few arbiter muxes. `ifdef TCU_META_ENABLE localparam LSU_SCHED_NUM_CLIENTS = 2; `else @@ -384,18 +688,16 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( for (genvar block_idx = 0; block_idx < `VX_CFG_NUM_LSU_BLOCKS; ++block_idx) begin : g_lsu_scheduler VX_lsu_sched_if sched_client_if [LSU_SCHED_NUM_CLIENTS](); - // Client 0 — LSU (per-block). Forward the per-block lsu_client_if - // master onto the scheduler's slave port via per-signal assigns - // (interface arrays can't be wired with a single assign). - assign sched_client_if[0].req_valid = lsu_client_if[block_idx].req_valid; - assign sched_client_if[0].req_data = lsu_client_if[block_idx].req_data; - assign lsu_client_if[block_idx].req_ready = sched_client_if[0].req_ready; - assign lsu_client_if[block_idx].rsp_valid = sched_client_if[0].rsp_valid; - assign lsu_client_if[block_idx].rsp_data = sched_client_if[0].rsp_data; - assign sched_client_if[0].rsp_ready = lsu_client_if[block_idx].rsp_ready; + // client 0: the block's LSU + assign sched_client_if[0].req_valid = lsu_client_if[block_idx].req_valid; + assign sched_client_if[0].req_data = lsu_client_if[block_idx].req_data; + assign lsu_client_if[block_idx].req_ready = sched_client_if[0].req_ready; + assign lsu_client_if[block_idx].rsp_valid = sched_client_if[0].rsp_valid; + assign lsu_client_if[block_idx].rsp_data = sched_client_if[0].rsp_data; + assign sched_client_if[0].rsp_ready = lsu_client_if[block_idx].rsp_ready; `ifdef TCU_META_ENABLE - // Client 1 — TCU AGU on block 0; tied off on other blocks. + // client 1: the TCU metadata AGU on block 0 if (block_idx == 0) begin : g_tcu_client assign sched_client_if[1].req_valid = tcu_mem_if.req_valid; assign sched_client_if[1].req_data = tcu_mem_if.req_data; @@ -414,12 +716,12 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( `endif VX_lsu_scheduler #( - .INSTANCE_ID (`SFORMATF(("%s-lsusched%0d", INSTANCE_ID, block_idx))), - .NUM_CLIENTS (LSU_SCHED_NUM_CLIENTS), - .NUM_LANES (`VX_CFG_NUM_LSU_LANES), - .CORE_QUEUE_SIZE(`VX_CFG_LSU_QUEUE_IN_SIZE), - .MEM_QUEUE_SIZE (LSU_QUEUE_OUT_SIZE), - .PENDING_SIZE (`VX_CFG_LSU_PENDING_SIZE) + .INSTANCE_ID (`SFORMATF(("%s-lsusched%0d", INSTANCE_ID, block_idx))), + .NUM_CLIENTS (LSU_SCHED_NUM_CLIENTS), + .NUM_LANES (`VX_CFG_NUM_LSU_LANES), + .CORE_QUEUE_SIZE (`VX_CFG_LSU_QUEUE_IN_SIZE), + .MEM_QUEUE_SIZE (LSU_QUEUE_OUT_SIZE), + .PENDING_SIZE (`VX_CFG_LSU_PENDING_SIZE) ) lsu_scheduler ( .clk (clk), .reset (reset), @@ -432,46 +734,33 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( VX_mem_unit #( .INSTANCE_ID (INSTANCE_ID) ) mem_unit ( - .clk (clk), - .reset (reset), + .clk (clk), + .reset (reset), `ifdef PERF_ENABLE - .lmem_perf (lmem_perf), - .coalescer_perf(coalescer_perf), + .lmem_perf (lmem_perf), + .coalescer_perf (coalescer_perf), `endif `ifdef VX_CFG_TCU_WGMMA_ENABLE - .tcu_lmem_if (tcu_lmem_if), + .tcu_lmem_if (tcu_lmem_if), `endif `ifdef VX_CFG_EXT_DXA_ENABLE - .dxa_lmem_bus_if(dxa_lmem_bus_if), + .dxa_lmem_bus_if (dxa_lmem_bus_if), .dxa_txbar_bus_if(dxa_txbar_bus_if), `endif - .empty (mem_unit_empty), - .lsu_mem_if (lsu_mem_if), - .dcr_flush_if (dcr_flush_dcache_if), - .flush_l1_only (dcr_flush_l1_only), - .dcache_bus_if (mmu_dcache_if) + .empty (mem_unit_empty), + .lsu_mem_if (lsu_mem_if), + .dcr_flush_if (dcr_flush_dcache_if), + .flush_l1_only (dcr_flush_l1_only), + .dcache_bus_if (dcache_bus_if) ); - // Address translation is relocated to the socket (one MMU per L1 cache). - // The core's dcache/icache buses carry the addresses the mem_unit and fetch - // produce straight through — virtual under VM, physical otherwise. Same - // widths on both sides (BASE == full for both L1s). - for (genvar i = 0; i < DCACHE_NUM_REQS; ++i) begin : g_dcache_passthru - `ASSIGN_VX_MEM_BUS_IF (dcache_bus_if[i], mmu_dcache_if[i]); - end - `ASSIGN_VX_MEM_BUS_IF (icache_bus_if, mmu_icache_if[0]); - `ifdef VX_CFG_VM_ENABLE `ifdef PERF_ENABLE - // The relocated MMU's perf counters live at the socket and are not yet routed - // through sysmem_perf, so the in-core pipeline view reads zero. + // the MMU perf counters are kept at the socket assign pipeline_perf.mmu = '0; `endif // Translation drains at the socket MMU; the socket returns its empty status. wire mmu_empty = mmu_drained; - // The socket MMU takes its satp from the shared DCR root, so the per-core CSR - // satp is not a translation source. - `UNUSED_VAR (sched_csr_if.csr_satp) `else wire mmu_empty = 1'b1; `endif @@ -485,12 +774,12 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( // buffer, which mem_unit_empty does not cover. wire [DCACHE_NUM_REQS-1:0] dcache_req_valid; for (genvar i = 0; i < DCACHE_NUM_REQS; ++i) begin : g_dcache_req_valid - assign dcache_req_valid[i] = mmu_dcache_if[i].req_valid; + assign dcache_req_valid[i] = dcache_bus_if[i].req_valid; end assign busy = sched_busy || ~(&lsu_sched_empty) || ~mem_unit_empty || (| dcache_req_valid) || ~mmu_empty; // BAR (vx_barrier / vx_barrier_arrive) drains LSU + MMU before suspending or registering arrival. - assign warp_ctl_if.lsu_sched_drained = (&lsu_sched_empty) && mmu_empty; + assign lsu_drained = (&lsu_sched_empty) && mmu_empty; `ifdef PERF_ENABLE @@ -498,7 +787,9 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( wire [`CLOG2(LSU_NUM_REQS+1)-1:0] perf_dcache_wr_req_per_cycle; wire [`CLOG2(LSU_NUM_REQS+1)-1:0] perf_dcache_rsp_per_cycle; - wire [1:0] perf_icache_pending_read_cycle; + wire [`CLOG2(ICACHE_NUM_REQS+1)-1:0] perf_icache_req_per_cycle; + wire [`CLOG2(ICACHE_NUM_REQS+1)-1:0] perf_icache_rsp_per_cycle; + wire [`CLOG2(ICACHE_NUM_REQS+1)+1-1:0] perf_icache_pending_read_cycle; wire [`CLOG2(LSU_NUM_REQS+1)+1-1:0] perf_dcache_pending_read_cycle; reg [PERF_CTR_BITS-1:0] perf_icache_pending_reads; @@ -508,8 +799,14 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( reg [PERF_CTR_BITS-1:0] perf_loads; reg [PERF_CTR_BITS-1:0] perf_stores; - wire perf_icache_req_fire = icache_bus_if.req_valid && icache_bus_if.req_ready; - wire perf_icache_rsp_fire = icache_bus_if.rsp_valid && icache_bus_if.rsp_ready; + wire [ICACHE_NUM_REQS-1:0] perf_icache_req_fire; + wire [ICACHE_NUM_REQS-1:0] perf_icache_rsp_fire; + for (genvar i = 0; i < ICACHE_NUM_REQS; ++i) begin : g_perf_icache + assign perf_icache_req_fire[i] = icache_bus_if[i].req_valid && icache_bus_if[i].req_ready; + assign perf_icache_rsp_fire[i] = icache_bus_if[i].rsp_valid && icache_bus_if[i].rsp_ready; + end + `POP_COUNT(perf_icache_req_per_cycle, perf_icache_req_fire); + `POP_COUNT(perf_icache_rsp_per_cycle, perf_icache_rsp_fire); wire [LSU_NUM_REQS-1:0] perf_dcache_rd_req_fire, perf_dcache_rd_req_fire_r; wire [LSU_NUM_REQS-1:0] perf_dcache_wr_req_fire, perf_dcache_wr_req_fire_r; @@ -530,7 +827,7 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( `POP_COUNT(perf_dcache_wr_req_per_cycle, perf_dcache_wr_req_fire_r); `POP_COUNT(perf_dcache_rsp_per_cycle, perf_dcache_rsp_fire); - assign perf_icache_pending_read_cycle = perf_icache_req_fire - perf_icache_rsp_fire; + assign perf_icache_pending_read_cycle = perf_icache_req_per_cycle - perf_icache_rsp_per_cycle; assign perf_dcache_pending_read_cycle = perf_dcache_rd_req_per_cycle - perf_dcache_rsp_per_cycle; always @(posedge clk) begin @@ -554,7 +851,7 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( perf_icache_lat <= '0; perf_dcache_lat <= '0; end else begin - perf_ifetches <= perf_ifetches + PERF_CTR_BITS'(perf_icache_req_fire); + perf_ifetches <= perf_ifetches + PERF_CTR_BITS'(perf_icache_req_per_cycle); perf_loads <= perf_loads + PERF_CTR_BITS'(perf_dcache_rd_req_per_cycle); perf_stores <= perf_stores + PERF_CTR_BITS'(perf_dcache_wr_req_per_cycle); perf_icache_lat <= perf_icache_lat + perf_icache_pending_reads; diff --git a/hw/rtl/core/VX_csr_cta.sv b/hw/rtl/core/VX_csr_cta.sv new file mode 100644 index 0000000000..8ceb33a7c4 --- /dev/null +++ b/hw/rtl/core/VX_csr_cta.sv @@ -0,0 +1,75 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// Per-warp CTA state for one group of warps, addressed by group-local warp id: +// the rank, lane record and CTA id of the group's own warps. Written by the +// dispatcher's broadcast; the parent presents only writes that belong to this +// group. The CTA contexts every warp may read are held core-wide. +module VX_csr_cta import VX_gpu_pkg::*; #( + parameter NUM_WARPS = 1, + parameter LW_WIDTH = `LOG2UP(NUM_WARPS) +) ( + input wire clk, + input wire reset, + + // this group's warps + input wire warp_write, + input wire [LW_WIDTH-1:0] warp_waddr, + input cta_warp_t warp_wdata, + input wire id_write, + input wire [LW_WIDTH-1:0] id_waddr, + input wire [NCTA_WIDTH-1:0] id_wdata, + + // CSR read: rdata is valid one cycle after the address, which the CSR unit + // holds stable for that cycle + input wire [LW_WIDTH-1:0] rd_lid, + output wire [NW_WIDTH-1:0] rd_cta_rank, + output cta_lane_t [`VX_CFG_NUM_THREADS-1:0] rd_lane, + + // CTA id of a warp, combinational + input wire [LW_WIDTH-1:0] sched_lid, + output wire [NCTA_WIDTH-1:0] sched_cta_id +); + cta_warp_t warp_rdata; + + VX_dp_ram #( + .DATAW ($bits(cta_warp_t)), + .SIZE (NUM_WARPS), + .RDW_MODE ("R"), + .OUT_REG (1) + ) cta_warp_ram ( + .clk (clk), + .reset (reset), + .read (1'b1), + .write (warp_write), + .wren (1'b1), + .waddr (warp_waddr), + .wdata (warp_wdata), + .raddr (rd_lid), + .rdata (warp_rdata) + ); + + assign rd_cta_rank = warp_rdata.cta_rank; + assign rd_lane = warp_rdata.lane_launch; + + reg [NUM_WARPS-1:0][NCTA_WIDTH-1:0] cta_id_r; + always @(posedge clk) begin + if (id_write) begin + cta_id_r[id_waddr] <= id_wdata; + end + end + assign sched_cta_id = cta_id_r[sched_lid]; + +endmodule diff --git a/hw/rtl/core/VX_csr_data.sv b/hw/rtl/core/VX_csr_data.sv index f09c17bb7e..e7a8799dc6 100644 --- a/hw/rtl/core/VX_csr_data.sv +++ b/hw/rtl/core/VX_csr_data.sv @@ -17,34 +17,24 @@ `include "VX_fpu_define.vh" `endif -`ifdef VX_CFG_XLEN_64 - `define CSR_READ_64(addr, dst, src) \ - addr : dst = `VX_CFG_XLEN'(src) -`else - `define CSR_READ_64(addr, dst, src) \ - addr : dst = src[31:0]; \ - addr+12'h80 : dst = 32'(src[$bits(src)-1:32]) -`endif - module VX_csr_data import VX_gpu_pkg::*; `ifdef VX_CFG_EXT_F_ENABLE import VX_fpu_pkg::*; `endif #( - parameter `STRING INSTANCE_ID = "", - parameter CORE_ID = 0 + parameter `STRING INSTANCE_ID = "" ) ( input wire clk, input wire reset, -`ifdef PERF_ENABLE - input sysmem_perf_t sysmem_perf, - input pipeline_perf_t pipeline_perf, -`endif + input wire [CORE_ID_WIDTH-1:0] core_id, `ifdef VX_CFG_EXT_F_ENABLE - VX_fpu_csr_if.slave fpu_csr_if [`VX_CFG_NUM_FPU_BLOCKS], + // FP CSR of the requesting warp: read live, written at fire + input wire [INST_FRM_BITS+`FP_FLAGS_BITS-1:0] fcsr_rd_data, + output wire [2:0] fcsr_wr_sel, // {fcsr, frm, fflags} + output wire [INST_FRM_BITS+`FP_FLAGS_BITS-1:0] fcsr_wr_data, `endif VX_sched_csr_if.slave sched_csr_if, @@ -58,12 +48,7 @@ import VX_fpu_pkg::*; input wire [`VX_CSR_ADDR_BITS-1:0] req_addr, input wire [INST_SFU_BITS-1:0] req_op, input wire [`VX_CFG_XLEN-1:0] req_src, - output wire [`VX_CFG_XLEN-1:0] read_data, - - // Host performance-counter reads - input wire [7:0] dcr_mpm_class, - input wire [`VX_CSR_ADDR_BITS-1:0] dcr_addr, - output wire [`VX_CFG_XLEN-1:0] dcr_data + output wire [`VX_CFG_XLEN-1:0] read_data ); `UNUSED_SPARAM (INSTANCE_ID) @@ -91,7 +76,8 @@ import VX_fpu_pkg::*; localparam LIVE_LMEM_ADDR = 20; localparam LIVE_CLUSTER = 21; localparam LIVE_ENTRY = 22; - localparam NUM_LIVE = 23; + localparam LIVE_SATP = 23; + localparam NUM_LIVE = 24; // Writable CSRs localparam WR_MSCRATCH = 0; @@ -114,10 +100,6 @@ import VX_fpu_pkg::*; endcase endfunction -`ifdef VX_CFG_VM_ENABLE - reg [`VX_CFG_XLEN-1:0] satp; -`endif - // Decode //////////////////////////////////////////////////////////////// // Scheduler CSRs read interface @@ -161,7 +143,7 @@ import VX_fpu_pkg::*; `VX_CSR_CTA_ENTRY : live_sel_w[LIVE_ENTRY] = 1; `VX_CSR_WARP_ID : stable_w = `VX_CFG_XLEN'(req_wid); - `VX_CSR_CORE_ID : stable_w = `VX_CFG_XLEN'(CORE_ID); + `VX_CSR_CORE_ID : stable_w = `VX_CFG_XLEN'(core_id); `VX_CSR_ACTIVE_THREADS: live_sel_w[LIVE_THREADS] = 1; `VX_CSR_ACTIVE_WARPS: live_sel_w[LIVE_WARPS] = 1; `VX_CSR_NUM_THREADS: stable_w = `VX_CFG_XLEN'(`VX_CFG_NUM_THREADS); @@ -178,7 +160,7 @@ import VX_fpu_pkg::*; `endif `ifdef VX_CFG_VM_ENABLE - `VX_CSR_SATP : begin stable_w = satp; wr_sel_w[WR_SATP] = 1; end + `VX_CSR_SATP : begin live_sel_w[LIVE_SATP] = 1; wr_sel_w[WR_SATP] = 1; end `endif // Machine-mode trap CSRs (stored in the scheduler). @@ -228,8 +210,7 @@ import VX_fpu_pkg::*; // Live values /////////////////////////////////////////////////////////// `ifdef VX_CFG_EXT_F_ENABLE - reg [`VX_CFG_NUM_WARPS-1:0][INST_FRM_BITS+`FP_FLAGS_BITS-1:0] fcsr, fcsr_n; - wire [INST_FRM_BITS+`FP_FLAGS_BITS-1:0] req_fcsr = fcsr[req_wid]; + wire [INST_FRM_BITS+`FP_FLAGS_BITS-1:0] req_fcsr = fcsr_rd_data; `endif wire [NUM_LIVE-1:0][`VX_CFG_XLEN-1:0] live_src; @@ -245,7 +226,7 @@ import VX_fpu_pkg::*; assign live_src[LIVE_INSTRET] = sched_csr_if.instret[31:0]; assign live_src[LIVE_INSTRET_H] = 32'(sched_csr_if.instret[PERF_CTR_BITS-1:32]); `endif - assign live_src[LIVE_THREADS] = `VX_CFG_XLEN'(sched_csr_if.thread_masks[req_wid]); + assign live_src[LIVE_THREADS] = `VX_CFG_XLEN'(sched_csr_if.thread_mask); assign live_src[LIVE_WARPS] = `VX_CFG_XLEN'(sched_csr_if.active_warps); `ifdef VX_CFG_EXT_F_ENABLE assign live_src[LIVE_FFLAGS] = `VX_CFG_XLEN'(req_fcsr[`FP_FLAGS_BITS-1:0]); @@ -268,6 +249,11 @@ import VX_fpu_pkg::*; assign live_src[LIVE_LMEM_ADDR] = `VX_CFG_XLEN'(sched_csr_if.cta_csrs.lmem_addr); assign live_src[LIVE_CLUSTER] = `VX_CFG_XLEN'(sched_csr_if.cta_csrs.cluster_size); assign live_src[LIVE_ENTRY] = `VX_CFG_XLEN'(to_fullPC(sched_csr_if.cta_csrs.entry)); +`ifdef VX_CFG_VM_ENABLE + assign live_src[LIVE_SATP] = sched_csr_if.csr_satp; +`else + assign live_src[LIVE_SATP] = '0; +`endif reg [`VX_CFG_XLEN-1:0] live_data; always @(*) begin @@ -294,66 +280,17 @@ import VX_fpu_pkg::*; `ifdef VX_CFG_EXT_F_ENABLE // The FP flags keep moving under in-flight FP instructions, so their // read-modify-write stays in the fire cycle. - wire [INST_FRM_BITS+`FP_FLAGS_BITS-1:0] fcsr_wr_data = - (INST_FRM_BITS+`FP_FLAGS_BITS)'(csr_rmw(req_op, live_data, req_src)); - - wire [`VX_CFG_NUM_FPU_BLOCKS-1:0] fpu_write_enable; - wire [`VX_CFG_NUM_FPU_BLOCKS-1:0][NW_WIDTH-1:0] fpu_write_wid; - fflags_t [`VX_CFG_NUM_FPU_BLOCKS-1:0] fpu_write_fflags; - - for (genvar i = 0; i < `VX_CFG_NUM_FPU_BLOCKS; ++i) begin : g_fpu_write - assign fpu_write_enable[i] = fpu_csr_if[i].write_enable; - assign fpu_write_wid[i] = fpu_csr_if[i].write_wid; - assign fpu_write_fflags[i] = fpu_csr_if[i].write_fflags; - end - - always @(*) begin - fcsr_n = fcsr; - for (integer i = 0; i < `VX_CFG_NUM_FPU_BLOCKS; ++i) begin - if (fpu_write_enable[i]) begin - fcsr_n[fpu_write_wid[i]][`FP_FLAGS_BITS-1:0] = fcsr[fpu_write_wid[i]][`FP_FLAGS_BITS-1:0] - | fpu_write_fflags[i]; - end - end - if (write_sel[WR_FFLAGS]) begin - fcsr_n[req_wid][`FP_FLAGS_BITS-1:0] = fcsr_wr_data[`FP_FLAGS_BITS-1:0]; - end - if (write_sel[WR_FRM]) begin - fcsr_n[req_wid][INST_FRM_BITS+`FP_FLAGS_BITS-1:`FP_FLAGS_BITS] = fcsr_wr_data[INST_FRM_BITS-1:0]; - end - if (write_sel[WR_FCSR]) begin - fcsr_n[req_wid] = fcsr_wr_data; - end - end - - for (genvar i = 0; i < `VX_CFG_NUM_FPU_BLOCKS; ++i) begin : g_fpu_csr_read_frm - assign fpu_csr_if[i].read_frm = fcsr[fpu_csr_if[i].read_wid][INST_FRM_BITS+`FP_FLAGS_BITS-1:`FP_FLAGS_BITS]; - end - - always @(posedge clk) begin - if (reset) begin - fcsr <= '0; - end else begin - fcsr <= fcsr_n; - end - end + assign fcsr_wr_data = (INST_FRM_BITS+`FP_FLAGS_BITS)'(csr_rmw(req_op, live_data, req_src)); + assign fcsr_wr_sel = {write_sel[WR_FCSR], write_sel[WR_FRM], write_sel[WR_FFLAGS]}; `else `UNUSED_VAR ({write_sel[WR_FCSR], write_sel[WR_FRM], write_sel[WR_FFLAGS]}) `endif `ifdef VX_CFG_VM_ENABLE - // Per-core SATP CSR. Initialized to 0 (BARE mode); kernel writes - // it from vx_start.S after the runtime has installed the page table. - // Surfaced on sched_csr_if.satp so VX_core can pick it up directly - // off the shared interface instead of routing through SFU/execute. - always @(posedge clk) begin - if (reset) begin - satp <= '0; - end else if (write_sel[WR_SATP]) begin - satp <= wr_data_r; - end - end - assign sched_csr_if.csr_satp = satp; + // The per-core SATP CSR lives in the scheduler, shared by every SFU block, + // so its read-modify-write stays in the fire cycle. + assign sched_csr_if.satp_wr_valid = write_sel[WR_SATP]; + assign sched_csr_if.satp_wr_data = csr_rmw(req_op, live_data, req_src); `else `UNUSED_VAR (write_sel[WR_SATP]) `endif @@ -386,210 +323,4 @@ import VX_fpu_pkg::*; end end - // Host performance-counter reads //////////////////////////////////////// - - reg [`VX_CFG_XLEN-1:0] dcr_data_w; - - always @(*) begin - dcr_data_w = '0; - case (dcr_addr) - `CSR_READ_64(`VX_CSR_MCYCLE, dcr_data_w, sched_csr_if.cycles); - `CSR_READ_64(`VX_CSR_MINSTRET, dcr_data_w, sched_csr_if.instret); - default: begin - `ifdef PERF_ENABLE - if ((dcr_addr >= `VX_CSR_MPM_USER && dcr_addr < (`VX_CSR_MPM_USER + 32)) - || (dcr_addr >= `VX_CSR_MPM_USER_H && dcr_addr < (`VX_CSR_MPM_USER_H + 32))) begin - case (dcr_mpm_class) - `VX_DCR_MPM_CLASS_CORE: begin - case (dcr_addr) - // PERF: pipeline - `CSR_READ_64(`VX_CSR_MPM_SCHED_IDLE, dcr_data_w, pipeline_perf.sched.idles); - `CSR_READ_64(`VX_CSR_MPM_ACTIVE_WARPS, dcr_data_w, pipeline_perf.sched.active_warps); - `CSR_READ_64(`VX_CSR_MPM_STALLED_WARPS, dcr_data_w, pipeline_perf.sched.stalled_warps); - `CSR_READ_64(`VX_CSR_MPM_ISSUED_WARPS, dcr_data_w, pipeline_perf.sched.issued_warps); - `CSR_READ_64(`VX_CSR_MPM_ISSUED_THREADS, dcr_data_w, pipeline_perf.sched.issued_threads); - `CSR_READ_64(`VX_CSR_MPM_STALL_FETCH, dcr_data_w, pipeline_perf.fetch.stalls); - `CSR_READ_64(`VX_CSR_MPM_STALL_IBUF, dcr_data_w, pipeline_perf.issue.ibf_stalls); - `CSR_READ_64(`VX_CSR_MPM_STALL_SCRB, dcr_data_w, pipeline_perf.issue.scb_stalls); - `CSR_READ_64(`VX_CSR_MPM_STALL_OPDS, dcr_data_w, pipeline_perf.issue.opd_stalls); - `CSR_READ_64(`VX_CSR_MPM_STALL_ALU, dcr_data_w, pipeline_perf.issue.dispatch_stalls[EX_ALU]); - `CSR_READ_64(`VX_CSR_MPM_INSTR_ALU, dcr_data_w, pipeline_perf.issue.dispatch_instrs[EX_ALU]); - `CSR_READ_64(`VX_CSR_MPM_STALL_LSU, dcr_data_w, pipeline_perf.issue.dispatch_stalls[EX_LSU]); - `CSR_READ_64(`VX_CSR_MPM_INSTR_LSU, dcr_data_w, pipeline_perf.issue.dispatch_instrs[EX_LSU]); - `CSR_READ_64(`VX_CSR_MPM_STALL_SFU, dcr_data_w, pipeline_perf.issue.dispatch_stalls[EX_SFU]); - `CSR_READ_64(`VX_CSR_MPM_INSTR_SFU, dcr_data_w, pipeline_perf.issue.dispatch_instrs[EX_SFU]); - `ifdef VX_CFG_EXT_F_ENABLE - `CSR_READ_64(`VX_CSR_MPM_STALL_FPU, dcr_data_w, pipeline_perf.issue.dispatch_stalls[EX_FPU]); - `CSR_READ_64(`VX_CSR_MPM_INSTR_FPU, dcr_data_w, pipeline_perf.issue.dispatch_instrs[EX_FPU]); - `endif - `ifdef VX_CFG_EXT_TCU_ENABLE - `CSR_READ_64(`VX_CSR_MPM_STALL_TCU, dcr_data_w, pipeline_perf.issue.dispatch_stalls[EX_TCU]); - `CSR_READ_64(`VX_CSR_MPM_INSTR_TCU, dcr_data_w, pipeline_perf.issue.dispatch_instrs[EX_TCU]); - `endif - // PERF: branches - `CSR_READ_64(`VX_CSR_MPM_BRANCHES, dcr_data_w, pipeline_perf.sched.branches); - `CSR_READ_64(`VX_CSR_MPM_DIVERGENCE, dcr_data_w, pipeline_perf.sched.divergence); - // PERF: memory (core-issued requests; DRAM traffic is in the MEM class) - `CSR_READ_64(`VX_CSR_MPM_IFETCHES, dcr_data_w, pipeline_perf.ifetches); - `CSR_READ_64(`VX_CSR_MPM_LOADS, dcr_data_w, pipeline_perf.loads); - `CSR_READ_64(`VX_CSR_MPM_STORES, dcr_data_w, pipeline_perf.stores); - `CSR_READ_64(`VX_CSR_MPM_IFETCH_LT, dcr_data_w, pipeline_perf.ifetch_latency); - `CSR_READ_64(`VX_CSR_MPM_LOAD_LT, dcr_data_w, pipeline_perf.load_latency); - default:; - endcase - end - `VX_DCR_MPM_CLASS_ICACHE: begin - case (dcr_addr) - `CSR_READ_64(`VX_CSR_MPM_ICACHE_READS, dcr_data_w, sysmem_perf.icache.reads); - `CSR_READ_64(`VX_CSR_MPM_ICACHE_MISS_R, dcr_data_w, sysmem_perf.icache.read_misses); - `CSR_READ_64(`VX_CSR_MPM_ICACHE_MSHR_ST, dcr_data_w, sysmem_perf.icache.mshr_stalls); - default:; - endcase - end - `VX_DCR_MPM_CLASS_DCACHE: begin - case (dcr_addr) - `CSR_READ_64(`VX_CSR_MPM_DCACHE_READS, dcr_data_w, sysmem_perf.dcache.reads); - `CSR_READ_64(`VX_CSR_MPM_DCACHE_WRITES, dcr_data_w, sysmem_perf.dcache.writes); - `CSR_READ_64(`VX_CSR_MPM_DCACHE_MISS_R, dcr_data_w, sysmem_perf.dcache.read_misses); - `CSR_READ_64(`VX_CSR_MPM_DCACHE_MISS_W, dcr_data_w, sysmem_perf.dcache.write_misses); - `CSR_READ_64(`VX_CSR_MPM_DCACHE_EVICTS, dcr_data_w, sysmem_perf.dcache.evictions); - `CSR_READ_64(`VX_CSR_MPM_DCACHE_BANK_ST, dcr_data_w, sysmem_perf.dcache.bank_stalls); - `CSR_READ_64(`VX_CSR_MPM_DCACHE_MSHR_ST, dcr_data_w, sysmem_perf.dcache.mshr_stalls); - default:; - endcase - end - `VX_DCR_MPM_CLASS_L2CACHE: begin - case (dcr_addr) - `CSR_READ_64(`VX_CSR_MPM_L2CACHE_READS, dcr_data_w, sysmem_perf.l2cache.reads); - `CSR_READ_64(`VX_CSR_MPM_L2CACHE_WRITES, dcr_data_w, sysmem_perf.l2cache.writes); - `CSR_READ_64(`VX_CSR_MPM_L2CACHE_MISS_R, dcr_data_w, sysmem_perf.l2cache.read_misses); - `CSR_READ_64(`VX_CSR_MPM_L2CACHE_MISS_W, dcr_data_w, sysmem_perf.l2cache.write_misses); - `CSR_READ_64(`VX_CSR_MPM_L2CACHE_EVICTS, dcr_data_w, sysmem_perf.l2cache.evictions); - `CSR_READ_64(`VX_CSR_MPM_L2CACHE_BANK_ST, dcr_data_w, sysmem_perf.l2cache.bank_stalls); - `CSR_READ_64(`VX_CSR_MPM_L2CACHE_MSHR_ST, dcr_data_w, sysmem_perf.l2cache.mshr_stalls); - default:; - endcase - end - `VX_DCR_MPM_CLASS_L3CACHE: begin - case (dcr_addr) - `CSR_READ_64(`VX_CSR_MPM_L3CACHE_READS, dcr_data_w, sysmem_perf.l3cache.reads); - `CSR_READ_64(`VX_CSR_MPM_L3CACHE_WRITES, dcr_data_w, sysmem_perf.l3cache.writes); - `CSR_READ_64(`VX_CSR_MPM_L3CACHE_MISS_R, dcr_data_w, sysmem_perf.l3cache.read_misses); - `CSR_READ_64(`VX_CSR_MPM_L3CACHE_MISS_W, dcr_data_w, sysmem_perf.l3cache.write_misses); - `CSR_READ_64(`VX_CSR_MPM_L3CACHE_EVICTS, dcr_data_w, sysmem_perf.l3cache.evictions); - `CSR_READ_64(`VX_CSR_MPM_L3CACHE_BANK_ST, dcr_data_w, sysmem_perf.l3cache.bank_stalls); - `CSR_READ_64(`VX_CSR_MPM_L3CACHE_MSHR_ST, dcr_data_w, sysmem_perf.l3cache.mshr_stalls); - default:; - endcase - end - `VX_DCR_MPM_CLASS_MEM: begin - case (dcr_addr) - // PERF: off-chip memory - `CSR_READ_64(`VX_CSR_MPM_MEM_READS, dcr_data_w, sysmem_perf.mem.reads); - `CSR_READ_64(`VX_CSR_MPM_MEM_WRITES, dcr_data_w, sysmem_perf.mem.writes); - `CSR_READ_64(`VX_CSR_MPM_MEM_LT, dcr_data_w, sysmem_perf.mem.latency); - // PERF: lmem - `CSR_READ_64(`VX_CSR_MPM_LMEM_READS, dcr_data_w, sysmem_perf.lmem.reads); - `CSR_READ_64(`VX_CSR_MPM_LMEM_WRITES, dcr_data_w, sysmem_perf.lmem.writes); - `CSR_READ_64(`VX_CSR_MPM_LMEM_BANK_ST, dcr_data_w, sysmem_perf.lmem.bank_stalls); - // PERF: coalescer - `CSR_READ_64(`VX_CSR_MPM_COALESCER_MISS, dcr_data_w, sysmem_perf.coalescer.misses); - `ifdef VX_CFG_VM_ENABLE - // PERF: VM/MMU (icache + dcache MMU summed) - `CSR_READ_64(`VX_CSR_MPM_TLB_READS, dcr_data_w, pipeline_perf.mmu.tlb_reads); - `CSR_READ_64(`VX_CSR_MPM_TLB_HITS, dcr_data_w, pipeline_perf.mmu.tlb_hits); - `CSR_READ_64(`VX_CSR_MPM_TLB_MISSES, dcr_data_w, pipeline_perf.mmu.tlb_misses); - `CSR_READ_64(`VX_CSR_MPM_TLB_EVICTS, dcr_data_w, pipeline_perf.mmu.tlb_evictions); - `CSR_READ_64(`VX_CSR_MPM_PTW_WALKS, dcr_data_w, pipeline_perf.mmu.ptw_walks); - `CSR_READ_64(`VX_CSR_MPM_PTW_LATENCY, dcr_data_w, pipeline_perf.mmu.ptw_latency); - `endif - default:; - endcase - end - `ifdef VX_CFG_EXT_DXA_ENABLE - `VX_DCR_MPM_CLASS_DXA: begin - case (dcr_addr) - `CSR_READ_64(`VX_CSR_MPM_DXA_TRANSFERS, dcr_data_w, sysmem_perf.dxa.transfers); - `CSR_READ_64(`VX_CSR_MPM_DXA_GMEM_READS, dcr_data_w, sysmem_perf.dxa.gmem_reads); - `CSR_READ_64(`VX_CSR_MPM_DXA_GMEM_DEDUP, dcr_data_w, sysmem_perf.dxa.gmem_dedup); - `CSR_READ_64(`VX_CSR_MPM_DXA_LMEM_WRITES,dcr_data_w, sysmem_perf.dxa.lmem_writes); - `CSR_READ_64(`VX_CSR_MPM_DXA_GMEM_LT, dcr_data_w, sysmem_perf.dxa.gmem_latency); - `CSR_READ_64(`VX_CSR_MPM_DXA_NOSLOT_STALLS, dcr_data_w, sysmem_perf.dxa.noslot_stalls); - default:; - endcase - end - `endif - `ifdef VX_CFG_EXT_TCU_ENABLE - `VX_DCR_MPM_CLASS_TCU: begin - case (dcr_addr) - `CSR_READ_64(`VX_CSR_MPM_TCU_TBUF_STALLS, dcr_data_w, pipeline_perf.tcu.tbuf_stalls); - `CSR_READ_64(`VX_CSR_MPM_TCU_TBUF_CACHE_HITS, dcr_data_w, pipeline_perf.tcu.tbuf_cache_hits); - `CSR_READ_64(`VX_CSR_MPM_TCU_LMEM_READS, dcr_data_w, pipeline_perf.tcu.lmem_reads); - default:; - endcase - end - `endif - `ifdef VX_CFG_EXT_TEX_ENABLE - `VX_DCR_MPM_CLASS_TEX: begin - case (dcr_addr) - `CSR_READ_64(`VX_CSR_MPM_TEX_READS, dcr_data_w, sysmem_perf.tex.mem_reads); - `CSR_READ_64(`VX_CSR_MPM_TEX_LAT, dcr_data_w, sysmem_perf.tex.mem_latency); - `CSR_READ_64(`VX_CSR_MPM_TEX_ST, dcr_data_w, sysmem_perf.tex.stall_cycles); - `CSR_READ_64(`VX_CSR_MPM_TCACHE_READS, dcr_data_w, sysmem_perf.tcache.reads); - `CSR_READ_64(`VX_CSR_MPM_TCACHE_MISS_R, dcr_data_w, sysmem_perf.tcache.read_misses); - `CSR_READ_64(`VX_CSR_MPM_TCACHE_BANK_ST, dcr_data_w, sysmem_perf.tcache.bank_stalls); - `CSR_READ_64(`VX_CSR_MPM_TCACHE_MSHR_ST, dcr_data_w, sysmem_perf.tcache.mshr_stalls); - default:; - endcase - end - `endif - `ifdef VX_CFG_EXT_RASTER_ENABLE - `VX_DCR_MPM_CLASS_RASTER: begin - case (dcr_addr) - `CSR_READ_64(`VX_CSR_MPM_RASTER_READS, dcr_data_w, sysmem_perf.raster.mem_reads); - `CSR_READ_64(`VX_CSR_MPM_RASTER_LAT, dcr_data_w, sysmem_perf.raster.mem_latency); - `CSR_READ_64(`VX_CSR_MPM_RASTER_ST, dcr_data_w, sysmem_perf.raster.stall_cycles); - `CSR_READ_64(`VX_CSR_MPM_RCACHE_READS, dcr_data_w, sysmem_perf.rcache.reads); - `CSR_READ_64(`VX_CSR_MPM_RCACHE_MISS_R, dcr_data_w, sysmem_perf.rcache.read_misses); - `CSR_READ_64(`VX_CSR_MPM_RCACHE_BANK_ST, dcr_data_w, sysmem_perf.rcache.bank_stalls); - `CSR_READ_64(`VX_CSR_MPM_RCACHE_MSHR_ST, dcr_data_w, sysmem_perf.rcache.mshr_stalls); - default:; - endcase - end - `endif - `ifdef VX_CFG_EXT_OM_ENABLE - `VX_DCR_MPM_CLASS_OM: begin - case (dcr_addr) - `CSR_READ_64(`VX_CSR_MPM_OM_READS, dcr_data_w, sysmem_perf.om.mem_reads); - `CSR_READ_64(`VX_CSR_MPM_OM_WRITES, dcr_data_w, sysmem_perf.om.mem_writes); - `CSR_READ_64(`VX_CSR_MPM_OM_LAT, dcr_data_w, sysmem_perf.om.mem_latency); - `CSR_READ_64(`VX_CSR_MPM_OM_ST, dcr_data_w, sysmem_perf.om.stall_cycles); - `CSR_READ_64(`VX_CSR_MPM_OCACHE_READS, dcr_data_w, sysmem_perf.ocache.reads); - `CSR_READ_64(`VX_CSR_MPM_OCACHE_WRITES, dcr_data_w, sysmem_perf.ocache.writes); - `CSR_READ_64(`VX_CSR_MPM_OCACHE_MISS_R, dcr_data_w, sysmem_perf.ocache.read_misses); - `CSR_READ_64(`VX_CSR_MPM_OCACHE_MISS_W, dcr_data_w, sysmem_perf.ocache.write_misses); - `CSR_READ_64(`VX_CSR_MPM_OCACHE_BANK_ST, dcr_data_w, sysmem_perf.ocache.bank_stalls); - `CSR_READ_64(`VX_CSR_MPM_OCACHE_MSHR_ST, dcr_data_w, sysmem_perf.ocache.mshr_stalls); - default:; - endcase - end - `endif - default:; - endcase - end - `endif - end - endcase - end - - assign dcr_data = dcr_data_w; -`ifndef PERF_ENABLE - `UNUSED_VAR (dcr_mpm_class) -`endif - -`ifdef PERF_ENABLE - `UNUSED_VAR (sysmem_perf.icache); - `UNUSED_VAR (sysmem_perf.lmem); -`endif - endmodule diff --git a/hw/rtl/core/VX_csr_fpu.sv b/hw/rtl/core/VX_csr_fpu.sv new file mode 100644 index 0000000000..7e3889c40f --- /dev/null +++ b/hw/rtl/core/VX_csr_fpu.sv @@ -0,0 +1,103 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" +`include "VX_fpu_define.vh" + +// Per-warp FP CSRs (frm + fflags) for one residue class of warps modulo WID_STRIDE, +// stored by wid / WID_STRIDE; every request presented belongs to that class. +// CSR port c serves the warps with (wid / WID_STRIDE) mod NUM_CSR_PORTS == c and +// FPU port f those with (wid / WID_STRIDE) mod NUM_FPU_PORTS == f, so each warp's +// state is written by exactly one port of each kind. +module VX_csr_fpu import VX_gpu_pkg::*, VX_fpu_pkg::*; #( + parameter WID_STRIDE = 1, + parameter NUM_CSR_PORTS = 1, + parameter NUM_FPU_PORTS = 1 +) ( + input wire clk, + input wire reset, + + input wire [NUM_CSR_PORTS-1:0][NW_WIDTH-1:0] csr_wid, + output wire [NUM_CSR_PORTS-1:0][INST_FRM_BITS+`FP_FLAGS_BITS-1:0] csr_rd_data, + input wire [NUM_CSR_PORTS-1:0][2:0] csr_wr_sel, // {fcsr, frm, fflags} + input wire [NUM_CSR_PORTS-1:0][INST_FRM_BITS+`FP_FLAGS_BITS-1:0] csr_wr_data, + + VX_fpu_csr_if.slave fpu_csr_if [NUM_FPU_PORTS] +); + localparam FCSR_W = INST_FRM_BITS + `FP_FLAGS_BITS; + localparam NUM_ENTRIES = `VX_CFG_NUM_WARPS / WID_STRIDE; + localparam STRIDE_BITS = `CLOG2(WID_STRIDE); + localparam ENTRY_W = `LOG2UP(NUM_ENTRIES); + + `STATIC_ASSERT (`IS_POW2(WID_STRIDE), ("invalid parameter")) + + function automatic logic [ENTRY_W-1:0] entry_of(input logic [NW_WIDTH-1:0] wid); + entry_of = ENTRY_W'(wid >> STRIDE_BITS); + endfunction + + wire [NUM_FPU_PORTS-1:0] fpu_wr_enable; + wire [NUM_FPU_PORTS-1:0][NW_WIDTH-1:0] fpu_wr_wid; + fflags_t [NUM_FPU_PORTS-1:0] fpu_wr_fflags; + for (genvar f = 0; f < NUM_FPU_PORTS; ++f) begin : g_fpu_ports + assign fpu_wr_enable[f] = fpu_csr_if[f].write_enable; + assign fpu_wr_wid[f] = fpu_csr_if[f].write_wid; + assign fpu_wr_fflags[f] = fpu_csr_if[f].write_fflags; + end + + wire [NUM_ENTRIES-1:0][FCSR_W-1:0] fcsr; + + for (genvar l = 0; l < NUM_ENTRIES; ++l) begin : g_entries + localparam C = l % NUM_CSR_PORTS; + localparam F = l % NUM_FPU_PORTS; + localparam [ENTRY_W-1:0] ENTRY = ENTRY_W'(l); + + reg [FCSR_W-1:0] fcsr_r, fcsr_n; + + always @(*) begin + fcsr_n = fcsr_r; + if (fpu_wr_enable[F] && (entry_of(fpu_wr_wid[F]) == ENTRY)) begin + fcsr_n[`FP_FLAGS_BITS-1:0] = fcsr_r[`FP_FLAGS_BITS-1:0] | fpu_wr_fflags[F]; + end + if (entry_of(csr_wid[C]) == ENTRY) begin + if (csr_wr_sel[C][0]) begin + fcsr_n[`FP_FLAGS_BITS-1:0] = csr_wr_data[C][`FP_FLAGS_BITS-1:0]; + end + if (csr_wr_sel[C][1]) begin + fcsr_n[FCSR_W-1:`FP_FLAGS_BITS] = csr_wr_data[C][INST_FRM_BITS-1:0]; + end + if (csr_wr_sel[C][2]) begin + fcsr_n = csr_wr_data[C]; + end + end + end + + always @(posedge clk) begin + if (reset) begin + fcsr_r <= '0; + end else begin + fcsr_r <= fcsr_n; + end + end + + assign fcsr[l] = fcsr_r; + end + + for (genvar c = 0; c < NUM_CSR_PORTS; ++c) begin : g_csr_rd + assign csr_rd_data[c] = fcsr[entry_of(csr_wid[c])]; + end + + for (genvar f = 0; f < NUM_FPU_PORTS; ++f) begin : g_fpu_rd + assign fpu_csr_if[f].read_frm = fcsr[entry_of(fpu_csr_if[f].read_wid)][FCSR_W-1:`FP_FLAGS_BITS]; + end + +endmodule diff --git a/hw/rtl/core/VX_csr_mpm.sv b/hw/rtl/core/VX_csr_mpm.sv new file mode 100644 index 0000000000..9105e11851 --- /dev/null +++ b/hw/rtl/core/VX_csr_mpm.sv @@ -0,0 +1,257 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +`ifdef VX_CFG_XLEN_64 + `define CSR_READ_64(addr, dst, src) \ + addr : dst = `VX_CFG_XLEN'(src) +`else + `define CSR_READ_64(addr, dst, src) \ + addr : dst = src[31:0]; \ + addr+12'h80 : dst = 32'(src[$bits(src)-1:32]) +`endif + +// Host-side machine performance-monitor reads (DCR path). The counters are not +// reachable from kernel CSR instructions, so one instance serves the whole core. +module VX_csr_mpm import VX_gpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "" +) ( +`ifdef PERF_ENABLE + input sysmem_perf_t sysmem_perf, + input pipeline_perf_t pipeline_perf, +`endif + input wire [PERF_CTR_BITS-1:0] cycles, + input wire [PERF_CTR_BITS-1:0] instret, + + VX_dcr_csr_if.slave dcr_csr_if +); + `UNUSED_SPARAM (INSTANCE_ID) + + wire [7:0] dcr_mpm_class = dcr_csr_if.mpm_class; + wire [`VX_CSR_ADDR_BITS-1:0] dcr_addr = dcr_csr_if.addr; + wire [`VX_CFG_XLEN-1:0] dcr_data; + + // Host counter reads decode their own address and never wait on a request. + assign dcr_csr_if.ready = 1'b1; + assign dcr_csr_if.value = VX_DCR_DATA_WIDTH'(dcr_data); + `UNUSED_VAR (dcr_data) + `UNUSED_VAR (dcr_csr_if.valid) + + // Host performance-counter reads //////////////////////////////////////// + + reg [`VX_CFG_XLEN-1:0] dcr_data_w; + + always @(*) begin + dcr_data_w = '0; + case (dcr_addr) + `CSR_READ_64(`VX_CSR_MCYCLE, dcr_data_w, cycles); + `CSR_READ_64(`VX_CSR_MINSTRET, dcr_data_w, instret); + default: begin + `ifdef PERF_ENABLE + if ((dcr_addr >= `VX_CSR_MPM_USER && dcr_addr < (`VX_CSR_MPM_USER + 32)) + || (dcr_addr >= `VX_CSR_MPM_USER_H && dcr_addr < (`VX_CSR_MPM_USER_H + 32))) begin + case (dcr_mpm_class) + `VX_DCR_MPM_CLASS_CORE: begin + case (dcr_addr) + // PERF: pipeline + `CSR_READ_64(`VX_CSR_MPM_SCHED_IDLE, dcr_data_w, pipeline_perf.sched.idles); + `CSR_READ_64(`VX_CSR_MPM_ACTIVE_WARPS, dcr_data_w, pipeline_perf.sched.active_warps); + `CSR_READ_64(`VX_CSR_MPM_STALLED_WARPS, dcr_data_w, pipeline_perf.sched.stalled_warps); + `CSR_READ_64(`VX_CSR_MPM_ISSUED_WARPS, dcr_data_w, pipeline_perf.sched.issued_warps); + `CSR_READ_64(`VX_CSR_MPM_ISSUED_THREADS, dcr_data_w, pipeline_perf.sched.issued_threads); + `CSR_READ_64(`VX_CSR_MPM_STALL_FETCH, dcr_data_w, pipeline_perf.fetch.stalls); + `CSR_READ_64(`VX_CSR_MPM_STALL_IBUF, dcr_data_w, pipeline_perf.issue.ibf_stalls); + `CSR_READ_64(`VX_CSR_MPM_STALL_SCRB, dcr_data_w, pipeline_perf.issue.scb_stalls); + `CSR_READ_64(`VX_CSR_MPM_STALL_OPDS, dcr_data_w, pipeline_perf.issue.opd_stalls); + `CSR_READ_64(`VX_CSR_MPM_STALL_ALU, dcr_data_w, pipeline_perf.issue.dispatch_stalls[EX_ALU]); + `CSR_READ_64(`VX_CSR_MPM_INSTR_ALU, dcr_data_w, pipeline_perf.issue.dispatch_instrs[EX_ALU]); + `CSR_READ_64(`VX_CSR_MPM_STALL_LSU, dcr_data_w, pipeline_perf.issue.dispatch_stalls[EX_LSU]); + `CSR_READ_64(`VX_CSR_MPM_INSTR_LSU, dcr_data_w, pipeline_perf.issue.dispatch_instrs[EX_LSU]); + `CSR_READ_64(`VX_CSR_MPM_STALL_SFU, dcr_data_w, pipeline_perf.issue.dispatch_stalls[EX_SFU]); + `CSR_READ_64(`VX_CSR_MPM_INSTR_SFU, dcr_data_w, pipeline_perf.issue.dispatch_instrs[EX_SFU]); + `ifdef VX_CFG_EXT_F_ENABLE + `CSR_READ_64(`VX_CSR_MPM_STALL_FPU, dcr_data_w, pipeline_perf.issue.dispatch_stalls[EX_FPU]); + `CSR_READ_64(`VX_CSR_MPM_INSTR_FPU, dcr_data_w, pipeline_perf.issue.dispatch_instrs[EX_FPU]); + `endif + `ifdef VX_CFG_EXT_TCU_ENABLE + `CSR_READ_64(`VX_CSR_MPM_STALL_TCU, dcr_data_w, pipeline_perf.issue.dispatch_stalls[EX_TCU]); + `CSR_READ_64(`VX_CSR_MPM_INSTR_TCU, dcr_data_w, pipeline_perf.issue.dispatch_instrs[EX_TCU]); + `endif + // PERF: branches + `CSR_READ_64(`VX_CSR_MPM_BRANCHES, dcr_data_w, pipeline_perf.sched.branches); + `CSR_READ_64(`VX_CSR_MPM_DIVERGENCE, dcr_data_w, pipeline_perf.sched.divergence); + // PERF: memory (core-issued requests; DRAM traffic is in the MEM class) + `CSR_READ_64(`VX_CSR_MPM_IFETCHES, dcr_data_w, pipeline_perf.ifetches); + `CSR_READ_64(`VX_CSR_MPM_LOADS, dcr_data_w, pipeline_perf.loads); + `CSR_READ_64(`VX_CSR_MPM_STORES, dcr_data_w, pipeline_perf.stores); + `CSR_READ_64(`VX_CSR_MPM_IFETCH_LT, dcr_data_w, pipeline_perf.ifetch_latency); + `CSR_READ_64(`VX_CSR_MPM_LOAD_LT, dcr_data_w, pipeline_perf.load_latency); + default:; + endcase + end + `VX_DCR_MPM_CLASS_ICACHE: begin + case (dcr_addr) + `CSR_READ_64(`VX_CSR_MPM_ICACHE_READS, dcr_data_w, sysmem_perf.icache.reads); + `CSR_READ_64(`VX_CSR_MPM_ICACHE_MISS_R, dcr_data_w, sysmem_perf.icache.read_misses); + `CSR_READ_64(`VX_CSR_MPM_ICACHE_MSHR_ST, dcr_data_w, sysmem_perf.icache.mshr_stalls); + default:; + endcase + end + `VX_DCR_MPM_CLASS_DCACHE: begin + case (dcr_addr) + `CSR_READ_64(`VX_CSR_MPM_DCACHE_READS, dcr_data_w, sysmem_perf.dcache.reads); + `CSR_READ_64(`VX_CSR_MPM_DCACHE_WRITES, dcr_data_w, sysmem_perf.dcache.writes); + `CSR_READ_64(`VX_CSR_MPM_DCACHE_MISS_R, dcr_data_w, sysmem_perf.dcache.read_misses); + `CSR_READ_64(`VX_CSR_MPM_DCACHE_MISS_W, dcr_data_w, sysmem_perf.dcache.write_misses); + `CSR_READ_64(`VX_CSR_MPM_DCACHE_EVICTS, dcr_data_w, sysmem_perf.dcache.evictions); + `CSR_READ_64(`VX_CSR_MPM_DCACHE_BANK_ST, dcr_data_w, sysmem_perf.dcache.bank_stalls); + `CSR_READ_64(`VX_CSR_MPM_DCACHE_MSHR_ST, dcr_data_w, sysmem_perf.dcache.mshr_stalls); + default:; + endcase + end + `VX_DCR_MPM_CLASS_L2CACHE: begin + case (dcr_addr) + `CSR_READ_64(`VX_CSR_MPM_L2CACHE_READS, dcr_data_w, sysmem_perf.l2cache.reads); + `CSR_READ_64(`VX_CSR_MPM_L2CACHE_WRITES, dcr_data_w, sysmem_perf.l2cache.writes); + `CSR_READ_64(`VX_CSR_MPM_L2CACHE_MISS_R, dcr_data_w, sysmem_perf.l2cache.read_misses); + `CSR_READ_64(`VX_CSR_MPM_L2CACHE_MISS_W, dcr_data_w, sysmem_perf.l2cache.write_misses); + `CSR_READ_64(`VX_CSR_MPM_L2CACHE_EVICTS, dcr_data_w, sysmem_perf.l2cache.evictions); + `CSR_READ_64(`VX_CSR_MPM_L2CACHE_BANK_ST, dcr_data_w, sysmem_perf.l2cache.bank_stalls); + `CSR_READ_64(`VX_CSR_MPM_L2CACHE_MSHR_ST, dcr_data_w, sysmem_perf.l2cache.mshr_stalls); + default:; + endcase + end + `VX_DCR_MPM_CLASS_L3CACHE: begin + case (dcr_addr) + `CSR_READ_64(`VX_CSR_MPM_L3CACHE_READS, dcr_data_w, sysmem_perf.l3cache.reads); + `CSR_READ_64(`VX_CSR_MPM_L3CACHE_WRITES, dcr_data_w, sysmem_perf.l3cache.writes); + `CSR_READ_64(`VX_CSR_MPM_L3CACHE_MISS_R, dcr_data_w, sysmem_perf.l3cache.read_misses); + `CSR_READ_64(`VX_CSR_MPM_L3CACHE_MISS_W, dcr_data_w, sysmem_perf.l3cache.write_misses); + `CSR_READ_64(`VX_CSR_MPM_L3CACHE_EVICTS, dcr_data_w, sysmem_perf.l3cache.evictions); + `CSR_READ_64(`VX_CSR_MPM_L3CACHE_BANK_ST, dcr_data_w, sysmem_perf.l3cache.bank_stalls); + `CSR_READ_64(`VX_CSR_MPM_L3CACHE_MSHR_ST, dcr_data_w, sysmem_perf.l3cache.mshr_stalls); + default:; + endcase + end + `VX_DCR_MPM_CLASS_MEM: begin + case (dcr_addr) + // PERF: off-chip memory + `CSR_READ_64(`VX_CSR_MPM_MEM_READS, dcr_data_w, sysmem_perf.mem.reads); + `CSR_READ_64(`VX_CSR_MPM_MEM_WRITES, dcr_data_w, sysmem_perf.mem.writes); + `CSR_READ_64(`VX_CSR_MPM_MEM_LT, dcr_data_w, sysmem_perf.mem.latency); + // PERF: lmem + `CSR_READ_64(`VX_CSR_MPM_LMEM_READS, dcr_data_w, sysmem_perf.lmem.reads); + `CSR_READ_64(`VX_CSR_MPM_LMEM_WRITES, dcr_data_w, sysmem_perf.lmem.writes); + `CSR_READ_64(`VX_CSR_MPM_LMEM_BANK_ST, dcr_data_w, sysmem_perf.lmem.bank_stalls); + // PERF: coalescer + `CSR_READ_64(`VX_CSR_MPM_COALESCER_MISS, dcr_data_w, sysmem_perf.coalescer.misses); + `ifdef VX_CFG_VM_ENABLE + // PERF: VM/MMU (icache + dcache MMU summed) + `CSR_READ_64(`VX_CSR_MPM_TLB_READS, dcr_data_w, pipeline_perf.mmu.tlb_reads); + `CSR_READ_64(`VX_CSR_MPM_TLB_HITS, dcr_data_w, pipeline_perf.mmu.tlb_hits); + `CSR_READ_64(`VX_CSR_MPM_TLB_MISSES, dcr_data_w, pipeline_perf.mmu.tlb_misses); + `CSR_READ_64(`VX_CSR_MPM_TLB_EVICTS, dcr_data_w, pipeline_perf.mmu.tlb_evictions); + `CSR_READ_64(`VX_CSR_MPM_PTW_WALKS, dcr_data_w, pipeline_perf.mmu.ptw_walks); + `CSR_READ_64(`VX_CSR_MPM_PTW_LATENCY, dcr_data_w, pipeline_perf.mmu.ptw_latency); + `endif + default:; + endcase + end + `ifdef VX_CFG_EXT_DXA_ENABLE + `VX_DCR_MPM_CLASS_DXA: begin + case (dcr_addr) + `CSR_READ_64(`VX_CSR_MPM_DXA_TRANSFERS, dcr_data_w, sysmem_perf.dxa.transfers); + `CSR_READ_64(`VX_CSR_MPM_DXA_GMEM_READS, dcr_data_w, sysmem_perf.dxa.gmem_reads); + `CSR_READ_64(`VX_CSR_MPM_DXA_GMEM_DEDUP, dcr_data_w, sysmem_perf.dxa.gmem_dedup); + `CSR_READ_64(`VX_CSR_MPM_DXA_LMEM_WRITES,dcr_data_w, sysmem_perf.dxa.lmem_writes); + `CSR_READ_64(`VX_CSR_MPM_DXA_GMEM_LT, dcr_data_w, sysmem_perf.dxa.gmem_latency); + `CSR_READ_64(`VX_CSR_MPM_DXA_NOSLOT_STALLS, dcr_data_w, sysmem_perf.dxa.noslot_stalls); + default:; + endcase + end + `endif + `ifdef VX_CFG_EXT_TCU_ENABLE + `VX_DCR_MPM_CLASS_TCU: begin + case (dcr_addr) + `CSR_READ_64(`VX_CSR_MPM_TCU_TBUF_STALLS, dcr_data_w, pipeline_perf.tcu.tbuf_stalls); + `CSR_READ_64(`VX_CSR_MPM_TCU_TBUF_CACHE_HITS, dcr_data_w, pipeline_perf.tcu.tbuf_cache_hits); + `CSR_READ_64(`VX_CSR_MPM_TCU_LMEM_READS, dcr_data_w, pipeline_perf.tcu.lmem_reads); + default:; + endcase + end + `endif + `ifdef VX_CFG_EXT_TEX_ENABLE + `VX_DCR_MPM_CLASS_TEX: begin + case (dcr_addr) + `CSR_READ_64(`VX_CSR_MPM_TEX_READS, dcr_data_w, sysmem_perf.tex.mem_reads); + `CSR_READ_64(`VX_CSR_MPM_TEX_LAT, dcr_data_w, sysmem_perf.tex.mem_latency); + `CSR_READ_64(`VX_CSR_MPM_TEX_ST, dcr_data_w, sysmem_perf.tex.stall_cycles); + `CSR_READ_64(`VX_CSR_MPM_TCACHE_READS, dcr_data_w, sysmem_perf.tcache.reads); + `CSR_READ_64(`VX_CSR_MPM_TCACHE_MISS_R, dcr_data_w, sysmem_perf.tcache.read_misses); + `CSR_READ_64(`VX_CSR_MPM_TCACHE_BANK_ST, dcr_data_w, sysmem_perf.tcache.bank_stalls); + `CSR_READ_64(`VX_CSR_MPM_TCACHE_MSHR_ST, dcr_data_w, sysmem_perf.tcache.mshr_stalls); + default:; + endcase + end + `endif + `ifdef VX_CFG_EXT_RASTER_ENABLE + `VX_DCR_MPM_CLASS_RASTER: begin + case (dcr_addr) + `CSR_READ_64(`VX_CSR_MPM_RASTER_READS, dcr_data_w, sysmem_perf.raster.mem_reads); + `CSR_READ_64(`VX_CSR_MPM_RASTER_LAT, dcr_data_w, sysmem_perf.raster.mem_latency); + `CSR_READ_64(`VX_CSR_MPM_RASTER_ST, dcr_data_w, sysmem_perf.raster.stall_cycles); + `CSR_READ_64(`VX_CSR_MPM_RCACHE_READS, dcr_data_w, sysmem_perf.rcache.reads); + `CSR_READ_64(`VX_CSR_MPM_RCACHE_MISS_R, dcr_data_w, sysmem_perf.rcache.read_misses); + `CSR_READ_64(`VX_CSR_MPM_RCACHE_BANK_ST, dcr_data_w, sysmem_perf.rcache.bank_stalls); + `CSR_READ_64(`VX_CSR_MPM_RCACHE_MSHR_ST, dcr_data_w, sysmem_perf.rcache.mshr_stalls); + default:; + endcase + end + `endif + `ifdef VX_CFG_EXT_OM_ENABLE + `VX_DCR_MPM_CLASS_OM: begin + case (dcr_addr) + `CSR_READ_64(`VX_CSR_MPM_OM_READS, dcr_data_w, sysmem_perf.om.mem_reads); + `CSR_READ_64(`VX_CSR_MPM_OM_WRITES, dcr_data_w, sysmem_perf.om.mem_writes); + `CSR_READ_64(`VX_CSR_MPM_OM_LAT, dcr_data_w, sysmem_perf.om.mem_latency); + `CSR_READ_64(`VX_CSR_MPM_OM_ST, dcr_data_w, sysmem_perf.om.stall_cycles); + `CSR_READ_64(`VX_CSR_MPM_OCACHE_READS, dcr_data_w, sysmem_perf.ocache.reads); + `CSR_READ_64(`VX_CSR_MPM_OCACHE_WRITES, dcr_data_w, sysmem_perf.ocache.writes); + `CSR_READ_64(`VX_CSR_MPM_OCACHE_MISS_R, dcr_data_w, sysmem_perf.ocache.read_misses); + `CSR_READ_64(`VX_CSR_MPM_OCACHE_MISS_W, dcr_data_w, sysmem_perf.ocache.write_misses); + `CSR_READ_64(`VX_CSR_MPM_OCACHE_BANK_ST, dcr_data_w, sysmem_perf.ocache.bank_stalls); + `CSR_READ_64(`VX_CSR_MPM_OCACHE_MSHR_ST, dcr_data_w, sysmem_perf.ocache.mshr_stalls); + default:; + endcase + end + `endif + default:; + endcase + end + `endif + end + endcase + end + + assign dcr_data = dcr_data_w; +`ifndef PERF_ENABLE + `UNUSED_VAR (dcr_mpm_class) +`endif + +`ifdef PERF_ENABLE + `UNUSED_VAR (sysmem_perf.icache); + `UNUSED_VAR (sysmem_perf.lmem); +`endif + +endmodule diff --git a/hw/rtl/core/VX_csr_unit.sv b/hw/rtl/core/VX_csr_unit.sv index 925d1e7cff..d4f501574b 100644 --- a/hw/rtl/core/VX_csr_unit.sv +++ b/hw/rtl/core/VX_csr_unit.sv @@ -13,25 +13,27 @@ `include "VX_define.vh" +`ifdef VX_CFG_EXT_F_ENABLE +`include "VX_fpu_define.vh" +`endif + module VX_csr_unit import VX_gpu_pkg::*; #( parameter `STRING INSTANCE_ID = "", - parameter CORE_ID = 0, parameter NUM_LANES = 1 ) ( input wire clk, input wire reset, -`ifdef PERF_ENABLE - input sysmem_perf_t sysmem_perf, - input pipeline_perf_t pipeline_perf, -`endif + input wire [CORE_ID_WIDTH-1:0] core_id, `ifdef VX_CFG_EXT_F_ENABLE - VX_fpu_csr_if.slave fpu_csr_if [`VX_CFG_NUM_FPU_BLOCKS], + output wire [NW_WIDTH-1:0] fcsr_wid, + input wire [INST_FRM_BITS+`FP_FLAGS_BITS-1:0] fcsr_rd_data, + output wire [2:0] fcsr_wr_sel, + output wire [INST_FRM_BITS+`FP_FLAGS_BITS-1:0] fcsr_wr_data, `endif VX_sched_csr_if.slave sched_csr_if, - VX_dcr_csr_if.slave dcr_csr_if, VX_execute_if.slave execute_if, VX_result_if.master result_if ); @@ -75,30 +77,21 @@ module VX_csr_unit import VX_gpu_pkg::*; #( wire [`VX_CFG_XLEN-1:0] csr_req_src = execute_if.data.op_args.csr.use_imm ? `VX_CFG_XLEN'(csr_imm) : rs1_data[0]; - wire [`VX_CFG_XLEN-1:0] csr_scalar_data, dcr_read_data; - - // Host counter reads decode their own address and never wait on a request. - assign dcr_csr_if.ready = 1'b1; - assign dcr_csr_if.value = VX_DCR_DATA_WIDTH'(dcr_read_data); - `UNUSED_VAR (dcr_read_data) - `UNUSED_VAR (dcr_csr_if.valid) + wire [`VX_CFG_XLEN-1:0] csr_scalar_data; VX_csr_data #( - .INSTANCE_ID (INSTANCE_ID), - .CORE_ID (CORE_ID) + .INSTANCE_ID (INSTANCE_ID) ) csr_data ( .clk (clk), .reset (reset), - - `ifdef PERF_ENABLE - .sysmem_perf (sysmem_perf), - .pipeline_perf (pipeline_perf), - `endif + .core_id (core_id), .sched_csr_if (sched_csr_if), `ifdef VX_CFG_EXT_F_ENABLE - .fpu_csr_if (fpu_csr_if), + .fcsr_rd_data (fcsr_rd_data), + .fcsr_wr_sel (fcsr_wr_sel), + .fcsr_wr_data (fcsr_wr_data), `endif .req_fire (csr_req_fire), @@ -108,18 +101,18 @@ module VX_csr_unit import VX_gpu_pkg::*; #( .req_addr (csr_addr), .req_op (execute_if.data.op_type), .req_src (csr_req_src), - .read_data (csr_scalar_data), - - .dcr_mpm_class (dcr_csr_if.mpm_class), - .dcr_addr (dcr_csr_if.addr), - .dcr_data (dcr_read_data) + .read_data (csr_scalar_data) ); +`ifdef VX_CFG_EXT_F_ENABLE + assign fcsr_wid = execute_if.data.header.wid; +`endif + // Per-lane CSRs // Thread ids are the lane index on top of a per-request base. wire [`VX_CFG_XLEN-1:0] wtid_base = (PID_BITS != 0) ? `VX_CFG_XLEN'(execute_if.data.header.pid * NUM_LANES) : '0; - wire [`VX_CFG_XLEN-1:0] gtid_base = (`VX_CFG_XLEN'(CORE_ID) << (NW_BITS + NT_BITS)) + wire [`VX_CFG_XLEN-1:0] gtid_base = (`VX_CFG_XLEN'(core_id) << (NW_BITS + NT_BITS)) + (`VX_CFG_XLEN'(execute_if.data.header.wid) << NT_BITS) + wtid_base; diff --git a/hw/rtl/core/VX_cta_dispatch.sv b/hw/rtl/core/VX_cta_dispatch.sv index 4991f97899..8dd1147587 100644 --- a/hw/rtl/core/VX_cta_dispatch.sv +++ b/hw/rtl/core/VX_cta_dispatch.sv @@ -13,9 +13,12 @@ `include "VX_define.vh" -// Dispatches warps for incoming CTAs from the KMU. +// Dispatches warps for incoming CTAs from the KMU. Warps launch on NUM_LANES +// lanes in parallel: lane l owns the warps with wid % NUM_LANES == l, and each +// lane launches one of its free warps per cycle until the CTA is fully launched. module VX_cta_dispatch import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "" + parameter `STRING INSTANCE_ID = "", + parameter NUM_LANES = 1 ) ( input wire clk, input wire reset, @@ -28,31 +31,27 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( input wire warp_done, input wire [NW_WIDTH-1:0] warp_done_wid, - // to scheduler (one warp per cycle) - output wire cta_fire, - output wire [NW_WIDTH-1:0] cta_wid, + // to scheduler (one warp per lane per cycle) + output wire [NUM_LANES-1:0] cta_fire, + output wire [NUM_LANES-1:0][NW_WIDTH-1:0] cta_wid, output wire [PC_BITS-1:0] cta_PC, - output wire [`VX_CFG_NUM_THREADS-1:0] cta_tmask, + output wire [NUM_LANES-1:0][`VX_CFG_NUM_THREADS-1:0] cta_tmask, output wire [`VX_CFG_MEM_ADDR_WIDTH-1:0] cta_param, - output wire cta_init, + output wire [NUM_LANES-1:0] cta_init, - // CTA-CSR read-back - input wire [NW_WIDTH-1:0] csr_rd_wid, - input wire [NCTA_WIDTH-1:0] csr_rd_cta_id, - output cta_csrs_t cta_rd_csrs, - // The per-lane launch record (see cta_lane_t): a compute warp's thread index and a - // fragment warp's stamp overlay the same bits, and the CSR unit reads the view the - // CSR being accessed calls for. - output cta_lane_t [`VX_CFG_NUM_THREADS-1:0] cta_rd_lane, - - // CTA id of the scheduled warp - input wire [NW_WIDTH-1:0] schedule_wid, - output wire [NCTA_WIDTH-1:0] schedule_cta_id, + // CTA state for the CTA-CSR stores (see VX_csr_cta): the CTA context, + // and each lane's warp records + output cta_bcast_t cta_bcast, + output cta_lane_bcast_t [NUM_LANES-1:0] cta_lane_bcast, output wire busy ); `UNUSED_SPARAM (INSTANCE_ID) + `STATIC_ASSERT ((`VX_CFG_NUM_WARPS % NUM_LANES) == 0, ("invalid parameter")) localparam NUM_CTA_SLOTS= `VX_CFG_NUM_WARPS; + localparam LANE_WARPS = `VX_CFG_NUM_WARPS / NUM_LANES; + localparam LANE_WIDTH = `LOG2UP(NUM_LANES); + localparam LCNT_WIDTH = `CLOG2(NUM_LANES+1); localparam CS_BITS = NW_WIDTH; // UP(NW_BITS): at least 1 to avoid zero-width when NUM_WARPS=1 localparam LMEM_SIZE = (1 << `VX_CFG_LMEM_LOG_SIZE); @@ -114,7 +113,6 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( // so a slot relaunched with lanes it has not run before must run it again. reg [7:0] cur_ctx_id_r; reg [`VX_CFG_NUM_WARPS-1:0][`VX_CFG_NUM_THREADS-1:0] warp_init_lanes_r; - reg warp_skip_init_r; // ------------------------------------------------------------------------- // FSM + per-dispatch registers @@ -131,28 +129,50 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( reg [CTA_TID_WIDTH:0] block_size_r; reg [2:0][CTA_TID_WIDTH-1:0] warp_step_r; reg [NW_WIDTH:0] cluster_size_r; - reg warp_fire_r; - reg [NW_WIDTH-1:0] warp_id_r; - reg [`VX_CFG_NUM_THREADS-1:0] warp_tmask_r; reg [NW_WIDTH-1:0] cta_rank_r; - reg [2:0][CTA_TID_WIDTH-1:0] thread_idx_r; reg [CS_BITS-1:0] cur_slot_r; + // Warps launched last cycle, one per lane; `order` is the warp's position + // among them, so its CTA rank is cta_rank_r + order. + reg [NUM_LANES-1:0] warp_fire_r; + reg [NUM_LANES-1:0][NW_WIDTH-1:0] warp_id_r; + reg [NUM_LANES-1:0][`VX_CFG_NUM_THREADS-1:0] warp_tmask_r; + reg [NUM_LANES-1:0][LANE_WIDTH-1:0] warp_order_r; + reg [NUM_LANES-1:0] warp_skip_init_r; + reg [LCNT_WIDTH-1:0] nfired_r; + wire fire_any_r = (| warp_fire_r); + + // Thread-index window: thread_idx_r[k] is the base thread index of the warp + // ranked cta_rank_r + k. Launching n warps shifts the window by n; the entries + // shifted in advance by group_step_r (NUM_LANES warps), so no two advances + // chain within a cycle. + reg [NUM_LANES-1:0][2:0][CTA_TID_WIDTH-1:0] thread_idx_r; + reg [2:0][CTA_TID_WIDTH-1:0] group_step_r; + // ------------------------------------------------------------------------- - // Free-warp selection + // Free-warp selection: one candidate per lane // ------------------------------------------------------------------------- reg [`VX_CFG_NUM_WARPS-1:0] dispatched_warps; - wire [NW_WIDTH-1:0] warp_id_n; - wire warp_ready; - VX_priority_encoder #( - .N (`VX_CFG_NUM_WARPS), - .REVERSE (0) - ) priority_enc ( - .data_in (~(active_warps | dispatched_warps)), - `UNUSED_PIN(onehot_out), - .index_out (warp_id_n), - .valid_out (warp_ready) - ); + wire [`VX_CFG_NUM_WARPS-1:0] free_warps = ~(active_warps | dispatched_warps); + wire [NUM_LANES-1:0] lane_ready; + wire [NUM_LANES-1:0][NW_WIDTH-1:0] lane_wid; + for (genvar l = 0; l < NUM_LANES; ++l) begin : g_lane_select + wire [LANE_WARPS-1:0] lane_free; + for (genvar j = 0; j < LANE_WARPS; ++j) begin : g_free + assign lane_free[j] = free_warps[j * NUM_LANES + l]; + end + wire [`LOG2UP(LANE_WARPS)-1:0] lane_idx; + VX_priority_encoder #( + .N (LANE_WARPS), + .REVERSE (0) + ) priority_enc ( + .data_in (lane_free), + `UNUSED_PIN(onehot_out), + .index_out (lane_idx), + .valid_out (lane_ready[l]) + ); + assign lane_wid[l] = NW_WIDTH'(lane_idx) * NW_WIDTH'(NUM_LANES) + NW_WIDTH'(l); + end wire kmu_bus_if_fire = kmu_bus_if.valid && kmu_bus_if.ready; @@ -191,7 +211,6 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( // pipeline; `is_frag_warp`/`frag_stamps_sel` pick the lane-record view being // written. All benign (0) in a build with no rasterizer. wire frag_inflight; - wire is_frag_warp; wire [`VX_CFG_NUM_THREADS-1:0][FRAG_LANE_BITS-1:0] frag_stamps_sel; `ifdef VX_CFG_EXT_RASTER_ENABLE @@ -213,54 +232,126 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( // ------------------------------------------------------------------------- // Power-of-two NUM_THREADS arithmetic — all combinational, zero adders // ------------------------------------------------------------------------- - wire [NW_WIDTH:0] cta_num_warps; wire [NW_WIDTH:0] kmu_num_warps; wire [CTA_TID_WIDTH:0] block_size_next; wire [CTA_TID_WIDTH:0] block_size_cur; wire [`VX_CFG_NUM_THREADS-1:0] partial_tmask; - // block_size_r retires the previously latched warp's threads a cycle late, so - // the count left for the warp being latched now excludes them. - assign block_size_cur = warp_fire_r ? block_size_next : block_size_r; + // block_size_r retires the warps latched last cycle a cycle late, so the count + // left for the warps being latched now excludes them. + assign block_size_cur = fire_any_r ? block_size_next : block_size_r; if (NT_BITS > 0) begin : g_nt_nonzero // Ceiling division block_size / NUM_THREADS: upper bits + OR of lower bits. - assign cta_num_warps = (NW_WIDTH+1)'(block_size_r[CTA_TID_WIDTH:NT_BITS]) + (NW_WIDTH+1)'(|block_size_r[NT_BITS-1:0]); // From KMU data at accept time (used to initialise table + cta_size output) assign kmu_num_warps = (NW_WIDTH+1)'(eff_block_size[CTA_TID_WIDTH:NT_BITS]) + (NW_WIDTH+1)'(|eff_block_size[NT_BITS-1:0]); - // Shared block_size decrement: low NT_BITS bits unchanged; upper bits decrement by 1. - assign block_size_next = {block_size_r[CTA_TID_WIDTH:NT_BITS] - 1'b1, block_size_r[NT_BITS-1:0]}; + // Shared block_size decrement: low NT_BITS bits unchanged; upper bits drop by + // the warps launched. + assign block_size_next = {block_size_r[CTA_TID_WIDTH:NT_BITS] - (CTA_TID_WIDTH+1-NT_BITS)'(nfired_r), block_size_r[NT_BITS-1:0]}; // Partial-warp mask: (1 << count) - 1 where count = block_size_cur[NT_BITS-1:0] assign partial_tmask = (`VX_CFG_NUM_THREADS'(1) << block_size_cur[NT_BITS-1:0]) - `VX_CFG_NUM_THREADS'(1); end else begin : g_nt_zero // NT_BITS=0: NUM_THREADS=1, each warp has exactly 1 thread, no partial warps. - assign cta_num_warps = (NW_WIDTH+1)'(block_size_r); assign kmu_num_warps = (NW_WIDTH+1)'(eff_block_size); - assign block_size_next = (CTA_TID_WIDTH+1)'(block_size_r - 1'b1); + assign block_size_next = (CTA_TID_WIDTH+1)'(block_size_r - (CTA_TID_WIDTH+1)'(nfired_r)); assign partial_tmask = `VX_CFG_NUM_THREADS'(0); end - // Full-warp test: upper bits non-zero (no comparator) - wire is_full_warp = |block_size_cur[CTA_TID_WIDTH:NT_BITS]; - // The lanes this launch activates: full warp = all ones, partial = (1< 0) begin : g_rem_nt + assign rem_warps_cur = (NW_WIDTH+1)'(full_warps_cur) + (NW_WIDTH+1)'(|block_size_cur[NT_BITS-1:0]); + end else begin : g_rem_nt1 + assign rem_warps_cur = (NW_WIDTH+1)'(full_warps_cur); + end - // Does slot w still owe the startup for any lane this launch activates? Resolved - // per slot, off the warp_id_n path, so the encoder still feeds only a 1-bit mux. + // Launch selection: lanes with a free warp, taken in round-robin order from + // lane_rr_r, up to the warps left. order[l] is the lane's position among + // the selected lanes. + reg [LANE_WIDTH-1:0] lane_rr_r; + wire launch_en; + reg [NUM_LANES-1:0] lane_sel; + reg [NUM_LANES-1:0][LANE_WIDTH-1:0] lane_order; + reg [LCNT_WIDTH-1:0] lane_cnt; + always @(*) begin + lane_sel = '0; + lane_order = '0; + lane_cnt = '0; + for (integer o = 0; o < NUM_LANES; ++o) begin + if (launch_en && lane_ready[(int'(lane_rr_r) + o) % NUM_LANES] + && ((NW_WIDTH+1)'(lane_cnt) < rem_warps_cur)) begin + lane_sel[(int'(lane_rr_r) + o) % NUM_LANES] = 1'b1; + lane_order[(int'(lane_rr_r) + o) % NUM_LANES] = LANE_WIDTH'(lane_cnt); + lane_cnt = lane_cnt + LCNT_WIDTH'(1); + end + end + end + + // A warp is full unless it is the CTA's last and the block size is not a + // multiple of the warp width. + wire [NUM_LANES-1:0][`VX_CFG_NUM_THREADS-1:0] lane_tmask; + for (genvar l = 0; l < NUM_LANES; ++l) begin : g_lane_tmask + wire is_full_warp = ((CTA_TID_WIDTH+1-NT_BITS)'(lane_order[l]) < full_warps_cur); + assign lane_tmask[l] = is_full_warp ? {`VX_CFG_NUM_THREADS{1'b1}} : partial_tmask; + end + + // Does slot w still owe the startup for any lane a full launch activates? A + // partial last warp re-runs the check on its own mask. wire [`VX_CFG_NUM_WARPS-1:0] slot_needs_init; for (genvar w = 0; w < `VX_CFG_NUM_WARPS; ++w) begin : g_needs_init - assign slot_needs_init[w] = |(launch_tmask & ~warp_init_lanes_r[w]); + localparam l = w % NUM_LANES; + assign slot_needs_init[w] = |(lane_tmask[l] & ~warp_init_lanes_r[w]); end - // Last-warp test: ceiling(remaining/NUM_THREADS) == 1. - // Covers (upper==1, lower==0) full-last and (upper==0, lower!=0) partial-only cases. - wire is_last_warp = (cta_num_warps == (NW_WIDTH+1)'(1)); - // 3D thread-index carry-propagation (combinational on registered state) - wire [CTA_TID_WIDTH:0] next_x = {1'b0, thread_idx_r[0]} + {1'b0, warp_step_r[0]}; - wire wrap_x = (next_x >= block_dim_r[0]); - wire [CTA_TID_WIDTH:0] next_y = ({1'b0, thread_idx_r[1]} + {1'b0, warp_step_r[1]}) + (CTA_TID_WIDTH+1)'(wrap_x); - wire wrap_y = (next_y >= block_dim_r[1]); + // One thread-index advance by `step`, with a single wrap per dimension: every + // step is normalized below the block dimensions. + function automatic logic [2:0][CTA_TID_WIDTH-1:0] tid_advance ( + input logic [2:0][CTA_TID_WIDTH-1:0] base, + input logic [2:0][CTA_TID_WIDTH-1:0] step, + input logic [CTA_TID_WIDTH:0] bdim_x, + input logic [CTA_TID_WIDTH:0] bdim_y + ); + logic [CTA_TID_WIDTH:0] nx, ny; + logic wx, wy; + nx = {1'b0, base[0]} + {1'b0, step[0]}; + wx = (nx >= bdim_x); + ny = ({1'b0, base[1]} + {1'b0, step[1]}) + (CTA_TID_WIDTH+1)'(wx); + wy = (ny >= bdim_y); + tid_advance[0] = wx ? CTA_TID_WIDTH'(nx - bdim_x) : CTA_TID_WIDTH'(nx); + tid_advance[1] = wy ? CTA_TID_WIDTH'(ny - bdim_y) : CTA_TID_WIDTH'(ny); + tid_advance[2] = base[2] + step[2] + CTA_TID_WIDTH'(wy); + endfunction + + // The window after the warps latched last cycle launch: entries shift down by + // nfired_r, and the ones shifted in are the entries they replace advanced by + // one group step. + wire [NUM_LANES-1:0][2:0][CTA_TID_WIDTH-1:0] thread_idx_grp; + wire [NUM_LANES-1:0][2:0][CTA_TID_WIDTH-1:0] thread_idx_adv; + for (genvar k = 0; k < NUM_LANES; ++k) begin : g_tid_window + assign thread_idx_grp[k] = tid_advance(thread_idx_r[k], group_step_r, block_dim_r[0], block_dim_r[1]); + reg [2:0][CTA_TID_WIDTH-1:0] shifted; + always @(*) begin + shifted = thread_idx_r[k]; + for (integer n = 1; n <= NUM_LANES; ++n) begin + if (int'(nfired_r) == n) begin + shifted = ((k + n) < NUM_LANES) ? thread_idx_r[(k + n) % NUM_LANES] + : thread_idx_grp[(k + n) % NUM_LANES]; + end + end + end + assign thread_idx_adv[k] = shifted; + end + + // Kernel constants of the window: the base thread indices of ranks 0..NUM_LANES-1 + // and the NUM_LANES-warp step, built by stepping warp_step_r once per cycle + // whenever a launch's block shape differs from the last one's. + reg [NUM_LANES-1:0][2:0][CTA_TID_WIDTH-1:0] thread_idx_init_r; + reg [2:0][CTA_TID_WIDTH-1:0] shape_step_r; + reg [2:0][CTA_TID_WIDTH:0] shape_dim_r; + reg [LCNT_WIDTH-1:0] init_cnt_r; + reg [2:0][CTA_TID_WIDTH-1:0] init_acc_r; + wire init_busy = (init_cnt_r != '0); // ------------------------------------------------------------------------- // Retirement decode — pipeline: @@ -411,12 +502,21 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( always_ff @(posedge clk) begin if (reset) begin state <= IDLE; - warp_fire_r <= 0; + warp_fire_r <= '0; warp_id_r <= '0; warp_tmask_r <= '0; + warp_order_r <= '0; + nfired_r <= '0; + lane_rr_r <= '0; + init_cnt_r <= '0; + shape_step_r <= '0; + shape_dim_r <= '0; + thread_idx_init_r <= '0; + group_step_r <= '0; + init_acc_r <= '0; cur_ctx_id_r <= '0; warp_init_lanes_r <= '0; - warp_skip_init_r <= 0; + warp_skip_init_r <= '0; tail_r <= '0; cur_lmem_base_r <= '0; slot_valid_r <= '0; @@ -447,8 +547,10 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( // ---- wid → cta-slot map: latch on warp dispatch ---------------- // (Used internally by the retirement path to recover the slot // index of a finishing warp; no longer exposed externally.) - if ((state == DISPATCH) && warp_ready) begin - cta_slot_per_warp_r[warp_id_n] <= cur_slot_r; + for (integer l = 0; l < NUM_LANES; ++l) begin + if (lane_sel[l]) begin + cta_slot_per_warp_r[lane_wid[l]] <= cur_slot_r; + end end // ---- Warp retirement ------------------------------------------- @@ -487,7 +589,18 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( warp_step_r <= kmu_req.args.compute.warp_step; cluster_size_r <= eff_cluster_size; cta_rank_r <= '0; - thread_idx_r <= '0; + thread_idx_r <= thread_idx_init_r; + group_step_r <= (NUM_LANES == 1) ? kmu_req.args.compute.warp_step : group_step_r; + // A new block shape rebuilds the window constants before launching. + if ((NUM_LANES > 1) && !kmu_is_frag + && ((kmu_req.args.compute.warp_step != shape_step_r) + || (kmu_req.args.compute.block_dim != shape_dim_r))) begin + shape_step_r <= kmu_req.args.compute.warp_step; + shape_dim_r <= kmu_req.args.compute.block_dim; + init_cnt_r <= LCNT_WIDTH'(1); + init_acc_r <= '0; + thread_idx_r <= '0; + end // Fixed-stride placement: this CTA lands in slot base_slot // (the round-robin tail, pre-wrapped for a cluster window) @@ -506,30 +619,44 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( end DISPATCH: begin - if (warp_ready) begin - warp_fire_r <= 1; - warp_id_r <= warp_id_n; - dispatched_warps[warp_id_n] <= 1'b1; - warp_tmask_r <= launch_tmask; - warp_skip_init_r <= ~slot_needs_init[warp_id_n]; - end else begin - warp_fire_r <= 0; + for (integer l = 0; l < NUM_LANES; ++l) begin + warp_fire_r[l] <= lane_sel[l]; + if (lane_sel[l]) begin + warp_id_r[l] <= lane_wid[l]; + warp_tmask_r[l] <= lane_tmask[l]; + warp_order_r[l] <= lane_order[l]; + warp_skip_init_r[l] <= ~slot_needs_init[lane_wid[l]]; + dispatched_warps[lane_wid[l]] <= 1'b1; + end end + nfired_r <= lane_cnt; + lane_rr_r <= LANE_WIDTH'((int'(lane_rr_r) + int'(lane_cnt)) % NUM_LANES); - if (warp_fire_r) begin - cta_rank_r <= cta_rank_r + NW_WIDTH'(1); - // Single shared adder result used for both decrement and last-warp test + if (fire_any_r) begin + cta_rank_r <= cta_rank_r + NW_WIDTH'(nfired_r); block_size_r <= block_size_next; + thread_idx_r <= thread_idx_adv; + for (integer l = 0; l < NUM_LANES; ++l) begin + if (warp_fire_r[l]) begin + warp_init_lanes_r[warp_id_r[l]] <= warp_init_lanes_r[warp_id_r[l]] | warp_tmask_r[l]; + end + end + if (rem_warps_cur == '0) begin + state <= IDLE; + end + end - warp_init_lanes_r[warp_id_r] <= warp_init_lanes_r[warp_id_r] | warp_tmask_r; - thread_idx_r[0] <= wrap_x ? CTA_TID_WIDTH'(next_x - block_dim_r[0]) : CTA_TID_WIDTH'(next_x); - thread_idx_r[1] <= wrap_y ? CTA_TID_WIDTH'(next_y - block_dim_r[1]) : CTA_TID_WIDTH'(next_y); - thread_idx_r[2] <= thread_idx_r[2] + warp_step_r[2] + CTA_TID_WIDTH'(wrap_y); - - if (is_last_warp) begin - warp_fire_r <= 0; - state <= IDLE; + // window constants: step warp_step_r once per cycle + if (init_busy) begin + if (int'(init_cnt_r) < NUM_LANES) begin + thread_idx_r[init_cnt_r] <= tid_advance(init_acc_r, warp_step_r, block_dim_r[0], block_dim_r[1]); + thread_idx_init_r[init_cnt_r] <= tid_advance(init_acc_r, warp_step_r, block_dim_r[0], block_dim_r[1]); + init_cnt_r <= init_cnt_r + LCNT_WIDTH'(1); + end else begin + group_step_r <= tid_advance(init_acc_r, warp_step_r, block_dim_r[0], block_dim_r[1]); + init_cnt_r <= '0; end + init_acc_r <= tid_advance(init_acc_r, warp_step_r, block_dim_r[0], block_dim_r[1]); end end default:; @@ -544,6 +671,8 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( // and cta_param. cta_csrs_t cta_csrs; + assign launch_en = (state == DISPATCH) && !init_busy; + assign cta_fire = warp_fire_r; assign cta_wid = warp_id_r; assign cta_PC = warp_PC; @@ -584,67 +713,21 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( // ------------------------------------------------------------------------- // CTA context storage + per-thread coordinate expansion // ------------------------------------------------------------------------- - // Per-CTA context (cta_ctx_ram) and per-warp residue (cta_warp_ram), written - // at launch and read back via csr_rd_wid / csr_rd_cta_id so CTA CSRs resolve - // in one cycle with no divider. BRAM-backed to avoid per-warp full-record flops. + // Per-CTA context and per-warp residue, broadcast at launch to the CTA-CSR + // stores, which resolve CTA CSRs in one cycle with no divider. - // Per-CTA context. + // Per-CTA context write port, shared by every lane. wire cta_ctx_write; wire [NCTA_WIDTH-1:0] cta_ctx_waddr; cta_ctx_t cta_ctx_wdata; - wire [NCTA_WIDTH-1:0] cta_ctx_raddr; - cta_ctx_t cta_ctx_rdata; - - VX_dp_ram #( - .DATAW ($bits(cta_ctx_t)), - .SIZE (NUM_CTA_MAX), - .RDW_MODE ("R"), - .OUT_REG (1) - ) cta_ctx_ram ( - .clk (clk), - .reset (reset), - .read (1'b1), - .write (cta_ctx_write), - .wren (1'b1), - .waddr (cta_ctx_waddr), - .wdata (cta_ctx_wdata), - .raddr (cta_ctx_raddr), - .rdata (cta_ctx_rdata) - ); - - // Per-warp residue. - wire cta_warp_write; - wire [NW_WIDTH-1:0] cta_warp_waddr; - cta_warp_t cta_warp_wdata; - wire [NW_WIDTH-1:0] cta_warp_raddr; - cta_warp_t cta_warp_rdata; - - VX_dp_ram #( - .DATAW ($bits(cta_warp_t)), - .SIZE (`VX_CFG_NUM_WARPS), - .RDW_MODE ("R"), - .OUT_REG (1) - ) cta_warp_ram ( - .clk (clk), - .reset (reset), - .read (1'b1), - .write (cta_warp_write), - .wren (1'b1), - .waddr (cta_warp_waddr), - .wdata (cta_warp_wdata), - .raddr (cta_warp_raddr), - .rdata (cta_warp_rdata) - ); // Per-lane CTA thread coordinates, expanded divide-free from the warp base // (lane 0 = base; each later lane steps +1 along X with a single wrap into Y // then Z) and stored in cta_warp_ram so CTA_THREAD_ID reads cost one cycle // with no divider. The expansion is a serial ripple, so it is pipelined - // TID_STEP lanes/cycle to meet timing. CTA dispatch is infrequent and + // TID_STEP lanes/cycle to meet timing; each launch lane has its own pipeline. // cta_warp_ram is read many cycles after a warp launches (fetch/decode/issue // latency >> TID_STAGES), so the added write latency is hidden. - wire [2:0][CTA_TID_WIDTH-1:0] cta_base_tid = thread_idx_r; - localparam TID_STEP = 2; localparam TID_STAGES = (`VX_CFG_NUM_THREADS - 1 + TID_STEP - 1) / TID_STEP; @@ -666,121 +749,139 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( typedef logic [`VX_CFG_NUM_THREADS-1:0][2:0][CTA_TID_WIDTH-1:0] cta_tid_arr_t; - cta_tid_arr_t tidp_tid [TID_STAGES+1]; - wire [TID_STAGES:0] tidp_valid; - wire [TID_STAGES:0][NW_WIDTH-1:0] tidp_wid; - wire [TID_STAGES:0][NW_WIDTH-1:0] tidp_rank; - // block_dim spans 1..NUM_WARPS*NUM_THREADS inclusive, one bit more than a - // thread index -- truncating it to CTA_TID_WIDTH turns a full-width dimension - // into 0 and makes every lane wrap. - wire [TID_STAGES:0][CTA_TID_WIDTH:0] tidp_bdx; - wire [TID_STAGES:0][CTA_TID_WIDTH:0] tidp_bdy; - - // stage 0: combinational inputs; seed lane 0 with the warp base. - cta_tid_arr_t tidp_tid0; - always @(*) begin - tidp_tid0 = '0; - tidp_tid0[0] = cta_base_tid; - end - assign tidp_tid[0] = tidp_tid0; - assign tidp_valid[0] = cta_fire; - assign tidp_wid[0] = cta_wid; - assign tidp_rank[0] = cta_csrs.cta_rank; - assign tidp_bdx[0] = cta_csrs.block_dim[0]; - assign tidp_bdy[0] = cta_csrs.block_dim[1]; - - for (genvar s = 1; s <= TID_STAGES; ++s) begin : g_tid_pipe - cta_tid_arr_t nxt; + wire [NUM_LANES-1:0] lane_frag_inflight; + + for (genvar l = 0; l < NUM_LANES; ++l) begin : g_lanes + cta_tid_arr_t tidp_tid [TID_STAGES+1]; + wire [TID_STAGES:0] tidp_valid; + wire [TID_STAGES:0][NW_WIDTH-1:0] tidp_wid; + wire [TID_STAGES:0][NW_WIDTH-1:0] tidp_rank; + // block_dim spans 1..NUM_WARPS*NUM_THREADS inclusive, one bit more than a + // thread index -- truncating it to CTA_TID_WIDTH turns a full-width dimension + // into 0 and makes every lane wrap. + wire [TID_STAGES:0][CTA_TID_WIDTH:0] tidp_bdx; + wire [TID_STAGES:0][CTA_TID_WIDTH:0] tidp_bdy; + + // stage 0: combinational inputs; seed lane 0 with the warp base. + cta_tid_arr_t tidp_tid0; always @(*) begin - nxt = tidp_tid[s-1]; - for (integer k = 0; k < TID_STEP; k = k + 1) begin - if (((s-1)*TID_STEP + k + 1) < `VX_CFG_NUM_THREADS) begin - nxt[(s-1)*TID_STEP + k + 1] = tid_next(nxt[(s-1)*TID_STEP + k], tidp_bdx[s-1], tidp_bdy[s-1]); + tidp_tid0 = '0; + tidp_tid0[0] = thread_idx_r[warp_order_r[l]]; + end + assign tidp_tid[0] = tidp_tid0; + assign tidp_valid[0] = warp_fire_r[l]; + assign tidp_wid[0] = warp_id_r[l]; + assign tidp_rank[0] = cta_csrs.cta_rank + NW_WIDTH'(warp_order_r[l]); + assign tidp_bdx[0] = cta_csrs.block_dim[0]; + assign tidp_bdy[0] = cta_csrs.block_dim[1]; + + for (genvar s = 1; s <= TID_STAGES; ++s) begin : g_tid_pipe + cta_tid_arr_t nxt; + always @(*) begin + nxt = tidp_tid[s-1]; + for (integer k = 0; k < TID_STEP; k = k + 1) begin + if (((s-1)*TID_STEP + k + 1) < `VX_CFG_NUM_THREADS) begin + nxt[(s-1)*TID_STEP + k + 1] = tid_next(nxt[(s-1)*TID_STEP + k], tidp_bdx[s-1], tidp_bdy[s-1]); + end end end - end - reg v_r; - reg [NW_WIDTH-1:0] wid_r, rank_r; - reg [CTA_TID_WIDTH:0] bdx_r, bdy_r; - cta_tid_arr_t tid_r; - always @(posedge clk) begin - if (reset) begin - v_r <= 1'b0; - end else begin - v_r <= tidp_valid[s-1]; + reg v_r; + reg [NW_WIDTH-1:0] wid_r, rank_r; + reg [CTA_TID_WIDTH:0] bdx_r, bdy_r; + cta_tid_arr_t tid_r; + always @(posedge clk) begin + if (reset) begin + v_r <= 1'b0; + end else begin + v_r <= tidp_valid[s-1]; + end + tid_r <= nxt; + wid_r <= tidp_wid[s-1]; + rank_r <= tidp_rank[s-1]; + bdx_r <= tidp_bdx[s-1]; + bdy_r <= tidp_bdy[s-1]; end - tid_r <= nxt; - wid_r <= tidp_wid[s-1]; - rank_r <= tidp_rank[s-1]; - bdx_r <= tidp_bdx[s-1]; - bdy_r <= tidp_bdy[s-1]; + assign tidp_tid[s] = tid_r; + assign tidp_valid[s] = v_r; + assign tidp_wid[s] = wid_r; + assign tidp_rank[s] = rank_r; + assign tidp_bdx[s] = bdx_r; + assign tidp_bdy[s] = bdy_r; end - assign tidp_tid[s] = tid_r; - assign tidp_valid[s] = v_r; - assign tidp_wid[s] = wid_r; - assign tidp_rank[s] = rank_r; - assign tidp_bdx[s] = bdx_r; - assign tidp_bdy[s] = bdy_r; - end -`ifdef VX_CFG_EXT_RASTER_ENABLE - // Which in-flight warps came from a fragment launch. One bit per stage rather - // than the stamps themselves: the write side needs to know WHICH view of the - // lane record to land, and the header side needs to know whether frag_stamps_r - // is still owed to someone. Shaped exactly like tidp_valid -- stage 0 - // combinational, the rest registered -- so the bit and the warp it describes - // stay in step. - wire [TID_STAGES:0] tidp_frag; - assign tidp_frag[0] = cta_fire && is_frag_r; - for (genvar s = 1; s <= TID_STAGES; ++s) begin : g_frag_pipe - reg f_r; - always @(posedge clk) begin - if (reset) begin - f_r <= 1'b0; - end else begin - f_r <= tidp_frag[s-1]; + wire is_frag_warp; + `ifdef VX_CFG_EXT_RASTER_ENABLE + // Which in-flight warps came from a fragment launch. One bit per stage rather + // than the stamps themselves: the write side needs to know WHICH view of the + // lane record to land, and the header side needs to know whether frag_stamps_r + // is still owed to someone. Shaped exactly like tidp_valid -- stage 0 + // combinational, the rest registered -- so the bit and the warp it describes + // stay in step. + wire [TID_STAGES:0] tidp_frag; + assign tidp_frag[0] = warp_fire_r[l] && is_frag_r; + for (genvar s = 1; s <= TID_STAGES; ++s) begin : g_frag_pipe + reg f_r; + always @(posedge clk) begin + if (reset) begin + f_r <= 1'b0; + end else begin + f_r <= tidp_frag[s-1]; + end end + assign tidp_frag[s] = f_r; end - assign tidp_frag[s] = f_r; + assign lane_frag_inflight[l] = |tidp_frag; + assign is_frag_warp = tidp_frag[TID_STAGES]; + `else + assign lane_frag_inflight[l] = 1'b0; + assign is_frag_warp = 1'b0; + // its only consumer (the lane-launch view mux) is RASTER-only + `UNUSED_VAR (is_frag_warp) + `endif + + // block_dim is consumed only by stages still rippling; the final stage's + // forwarded copy is unused. + `UNUSED_VAR (tidp_bdx[TID_STAGES]) + `UNUSED_VAR (tidp_bdy[TID_STAGES]) + + cta_warp_t cta_warp_wdata; + assign cta_warp_wdata.cta_rank = tidp_rank[TID_STAGES]; + + // The lane record is an overlay: a fragment launch lands its stamp slice, and a + // compute launch lands the expanded thread index. Which of the two applies is a + // property of the WARP being written, not of whatever launch the header stage + // happens to be holding now, so it rides the pipeline with it. + for (genvar i = 0; i < `VX_CFG_NUM_THREADS; ++i) begin : g_lane_launch + cta_lane_t lane_compute; + assign lane_compute = LANE_LAUNCH_BITS'(tidp_tid[TID_STAGES][i]); + `ifdef VX_CFG_EXT_RASTER_ENABLE + cta_lane_t lane_fragment; + assign lane_fragment = LANE_LAUNCH_BITS'(frag_stamps_sel[i]); + assign cta_warp_wdata.lane_launch[i] = is_frag_warp ? lane_fragment : lane_compute; + `else + assign cta_warp_wdata.lane_launch[i] = lane_compute; + `endif + end + + assign cta_lane_bcast[l].warp_write = tidp_valid[TID_STAGES]; + assign cta_lane_bcast[l].warp_waddr = tidp_wid[TID_STAGES]; + assign cta_lane_bcast[l].warp_wdata = cta_warp_wdata; + assign cta_lane_bcast[l].id_write = warp_fire_r[l]; + assign cta_lane_bcast[l].id_wid = warp_id_r[l]; + assign cta_lane_bcast[l].id_cta = cta_csrs.cta_id; end - assign frag_inflight = |tidp_frag; - assign is_frag_warp = tidp_frag[TID_STAGES]; + +`ifdef VX_CFG_EXT_RASTER_ENABLE + assign frag_inflight = |lane_frag_inflight; assign frag_stamps_sel = frag_stamps_r; `else assign frag_inflight = 1'b0; - assign is_frag_warp = 1'b0; assign frag_stamps_sel = '0; - // their only consumer (the lane-launch view mux) is RASTER-only - `UNUSED_VAR (is_frag_warp) + `UNUSED_VAR (lane_frag_inflight) `UNUSED_VAR (frag_stamps_sel) `endif - // block_dim is consumed only by stages still rippling; the final stage's - // forwarded copy is unused. - `UNUSED_VAR (tidp_bdx[TID_STAGES]) - `UNUSED_VAR (tidp_bdy[TID_STAGES]) - - assign cta_warp_write = tidp_valid[TID_STAGES]; - assign cta_warp_waddr = tidp_wid[TID_STAGES]; - assign cta_warp_wdata.cta_rank = tidp_rank[TID_STAGES]; - - // The lane record is an overlay: a fragment launch lands its stamp slice, and a - // compute launch lands the expanded thread index. Which of the two applies is a - // property of the WARP being written, not of whatever launch the header stage - // happens to be holding now, so it rides the pipeline with it. - for (genvar i = 0; i < `VX_CFG_NUM_THREADS; ++i) begin : g_lane_launch - cta_lane_t lane_compute; - assign lane_compute = LANE_LAUNCH_BITS'(tidp_tid[TID_STAGES][i]); - `ifdef VX_CFG_EXT_RASTER_ENABLE - cta_lane_t lane_fragment; - assign lane_fragment = LANE_LAUNCH_BITS'(frag_stamps_sel[i]); - assign cta_warp_wdata.lane_launch[i] = is_frag_warp ? lane_fragment : lane_compute; - `else - assign cta_warp_wdata.lane_launch[i] = lane_compute; - `endif - end - - assign cta_ctx_write = cta_fire; + assign cta_ctx_write = fire_any_r; assign cta_ctx_waddr = cta_csrs.cta_id; assign cta_ctx_wdata.cta_size = cta_csrs.cta_size; assign cta_ctx_wdata.block_idx = cta_csrs.block_idx; @@ -791,32 +892,9 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( assign cta_ctx_wdata.cluster_size = cta_csrs.cluster_size; assign cta_ctx_wdata.entry = cta_csrs.entry; - // rdata is valid one cycle after raddr; csr_unit holds its read address - // stable for that cycle. - assign cta_warp_raddr = csr_rd_wid; - assign cta_ctx_raddr = csr_rd_cta_id; - - assign cta_rd_csrs.cta_id = csr_rd_cta_id; - assign cta_rd_csrs.cta_rank = cta_warp_rdata.cta_rank; - - assign cta_rd_lane = cta_warp_rdata.lane_launch; - assign cta_rd_csrs.cta_size = cta_ctx_rdata.cta_size; - assign cta_rd_csrs.block_idx = cta_ctx_rdata.block_idx; - assign cta_rd_csrs.block_dim = cta_ctx_rdata.block_dim; - assign cta_rd_csrs.grid_dim = cta_ctx_rdata.grid_dim; - assign cta_rd_csrs.param = cta_ctx_rdata.param; - assign cta_rd_csrs.lmem_addr = cta_ctx_rdata.lmem_addr; - assign cta_rd_csrs.cluster_size = cta_ctx_rdata.cluster_size; - assign cta_rd_csrs.entry = cta_ctx_rdata.entry; - - // Per-warp -> CTA-id map, latched on launch and indexed by schedule_wid. - reg [`VX_CFG_NUM_WARPS-1:0][NCTA_WIDTH-1:0] cta_id_per_warp_r; - always @(posedge clk) begin - if (cta_fire) begin - cta_id_per_warp_r[cta_wid] <= cta_csrs.cta_id; - end - end - assign schedule_cta_id = cta_id_per_warp_r[schedule_wid]; + assign cta_bcast.ctx_write = cta_ctx_write; + assign cta_bcast.ctx_waddr = cta_ctx_waddr; + assign cta_bcast.ctx_wdata = cta_ctx_wdata; `ifdef DBG_TRACE_PIPELINE // Pipeline warp_done_wid alongside the retirement chain for trace logging. @@ -840,13 +918,15 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( kmu_req.param, stride, kmu_num_warps, usable_slots_r)) end - // Warp dispatched to scheduler - if (warp_fire_r) begin - `TRACE(1, ("%t: %s dispatch: wid=%0d, cta_id=%0d, PC=0x%0h, tmask=%b, param=0x%0h, lmem_addr=0x%0h, init=%b\n", - $time, INSTANCE_ID, warp_id_r, cur_slot_r, to_fullPC(warp_PC), - warp_tmask_r, param_r, - (`VX_CFG_MEM_ADDR_WIDTH'(`VX_MEM_LMEM_BASE_ADDR) | `VX_CFG_MEM_ADDR_WIDTH'(cur_lmem_base_r)), - ~warp_skip_init_r)) + // Warps dispatched to the schedulers + for (integer l = 0; l < NUM_LANES; ++l) begin + if (warp_fire_r[l]) begin + `TRACE(1, ("%t: %s dispatch: wid=%0d, cta_id=%0d, rank=%0d, PC=0x%0h, tmask=%b, param=0x%0h, lmem_addr=0x%0h, init=%b\n", + $time, INSTANCE_ID, warp_id_r[l], cur_slot_r, cta_rank_r + NW_WIDTH'(warp_order_r[l]), to_fullPC(warp_PC), + warp_tmask_r[l], param_r, + (`VX_CFG_MEM_ADDR_WIDTH'(`VX_MEM_LMEM_BASE_ADDR) | `VX_CFG_MEM_ADDR_WIDTH'(cur_lmem_base_r)), + ~warp_skip_init_r[l])) + end end // Warp retirement / CTA done if (warp_done_r_dly && slot_valid_r[done_slot_r_dly]) begin diff --git a/hw/rtl/core/VX_dcr_data.sv b/hw/rtl/core/VX_dcr_data.sv index 44f5624b82..c835d176a5 100644 --- a/hw/rtl/core/VX_dcr_data.sv +++ b/hw/rtl/core/VX_dcr_data.sv @@ -14,12 +14,13 @@ `include "VX_define.vh" module VX_dcr_data import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - parameter CORE_ID = 0 + parameter `STRING INSTANCE_ID = "" ) ( input wire clk, input wire reset, + input wire [CORE_ID_WIDTH-1:0] core_id, + // Inputs VX_dcr_bus_if.slave dcr_bus_if, @@ -50,7 +51,7 @@ module VX_dcr_data import VX_gpu_pkg::*; #( wire is_mpm_read = dcr_bus_if.req_valid && ~dcr_bus_if.req_data.rw && (dcr_bus_if.req_data.addr == `VX_DCR_BASE_MPM_VALUE) - && (mpm_target_cid == 16'(CORE_ID)) + && (mpm_target_cid == 16'(core_id)) && ~dcr_csr_pending_r; always @(posedge clk) begin @@ -89,7 +90,7 @@ module VX_dcr_data import VX_gpu_pkg::*; #( && (dcr_bus_if.req_data.addr == `VX_DCR_BASE_CACHE_FLUSH); wire is_flush_write = is_flush_req && dcr_bus_if.req_data.rw; wire is_flush_read = is_flush_req && ~dcr_bus_if.req_data.rw - && (mpm_target_cid == 16'(CORE_ID)); + && (mpm_target_cid == 16'(core_id)); wire flush_done = flush_pending_r && dcr_flush_if.done; wire flush_start = ~flush_pending_r && is_flush_write; diff --git a/hw/rtl/core/VX_decompressor.sv b/hw/rtl/core/VX_decompressor.sv index de3aea297e..229400d48e 100644 --- a/hw/rtl/core/VX_decompressor.sv +++ b/hw/rtl/core/VX_decompressor.sv @@ -66,7 +66,9 @@ // state[]/buf_pc[]/BRAM/follow_req/rsp_ready and sched_buffered_match all // gated on s1_can_accept. module VX_decompressor import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "" + parameter `STRING INSTANCE_ID = "", + parameter NUM_WARPS = `VX_CFG_NUM_WARPS, + parameter WID_W = `LOG2UP(NUM_WARPS) ) ( input wire clk, input wire reset, @@ -76,7 +78,7 @@ module VX_decompressor import VX_gpu_pkg::*; #( input wire [PC_BITS-1:0] sched_PC, input wire [`VX_CFG_NUM_THREADS-1:0] sched_tmask, input wire [NCTA_WIDTH-1:0] sched_cta_id, - input wire [NW_WIDTH-1:0] sched_wid, + input wire [WID_W-1:0] sched_wid, output wire sched_buffered_match, // Ungated buffered-hit: high whenever the scheduled PC is already in the // decompressor buffer, regardless of stage-0 stall. Used by VX_fetch to @@ -89,19 +91,19 @@ module VX_decompressor import VX_gpu_pkg::*; #( input wire [PC_BITS-1:0] rsp_PC, input wire [`VX_CFG_NUM_THREADS-1:0] rsp_tmask, input wire [NCTA_WIDTH-1:0] rsp_cta_id, - input wire [NW_WIDTH-1:0] rsp_wid, + input wire [WID_W-1:0] rsp_wid, input wire [UUID_WIDTH-1:0] rsp_uuid, output logic rsp_ready, // One-outstanding-per-warp tracking (from VX_fetch): high while a warp has // an icache request in flight. Used to gate the persistent follow arbiter. - input wire [`VX_CFG_NUM_WARPS-1:0] inflight, + input wire [NUM_WARPS-1:0] inflight, // Follow-up icache request (cross-word 32-bit). output logic follow_req_valid, output logic [PC_BITS-1:0] follow_req_PC, output logic [`VX_CFG_NUM_THREADS-1:0] follow_req_tmask, - output logic [NW_WIDTH-1:0] follow_req_wid, + output logic [WID_W-1:0] follow_req_wid, output logic [UUID_WIDTH-1:0] follow_req_uuid, // Decoded fetch interface. @@ -151,7 +153,7 @@ module VX_decompressor import VX_gpu_pkg::*; #( typedef struct packed { logic [31:0] payload32; logic [PC_BITS-1:0] PC; - logic [NW_WIDTH-1:0] wid; + logic [WID_W-1:0] wid; logic [UUID_WIDTH-1:0] uuid; logic [`VX_CFG_NUM_THREADS-1:0] tmask; logic [NCTA_WIDTH-1:0] cta_id; @@ -338,25 +340,25 @@ module VX_decompressor import VX_gpu_pkg::*; #( // State storage // ------------------------------------------------------------------ - buf_state_e state [`VX_CFG_NUM_WARPS]; - buf_state_e state_n [`VX_CFG_NUM_WARPS]; - logic [PC_BITS-1:0] buf_pc [`VX_CFG_NUM_WARPS]; - logic [PC_BITS-1:0] buf_pc_n [`VX_CFG_NUM_WARPS]; + buf_state_e state [NUM_WARPS]; + buf_state_e state_n [NUM_WARPS]; + logic [PC_BITS-1:0] buf_pc [NUM_WARPS]; + logic [PC_BITS-1:0] buf_pc_n [NUM_WARPS]; // Follow metadata captured on BUF_32HI entry (the persistent follow arbiter // re-issues the request from registered state, so it needs its own copy). - logic [UUID_WIDTH-1:0] hi_uuid [`VX_CFG_NUM_WARPS]; - logic [UUID_WIDTH-1:0] hi_uuid_n [`VX_CFG_NUM_WARPS]; - logic [`VX_CFG_NUM_THREADS-1:0] hi_tmask [`VX_CFG_NUM_WARPS]; - logic [`VX_CFG_NUM_THREADS-1:0] hi_tmask_n [`VX_CFG_NUM_WARPS]; + logic [UUID_WIDTH-1:0] hi_uuid [NUM_WARPS]; + logic [UUID_WIDTH-1:0] hi_uuid_n [NUM_WARPS]; + logic [`VX_CFG_NUM_THREADS-1:0] hi_tmask [NUM_WARPS]; + logic [`VX_CFG_NUM_THREADS-1:0] hi_tmask_n [NUM_WARPS]; // High half of the cross-word 32-bit, latched from the follow response and // held (BUF_32RDY) until the scheduler-gated emit combines it with the // BRAM-stashed low half. - logic [15:0] hi_hw [`VX_CFG_NUM_WARPS]; - logic [15:0] hi_hw_n [`VX_CFG_NUM_WARPS]; + logic [15:0] hi_hw [NUM_WARPS]; + logic [15:0] hi_hw_n [NUM_WARPS]; // drain[w]: the BUF_32HI straddle on warp w was squashed by a redirect // while its follow was in flight; discard the follow response (no emit). - logic [`VX_CFG_NUM_WARPS-1:0] drain, drain_n; + logic [NUM_WARPS-1:0] drain, drain_n; // spec[w]: the BUF_32HI straddle is speculative — it trails an RVC emitted // from the same word (have_D_emit_lo), which may be a taken branch, so the // warp is NOT yet committed to the straddle PC. Speculative straddles defer @@ -365,18 +367,18 @@ module VX_decompressor import VX_gpu_pkg::*; #( // (have_D_xword: the warp is already at the straddle PC and was acked by its // own fetch, so the scheduler will not re-present it) emit on the follow // response (have_B_resp). - logic [`VX_CFG_NUM_WARPS-1:0] spec, spec_n; + logic [NUM_WARPS-1:0] spec, spec_n; logic buf_we; - logic [NW_WIDTH-1:0] buf_waddr; + logic [WID_W-1:0] buf_waddr; buf_data_t buf_wdata; - logic [NW_WIDTH-1:0] buf_raddr; + logic [WID_W-1:0] buf_raddr; buf_data_t buf_rdata; // registered (OUT_REG=1) logic buf_read; // gates rdata_r update during stall VX_dp_ram #( .DATAW ($bits(buf_data_t)), - .SIZE (`VX_CFG_NUM_WARPS), + .SIZE (NUM_WARPS), .OUT_REG (1), .LUTRAM (1), .RDW_MODE("R") @@ -530,7 +532,7 @@ module VX_decompressor import VX_gpu_pkg::*; #( buf_we = 1'b0; buf_waddr = rsp_wid; buf_wdata = '0; - for (int w = 0; w < `VX_CFG_NUM_WARPS; ++w) begin + for (int w = 0; w < NUM_WARPS; ++w) begin state_n[w] = state[w]; buf_pc_n[w] = buf_pc[w]; hi_uuid_n[w] = hi_uuid[w]; @@ -552,8 +554,8 @@ module VX_decompressor import VX_gpu_pkg::*; #( // BUF_RVC : no outstanding fetch -> clear directly. // (buf_pc == sched_PC is normal straddler continuation after an // RVC-then-straddler word — must NOT drop.) - for (int w = 0; w < `VX_CFG_NUM_WARPS; ++w) begin - if (sched_valid && (sched_wid == w[NW_WIDTH-1:0]) + for (int w = 0; w < NUM_WARPS; ++w) begin + if (sched_valid && (sched_wid == WID_W'(w)) && (buf_pc[w] != sched_PC)) begin if (state[w] == BUF_32HI) begin drain_n[w] = 1'b1; @@ -684,14 +686,14 @@ module VX_decompressor import VX_gpu_pkg::*; #( // alias. Lowest-wid priority (reverse scan). // ------------------------------------------------------------------ logic follow_sel_valid; - logic [NW_WIDTH-1:0] follow_sel_wid; + logic [WID_W-1:0] follow_sel_wid; always_comb begin follow_sel_valid = 1'b0; follow_sel_wid = '0; - for (int w = `VX_CFG_NUM_WARPS-1; w >= 0; --w) begin - if ((state[w] == BUF_32HI) && ~inflight[w[NW_WIDTH-1:0]]) begin + for (int w = NUM_WARPS-1; w >= 0; --w) begin + if ((state[w] == BUF_32HI) && ~inflight[w]) begin follow_sel_valid = 1'b1; - follow_sel_wid = w[NW_WIDTH-1:0]; + follow_sel_wid = WID_W'(w); end end end @@ -716,7 +718,7 @@ module VX_decompressor import VX_gpu_pkg::*; #( always_comb begin fsm_data = '0; fsm_data.PC = s1_ctx.PC; - fsm_data.wid = s1_ctx.wid; + fsm_data.wid = NW_WIDTH'(s1_ctx.wid); // cta_id: response-gated straddle (S1_BR) carries the BRAM-stashed CTA; // all other kinds captured the CTA into ctx (sched or rsp) in stage 0. fsm_data.cta_id = (s1_kind == S1_BR) ? buf_rdata.cta_id : s1_ctx.cta_id; @@ -761,7 +763,7 @@ module VX_decompressor import VX_gpu_pkg::*; #( always_comb begin fast_data = '0; fast_data.uuid = rsp_uuid; - fast_data.wid = rsp_wid; + fast_data.wid = NW_WIDTH'(rsp_wid); fast_data.tmask = rsp_tmask; fast_data.cta_id = rsp_cta_id; fast_data.PC = rsp_PC; @@ -782,7 +784,7 @@ module VX_decompressor import VX_gpu_pkg::*; #( always_ff @(posedge clk) begin if (reset) begin - for (int w = 0; w < `VX_CFG_NUM_WARPS; ++w) begin + for (int w = 0; w < NUM_WARPS; ++w) begin state[w] <= BUF_EMPTY; buf_pc[w] <= '0; hi_uuid[w] <= '0; @@ -794,7 +796,7 @@ module VX_decompressor import VX_gpu_pkg::*; #( s1_kind <= S1_NONE; s1_ctx <= '0; end else begin - for (int w = 0; w < `VX_CFG_NUM_WARPS; ++w) begin + for (int w = 0; w < NUM_WARPS; ++w) begin state[w] <= state_n[w]; buf_pc[w] <= buf_pc_n[w]; hi_uuid[w] <= hi_uuid_n[w]; diff --git a/hw/rtl/core/VX_dispatcher.sv b/hw/rtl/core/VX_dispatcher.sv index d0c3a81d62..9c17114d77 100644 --- a/hw/rtl/core/VX_dispatcher.sv +++ b/hw/rtl/core/VX_dispatcher.sv @@ -14,12 +14,13 @@ `include "VX_define.vh" module VX_dispatcher import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - parameter ISSUE_ID = 0 + parameter `STRING INSTANCE_ID = "" ) ( input wire clk, input wire reset, + input wire [ISSUE_ISW_W-1:0] slice_id, + `ifdef PERF_ENABLE output wire [NUM_EX_UNITS-1:0][PERF_CTR_BITS-1:0] perf_stalls, output wire [NUM_EX_UNITS-1:0][PERF_CTR_BITS-1:0] perf_instrs, @@ -32,7 +33,9 @@ module VX_dispatcher import VX_gpu_pkg::*; #( VX_dispatch_if.master dispatch_if [NUM_EX_UNITS] ); `UNUSED_SPARAM (INSTANCE_ID) - `UNUSED_PARAM (ISSUE_ID) +`ifndef DBG_TRACE_PIPELINE + `UNUSED_VAR (slice_id) +`endif localparam OUT_DATAW = $bits(dispatch_t); @@ -45,7 +48,7 @@ module VX_dispatcher import VX_gpu_pkg::*; #( end // All execution units pass operand data straight through; LSU address - // generation (pack-load included) lives in VX_lsu_slice. + // generation (pack-load included) lives in VX_lsu_block. for (genvar i = 0; i < NUM_EX_UNITS; ++i) begin : g_buffers VX_elastic_buffer #( .DATAW (OUT_DATAW), @@ -107,7 +110,7 @@ module VX_dispatcher import VX_gpu_pkg::*; #( for (genvar ex = 0; ex < NUM_EX_UNITS; ++ex) begin : g_dispatch_trace always @(posedge clk) begin if (dispatch_if[ex].valid && dispatch_if[ex].ready) begin - `TRACE(1, ("%t: %s dispatch: wid=%0d, cta_id=%0d, sid=%0d, PC=0x%0h, ex=", $time, INSTANCE_ID, wis_to_wid(dispatch_if[ex].data.wis, ISSUE_ID), dispatch_if[ex].data.cta_id, dispatch_if[ex].data.sid, to_fullPC(dispatch_if[ex].data.PC))) + `TRACE(1, ("%t: %s dispatch: wid=%0d, cta_id=%0d, sid=%0d, PC=0x%0h, ex=", $time, INSTANCE_ID, wis_to_wid(dispatch_if[ex].data.wis, slice_id), dispatch_if[ex].data.cta_id, dispatch_if[ex].data.sid, to_fullPC(dispatch_if[ex].data.PC))) VX_trace_pkg::trace_ex_type(1, ex); `TRACE(1, (", op=")) VX_trace_pkg::trace_ex_op(1, ex, dispatch_if[ex].data.op_type, dispatch_if[ex].data.op_args); diff --git a/hw/rtl/core/VX_execute.sv b/hw/rtl/core/VX_execute.sv index 4f0222e066..b1601ff6da 100644 --- a/hw/rtl/core/VX_execute.sv +++ b/hw/rtl/core/VX_execute.sv @@ -14,17 +14,16 @@ `include "VX_define.vh" module VX_execute import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - parameter CORE_ID = 0 + parameter `STRING INSTANCE_ID = "" ) ( `SCOPE_IO_DECL input wire clk, input wire reset, + input wire [CORE_ID_WIDTH-1:0] core_id, + `ifdef PERF_ENABLE - input sysmem_perf_t sysmem_perf, - input pipeline_perf_t pipeline_perf, `ifdef VX_CFG_EXT_TCU_ENABLE output tcu_perf_t tcu_perf, `endif @@ -49,7 +48,6 @@ module VX_execute import VX_gpu_pkg::*; #( `ifdef VX_CFG_EXT_DXA_ENABLE VX_dxa_req_bus_if.master dxa_req_bus_if, - VX_txbar_bus_if.slave dxa_txbar_bus_if, `endif `ifdef VX_CFG_EXT_TEX_ENABLE @@ -65,21 +63,19 @@ module VX_execute import VX_gpu_pkg::*; #( `ifdef VX_CFG_EXT_RTU_ENABLE VX_rtu_bus_if.master rtu_bus_if, - VX_sched_unlock_if.master sched_unlock_if, + VX_sched_unlock_if.master sched_unlock_if [`VX_CFG_NUM_SFU_BLOCKS], `endif - // scheduler interfaces - VX_sched_csr_if.slave sched_csr_if, - VX_branch_ctl_if.master branch_ctl_if [`VX_CFG_NUM_ALU_BLOCKS], - VX_warp_ctl_if.master warp_ctl_if, - `ifdef VX_CFG_TCU_WGMMA_ENABLE // TCU tile-buffer local-memory read port VX_mem_bus_if.master tcu_lmem_if, `endif - // DCR-CSR interface - VX_dcr_csr_if dcr_csr_if + // scheduler interfaces + VX_sched_csr_if.slave sched_csr_if [`VX_CFG_NUM_SFU_BLOCKS], + VX_branch_ctl_if.master branch_ctl_if [`VX_CFG_NUM_ALU_BLOCKS], + VX_warp_shared_if.master warp_shared_if [`VX_CFG_NUM_SFU_BLOCKS], + VX_warp_ctl_if.master warp_ctl_if [`VX_CFG_NUM_SFU_BLOCKS] ); `ifdef VX_CFG_EXT_F_ENABLE @@ -99,12 +95,12 @@ module VX_execute import VX_gpu_pkg::*; #( `SCOPE_IO_SWITCH (1); VX_lsu_unit #( - .INSTANCE_ID (`SFORMATF(("%s-lsu", INSTANCE_ID))), - .CORE_ID (CORE_ID) + .INSTANCE_ID (`SFORMATF(("%s-lsu", INSTANCE_ID))) ) lsu_unit ( `SCOPE_IO_BIND (0) .clk (clk), .reset (reset), + .core_id (core_id), .dispatch_if (dispatch_if[EX_LSU * `VX_CFG_ISSUE_WIDTH +: `VX_CFG_ISSUE_WIDTH]), .commit_if (commit_if[EX_LSU * `VX_CFG_ISSUE_WIDTH +: `VX_CFG_ISSUE_WIDTH]), .per_block_client_if (lsu_client_if) @@ -143,15 +139,11 @@ module VX_execute import VX_gpu_pkg::*; #( `endif VX_sfu_unit #( - .INSTANCE_ID (`SFORMATF(("%s-sfu", INSTANCE_ID))), - .CORE_ID (CORE_ID) + .INSTANCE_ID (`SFORMATF(("%s-sfu", INSTANCE_ID))) ) sfu_unit ( .clk (clk), .reset (reset), - `ifdef PERF_ENABLE - .sysmem_perf (sysmem_perf), - .pipeline_perf (pipeline_perf), - `endif + .core_id (core_id), .dispatch_if (dispatch_if[EX_SFU * `VX_CFG_ISSUE_WIDTH +: `VX_CFG_ISSUE_WIDTH]), .commit_if (commit_if[EX_SFU * `VX_CFG_ISSUE_WIDTH +: `VX_CFG_ISSUE_WIDTH]), `ifdef VX_CFG_EXT_F_ENABLE @@ -159,22 +151,17 @@ module VX_execute import VX_gpu_pkg::*; #( `endif `ifdef VX_CFG_EXT_DXA_ENABLE .dxa_req_bus_if (dxa_req_bus_if), - .dxa_txbar_bus_if(dxa_txbar_bus_if), `endif `ifdef VX_CFG_EXT_TEX_ENABLE .tex_bus_if (tex_bus_if), `endif - `ifdef VX_CFG_EXT_OM_ENABLE - `endif - `ifdef VX_CFG_EXT_RASTER_ENABLE - `endif `ifdef VX_CFG_EXT_RTU_ENABLE .rtu_bus_if (rtu_bus_if), .sched_unlock_if (sched_unlock_if), `endif .sched_csr_if (sched_csr_if), - .warp_ctl_if (warp_ctl_if), - .dcr_csr_if (dcr_csr_if) + .warp_shared_if (warp_shared_if), + .warp_ctl_if (warp_ctl_if) ); endmodule diff --git a/hw/rtl/core/VX_execute_slice.sv b/hw/rtl/core/VX_execute_slice.sv new file mode 100644 index 0000000000..541c84efe7 --- /dev/null +++ b/hw/rtl/core/VX_execute_slice.sv @@ -0,0 +1,401 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +`ifdef VX_CFG_EXT_F_ENABLE +`include "VX_fpu_define.vh" +`endif + +// The execution units of one issue slice: one ALU, LSU, SFU, FPU and TCU block +// each, and the FP CSRs of the slice's warps. +module VX_execute_slice import VX_gpu_pkg::*; +`ifdef VX_CFG_EXT_F_ENABLE +import VX_fpu_pkg::*; +`endif +`ifdef VX_CFG_EXT_TCU_ENABLE +import VX_tcu_pkg::*; +`endif +#( + parameter `STRING INSTANCE_ID = "", + parameter `STRING SLICE_STR = "" +) ( + `SCOPE_IO_DECL + + input wire clk, + input wire reset, + + input wire [CORE_ID_WIDTH-1:0] core_id, + input wire [ISSUE_ISW_W-1:0] slice_id, + + VX_dispatch_if.slave dispatch_if [NUM_EX_UNITS], + VX_commit_if.master commit_if [NUM_EX_UNITS], + + // ALU + VX_branch_ctl_if.master branch_ctl_if, + + // LSU + VX_lsu_sched_if.master lsu_client_if, + + // SFU +`ifdef VX_CFG_EXT_DXA_ENABLE + VX_dxa_req_bus_if.master dxa_req_bus_if, +`endif +`ifdef VX_CFG_EXT_TEX_ENABLE + VX_tex_bus_if.master tex_bus_if, +`endif +`ifdef VX_CFG_EXT_RTU_ENABLE + VX_rtu_bus_if.master rtu_bus_if, + VX_sched_unlock_if.master sched_unlock_if, +`endif + + // TCU +`ifdef VX_CFG_TCU_WGMMA_ENABLE + output wire tcu_exec_valid, + output wire tcu_exec_ready, + output tcu_execute_t tcu_exec_data, + input wire [TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs1_data, + input wire [TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs2_data, + input wire tcu_tbuf_ready, +`endif +`ifdef TCU_META_ENABLE + output wire tcu_agu_ld_valid, + output tcu_execute_t tcu_agu_ld_data, + input wire tcu_agu_ld_ready, + input wire tcu_agu_result_valid, + input tcu_result_t tcu_agu_result_data, + output wire tcu_agu_result_ready, + input wire tcu_meta_wr_en, + input wire [NW_WIDTH-1:0] tcu_meta_wr_wid, + input wire [4:0] tcu_meta_wr_idx, + input wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] tcu_meta_wr_data, +`endif + + // SFU warp control and CSRs + VX_warp_shared_if.master warp_shared_if, + VX_warp_ctl_if.master warp_ctl_if, + VX_sched_csr_if.slave sched_csr_if +); + localparam ALU_PARTIAL_BW = (`VX_CFG_NUM_ALU_LANES != `VX_CFG_SIMD_WIDTH); +`ifdef VX_CFG_EXT_F_ENABLE + localparam FPU_PARTIAL_BW = (`VX_CFG_NUM_FPU_LANES != `VX_CFG_SIMD_WIDTH); +`endif + +`ifdef SCOPE + localparam SCOPE_COUNT = 1 + `VX_CFG_EXT_TCU_ENABLED; +`endif + `SCOPE_IO_SWITCH (SCOPE_COUNT); + + // ALU + + VX_execute_if #( + .data_t (alu_execute_t) + ) alu_execute_if[1](); + + VX_result_if #( + .data_t (alu_result_t) + ) alu_result_if[1](); + + VX_lane_dispatch #( + .BLOCK_SIZE (1), + .NUM_LANES (`VX_CFG_NUM_ALU_LANES), + .OUT_BUF (ALU_PARTIAL_BW ? 3 : 0), + .NUM_PORTS (1) + ) alu_dispatch ( + .clk (clk), + .reset (reset), + .port_base (slice_id), + .dispatch_if(dispatch_if[EX_ALU +: 1]), + .execute_if (alu_execute_if) + ); + + VX_alu_block #( + .INSTANCE_ID (`SFORMATF(("%s-alu", INSTANCE_ID))), + .BLOCK_STR (SLICE_STR), + .NUM_LANES (`VX_CFG_NUM_ALU_LANES), + .PARTIAL_BW (ALU_PARTIAL_BW) + ) alu_block ( + .clk (clk), + .reset (reset), + .block_idx (slice_id), + .execute_if (alu_execute_if[0]), + .result_if (alu_result_if[0]), + .branch_ctl_if (branch_ctl_if) + ); + + VX_lane_gather #( + .BLOCK_SIZE (1), + .NUM_LANES (`VX_CFG_NUM_ALU_LANES), + .OUT_BUF (ALU_PARTIAL_BW ? 3 : 0), + .NUM_PORTS (1) + ) alu_gather ( + .clk (clk), + .reset (reset), + .result_if (alu_result_if), + .commit_if (commit_if[EX_ALU +: 1]) + ); + + // LSU + + VX_execute_if #( + .data_t (lsu_execute_t) + ) lsu_execute_if[1](); + + VX_result_if #( + .data_t (lsu_result_t) + ) lsu_result_if[1](); + + VX_lane_dispatch #( + .BLOCK_SIZE (1), + .NUM_LANES (`VX_CFG_NUM_LSU_LANES), + .OUT_BUF (3), + .NUM_PORTS (1) + ) lsu_dispatch ( + .clk (clk), + .reset (reset), + .port_base (slice_id), + .dispatch_if(dispatch_if[EX_LSU +: 1]), + .execute_if (lsu_execute_if) + ); + + VX_lsu_block #( + .INSTANCE_ID (`SFORMATF(("%s-lsu%s", INSTANCE_ID, SLICE_STR))) + ) lsu_block ( + `SCOPE_IO_BIND (0) + .clk (clk), + .reset (reset), + .core_id (core_id), + .execute_if (lsu_execute_if[0]), + .result_if (lsu_result_if[0]), + .client_if (lsu_client_if) + ); + + VX_lane_gather #( + .BLOCK_SIZE (1), + .NUM_LANES (`VX_CFG_NUM_LSU_LANES), + .OUT_BUF (3), + .NUM_PORTS (1) + ) lsu_gather ( + .clk (clk), + .reset (reset), + .result_if (lsu_result_if), + .commit_if (commit_if[EX_LSU +: 1]) + ); + + // SFU + + VX_execute_if #( + .data_t (sfu_execute_t) + ) sfu_execute_if[1](); + + VX_result_if #( + .data_t (sfu_result_t) + ) sfu_result_if[1](); + + VX_lane_dispatch #( + .BLOCK_SIZE (1), + .NUM_LANES (`VX_CFG_NUM_SFU_LANES), + .OUT_BUF (3), + .NUM_PORTS (1) + ) sfu_dispatch ( + .clk (clk), + .reset (reset), + .port_base (slice_id), + .dispatch_if(dispatch_if[EX_SFU +: 1]), + .execute_if (sfu_execute_if) + ); + +`ifdef VX_CFG_EXT_F_ENABLE + wire [NW_WIDTH-1:0] fcsr_wid; + wire [INST_FRM_BITS+`FP_FLAGS_BITS-1:0] fcsr_rd_data; + wire [2:0] fcsr_wr_sel; + wire [INST_FRM_BITS+`FP_FLAGS_BITS-1:0] fcsr_wr_data; +`endif + + VX_sfu_block #( + .INSTANCE_ID ((`VX_CFG_NUM_SFU_BLOCKS > 1) ? `SFORMATF(("%s-sfu%s", INSTANCE_ID, SLICE_STR)) : `SFORMATF(("%s-sfu", INSTANCE_ID))) + ) sfu_block ( + .clk (clk), + .reset (reset), + .core_id (core_id), + .block_idx (SFU_BLOCK_W'(slice_id)), + .execute_if (sfu_execute_if[0]), + .result_if (sfu_result_if[0]), + `ifdef VX_CFG_EXT_F_ENABLE + .fcsr_wid (fcsr_wid), + .fcsr_rd_data (fcsr_rd_data), + .fcsr_wr_sel (fcsr_wr_sel), + .fcsr_wr_data (fcsr_wr_data), + `endif + `ifdef VX_CFG_EXT_DXA_ENABLE + .dxa_req_bus_if (dxa_req_bus_if), + `endif + `ifdef VX_CFG_EXT_TEX_ENABLE + .tex_bus_if (tex_bus_if), + `endif + `ifdef VX_CFG_EXT_RTU_ENABLE + .rtu_bus_if (rtu_bus_if), + .sched_unlock_if(sched_unlock_if), + `endif + .sched_csr_if (sched_csr_if), + .warp_shared_if (warp_shared_if), + .warp_ctl_if (warp_ctl_if) + ); + + VX_lane_gather #( + .BLOCK_SIZE (1), + .NUM_LANES (`VX_CFG_NUM_SFU_LANES), + .OUT_BUF (3), + .NUM_PORTS (1) + ) sfu_gather ( + .clk (clk), + .reset (reset), + .result_if (sfu_result_if), + .commit_if (commit_if[EX_SFU +: 1]) + ); + + // FPU + +`ifdef VX_CFG_EXT_F_ENABLE + VX_fpu_csr_if fpu_csr_if[1](); + + VX_execute_if #( + .data_t (fpu_execute_t) + ) fpu_execute_if[1](); + + VX_result_if #( + .data_t (fpu_result_t) + ) fpu_result_if[1](); + + VX_lane_dispatch #( + .BLOCK_SIZE (1), + .NUM_LANES (`VX_CFG_NUM_FPU_LANES), + .OUT_BUF (FPU_PARTIAL_BW ? 3 : 0), + .NUM_PORTS (1) + ) fpu_dispatch ( + .clk (clk), + .reset (reset), + .port_base (slice_id), + .dispatch_if(dispatch_if[EX_FPU +: 1]), + .execute_if (fpu_execute_if) + ); + + VX_fpu_block #( + .INSTANCE_ID (`SFORMATF(("%s-fpu", INSTANCE_ID))), + .NUM_LANES (`VX_CFG_NUM_FPU_LANES), + .PARTIAL_BW (FPU_PARTIAL_BW) + ) fpu_block ( + .clk (clk), + .reset (reset), + .block_idx (slice_id), + .execute_if (fpu_execute_if[0]), + .result_if (fpu_result_if[0]), + .fpu_csr_if (fpu_csr_if[0]) + ); + + VX_lane_gather #( + .BLOCK_SIZE (1), + .NUM_LANES (`VX_CFG_NUM_FPU_LANES), + .OUT_BUF (FPU_PARTIAL_BW ? 3 : 0), + .NUM_PORTS (1) + ) fpu_gather ( + .clk (clk), + .reset (reset), + .result_if (fpu_result_if), + .commit_if (commit_if[EX_FPU +: 1]) + ); + + VX_csr_fpu #( + .WID_STRIDE (`VX_CFG_ISSUE_WIDTH), + .NUM_CSR_PORTS (1), + .NUM_FPU_PORTS (1) + ) csr_fpu ( + .clk (clk), + .reset (reset), + .csr_wid (fcsr_wid), + .csr_rd_data (fcsr_rd_data), + .csr_wr_sel (fcsr_wr_sel), + .csr_wr_data (fcsr_wr_data), + .fpu_csr_if (fpu_csr_if) + ); +`endif + + // TCU + +`ifdef VX_CFG_EXT_TCU_ENABLE + VX_execute_if #( + .data_t (tcu_execute_t) + ) tcu_execute_if[1](); + + VX_result_if #( + .data_t (tcu_result_t) + ) tcu_result_if[1](); + + VX_lane_dispatch #( + .BLOCK_SIZE (1), + .NUM_LANES (`VX_CFG_NUM_TCU_LANES), + .OUT_BUF (3), + .NUM_PORTS (1) + ) tcu_dispatch ( + .clk (clk), + .reset (reset), + .port_base (slice_id), + .dispatch_if(dispatch_if[EX_TCU +: 1]), + .execute_if (tcu_execute_if) + ); + + VX_tcu_block #( + .INSTANCE_ID (`SFORMATF(("%s-tcu", INSTANCE_ID))), + .BLOCK_STR (SLICE_STR) + ) tcu_block ( + `SCOPE_IO_BIND (1) + .clk (clk), + .reset (reset), + `ifdef VX_CFG_TCU_WGMMA_ENABLE + .exec_valid (tcu_exec_valid), + .exec_ready (tcu_exec_ready), + .exec_data (tcu_exec_data), + .tbuf_rs1_data (tcu_tbuf_rs1_data), + .tbuf_rs2_data (tcu_tbuf_rs2_data), + .tbuf_ready (tcu_tbuf_ready), + `endif + `ifdef TCU_META_ENABLE + .agu_ld_valid (tcu_agu_ld_valid), + .agu_ld_data (tcu_agu_ld_data), + .agu_ld_ready (tcu_agu_ld_ready), + .agu_result_valid (tcu_agu_result_valid), + .agu_result_data (tcu_agu_result_data), + .agu_result_ready (tcu_agu_result_ready), + .meta_wr_en (tcu_meta_wr_en), + .meta_wr_wid (tcu_meta_wr_wid), + .meta_wr_idx (tcu_meta_wr_idx), + .meta_wr_data (tcu_meta_wr_data), + `endif + .execute_if (tcu_execute_if[0]), + .result_if (tcu_result_if[0]) + ); + + VX_lane_gather #( + .BLOCK_SIZE (1), + .NUM_LANES (`VX_CFG_NUM_TCU_LANES), + .OUT_BUF (3), + .NUM_PORTS (1) + ) tcu_gather ( + .clk (clk), + .reset (reset), + .result_if (tcu_result_if), + .commit_if (commit_if[EX_TCU +: 1]) + ); +`endif + +endmodule diff --git a/hw/rtl/core/VX_fetch.sv b/hw/rtl/core/VX_fetch.sv index 72cc0b6754..a4f8847e7f 100644 --- a/hw/rtl/core/VX_fetch.sv +++ b/hw/rtl/core/VX_fetch.sv @@ -13,8 +13,11 @@ `include "VX_define.vh" +// NUM_WARPS is the number of warps this unit fetches for; their ids arrive in +// the low bits of schedule_if.data.wid and leave the same way on fetch_if. module VX_fetch import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "" + parameter `STRING INSTANCE_ID = "", + parameter NUM_WARPS = `VX_CFG_NUM_WARPS ) ( `SCOPE_IO_DECL @@ -37,6 +40,14 @@ module VX_fetch import VX_gpu_pkg::*; #( `UNUSED_SPARAM (INSTANCE_ID) `UNUSED_VAR (reset) + localparam WID_W = `LOG2UP(NUM_WARPS); + `STATIC_ASSERT (WID_W == ICACHE_TAG_ID_BITS, ("invalid parameter")) + + wire [WID_W-1:0] sched_wid = schedule_if.data.wid[WID_W-1:0]; + if (NW_WIDTH > WID_W) begin : g_sched_wid_unused + `UNUSED_VAR (schedule_if.data.wid[NW_WIDTH-1:WID_W]) + end + wire icache_req_valid; wire icache_req_ready; wire [ICACHE_ADDR_WIDTH-1:0] icache_req_addr; @@ -44,25 +55,25 @@ module VX_fetch import VX_gpu_pkg::*; #( // tag bits VX_mem_bus_arb adds downstream live in ICACHE_TAG_WIDTH but // are not driven by this stage. wire [ICACHE_FETCH_TAG_WIDTH-1:0] icache_req_tag; - wire [NW_WIDTH-1:0] icache_req_wid; + wire [WID_W-1:0] icache_req_wid; wire [UUID_WIDTH-1:0] icache_req_uuid; wire [UUID_WIDTH-1:0] rsp_uuid; wire [PC_BITS-1:0] rsp_PC; wire [`VX_CFG_NUM_THREADS-1:0] rsp_tmask; - wire [NW_WIDTH-1:0] req_tag; - wire [NW_WIDTH-1:0] rsp_tag; + wire [WID_W-1:0] req_tag; + wire [WID_W-1:0] rsp_tag; wire [NCTA_WIDTH-1:0] rsp_cta_id; wire icache_req_fire = icache_req_valid && icache_req_ready; - assign req_tag = schedule_if.data.wid; + assign req_tag = sched_wid; assign {rsp_uuid, rsp_tag} = icache_bus_if.rsp_data.tag; VX_dp_ram #( .DATAW (PC_BITS + `VX_CFG_NUM_THREADS + NCTA_WIDTH), - .SIZE (`VX_CFG_NUM_WARPS), + .SIZE (NUM_WARPS), .RDW_MODE ("R"), .LUTRAM (1) ) tag_store ( @@ -87,12 +98,12 @@ module VX_fetch import VX_gpu_pkg::*; #( wire [PC_BITS-1:0] icache_req_PC; wire [`VX_CFG_NUM_THREADS-1:0] icache_req_tmask; - wire [NW_WIDTH-1:0] rsp_wid; + wire [WID_W-1:0] rsp_wid; wire follow_req_valid; wire [PC_BITS-1:0] follow_req_PC; wire [`VX_CFG_NUM_THREADS-1:0] follow_req_tmask; - wire [NW_WIDTH-1:0] follow_req_wid; + wire [WID_W-1:0] follow_req_wid; wire [UUID_WIDTH-1:0] follow_req_uuid; wire sched_buffered_match; @@ -102,23 +113,23 @@ module VX_fetch import VX_gpu_pkg::*; #( // One-outstanding-per-warp (Invariant B): a warp with an icache request in // flight must not issue another, else its two responses alias the single // tag_store slot. Set on request fire, cleared on response fire. - reg [`VX_CFG_NUM_WARPS-1:0] inflight; + reg [NUM_WARPS-1:0] inflight; wire icache_rsp_fire = icache_bus_if.rsp_valid && icache_bus_if.rsp_ready; always @(posedge clk) begin if (reset) begin inflight <= '0; end else begin - for (int w = 0; w < `VX_CFG_NUM_WARPS; ++w) begin - if (icache_req_fire && (icache_req_wid == w[NW_WIDTH-1:0])) + for (int w = 0; w < NUM_WARPS; ++w) begin + if (icache_req_fire && (icache_req_wid == WID_W'(w))) inflight[w] <= 1'b1; - else if (icache_rsp_fire && (rsp_wid == w[NW_WIDTH-1:0])) + else if (icache_rsp_fire && (rsp_wid == WID_W'(w))) inflight[w] <= 1'b0; end end end `UNUSED_VAR (icache_req_tmask) - // ibuffer occupancy is already gated by VX_scheduler (schedule_warps + // ibuffer occupancy is already gated by VX_warp_sched (schedule_warps // masks out warps with full ibufs), so schedule_if.valid implies space. // Suppress with the UNGATED sched_buffered (not the st0-gated match): a // word already in the decompressor buffer must never be re-fetched, even @@ -135,7 +146,7 @@ module VX_fetch import VX_gpu_pkg::*; #( assign icache_req_valid = follow_req_valid || sched_req_valid; assign icache_req_PC = follow_req_valid ? follow_req_PC : schedule_if.data.PC; assign icache_req_tmask = follow_req_valid ? follow_req_tmask : schedule_if.data.tmask; - assign icache_req_wid = follow_req_valid ? follow_req_wid : schedule_if.data.wid; + assign icache_req_wid = follow_req_valid ? follow_req_wid : sched_wid; assign icache_req_uuid = follow_req_valid ? follow_req_uuid : schedule_if.data.uuid; // NOTE: bind the full PC to a net before part-selecting — Vivado synthesis // rejects a bit-select applied directly to a function-call result. @@ -156,7 +167,8 @@ module VX_fetch import VX_gpu_pkg::*; #( || sched_buffered_match; VX_decompressor #( - .INSTANCE_ID (INSTANCE_ID) + .INSTANCE_ID (INSTANCE_ID), + .NUM_WARPS (NUM_WARPS) ) decompressor ( .clk (clk), .reset (reset), @@ -164,7 +176,7 @@ module VX_fetch import VX_gpu_pkg::*; #( .sched_PC (schedule_if.data.PC), .sched_tmask (schedule_if.data.tmask), .sched_cta_id (schedule_if.data.cta_id), - .sched_wid (schedule_if.data.wid), + .sched_wid (sched_wid), .sched_buffered_match (sched_buffered_match), .sched_buffered (sched_buffered), .inflight (inflight), @@ -191,7 +203,7 @@ module VX_fetch import VX_gpu_pkg::*; #( assign icache_req_valid = schedule_if.valid; assign icache_req_addr = schedule_if.data.PC[2-(`VX_CFG_XLEN-PC_BITS) +: ICACHE_ADDR_WIDTH]; - assign icache_req_wid = schedule_if.data.wid; + assign icache_req_wid = sched_wid; assign icache_req_uuid = schedule_if.data.uuid; assign icache_req_tag = {icache_req_uuid, icache_req_wid}; assign schedule_if.ibuf_pop = fetch_if.ibuf_pop; @@ -199,7 +211,7 @@ module VX_fetch import VX_gpu_pkg::*; #( assign fetch_if.valid = icache_bus_if.rsp_valid; assign fetch_if.data.tmask = rsp_tmask; - assign fetch_if.data.wid = rsp_tag; + assign fetch_if.data.wid = NW_WIDTH'(rsp_tag); assign fetch_if.data.cta_id = rsp_cta_id; assign fetch_if.data.PC = rsp_PC; assign fetch_if.data.instr = icache_bus_if.rsp_data.data; diff --git a/hw/rtl/core/VX_fetch_slice.sv b/hw/rtl/core/VX_fetch_slice.sv new file mode 100644 index 0000000000..41d788c064 --- /dev/null +++ b/hw/rtl/core/VX_fetch_slice.sv @@ -0,0 +1,149 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// One fetch slice: warp scheduling, fetch and decode of the warps with +// wid mod FETCH_WIDTH == slice_id, on the slice's own icache port. The front end +// names the slice's warps by wfs; decoded instructions leave with their +// core-wide wid, and ibuffer pops return indexed by wfs. +module VX_fetch_slice import VX_gpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "", + parameter `STRING SLICE_STR = "" +) ( + `SCOPE_IO_DECL + + input wire clk, + input wire reset, + + input wire [CORE_ID_WIDTH-1:0] core_id, + input wire [FETCH_FSW_W-1:0] slice_id, + +`ifdef PERF_ENABLE + output fetch_perf_t fetch_perf, +`endif + + // core scheduler + VX_slice_sched_if.slave slice_sched_if, + + // instruction fetch + VX_mem_bus_if.master icache_bus_if, + + // decoded instructions + VX_decode_if.master decode_if, + + // ALU ports + input sched_branch_t [FETCH_ALU_PORTS-1:0] branch, + + // SFU ports + input sched_wctl_t [FETCH_SFU_PORTS-1:0] wctl, + input wire [FETCH_SFU_PORTS-1:0][NW_WIDTH-1:0] dvstack_wid, + output wire [FETCH_SFU_PORTS-1:0][DV_STACK_SIZEW-1:0] dvstack_ptr, + output wire [FETCH_SFU_PORTS-1:0][`VX_CFG_MEM_ADDR_WIDTH-1:0] dvstack_mscratch, + input wire [FETCH_SFU_PORTS-1:0][NW_WIDTH-1:0] csr_rd_wid, + output wire sched_csr_rd_t [FETCH_SFU_PORTS-1:0] csr_rd, + input sched_csr_wr_t [FETCH_SFU_PORTS-1:0] csr_wr, +`ifdef VX_CFG_EXT_RTU_ENABLE + input wire [FETCH_SFU_PORTS-1:0] rtu_unlock_valid, + input wire [FETCH_SFU_PORTS-1:0][NW_WIDTH-1:0] rtu_unlock_wid, +`endif + + // per-warp pipeline events, indexed by wfs + input wire [PER_FETCH_WARPS-1:0] issued, + input wire [PER_FETCH_WARPS-1:0] committed, + output wire [PER_FETCH_WARPS-1:0] pending_alm_empty +); + localparam SW = PER_FETCH_WARPS; + localparam LW = FETCH_WFS_W; + + `SCOPE_IO_SWITCH (1); + + VX_schedule_if schedule_if(); + VX_fetch_if fetch_if(); + VX_decode_if fe_decode_if(); + VX_decode_sched_if decode_sched_if(); + + VX_warp_sched #( + .INSTANCE_ID (`SFORMATF(("%s-scheduler%s", INSTANCE_ID, SLICE_STR))) + ) warp_sched ( + .clk (clk), + .reset (reset), + .core_id (core_id), + .slice_id (slice_id), + .sched_if (slice_sched_if), + .branch (branch), + .wctl (wctl), + .dvstack_wid (dvstack_wid), + .dvstack_ptr (dvstack_ptr), + .dvstack_mscratch (dvstack_mscratch), + .csr_rd_wid (csr_rd_wid), + .csr_rd (csr_rd), + .csr_wr (csr_wr), + `ifdef VX_CFG_EXT_RTU_ENABLE + .rtu_unlock_valid (rtu_unlock_valid), + .rtu_unlock_wid (rtu_unlock_wid), + `endif + .decode_sched_if (decode_sched_if), + .issued (issued), + .committed (committed), + .ibuf_pop (decode_if.ibuf_pop[SW-1:0]), + .pending_alm_empty(pending_alm_empty), + .schedule_if (schedule_if) + ); + + VX_fetch #( + .INSTANCE_ID (`SFORMATF(("%s-fetch%s", INSTANCE_ID, SLICE_STR))), + .NUM_WARPS (SW) + ) fetch ( + `SCOPE_IO_BIND (0) + .clk (clk), + .reset (reset), + `ifdef PERF_ENABLE + .fetch_perf (fetch_perf), + `endif + .icache_bus_if (icache_bus_if), + .schedule_if (schedule_if), + .fetch_if (fetch_if) + ); + + VX_decode #( + .INSTANCE_ID (`SFORMATF(("%s-decode%s", INSTANCE_ID, SLICE_STR))) + ) decode ( + .clk (clk), + .reset (reset), + .fetch_if (fetch_if), + .decode_if (fe_decode_if), + .decode_sched_if(decode_sched_if) + ); + + // decoded instructions take their core-wide wid + decode_t decode_data; + always @(*) begin + decode_data = fe_decode_if.data; + decode_data.wid = wfs_to_wid(fe_decode_if.data.wid[LW-1:0], slice_id); + end + if (NW_WIDTH > LW) begin : g_fe_wid_unused + `UNUSED_VAR (fe_decode_if.data.wid[NW_WIDTH-1:LW]) + end + + assign decode_if.valid = fe_decode_if.valid; + assign decode_if.data = decode_data; + assign fe_decode_if.ready = decode_if.ready; + + assign fe_decode_if.ibuf_pop[SW-1:0] = decode_if.ibuf_pop[SW-1:0]; + if (SW < `VX_CFG_NUM_WARPS) begin : g_ibuf_pop_pad + assign fe_decode_if.ibuf_pop[`VX_CFG_NUM_WARPS-1:SW] = '0; + `UNUSED_VAR (decode_if.ibuf_pop[`VX_CFG_NUM_WARPS-1:SW]) + end + +endmodule diff --git a/hw/rtl/core/VX_fetch_xbar.sv b/hw/rtl/core/VX_fetch_xbar.sv new file mode 100644 index 0000000000..85c7babd07 --- /dev/null +++ b/hw/rtl/core/VX_fetch_xbar.sv @@ -0,0 +1,229 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// Connects the fetch slices (fetch slice f owns wid mod FETCH_WIDTH == f) to the +// issue slices (wid mod ISSUE_WIDTH) and the ALU/SFU blocks (wid mod B). Issue +// slice i takes its instructions from fetch slice i mod FETCH_WIDTH. A block +// serving several fetch slices sends each its own warps' events and answers its +// reads from the slice owning the read wid. Purely combinational. +module VX_fetch_xbar import VX_gpu_pkg::*; ( + // instruction path + VX_decode_if.slave fs_decode_if [`VX_CFG_FETCH_WIDTH], + VX_decode_if.master is_decode_if [`VX_CFG_ISSUE_WIDTH], + + // issue and commit events, and per-warp drain status + input wire [`VX_CFG_ISSUE_WIDTH-1:0] issue_valid, + input wire [`VX_CFG_ISSUE_WIDTH-1:0][ISSUE_WIS_W-1:0] issue_wis, + input wire [`VX_CFG_NUM_WARPS-1:0] committed_warps, + output wire [`VX_CFG_FETCH_WIDTH-1:0][PER_FETCH_WARPS-1:0] fs_issued, + output wire [`VX_CFG_FETCH_WIDTH-1:0][PER_FETCH_WARPS-1:0] fs_committed, + input wire [`VX_CFG_FETCH_WIDTH-1:0][PER_FETCH_WARPS-1:0] fs_pending_alm_empty, + + // FU blocks + VX_branch_ctl_if.slave branch_ctl_if [`VX_CFG_NUM_ALU_BLOCKS], + VX_warp_ctl_if.slave warp_ctl_if [`VX_CFG_NUM_SFU_BLOCKS], + VX_sched_csr_if.master sched_csr_if [`VX_CFG_NUM_SFU_BLOCKS], +`ifdef VX_CFG_EXT_RTU_ENABLE + VX_sched_unlock_if.slave sched_unlock_if [`VX_CFG_NUM_SFU_BLOCKS], +`endif + + // core scheduler state + input wire [PERF_CTR_BITS-1:0] cycles, + input wire [PERF_CTR_BITS-1:0] instret, + input wire [`VX_CFG_NUM_WARPS-1:0] active_warps, + input wire lsu_sched_drained, + input cta_ctx_t [`VX_CFG_NUM_SFU_BLOCKS-1:0] csr_rd_ctx, +`ifdef VX_CFG_VM_ENABLE + input wire [`VX_CFG_NUM_SFU_BLOCKS-1:0][`VX_CFG_XLEN-1:0] satp, + output wire [`VX_CFG_NUM_SFU_BLOCKS-1:0] satp_wr_valid, + output wire [`VX_CFG_NUM_SFU_BLOCKS-1:0][`VX_CFG_XLEN-1:0] satp_wr_data, +`endif + + // fetch slices + output sched_branch_t [`VX_CFG_FETCH_WIDTH-1:0][FETCH_ALU_PORTS-1:0] fs_branch, + output sched_wctl_t [`VX_CFG_FETCH_WIDTH-1:0][FETCH_SFU_PORTS-1:0] fs_wctl, + output wire [`VX_CFG_FETCH_WIDTH-1:0][FETCH_SFU_PORTS-1:0][NW_WIDTH-1:0] fs_dvstack_wid, + input wire [`VX_CFG_FETCH_WIDTH-1:0][FETCH_SFU_PORTS-1:0][DV_STACK_SIZEW-1:0] fs_dvstack_ptr, + input wire [`VX_CFG_FETCH_WIDTH-1:0][FETCH_SFU_PORTS-1:0][`VX_CFG_MEM_ADDR_WIDTH-1:0] fs_dvstack_mscratch, + output wire [`VX_CFG_FETCH_WIDTH-1:0][FETCH_SFU_PORTS-1:0][NW_WIDTH-1:0] fs_csr_rd_wid, + input sched_csr_rd_t [`VX_CFG_FETCH_WIDTH-1:0][FETCH_SFU_PORTS-1:0] fs_csr_rd, +`ifdef VX_CFG_EXT_RTU_ENABLE + output wire [`VX_CFG_FETCH_WIDTH-1:0][FETCH_SFU_PORTS-1:0] fs_rtu_unlock_valid, + output wire [`VX_CFG_FETCH_WIDTH-1:0][FETCH_SFU_PORTS-1:0][NW_WIDTH-1:0] fs_rtu_unlock_wid, +`endif + output sched_csr_wr_t [`VX_CFG_FETCH_WIDTH-1:0][FETCH_SFU_PORTS-1:0] fs_csr_wr +); + localparam FW = `VX_CFG_FETCH_WIDTH; + localparam IW = `VX_CFG_ISSUE_WIDTH; + localparam NW = `VX_CFG_NUM_WARPS; + localparam SW = PER_FETCH_WARPS; + localparam BA = `VX_CFG_NUM_ALU_BLOCKS; + localparam BS = `VX_CFG_NUM_SFU_BLOCKS; + localparam NPA = FETCH_ALU_PORTS; + localparam NPS = FETCH_SFU_PORTS; + + `STATIC_ASSERT (`IS_POW2(FW) && (FW <= IW) && ((IW % FW) == 0), ("invalid parameter")) + + // instruction path + + wire [IW-1:0] is_ready; + for (genvar i = 0; i < IW; ++i) begin : g_issue_slices + localparam F = i % FW; + assign is_decode_if[i].valid = fs_decode_if[F].valid && (wid_to_isw(fs_decode_if[F].data.wid) == ISSUE_ISW_W'(i)); + assign is_decode_if[i].data = fs_decode_if[F].data; + assign is_ready[i] = is_decode_if[i].ready; + if (PER_ISSUE_WARPS < NW) begin : g_ibuf_pop_unused + `UNUSED_VAR (is_decode_if[i].ibuf_pop[NW-1:PER_ISSUE_WARPS]) + end + end + + for (genvar f = 0; f < FW; ++f) begin : g_fetch_slices + assign fs_decode_if[f].ready = is_ready[wid_to_isw(fs_decode_if[f].data.wid)]; + for (genvar l = 0; l < SW; ++l) begin : g_ibuf_pop + localparam W = l * FW + f; + assign fs_decode_if[f].ibuf_pop[l] = is_decode_if[W % IW].ibuf_pop[W / IW]; + end + if (SW < NW) begin : g_ibuf_pop_pad + assign fs_decode_if[f].ibuf_pop[NW-1:SW] = '0; + end + end + + // issue and commit events, and per-warp drain status + + wire [NW-1:0] issued_warps; + wire [NW-1:0] pending_alm_empty; + for (genvar w = 0; w < NW; ++w) begin : g_warps + assign issued_warps[w] = issue_valid[w % IW] && (issue_wis[w % IW] == ISSUE_WIS_W'(w / IW)); + assign fs_issued[w % FW][w / FW] = issued_warps[w]; + assign fs_committed[w % FW][w / FW] = committed_warps[w]; + assign pending_alm_empty[w] = fs_pending_alm_empty[w % FW][w / FW]; + end + + // ALU blocks: fetch slice f port q is block q * FW + f, or block f mod BA when shared + + for (genvar f = 0; f < FW; ++f) begin : g_alu_slices + for (genvar q = 0; q < NPA; ++q) begin : g_ports + localparam B = (BA >= FW) ? (q * FW + f) : (f % BA); + wire own; + if (BA >= FW) begin : g_own + assign own = 1'b1; + end else begin : g_shared + assign own = (wid_to_fsw(branch_ctl_if[B].wid) == FETCH_FSW_W'(f)); + end + assign fs_branch[f][q].valid = branch_ctl_if[B].valid && own; + assign fs_branch[f][q].wid = branch_ctl_if[B].wid; + assign fs_branch[f][q].taken = branch_ctl_if[B].taken; + assign fs_branch[f][q].dest = branch_ctl_if[B].dest; + assign fs_branch[f][q].is_trap = branch_ctl_if[B].is_trap; + assign fs_branch[f][q].is_mret = branch_ctl_if[B].is_mret; + assign fs_branch[f][q].trap_cause = branch_ctl_if[B].trap_cause; + end + end + + // SFU blocks: same port mapping; reads are answered by the slice owning the read wid + + for (genvar f = 0; f < FW; ++f) begin : g_sfu_slices + for (genvar q = 0; q < NPS; ++q) begin : g_ports + localparam B = (BS >= FW) ? (q * FW + f) : (f % BS); + wire own_ctl, own_wr; + `ifdef VX_CFG_EXT_RTU_ENABLE + wire own_unlock; + `endif + if (BS >= FW) begin : g_own + assign own_ctl = 1'b1; + assign own_wr = 1'b1; + `ifdef VX_CFG_EXT_RTU_ENABLE + assign own_unlock = 1'b1; + `endif + end else begin : g_shared + assign own_ctl = (wid_to_fsw(warp_ctl_if[B].wid) == FETCH_FSW_W'(f)); + assign own_wr = (wid_to_fsw(sched_csr_if[B].csr_wr_wid) == FETCH_FSW_W'(f)); + `ifdef VX_CFG_EXT_RTU_ENABLE + assign own_unlock = (wid_to_fsw(sched_unlock_if[B].wid) == FETCH_FSW_W'(f)); + `endif + end + + assign fs_wctl[f][q].tmc_valid = warp_ctl_if[B].tmc_valid && own_ctl; + assign fs_wctl[f][q].split_valid = warp_ctl_if[B].split_valid && own_ctl; + assign fs_wctl[f][q].sjoin_valid = warp_ctl_if[B].sjoin_valid && own_ctl; + assign fs_wctl[f][q].wsync_valid = warp_ctl_if[B].wsync_valid && own_ctl; + assign fs_wctl[f][q].wid = warp_ctl_if[B].wid; + assign fs_wctl[f][q].tmc = warp_ctl_if[B].tmc; + assign fs_wctl[f][q].split = warp_ctl_if[B].split; + assign fs_wctl[f][q].sjoin = warp_ctl_if[B].sjoin; + assign fs_dvstack_wid[f][q] = warp_ctl_if[B].dvstack_wid; + + assign fs_csr_rd_wid[f][q] = sched_csr_if[B].csr_rd_wid; + assign fs_csr_wr[f][q].valid = sched_csr_if[B].csr_wr_valid && own_wr; + assign fs_csr_wr[f][q].wid = sched_csr_if[B].csr_wr_wid; + assign fs_csr_wr[f][q].data = sched_csr_if[B].csr_wr_data; + assign fs_csr_wr[f][q].trap_valid = sched_csr_if[B].trap_csr_wr_valid & {NUM_TRAP_CSRS{own_wr}}; + assign fs_csr_wr[f][q].trap_data = sched_csr_if[B].trap_csr_wr_data; + + `ifdef VX_CFG_EXT_RTU_ENABLE + assign fs_rtu_unlock_valid[f][q] = sched_unlock_if[B].valid && own_unlock; + assign fs_rtu_unlock_wid[f][q] = sched_unlock_if[B].wid; + `endif + end + end + + for (genvar b = 0; b < BS; ++b) begin : g_sfu_blocks + sched_csr_rd_t csr_rd; + if (BS >= FW) begin : g_own + assign warp_ctl_if[b].dvstack_ptr = fs_dvstack_ptr[b % FW][b / FW]; + assign warp_ctl_if[b].mscratch = fs_dvstack_mscratch[b % FW][b / FW]; + assign csr_rd = fs_csr_rd[b % FW][b / FW]; + end else begin : g_shared + wire [FETCH_FSW_W-1:0] stack_fsw = wid_to_fsw(warp_ctl_if[b].dvstack_wid); + wire [FETCH_FSW_W-1:0] csr_fsw = wid_to_fsw(sched_csr_if[b].csr_rd_wid); + assign warp_ctl_if[b].dvstack_ptr = fs_dvstack_ptr[stack_fsw][0]; + assign warp_ctl_if[b].mscratch = fs_dvstack_mscratch[stack_fsw][0]; + assign csr_rd = fs_csr_rd[csr_fsw][0]; + end + + for (genvar k = 0; k < SFU_BLOCK_WARPS; ++k) begin : g_pending + assign warp_ctl_if[b].warp_pending_alm_empty[k] = pending_alm_empty[k * BS + b]; + end + assign warp_ctl_if[b].lsu_sched_drained = lsu_sched_drained; + + assign sched_csr_if[b].cycles = cycles; + assign sched_csr_if[b].instret = instret; + assign sched_csr_if[b].active_warps = active_warps; + assign sched_csr_if[b].thread_mask = csr_rd.thread_mask; + assign sched_csr_if[b].mscratch = csr_rd.mscratch; + assign sched_csr_if[b].csr_mstatus = csr_rd.mstatus; + assign sched_csr_if[b].csr_mtvec = csr_rd.mtvec; + assign sched_csr_if[b].csr_mepc = csr_rd.mepc; + assign sched_csr_if[b].csr_mcause = csr_rd.mcause; + assign sched_csr_if[b].csr_mtval = csr_rd.mtval; + assign sched_csr_if[b].cta_csrs.cta_id = sched_csr_if[b].csr_rd_cta_id; + assign sched_csr_if[b].cta_csrs.cta_rank = csr_rd.cta_rank; + assign sched_csr_if[b].cta_csrs.cta_size = csr_rd_ctx[b].cta_size; + assign sched_csr_if[b].cta_csrs.block_idx = csr_rd_ctx[b].block_idx; + assign sched_csr_if[b].cta_csrs.block_dim = csr_rd_ctx[b].block_dim; + assign sched_csr_if[b].cta_csrs.grid_dim = csr_rd_ctx[b].grid_dim; + assign sched_csr_if[b].cta_csrs.entry = csr_rd_ctx[b].entry; + assign sched_csr_if[b].cta_csrs.param = csr_rd_ctx[b].param; + assign sched_csr_if[b].cta_csrs.lmem_addr = csr_rd_ctx[b].lmem_addr; + assign sched_csr_if[b].cta_csrs.cluster_size = csr_rd_ctx[b].cluster_size; + assign sched_csr_if[b].cta_lane = csr_rd.cta_lane; + `ifdef VX_CFG_VM_ENABLE + assign sched_csr_if[b].csr_satp = satp[b]; + assign satp_wr_valid[b] = sched_csr_if[b].satp_wr_valid; + assign satp_wr_data[b] = sched_csr_if[b].satp_wr_data; + `endif + end + +endmodule diff --git a/hw/rtl/core/VX_ibuffer.sv b/hw/rtl/core/VX_ibuffer.sv index 377075c7f1..6cfd72f7bd 100644 --- a/hw/rtl/core/VX_ibuffer.sv +++ b/hw/rtl/core/VX_ibuffer.sv @@ -14,8 +14,7 @@ `include "VX_define.vh" module VX_ibuffer import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - parameter ISSUE_ID = 0 + parameter `STRING INSTANCE_ID = "" ) ( input wire clk, input wire reset, @@ -30,7 +29,6 @@ module VX_ibuffer import VX_gpu_pkg::*; #( // outputs VX_ibuffer_if.master ibuffer_if [PER_ISSUE_WARPS] ); - `UNUSED_PARAM (ISSUE_ID) localparam OUT_DATAW = $bits(ibuffer_t); diff --git a/hw/rtl/core/VX_ipdom_stack.sv b/hw/rtl/core/VX_ipdom_stack.sv index a702d61d09..a5b8a8b3f7 100644 --- a/hw/rtl/core/VX_ipdom_stack.sv +++ b/hw/rtl/core/VX_ipdom_stack.sv @@ -14,31 +14,33 @@ `include "VX_define.vh" module VX_ipdom_stack import VX_gpu_pkg::*; #( - parameter WIDTH = 1, - parameter DEPTH = 1, - parameter ADDRW = `LOG2UP(DEPTH) + parameter WIDTH = 1, + parameter DEPTH = 1, + parameter NUM_WARPS = `VX_CFG_NUM_WARPS, + parameter WID_W = `LOG2UP(NUM_WARPS), + parameter ADDRW = `LOG2UP(DEPTH) ) ( input wire clk, input wire reset, - input wire [NW_WIDTH-1:0] wid, + input wire [WID_W-1:0] wid, input wire [WIDTH-1:0] d_val, input wire [ADDRW-1:0] rd_ptr, input wire push, input wire pop, output wire [WIDTH-1:0] q_val, output wire q_idx, - output wire [`VX_CFG_NUM_WARPS-1:0][ADDRW-1:0] wr_ptr, + output wire [NUM_WARPS-1:0][ADDRW-1:0] wr_ptr, output wire empty, output wire full ); localparam BRAM_DATAW = 1 + WIDTH; - localparam BRAM_SIZE = DEPTH * `VX_CFG_NUM_WARPS; + localparam BRAM_SIZE = DEPTH * NUM_WARPS; localparam BRAW_ADDRW = `LOG2UP(BRAM_SIZE); - wire [`VX_CFG_NUM_WARPS-1:0][ADDRW-1:0] wr_ptr_w; - wire [`VX_CFG_NUM_WARPS-1:0] empty_w, full_w; + wire [NUM_WARPS-1:0][ADDRW-1:0] wr_ptr_w; + wire [NUM_WARPS-1:0] empty_w, full_w; - for (genvar i = 0; i < `VX_CFG_NUM_WARPS; i++) begin : g_addressing + for (genvar i = 0; i < NUM_WARPS; i++) begin : g_addressing reg [ADDRW-1:0] wr_ptr_r; reg empty_r, full_r; @@ -75,14 +77,14 @@ module VX_ipdom_stack import VX_gpu_pkg::*; #( wire [BRAW_ADDRW-1:0] raddr, waddr; - if (DEPTH > 1 && `VX_CFG_NUM_WARPS > 1) begin : g_DW + if (DEPTH > 1 && NUM_WARPS > 1) begin : g_DW assign waddr = push ? {wr_ptr_w[wid], wid} : {rd_ptr, wid}; assign raddr = {rd_ptr, wid}; end else if (DEPTH > 1) begin : g_D `UNUSED_VAR (wid) assign waddr = push ? wr_ptr_w : rd_ptr; assign raddr = rd_ptr; - end else if (`VX_CFG_NUM_WARPS > 1) begin : g_W + end else if (NUM_WARPS > 1) begin : g_W `UNUSED_VAR (rd_ptr) `UNUSED_VAR (wr_ptr_w) assign waddr = wid; diff --git a/hw/rtl/core/VX_issue.sv b/hw/rtl/core/VX_issue.sv index ddc745b577..03bff6cd98 100644 --- a/hw/rtl/core/VX_issue.sv +++ b/hw/rtl/core/VX_issue.sv @@ -25,7 +25,7 @@ module VX_issue import VX_gpu_pkg::*; #( output issue_perf_t issue_perf, `endif - VX_decode_if.slave decode_if, + VX_decode_if.slave decode_if [`VX_CFG_ISSUE_WIDTH], VX_writeback_if.slave writeback_if [`VX_CFG_ISSUE_WIDTH], VX_dispatch_if.master dispatch_if [NUM_EX_UNITS * `VX_CFG_ISSUE_WIDTH], VX_issue_sched_if.master issue_sched_if [`VX_CFG_ISSUE_WIDTH] @@ -43,11 +43,6 @@ module VX_issue import VX_gpu_pkg::*; #( end `endif - wire [ISSUE_ISW_W-1:0] decode_isw = wid_to_isw(decode_if.data.wid); - - wire [`VX_CFG_ISSUE_WIDTH-1:0] decode_ready_in; - assign decode_if.ready = decode_ready_in[decode_isw]; - `SCOPE_IO_SWITCH (`VX_CFG_ISSUE_WIDTH); wire [`VX_CFG_ISSUE_WIDTH-1:0] issued_warps; @@ -56,27 +51,18 @@ module VX_issue import VX_gpu_pkg::*; #( for (genvar issue_id = 0; issue_id < `VX_CFG_ISSUE_WIDTH; ++issue_id) begin : g_slices VX_dispatch_if per_issue_dispatch_if[NUM_EX_UNITS](); - VX_decode_if slice_decode_if(); - - assign slice_decode_if.valid = decode_if.valid && (decode_isw == issue_id); - assign slice_decode_if.data = decode_if.data; - assign decode_ready_in[issue_id] = slice_decode_if.ready; - - for (genvar w = 0; w < PER_ISSUE_WARPS; ++w) begin : g_ibuf_pop - assign decode_if.ibuf_pop[wis_to_wid(ISSUE_WIS_W'(w), ISSUE_ISW_W'(issue_id))] = slice_decode_if.ibuf_pop[w]; - end VX_issue_slice #( - .INSTANCE_ID (`SFORMATF(("%s%0d", INSTANCE_ID, issue_id))), - .ISSUE_ID (issue_id) + .INSTANCE_ID (`SFORMATF(("%s%0d", INSTANCE_ID, issue_id))) ) issue_slice ( `SCOPE_IO_BIND(issue_id) .clk (clk), .reset (reset), + .slice_id (ISSUE_ISW_W'(issue_id)), `ifdef PERF_ENABLE .issue_perf (per_issue_perf[issue_id]), `endif - .decode_if (slice_decode_if), + .decode_if (decode_if[issue_id]), .writeback_if (writeback_if[issue_id]), .dispatch_if (per_issue_dispatch_if), .warp_issued (issued_warps[issue_id]), diff --git a/hw/rtl/core/VX_issue_slice.sv b/hw/rtl/core/VX_issue_slice.sv index b827b1e0ee..4f147c68e2 100644 --- a/hw/rtl/core/VX_issue_slice.sv +++ b/hw/rtl/core/VX_issue_slice.sv @@ -14,14 +14,15 @@ `include "VX_define.vh" module VX_issue_slice import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - parameter ISSUE_ID = 0 + parameter `STRING INSTANCE_ID = "" ) ( `SCOPE_IO_DECL input wire clk, input wire reset, + input wire [ISSUE_ISW_W-1:0] slice_id, + `ifdef PERF_ENABLE output issue_perf_t issue_perf, `endif @@ -39,8 +40,7 @@ module VX_issue_slice import VX_gpu_pkg::*; #( wire [NUM_EX_UNITS-1:0] fu_release; VX_ibuffer #( - .INSTANCE_ID (`SFORMATF(("%s-ibuffer", INSTANCE_ID))), - .ISSUE_ID (ISSUE_ID) + .INSTANCE_ID (`SFORMATF(("%s-ibuffer", INSTANCE_ID))) ) ibuffer ( .clk (clk), .reset (reset), @@ -52,8 +52,7 @@ module VX_issue_slice import VX_gpu_pkg::*; #( ); VX_scoreboard #( - .INSTANCE_ID (`SFORMATF(("%s-scoreboard", INSTANCE_ID))), - .ISSUE_ID (ISSUE_ID) + .INSTANCE_ID (`SFORMATF(("%s-scoreboard", INSTANCE_ID))) ) scoreboard ( .clk (clk), .reset (reset), @@ -67,8 +66,7 @@ module VX_issue_slice import VX_gpu_pkg::*; #( ); VX_operands #( - .INSTANCE_ID (`SFORMATF(("%s-operands", INSTANCE_ID))), - .ISSUE_ID (ISSUE_ID) + .INSTANCE_ID (`SFORMATF(("%s-operands", INSTANCE_ID))) ) operands ( .clk (clk), .reset (reset), @@ -81,11 +79,11 @@ module VX_issue_slice import VX_gpu_pkg::*; #( ); VX_dispatcher #( - .INSTANCE_ID (`SFORMATF(("%s-dispatcher", INSTANCE_ID))), - .ISSUE_ID (ISSUE_ID) + .INSTANCE_ID (`SFORMATF(("%s-dispatcher", INSTANCE_ID))) ) dispatcher ( .clk (clk), .reset (reset), + .slice_id (slice_id), `ifdef PERF_ENABLE .perf_stalls (issue_perf.dispatch_stalls), .perf_instrs (issue_perf.dispatch_instrs), diff --git a/hw/rtl/core/VX_lane_dispatch.sv b/hw/rtl/core/VX_lane_dispatch.sv index b6ae6126a1..5ff16da692 100644 --- a/hw/rtl/core/VX_lane_dispatch.sv +++ b/hw/rtl/core/VX_lane_dispatch.sv @@ -17,19 +17,25 @@ module VX_lane_dispatch import VX_gpu_pkg::*; #( parameter BLOCK_SIZE = 1, parameter NUM_LANES = 1, parameter OUT_BUF = 0, - parameter MAX_FANOUT = `MAX_FANOUT + parameter MAX_FANOUT = `MAX_FANOUT, + // dispatch ports served, starting at issue slice port_base + parameter NUM_PORTS = `VX_CFG_ISSUE_WIDTH ) ( input wire clk, input wire reset, + input wire [ISSUE_ISW_W-1:0] port_base, + // inputs - VX_dispatch_if.slave dispatch_if [`VX_CFG_ISSUE_WIDTH], + VX_dispatch_if.slave dispatch_if [NUM_PORTS], // outputs VX_execute_if.master execute_if [BLOCK_SIZE] ); - `STATIC_ASSERT (`IS_DIVISBLE(`VX_CFG_ISSUE_WIDTH, BLOCK_SIZE), ("invalid parameter")) + `STATIC_ASSERT (`IS_DIVISBLE(NUM_PORTS, BLOCK_SIZE), ("invalid parameter")) `STATIC_ASSERT (`IS_DIVISBLE(`VX_CFG_SIMD_WIDTH, NUM_LANES), ("invalid parameter")) + // a batched dispatch derives the issue slot from the batch index alone + `STATIC_ASSERT ((NUM_PORTS == `VX_CFG_ISSUE_WIDTH) || (BLOCK_SIZE == NUM_PORTS), ("invalid parameter")) `DECL_EXECUTE_T (pe, NUM_LANES); localparam IN_DATAW = $bits(dispatch_t); @@ -41,9 +47,9 @@ module VX_lane_dispatch import VX_gpu_pkg::*; #( localparam LPID_WIDTH = `UP(LPID_BITS); localparam GPID_BITS = `CLOG2(`VX_CFG_NUM_THREADS / NUM_LANES); localparam GPID_WIDTH = `UP(GPID_BITS); - localparam BATCH_COUNT = `VX_CFG_ISSUE_WIDTH / BLOCK_SIZE; + localparam BATCH_COUNT = NUM_PORTS / BLOCK_SIZE; localparam BATCH_COUNT_W= `LOG2UP(BATCH_COUNT); - localparam ISSUE_W = `LOG2UP(`VX_CFG_ISSUE_WIDTH); + localparam ISSUE_W = `LOG2UP(NUM_PORTS); localparam FANOUT_ENABLE= (`VX_CFG_SIMD_WIDTH > (MAX_FANOUT + MAX_FANOUT /2)); localparam DATA_IN_TMASK_OFF = IN_DATAW - (UUID_WIDTH + ISSUE_WIS_W + NCTA_WIDTH + SIMD_IDX_W + `VX_CFG_SIMD_WIDTH); @@ -54,11 +60,11 @@ module VX_lane_dispatch import VX_gpu_pkg::*; #( logic [NUM_LANES-1:0] tmask; } packet_t; - wire [`VX_CFG_ISSUE_WIDTH-1:0] dispatch_valid; - wire [`VX_CFG_ISSUE_WIDTH-1:0][IN_DATAW-1:0] dispatch_data; - wire [`VX_CFG_ISSUE_WIDTH-1:0] dispatch_ready; + wire [NUM_PORTS-1:0] dispatch_valid; + wire [NUM_PORTS-1:0][IN_DATAW-1:0] dispatch_data; + wire [NUM_PORTS-1:0] dispatch_ready; - for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_dispatch_data + for (genvar i = 0; i < NUM_PORTS; ++i) begin : g_dispatch_data assign dispatch_valid[i] = dispatch_if[i].valid; assign dispatch_data[i] = dispatch_if[i].data; assign dispatch_if[i].ready = dispatch_ready[i]; @@ -110,6 +116,10 @@ module VX_lane_dispatch import VX_gpu_pkg::*; #( `UNUSED_VAR (batch_done) end + if (BATCH_COUNT != 1) begin : g_no_port_base + `UNUSED_VAR (port_base) + end + wire [BLOCK_SIZE-1:0][ISSUE_W-1:0] issue_indices; for (genvar block_idx = 0; block_idx < BLOCK_SIZE; ++block_idx) begin : g_issue_indices assign issue_indices[block_idx] = ISSUE_W'(batch_idx * BLOCK_SIZE) + ISSUE_W'(block_idx); @@ -198,7 +208,7 @@ module VX_lane_dispatch import VX_gpu_pkg::*; #( assign isw = batch_idx; end end else begin : g_isw - assign isw = block_idx; + assign isw = port_base + ISSUE_ISW_W'(block_idx); end wire [NW_WIDTH-1:0] block_wid = wis_to_wid(dispatch_wis, isw); @@ -236,7 +246,7 @@ module VX_lane_dispatch import VX_gpu_pkg::*; #( end // release the dispatch interface when all packets are sent - reg [`VX_CFG_ISSUE_WIDTH-1:0] ready_in; + reg [NUM_PORTS-1:0] ready_in; always @(*) begin ready_in = 0; for (integer block_idx = 0; block_idx < BLOCK_SIZE; ++block_idx) begin diff --git a/hw/rtl/core/VX_lane_gather.sv b/hw/rtl/core/VX_lane_gather.sv index 0803f31bc8..905049c5e5 100644 --- a/hw/rtl/core/VX_lane_gather.sv +++ b/hw/rtl/core/VX_lane_gather.sv @@ -16,7 +16,9 @@ module VX_lane_gather import VX_gpu_pkg::*; #( parameter BLOCK_SIZE = 1, parameter NUM_LANES = 1, - parameter OUT_BUF = 0 + parameter OUT_BUF = 0, + // commit ports served + parameter NUM_PORTS = `VX_CFG_ISSUE_WIDTH ) ( input wire clk, input wire reset, @@ -25,9 +27,9 @@ module VX_lane_gather import VX_gpu_pkg::*; #( VX_result_if.slave result_if [BLOCK_SIZE], // outputs - VX_commit_if.master commit_if [`VX_CFG_ISSUE_WIDTH] + VX_commit_if.master commit_if [NUM_PORTS] ); - `STATIC_ASSERT (`IS_DIVISBLE(`VX_CFG_ISSUE_WIDTH, BLOCK_SIZE), ("invalid parameter")) + `STATIC_ASSERT (`IS_DIVISBLE(NUM_PORTS, BLOCK_SIZE), ("invalid parameter")) `STATIC_ASSERT (`IS_DIVISBLE(`VX_CFG_SIMD_WIDTH, NUM_LANES), ("invalid parameter")) `DECL_EXECUTE_T (pe, NUM_LANES); @@ -38,34 +40,38 @@ module VX_lane_gather import VX_gpu_pkg::*; #( localparam LPID_WIDTH = `UP(LPID_BITS); localparam DATAW = $bits(pe_result_t); localparam DATA_WIS_OFF = DATAW - (UUID_WIDTH + NW_WIDTH); + localparam PORT_W = `LOG2UP(NUM_PORTS); + + // a partial gather recovers the issue slice from the warp id + `STATIC_ASSERT ((BLOCK_SIZE == NUM_PORTS) || (NUM_PORTS == `VX_CFG_ISSUE_WIDTH), ("invalid parameter")) wire [BLOCK_SIZE-1:0] result_in_valid; wire [BLOCK_SIZE-1:0][DATAW-1:0] result_in_data; wire [BLOCK_SIZE-1:0] result_in_ready; - wire [BLOCK_SIZE-1:0][ISSUE_ISW_W-1:0] result_in_isw; + wire [BLOCK_SIZE-1:0][PORT_W-1:0] result_in_isw; for (genvar i = 0; i < BLOCK_SIZE; ++i) begin : g_commit_in assign result_in_valid[i] = result_if[i].valid; assign result_in_data[i] = result_if[i].data; assign result_if[i].ready = result_in_ready[i]; - if (BLOCK_SIZE != `VX_CFG_ISSUE_WIDTH) begin : g_result_in_isw_partial + if (BLOCK_SIZE != NUM_PORTS) begin : g_result_in_isw_partial if (BLOCK_SIZE != 1) begin : g_block - assign result_in_isw[i] = {result_in_data[i][DATA_WIS_OFF+BLOCK_SIZE_W +: (ISSUE_ISW_W-BLOCK_SIZE_W)], BLOCK_SIZE_W'(i)}; + assign result_in_isw[i] = PORT_W'({result_in_data[i][DATA_WIS_OFF+BLOCK_SIZE_W +: (ISSUE_ISW_W-BLOCK_SIZE_W)], BLOCK_SIZE_W'(i)}); end else begin : g_no_block - assign result_in_isw[i] = result_in_data[i][DATA_WIS_OFF +: ISSUE_ISW_W]; + assign result_in_isw[i] = PORT_W'(result_in_data[i][DATA_WIS_OFF +: ISSUE_ISW_W]); end end else begin : g_result_in_isw_full - assign result_in_isw[i] = BLOCK_SIZE_W'(i); + assign result_in_isw[i] = PORT_W'(i); end end - reg [`VX_CFG_ISSUE_WIDTH-1:0] result_out_valid; - reg [`VX_CFG_ISSUE_WIDTH-1:0][DATAW-1:0] result_out_data; - wire [`VX_CFG_ISSUE_WIDTH-1:0] result_out_ready; + reg [NUM_PORTS-1:0] result_out_valid; + reg [NUM_PORTS-1:0][DATAW-1:0] result_out_data; + wire [NUM_PORTS-1:0] result_out_ready; always @(*) begin result_out_valid = '0; - for (integer i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin + for (integer i = 0; i < NUM_PORTS; ++i) begin result_out_data[i] = 'x; end for (integer i = 0; i < BLOCK_SIZE; ++i) begin @@ -78,7 +84,7 @@ module VX_lane_gather import VX_gpu_pkg::*; #( assign result_in_ready[i] = result_out_ready[result_in_isw[i]]; end - for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin: g_out_bufs + for (genvar i = 0; i < NUM_PORTS; ++i) begin: g_out_bufs VX_result_if #( .data_t (pe_result_t) ) result_tmp_if(); diff --git a/hw/rtl/core/VX_lsu_agu.sv b/hw/rtl/core/VX_lsu_agu.sv index a858877ee9..c9dde01271 100644 --- a/hw/rtl/core/VX_lsu_agu.sv +++ b/hw/rtl/core/VX_lsu_agu.sv @@ -14,7 +14,7 @@ `include "VX_define.vh" // VX_lsu_agu: per-lane LSU address generator. Owns all LSU address forms -// so VX_lsu_slice contains no address arithmetic: +// so VX_lsu_block contains no address arithmetic: // - plain load/store/fence: addr = base + sext(offset) // - pack-load uop: addr = base + uop_idx * stride // (uop_idx = offset[1:0], stride = rs2; a 2-bit shift-and-add diff --git a/hw/rtl/core/VX_lsu_slice.sv b/hw/rtl/core/VX_lsu_block.sv similarity index 98% rename from hw/rtl/core/VX_lsu_slice.sv rename to hw/rtl/core/VX_lsu_block.sv index 066df5073e..f127d9f260 100644 --- a/hw/rtl/core/VX_lsu_slice.sv +++ b/hw/rtl/core/VX_lsu_block.sv @@ -13,15 +13,16 @@ `include "VX_define.vh" -module VX_lsu_slice import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - parameter CORE_ID = 0 +module VX_lsu_block import VX_gpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "" ) ( `SCOPE_IO_DECL input wire clk, input wire reset, + input wire [CORE_ID_WIDTH-1:0] core_id, + // Inputs VX_execute_if.slave execute_if, @@ -47,7 +48,9 @@ module VX_lsu_slice import VX_gpu_pkg::*; #( localparam [MEM_ADDRW-1:0] LMEM_BLK_BASE = MEM_ADDRW'(`VX_CFG_XLEN'(`VX_MEM_LMEM_BASE_ADDR) >> MEM_ASHIFT); `STATIC_ASSERT (MEM_ADDRW > LMEM_BLK_LSB, ("LMEM window wider than the block address space")) `endif - `UNUSED_PARAM (CORE_ID) +`ifndef VX_CFG_EXT_A_ENABLE + `UNUSED_VAR (core_id) +`endif `UNUSED_SPARAM (INSTANCE_ID) // tag_width = header + op_type + align + pkt_addr + fence @@ -141,7 +144,7 @@ module VX_lsu_slice import VX_gpu_pkg::*; #( `endif // make_hart_id(cid, wid, tid) — packed concatenation, low bits = tid. assign lane_amo.hart_id = HART_ID_WIDTH'( - (HART_ID_WIDTH'(CORE_ID) << (NW_BITS + NT_BITS)) + (HART_ID_WIDTH'(core_id) << (NW_BITS + NT_BITS)) | (HART_ID_WIDTH'(execute_if.data.header.wid) << NT_BITS) | HART_ID_WIDTH'(i) ); diff --git a/hw/rtl/core/VX_lsu_unit.sv b/hw/rtl/core/VX_lsu_unit.sv index 67e5d22eea..2623bc8162 100644 --- a/hw/rtl/core/VX_lsu_unit.sv +++ b/hw/rtl/core/VX_lsu_unit.sv @@ -14,14 +14,15 @@ `include "VX_define.vh" module VX_lsu_unit import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - parameter CORE_ID = 0 + parameter `STRING INSTANCE_ID = "" ) ( `SCOPE_IO_DECL input wire clk, input wire reset, + input wire [CORE_ID_WIDTH-1:0] core_id, + // Inputs VX_dispatch_if.slave dispatch_if [`VX_CFG_ISSUE_WIDTH], @@ -47,6 +48,7 @@ module VX_lsu_unit import VX_gpu_pkg::*; #( ) lane_dispatch ( .clk (clk), .reset (reset), + .port_base ('0), .dispatch_if(dispatch_if), .execute_if (per_block_execute_if) ); @@ -56,13 +58,13 @@ module VX_lsu_unit import VX_gpu_pkg::*; #( ) per_block_result_if[BLOCK_SIZE](); for (genvar block_idx = 0; block_idx < BLOCK_SIZE; ++block_idx) begin : g_blocks - VX_lsu_slice #( - .INSTANCE_ID (`SFORMATF(("%s%0d", INSTANCE_ID, block_idx))), - .CORE_ID (CORE_ID) - ) lsu_slice( + VX_lsu_block #( + .INSTANCE_ID (`SFORMATF(("%s%0d", INSTANCE_ID, block_idx))) + ) lsu_block( `SCOPE_IO_BIND (block_idx) .clk (clk), .reset (reset), + .core_id (core_id), .execute_if (per_block_execute_if[block_idx]), .result_if (per_block_result_if[block_idx]), .client_if (per_block_client_if[block_idx]) diff --git a/hw/rtl/core/VX_operands.sv b/hw/rtl/core/VX_operands.sv index 56e1ba7723..2c1f11b9bc 100644 --- a/hw/rtl/core/VX_operands.sv +++ b/hw/rtl/core/VX_operands.sv @@ -14,8 +14,7 @@ `include "VX_define.vh" module VX_operands import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - parameter ISSUE_ID = 0 + parameter `STRING INSTANCE_ID = "" ) ( input wire clk, input wire reset, @@ -28,7 +27,6 @@ module VX_operands import VX_gpu_pkg::*; #( VX_scoreboard_if.slave scoreboard_if, VX_operands_if.master operands_if ); - `UNUSED_SPARAM (ISSUE_ID) localparam OUT_DATAW = $bits(operands_t); diff --git a/hw/rtl/core/VX_scheduler.sv b/hw/rtl/core/VX_scheduler.sv index 66e02120de..c2d36fda22 100644 --- a/hw/rtl/core/VX_scheduler.sv +++ b/hw/rtl/core/VX_scheduler.sv @@ -13,559 +13,261 @@ `include "VX_define.vh" +// The core-level part of scheduling: CTA dispatch and the core-wide warp-control +// resources. Each fetch slice schedules and fetches its own warps (VX_warp_sched, +// VX_fetch); core-wide events are handed to it with slice-local warp ids. module VX_scheduler import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - parameter CORE_ID = 0 + parameter `STRING INSTANCE_ID = "" ) ( input wire clk, input wire reset, + input wire [CORE_ID_WIDTH-1:0] core_id, + `ifdef PERF_ENABLE output sched_perf_t sched_perf, `endif - // inputs - VX_warp_ctl_if.slave warp_ctl_if, -`ifdef VX_CFG_EXT_RTU_ENABLE - VX_sched_unlock_if.slave sched_unlock_if, // RTU TRACE wstall release + VX_slice_sched_if.master slice_if [`VX_CFG_FETCH_WIDTH], + VX_warp_shared_if.slave warp_shared_if [`VX_CFG_NUM_SFU_BLOCKS], +`ifdef VX_CFG_EXT_DXA_ENABLE + VX_txbar_bus_if.slave txbar_bus_if, `endif - VX_branch_ctl_if.slave branch_ctl_if [`VX_CFG_NUM_ALU_BLOCKS], - VX_decode_sched_if.slave decode_sched_if, - VX_issue_sched_if.slave issue_sched_if [`VX_CFG_ISSUE_WIDTH], - VX_commit_sched_if.slave commit_sched_if, + input wire [`VX_CFG_NUM_WARPS-1:0] committed_warps, + + // CTA context read by each SFU block's CSR unit, valid a cycle after the address + input wire [`VX_CFG_NUM_SFU_BLOCKS-1:0][NCTA_WIDTH-1:0] csr_rd_cta_id, + output cta_ctx_t [`VX_CFG_NUM_SFU_BLOCKS-1:0] csr_rd_ctx, - // KMU bus VX_kmu_bus_if.slave kmu_bus_if, - // outputs - VX_schedule_if.master schedule_if, - VX_sched_csr_if.master sched_csr_if, VX_gbar_bus_if.master gbar_bus_if, - // status + // per-core CSR state + output wire [PERF_CTR_BITS-1:0] cycles, + output wire [PERF_CTR_BITS-1:0] instret, + output wire [`VX_CFG_NUM_WARPS-1:0] active_warps, +`ifdef VX_CFG_VM_ENABLE + output wire [`VX_CFG_NUM_SFU_BLOCKS-1:0][`VX_CFG_XLEN-1:0] satp, + input wire [`VX_CFG_NUM_SFU_BLOCKS-1:0] satp_wr_valid, + input wire [`VX_CFG_NUM_SFU_BLOCKS-1:0][`VX_CFG_XLEN-1:0] satp_wr_data, +`endif + output wire busy ); `UNUSED_SPARAM (INSTANCE_ID) - `UNUSED_PARAM (CORE_ID) - - reg [`VX_CFG_NUM_WARPS-1:0] active_warps, active_warps_n; // updated when a warp is activated or disabled - reg [`VX_CFG_NUM_WARPS-1:0] stalled_warps, stalled_warps_n; // set when branch/gpgpu instructions are issued - - reg [`VX_CFG_NUM_WARPS-1:0][`VX_CFG_NUM_THREADS-1:0] thread_masks, thread_masks_n; - reg [`VX_CFG_NUM_WARPS-1:0][PC_BITS-1:0] warp_pcs, warp_pcs_n; - reg [`VX_CFG_NUM_WARPS-1:0][`VX_CFG_MEM_ADDR_WIDTH-1:0] mscratch_r; - - // Per-warp machine-mode trap CSRs. csrw writes arrive on - // sched_csr_if.trap_csr_wr_*; ECALL/EBREAK hardware-write mepc/mcause/ - // mtval; MRET restores the warp PC from mepc. - reg [`VX_CFG_NUM_WARPS-1:0][`VX_CFG_XLEN-1:0] mstatus_r, mtvec_r, mepc_r, mcause_r, mtval_r; - wire [NW_WIDTH-1:0] schedule_wid; - wire [`VX_CFG_NUM_THREADS-1:0] schedule_tmask; - wire [PC_BITS-1:0] schedule_pc; - wire schedule_valid; - wire schedule_ready; + localparam FW = `VX_CFG_FETCH_WIDTH; + localparam SW = PER_FETCH_WARPS; + + // slice status in wid order (wid = {wfs, fsw}) + wire [`VX_CFG_NUM_WARPS-1:0] stalled_warps; + wire [`VX_CFG_NUM_WARPS-1:0] ibuf_full_n, pending_warp_empty; + for (genvar w = 0; w < `VX_CFG_NUM_WARPS; ++w) begin : g_interleave + assign active_warps[w] = slice_if[w % FW].active[w / FW]; + assign stalled_warps[w] = slice_if[w % FW].stalled[w / FW]; + assign ibuf_full_n[w] = slice_if[w % FW].ibuf_full_n[w / FW]; + assign pending_warp_empty[w] = slice_if[w % FW].pending_empty[w / FW]; + end - // CTA dispatcher - wire cta_fire; - wire [NW_WIDTH-1:0] cta_wid; + // CTA dispatcher: lane p launches fetch slice p's warps + wire [FW-1:0] cta_fire; + wire [FW-1:0][NW_WIDTH-1:0] cta_wid; wire [PC_BITS-1:0] cta_PC; - wire [`VX_CFG_NUM_THREADS-1:0] cta_tmask; + wire [FW-1:0][`VX_CFG_NUM_THREADS-1:0] cta_tmask; wire [`VX_CFG_MEM_ADDR_WIDTH-1:0] cta_param; + wire [FW-1:0] cta_init; + cta_bcast_t cta_bcast; + cta_lane_bcast_t [FW-1:0] cta_lane_bcast; wire cta_dispatcher_busy; - wire cta_init; - - // CTA-CSR read-back from the dispatcher (it owns the per-CTA/per-warp tables). - cta_csrs_t cta_rd_csrs; - cta_lane_t [`VX_CFG_NUM_THREADS-1:0] cta_rd_lane; - wire [NCTA_WIDTH-1:0] schedule_cta_id; - - // Warp retirement: TMC with tmask==0 permanently deactivates the warp - wire cta_warp_done = warp_ctl_if.tmc_valid && (warp_ctl_if.tmc.tmask == 0); + wire cta_warp_done; + wire [NW_WIDTH-1:0] cta_warp_done_wid; VX_cta_dispatch #( - .INSTANCE_ID (`SFORMATF(("%s-cta_dispatch", INSTANCE_ID))) + .INSTANCE_ID (`SFORMATF(("%s-cta_dispatch", INSTANCE_ID))), + .NUM_LANES (FW) ) cta_dispatcher ( - .clk (clk), - .reset (reset), - .kmu_bus_if (kmu_bus_if), - .active_warps(active_warps), - .warp_done (cta_warp_done), - .warp_done_wid(warp_ctl_if.wid), - .cta_fire (cta_fire), - .cta_wid (cta_wid), - .cta_PC (cta_PC), - .cta_tmask (cta_tmask), - .cta_param (cta_param), - .cta_init (cta_init), - .csr_rd_wid (sched_csr_if.csr_rd_wid), - .csr_rd_cta_id(sched_csr_if.csr_rd_cta_id), - .cta_rd_csrs(cta_rd_csrs), - .cta_rd_lane(cta_rd_lane), - .schedule_wid(schedule_wid), - .schedule_cta_id(schedule_cta_id), - .busy (cta_dispatcher_busy) + .clk (clk), + .reset (reset), + .kmu_bus_if (kmu_bus_if), + .active_warps (active_warps), + .warp_done (cta_warp_done), + .warp_done_wid(cta_warp_done_wid), + .cta_fire (cta_fire), + .cta_wid (cta_wid), + .cta_PC (cta_PC), + .cta_tmask (cta_tmask), + .cta_param (cta_param), + .cta_init (cta_init), + .cta_bcast (cta_bcast), + .cta_lane_bcast(cta_lane_bcast), + .busy (cta_dispatcher_busy) ); - assign sched_csr_if.cta_csrs = cta_rd_csrs; - assign sched_csr_if.cta_lane = cta_rd_lane; - - assign sched_csr_if.mscratch = mscratch_r[sched_csr_if.csr_rd_wid]; - assign sched_csr_if.csr_mstatus = mstatus_r[sched_csr_if.csr_rd_wid]; - assign sched_csr_if.csr_mtvec = mtvec_r [sched_csr_if.csr_rd_wid]; - assign sched_csr_if.csr_mepc = mepc_r [sched_csr_if.csr_rd_wid]; - assign sched_csr_if.csr_mcause = mcause_r [sched_csr_if.csr_rd_wid]; - assign sched_csr_if.csr_mtval = mtval_r [sched_csr_if.csr_rd_wid]; - - // split/join - wire join_valid; - wire join_is_dvg; - wire join_is_else; - wire [NW_WIDTH-1:0] join_wid; - wire [`VX_CFG_NUM_THREADS-1:0] join_tmask; - wire [PC_BITS-1:0] join_pc; - - reg [PERF_CTR_BITS-1:0] cycles; - - wire schedule_fire = schedule_valid && schedule_ready; - wire schedule_if_fire = schedule_if.valid && schedule_if.ready; -`ifdef VX_CFG_EXT_C_ENABLE - // PC advance is driven by decompress_finished under EXT_C; - `UNUSED_VAR (schedule_if_fire) -`endif - - // branch - wire [`VX_CFG_NUM_ALU_BLOCKS-1:0] branch_valid; - wire [`VX_CFG_NUM_ALU_BLOCKS-1:0][NW_WIDTH-1:0] branch_wid; - wire [`VX_CFG_NUM_ALU_BLOCKS-1:0] branch_taken; - wire [`VX_CFG_NUM_ALU_BLOCKS-1:0][PC_BITS-1:0] branch_dest; - wire [`VX_CFG_NUM_ALU_BLOCKS-1:0] branch_is_trap; - wire [`VX_CFG_NUM_ALU_BLOCKS-1:0] branch_is_mret; - wire [`VX_CFG_NUM_ALU_BLOCKS-1:0][3:0] branch_trap_cause; - for (genvar i = 0; i < `VX_CFG_NUM_ALU_BLOCKS; ++i) begin : g_branch_init - assign branch_valid[i] = branch_ctl_if[i].valid; - assign branch_wid[i] = branch_ctl_if[i].wid; - assign branch_taken[i] = branch_ctl_if[i].taken; - assign branch_dest[i] = branch_ctl_if[i].dest; - assign branch_is_trap[i] = branch_ctl_if[i].is_trap; - assign branch_is_mret[i] = branch_ctl_if[i].is_mret; - assign branch_trap_cause[i] = branch_ctl_if[i].trap_cause; + // CTA contexts, one read port per SFU block + for (genvar b = 0; b < `VX_CFG_NUM_SFU_BLOCKS; ++b) begin : g_cta_ctx + VX_dp_ram #( + .DATAW ($bits(cta_ctx_t)), + .SIZE (NUM_CTA_MAX), + .RDW_MODE ("R"), + .OUT_REG (1) + ) cta_ctx_ram ( + .clk (clk), + .reset (reset), + .read (1'b1), + .write (cta_bcast.ctx_write), + .wren (1'b1), + .waddr (cta_bcast.ctx_waddr), + .wdata (cta_bcast.ctx_wdata), + .raddr (csr_rd_cta_id[b]), + .rdata (csr_rd_ctx[b]) + ); end - // barriers - wire [`VX_CFG_NUM_WARPS-1:0] bar_unlock_mask; + // core-wide warp-control resources wire bar_unlock_valid; - - // wspawn - wspawn_t wspawn; - reg wspawn_valid; - reg [NW_WIDTH-1:0] wspawn_wid; - reg is_single_warp; - - wire [`CLOG2(`VX_CFG_NUM_WARPS+1)-1:0] active_warps_cnt; - `POP_COUNT(active_warps_cnt, active_warps); - - always @(*) begin - active_warps_n = active_warps; - stalled_warps_n = stalled_warps; - thread_masks_n = thread_masks; - warp_pcs_n = warp_pcs; - - // dispatch warps - if (cta_fire) begin - active_warps_n[cta_wid] = 1; - // Reusing a warp for the next CTA skips the one-time prologue and rewinds to the - // kernel's per-CTA dispatch window: a fixed 20-byte (5-instruction) sequence that - // reloads the entry pointer and kargs before re-calling. - warp_pcs_n[cta_wid] = cta_init ? cta_PC : (warp_pcs[cta_wid] - from_fullPC(`VX_CFG_XLEN'(20))); - thread_masks_n[cta_wid] = cta_tmask; - end - - // decode unlock - if (decode_sched_if.valid && decode_sched_if.unlock) begin - stalled_warps_n[decode_sched_if.wid] = 0; - end - - // wspawn handling - if (wspawn_valid && is_single_warp) begin - active_warps_n |= wspawn.wmask; - for (integer i = 0; i < `VX_CFG_NUM_WARPS; ++i) begin - if (wspawn.wmask[i] && (NW_WIDTH'(i) != wspawn_wid)) begin - thread_masks_n[i][0] = 1; - warp_pcs_n[i] = wspawn.pc; - end - end - stalled_warps_n[wspawn_wid] = 0; // unlock warp - end - - // TMC handling - if (warp_ctl_if.tmc_valid) begin - active_warps_n[warp_ctl_if.wid] = (warp_ctl_if.tmc.tmask != 0); - thread_masks_n[warp_ctl_if.wid] = warp_ctl_if.tmc.tmask; - stalled_warps_n[warp_ctl_if.wid] = 0; // unlock warp - end - - // split handling - if (warp_ctl_if.split_valid) begin - if (warp_ctl_if.split.is_dvg) begin - thread_masks_n[warp_ctl_if.wid] = warp_ctl_if.split.then_tmask; - end - stalled_warps_n[warp_ctl_if.wid] = 0; // unlock warp - end - - // join handling - if (join_valid) begin - if (join_is_dvg) begin - if (join_is_else) begin - warp_pcs_n[join_wid] = join_pc; - end - thread_masks_n[join_wid] = join_tmask; - end - stalled_warps_n[join_wid] = 0; // unlock warp - end - - // barrier unlock handling - if (bar_unlock_valid) begin - stalled_warps_n &= ~bar_unlock_mask; - end - - // wsync unlock: warp pipeline drained - if (warp_ctl_if.wsync_valid) begin - stalled_warps_n[warp_ctl_if.wid] = 0; - end - - // Branch handling - for (integer i = 0; i < `VX_CFG_NUM_ALU_BLOCKS; ++i) begin - if (branch_valid[i]) begin - if (branch_is_trap[i]) begin - // ECALL/EBREAK: redirect to trap vector (mtvec[1:0] = MODE field; mask off to get base address). - warp_pcs_n[branch_wid[i]] = from_fullPC(mtvec_r[branch_wid[i]] & ~`VX_CFG_XLEN'(3)); - end else if (branch_is_mret[i]) begin - // MRET/SRET/URET: restore the saved PC from mepc. ECALL/EBREAK - // are the only traps and they do not narrow the tmask, so there - // is nothing to restore beyond the PC. - warp_pcs_n[branch_wid[i]] = from_fullPC(mepc_r[branch_wid[i]]); - end else if (branch_taken[i]) begin - warp_pcs_n[branch_wid[i]] = branch_dest[i]; - end - stalled_warps_n[branch_wid[i]] = 0; // unlock warp - end - end - - // stall the warp until decode stage - if (schedule_fire) begin - stalled_warps_n[schedule_wid] = 1; - end - - // advance PC. - `ifdef VX_CFG_EXT_C_ENABLE - // With RVC, the decompressor may emit a 2-byte instruction. Advance - // from the committed warp PC rather than the redirect-muxed next-PC: - // every redirect source (branch/trap/mret, split/join, wspawn) stalls - // its warp from schedule until it resolves, so no redirect writes a - // warp's PC on the same cycle that warp decode-advances. Reading the - // registered PC keeps the +2/+4 adder off the branch/trap redirect - // cone, matching the non-RVC path which advances the pipeline-carried - // PC instead of the combinational next-PC. - if (decode_sched_if.valid) begin - warp_pcs_n[decode_sched_if.wid] = - warp_pcs[decode_sched_if.wid] - + from_fullPC(decode_sched_if.is_rvc ? `VX_CFG_XLEN'(2) : `VX_CFG_XLEN'(4)); - end - `else - if (schedule_if_fire) begin - warp_pcs_n[schedule_if.data.wid] = schedule_if.data.PC + from_fullPC(`VX_CFG_XLEN'(4)); - end + wire [`VX_CFG_NUM_WARPS-1:0] bar_unlock_mask; + wire wspawn_apply; + wire [`VX_CFG_NUM_WARPS-1:0] wspawn_wmask; + wire [PC_BITS-1:0] wspawn_pc; + wire [NW_WIDTH-1:0] wspawn_wid; + wire [`VX_CFG_MEM_ADDR_WIDTH-1:0] wspawn_mscratch; + + VX_warp_shared_unit #( + .INSTANCE_ID (INSTANCE_ID), + .NUM_REQS (`VX_CFG_NUM_SFU_BLOCKS) + ) warp_shared_unit ( + .clk (clk), + .reset (reset), + .core_id (core_id), + .warp_shared_if (warp_shared_if), + `ifdef VX_CFG_EXT_DXA_ENABLE + .txbar_bus_if (txbar_bus_if), `endif + .active_warps (active_warps), + .bar_unlock_valid(bar_unlock_valid), + .bar_unlock_mask (bar_unlock_mask), + .gbar_bus_if (gbar_bus_if), + .wspawn_apply (wspawn_apply), + .wspawn_wmask (wspawn_wmask), + .wspawn_pc (wspawn_pc), + .wspawn_wid (wspawn_wid), + .wspawn_mscratch (wspawn_mscratch), + .warp_done (cta_warp_done), + .warp_done_wid (cta_warp_done_wid), + `UNUSED_PIN (wspawn_pending) + ); - `ifdef VX_CFG_EXT_RTU_ENABLE - // A wstall'd TRACE retires (its traversal's first response landed and the - // arm op wrote back the handle): resume the warp so it proceeds to WAIT, - // which returns the response status (terminal or candidate). No trap, no - // redirect — the candidate is serviced inline by the warp's loop. - if (sched_unlock_if.valid) begin - stalled_warps_n[sched_unlock_if.wid] = 1'b0; - end - `endif - end + // counters + reg [PERF_CTR_BITS-1:0] cycles_r; + reg [PERF_CTR_BITS-1:0] instret_r; + wire [`CLOG2(`VX_CFG_NUM_WARPS+1)-1:0] committed_warps_cnt; + `POP_COUNT(committed_warps_cnt, committed_warps); always @(posedge clk) begin if (reset) begin - stalled_warps <= '0; - warp_pcs <= '0; - active_warps <= '0; - thread_masks <= '0; - cycles <= '0; - wspawn_valid <= 0; - warp_pcs <= '0; - active_warps <= '0; - thread_masks <= '0; - is_single_warp <= 0; - mscratch_r <= '0; - mstatus_r <= '0; - mtvec_r <= '0; - mepc_r <= '0; - mcause_r <= '0; - mtval_r <= '0; + cycles_r <= '0; + instret_r <= '0; end else begin - active_warps <= active_warps_n; - stalled_warps <= stalled_warps_n; - thread_masks <= thread_masks_n; - warp_pcs <= warp_pcs_n; - is_single_warp <= (active_warps_cnt == $bits(active_warps_cnt)'(1)); - - // wspawn handling - if (warp_ctl_if.wspawn_valid) begin - wspawn_valid <= 1; - wspawn.wmask <= warp_ctl_if.wspawn.wmask; - wspawn.pc <= warp_ctl_if.wspawn.pc; - wspawn_wid <= warp_ctl_if.wid; - end - if (wspawn_valid && is_single_warp) begin - wspawn_valid <= 0; - // copy mscratch from spawning warp to all newly spawned warps - for (integer i = 0; i < `VX_CFG_NUM_WARPS; ++i) begin - if (wspawn.wmask[i] && (NW_WIDTH'(i) != wspawn_wid)) begin - mscratch_r[i] <= mscratch_r[wspawn_wid]; - end - end - end - - // CTA dispatch: latch this warp's mscratch (param). The per-CTA / - // per-warp tables and the wid->cta_id map live in VX_cta_dispatch. - if (cta_fire) begin - mscratch_r[cta_wid] <= cta_param; - end - - // MSCRATCH write-back from CSR unit (CSR instruction) - if (sched_csr_if.csr_wr_valid) begin - mscratch_r[sched_csr_if.csr_wr_wid] <= sched_csr_if.csr_wr_data; - end - - // Trap CSR write-back from CSR unit (csrw mstatus/mtvec/mepc/...) - if (sched_csr_if.trap_csr_wr_valid[0]) begin - mstatus_r[sched_csr_if.csr_wr_wid] <= sched_csr_if.trap_csr_wr_data; - end - if (sched_csr_if.trap_csr_wr_valid[1]) begin - mtvec_r [sched_csr_if.csr_wr_wid] <= sched_csr_if.trap_csr_wr_data; - end - if (sched_csr_if.trap_csr_wr_valid[2]) begin - mepc_r [sched_csr_if.csr_wr_wid] <= sched_csr_if.trap_csr_wr_data; - end - if (sched_csr_if.trap_csr_wr_valid[3]) begin - mcause_r [sched_csr_if.csr_wr_wid] <= sched_csr_if.trap_csr_wr_data; - end - if (sched_csr_if.trap_csr_wr_valid[4]) begin - mtval_r [sched_csr_if.csr_wr_wid] <= sched_csr_if.trap_csr_wr_data; - end - - // Hardware trap entry (ECALL/EBREAK): snapshot the faulting PC - // into mepc and the cause into mcause. Ordered after the - // software write so a hardware trap wins a same-cycle conflict. - for (integer i = 0; i < `VX_CFG_NUM_ALU_BLOCKS; ++i) begin - if (branch_valid[i] && branch_is_trap[i]) begin - mepc_r [branch_wid[i]] <= to_fullPC(branch_dest[i]); - mcause_r[branch_wid[i]] <= `VX_CFG_XLEN'(branch_trap_cause[i]); - mtval_r [branch_wid[i]] <= '0; - end - end - if (busy) begin - cycles <= cycles + 1; + cycles_r <= cycles_r + 1; end + instret_r <= instret_r + PERF_CTR_BITS'(committed_warps_cnt); end end + assign cycles = cycles_r; + assign instret = instret_r; - // Barrier unit - - VX_bar_unit #( - .INSTANCE_ID (`SFORMATF(("%s-barrier", INSTANCE_ID))), - .CORE_ID (CORE_ID) - ) bar_unit ( - .clk (clk), - .reset (reset), - .req_valid (warp_ctl_if.bar_valid), - .req_wid (warp_ctl_if.wid), - .req_data (warp_ctl_if.bar), - .read_addr (warp_ctl_if.bar_addr), - .read_phase (warp_ctl_if.bar_phase), - .active_warps(active_warps), - .gbar_bus_if(gbar_bus_if), - .unlock_valid(bar_unlock_valid), - .unlock_mask(bar_unlock_mask) - ); - - // split/join handling - - VX_split_join #( - .INSTANCE_ID (`SFORMATF(("%s-splitjoin", INSTANCE_ID))), - .OUT_REG (1) - ) split_join ( - .clk (clk), - .reset (reset), - .split_valid(warp_ctl_if.split_valid), - .sjoin_valid(warp_ctl_if.sjoin_valid), - .wid (warp_ctl_if.wid), - .split (warp_ctl_if.split), - .sjoin (warp_ctl_if.sjoin), - .join_valid (join_valid), - .join_is_dvg(join_is_dvg), - .join_is_else(join_is_else), - .join_wid (join_wid), - .join_tmask (join_tmask), - .join_pc (join_pc), - .stack_wid (warp_ctl_if.dvstack_wid), - .stack_ptr (warp_ctl_if.dvstack_ptr) - ); - - // schedule the next ready warp - - wire [`VX_CFG_NUM_WARPS-1:0] ready_warps = active_warps & ~stalled_warps; - - // Per-warp ibuffer occupancy counter (registered full[i] keeps arbitration - // off the critical path; full_n feeds an externally registered aggregate - // so all_full is valid the same cycle as full[i]). - localparam IBUF_CW = $clog2(`VX_CFG_IBUF_SIZE + 1); - - wire [`VX_CFG_NUM_WARPS-1:0] schedule_onehot; - logic [`VX_CFG_NUM_WARPS-1:0] ibuf_full, ibuf_full_n; - - for (genvar i = 0; i < `VX_CFG_NUM_WARPS; ++i) begin : g_ibuf_cnt - logic [IBUF_CW-1:0] size_r, size_n; - wire incr = schedule_fire && schedule_onehot[i]; - wire decr = schedule_if.ibuf_pop[i]; - assign size_n = size_r + IBUF_CW'(incr) - IBUF_CW'(decr); - assign ibuf_full_n[i] = (size_n == IBUF_CW'(`VX_CFG_IBUF_SIZE)); - always @(posedge clk) begin - if (reset) begin - size_r <= '0; - ibuf_full[i] <= 1'b0; - end else begin - size_r <= size_n; - ibuf_full[i] <= ibuf_full_n[i]; - end + // from the slices' next-state full, so it is valid the same cycle as their full flags + reg all_ibuf_full; + always @(posedge clk) begin + if (reset) begin + all_ibuf_full <= 1'b0; + end else begin + all_ibuf_full <= (& ibuf_full_n); end end - wire [`VX_CFG_NUM_WARPS-1:0] preferred_warps = ready_warps & ~ibuf_full; -`ifndef L1_ENABLE - // without L1, we should ensure the icache never stalls, - // because it could deadlock dcache response since they share the same bus. - wire [`VX_CFG_NUM_WARPS-1:0] schedule_warps = preferred_warps; -`else - reg all_ibuf_full; +`ifdef VX_CFG_VM_ENABLE + // Per-core SATP CSR, written by any SFU block's CSR unit. Accesses in one + // cycle take effect in block order: each block reads the value the lower + // blocks leave. + reg [`VX_CFG_XLEN-1:0] satp_r; + wire [`VX_CFG_NUM_SFU_BLOCKS:0][`VX_CFG_XLEN-1:0] satp_chain; + assign satp_chain[0] = satp_r; + for (genvar b = 0; b < `VX_CFG_NUM_SFU_BLOCKS; ++b) begin : g_satp + assign satp[b] = satp_chain[b]; + assign satp_chain[b+1] = satp_wr_valid[b] ? satp_wr_data[b] : satp_chain[b]; + end always @(posedge clk) begin - if (reset) all_ibuf_full <= 1'b0; - else all_ibuf_full <= (& ibuf_full_n); + if (reset) begin + satp_r <= '0; + end else begin + satp_r <= satp_chain[`VX_CFG_NUM_SFU_BLOCKS]; + end end - wire [`VX_CFG_NUM_WARPS-1:0] schedule_warps = all_ibuf_full ? ready_warps : preferred_warps; `endif - VX_priority_encoder #( - .N (`VX_CFG_NUM_WARPS) - ) wid_select ( - .data_in (schedule_warps), - .index_out (schedule_wid), - .valid_out (schedule_valid), - .onehot_out(schedule_onehot) - ); + // per-slice view of the core-wide events - wire [`VX_CFG_NUM_WARPS-1:0][(`VX_CFG_NUM_THREADS + PC_BITS)-1:0] schedule_data; - for (genvar i = 0; i < `VX_CFG_NUM_WARPS; ++i) begin : g_schedule_data - assign schedule_data[i] = {thread_masks[i], warp_pcs[i]}; - end + wire [FETCH_FSW_W-1:0] wspawn_fsw = wid_to_fsw(wspawn_wid); - assign {schedule_tmask, schedule_pc} = { - schedule_data[schedule_wid][(`VX_CFG_NUM_THREADS + PC_BITS)-1:(`VX_CFG_NUM_THREADS + PC_BITS)-4], - schedule_data[schedule_wid][(`VX_CFG_NUM_THREADS + PC_BITS)-5:0] - }; - - wire [UUID_WIDTH-1:0] instr_uuid; -`ifdef UUID_ENABLE - VX_uuid_gen #( - .CORE_ID (CORE_ID) - ) uuid_gen ( - .clk (clk), - .reset (reset), - .incr (schedule_fire), - .wid (schedule_wid), - .uuid (instr_uuid) - ); -`else - assign instr_uuid = '0; -`endif + for (genvar p = 0; p < FW; ++p) begin : g_slices + localparam [FETCH_FSW_W-1:0] FSW = FETCH_FSW_W'(p); - // schedule_cta_id is produced by VX_cta_dispatch from its wid->cta_id map. - - VX_elastic_buffer #( - .DATAW (`VX_CFG_NUM_THREADS + PC_BITS + NW_WIDTH + NCTA_WIDTH + UUID_WIDTH), - .SIZE (2), // need to buffer out ready_in - .OUT_REG (1) // should be registered for BRAM acces in fetch unit - ) out_buf ( - .clk (clk), - .reset (reset), - .valid_in (schedule_valid), - .ready_in (schedule_ready), - .data_in ({schedule_tmask, schedule_pc, schedule_wid, schedule_cta_id, instr_uuid}), - .data_out ({schedule_if.data.tmask, schedule_if.data.PC, schedule_if.data.wid, schedule_if.data.cta_id, schedule_if.data.uuid}), - .valid_out (schedule_if.valid), - .ready_out (schedule_if.ready) - ); + assign slice_if[p].cta_fire = cta_fire[p]; + assign slice_if[p].cta_lid = wid_to_wfs(cta_wid[p]); + assign slice_if[p].cta_PC = cta_PC; + assign slice_if[p].cta_tmask = cta_tmask[p]; + assign slice_if[p].cta_param = cta_param; + assign slice_if[p].cta_init = cta_init[p]; - // Track committed instructions + assign slice_if[p].warp_write = cta_lane_bcast[p].warp_write; + assign slice_if[p].warp_waddr = wid_to_wfs(cta_lane_bcast[p].warp_waddr); + assign slice_if[p].warp_wdata = cta_lane_bcast[p].warp_wdata; + assign slice_if[p].id_write = cta_lane_bcast[p].id_write; + assign slice_if[p].id_waddr = wid_to_wfs(cta_lane_bcast[p].id_wid); + assign slice_if[p].id_wdata = cta_lane_bcast[p].id_cta; + `UNUSED_VAR ({wid_to_fsw(cta_wid[p]), wid_to_fsw(cta_lane_bcast[p].warp_waddr), wid_to_fsw(cta_lane_bcast[p].id_wid)}) - reg [PERF_CTR_BITS-1:0] instret; + assign slice_if[p].wspawn_apply = wspawn_apply; + assign slice_if[p].wspawn_pc = wspawn_pc; + assign slice_if[p].wspawn_src = (wspawn_fsw == FSW); + assign slice_if[p].wspawn_src_lid = wid_to_wfs(wspawn_wid); + assign slice_if[p].wspawn_mscratch = wspawn_mscratch; - wire [`VX_CFG_NUM_WARPS-1:0] committed_warps_v = commit_sched_if.committed_warps; - wire [`CLOG2(`VX_CFG_NUM_WARPS+1)-1:0] committed_warps_cnt_v; - `POP_COUNT(committed_warps_cnt_v, committed_warps_v); + assign slice_if[p].bar_unlock_valid = bar_unlock_valid; - always @(posedge clk) begin - if (reset) begin - instret <= '0; - end else begin - instret <= instret + PERF_CTR_BITS'(committed_warps_cnt_v); + assign slice_if[p].retire_valid = cta_warp_done && (wid_to_fsw(cta_warp_done_wid) == FSW); + assign slice_if[p].retire_lid = wid_to_wfs(cta_warp_done_wid); + + for (genvar l = 0; l < SW; ++l) begin : g_lw + assign slice_if[p].wspawn_wmask[l] = wspawn_wmask[l * FW + p]; + assign slice_if[p].bar_unlock_mask[l] = bar_unlock_mask[l * FW + p]; end + + assign slice_if[p].all_ibuf_full = all_ibuf_full; end - // Track pending instructions per warp - - wire [`VX_CFG_NUM_WARPS-1:0] pending_warp_empty; - wire [`VX_CFG_NUM_WARPS-1:0] pending_warp_alm_empty; - - for (genvar i = 0; i < `VX_CFG_NUM_WARPS; ++i) begin : g_pending_warps - localparam logic [ISSUE_ISW_W-1:0] isw = wid_to_isw(i); - localparam logic [ISSUE_WIS_W-1:0] wis = wid_to_wis(i); - - VX_pending_size #( - .SIZE (256), - .ALM_EMPTY (1) - ) per_warp_ctr ( - .clk (clk), - .reset (reset), - .incr (issue_sched_if[isw].valid && (issue_sched_if[isw].wis == ISSUE_WIS_W'(wis))), - .decr (commit_sched_if.committed_warps[i]), - .empty (pending_warp_empty[i]), - .alm_empty (pending_warp_alm_empty[i]), - `UNUSED_PIN (full), - `UNUSED_PIN (alm_full), - `UNUSED_PIN (size) - ); + wire [FW-1:0] sched_valid, sched_fire; + wire [FW-1:0][`VX_CFG_NUM_THREADS-1:0] sched_tmask; + for (genvar p = 0; p < FW; ++p) begin : g_sched_status + assign sched_valid[p] = slice_if[p].sched_valid; + assign sched_fire[p] = slice_if[p].sched_fire; + assign sched_tmask[p] = slice_if[p].sched_tmask; end +`ifndef PERF_ENABLE + `UNUSED_VAR ({sched_valid, sched_fire, sched_tmask}) +`endif + // active_warps is the slices' registered state, so a CTA launch shows there one cycle after + // cta_fire; counting the launch itself keeps busy from dropping between the dispatcher going + // idle and the launched warps becoming visible (a CTA launched in a single cycle). wire busy_buf; - `BUFFER_EX(busy_buf, (active_warps_n != 0 || ~(&pending_warp_empty)), 1'b1, 1, 1); + `BUFFER_EX(busy_buf, (active_warps != 0 || (| cta_fire) || ~(&pending_warp_empty)), 1'b1, 1, 1); assign busy = busy_buf || cta_dispatcher_busy; - assign warp_ctl_if.warp_pending_alm_empty = pending_warp_alm_empty; - - // export CSRs - assign sched_csr_if.cycles = cycles; - assign sched_csr_if.instret = instret; - assign sched_csr_if.active_warps = active_warps; - assign sched_csr_if.thread_masks = thread_masks; - - // timeout handling + // timeout handling reg [31:0] timeout_ctr; reg timeout_enable; always @(posedge clk) begin @@ -573,7 +275,7 @@ module VX_scheduler import VX_gpu_pkg::*; #( timeout_ctr <= '0; timeout_enable <= 0; end else begin - if (decode_sched_if.valid && decode_sched_if.unlock) begin + if (| cta_fire) begin timeout_enable <= 1; end if (timeout_enable && active_warps !=0 && active_warps == stalled_warps) begin @@ -590,15 +292,15 @@ module VX_scheduler import VX_gpu_pkg::*; #( localparam SCHED_STALL_TIMEOUT = STALL_TIMEOUT; `endif `ifdef EXT_SCHED_TIMEOUT_DUMP + wire timeout_dump = !reset && (timeout_ctr == (SCHED_STALL_TIMEOUT - 1)); always @(posedge clk) begin - if (!reset && (timeout_ctr == (SCHED_STALL_TIMEOUT - 1))) begin + if (timeout_dump) begin $display("*** %s scheduler-timeout dump: active=%b stalled=%b", INSTANCE_ID, active_warps, stalled_warps); - for (integer wi = 0; wi < `VX_CFG_NUM_WARPS; ++wi) begin - $display(" wid=%0d stalled=%0d pc=0x%0h tmask=%b", - wi, stalled_warps[wi], to_fullPC(warp_pcs[wi]), thread_masks[wi]); - end end end + for (genvar p = 0; p < FW; ++p) begin : g_timeout_dump + assign slice_if[p].timeout_dump = timeout_dump; + end `endif `RUNTIME_ASSERT(timeout_ctr < SCHED_STALL_TIMEOUT, ("*** %s timeout: active_warps=%b, stalled_warps=%b", INSTANCE_ID, active_warps, stalled_warps)) @@ -611,17 +313,34 @@ module VX_scheduler import VX_gpu_pkg::*; #( reg [PERF_CTR_BITS-1:0] perf_branches; reg [PERF_CTR_BITS-1:0] perf_divergence; - wire [`CLOG2(`VX_CFG_NUM_WARPS+1)-1:0] stalled_warps_cnt; - wire [`CLOG2(`VX_CFG_NUM_ALU_BLOCKS+1)-1:0] branches_cnt; - wire [`CLOG2(`VX_CFG_NUM_THREADS+1)-1:0] issued_threads_cnt; + localparam NPA = FETCH_ALU_PORTS; + localparam NPS = FETCH_SFU_PORTS; - wire schedule_idle = ~schedule_valid; - wire has_divergence = warp_ctl_if.split_valid && warp_ctl_if.split.is_dvg; - wire [`VX_CFG_NUM_THREADS-1:0] issued_threads = {`VX_CFG_NUM_THREADS{schedule_if_fire}} & schedule_if.data.tmask; + wire [FW-1:0][NPA-1:0] slice_branch; + wire [FW-1:0][NPS-1:0] slice_divergence; + for (genvar p = 0; p < FW; ++p) begin : g_perf_slices + assign slice_branch[p] = slice_if[p].branch_valid; + assign slice_divergence[p] = slice_if[p].divergence; + end + wire [`CLOG2(`VX_CFG_NUM_WARPS+1)-1:0] active_warps_cnt, stalled_warps_cnt; + wire [`CLOG2(FW*NPA+1)-1:0] branches_cnt; + wire [`CLOG2(FW*NPS+1)-1:0] divergence_cnt; + wire [`CLOG2(FW+1)-1:0] issued_warps_cnt; + wire [`CLOG2(FW*`VX_CFG_NUM_THREADS+1)-1:0] issued_threads_cnt; + wire [FW-1:0][`VX_CFG_NUM_THREADS-1:0] issued_threads; + for (genvar p = 0; p < FW; ++p) begin : g_issued_threads + assign issued_threads[p] = {`VX_CFG_NUM_THREADS{sched_fire[p]}} & sched_tmask[p]; + end + + `POP_COUNT(active_warps_cnt, active_warps); `POP_COUNT(stalled_warps_cnt, stalled_warps); `POP_COUNT(issued_threads_cnt, issued_threads); - `POP_COUNT(branches_cnt, branch_valid); + `POP_COUNT(issued_warps_cnt, sched_fire); + `POP_COUNT(branches_cnt, slice_branch); + `POP_COUNT(divergence_cnt, slice_divergence); + + wire sched_idle = ~(| sched_valid); always @(posedge clk) begin if (reset) begin @@ -633,13 +352,13 @@ module VX_scheduler import VX_gpu_pkg::*; #( perf_branches <= '0; perf_divergence <= '0; end else begin - perf_sched_idles <= perf_sched_idles + PERF_CTR_BITS'(schedule_idle); + perf_sched_idles <= perf_sched_idles + PERF_CTR_BITS'(sched_idle); perf_active_warps <= perf_active_warps + PERF_CTR_BITS'(active_warps_cnt); perf_stalled_warps <= perf_stalled_warps + PERF_CTR_BITS'(stalled_warps_cnt); - perf_issued_warps <= perf_issued_warps + PERF_CTR_BITS'(schedule_if_fire); + perf_issued_warps <= perf_issued_warps + PERF_CTR_BITS'(issued_warps_cnt); perf_issued_threads<= perf_issued_threads + PERF_CTR_BITS'(issued_threads_cnt); perf_branches <= perf_branches + PERF_CTR_BITS'(branches_cnt); - perf_divergence <= perf_divergence + PERF_CTR_BITS'(has_divergence); + perf_divergence <= perf_divergence + PERF_CTR_BITS'(divergence_cnt); end end @@ -650,25 +369,9 @@ module VX_scheduler import VX_gpu_pkg::*; #( assign sched_perf.issued_threads= perf_issued_threads; assign sched_perf.branches = perf_branches; assign sched_perf.divergence = perf_divergence; -`endif - -`ifdef DBG_TRACE_PIPELINE - for (genvar w = 0; w < `VX_CFG_NUM_WARPS; ++w) begin : g_trace_warp_status - always @(posedge clk) begin - if (active_warps_n[w] != active_warps[w] - || (active_warps[w] && (stalled_warps_n[w] != stalled_warps[w] - || thread_masks_n[w] != thread_masks[w]))) begin - `TRACE(1, ("%t: %s warp-state: wid=%0d, active=%b, stalled=%b, tmask=%b\n", - $time, INSTANCE_ID, w, active_warps_n[w], stalled_warps_n[w], thread_masks_n[w] - )) - end - end - end - - always @(posedge clk) begin - if (schedule_fire) begin - `TRACE(1, ("%t: %s dispatch: wid=%0d, cta_id=%0d, PC=0x%0h, tmask=%b (#%0d)\n", $time, INSTANCE_ID, schedule_wid, schedule_cta_id, to_fullPC(schedule_pc), schedule_tmask, instr_uuid)) - end +`else + for (genvar p = 0; p < FW; ++p) begin : g_perf_unused + `UNUSED_VAR ({slice_if[p].branch_valid, slice_if[p].divergence}) end `endif diff --git a/hw/rtl/core/VX_scoreboard.sv b/hw/rtl/core/VX_scoreboard.sv index 54f9f1e6e8..922ef621d9 100644 --- a/hw/rtl/core/VX_scoreboard.sv +++ b/hw/rtl/core/VX_scoreboard.sv @@ -14,8 +14,7 @@ `include "VX_define.vh" module VX_scoreboard import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - parameter ISSUE_ID = 0 + parameter `STRING INSTANCE_ID = "" ) ( input wire clk, input wire reset, @@ -30,7 +29,6 @@ module VX_scoreboard import VX_gpu_pkg::*; #( VX_scoreboard_if.master scoreboard_if ); `UNUSED_SPARAM (INSTANCE_ID) - `UNUSED_PARAM (ISSUE_ID) `UNUSED_VAR ({writeback_if.data.sop, writeback_if.data.eop, writeback_if.data.wis}) localparam NUM_OPDS = NUM_SRC_OPDS + 1; diff --git a/hw/rtl/core/VX_sfu_block.sv b/hw/rtl/core/VX_sfu_block.sv new file mode 100644 index 0000000000..15ce7ad91d --- /dev/null +++ b/hw/rtl/core/VX_sfu_block.sv @@ -0,0 +1,231 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +`ifdef VX_CFG_EXT_F_ENABLE +`include "VX_fpu_define.vh" +`endif + +// One SFU block: the PEs serving the warps with wid mod NUM_SFU_BLOCKS == block_idx. +module VX_sfu_block import VX_gpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "" +) ( + input wire clk, + input wire reset, + + input wire [CORE_ID_WIDTH-1:0] core_id, + input wire [SFU_BLOCK_W-1:0] block_idx, + + VX_execute_if.slave execute_if, + VX_result_if.master result_if, + +`ifdef VX_CFG_EXT_F_ENABLE + output wire [NW_WIDTH-1:0] fcsr_wid, + input wire [INST_FRM_BITS+`FP_FLAGS_BITS-1:0] fcsr_rd_data, + output wire [2:0] fcsr_wr_sel, + output wire [INST_FRM_BITS+`FP_FLAGS_BITS-1:0] fcsr_wr_data, +`endif + +`ifdef VX_CFG_EXT_DXA_ENABLE + VX_dxa_req_bus_if.master dxa_req_bus_if, +`endif + +`ifdef VX_CFG_EXT_TEX_ENABLE + VX_tex_bus_if.master tex_bus_if, +`endif + +`ifdef VX_CFG_EXT_RTU_ENABLE + VX_rtu_bus_if.master rtu_bus_if, + VX_sched_unlock_if.master sched_unlock_if, +`endif + + VX_sched_csr_if.slave sched_csr_if, + VX_warp_shared_if.master warp_shared_if, + VX_warp_ctl_if.master warp_ctl_if +); + localparam NUM_LANES = `VX_CFG_NUM_SFU_LANES; +`ifndef VX_CFG_EXT_RTU_ENABLE + `UNUSED_VAR (block_idx) +`endif + localparam PE_COUNT = 2 + `VX_CFG_EXT_DXA_ENABLED + `VX_CFG_EXT_TEX_ENABLED + `VX_CFG_EXT_OM_ENABLED + `VX_CFG_EXT_RASTER_ENABLED + `VX_CFG_EXT_RTU_ENABLED; + localparam PE_SEL_BITS = `CLOG2(PE_COUNT); + localparam PE_IDX_WCTL = 0; + localparam PE_IDX_CSRS = 1; +`ifdef VX_CFG_EXT_DXA_ENABLE + localparam PE_IDX_DXA = 2; +`endif +`ifdef VX_CFG_EXT_TEX_ENABLE + localparam PE_IDX_TEX = 2 + `VX_CFG_EXT_DXA_ENABLED; +`endif +`ifdef VX_CFG_EXT_OM_ENABLE + localparam PE_IDX_OM = 2 + `VX_CFG_EXT_DXA_ENABLED + `VX_CFG_EXT_TEX_ENABLED; +`endif +`ifdef VX_CFG_EXT_RASTER_ENABLE + localparam PE_IDX_RASTER = 2 + `VX_CFG_EXT_DXA_ENABLED + `VX_CFG_EXT_TEX_ENABLED + `VX_CFG_EXT_OM_ENABLED; +`endif +`ifdef VX_CFG_EXT_RTU_ENABLE + localparam PE_IDX_RTUW = 2 + `VX_CFG_EXT_DXA_ENABLED + `VX_CFG_EXT_TEX_ENABLED + `VX_CFG_EXT_OM_ENABLED + `VX_CFG_EXT_RASTER_ENABLED; +`endif + + VX_execute_if #( + .data_t (sfu_execute_t) + ) pe_execute_if[PE_COUNT](); + + VX_result_if#( + .data_t (sfu_result_t) + ) pe_result_if[PE_COUNT](); + + reg [PE_SEL_BITS-1:0] pe_select; + always @(*) begin + pe_select = PE_SEL_BITS'(PE_IDX_WCTL); + if (inst_sfu_is_csr(execute_if.data.op_type)) begin + pe_select = PE_SEL_BITS'(PE_IDX_CSRS); + end + `ifdef VX_CFG_EXT_DXA_ENABLE + if (execute_if.data.op_type == INST_SFU_DXA) begin + pe_select = PE_SEL_BITS'(PE_IDX_DXA); + end + `endif + `ifdef VX_CFG_EXT_TEX_ENABLE + if (execute_if.data.op_type == INST_SFU_TEX) begin + pe_select = PE_SEL_BITS'(PE_IDX_TEX); + end + `endif + `ifdef VX_CFG_EXT_OM_ENABLE + if (execute_if.data.op_type == INST_SFU_OM) begin + pe_select = PE_SEL_BITS'(PE_IDX_OM); + end + `endif + `ifdef VX_CFG_EXT_RASTER_ENABLE + if (execute_if.data.op_type == INST_SFU_RASTER) begin + pe_select = PE_SEL_BITS'(PE_IDX_RASTER); + end + `endif + `ifdef VX_CFG_EXT_RTU_ENABLE + if (execute_if.data.op_type == INST_SFU_RTUW) begin + pe_select = PE_SEL_BITS'(PE_IDX_RTUW); + end + `endif + end + + VX_pe_switch #( + .PE_COUNT (PE_COUNT), + .NUM_LANES (NUM_LANES), + .ARBITER ("R"), + .REQ_OUT_BUF(0), + .RSP_OUT_BUF(3) + ) pe_switch ( + .clk (clk), + .reset (reset), + .pe_sel (pe_select), + .execute_in_if (execute_if), + .result_out_if (result_if), + .execute_out_if (pe_execute_if), + .result_in_if (pe_result_if) + ); + + VX_wctl_unit #( + .INSTANCE_ID (`SFORMATF(("%s-wctl", INSTANCE_ID))), + .NUM_LANES (NUM_LANES) + ) wctl_unit ( + .clk (clk), + .reset (reset), + .execute_if (pe_execute_if[PE_IDX_WCTL]), + .warp_shared_if (warp_shared_if), + .warp_ctl_if (warp_ctl_if), + .result_if (pe_result_if[PE_IDX_WCTL]) + ); + + VX_csr_unit #( + .INSTANCE_ID (`SFORMATF(("%s-csr", INSTANCE_ID))), + .NUM_LANES (NUM_LANES) + ) csr_unit ( + .clk (clk), + .reset (reset), + .core_id (core_id), + .execute_if (pe_execute_if[PE_IDX_CSRS]), + `ifdef VX_CFG_EXT_F_ENABLE + .fcsr_wid (fcsr_wid), + .fcsr_rd_data (fcsr_rd_data), + .fcsr_wr_sel (fcsr_wr_sel), + .fcsr_wr_data (fcsr_wr_data), + `endif + .sched_csr_if (sched_csr_if), + .result_if (pe_result_if[PE_IDX_CSRS]) + ); + +`ifdef VX_CFG_EXT_DXA_ENABLE + VX_dxa_unit #( + .INSTANCE_ID (`SFORMATF(("%s-dxa", INSTANCE_ID))) + ) dxa_unit ( + .clk (clk), + .reset (reset), + .core_id (core_id), + .execute_if (pe_execute_if[PE_IDX_DXA]), + .result_if (pe_result_if[PE_IDX_DXA]), + .dxa_req_bus_if (dxa_req_bus_if) + ); +`endif + +`ifdef VX_CFG_EXT_TEX_ENABLE + VX_tex_unit #( + .INSTANCE_ID (`SFORMATF(("%s-tex", INSTANCE_ID))), + .NUM_LANES (NUM_LANES) + ) tex_unit ( + .clk (clk), + .reset (reset), + .execute_if (pe_execute_if[PE_IDX_TEX]), + .result_if (pe_result_if[PE_IDX_TEX]), + .tex_bus_if (tex_bus_if) + ); +`endif + +`ifdef VX_CFG_EXT_OM_ENABLE + // A fragment leaves the shader as a store to the OM aperture through the LSU, + // so the SFU services no OM op; the PE slot is kept for index stability. + assign pe_execute_if[PE_IDX_OM].ready = 1'b1; + assign pe_result_if[PE_IDX_OM].valid = 1'b0; + assign pe_result_if[PE_IDX_OM].data = '0; + `UNUSED_VAR (pe_execute_if[PE_IDX_OM].valid) + `UNUSED_VAR (pe_execute_if[PE_IDX_OM].data) +`endif + +`ifdef VX_CFG_EXT_RASTER_ENABLE + // The fragment payload reaches the warp at launch, so the SFU services no + // raster op; the PE slot is kept for index stability. + assign pe_execute_if[PE_IDX_RASTER].ready = 1'b1; + assign pe_result_if[PE_IDX_RASTER].valid = 1'b0; + assign pe_result_if[PE_IDX_RASTER].data = '0; + `UNUSED_VAR (pe_execute_if[PE_IDX_RASTER].valid) + `UNUSED_VAR (pe_execute_if[PE_IDX_RASTER].data) + `UNUSED_VAR (pe_result_if[PE_IDX_RASTER].ready) +`endif + +`ifdef VX_CFG_EXT_RTU_ENABLE + VX_rtu_unit #( + .INSTANCE_ID (`SFORMATF(("%s-rtuw", INSTANCE_ID))), + .RTU_TAG_WIDTH (RTU_REQ_TAG_WIDTH), + .NUM_LANES (NUM_LANES) + ) rtu_unit ( + .clk (clk), + .reset (reset), + .block_idx (block_idx), + .execute_if (pe_execute_if[PE_IDX_RTUW]), + .result_if (pe_result_if[PE_IDX_RTUW]), + .rtu_bus_if (rtu_bus_if), + .sched_unlock_if (sched_unlock_if) + ); +`endif + +endmodule diff --git a/hw/rtl/core/VX_sfu_shared.sv b/hw/rtl/core/VX_sfu_shared.sv new file mode 100644 index 0000000000..2d8cfb9c64 --- /dev/null +++ b/hw/rtl/core/VX_sfu_shared.sv @@ -0,0 +1,156 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// Merges the SFU blocks' requests onto the core's single DXA, TEX and RTU buses. +module VX_sfu_shared import VX_gpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "" +) ( +`ifdef VX_CFG_EXT_DXA_ENABLE + VX_dxa_req_bus_if.slave block_dxa_req_bus_if [`VX_CFG_NUM_SFU_BLOCKS], + VX_dxa_req_bus_if.master dxa_req_bus_if, +`endif + +`ifdef VX_CFG_EXT_TEX_ENABLE + VX_tex_bus_if.slave block_tex_bus_if [`VX_CFG_NUM_SFU_BLOCKS], + VX_tex_bus_if.master tex_bus_if, +`endif + +`ifdef VX_CFG_EXT_RTU_ENABLE + VX_rtu_bus_if.slave block_rtu_bus_if [`VX_CFG_NUM_SFU_BLOCKS], + VX_rtu_bus_if.master rtu_bus_if, +`endif + + input wire clk, + input wire reset +); + `UNUSED_SPARAM (INSTANCE_ID) + + localparam NB = `VX_CFG_NUM_SFU_BLOCKS; + // the merged request leaves the core: register it when it is a real merge + localparam REQ_OUT_BUF = (NB > 1) ? 3 : 0; + +`ifndef VX_CFG_EXT_DXA_ENABLE +`ifndef VX_CFG_EXT_TEX_ENABLE +`ifndef VX_CFG_EXT_RTU_ENABLE + `UNUSED_VAR ({clk, reset}) + `UNUSED_PARAM (REQ_OUT_BUF) +`endif +`endif +`endif + +`ifdef VX_CFG_EXT_DXA_ENABLE + VX_dxa_req_bus_if dxa_arb_out_if[1](); + + VX_dxa_req_arb #( + .NUM_INPUTS (NB), + .NUM_OUTPUTS (1), + .OUT_BUF (REQ_OUT_BUF), + .ARBITER ("R") + ) dxa_arb ( + .clk (clk), + .reset (reset), + .bus_in_if (block_dxa_req_bus_if), + .bus_out_if (dxa_arb_out_if) + ); + + assign dxa_req_bus_if.req_valid = dxa_arb_out_if[0].req_valid; + assign dxa_req_bus_if.req_data = dxa_arb_out_if[0].req_data; + assign dxa_arb_out_if[0].req_ready = dxa_req_bus_if.req_ready; +`endif + +`ifdef VX_CFG_EXT_TEX_ENABLE + VX_tex_bus_if #( + .NUM_LANES (`VX_CFG_NUM_SFU_LANES), + .TAG_WIDTH (TEX_CORE_TAG_WIDTH) + ) tex_arb_out_if[1](); + + VX_tex_bus_arb #( + .NUM_INPUTS (NB), + .NUM_OUTPUTS (1), + .NUM_LANES (`VX_CFG_NUM_SFU_LANES), + .TAG_WIDTH (TEX_REQ_TAG_WIDTH), + .ARBITER ("R"), + .OUT_BUF_REQ (REQ_OUT_BUF), + .OUT_BUF_RSP (0) + ) tex_arb ( + .clk (clk), + .reset (reset), + .bus_in_if (block_tex_bus_if), + .bus_out_if (tex_arb_out_if) + ); + + assign tex_bus_if.req_valid = tex_arb_out_if[0].req_valid; + assign tex_bus_if.req_data = tex_arb_out_if[0].req_data; + assign tex_arb_out_if[0].req_ready = tex_bus_if.req_ready; + assign tex_arb_out_if[0].rsp_valid = tex_bus_if.rsp_valid; + assign tex_arb_out_if[0].rsp_data = tex_bus_if.rsp_data; + assign tex_bus_if.rsp_ready = tex_arb_out_if[0].rsp_ready; +`endif + +`ifdef VX_CFG_EXT_RTU_ENABLE + localparam RTU_SEL_W = `UP(`ARB_SEL_BITS(NB, 1)); + + VX_rtu_bus_if #( + .NUM_LANES (`VX_CFG_NUM_SFU_LANES), + .TAG_WIDTH (RTU_CORE_TAG_WIDTH), + .SRC_WIDTH (RTU_SEL_W) + ) rtu_arb_out_if[1](); + + VX_rtu_bus_arb #( + .NUM_INPUTS (NB), + .NUM_OUTPUTS (1), + .NUM_LANES (`VX_CFG_NUM_SFU_LANES), + .TAG_WIDTH (RTU_REQ_TAG_WIDTH), + .ARBITER ("R"), + .OUT_BUF_ARM (REQ_OUT_BUF), + .OUT_BUF_REQ (REQ_OUT_BUF), + .OUT_BUF_WIN (0) + ) rtu_arb ( + .clk (clk), + .reset (reset), + .bus_in_if (block_rtu_bus_if), + .bus_out_if (rtu_arb_out_if) + ); + + // A warp lives in exactly one SFU block, so {wid} alone names a beat's owner + // on the core's bus; the socket re-derives the core index. + `UNUSED_VAR (rtu_arb_out_if[0].arm_data.src) + `UNUSED_VAR (rtu_arb_out_if[0].req_data.src) + + assign rtu_bus_if.arm_valid = rtu_arb_out_if[0].arm_valid; + assign rtu_bus_if.arm_data.src = 1'b0; + assign rtu_bus_if.arm_data.wid = rtu_arb_out_if[0].arm_data.wid; + assign rtu_bus_if.arm_data.mask = rtu_arb_out_if[0].arm_data.mask; + assign rtu_bus_if.arm_data.scene_base = rtu_arb_out_if[0].arm_data.scene_base; + assign rtu_bus_if.arm_data.flags = rtu_arb_out_if[0].arm_data.flags; + assign rtu_bus_if.arm_data.cull_mask = rtu_arb_out_if[0].arm_data.cull_mask; + assign rtu_bus_if.arm_data.payload_ptr = rtu_arb_out_if[0].arm_data.payload_ptr; + assign rtu_bus_if.arm_data.tag = rtu_arb_out_if[0].arm_data.tag; + assign rtu_arb_out_if[0].arm_ready = rtu_bus_if.arm_ready; + + assign rtu_bus_if.req_valid = rtu_arb_out_if[0].req_valid; + assign rtu_bus_if.req_data.kind = rtu_arb_out_if[0].req_data.kind; + assign rtu_bus_if.req_data.src = 1'b0; + assign rtu_bus_if.req_data.wid = rtu_arb_out_if[0].req_data.wid; + assign rtu_bus_if.req_data.data = rtu_arb_out_if[0].req_data.data; + assign rtu_bus_if.req_data.cb_action = rtu_arb_out_if[0].req_data.cb_action; + assign rtu_arb_out_if[0].req_ready = rtu_bus_if.req_ready; + + assign rtu_arb_out_if[0].win_valid = rtu_bus_if.win_valid; + assign rtu_arb_out_if[0].win_data = rtu_bus_if.win_data; + assign rtu_bus_if.win_ready = rtu_arb_out_if[0].win_ready; +`endif + +endmodule diff --git a/hw/rtl/core/VX_sfu_unit.sv b/hw/rtl/core/VX_sfu_unit.sv index 660303ee31..ecbbf46146 100644 --- a/hw/rtl/core/VX_sfu_unit.sv +++ b/hw/rtl/core/VX_sfu_unit.sv @@ -13,21 +13,17 @@ `include "VX_define.vh" -module VX_sfu_unit import VX_gpu_pkg::*; -`ifdef VX_CFG_EXT_RASTER_ENABLE -import VX_raster_pkg::*; +`ifdef VX_CFG_EXT_F_ENABLE +`include "VX_fpu_define.vh" `endif -#( - parameter `STRING INSTANCE_ID = "", - parameter CORE_ID = 0 + +module VX_sfu_unit import VX_gpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "" ) ( input wire clk, input wire reset, -`ifdef PERF_ENABLE - input sysmem_perf_t sysmem_perf, - input pipeline_perf_t pipeline_perf, -`endif + input wire [CORE_ID_WIDTH-1:0] core_id, // Inputs VX_dispatch_if.slave dispatch_if [`VX_CFG_ISSUE_WIDTH], @@ -38,55 +34,26 @@ import VX_raster_pkg::*; `ifdef VX_CFG_EXT_DXA_ENABLE VX_dxa_req_bus_if.master dxa_req_bus_if, - VX_txbar_bus_if.slave dxa_txbar_bus_if, `endif `ifdef VX_CFG_EXT_TEX_ENABLE VX_tex_bus_if.master tex_bus_if, `endif -`ifdef VX_CFG_EXT_OM_ENABLE -`endif - - // RASTER has no SFU port at all now: the fragment shader is launched by the - // raster engine (push, not pull) and its stamp arrives in the launch, so - // nothing about a fragment passes through this unit. - `ifdef VX_CFG_EXT_RTU_ENABLE VX_rtu_bus_if.master rtu_bus_if, - VX_sched_unlock_if.master sched_unlock_if, + VX_sched_unlock_if.master sched_unlock_if [`VX_CFG_NUM_SFU_BLOCKS], `endif - VX_sched_csr_if.slave sched_csr_if, - - VX_dcr_csr_if dcr_csr_if, + VX_sched_csr_if.slave sched_csr_if [`VX_CFG_NUM_SFU_BLOCKS], // Outputs VX_commit_if.master commit_if [`VX_CFG_ISSUE_WIDTH], - VX_warp_ctl_if.master warp_ctl_if + VX_warp_shared_if.master warp_shared_if [`VX_CFG_NUM_SFU_BLOCKS], + VX_warp_ctl_if.master warp_ctl_if [`VX_CFG_NUM_SFU_BLOCKS] ); - `UNUSED_SPARAM (INSTANCE_ID) - localparam BLOCK_SIZE = 1; - localparam NUM_LANES = `VX_CFG_NUM_SFU_LANES; - localparam PE_COUNT = 2 + `VX_CFG_EXT_DXA_ENABLED + `VX_CFG_EXT_TEX_ENABLED + `VX_CFG_EXT_OM_ENABLED + `VX_CFG_EXT_RASTER_ENABLED + `VX_CFG_EXT_RTU_ENABLED; - localparam PE_SEL_BITS = `CLOG2(PE_COUNT); - localparam PE_IDX_WCTL = 0; - localparam PE_IDX_CSRS = 1; -`ifdef VX_CFG_EXT_DXA_ENABLE - localparam PE_IDX_DXA = 2; -`endif -`ifdef VX_CFG_EXT_TEX_ENABLE - localparam PE_IDX_TEX = 2 + `VX_CFG_EXT_DXA_ENABLED; -`endif -`ifdef VX_CFG_EXT_OM_ENABLE - localparam PE_IDX_OM = 2 + `VX_CFG_EXT_DXA_ENABLED + `VX_CFG_EXT_TEX_ENABLED; -`endif -`ifdef VX_CFG_EXT_RASTER_ENABLE - localparam PE_IDX_RASTER = 2 + `VX_CFG_EXT_DXA_ENABLED + `VX_CFG_EXT_TEX_ENABLED + `VX_CFG_EXT_OM_ENABLED; -`endif -`ifdef VX_CFG_EXT_RTU_ENABLE - localparam PE_IDX_RTUW = 2 + `VX_CFG_EXT_DXA_ENABLED + `VX_CFG_EXT_TEX_ENABLED + `VX_CFG_EXT_OM_ENABLED + `VX_CFG_EXT_RASTER_ENABLED; -`endif + localparam BLOCK_SIZE = `VX_CFG_NUM_SFU_BLOCKS; + localparam NUM_LANES = `VX_CFG_NUM_SFU_LANES; VX_execute_if #( .data_t (sfu_execute_t) @@ -103,193 +70,99 @@ import VX_raster_pkg::*; ) lane_dispatch ( .clk (clk), .reset (reset), + .port_base ('0), .dispatch_if(dispatch_if), .execute_if (per_block_execute_if) ); - VX_execute_if #( - .data_t (sfu_execute_t) - ) pe_execute_if[PE_COUNT](); - - VX_result_if#( - .data_t (sfu_result_t) - ) pe_result_if[PE_COUNT](); - - reg [PE_SEL_BITS-1:0] pe_select; - always @(*) begin - pe_select = PE_SEL_BITS'(PE_IDX_WCTL); - if (inst_sfu_is_csr(per_block_execute_if[0].data.op_type)) begin - pe_select = PE_SEL_BITS'(PE_IDX_CSRS); - end - `ifdef VX_CFG_EXT_DXA_ENABLE - if (per_block_execute_if[0].data.op_type == INST_SFU_DXA) begin - pe_select = PE_SEL_BITS'(PE_IDX_DXA); - end - `endif - `ifdef VX_CFG_EXT_TEX_ENABLE - if (per_block_execute_if[0].data.op_type == INST_SFU_TEX) begin - pe_select = PE_SEL_BITS'(PE_IDX_TEX); - end - `endif - `ifdef VX_CFG_EXT_OM_ENABLE - if (per_block_execute_if[0].data.op_type == INST_SFU_OM) begin - pe_select = PE_SEL_BITS'(PE_IDX_OM); - end - `endif - `ifdef VX_CFG_EXT_RASTER_ENABLE - if (per_block_execute_if[0].data.op_type == INST_SFU_RASTER) begin - pe_select = PE_SEL_BITS'(PE_IDX_RASTER); - end - `endif - `ifdef VX_CFG_EXT_RTU_ENABLE - if (per_block_execute_if[0].data.op_type == INST_SFU_RTUW) begin - pe_select = PE_SEL_BITS'(PE_IDX_RTUW); - end - `endif - end - - VX_pe_switch #( - .PE_COUNT (PE_COUNT), - .NUM_LANES (NUM_LANES), - .ARBITER ("R"), - .REQ_OUT_BUF(0), - .RSP_OUT_BUF(3) - ) pe_switch ( - .clk (clk), - .reset (reset), - .pe_sel (pe_select), - .execute_in_if (per_block_execute_if[0]), - .result_out_if (per_block_result_if[0]), - .execute_out_if (pe_execute_if), - .result_in_if (pe_result_if) - ); - - VX_txbar_bus_if txbar_bus_if(); - - VX_wctl_unit #( - .INSTANCE_ID (`SFORMATF(("%s-wctl", INSTANCE_ID))), - .NUM_LANES (NUM_LANES) - ) wctl_unit ( - .clk (clk), - .reset (reset), - .execute_if (pe_execute_if[PE_IDX_WCTL]), - .warp_ctl_if(warp_ctl_if), - .txbar_bus_if(txbar_bus_if), - .result_if (pe_result_if[PE_IDX_WCTL]) - ); - - VX_csr_unit #( - .INSTANCE_ID (`SFORMATF(("%s-csr", INSTANCE_ID))), - .CORE_ID (CORE_ID), - .NUM_LANES (NUM_LANES) - ) csr_unit ( - .clk (clk), - .reset (reset), - - .execute_if (pe_execute_if[PE_IDX_CSRS]), - - `ifdef PERF_ENABLE - .sysmem_perf (sysmem_perf), - .pipeline_perf (pipeline_perf), - `endif - - `ifdef VX_CFG_EXT_F_ENABLE - .fpu_csr_if (fpu_csr_if), - `endif - - .sched_csr_if (sched_csr_if), - .result_if (pe_result_if[PE_IDX_CSRS]), - .dcr_csr_if (dcr_csr_if) - ); +`ifdef VX_CFG_EXT_F_ENABLE + wire [BLOCK_SIZE-1:0][NW_WIDTH-1:0] fcsr_wid; + wire [BLOCK_SIZE-1:0][INST_FRM_BITS+`FP_FLAGS_BITS-1:0] fcsr_rd_data; + wire [BLOCK_SIZE-1:0][2:0] fcsr_wr_sel; + wire [BLOCK_SIZE-1:0][INST_FRM_BITS+`FP_FLAGS_BITS-1:0] fcsr_wr_data; +`endif `ifdef VX_CFG_EXT_DXA_ENABLE - VX_dxa_unit #( - .INSTANCE_ID (`SFORMATF(("%s-dxa", INSTANCE_ID))), - .CORE_ID (CORE_ID) - ) dxa_unit ( - .clk (clk), - .reset (reset), - .execute_if (pe_execute_if[PE_IDX_DXA]), - .result_if (pe_result_if[PE_IDX_DXA]), - .dxa_req_bus_if (dxa_req_bus_if) - ); - - // The only txbar producer is the SMEM-completion path through - // dxa_txbar_bus_if (DXA release); no arbitration is needed. - assign txbar_bus_if.valid = dxa_txbar_bus_if.valid; - assign txbar_bus_if.data = dxa_txbar_bus_if.data; - assign dxa_txbar_bus_if.ready = txbar_bus_if.ready; -`else - assign txbar_bus_if.valid = 1'b0; - assign txbar_bus_if.data = 'x; - `UNUSED_VAR (txbar_bus_if.ready) + VX_dxa_req_bus_if block_dxa_req_bus_if[BLOCK_SIZE](); `endif - -// The hit window has no FF consumers left. TEX was the last one — it spilled -// its 2x2 quad's eight (u,v) operands into window slots because no RISC-V encoding -// holds eight inputs, and that cost the window two full RAM mirrors. It now takes -// u/v/lod in registers (vx_tex, R4-type) and its shader computes the mip LOD with -// vx_tex_quad_lod(), so the window is down to a single mirror and its only tenant -// is the RTU. - `ifdef VX_CFG_EXT_TEX_ENABLE - VX_tex_unit #( - .INSTANCE_ID (`SFORMATF(("%s-tex", INSTANCE_ID))), - .CORE_ID (CORE_ID), - .NUM_LANES (NUM_LANES) - ) tex_unit ( - .clk (clk), - .reset (reset), - .execute_if (pe_execute_if[PE_IDX_TEX]), - .result_if (pe_result_if[PE_IDX_TEX]), - .tex_bus_if (tex_bus_if) - ); -`endif - -`ifdef VX_CFG_EXT_OM_ENABLE - // OM export (v2): a fragment leaves the shader as a STORE to the OM aperture - // (vx_om_export -> VX_gfx_uops -> the LSU), so the SFU services no OM op. The - // PE slot is retained for index stability and tied off, as RASTER's is. - assign pe_execute_if[PE_IDX_OM].ready = 1'b1; - assign pe_result_if[PE_IDX_OM].valid = 1'b0; - assign pe_result_if[PE_IDX_OM].data = '0; - `UNUSED_VAR (pe_execute_if[PE_IDX_OM].valid) - `UNUSED_VAR (pe_execute_if[PE_IDX_OM].data) + VX_tex_bus_if #( + .NUM_LANES (NUM_LANES), + .TAG_WIDTH (TEX_REQ_TAG_WIDTH) + ) block_tex_bus_if[BLOCK_SIZE](); `endif +`ifdef VX_CFG_EXT_RTU_ENABLE + VX_rtu_bus_if #( + .NUM_LANES (NUM_LANES), + .TAG_WIDTH (RTU_REQ_TAG_WIDTH) + ) block_rtu_bus_if[BLOCK_SIZE](); +`endif + + for (genvar block_idx = 0; block_idx < BLOCK_SIZE; ++block_idx) begin : g_blocks + VX_sfu_block #( + .INSTANCE_ID ((BLOCK_SIZE > 1) ? `SFORMATF(("%s%0d", INSTANCE_ID, block_idx)) : INSTANCE_ID) + ) sfu_block ( + .clk (clk), + .reset (reset), + .core_id (core_id), + .block_idx (SFU_BLOCK_W'(block_idx)), + .execute_if (per_block_execute_if[block_idx]), + .result_if (per_block_result_if[block_idx]), + `ifdef VX_CFG_EXT_F_ENABLE + .fcsr_wid (fcsr_wid[block_idx]), + .fcsr_rd_data (fcsr_rd_data[block_idx]), + .fcsr_wr_sel (fcsr_wr_sel[block_idx]), + .fcsr_wr_data (fcsr_wr_data[block_idx]), + `endif + `ifdef VX_CFG_EXT_DXA_ENABLE + .dxa_req_bus_if (block_dxa_req_bus_if[block_idx]), + `endif + `ifdef VX_CFG_EXT_TEX_ENABLE + .tex_bus_if (block_tex_bus_if[block_idx]), + `endif + `ifdef VX_CFG_EXT_RTU_ENABLE + .rtu_bus_if (block_rtu_bus_if[block_idx]), + .sched_unlock_if(sched_unlock_if[block_idx]), + `endif + .sched_csr_if (sched_csr_if[block_idx]), + .warp_shared_if (warp_shared_if[block_idx]), + .warp_ctl_if (warp_ctl_if[block_idx]) + ); + end -`ifdef VX_CFG_EXT_RASTER_ENABLE - // RASTER push (v2): the fragment payload reaches the warp at launch via the - // core-level distributor's window seed (rast_win_wr_*), so the SFU services - // no raster op. The PE slot is retained for index stability and tied off — - // no kernel issues INST_SFU_RASTER under the push model. - assign pe_execute_if[PE_IDX_RASTER].ready = 1'b1; - assign pe_result_if[PE_IDX_RASTER].valid = 1'b0; - assign pe_result_if[PE_IDX_RASTER].data = '0; - `UNUSED_VAR (pe_execute_if[PE_IDX_RASTER].valid) - `UNUSED_VAR (pe_execute_if[PE_IDX_RASTER].data) - `UNUSED_VAR (pe_result_if[PE_IDX_RASTER].ready) +`ifdef VX_CFG_EXT_F_ENABLE + VX_csr_fpu #( + .NUM_CSR_PORTS (BLOCK_SIZE), + .NUM_FPU_PORTS (`VX_CFG_NUM_FPU_BLOCKS) + ) csr_fpu ( + .clk (clk), + .reset (reset), + .csr_wid (fcsr_wid), + .csr_rd_data (fcsr_rd_data), + .csr_wr_sel (fcsr_wr_sel), + .csr_wr_data (fcsr_wr_data), + .fpu_csr_if (fpu_csr_if) + ); `endif -`ifdef VX_CFG_EXT_RTU_ENABLE - VX_rtu_unit #( - .INSTANCE_ID (`SFORMATF(("%s-rtuw", INSTANCE_ID))), - .CORE_ID (CORE_ID), - `ifdef VX_CFG_EXT_RTU_ENABLE - .RTU_TAG_WIDTH (RTU_REQ_TAG_WIDTH), + VX_sfu_shared #( + .INSTANCE_ID (`SFORMATF(("%s-shared", INSTANCE_ID))) + ) sfu_shared ( + `ifdef VX_CFG_EXT_DXA_ENABLE + .block_dxa_req_bus_if (block_dxa_req_bus_if), + .dxa_req_bus_if (dxa_req_bus_if), + `endif + `ifdef VX_CFG_EXT_TEX_ENABLE + .block_tex_bus_if (block_tex_bus_if), + .tex_bus_if (tex_bus_if), `endif - .NUM_LANES (NUM_LANES) - ) rtu_unit ( - .clk (clk), - .reset (reset), - .execute_if (pe_execute_if[PE_IDX_RTUW]), - .result_if (pe_result_if[PE_IDX_RTUW]) `ifdef VX_CFG_EXT_RTU_ENABLE - , - .rtu_bus_if (rtu_bus_if), - .sched_unlock_if (sched_unlock_if) + .block_rtu_bus_if (block_rtu_bus_if), + .rtu_bus_if (rtu_bus_if), `endif + .clk (clk), + .reset (reset) ); -`endif VX_lane_gather #( .BLOCK_SIZE (BLOCK_SIZE), diff --git a/hw/rtl/core/VX_split_join.sv b/hw/rtl/core/VX_split_join.sv index 7da6d4df6d..36fe514539 100644 --- a/hw/rtl/core/VX_split_join.sv +++ b/hw/rtl/core/VX_split_join.sv @@ -15,20 +15,22 @@ module VX_split_join import VX_gpu_pkg::*; #( parameter `STRING INSTANCE_ID = "", - parameter OUT_REG = 0 + parameter NUM_WARPS = `VX_CFG_NUM_WARPS, + parameter WID_W = `LOG2UP(NUM_WARPS), + parameter OUT_REG = 0 ) ( input wire clk, input wire reset, input wire split_valid, input wire sjoin_valid, - input wire [NW_WIDTH-1:0] wid, + input wire [WID_W-1:0] wid, input split_t split, input join_t sjoin, - input wire [NW_WIDTH-1:0] stack_wid, + input wire [WID_W-1:0] stack_wid, output wire join_valid, output wire join_is_dvg, output wire join_is_else, - output wire [NW_WIDTH-1:0] join_wid, + output wire [WID_W-1:0] join_wid, output wire [`VX_CFG_NUM_THREADS-1:0] join_tmask, output wire [PC_BITS-1:0] join_pc, output wire [DV_STACK_SIZEW-1:0] stack_ptr @@ -36,7 +38,7 @@ module VX_split_join import VX_gpu_pkg::*; #( `UNUSED_SPARAM (INSTANCE_ID) if (NT_BITS != 0) begin : g_enable - wire [`VX_CFG_NUM_WARPS-1:0][DV_STACK_SIZEW-1:0] ipdom_wr_ptr; + wire [NUM_WARPS-1:0][DV_STACK_SIZEW-1:0] ipdom_wr_ptr; wire [`VX_CFG_NUM_THREADS-1:0] orig_tmask; wire [PC_BITS-1:0] next_pc; wire ipdom_idx; @@ -46,8 +48,10 @@ module VX_split_join import VX_gpu_pkg::*; #( wire sjoin_is_dvg = (sjoin.stack_ptr != ipdom_wr_ptr[wid]); VX_ipdom_stack #( - .WIDTH (`VX_CFG_NUM_THREADS + PC_BITS), - .DEPTH (DV_STACK_SIZE) + .WIDTH (`VX_CFG_NUM_THREADS + PC_BITS), + .DEPTH (DV_STACK_SIZE), + .NUM_WARPS (NUM_WARPS), + .WID_W (WID_W) ) ipdom_stack ( .clk (clk), .reset (reset), @@ -66,7 +70,7 @@ module VX_split_join import VX_gpu_pkg::*; #( wire [`VX_CFG_NUM_THREADS-1:0] join_tmask_n = ipdom_idx ? orig_tmask : (~sjoin.tmask & orig_tmask); VX_pipe_register #( - .DATAW (1 + NW_WIDTH + 1 + 1 + `VX_CFG_NUM_THREADS + PC_BITS), + .DATAW (1 + WID_W + 1 + 1 + `VX_CFG_NUM_THREADS + PC_BITS), .RESETW (1), .DEPTH (OUT_REG) ) pipe_reg ( diff --git a/hw/rtl/core/VX_uop_packld.sv b/hw/rtl/core/VX_uop_packld.sv index f33483f579..9ec971e633 100644 --- a/hw/rtl/core/VX_uop_packld.sv +++ b/hw/rtl/core/VX_uop_packld.sv @@ -24,7 +24,7 @@ // eff_rs1[lane] = rs1[lane] + rs2[lane] * uop_idx (shift-and-add) // The pack field is cleared in op_args forwarded to the LSU slice so it sees a // plain LBU or LHU. The NaN-boxing of the FLW result is handled automatically -// because is_float==1 and rd is a float register (triggers nan-box in lsu_slice). +// because is_float==1 and rd is a float register (triggers nan-box in lsu_block). module VX_uop_packld import VX_gpu_pkg::*; ( input clk, diff --git a/hw/rtl/core/VX_uuid_gen.sv b/hw/rtl/core/VX_uuid_gen.sv index 61ec538e35..34ea3aab1d 100644 --- a/hw/rtl/core/VX_uuid_gen.sv +++ b/hw/rtl/core/VX_uuid_gen.sv @@ -13,31 +13,45 @@ `include "VX_define.vh" +// Per-warp instruction ids. NUM_WARPS is the number of warps counted here: all +// of the core's, or the PER_FETCH_WARPS of one fetch slice (wid = {wfs, slice}). module VX_uuid_gen import VX_gpu_pkg::*; #( - parameter CORE_ID = 0 + parameter NUM_WARPS = `VX_CFG_NUM_WARPS ) ( input wire clk, input wire reset, + + input wire [CORE_ID_WIDTH-1:0] core_id, input wire incr, input wire [NW_WIDTH-1:0] wid, output wire [UUID_WIDTH-1:0] uuid ); localparam GNW_WIDTH = UUID_WIDTH - 32; - reg [31:0] uuid_cntrs [0:`VX_CFG_NUM_WARPS-1]; - reg [`VX_CFG_NUM_WARPS-1:0] has_uuid_cntrs; + localparam LW = `LOG2UP(NUM_WARPS); + + wire [LW-1:0] lid; + if (NUM_WARPS == `VX_CFG_NUM_WARPS) begin : g_core_warps + assign lid = LW'(wid); + end else begin : g_slice_warps + `STATIC_ASSERT (NUM_WARPS == PER_FETCH_WARPS, ("invalid parameter")) + assign lid = wid_to_wfs(wid); + end + + reg [31:0] uuid_cntrs [0:NUM_WARPS-1]; + reg [NUM_WARPS-1:0] has_uuid_cntrs; always @(posedge clk) begin if (reset) begin has_uuid_cntrs <= '0; end else if (incr) begin - has_uuid_cntrs[wid] <= 1; + has_uuid_cntrs[lid] <= 1; end if (incr) begin - uuid_cntrs[wid] <= has_uuid_cntrs[wid] ? (uuid_cntrs[wid] + 1) : 1; + uuid_cntrs[lid] <= has_uuid_cntrs[lid] ? (uuid_cntrs[lid] + 1) : 1; end end - wire [GNW_WIDTH-1:0] g_wid = (GNW_WIDTH'(CORE_ID) << NW_BITS) + GNW_WIDTH'(wid); - assign uuid = {g_wid, (has_uuid_cntrs[wid] ? uuid_cntrs[wid] : 0)}; + wire [GNW_WIDTH-1:0] g_wid = (GNW_WIDTH'(core_id) << NW_BITS) + GNW_WIDTH'(wid); + assign uuid = {g_wid, (has_uuid_cntrs[lid] ? uuid_cntrs[lid] : 0)}; endmodule diff --git a/hw/rtl/core/VX_warp_sched.sv b/hw/rtl/core/VX_warp_sched.sv new file mode 100644 index 0000000000..eeeb663bc4 --- /dev/null +++ b/hw/rtl/core/VX_warp_sched.sv @@ -0,0 +1,514 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// The warp scheduler of one fetch slice: the state of the slice's warps +// (wid = {wfs, slice_id}), its warp selection and a registered fetch request to +// the slice's fetch unit, which names warps by wfs. FU block port q carries the +// traffic of the local warps with wfs mod PORTS == q, addressed by core-wide wid; +// core-wide events arrive through sched_if already localized. +module VX_warp_sched import VX_gpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "" +) ( + input wire clk, + input wire reset, + + input wire [CORE_ID_WIDTH-1:0] core_id, + input wire [FETCH_FSW_W-1:0] slice_id, + + VX_slice_sched_if.slave sched_if, + + // ALU ports + input sched_branch_t [FETCH_ALU_PORTS-1:0] branch, + + // SFU ports + input sched_wctl_t [FETCH_SFU_PORTS-1:0] wctl, + input wire [FETCH_SFU_PORTS-1:0][NW_WIDTH-1:0] dvstack_wid, + output wire [FETCH_SFU_PORTS-1:0][DV_STACK_SIZEW-1:0] dvstack_ptr, + output wire [FETCH_SFU_PORTS-1:0][`VX_CFG_MEM_ADDR_WIDTH-1:0] dvstack_mscratch, + input wire [FETCH_SFU_PORTS-1:0][NW_WIDTH-1:0] csr_rd_wid, + output wire sched_csr_rd_t [FETCH_SFU_PORTS-1:0] csr_rd, + input sched_csr_wr_t [FETCH_SFU_PORTS-1:0] csr_wr, +`ifdef VX_CFG_EXT_RTU_ENABLE + input wire [FETCH_SFU_PORTS-1:0] rtu_unlock_valid, + input wire [FETCH_SFU_PORTS-1:0][NW_WIDTH-1:0] rtu_unlock_wid, +`endif + + // per-warp pipeline events, indexed by wfs + VX_decode_sched_if.slave decode_sched_if, + input wire [PER_FETCH_WARPS-1:0] issued, + input wire [PER_FETCH_WARPS-1:0] committed, + input wire [PER_FETCH_WARPS-1:0] ibuf_pop, + output wire [PER_FETCH_WARPS-1:0] pending_alm_empty, + + // fetch request + VX_schedule_if.master schedule_if +); + `UNUSED_SPARAM (INSTANCE_ID) + + localparam SW = PER_FETCH_WARPS; + localparam LW = FETCH_WFS_W; + localparam NPA = FETCH_ALU_PORTS; + localparam NPS = FETCH_SFU_PORTS; + localparam NPS_BITS = `CLOG2(NPS); + localparam PSW = SW / NPS; // warps per SFU port + localparam KW = `LOG2UP(PSW); + localparam IBUF_CW = $clog2(`VX_CFG_IBUF_SIZE + 1); + + `UNUSED_VAR (schedule_if.ibuf_pop) + + // wfs -> SFU port-local index + function automatic logic [KW-1:0] wfs_to_k(input logic [LW-1:0] wfs); + wfs_to_k = KW'(wfs >> NPS_BITS); + endfunction + + wire [LW-1:0] dec_lid = decode_sched_if.wid[LW-1:0]; + if (NW_WIDTH > LW) begin : g_dec_wid_unused + `UNUSED_VAR (decode_sched_if.wid[NW_WIDTH-1:LW]) + end + + wire [NPA-1:0][LW-1:0] br_lid; + for (genvar q = 0; q < NPA; ++q) begin : g_br_lid + assign br_lid[q] = wid_to_wfs(branch[q].wid); + `UNUSED_VAR (wid_to_fsw(branch[q].wid)) + end + + wire [NPS-1:0][LW-1:0] ctl_lid, csr_rd_lid, csr_wr_lid; +`ifdef VX_CFG_EXT_RTU_ENABLE + wire [NPS-1:0][LW-1:0] rtu_lid; +`endif + + // split/join, one stack per SFU port over the port's warps + wire [NPS-1:0] join_valid; + wire [NPS-1:0] join_is_dvg; + wire [NPS-1:0] join_is_else; + wire [NPS-1:0][KW-1:0] join_k; + wire [NPS-1:0][`VX_CFG_NUM_THREADS-1:0] join_tmask; + wire [NPS-1:0][PC_BITS-1:0] join_pc; + + for (genvar q = 0; q < NPS; ++q) begin : g_sfu_ports + assign ctl_lid[q] = wid_to_wfs(wctl[q].wid); + assign csr_rd_lid[q] = wid_to_wfs(csr_rd_wid[q]); + assign csr_wr_lid[q] = wid_to_wfs(csr_wr[q].wid); + `UNUSED_VAR ({wid_to_fsw(wctl[q].wid), wid_to_fsw(csr_rd_wid[q]), wid_to_fsw(csr_wr[q].wid), wid_to_fsw(dvstack_wid[q])}) + `ifdef VX_CFG_EXT_RTU_ENABLE + assign rtu_lid[q] = wid_to_wfs(rtu_unlock_wid[q]); + `UNUSED_VAR (wid_to_fsw(rtu_unlock_wid[q])) + `endif + + VX_split_join #( + .INSTANCE_ID ((NPS > 1) ? `SFORMATF(("%s-splitjoin%0d", INSTANCE_ID, q)) : `SFORMATF(("%s-splitjoin", INSTANCE_ID))), + .NUM_WARPS (PSW), + .WID_W (KW), + .OUT_REG (1) + ) split_join ( + .clk (clk), + .reset (reset), + .split_valid (wctl[q].split_valid), + .sjoin_valid (wctl[q].sjoin_valid), + .wid (wfs_to_k(ctl_lid[q])), + .split (wctl[q].split), + .sjoin (wctl[q].sjoin), + .join_valid (join_valid[q]), + .join_is_dvg (join_is_dvg[q]), + .join_is_else(join_is_else[q]), + .join_wid (join_k[q]), + .join_tmask (join_tmask[q]), + .join_pc (join_pc[q]), + .stack_wid (wfs_to_k(wid_to_wfs(dvstack_wid[q]))), + .stack_ptr (dvstack_ptr[q]) + ); + end + + wire [SW-1:0] active_warps, stalled_warps; + wire [SW-1:0][`VX_CFG_NUM_THREADS-1:0] thread_masks; + wire [SW-1:0][PC_BITS-1:0] warp_pcs; + wire [SW-1:0] ibuf_full, ibuf_full_n; + wire [SW-1:0] pending_empty; + wire [SW-1:0] cand_onehot; + wire cand_valid; + wire sched_fire; + + reg [SW-1:0][`VX_CFG_MEM_ADDR_WIDTH-1:0] mscratch_r; + reg [SW-1:0][`VX_CFG_XLEN-1:0] mstatus_r, mtvec_r, mepc_r, mcause_r, mtval_r; + + for (genvar l = 0; l < SW; ++l) begin : g_warps + localparam [LW-1:0] LID = LW'(l); + localparam QA = l % NPA; + localparam QS = l % NPS; + localparam [KW-1:0] K = KW'(l / NPS); + + reg active_r, active_n; + reg stalled_r, stalled_n; + reg [`VX_CFG_NUM_THREADS-1:0] tmask_r, tmask_n; + reg [PC_BITS-1:0] pc_r, pc_n; + + wire ctl_hit = (ctl_lid[QS] == LID); + wire br_hit = branch[QA].valid && (br_lid[QA] == LID); + wire join_hit = join_valid[QS] && (join_k[QS] == K); + + always @(*) begin + active_n = active_r; + stalled_n = stalled_r; + tmask_n = tmask_r; + pc_n = pc_r; + + if (sched_if.cta_fire && (sched_if.cta_lid == LID)) begin + active_n = 1; + // Reusing a warp for the next CTA skips the one-time prologue and rewinds to the + // kernel's per-CTA dispatch window: a fixed 20-byte (5-instruction) sequence that + // reloads the entry pointer and kargs before re-calling. + pc_n = sched_if.cta_init ? sched_if.cta_PC : (pc_r - from_fullPC(`VX_CFG_XLEN'(20))); + tmask_n = sched_if.cta_tmask; + end + + if (decode_sched_if.valid && decode_sched_if.unlock && (dec_lid == LID)) begin + stalled_n = 0; + end + + if (sched_if.wspawn_apply) begin + if (sched_if.wspawn_wmask[l]) begin + active_n = 1; + if (!(sched_if.wspawn_src && (sched_if.wspawn_src_lid == LID))) begin + tmask_n[0] = 1; + pc_n = sched_if.wspawn_pc; + end + end + if (sched_if.wspawn_src && (sched_if.wspawn_src_lid == LID)) begin + stalled_n = 0; + end + end + + if (wctl[QS].tmc_valid && ctl_hit) begin + active_n = (wctl[QS].tmc.tmask != 0); + tmask_n = wctl[QS].tmc.tmask; + stalled_n = 0; + end + + if (wctl[QS].split_valid && ctl_hit) begin + if (wctl[QS].split.is_dvg) begin + tmask_n = wctl[QS].split.then_tmask; + end + stalled_n = 0; + end + + if (join_hit) begin + if (join_is_dvg[QS]) begin + if (join_is_else[QS]) begin + pc_n = join_pc[QS]; + end + tmask_n = join_tmask[QS]; + end + stalled_n = 0; + end + + if (sched_if.bar_unlock_valid && sched_if.bar_unlock_mask[l]) begin + stalled_n = 0; + end + + // an exiting warp is retired as its CTA accounting takes it, before + // the slot can be relaunched + if (sched_if.retire_valid && (sched_if.retire_lid == LID)) begin + active_n = 0; + tmask_n = '0; + stalled_n = 0; + end + + // wsync unlock: warp pipeline drained + if (wctl[QS].wsync_valid && ctl_hit) begin + stalled_n = 0; + end + + if (br_hit) begin + if (branch[QA].is_trap) begin + // ECALL/EBREAK: redirect to trap vector (mtvec[1:0] = MODE field; mask off to get base address). + pc_n = from_fullPC(mtvec_r[l] & ~`VX_CFG_XLEN'(3)); + end else if (branch[QA].is_mret) begin + // MRET/SRET/URET: restore the saved PC from mepc. + pc_n = from_fullPC(mepc_r[l]); + end else if (branch[QA].taken) begin + pc_n = branch[QA].dest; + end + stalled_n = 0; + end + + // stall the warp until decode stage + if (sched_fire && cand_onehot[l]) begin + stalled_n = 1; + `ifndef VX_CFG_EXT_C_ENABLE + pc_n = pc_r + from_fullPC(`VX_CFG_XLEN'(4)); + `endif + end + + `ifdef VX_CFG_EXT_C_ENABLE + // Every redirect source stalls its warp from schedule until it resolves, so + // no redirect writes a warp's PC on the cycle that warp decode-advances. + if (decode_sched_if.valid && (dec_lid == LID)) begin + pc_n = pc_r + from_fullPC(decode_sched_if.is_rvc ? `VX_CFG_XLEN'(2) : `VX_CFG_XLEN'(4)); + end + `endif + + `ifdef VX_CFG_EXT_RTU_ENABLE + // A wstall'd TRACE retires: resume the warp so it proceeds to WAIT. + if (rtu_unlock_valid[QS] && (rtu_lid[QS] == LID)) begin + stalled_n = 1'b0; + end + `endif + end + + always @(posedge clk) begin + if (reset) begin + active_r <= 0; + stalled_r <= 0; + tmask_r <= '0; + pc_r <= '0; + mscratch_r[l] <= '0; + mstatus_r[l] <= '0; + mtvec_r[l] <= '0; + mepc_r[l] <= '0; + mcause_r[l] <= '0; + mtval_r[l] <= '0; + end else begin + active_r <= active_n; + stalled_r <= stalled_n; + tmask_r <= tmask_n; + pc_r <= pc_n; + + // spawned warps inherit the spawning warp's mscratch + if (sched_if.wspawn_apply && sched_if.wspawn_wmask[l] + && !(sched_if.wspawn_src && (sched_if.wspawn_src_lid == LID))) begin + mscratch_r[l] <= sched_if.wspawn_mscratch; + end + // CTA dispatch: latch this warp's mscratch (param) + if (sched_if.cta_fire && (sched_if.cta_lid == LID)) begin + mscratch_r[l] <= sched_if.cta_param; + end + if (csr_wr[QS].valid && (csr_wr_lid[QS] == LID)) begin + mscratch_r[l] <= csr_wr[QS].data; + end + + if (csr_wr_lid[QS] == LID) begin + if (csr_wr[QS].trap_valid[0]) begin + mstatus_r[l] <= csr_wr[QS].trap_data; + end + if (csr_wr[QS].trap_valid[1]) begin + mtvec_r[l] <= csr_wr[QS].trap_data; + end + if (csr_wr[QS].trap_valid[2]) begin + mepc_r[l] <= csr_wr[QS].trap_data; + end + if (csr_wr[QS].trap_valid[3]) begin + mcause_r[l] <= csr_wr[QS].trap_data; + end + if (csr_wr[QS].trap_valid[4]) begin + mtval_r[l] <= csr_wr[QS].trap_data; + end + end + + // Hardware trap entry (ECALL/EBREAK) wins a same-cycle software write. + if (br_hit && branch[QA].is_trap) begin + mepc_r[l] <= to_fullPC(branch[QA].dest); + mcause_r[l] <= `VX_CFG_XLEN'(branch[QA].trap_cause); + mtval_r[l] <= '0; + end + end + end + + assign active_warps[l] = active_r; + assign stalled_warps[l] = stalled_r; + assign thread_masks[l] = tmask_r; + assign warp_pcs[l] = pc_r; + + // ibuffer occupancy: registered full keeps arbitration off the critical path + logic [IBUF_CW-1:0] ibuf_size_r, ibuf_size_n; + logic ibuf_full_r; + wire ibuf_incr = sched_fire && cand_onehot[l]; + assign ibuf_size_n = ibuf_size_r + IBUF_CW'(ibuf_incr) - IBUF_CW'(ibuf_pop[l]); + always @(posedge clk) begin + if (reset) begin + ibuf_size_r <= '0; + ibuf_full_r <= 1'b0; + end else begin + ibuf_size_r <= ibuf_size_n; + ibuf_full_r <= ibuf_full_n[l]; + end + end + assign ibuf_full_n[l] = (ibuf_size_n == IBUF_CW'(`VX_CFG_IBUF_SIZE)); + assign ibuf_full[l] = ibuf_full_r; + + VX_pending_size #( + .SIZE (256), + .ALM_EMPTY (1) + ) pending_ctr ( + .clk (clk), + .reset (reset), + .incr (issued[l]), + .decr (committed[l]), + .empty (pending_empty[l]), + .alm_empty (pending_alm_empty[l]), + `UNUSED_PIN (full), + `UNUSED_PIN (alm_full), + `UNUSED_PIN (size) + ); + + `ifdef DBG_TRACE_PIPELINE + always @(posedge clk) begin + if (active_n != active_r + || (active_r && (stalled_n != stalled_r || tmask_n != tmask_r))) begin + `TRACE(1, ("%t: %s warp-state: wid=%0d, active=%b, stalled=%b, tmask=%b\n", + $time, INSTANCE_ID, wfs_to_wid(LID, slice_id), active_n, stalled_n, tmask_n)) + end + end + `endif + end + + // the slice's lowest ready warp + wire [SW-1:0] ready_warps = active_warps & ~stalled_warps; + wire [SW-1:0] preferred_warps = ready_warps & ~ibuf_full; +`ifndef L1_ENABLE + // without L1, the icache must never stall: it shares the bus with dcache responses + wire [SW-1:0] schedule_warps = preferred_warps; + `UNUSED_VAR (sched_if.all_ibuf_full) +`else + wire [SW-1:0] schedule_warps = sched_if.all_ibuf_full ? ready_warps : preferred_warps; +`endif + + wire [`UP(FETCH_WFS_BITS)-1:0] cand_index; + VX_priority_encoder #( + .N (SW) + ) wid_select ( + .data_in (schedule_warps), + .index_out (cand_index), + .valid_out (cand_valid), + .onehot_out(cand_onehot) + ); + wire [LW-1:0] cand_lid = LW'(cand_index); + + // CTA CSRs and CTA ids, one store per SFU port over the port's warps + wire [NPS-1:0][NCTA_WIDTH-1:0] port_cta_id; + + for (genvar q = 0; q < NPS; ++q) begin : g_csr_cta + wire [LW-1:0] warp_waddr = sched_if.warp_waddr; + wire [LW-1:0] id_waddr = sched_if.id_waddr; + + VX_csr_cta #( + .NUM_WARPS (PSW) + ) csr_cta ( + .clk (clk), + .reset (reset), + .warp_write (sched_if.warp_write && ((32'(warp_waddr) % NPS) == q)), + .warp_waddr (wfs_to_k(warp_waddr)), + .warp_wdata (sched_if.warp_wdata), + .id_write (sched_if.id_write && ((32'(id_waddr) % NPS) == q)), + .id_waddr (wfs_to_k(id_waddr)), + .id_wdata (sched_if.id_wdata), + .rd_lid (wfs_to_k(csr_rd_lid[q])), + .rd_cta_rank (csr_rd[q].cta_rank), + .rd_lane (csr_rd[q].cta_lane), + .sched_lid (wfs_to_k(cand_lid)), + .sched_cta_id (port_cta_id[q]) + ); + + assign csr_rd[q].thread_mask = thread_masks[csr_rd_lid[q]]; + assign csr_rd[q].mscratch = mscratch_r[csr_rd_lid[q]]; + assign csr_rd[q].mstatus = mstatus_r[csr_rd_lid[q]]; + assign csr_rd[q].mtvec = mtvec_r[csr_rd_lid[q]]; + assign csr_rd[q].mepc = mepc_r[csr_rd_lid[q]]; + assign csr_rd[q].mcause = mcause_r[csr_rd_lid[q]]; + assign csr_rd[q].mtval = mtval_r[csr_rd_lid[q]]; + + assign dvstack_mscratch[q] = mscratch_r[wid_to_wfs(dvstack_wid[q])]; + end + + wire [NCTA_WIDTH-1:0] cand_cta_id = port_cta_id[32'(cand_lid) % NPS]; + + // fetch request, registered at the slice boundary + wire sched_ready; + assign sched_fire = cand_valid && sched_ready; + + wire [UUID_WIDTH-1:0] sched_uuid; +`ifdef UUID_ENABLE + VX_uuid_gen #( + .NUM_WARPS (SW) + ) uuid_gen ( + .clk (clk), + .reset (reset), + .core_id (core_id), + .incr (sched_fire), + .wid (wfs_to_wid(cand_lid, slice_id)), + .uuid (sched_uuid) + ); +`else + assign sched_uuid = '0; + `UNUSED_VAR (core_id) +`ifndef DBG_TRACE_PIPELINE + `UNUSED_VAR (slice_id) +`endif +`endif + + VX_elastic_buffer #( + .DATAW (`VX_CFG_NUM_THREADS + PC_BITS + NW_WIDTH + NCTA_WIDTH + UUID_WIDTH), + .SIZE (2), + .OUT_REG (1) + ) sched_buf ( + .clk (clk), + .reset (reset), + .valid_in (cand_valid), + .ready_in (sched_ready), + .data_in ({thread_masks[cand_lid], warp_pcs[cand_lid], NW_WIDTH'(cand_lid), cand_cta_id, sched_uuid}), + .data_out ({schedule_if.data.tmask, schedule_if.data.PC, schedule_if.data.wid, schedule_if.data.cta_id, schedule_if.data.uuid}), + .valid_out (schedule_if.valid), + .ready_out (schedule_if.ready) + ); + + wire schedule_if_fire = schedule_if.valid && schedule_if.ready; + + // status + assign sched_if.active = active_warps; + assign sched_if.stalled = stalled_warps; + assign sched_if.ibuf_full_n = ibuf_full_n; + assign sched_if.pending_empty = pending_empty; + + // performance events + for (genvar q = 0; q < NPA; ++q) begin : g_branch_valid + assign sched_if.branch_valid[q] = branch[q].valid; + end + for (genvar q = 0; q < NPS; ++q) begin : g_divergence + assign sched_if.divergence[q] = wctl[q].split_valid && wctl[q].split.is_dvg; + end + assign sched_if.sched_valid = cand_valid; + assign sched_if.sched_fire = schedule_if_fire; + assign sched_if.sched_tmask = schedule_if.data.tmask; + +`ifdef EXT_SCHED_TIMEOUT_DUMP + always @(posedge clk) begin + if (sched_if.timeout_dump) begin + for (integer l = 0; l < SW; ++l) begin + $display(" wid=%0d stalled=%0d pc=0x%0h tmask=%b", + wfs_to_wid(LW'(l), slice_id), stalled_warps[l], to_fullPC(warp_pcs[l]), thread_masks[l]); + end + end + end +`endif + +`ifdef DBG_TRACE_PIPELINE + always @(posedge clk) begin + if (sched_fire) begin + `TRACE(1, ("%t: %s dispatch: wid=%0d, cta_id=%0d, PC=0x%0h, tmask=%b (#%0d)\n", $time, INSTANCE_ID, + wfs_to_wid(cand_lid, slice_id), cand_cta_id, to_fullPC(warp_pcs[cand_lid]), thread_masks[cand_lid], sched_uuid)) + end + end +`endif + +endmodule diff --git a/hw/rtl/core/VX_warp_shared_unit.sv b/hw/rtl/core/VX_warp_shared_unit.sv new file mode 100644 index 0000000000..de5ed3db5f --- /dev/null +++ b/hw/rtl/core/VX_warp_shared_unit.sv @@ -0,0 +1,268 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// The core-wide warp-control resources: barrier unit, wspawn and warp retirement. +// One request per cycle across the requesters, round-robin; a request that +// asks for it is completed back to its requester. Transaction-barrier +// completions use the barrier unit when no request does. +// +// Several requesters register their requests toward the arbiter, which acts on +// the grant and completes it registered. A sole requester is connected +// directly: it is completed in the cycle it asks, and its request is registered +// here instead, so every request takes effect at the same distance from its +// instruction in both cases. +module VX_warp_shared_unit import VX_gpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "", + parameter NUM_REQS = 1 +) ( + input wire clk, + input wire reset, + + input wire [CORE_ID_WIDTH-1:0] core_id, + + VX_warp_shared_if.slave warp_shared_if [NUM_REQS], +`ifdef VX_CFG_EXT_DXA_ENABLE + VX_txbar_bus_if.slave txbar_bus_if, +`endif + input wire [`VX_CFG_NUM_WARPS-1:0] active_warps, + + // barrier release + output wire bar_unlock_valid, + output wire [`VX_CFG_NUM_WARPS-1:0] bar_unlock_mask, + VX_gbar_bus_if.master gbar_bus_if, + + // wspawn, applied once a single warp remains + output wire wspawn_apply, + output wire [`VX_CFG_NUM_WARPS-1:0] wspawn_wmask, + output wire [PC_BITS-1:0] wspawn_pc, + output wire [NW_WIDTH-1:0] wspawn_wid, + output wire [`VX_CFG_MEM_ADDR_WIDTH-1:0] wspawn_mscratch, + + // warp retirement + output wire warp_done, + output wire [NW_WIDTH-1:0] warp_done_wid, + + output wire wspawn_pending +); + localparam REQ_DATAW = $bits(warp_shared_req_t); + localparam SEL_W = `LOG2UP(NUM_REQS); + localparam DIRECT = (NUM_REQS == 1); + + wire [NUM_REQS-1:0] req_valid_in; + wire [NUM_REQS-1:0][REQ_DATAW-1:0] req_data_in; + wire [NUM_REQS-1:0] req_ready_in; + for (genvar i = 0; i < NUM_REQS; ++i) begin : g_req_in + assign req_valid_in[i] = warp_shared_if[i].req_valid; + assign req_data_in[i] = warp_shared_if[i].req_data; + assign warp_shared_if[i].req_ready = req_ready_in[i]; + end + + wire req_valid; + warp_shared_req_t req; + wire [SEL_W-1:0] req_sel; + + VX_stream_arb #( + .NUM_INPUTS (NUM_REQS), + .NUM_OUTPUTS (1), + .DATAW (REQ_DATAW), + .ARBITER ("R"), + .OUT_BUF (0) + ) req_arb ( + .clk (clk), + .reset (reset), + .valid_in (req_valid_in), + .ready_in (req_ready_in), + .data_in (req_data_in), + .data_out (req), + .valid_out (req_valid), + .ready_out (1'b1), + .sel_out (req_sel) + ); + + wire is_bar = req_valid && (req.kind == WSH_BAR); + wire is_wspawn = req_valid && (req.kind == WSH_WSPAWN); + wire is_exit = req_valid && (req.kind == WSH_EXIT); + + // wspawn and retirement act on the registered request + localparam ACT_DATAW = NW_WIDTH + $bits(wspawn_t) + `VX_CFG_MEM_ADDR_WIDTH; + wire act_wspawn, act_exit; + wire [NW_WIDTH-1:0] act_wid; + wspawn_t act_wspawn_data; + wire [`VX_CFG_MEM_ADDR_WIDTH-1:0] act_mscratch; + if (DIRECT) begin : g_act_reg + VX_pipe_register #( + .DATAW (2 + ACT_DATAW), + .RESETW (2) + ) act_reg ( + .clk (clk), + .reset (reset), + .enable (1'b1), + .data_in ({is_wspawn, is_exit, req.wid, req.wspawn, req.mscratch}), + .data_out ({act_wspawn, act_exit, act_wid, act_wspawn_data, act_mscratch}) + ); + end else begin : g_act + assign {act_wspawn, act_exit, act_wid, act_wspawn_data, act_mscratch} + = {is_wspawn, is_exit, req.wid, req.wspawn, req.mscratch}; + end + + // barrier unit + + wire txbar_req_valid; + barrier_t txbar_req; + wire [BAR_ADDR_W-1:0] bar_read_addr; +`ifdef VX_CFG_EXT_DXA_ENABLE + assign txbar_bus_if.ready = ~is_bar; + wire txbar_fire = txbar_bus_if.valid && txbar_bus_if.ready; + assign bar_read_addr = is_bar ? req.bar_addr : txbar_bus_if.data.addr; + // a completion releases one pending transaction event + barrier_t txbar_req_w; + assign txbar_req_w.id = '0; + assign txbar_req_w.is_event = 1'b1; + assign txbar_req_w.is_sync = 1'b0; + assign txbar_req_w.is_global = 1'b0; + assign txbar_req_w.is_arrive = 1'b0; + assign txbar_req_w.phase = ~txbar_bus_if.data.is_done; + assign txbar_req_w.size_m1 = '0; + VX_pipe_register #( + .DATAW (1 + $bits(barrier_t)), + .RESETW (1) + ) txbar_reg ( + .clk (clk), + .reset (reset), + .enable (1'b1), + .data_in ({txbar_fire, txbar_req_w}), + .data_out ({txbar_req_valid, txbar_req}) + ); +`else + assign bar_read_addr = req.bar_addr; + assign txbar_req_valid = 1'b0; + assign txbar_req = '0; +`endif + + // the barrier request applies the cycle after its phase is read, as the + // read must see the barrier before this arrival + reg bar_valid_r; + barrier_t bar_r; + reg [NW_WIDTH-1:0] bar_wid_r; + always @(posedge clk) begin + if (reset) begin + bar_valid_r <= 1'b0; + end else begin + bar_valid_r <= is_bar; + end + bar_r <= req.bar; + bar_wid_r <= req.wid; + end + + wire bar_read_phase; + + VX_bar_unit #( + .INSTANCE_ID (`SFORMATF(("%s-barrier", INSTANCE_ID))) + ) bar_unit ( + .clk (clk), + .reset (reset), + .core_id (core_id), + .req_valid (bar_valid_r || txbar_req_valid), + .req_wid (bar_wid_r), + .req_data (bar_valid_r ? bar_r : txbar_req), + .read_addr (bar_read_addr), + .read_phase (bar_read_phase), + .active_warps (active_warps), + .gbar_bus_if (gbar_bus_if), + .unlock_valid (bar_unlock_valid), + .unlock_mask (bar_unlock_mask) + ); + + // wspawn + + reg wspawn_valid_r; + wspawn_t wspawn_r; + reg [NW_WIDTH-1:0] wspawn_wid_r; + reg [`VX_CFG_MEM_ADDR_WIDTH-1:0] wspawn_mscratch_r; + reg is_single_warp; + + wire [`CLOG2(`VX_CFG_NUM_WARPS+1)-1:0] active_warps_cnt; + `POP_COUNT(active_warps_cnt, active_warps); + + assign wspawn_apply = wspawn_valid_r && is_single_warp; + + always @(posedge clk) begin + if (reset) begin + wspawn_valid_r <= 1'b0; + is_single_warp <= 1'b0; + end else begin + is_single_warp <= (active_warps_cnt == $bits(active_warps_cnt)'(1)); + if (act_wspawn) begin + wspawn_valid_r <= 1'b1; + end + if (wspawn_apply) begin + wspawn_valid_r <= 1'b0; + end + end + if (act_wspawn) begin + wspawn_r <= act_wspawn_data; + wspawn_wid_r <= act_wid; + wspawn_mscratch_r <= act_mscratch; + end + end + + assign wspawn_wmask = wspawn_r.wmask; + assign wspawn_pc = wspawn_r.pc; + assign wspawn_wid = wspawn_wid_r; + assign wspawn_mscratch = wspawn_mscratch_r; + assign wspawn_pending = wspawn_valid_r; + + // warp retirement + + assign warp_done = act_exit; + assign warp_done_wid = act_wid; + + // completion + + wire [NUM_REQS-1:0] rsp_valid = NUM_REQS'(req_valid && req.has_rsp) << req_sel; + wire [NUM_REQS-1:0] rsp_valid_out; + wire rsp_phase_out; + if (DIRECT) begin : g_rsp_direct + assign rsp_valid_out = rsp_valid; + assign rsp_phase_out = bar_read_phase; + end else begin : g_rsp_reg + reg [NUM_REQS-1:0] rsp_valid_r; + reg rsp_phase_r; + always @(posedge clk) begin + if (reset) begin + rsp_valid_r <= '0; + end else begin + rsp_valid_r <= rsp_valid; + end + rsp_phase_r <= bar_read_phase; + end + assign rsp_valid_out = rsp_valid_r; + assign rsp_phase_out = rsp_phase_r; + end + + for (genvar i = 0; i < NUM_REQS; ++i) begin : g_rsp + assign warp_shared_if[i].rsp_valid = rsp_valid_out[i]; + assign warp_shared_if[i].rsp_phase = rsp_phase_out; + end + +`ifdef DBG_TRACE_PIPELINE + always @(posedge clk) begin + if (req_valid) begin + `TRACE(1, ("%t: %s request: sel=%0d, kind=%0d, wid=%0d\n", $time, INSTANCE_ID, req_sel, req.kind, req.wid)) + end + end +`endif + +endmodule diff --git a/hw/rtl/core/VX_wctl_unit.sv b/hw/rtl/core/VX_wctl_unit.sv index f2a3857ab5..7a2a0117de 100644 --- a/hw/rtl/core/VX_wctl_unit.sv +++ b/hw/rtl/core/VX_wctl_unit.sv @@ -22,16 +22,16 @@ module VX_wctl_unit import VX_gpu_pkg::*; #( // Inputs VX_execute_if.slave execute_if, - VX_txbar_bus_if.slave txbar_bus_if, // Outputs VX_warp_ctl_if.master warp_ctl_if, + VX_warp_shared_if.master warp_shared_if, VX_result_if.master result_if ); `UNUSED_SPARAM (INSTANCE_ID) localparam LANE_BITS = `CLOG2(NUM_LANES); localparam PID_BITS = `CLOG2(`VX_CFG_NUM_THREADS / NUM_LANES); - localparam WCTL_WIDTH = $bits(tmc_t) + $bits(wspawn_t) + $bits(split_t) + $bits(join_t) + $bits(barrier_t); + localparam WCTL_WIDTH = $bits(tmc_t) + $bits(split_t) + $bits(join_t); `UNUSED_VAR (execute_if.data.rs3_data) @@ -50,7 +50,7 @@ module VX_wctl_unit import VX_gpu_pkg::*; #( wire is_wsync = (execute_if.data.op_type == INST_SFU_WSYNC); wire wctl_valid; - wire wspawn_valid, tmc_valid, split_valid, sjoin_valid, bar_valid, wsync_valid; + wire tmc_valid, split_valid, sjoin_valid, wsync_valid; wire [`UP(LANE_BITS)-1:0] last_tid; if (LANE_BITS != 0) begin : g_last_tid @@ -82,9 +82,10 @@ module VX_wctl_unit import VX_gpu_pkg::*; #( logic [`VX_CFG_NUM_THREADS-1:0] else_tmask; if (PID_BITS != 0) begin : g_pid - reg [`VX_CFG_NUM_WARPS-1:0][2*`VX_CFG_NUM_THREADS-1:0] tmask_table; + reg [SFU_BLOCK_WARPS-1:0][2*`VX_CFG_NUM_THREADS-1:0] tmask_table; - wire [2*`VX_CFG_NUM_THREADS-1:0] tmask_r = tmask_table[execute_if.data.header.wid]; + wire [SFU_BLOCK_NW_W-1:0] tmask_lid = wid_to_sfu_local(execute_if.data.header.wid); + wire [2*`VX_CFG_NUM_THREADS-1:0] tmask_r = tmask_table[tmask_lid]; always @(*) begin {else_tmask, then_tmask} = execute_if.data.header.sop ? '0 : tmask_r; @@ -94,7 +95,7 @@ module VX_wctl_unit import VX_gpu_pkg::*; #( always @(posedge clk) begin if (execute_if.valid) begin - tmask_table[execute_if.data.header.wid] <= {else_tmask, then_tmask}; + tmask_table[tmask_lid] <= {else_tmask, then_tmask}; end end end else begin : g_no_pid @@ -108,7 +109,6 @@ module VX_wctl_unit import VX_gpu_pkg::*; #( // tmc / pred wire [`VX_CFG_NUM_THREADS-1:0] pred_mask = has_then ? then_tmask : rs2_data[`VX_CFG_NUM_THREADS-1:0]; - assign tmc_valid = wctl_valid && (is_tmc || is_pred); assign tmc.tmask = is_pred ? pred_mask : rs1_data[`VX_CFG_NUM_THREADS-1:0]; // split @@ -137,62 +137,144 @@ module VX_wctl_unit import VX_gpu_pkg::*; #( wire [BAR_ADDR_W-1:0] wctl_bar_addr; `CONCAT(wctl_bar_addr, rs1_data[NW_BITS-1:0], rs1_data[BAR_ID_SHIFT +: NB_BITS], NW_BITS, NB_BITS) - wire wctl_bar_enable = wctl_valid && is_bar; - // expect_tx variant of barrier_arrive: rs2[31]=1 flags pre-registration // of pending transaction events. Routed through the existing event path // (is_event + phase=1) so VX_bar_unit's increment branch handles it. - wire is_tx_expect = wctl_bar_enable + wire is_tx_expect = is_bar && execute_if.data.op_args.wctl.is_bar_arrive && rs2_data[31]; - assign bar_valid = wctl_bar_enable || txbar_bus_if.valid; assign bar.id = rs1_data[BAR_ID_SHIFT +: NB_BITS]; - assign bar.is_event = (txbar_bus_if.valid && ~wctl_bar_enable) || is_tx_expect; + assign bar.is_event = is_tx_expect; assign bar.is_sync = execute_if.data.op_args.wctl.is_sync_bar; - assign bar.is_global= wctl_bar_enable && rs1_data[31]; + assign bar.is_global= rs1_data[31]; // expect_tx is not a normal arrival (it does not advance arrive_count) assign bar.is_arrive= (execute_if.data.op_args.wctl.is_bar_arrive && ~rs2_data[31]) || execute_if.data.op_args.wctl.is_sync_bar; - // expect_tx → phase=1 (attach/increment); txbar release → phase=0 - assign bar.phase = is_tx_expect ? 1'b1 : (wctl_bar_enable ? rs2_data[0] : ~txbar_bus_if.data.is_done); + // expect_tx → phase=1 (attach/increment) + assign bar.phase = is_tx_expect ? 1'b1 : rs2_data[0]; // For expect_tx: size_m1 carries (count - 1); for normal arrive: (num_warps - 1). // bar_unit will + 1 when consuming on the event path so the actual increment is `count`. assign bar.size_m1 = rs2_data[BAR_SIZE_W-1:0] - BAR_SIZE_W'(1); - assign txbar_bus_if.ready = ~wctl_bar_enable; - // wspawn wire [`VX_CFG_NUM_WARPS-1:0] wspawn_wmask; for (genvar i = 0; i < `VX_CFG_NUM_WARPS; ++i) begin : g_wspawn_wmask assign wspawn_wmask[i] = (i < rs1_data[NW_BITS:0]) && (i != execute_if.data.header.wid); end - assign wspawn_valid = wctl_valid && is_wspawn; assign wspawn.wmask = wspawn_wmask; assign wspawn.pc = from_fullPC(rs2_data); // Scheduler response setup assign warp_ctl_if.dvstack_wid = execute_if.data.header.wid; - assign warp_ctl_if.bar_addr = wctl_bar_enable ? wctl_bar_addr : txbar_bus_if.data.addr; - - // Send scheduler request // WSYNC: hold execute until this warp's pipeline is drained (alm_empty=1 // means only WSYNC itself is pending — all prior instructions committed). wire wsync_drain = execute_if.valid && is_wsync - && !warp_ctl_if.warp_pending_alm_empty[execute_if.data.header.wid]; + && !warp_ctl_if.warp_pending_alm_empty[wid_to_sfu_local(execute_if.data.header.wid)]; // BAR (vx_barrier or vx_barrier_arrive) drains LSU before continuing. wire bar_drain = execute_if.valid && is_bar && !warp_ctl_if.lsu_sched_drained; + // Barriers, wspawn and warp retirement are core-wide requests to the shared + // unit. Only a barrier arrive writes a result, the barrier phase, so it alone + // completes once the unit has served it; every other request is posted, + // completing as it is accepted, and the warp resumes through its unlock path. + wire bar_rsp_valid = is_bar && execute_if.data.op_args.wctl.is_bar_arrive; + wire is_exit = (is_tmc || is_pred) && (tmc.tmask == 0); + wire is_shared = execute_if.data.header.eop && (is_bar || is_wspawn || is_exit); + wire sh_posted = is_shared && ~bar_rsp_valid; + + // an exit is retired by the shared unit, which deactivates the warp itself + assign tmc_valid = wctl_valid && (is_tmc || is_pred) && !is_shared; + + localparam SH_IDLE = 2'd0, SH_WAIT = 2'd1, SH_DONE = 2'd2; + reg [1:0] sh_state; + reg sh_phase; + + warp_shared_req_t sh_req; + assign sh_req.kind = is_bar ? WSH_BAR : (is_wspawn ? WSH_WSPAWN : WSH_EXIT); + assign sh_req.has_rsp = ~sh_posted; + assign sh_req.wid = execute_if.data.header.wid; + assign sh_req.bar = bar; + assign sh_req.bar_addr = wctl_bar_addr; + assign sh_req.wspawn = wspawn; + assign sh_req.mscratch = warp_ctl_if.mscratch; + + // A sole requester reaches the shared unit directly and is answered in the + // cycle it asks; with several, requests are registered toward the unit's + // arbiter and answered two cycles later. + localparam SH_DIRECT = (`VX_CFG_NUM_SFU_BLOCKS == 1); + + wire rsp_buf_ready; + // a posted or directly answered request is sent only in the cycle its instruction completes + wire sh_req_valid = execute_if.valid && is_shared && (sh_state == SH_IDLE) && !bar_drain + && ((!sh_posted && !SH_DIRECT) || rsp_buf_ready); + wire sh_req_ready; + + if (SH_DIRECT) begin : g_sh_direct + assign warp_shared_if.req_valid = sh_req_valid; + assign warp_shared_if.req_data = sh_req; + assign sh_req_ready = warp_shared_if.req_ready; + end else begin : g_sh_buf + VX_elastic_buffer #( + .DATAW ($bits(warp_shared_req_t)), + .SIZE (2), + .OUT_REG (1) + ) sh_req_buf ( + .clk (clk), + .reset (reset), + .valid_in (sh_req_valid), + .ready_in (sh_req_ready), + .data_in (sh_req), + .data_out (warp_shared_if.req_data), + .valid_out (warp_shared_if.req_valid), + .ready_out (warp_shared_if.req_ready) + ); + end + + wire sh_req_fire = sh_req_valid && sh_req_ready; + wire sh_ok = ~is_shared || (sh_state == SH_DONE) + || ((sh_state == SH_WAIT) && warp_shared_if.rsp_valid) + || (sh_req_fire && (sh_posted || warp_shared_if.rsp_valid)); + wire exec_ok = rsp_buf_ready && !wsync_drain && sh_ok && (is_shared || !bar_drain); + assign execute_if.ready = exec_ok; + wire execute_fire = execute_if.valid && execute_if.ready; assign wctl_valid = execute_fire && execute_if.data.header.eop; + always @(posedge clk) begin + if (reset) begin + sh_state <= SH_IDLE; + end else begin + case (sh_state) + SH_IDLE: begin + if (sh_req_fire && !sh_posted && !execute_fire) begin + sh_state <= warp_shared_if.rsp_valid ? SH_DONE : SH_WAIT; + end + end + SH_WAIT: begin + if (warp_shared_if.rsp_valid) begin + sh_state <= execute_fire ? SH_IDLE : SH_DONE; + end + end + default: begin + if (execute_fire) begin + sh_state <= SH_IDLE; + end + end + endcase + end + if (warp_shared_if.rsp_valid) begin + sh_phase <= warp_shared_if.rsp_phase; + end + end + assign wsync_valid = wctl_valid && is_wsync; VX_pipe_register #( - .DATAW (6 + NW_WIDTH + WCTL_WIDTH), + .DATAW (4 + NW_WIDTH + WCTL_WIDTH), .RESETW (1) ) wctl_reg ( .clk (clk), @@ -200,56 +282,46 @@ module VX_wctl_unit import VX_gpu_pkg::*; #( .enable (1'b1), .data_in ({ tmc_valid, - wspawn_valid, split_valid, sjoin_valid, - bar_valid, wsync_valid, execute_if.data.header.wid, tmc, - wspawn, split, - sjoin, - bar + sjoin }), .data_out ({ warp_ctl_if.tmc_valid, - warp_ctl_if.wspawn_valid, warp_ctl_if.split_valid, warp_ctl_if.sjoin_valid, - warp_ctl_if.bar_valid, warp_ctl_if.wsync_valid, warp_ctl_if.wid, warp_ctl_if.tmc, - warp_ctl_if.wspawn, warp_ctl_if.split, - warp_ctl_if.sjoin, - warp_ctl_if.bar + warp_ctl_if.sjoin }) ); // Send result + wire rsp_phase = warp_shared_if.rsp_valid ? warp_shared_if.rsp_phase : sh_phase; wire [DV_STACK_SIZEW-1:0] dvstack_ptr_r; - wire bar_rsp_valid = is_bar && execute_if.data.op_args.wctl.is_bar_arrive; wire bar_rsp_valid_r; wire bar_phase_r; - wire rsp_buf_ready; VX_elastic_buffer #( .DATAW ($bits(sfu_header_t) + DV_STACK_SIZEW + 1 + 1), .SIZE (2) ) rsp_buf ( .clk (clk), .reset (reset), - .valid_in (execute_if.valid && !wsync_drain && !bar_drain), + .valid_in (execute_if.valid && exec_ok), .ready_in (rsp_buf_ready), - .data_in ({execute_if.data.header, warp_ctl_if.dvstack_ptr, warp_ctl_if.bar_phase, bar_rsp_valid}), - .data_out ({result_if.data.header, dvstack_ptr_r, bar_phase_r, bar_rsp_valid_r}), + .data_in ({execute_if.data.header, warp_ctl_if.dvstack_ptr, rsp_phase, bar_rsp_valid}), + .data_out ({result_if.data.header, dvstack_ptr_r, bar_phase_r, bar_rsp_valid_r}), .valid_out (result_if.valid), .ready_out (result_if.ready) ); - assign execute_if.ready = rsp_buf_ready && !wsync_drain && !bar_drain; // Result data for (genvar i = 0; i < NUM_LANES; ++i) begin : g_result_if diff --git a/hw/rtl/dxa/VX_dxa_unit.sv b/hw/rtl/dxa/VX_dxa_unit.sv index ed9136fd95..cd3d8c8b13 100644 --- a/hw/rtl/dxa/VX_dxa_unit.sv +++ b/hw/rtl/dxa/VX_dxa_unit.sv @@ -14,12 +14,13 @@ `include "VX_define.vh" module VX_dxa_unit import VX_gpu_pkg::*, VX_dxa_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - parameter CORE_ID = 0 + parameter `STRING INSTANCE_ID = "" ) ( input wire clk, input wire reset, + input wire [CORE_ID_WIDTH-1:0] core_id, + VX_execute_if.slave execute_if, VX_result_if.master result_if, VX_dxa_req_bus_if.master dxa_req_bus_if @@ -58,7 +59,10 @@ module VX_dxa_unit import VX_gpu_pkg::*, VX_dxa_pkg::*; #( // Build dxa_req payload dxa_req_data_t dxa_req_data_in; - assign dxa_req_data_in.core_id = NC_WIDTH'(CORE_ID); + assign dxa_req_data_in.core_id = NC_WIDTH'(core_id); + if (CORE_ID_WIDTH > NC_WIDTH) begin : g_core_id_hi_unused + `UNUSED_VAR (core_id[CORE_ID_WIDTH-1:NC_WIDTH]) + end assign dxa_req_data_in.uuid = execute_if.data.header.uuid; assign dxa_req_data_in.wid = execute_if.data.header.wid; assign dxa_req_data_in.smem_addr = lmem_rel_byte_addr[DXA_SMEM_ADDR_W-1:0]; diff --git a/hw/rtl/dxa/VX_dxa_worker.sv b/hw/rtl/dxa/VX_dxa_worker.sv index 8f2a9500ea..fd8f0d5e25 100644 --- a/hw/rtl/dxa/VX_dxa_worker.sv +++ b/hw/rtl/dxa/VX_dxa_worker.sv @@ -45,6 +45,8 @@ module VX_dxa_worker import VX_gpu_pkg::*, VX_dxa_pkg::*; #( localparam MAX_OUTSTANDING = `VX_CFG_DXA_MAX_INFLIGHT; localparam TAG_W = `CLOG2(MAX_OUTSTANDING); localparam SEQ_W = `CLOG2(MAX_OUTSTANDING + 1); + // the outstanding-request queue and its tags need a power of two of at least 2 + `STATIC_ASSERT (`IS_POW2(MAX_OUTSTANDING) && (MAX_OUTSTANDING >= 2), ("invalid parameter")) // ════════════════════════════════════════════════════════════════════ // Inter-module wires diff --git a/hw/rtl/fpu/VX_fpu_block.sv b/hw/rtl/fpu/VX_fpu_block.sv new file mode 100644 index 0000000000..eb97c427e6 --- /dev/null +++ b/hw/rtl/fpu/VX_fpu_block.sv @@ -0,0 +1,261 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_fpu_define.vh" + +// One FPU block: the FP datapath with its tag store and fflags accumulation. +module VX_fpu_block import VX_gpu_pkg::*, VX_fpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "", + parameter NUM_LANES = 1, + parameter PARTIAL_BW = 0 +) ( + input wire clk, + input wire reset, + + input wire [`LOG2UP(`VX_CFG_NUM_FPU_BLOCKS)-1:0] block_idx, + + VX_execute_if.slave execute_if, + VX_result_if.master result_if, + VX_fpu_csr_if.master fpu_csr_if +); + `UNUSED_SPARAM (INSTANCE_ID) + localparam PID_BITS = `CLOG2(`VX_CFG_NUM_THREADS / NUM_LANES); + localparam TAG_WIDTH = `LOG2UP(`VX_CFG_FPU_QUEUE_SIZE); + + wire fpu_req_valid, fpu_req_ready; + wire fpu_rsp_valid, fpu_rsp_ready; + wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] fpu_rsp_result; + fflags_t fpu_rsp_fflags; + wire fpu_rsp_has_fflags; + + fpu_header_t fpu_hdr, fpu_hdr_wb, fpu_hdr_store; + + // Carry the decoded wb through the tag store: FPU compares may target + // integer x0 (e.g. feq.s x0, ...), which decode does not reserve in the + // scoreboard, so forcing wb=1 on readback would trip the invalid + // writeback assertion. + always_comb begin + fpu_hdr_store = execute_if.data.header; + end + + always_comb begin + fpu_hdr_wb = fpu_hdr; + end + + wire [TAG_WIDTH-1:0] fpu_req_tag, fpu_rsp_tag; + wire mdata_full; + + wire [INST_FMT_BITS-1:0] fpu_fmt = execute_if.data.op_args.fpu.fmt; + wire [INST_FRM_BITS-1:0] fpu_frm = execute_if.data.op_args.fpu.frm; + + wire execute_fire = execute_if.valid && execute_if.ready; + wire fpu_rsp_fire = fpu_rsp_valid && fpu_rsp_ready; + + VX_index_buffer #( + .DATAW ($bits(fpu_header_t)), + .SIZE (`VX_CFG_FPU_QUEUE_SIZE) + ) tag_store ( + .clk (clk), + .reset (reset), + .acquire_en (execute_fire), + .write_addr (fpu_req_tag), + .write_data (fpu_hdr_store), + .read_data (fpu_hdr), + .read_addr (fpu_rsp_tag), + .release_en (fpu_rsp_fire), + .full (mdata_full), + `UNUSED_PIN (empty) + ); + + // resolve dynamic FRM from CSR + wire [INST_FRM_BITS-1:0] fpu_req_frm; + `ASSIGN_BLOCKED_WID (fpu_csr_if.read_wid, execute_if.data.header.wid, block_idx, `VX_CFG_NUM_FPU_BLOCKS) + if (`VX_CFG_NUM_FPU_BLOCKS == 1) begin : g_block_idx_unused + `UNUSED_VAR (block_idx) + end + assign fpu_req_frm = (execute_if.data.op_type != INST_FPU_MISC + && fpu_frm == INST_FRM_DYN) ? fpu_csr_if.read_frm : fpu_frm; + + // submit FPU request + + assign fpu_req_valid = execute_if.valid && ~mdata_full; + assign execute_if.ready = fpu_req_ready && ~mdata_full; + +`ifdef VX_CFG_FPU_TYPE_DPI + + VX_fpu_dpi #( + .NUM_LANES (NUM_LANES), + .TAG_WIDTH (TAG_WIDTH), + .OUT_BUF (PARTIAL_BW ? 1 : 3) + ) fpu_dpi ( + .clk (clk), + .reset (reset), + + .valid_in (fpu_req_valid), + .mask_in (execute_if.data.header.tmask), + .op_type (execute_if.data.op_type), + .fmt (fpu_fmt), + .frm (fpu_req_frm), + .dataa (execute_if.data.rs1_data), + .datab (execute_if.data.rs2_data), + .datac (execute_if.data.rs3_data), + .tag_in (fpu_req_tag), + .ready_in (fpu_req_ready), + + .valid_out (fpu_rsp_valid), + .result (fpu_rsp_result), + .has_fflags (fpu_rsp_has_fflags), + .fflags (fpu_rsp_fflags), + .tag_out (fpu_rsp_tag), + .ready_out (fpu_rsp_ready) + ); + +`elsif VX_CFG_FPU_TYPE_FPNEW + + VX_fpu_fpnew #( + .NUM_LANES (NUM_LANES), + .TAG_WIDTH (TAG_WIDTH), + .OUT_BUF (PARTIAL_BW ? 1 : 3) + ) fpu_fpnew ( + .clk (clk), + .reset (reset), + + .valid_in (fpu_req_valid), + .mask_in (execute_if.data.header.tmask), + .op_type (execute_if.data.op_type), + .fmt (fpu_fmt), + .frm (fpu_req_frm), + .dataa (execute_if.data.rs1_data), + .datab (execute_if.data.rs2_data), + .datac (execute_if.data.rs3_data), + .tag_in (fpu_req_tag), + .ready_in (fpu_req_ready), + + .valid_out (fpu_rsp_valid), + .result (fpu_rsp_result), + .has_fflags (fpu_rsp_has_fflags), + .fflags (fpu_rsp_fflags), + .tag_out (fpu_rsp_tag), + .ready_out (fpu_rsp_ready) + ); + +`elsif VX_CFG_FPU_TYPE_DSP + + VX_fpu_dsp #( + .NUM_LANES (NUM_LANES), + .TAG_WIDTH (TAG_WIDTH), + .OUT_BUF (PARTIAL_BW ? 1 : 3) + ) fpu_dsp ( + .clk (clk), + .reset (reset), + + .valid_in (fpu_req_valid), + .mask_in (execute_if.data.header.tmask), + .op_type (execute_if.data.op_type), + .fmt (fpu_fmt), + .frm (fpu_req_frm), + .dataa (execute_if.data.rs1_data), + .datab (execute_if.data.rs2_data), + .datac (execute_if.data.rs3_data), + .tag_in (fpu_req_tag), + .ready_in (fpu_req_ready), + + .valid_out (fpu_rsp_valid), + .result (fpu_rsp_result), + .has_fflags (fpu_rsp_has_fflags), + .fflags (fpu_rsp_fflags), + .tag_out (fpu_rsp_tag), + .ready_out (fpu_rsp_ready) + ); + +`elsif VX_CFG_FPU_TYPE_STD + + VX_fpu_std #( + .NUM_LANES (NUM_LANES), + .TAG_WIDTH (TAG_WIDTH), + .OUT_BUF (PARTIAL_BW ? 1 : 3) + ) fpu_std ( + .clk (clk), + .reset (reset), + + .valid_in (fpu_req_valid), + .mask_in (execute_if.data.header.tmask), + .op_type (execute_if.data.op_type), + .fmt (fpu_fmt), + .frm (fpu_req_frm), + .dataa (execute_if.data.rs1_data), + .datab (execute_if.data.rs2_data), + .datac (execute_if.data.rs3_data), + .tag_in (fpu_req_tag), + .ready_in (fpu_req_ready), + + .valid_out (fpu_rsp_valid), + .result (fpu_rsp_result), + .has_fflags (fpu_rsp_has_fflags), + .fflags (fpu_rsp_fflags), + .tag_out (fpu_rsp_tag), + .ready_out (fpu_rsp_ready) + ); + +`endif + + // handle CSR update + fflags_t fpu_rsp_fflags_q; + + if (PID_BITS != 0) begin : g_fflags_pid + fflags_t fpu_rsp_fflags_r; + always @(posedge clk) begin + if (reset) begin + fpu_rsp_fflags_r <= '0; + end else if (fpu_rsp_fire) begin + fpu_rsp_fflags_r <= fpu_hdr.eop ? '0 : (fpu_rsp_fflags_r | fpu_rsp_fflags); + end + end + assign fpu_rsp_fflags_q = fpu_rsp_fflags_r | fpu_rsp_fflags; + end else begin : g_fflags_no_pid + assign fpu_rsp_fflags_q = fpu_rsp_fflags; + end + + VX_fpu_csr_if fpu_csr_tmp_if(); + assign fpu_csr_tmp_if.write_enable = fpu_rsp_fire && fpu_hdr.eop && fpu_rsp_has_fflags; + `ASSIGN_BLOCKED_WID (fpu_csr_tmp_if.write_wid, fpu_hdr.wid, block_idx, `VX_CFG_NUM_FPU_BLOCKS) + assign fpu_csr_tmp_if.write_fflags = fpu_rsp_fflags_q; + + VX_pipe_register #( + .DATAW (1 + NW_WIDTH + $bits(fflags_t)), + .RESETW (1) + ) fpu_csr_reg ( + .clk (clk), + .reset (reset), + .enable (1'b1), + .data_in ({fpu_csr_tmp_if.write_enable, fpu_csr_tmp_if.write_wid, fpu_csr_tmp_if.write_fflags}), + .data_out ({fpu_csr_if.write_enable, fpu_csr_if.write_wid, fpu_csr_if.write_fflags}) + ); + + // send response + + VX_elastic_buffer #( + .DATAW ($bits(fpu_header_t) + (NUM_LANES * `VX_CFG_XLEN)), + .SIZE (0) + ) rsp_buf ( + .clk (clk), + .reset (reset), + .valid_in (fpu_rsp_valid), + .ready_in (fpu_rsp_ready), + .data_in ({fpu_hdr_wb, fpu_rsp_result}), + .data_out (result_if.data), + .valid_out (result_if.valid), + .ready_out (result_if.ready) + ); + +endmodule diff --git a/hw/rtl/fpu/VX_fpu_unit.sv b/hw/rtl/fpu/VX_fpu_unit.sv index 9e2386d54f..b64bf5825e 100644 --- a/hw/rtl/fpu/VX_fpu_unit.sv +++ b/hw/rtl/fpu/VX_fpu_unit.sv @@ -29,8 +29,6 @@ module VX_fpu_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; #( `UNUSED_SPARAM (INSTANCE_ID) localparam BLOCK_SIZE = `VX_CFG_NUM_FPU_BLOCKS; localparam NUM_LANES = `VX_CFG_NUM_FPU_LANES; - localparam PID_BITS = `CLOG2(`VX_CFG_NUM_THREADS / NUM_LANES); - localparam TAG_WIDTH = `LOG2UP(`VX_CFG_FPU_QUEUE_SIZE); localparam PARTIAL_BW = (BLOCK_SIZE != `VX_CFG_ISSUE_WIDTH) || (NUM_LANES != `VX_CFG_SIMD_WIDTH); VX_execute_if #( @@ -44,6 +42,7 @@ module VX_fpu_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; #( ) lane_dispatch ( .clk (clk), .reset (reset), + .port_base ('0), .dispatch_if(dispatch_if), .execute_if (per_block_execute_if) ); @@ -53,227 +52,17 @@ module VX_fpu_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; #( ) per_block_result_if[BLOCK_SIZE](); for (genvar block_idx = 0; block_idx < BLOCK_SIZE; ++block_idx) begin : g_blocks - wire fpu_req_valid, fpu_req_ready; - wire fpu_rsp_valid, fpu_rsp_ready; - wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] fpu_rsp_result; - fflags_t fpu_rsp_fflags; - wire fpu_rsp_has_fflags; - - fpu_header_t fpu_hdr, fpu_hdr_wb, fpu_hdr_store; - - // Carry the decoded wb through the tag store: FPU compares may target - // integer x0 (e.g. feq.s x0, ...), which decode does not reserve in the - // scoreboard, so forcing wb=1 on readback would trip the invalid - // writeback assertion. - always_comb begin - fpu_hdr_store = per_block_execute_if[block_idx].data.header; - end - - always_comb begin - fpu_hdr_wb = fpu_hdr; - end - - wire [TAG_WIDTH-1:0] fpu_req_tag, fpu_rsp_tag; - wire mdata_full; - - wire [INST_FMT_BITS-1:0] fpu_fmt = per_block_execute_if[block_idx].data.op_args.fpu.fmt; - wire [INST_FRM_BITS-1:0] fpu_frm = per_block_execute_if[block_idx].data.op_args.fpu.frm; - - wire execute_fire = per_block_execute_if[block_idx].valid && per_block_execute_if[block_idx].ready; - wire fpu_rsp_fire = fpu_rsp_valid && fpu_rsp_ready; - - VX_index_buffer #( - .DATAW ($bits(fpu_header_t)), - .SIZE (`VX_CFG_FPU_QUEUE_SIZE) - ) tag_store ( - .clk (clk), - .reset (reset), - .acquire_en (execute_fire), - .write_addr (fpu_req_tag), - .write_data (fpu_hdr_store), - .read_data (fpu_hdr), - .read_addr (fpu_rsp_tag), - .release_en (fpu_rsp_fire), - .full (mdata_full), - `UNUSED_PIN (empty) - ); - - // resolve dynamic FRM from CSR - wire [INST_FRM_BITS-1:0] fpu_req_frm; - `ASSIGN_BLOCKED_WID (fpu_csr_if[block_idx].read_wid, per_block_execute_if[block_idx].data.header.wid, block_idx, `VX_CFG_NUM_FPU_BLOCKS) - assign fpu_req_frm = (per_block_execute_if[block_idx].data.op_type != INST_FPU_MISC - && fpu_frm == INST_FRM_DYN) ? fpu_csr_if[block_idx].read_frm : fpu_frm; - - // submit FPU request - - assign fpu_req_valid = per_block_execute_if[block_idx].valid && ~mdata_full; - assign per_block_execute_if[block_idx].ready = fpu_req_ready && ~mdata_full; - - `ifdef VX_CFG_FPU_TYPE_DPI - - VX_fpu_dpi #( - .NUM_LANES (NUM_LANES), - .TAG_WIDTH (TAG_WIDTH), - .OUT_BUF (PARTIAL_BW ? 1 : 3) - ) fpu_dpi ( + VX_fpu_block #( + .INSTANCE_ID (INSTANCE_ID), + .NUM_LANES (NUM_LANES), + .PARTIAL_BW (PARTIAL_BW) + ) fpu_block ( .clk (clk), .reset (reset), - - .valid_in (fpu_req_valid), - .mask_in (per_block_execute_if[block_idx].data.header.tmask), - .op_type (per_block_execute_if[block_idx].data.op_type), - .fmt (fpu_fmt), - .frm (fpu_req_frm), - .dataa (per_block_execute_if[block_idx].data.rs1_data), - .datab (per_block_execute_if[block_idx].data.rs2_data), - .datac (per_block_execute_if[block_idx].data.rs3_data), - .tag_in (fpu_req_tag), - .ready_in (fpu_req_ready), - - .valid_out (fpu_rsp_valid), - .result (fpu_rsp_result), - .has_fflags (fpu_rsp_has_fflags), - .fflags (fpu_rsp_fflags), - .tag_out (fpu_rsp_tag), - .ready_out (fpu_rsp_ready) - ); - - `elsif VX_CFG_FPU_TYPE_FPNEW - - VX_fpu_fpnew #( - .NUM_LANES (NUM_LANES), - .TAG_WIDTH (TAG_WIDTH), - .OUT_BUF (PARTIAL_BW ? 1 : 3) - ) fpu_fpnew ( - .clk (clk), - .reset (reset), - - .valid_in (fpu_req_valid), - .mask_in (per_block_execute_if[block_idx].data.header.tmask), - .op_type (per_block_execute_if[block_idx].data.op_type), - .fmt (fpu_fmt), - .frm (fpu_req_frm), - .dataa (per_block_execute_if[block_idx].data.rs1_data), - .datab (per_block_execute_if[block_idx].data.rs2_data), - .datac (per_block_execute_if[block_idx].data.rs3_data), - .tag_in (fpu_req_tag), - .ready_in (fpu_req_ready), - - .valid_out (fpu_rsp_valid), - .result (fpu_rsp_result), - .has_fflags (fpu_rsp_has_fflags), - .fflags (fpu_rsp_fflags), - .tag_out (fpu_rsp_tag), - .ready_out (fpu_rsp_ready) - ); - - `elsif VX_CFG_FPU_TYPE_DSP - - VX_fpu_dsp #( - .NUM_LANES (NUM_LANES), - .TAG_WIDTH (TAG_WIDTH), - .OUT_BUF (PARTIAL_BW ? 1 : 3) - ) fpu_dsp ( - .clk (clk), - .reset (reset), - - .valid_in (fpu_req_valid), - .mask_in (per_block_execute_if[block_idx].data.header.tmask), - .op_type (per_block_execute_if[block_idx].data.op_type), - .fmt (fpu_fmt), - .frm (fpu_req_frm), - .dataa (per_block_execute_if[block_idx].data.rs1_data), - .datab (per_block_execute_if[block_idx].data.rs2_data), - .datac (per_block_execute_if[block_idx].data.rs3_data), - .tag_in (fpu_req_tag), - .ready_in (fpu_req_ready), - - .valid_out (fpu_rsp_valid), - .result (fpu_rsp_result), - .has_fflags (fpu_rsp_has_fflags), - .fflags (fpu_rsp_fflags), - .tag_out (fpu_rsp_tag), - .ready_out (fpu_rsp_ready) - ); - - `elsif VX_CFG_FPU_TYPE_STD - - VX_fpu_std #( - .NUM_LANES (NUM_LANES), - .TAG_WIDTH (TAG_WIDTH), - .OUT_BUF (PARTIAL_BW ? 1 : 3) - ) fpu_std ( - .clk (clk), - .reset (reset), - - .valid_in (fpu_req_valid), - .mask_in (per_block_execute_if[block_idx].data.header.tmask), - .op_type (per_block_execute_if[block_idx].data.op_type), - .fmt (fpu_fmt), - .frm (fpu_req_frm), - .dataa (per_block_execute_if[block_idx].data.rs1_data), - .datab (per_block_execute_if[block_idx].data.rs2_data), - .datac (per_block_execute_if[block_idx].data.rs3_data), - .tag_in (fpu_req_tag), - .ready_in (fpu_req_ready), - - .valid_out (fpu_rsp_valid), - .result (fpu_rsp_result), - .has_fflags (fpu_rsp_has_fflags), - .fflags (fpu_rsp_fflags), - .tag_out (fpu_rsp_tag), - .ready_out (fpu_rsp_ready) - ); - - `endif - - // handle CSR update - fflags_t fpu_rsp_fflags_q; - - if (PID_BITS != 0) begin : g_fflags_pid - fflags_t fpu_rsp_fflags_r; - always @(posedge clk) begin - if (reset) begin - fpu_rsp_fflags_r <= '0; - end else if (fpu_rsp_fire) begin - fpu_rsp_fflags_r <= fpu_hdr.eop ? '0 : (fpu_rsp_fflags_r | fpu_rsp_fflags); - end - end - assign fpu_rsp_fflags_q = fpu_rsp_fflags_r | fpu_rsp_fflags; - end else begin : g_fflags_no_pid - assign fpu_rsp_fflags_q = fpu_rsp_fflags; - end - - VX_fpu_csr_if fpu_csr_tmp_if(); - assign fpu_csr_tmp_if.write_enable = fpu_rsp_fire && fpu_hdr.eop && fpu_rsp_has_fflags; - `ASSIGN_BLOCKED_WID (fpu_csr_tmp_if.write_wid, fpu_hdr.wid, block_idx, `VX_CFG_NUM_FPU_BLOCKS) - assign fpu_csr_tmp_if.write_fflags = fpu_rsp_fflags_q; - - VX_pipe_register #( - .DATAW (1 + NW_WIDTH + $bits(fflags_t)), - .RESETW (1) - ) fpu_csr_reg ( - .clk (clk), - .reset (reset), - .enable (1'b1), - .data_in ({fpu_csr_tmp_if.write_enable, fpu_csr_tmp_if.write_wid, fpu_csr_tmp_if.write_fflags}), - .data_out ({fpu_csr_if[block_idx].write_enable, fpu_csr_if[block_idx].write_wid, fpu_csr_if[block_idx].write_fflags}) - ); - - // send response - - VX_elastic_buffer #( - .DATAW ($bits(fpu_header_t) + (NUM_LANES * `VX_CFG_XLEN)), - .SIZE (0) - ) rsp_buf ( - .clk (clk), - .reset (reset), - .valid_in (fpu_rsp_valid), - .ready_in (fpu_rsp_ready), - .data_in ({fpu_hdr_wb, fpu_rsp_result}), - .data_out (per_block_result_if[block_idx].data), - .valid_out (per_block_result_if[block_idx].valid), - .ready_out (per_block_result_if[block_idx].ready) + .block_idx (`LOG2UP(BLOCK_SIZE)'(block_idx)), + .execute_if (per_block_execute_if[block_idx]), + .result_if (per_block_result_if[block_idx]), + .fpu_csr_if (fpu_csr_if[block_idx]) ); end diff --git a/hw/rtl/interfaces/VX_sched_csr_if.sv b/hw/rtl/interfaces/VX_sched_csr_if.sv index 5ac508cbfc..d773df084d 100644 --- a/hw/rtl/interfaces/VX_sched_csr_if.sv +++ b/hw/rtl/interfaces/VX_sched_csr_if.sv @@ -18,7 +18,7 @@ interface VX_sched_csr_if import VX_gpu_pkg::*; (); wire [PERF_CTR_BITS-1:0] cycles; wire [PERF_CTR_BITS-1:0] instret; wire [`VX_CFG_NUM_WARPS-1:0] active_warps; - wire [`VX_CFG_NUM_WARPS-1:0][`VX_CFG_NUM_THREADS-1:0] thread_masks; + wire [`VX_CFG_NUM_THREADS-1:0] thread_mask; // selected by csr_rd_wid // Read port: slave sends wid + cta_id, master returns selected mscratch // and cta_csrs. csr_rd_cta_id must carry the executing instruction's @@ -33,6 +33,8 @@ interface VX_sched_csr_if import VX_gpu_pkg::*; (); `ifdef VX_CFG_VM_ENABLE logic [`VX_CFG_XLEN-1:0] csr_satp; + logic satp_wr_valid; + logic [`VX_CFG_XLEN-1:0] satp_wr_data; `endif // Write port: slave notifies scheduler of MSCRATCH CSR writes @@ -57,7 +59,7 @@ interface VX_sched_csr_if import VX_gpu_pkg::*; (); output cycles, output instret, output active_warps, - output thread_masks, + output thread_mask, output mscratch, output cta_csrs, output cta_lane, @@ -67,7 +69,9 @@ interface VX_sched_csr_if import VX_gpu_pkg::*; (); output csr_mcause, output csr_mtval, `ifdef VX_CFG_VM_ENABLE - input csr_satp, + output csr_satp, + input satp_wr_valid, + input satp_wr_data, `endif input csr_rd_wid, input csr_rd_cta_id, @@ -82,7 +86,7 @@ interface VX_sched_csr_if import VX_gpu_pkg::*; (); input cycles, input instret, input active_warps, - input thread_masks, + input thread_mask, input mscratch, input cta_csrs, input cta_lane, @@ -92,7 +96,9 @@ interface VX_sched_csr_if import VX_gpu_pkg::*; (); input csr_mcause, input csr_mtval, `ifdef VX_CFG_VM_ENABLE - output csr_satp, + input csr_satp, + output satp_wr_valid, + output satp_wr_data, `endif output csr_rd_wid, output csr_rd_cta_id, diff --git a/hw/rtl/interfaces/VX_slice_sched_if.sv b/hw/rtl/interfaces/VX_slice_sched_if.sv new file mode 100644 index 0000000000..c5e056ab43 --- /dev/null +++ b/hw/rtl/interfaces/VX_slice_sched_if.sv @@ -0,0 +1,151 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// Core scheduler state exchanged with one fetch slice's warp scheduler. +// Core-wide events arrive already addressed to the slice: warp ids are +// slice-local (wid = {wfs, fsw}) and per-warp masks hold only its warps. +interface VX_slice_sched_if import VX_gpu_pkg::*; (); + + localparam SW = PER_FETCH_WARPS; + localparam LW = FETCH_WFS_W; + + // CTA launch + logic cta_fire; + logic [LW-1:0] cta_lid; + logic [PC_BITS-1:0] cta_PC; + logic [`VX_CFG_NUM_THREADS-1:0] cta_tmask; + logic [`VX_CFG_MEM_ADDR_WIDTH-1:0] cta_param; + logic cta_init; + + // CTA-CSR store writes + logic warp_write; + logic [LW-1:0] warp_waddr; + cta_warp_t warp_wdata; + logic id_write; + logic [LW-1:0] id_waddr; + logic [NCTA_WIDTH-1:0] id_wdata; + + // wspawn + logic wspawn_apply; + logic [SW-1:0] wspawn_wmask; + logic [PC_BITS-1:0] wspawn_pc; + logic wspawn_src; // the spawning warp is in this slice + logic [LW-1:0] wspawn_src_lid; + logic [`VX_CFG_MEM_ADDR_WIDTH-1:0] wspawn_mscratch; + + // barrier release + logic bar_unlock_valid; + logic [SW-1:0] bar_unlock_mask; + + // warp retirement, as the core-wide CTA accounting takes it + logic retire_valid; + logic [LW-1:0] retire_lid; + + // every warp of the core has a full ibuffer + logic all_ibuf_full; +`ifdef EXT_SCHED_TIMEOUT_DUMP + // the core-wide stall timeout is about to fire + logic timeout_dump; +`endif + + // slice status and performance events + logic [SW-1:0] active; + logic [SW-1:0] stalled; + logic [SW-1:0] pending_empty; + // next-state ibuffer full, registered core-wide into all_ibuf_full + logic [SW-1:0] ibuf_full_n; + logic [FETCH_ALU_PORTS-1:0] branch_valid; + logic [FETCH_SFU_PORTS-1:0] divergence; + logic sched_valid; + logic sched_fire; + logic [`VX_CFG_NUM_THREADS-1:0] sched_tmask; + + modport master ( + output cta_fire, + output cta_lid, + output cta_PC, + output cta_tmask, + output cta_param, + output cta_init, + output warp_write, + output warp_waddr, + output warp_wdata, + output id_write, + output id_waddr, + output id_wdata, + output wspawn_apply, + output wspawn_wmask, + output wspawn_pc, + output wspawn_src, + output wspawn_src_lid, + output wspawn_mscratch, + output bar_unlock_valid, + output bar_unlock_mask, + output retire_valid, + output retire_lid, + output all_ibuf_full, +`ifdef EXT_SCHED_TIMEOUT_DUMP + output timeout_dump, +`endif + input active, + input stalled, + input ibuf_full_n, + input pending_empty, + input branch_valid, + input divergence, + input sched_valid, + input sched_fire, + input sched_tmask + ); + + modport slave ( + input cta_fire, + input cta_lid, + input cta_PC, + input cta_tmask, + input cta_param, + input cta_init, + input warp_write, + input warp_waddr, + input warp_wdata, + input id_write, + input id_waddr, + input id_wdata, + input wspawn_apply, + input wspawn_wmask, + input wspawn_pc, + input wspawn_src, + input wspawn_src_lid, + input wspawn_mscratch, + input bar_unlock_valid, + input bar_unlock_mask, + input retire_valid, + input retire_lid, + input all_ibuf_full, +`ifdef EXT_SCHED_TIMEOUT_DUMP + input timeout_dump, +`endif + output active, + output stalled, + output ibuf_full_n, + output pending_empty, + output branch_valid, + output divergence, + output sched_valid, + output sched_fire, + output sched_tmask + ); + +endinterface diff --git a/hw/rtl/interfaces/VX_warp_ctl_if.sv b/hw/rtl/interfaces/VX_warp_ctl_if.sv index 9da68980c9..0e49757335 100644 --- a/hw/rtl/interfaces/VX_warp_ctl_if.sv +++ b/hw/rtl/interfaces/VX_warp_ctl_if.sv @@ -15,23 +15,17 @@ interface VX_warp_ctl_if import VX_gpu_pkg::*; (); - wire wspawn_valid; wire tmc_valid; wire split_valid; wire sjoin_valid; - wire bar_valid; wire wsync_valid; - wire [NW_WIDTH-1:0] wid; - tmc_t tmc; - wspawn_t wspawn; split_t split; join_t sjoin; - barrier_t bar; - // Per-warp pipeline drain status (from scheduler) - wire [`VX_CFG_NUM_WARPS-1:0] warp_pending_alm_empty; + // Pipeline drain status of the scheduler's warps, by SFU-block-local warp id + wire [SFU_BLOCK_WARPS-1:0] warp_pending_alm_empty; // LSU memory-scheduler drained signal (AND-reduced across all LSU blocks). // Used by VX_wctl_unit to gate BAR (both vx_barrier and vx_barrier_arrive) @@ -40,64 +34,41 @@ interface VX_warp_ctl_if import VX_gpu_pkg::*; (); // the CUDA __syncthreads / OpenCL barrier(CLK_LOCAL_MEM_FENCE) contract. wire lsu_sched_drained; - // Return IPDOM stack address + // IPDOM stack address and mscratch of the warp at dvstack_wid wire [NW_WIDTH-1:0] dvstack_wid; wire [DV_STACK_SIZEW-1:0] dvstack_ptr; - - // Return barrier phase - wire [BAR_ADDR_W-1:0] bar_addr; - wire bar_phase; + wire [`VX_CFG_MEM_ADDR_WIDTH-1:0] mscratch; modport master ( - output wspawn_valid, output tmc_valid, output split_valid, output sjoin_valid, - output bar_valid, output wsync_valid, - output wid, - - output wspawn, output tmc, output split, output sjoin, - output bar, - - output bar_addr, - input bar_phase, - input warp_pending_alm_empty, input lsu_sched_drained, - output dvstack_wid, - input dvstack_ptr + input dvstack_ptr, + input mscratch ); modport slave ( - input wspawn_valid, - input tmc_valid, - input split_valid, - input sjoin_valid, - input bar_valid, - input wsync_valid, - - input wid, - - input wspawn, - input tmc, - input split, - input sjoin, - input bar, - - input bar_addr, - output bar_phase, - + input tmc_valid, + input split_valid, + input sjoin_valid, + input wsync_valid, + input wid, + input tmc, + input split, + input sjoin, output warp_pending_alm_empty, output lsu_sched_drained, - - input dvstack_wid, - output dvstack_ptr + input dvstack_wid, + output dvstack_ptr, + output mscratch ); endinterface diff --git a/hw/rtl/interfaces/VX_warp_shared_if.sv b/hw/rtl/interfaces/VX_warp_shared_if.sv new file mode 100644 index 0000000000..86d1f13786 --- /dev/null +++ b/hw/rtl/interfaces/VX_warp_shared_if.sv @@ -0,0 +1,44 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// A core-wide warp-control request and its completion. A requester waits on at +// most one request with has_rsp set; rsp_valid completes it and carries the +// barrier phase. Other requests are posted and get no completion. +interface VX_warp_shared_if import VX_gpu_pkg::*; (); + + logic req_valid; + warp_shared_req_t req_data; + logic req_ready; + + logic rsp_valid; + logic rsp_phase; + + modport master ( + output req_valid, + output req_data, + input req_ready, + input rsp_valid, + input rsp_phase + ); + + modport slave ( + input req_valid, + input req_data, + output req_ready, + output rsp_valid, + output rsp_phase + ); + +endinterface diff --git a/hw/rtl/om/VX_om_core.sv b/hw/rtl/om/VX_om_core.sv index 459708e0d6..f8f610d39f 100644 --- a/hw/rtl/om/VX_om_core.sv +++ b/hw/rtl/om/VX_om_core.sv @@ -51,7 +51,7 @@ module VX_om_core import VX_gpu_pkg::*; import VX_om_pkg::*; #( om_rt_dcrs_t om_rt_dcrs [`VX_OM_MAX_RT]; VX_om_dcr #( - .INSTANCE_ID ($sformatf("%s-dcr", INSTANCE_ID)) + .INSTANCE_ID (`SFORMATF(("%s-dcr", INSTANCE_ID))) ) om_dcr ( .clk (clk), .reset (reset), @@ -88,7 +88,7 @@ module VX_om_core import VX_gpu_pkg::*; import VX_om_pkg::*; #( `UNUSED_VAR (mem_write_notify) VX_om_mem #( - .INSTANCE_ID ($sformatf("%s-mem", INSTANCE_ID)), + .INSTANCE_ID (`SFORMATF(("%s-mem", INSTANCE_ID))), .NUM_LANES (NUM_LANES), .TAG_WIDTH (MEM_TAG_WIDTH) ) om_mem ( @@ -144,7 +144,7 @@ module VX_om_core import VX_gpu_pkg::*; import VX_om_pkg::*; #( wire [NUM_LANES-1:0] ds_pass_out; VX_om_ds #( - .INSTANCE_ID ($sformatf("%s-ds", INSTANCE_ID)), + .INSTANCE_ID (`SFORMATF(("%s-ds", INSTANCE_ID))), .NUM_LANES (NUM_LANES), .TAG_WIDTH (DS_TAG_WIDTH) ) om_ds ( @@ -187,7 +187,7 @@ module VX_om_core import VX_gpu_pkg::*; import VX_om_pkg::*; #( wire [OM_RT_IDX_BITS-1:0] mem_rsp_rt; VX_om_blend #( - .INSTANCE_ID ($sformatf("%s-blend", INSTANCE_ID)), + .INSTANCE_ID (`SFORMATF(("%s-blend", INSTANCE_ID))), .NUM_LANES (NUM_LANES), .TAG_WIDTH (BLEND_TAG_WIDTH) ) om_blend ( diff --git a/hw/rtl/om/VX_om_mem.sv b/hw/rtl/om/VX_om_mem.sv index 67cc03cb6f..45841002fb 100644 --- a/hw/rtl/om/VX_om_mem.sv +++ b/hw/rtl/om/VX_om_mem.sv @@ -245,7 +245,7 @@ module VX_om_mem import VX_gpu_pkg::*; import VX_om_pkg::*; #( ) lsu_mem_if(); VX_mem_scheduler #( - .INSTANCE_ID ($sformatf("%s-memsched", INSTANCE_ID)), + .INSTANCE_ID (`SFORMATF(("%s-memsched", INSTANCE_ID))), .CORE_REQS (NUM_REQS), .MEM_CHANNELS (MEM_CHANNELS), .WORD_SIZE (4), diff --git a/hw/rtl/om/VX_om_steer.sv b/hw/rtl/om/VX_om_steer.sv index 0fd8ce9c8e..26310185c8 100644 --- a/hw/rtl/om/VX_om_steer.sv +++ b/hw/rtl/om/VX_om_steer.sv @@ -23,7 +23,7 @@ // // The select is the is_addr_om attr bit alone. No cache level decodes it -- the // aperture store is presented to the caches as an ordinary IO store (they carry -// attr through verbatim), so this bit arrives untouched. See VX_lsu_slice. +// attr through verbatim), so this bit arrives untouched. See VX_lsu_block. // // Flush ordering: the steer is a fork, so a flush token bound for L2 is held // here until the OM side has been flushed. Every aperture store this trunk diff --git a/hw/rtl/raster/VX_raster_be.sv b/hw/rtl/raster/VX_raster_be.sv index e33ca50d9c..8426c46cf0 100644 --- a/hw/rtl/raster/VX_raster_be.sv +++ b/hw/rtl/raster/VX_raster_be.sv @@ -119,7 +119,7 @@ module VX_raster_be import VX_raster_pkg::*; #( wire [PER_BLOCK_QUADS-1:0][`VX_RASTER_DIM_BITS-1:0] qe_yloc; VX_raster_qe #( - .INSTANCE_ID ($sformatf("%s-qe", INSTANCE_ID)), + .INSTANCE_ID (`SFORMATF(("%s-qe", INSTANCE_ID))), .NUM_QUADS (PER_BLOCK_QUADS) ) quad_evaluator ( .clk (clk), diff --git a/hw/rtl/raster/VX_raster_core.sv b/hw/rtl/raster/VX_raster_core.sv index 470e8ea205..be08681dbc 100644 --- a/hw/rtl/raster/VX_raster_core.sv +++ b/hw/rtl/raster/VX_raster_core.sv @@ -17,7 +17,6 @@ module VX_raster_core import VX_gpu_pkg::*; import VX_raster_pkg::*; #( parameter `STRING INSTANCE_ID = "", - parameter INSTANCE_IDX = 0, parameter NUM_INSTANCES = 1, parameter NUM_SLICES = 1, // number of slices parameter TILE_LOGSIZE = 5, // tile log size @@ -32,6 +31,8 @@ module VX_raster_core import VX_gpu_pkg::*; import VX_raster_pkg::*; #( input wire clk, input wire reset, + input wire [`LOG2UP(NUM_INSTANCES)-1:0] instance_idx, + // PERF `ifdef PERF_ENABLE VX_raster_perf_if.master perf_raster_if, @@ -79,7 +80,7 @@ module VX_raster_core import VX_gpu_pkg::*; import VX_raster_pkg::*; #( raster_dcrs_t raster_dcrs; VX_raster_dcr #( - .INSTANCE_ID ($sformatf("%s-dcr", INSTANCE_ID)) + .INSTANCE_ID (`SFORMATF(("%s-dcr", INSTANCE_ID))) ) raster_dcr ( .clk (clk), .reset (reset), @@ -134,14 +135,14 @@ module VX_raster_core import VX_gpu_pkg::*; import VX_raster_pkg::*; #( // Memory unit VX_raster_mem #( - .INSTANCE_ID ($sformatf("%s-mem", INSTANCE_ID)), - .INSTANCE_IDX (INSTANCE_IDX), + .INSTANCE_ID (`SFORMATF(("%s-mem", INSTANCE_ID))), .NUM_INSTANCES (NUM_INSTANCES), .TILE_LOGSIZE (TILE_LOGSIZE), .QUEUE_SIZE (MEM_FIFO_DEPTH) ) raster_mem ( .clk (clk), .reset (reset), + .instance_idx (instance_idx), .start (mem_unit_start), .busy (mem_unit_busy), @@ -330,7 +331,7 @@ module VX_raster_core import VX_gpu_pkg::*; import VX_raster_pkg::*; #( `endif VX_raster_slice #( - .INSTANCE_ID ($sformatf("%s-slice%d", INSTANCE_ID, slice_id)), + .INSTANCE_ID (`SFORMATF(("%s-slice%d", INSTANCE_ID, slice_id))), .TILE_LOGSIZE (TILE_LOGSIZE), .BLOCK_LOGSIZE (BLOCK_LOGSIZE), .OUTPUT_QUADS (OUTPUT_QUADS), @@ -372,7 +373,7 @@ module VX_raster_core import VX_gpu_pkg::*; import VX_raster_pkg::*; #( // depth read from the ocache (elastic, variable-latency). Pass-through // when the per-draw earlyz_safe DCR is clear. VX_raster_earlyz #( - .INSTANCE_ID ($sformatf("%s-earlyz%d", INSTANCE_ID, slice_id)), + .INSTANCE_ID (`SFORMATF(("%s-earlyz%d", INSTANCE_ID, slice_id))), .OUTPUT_QUADS (OUTPUT_QUADS) ) raster_earlyz ( .clk (clk), @@ -529,7 +530,7 @@ module VX_raster_core import VX_gpu_pkg::*; import VX_raster_pkg::*; #( // no-work case so such an engine drains immediately. localparam LOG2_NUM_INSTANCES = `CLOG2(NUM_INSTANCES); wire [`RASTER_TILE_BITS-1:0] my_tile_count = - (raster_dcrs.tile_count + `RASTER_TILE_BITS'(NUM_INSTANCES - 1 - INSTANCE_IDX)) >> LOG2_NUM_INSTANCES; + (raster_dcrs.tile_count + (`RASTER_TILE_BITS'(NUM_INSTANCES - 1) - `RASTER_TILE_BITS'(instance_idx))) >> LOG2_NUM_INSTANCES; wire has_no_tiles = (my_tile_count == '0); always @(posedge clk) begin diff --git a/hw/rtl/raster/VX_raster_earlyz.sv b/hw/rtl/raster/VX_raster_earlyz.sv index 2cb0529650..7759ba7232 100644 --- a/hw/rtl/raster/VX_raster_earlyz.sv +++ b/hw/rtl/raster/VX_raster_earlyz.sv @@ -297,7 +297,7 @@ module VX_raster_earlyz import VX_gpu_pkg::*; import VX_raster_pkg::*; import VX ) lsu_mem_if(); VX_mem_scheduler #( - .INSTANCE_ID ($sformatf("%s-memsched", INSTANCE_ID)), + .INSTANCE_ID (`SFORMATF(("%s-memsched", INSTANCE_ID))), .CORE_REQS (NUM_PIX), .MEM_CHANNELS (OCACHE_NUM_REQS), .WORD_SIZE (OCACHE_WORD_SIZE), diff --git a/hw/rtl/raster/VX_raster_mem.sv b/hw/rtl/raster/VX_raster_mem.sv index d4e5a82bda..002a9bc5ce 100644 --- a/hw/rtl/raster/VX_raster_mem.sv +++ b/hw/rtl/raster/VX_raster_mem.sv @@ -23,7 +23,6 @@ module VX_raster_mem import VX_gpu_pkg::*; import VX_raster_pkg::*; #( parameter `STRING INSTANCE_ID = "", - parameter INSTANCE_IDX = 0, parameter NUM_INSTANCES = 1, parameter TILE_LOGSIZE = 5, parameter QUEUE_SIZE = 8 @@ -31,6 +30,8 @@ module VX_raster_mem import VX_gpu_pkg::*; import VX_raster_pkg::*; #( input wire clk, input wire reset, + input wire [`LOG2UP(NUM_INSTANCES)-1:0] instance_idx, + // Device configurations raster_dcrs_t dcrs, @@ -144,8 +145,8 @@ module VX_raster_mem import VX_gpu_pkg::*; import VX_raster_pkg::*; #( `UNUSED_VAR (th_pids_count) // calculate tile start info - wire [`RASTER_TILE_BITS-1:0] start_tile_count = (dcrs.tile_count + `RASTER_TILE_BITS'(NUM_INSTANCES - 1 - INSTANCE_IDX)) >> LOG2_NUM_INSTANCES; - wire [W_ADDR_BITS-1:0] start_tbuf_addr = {dcrs.tbuf_addr, 4'b0} + W_ADDR_BITS'(INSTANCE_IDX * TILE_HEADER_SIZEW); + wire [`RASTER_TILE_BITS-1:0] start_tile_count = (dcrs.tile_count + (`RASTER_TILE_BITS'(NUM_INSTANCES - 1) - `RASTER_TILE_BITS'(instance_idx))) >> LOG2_NUM_INSTANCES; + wire [W_ADDR_BITS-1:0] start_tbuf_addr = {dcrs.tbuf_addr, 4'b0} + (W_ADDR_BITS'(instance_idx) * W_ADDR_BITS'(TILE_HEADER_SIZEW)); // The sorted-pid array is a single dense block that follows ALL bin headers // (num_bins == dcrs.tile_count of them, 3 words each), shared across the N @@ -345,7 +346,7 @@ module VX_raster_mem import VX_gpu_pkg::*; import VX_raster_pkg::*; #( `UNUSED_VAR (sched_rsp_tag) VX_mem_scheduler #( - .INSTANCE_ID ($sformatf("%s-memsched", INSTANCE_ID)), + .INSTANCE_ID (`SFORMATF(("%s-memsched", INSTANCE_ID))), .CORE_REQS (NUM_REQS), .MEM_CHANNELS (RCACHE_NUM_REQS), .WORD_SIZE (`RASTER_DATA_BITS / 8), diff --git a/hw/rtl/raster/VX_raster_slice.sv b/hw/rtl/raster/VX_raster_slice.sv index ae1e0a243f..4f1f920660 100644 --- a/hw/rtl/raster/VX_raster_slice.sv +++ b/hw/rtl/raster/VX_raster_slice.sv @@ -79,7 +79,7 @@ module VX_raster_slice import VX_raster_pkg::*; #( `endif VX_raster_te #( - .INSTANCE_ID ($sformatf("%s-te", INSTANCE_ID)), + .INSTANCE_ID (`SFORMATF(("%s-te", INSTANCE_ID))), .TILE_LOGSIZE (TILE_LOGSIZE), .BLOCK_LOGSIZE (BLOCK_LOGSIZE) ) tile_evaluator ( @@ -147,7 +147,7 @@ module VX_raster_slice import VX_raster_pkg::*; #( `endif VX_raster_be #( - .INSTANCE_ID ($sformatf("%s-be", INSTANCE_ID)), + .INSTANCE_ID (`SFORMATF(("%s-be", INSTANCE_ID))), .BLOCK_LOGSIZE (BLOCK_LOGSIZE), .OUTPUT_QUADS (OUTPUT_QUADS), .QUAD_FIFO_DEPTH (QUAD_FIFO_DEPTH) diff --git a/hw/rtl/rtu/VX_rtu_unit.sv b/hw/rtl/rtu/VX_rtu_unit.sv index cfeb22a315..ab93aceecf 100644 --- a/hw/rtl/rtu/VX_rtu_unit.sv +++ b/hw/rtl/rtu/VX_rtu_unit.sv @@ -38,13 +38,14 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( parameter `STRING INSTANCE_ID = "", - parameter CORE_ID = 0, parameter NUM_LANES = `VX_CFG_NUM_THREADS, parameter RTU_TAG_WIDTH = 1 ) ( input wire clk, input wire reset, + input wire [SFU_BLOCK_W-1:0] block_idx, + // SFU PE-style request/response interfaces VX_execute_if.slave execute_if, VX_result_if.master result_if, @@ -56,9 +57,12 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( VX_sched_unlock_if.master sched_unlock_if ); `UNUSED_SPARAM (INSTANCE_ID) - `UNUSED_PARAM (CORE_ID) - localparam RAM_SIZE = `VX_CFG_NUM_WARPS * RTUW_SLOT_COUNT; + // This SFU block serves the warps with wid mod NUM_SFU_BLOCKS == block_idx; + // its per-warp state is indexed by the block-local warp id. + localparam NLW = SFU_BLOCK_WARPS; + localparam RAM_SIZE = NLW * RTUW_SLOT_COUNT; + localparam RAM_ADDRW = `LOG2UP(RAM_SIZE); localparam RAM_DATAW = 32 * NUM_LANES; // The address concatenates {warp, slot}; the slot stride must be a power of @@ -80,6 +84,7 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( wire [2:0] uop = execute_if.data.op_args.rtuw.uop; wire [RTUW_SLOT_BITS-1:0] slot = execute_if.data.op_args.rtuw.slot[RTUW_SLOT_BITS-1:0]; wire [NW_WIDTH-1:0] wid = execute_if.data.header.wid; + wire [SFU_BLOCK_NW_W-1:0] lid = wid_to_sfu_local(wid); // Register the outgoing RTU bus at this module boundary so the core->socket // seam launches at a flop (see VX_rtu_bus_slice). We drive the internal @@ -107,7 +112,7 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( // Written by the RTU, read by the shader. Neither port is ever contended, so // the RTU is never made to wait for a write and a read never waits for it. wire win_v = rtu_bus_w.win_valid; - wire [NW_WIDTH-1:0] win_wid = rtu_bus_w.win_data.wid; + wire [SFU_BLOCK_NW_W-1:0] win_lid = wid_to_sfu_local(rtu_bus_w.win_data.wid); wire [RTUW_SLOT_BITS-1:0] win_slot = rtu_bus_w.win_data.slot; `UNUSED_VAR (rtu_bus_w.win_data.tag) @@ -115,7 +120,7 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( wire core_rden; wire [RTUW_SLOT_BITS-1:0] core_slot; - wire [NW_WIDTH-1:0] core_wid; + wire [SFU_BLOCK_NW_W-1:0] core_lid; wire [NUM_LANES-1:0][31:0] core_rdata; VX_dp_ram #( @@ -130,9 +135,9 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( .read (core_rden), .write (win_v), .wren (rtu_bus_w.win_data.mask), - .waddr ({win_wid, win_slot}), + .waddr (RAM_ADDRW'({win_lid, win_slot})), .wdata (rtu_bus_w.win_data.data), - .raddr ({core_wid, core_slot}), + .raddr (RAM_ADDRW'({core_lid, core_slot})), .rdata (core_rdata) ); @@ -147,7 +152,7 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( // longer one of them — see the parked-WAIT table below. wire is_read = (op == RTUW_OP_GETWF) || (op == RTUW_OP_GETW); - reg [`VX_CFG_NUM_WARPS-1:0] response_ready, trace_open; + reg [NLW-1:0] response_ready, trace_open; // ── the parked WAIT — a long-latency op MUST NOT hold the execute stage ──── // A WAIT used to sit at execute_if until its record landed, and execute_if is @@ -165,8 +170,8 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( // WAIT and the unlock below is what releases it — so the semantics are unchanged; // only the execute port is freed. A warp holds at most one trace, so one entry // per warp is exact, not a heuristic. - reg [`VX_CFG_NUM_WARPS-1:0] wait_pend; - sfu_header_t wait_hdr [`VX_CFG_NUM_WARPS]; + reg [NLW-1:0] wait_pend; + sfu_header_t wait_hdr [NLW]; // ── result stage (the RAM read is synchronous) ───────────────────────── // One op is accepted per cycle into s1; its word arrives from the RAM on the @@ -227,7 +232,7 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( // // A CONTINUE with no open candidate retires as a no-op (defensive — correct // kernels only issue one inside the yield loop). - wire cont_want = is_cont && trace_open[wid]; + wire cont_want = is_cont && trace_open[lid]; wire is_stream = is_ray || cont_want; wire is_2beat = cont_want || (is_trace && (uop == RTUW_UOP_ARM)); @@ -281,20 +286,22 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( // the two arbitrate for the result stage. The parked WAIT wins: it has already // retired from execute_if and the warp behind it is stalled on nothing else, // while a GETW at execute_if can simply wait a cycle. - wire [`VX_CFG_NUM_WARPS-1:0] wait_done = wait_pend & response_ready; + wire [NLW-1:0] wait_done = wait_pend & response_ready; - wire [NW_WIDTH-1:0] wake_wid; - wire wake_valid; - wire [`VX_CFG_NUM_WARPS-1:0] wake_1h; + wire [`LOG2UP(NLW)-1:0] wake_index; + wire wake_valid; + wire [NLW-1:0] wake_1h; VX_priority_encoder #( - .N (`VX_CFG_NUM_WARPS) + .N (NLW) ) wake_picker ( .data_in (wait_done), .onehot_out (wake_1h), - .index_out (wake_wid), + .index_out (wake_index), .valid_out (wake_valid) ); `UNUSED_VAR (wake_1h) + wire [SFU_BLOCK_NW_W-1:0] wake_lid = SFU_BLOCK_NW_W'(wake_index); + wire [NW_WIDTH-1:0] wake_wid = sfu_local_to_wid(wake_lid, block_idx); assign wake_fire = wake_valid && s1_ready; @@ -302,12 +309,12 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( // A stream op retires with its last beat; everything else needs only the // result stage. A WAIT retires the moment it is parked — it never blocks. wire read_fire = execute_if.valid && is_read && s1_grant; - wire wait_fire = execute_if.valid && is_wait && ~wait_pend[wid]; + wire wait_fire = execute_if.valid && is_wait && ~wait_pend[lid]; wire stream_fire = rb_fire && rb_end; - wire cont_nop = execute_if.valid && is_cont && ~trace_open[wid] && s1_grant; + wire cont_nop = execute_if.valid && is_cont && ~trace_open[lid] && s1_grant; wire op_fire = read_fire || wait_fire || stream_fire || cont_nop || arm_fire; - `RUNTIME_ASSERT(~(execute_if.valid && is_wait && wait_pend[wid]), + `RUNTIME_ASSERT(~(execute_if.valid && is_wait && wait_pend[lid]), ("%t: *** %s: wid=%0d issued a second WAIT with one already parked", $time, INSTANCE_ID, wid)) @@ -331,7 +338,7 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( // read reads the slot its op names. assign core_rden = read_fire || wake_fire; assign core_slot = wake_fire ? RTUW_SLOT_BITS'(`VX_RT_STATUS) : slot; - assign core_wid = wake_fire ? wake_wid : wid; + assign core_lid = wake_fire ? wake_lid : lid; // ── sequential state ─────────────────────────────────────────────────── always @(posedge clk) begin @@ -350,7 +357,7 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( if (wake_fire) begin s1_valid <= 1'b1; s1_from_ram <= 1'b1; - s1_header <= wait_hdr[wake_wid]; + s1_header <= wait_hdr[wake_lid]; end else if (op_fire && ~wait_fire) begin s1_valid <= 1'b1; s1_from_ram <= is_read; @@ -359,8 +366,8 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( // park the WAIT: it retires now, and answers later if (wait_fire) begin - wait_pend[wid] <= 1'b1; - wait_hdr[wid] <= execute_if.data.header; + wait_pend[lid] <= 1'b1; + wait_hdr[lid] <= execute_if.data.header; end // The last beat retires the op, so the counter always lands back at 0. @@ -370,18 +377,18 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( // The waking WAIT consumes the record, so the next one waits again. if (wake_fire) begin - wait_pend[wake_wid] <= 1'b0; - response_ready[wake_wid] <= 1'b0; + wait_pend[wake_lid] <= 1'b0; + response_ready[wake_lid] <= 1'b0; end // A CONTINUE resolves the open candidate once its last beat is away. if (stream_fire && cont_want) begin - trace_open[wid] <= 1'b0; + trace_open[lid] <= 1'b0; end // Last, so a record landing in the same cycle as the op it unblocks // wins: the record is the newer event. if (status_wr) begin - response_ready[win_wid] <= 1'b1; - trace_open[win_wid] <= rtu_bus_w.win_data.is_cand; + response_ready[win_lid] <= 1'b1; + trace_open[win_lid] <= rtu_bus_w.win_data.is_cand; end end end diff --git a/hw/rtl/tcu/VX_tcu_block.sv b/hw/rtl/tcu/VX_tcu_block.sv new file mode 100644 index 0000000000..ee6e19a637 --- /dev/null +++ b/hw/rtl/tcu/VX_tcu_block.sv @@ -0,0 +1,121 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// One TCU block: the MMA datapath, with TCU_LD handed to the core-wide AGU and +// WGMMA operands supplied by the core-wide tile buffers (VX_tcu_shared). +module VX_tcu_block import VX_gpu_pkg::*, VX_tcu_pkg::*; #( + parameter `STRING INSTANCE_ID = "", + parameter `STRING BLOCK_STR = "" +) ( + `SCOPE_IO_DECL + + input wire clk, + input wire reset, + +`ifdef VX_CFG_TCU_WGMMA_ENABLE + // MMA execute handshake observed by the WGMMA tile buffers + output wire exec_valid, + output wire exec_ready, + output tcu_execute_t exec_data, + input wire [TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tbuf_rs1_data, + input wire [TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tbuf_rs2_data, + input wire tbuf_ready, +`endif + +`ifdef TCU_META_ENABLE + // TCU_LD to and from the AGU + output wire agu_ld_valid, + output tcu_execute_t agu_ld_data, + input wire agu_ld_ready, + input wire agu_result_valid, + input tcu_result_t agu_result_data, + output wire agu_result_ready, + input wire meta_wr_en, + input wire [NW_WIDTH-1:0] meta_wr_wid, + input wire [4:0] meta_wr_idx, + input wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] meta_wr_data, +`endif + + VX_execute_if.slave execute_if, + VX_result_if.master result_if +); + `STATIC_ASSERT (`VX_CFG_NUM_TCU_BLOCKS == `VX_CFG_ISSUE_WIDTH, ("must be full issue execution")); + `STATIC_ASSERT (`VX_CFG_NUM_TCU_LANES == `VX_CFG_NUM_THREADS, ("must be full warp execution")); + + VX_execute_if #( + .data_t (tcu_execute_t) + ) core_execute_if(); + + VX_result_if #( + .data_t (tcu_result_t) + ) core_result_if(); + +`ifdef TCU_META_ENABLE + // TCU_LD goes to the AGU, every MMA op to the core + wire is_tcu_ld = (execute_if.data.op_type == INST_TCU_LD); + + assign agu_ld_valid = execute_if.valid && is_tcu_ld; + assign agu_ld_data = execute_if.data; + assign core_execute_if.valid = execute_if.valid && !is_tcu_ld; + assign core_execute_if.data = execute_if.data; + assign execute_if.ready = is_tcu_ld ? agu_ld_ready : core_execute_if.ready; + + // AGU wins same-cycle conflicts; the core stalls and retries next cycle. + assign result_if.valid = agu_result_valid || core_result_if.valid; + assign result_if.data = agu_result_valid ? agu_result_data : core_result_if.data; + assign agu_result_ready = result_if.ready; + assign core_result_if.ready = result_if.ready && !agu_result_valid; +`else + assign core_execute_if.valid = execute_if.valid; + assign core_execute_if.data = execute_if.data; + assign execute_if.ready = core_execute_if.ready; + + assign result_if.valid = core_result_if.valid; + assign result_if.data = core_result_if.data; + assign core_result_if.ready = result_if.ready; +`endif + +`ifdef VX_CFG_TCU_WGMMA_ENABLE + assign exec_valid = core_execute_if.valid; + assign exec_ready = core_execute_if.ready; + assign exec_data = core_execute_if.data; +`endif + + VX_tcu_core #( + .INSTANCE_ID (`SFORMATF(("%s-fused%s", INSTANCE_ID, BLOCK_STR))) + ) tcu_core ( + `ifdef SCOPE + .scope_reset (scope_reset), + .scope_bus_in (scope_bus_in), + .scope_bus_out (scope_bus_out), + `endif + .clk (clk), + .reset (reset), + `ifdef VX_CFG_TCU_WGMMA_ENABLE + .tbuf_rs1_data (tbuf_rs1_data), + .tbuf_rs2_data (tbuf_rs2_data), + .tbuf_ready (tbuf_ready), + `endif + `ifdef TCU_META_ENABLE + .ext_meta_wr_en (meta_wr_en), + .ext_meta_wr_wid (meta_wr_wid), + .ext_meta_wr_idx (meta_wr_idx), + .ext_meta_wr_data (meta_wr_data), + `endif + .execute_if (core_execute_if), + .result_if (core_result_if) + ); + +endmodule diff --git a/hw/rtl/tcu/VX_tcu_core.sv b/hw/rtl/tcu/VX_tcu_core.sv index f1a3c91b19..1762caafd7 100644 --- a/hw/rtl/tcu/VX_tcu_core.sv +++ b/hw/rtl/tcu/VX_tcu_core.sv @@ -303,8 +303,10 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( `endif `ifdef TCU_META_ENABLE + // a TCU block only executes one residue class of warps modulo NUM_TCU_BLOCKS VX_tcu_meta #( - .INSTANCE_ID (INSTANCE_ID) + .INSTANCE_ID (INSTANCE_ID), + .WID_STRIDE (`VX_CFG_NUM_TCU_BLOCKS) ) tcu_meta ( .clk (clk), .reset (reset), diff --git a/hw/rtl/tcu/VX_tcu_meta.sv b/hw/rtl/tcu/VX_tcu_meta.sv index 5c55e58bb0..8a31ca6bad 100644 --- a/hw/rtl/tcu/VX_tcu_meta.sv +++ b/hw/rtl/tcu/VX_tcu_meta.sv @@ -31,7 +31,10 @@ // VX_tcu_core consume the result in that same cycle. module VX_tcu_meta import VX_gpu_pkg::*, VX_tcu_pkg::*; #( - parameter `STRING INSTANCE_ID = "" + parameter `STRING INSTANCE_ID = "", + // stores one residue class of warps modulo WID_STRIDE, by wid / WID_STRIDE; + // the parent only presents writes for that class + parameter WID_STRIDE = 1 ) ( input wire clk, input wire reset, @@ -74,8 +77,11 @@ module VX_tcu_meta import VX_gpu_pkg::*, VX_tcu_pkg::*; #( localparam MERGED_COLS = SP_COLS + 2 * MX_COLS; localparam MERGED_BITS = MERGED_COLS * 32; - localparam META_DEPTH = `VX_CFG_NUM_WARPS; - localparam META_ADDRW = `CLOG2(META_DEPTH); + localparam META_DEPTH = `VX_CFG_NUM_WARPS / WID_STRIDE; + localparam META_ADDRW = `CLOG2(META_DEPTH); + localparam STRIDE_BITS = `CLOG2(WID_STRIDE); + + `STATIC_ASSERT (`IS_POW2(WID_STRIDE), ("invalid parameter")) // Namespace decode (mirrors the AGU / vx_tensor.h slot encoding). The // per-region write strobes are defined inside each region's block below so @@ -85,8 +91,8 @@ module VX_tcu_meta import VX_gpu_pkg::*, VX_tcu_pkg::*; #( wire [MERGED_BITS-1:0] merged_wdata; wire [MERGED_BITS-1:0] merged_rdata; - wire [`UP(META_ADDRW)-1:0] wr_addr = `UP(META_ADDRW)'(wr_wid); - wire [`UP(META_ADDRW)-1:0] rd_addr = `UP(META_ADDRW)'(rd_wid); + wire [`UP(META_ADDRW)-1:0] wr_addr = `UP(META_ADDRW)'(wr_wid >> STRIDE_BITS); + wire [`UP(META_ADDRW)-1:0] rd_addr = `UP(META_ADDRW)'(rd_wid >> STRIDE_BITS); // ----------------------------------------------------------------------- // Sparse region: per-thread metadata packed into PER_WARP_DEPTH banks of diff --git a/hw/rtl/tcu/VX_tcu_shared.sv b/hw/rtl/tcu/VX_tcu_shared.sv new file mode 100644 index 0000000000..77ff231a72 --- /dev/null +++ b/hw/rtl/tcu/VX_tcu_shared.sv @@ -0,0 +1,118 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// TCU state shared by every TCU block: the WGMMA tile buffers (a warpgroup's +// B operand is common to its blocks) and the warp-level metadata AGU. +module VX_tcu_shared import VX_gpu_pkg::*, VX_tcu_pkg::*; #( + parameter `STRING INSTANCE_ID = "", + parameter BLOCK_SIZE = `VX_CFG_NUM_TCU_BLOCKS +) ( +`ifdef PERF_ENABLE + output tcu_perf_t tcu_perf, +`endif + +`ifdef VX_CFG_TCU_WGMMA_ENABLE + input wire [BLOCK_SIZE-1:0] exec_valid, + input wire [BLOCK_SIZE-1:0] exec_ready, + input tcu_execute_t exec_data [BLOCK_SIZE], + output wire [BLOCK_SIZE-1:0][TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tbuf_rs1_data, + output wire [BLOCK_SIZE-1:0][TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tbuf_rs2_data, + output wire [BLOCK_SIZE-1:0] tbuf_ready_eff, + // Bank-parallel LMEM read port + VX_mem_bus_if.master tcu_lmem_if, +`endif + +`ifdef TCU_META_ENABLE + input wire [BLOCK_SIZE-1:0] agu_ld_valid, + input tcu_execute_t agu_ld_data [BLOCK_SIZE], + output wire [BLOCK_SIZE-1:0] agu_ld_ready, + output wire [BLOCK_SIZE-1:0] agu_result_valid, + output tcu_result_t agu_result_data [BLOCK_SIZE], + input wire [BLOCK_SIZE-1:0] agu_result_ready, + output wire meta_wr_en, + output wire [NW_WIDTH-1:0] meta_wr_wid, + output wire [4:0] meta_wr_idx, + output wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] meta_wr_data, + // TCU_LD memory client of the LSU scheduler + VX_lsu_sched_if.master tcu_mem_if, +`endif + + input wire clk, + input wire reset +); + `UNUSED_SPARAM (INSTANCE_ID) +`ifndef VX_CFG_TCU_WGMMA_ENABLE +`ifndef TCU_META_ENABLE + `UNUSED_PARAM (BLOCK_SIZE) + `UNUSED_VAR ({clk, reset}) +`endif +`endif + +`ifdef VX_CFG_TCU_WGMMA_ENABLE + VX_tcu_wgmma #( + .INSTANCE_ID (`SFORMATF(("%s-wgmma", INSTANCE_ID))), + .BLOCK_SIZE (BLOCK_SIZE) + ) wgmma ( + .clk (clk), + .reset (reset), + `ifdef PERF_ENABLE + .tcu_perf (tcu_perf), + `endif + .exec_valid (exec_valid), + .exec_ready (exec_ready), + .exec_data (exec_data), + .tcu_lmem_if (tcu_lmem_if), + .tbuf_rs1_data (tbuf_rs1_data), + .tbuf_rs2_data (tbuf_rs2_data), + .tbuf_ready_eff (tbuf_ready_eff) + ); +`else +`ifdef PERF_ENABLE + assign tcu_perf.tbuf_stalls = '0; + assign tcu_perf.tbuf_cache_hits = '0; + assign tcu_perf.lmem_reads = '0; + assign tcu_perf.wgmma_instrs = '0; + assign tcu_perf.wgmma_stalls = '0; +`endif +`endif + +`ifdef TCU_META_ENABLE + // The AGU streams NUM_TCU_LANES operands over the LSU memory client, whose + // request/response mask and data are sized by NUM_LSU_LANES; the two lane + // counts must match for the per-lane completion tracking to be correct. + `STATIC_ASSERT (`VX_CFG_NUM_TCU_LANES == `VX_CFG_NUM_LSU_LANES, ("VX_tcu_agu requires NUM_TCU_LANES == NUM_LSU_LANES")); + + VX_tcu_agu #( + .INSTANCE_ID (`SFORMATF(("%s-agu", INSTANCE_ID))), + .BLOCK_SIZE (BLOCK_SIZE), + .NUM_LANES (`VX_CFG_NUM_TCU_LANES) + ) agu ( + .clk (clk), + .reset (reset), + .per_block_ld_valid (agu_ld_valid), + .per_block_ld_data (agu_ld_data), + .per_block_ld_ready (agu_ld_ready), + .client_if (tcu_mem_if), + .meta_wr_en (meta_wr_en), + .meta_wr_wid (meta_wr_wid), + .meta_wr_idx (meta_wr_idx), + .meta_wr_data (meta_wr_data), + .result_valid (agu_result_valid), + .result_data (agu_result_data), + .result_ready (agu_result_ready) + ); +`endif + +endmodule diff --git a/hw/rtl/tcu/VX_tcu_unit.sv b/hw/rtl/tcu/VX_tcu_unit.sv index 593e39eee7..fe0e4838e7 100644 --- a/hw/rtl/tcu/VX_tcu_unit.sv +++ b/hw/rtl/tcu/VX_tcu_unit.sv @@ -59,6 +59,7 @@ module VX_tcu_unit import VX_gpu_pkg::*, VX_tcu_pkg::*; #( ) lane_dispatch ( .clk (clk), .reset (reset), + .port_base ('0), .dispatch_if(dispatch_if), .execute_if (per_block_execute_if) ); @@ -67,188 +68,96 @@ module VX_tcu_unit import VX_gpu_pkg::*, VX_tcu_pkg::*; #( .data_t (tcu_result_t) ) per_block_result_if[BLOCK_SIZE](); - // ----------------------------------------------------------------------- - // Split each per_block_execute_if between two consumers: - // - VX_tcu_agu: handles INST_TCU_LD (warp-level memory load). - // - VX_tcu_core: handles every MMA op_type. - // The ready signal is muxed by op_type so only one consumer drives at a time. - // ----------------------------------------------------------------------- - VX_execute_if #( - .data_t (tcu_execute_t) - ) core_execute_if[BLOCK_SIZE](); - - VX_result_if #( - .data_t (tcu_result_t) - ) core_result_if[BLOCK_SIZE](); - -`ifdef TCU_META_ENABLE - wire [BLOCK_SIZE-1:0] agu_ld_valid; - wire [BLOCK_SIZE-1:0] agu_ld_ready; - tcu_execute_t agu_ld_data [BLOCK_SIZE]; - - wire [BLOCK_SIZE-1:0] agu_result_valid; - tcu_result_t agu_result_data [BLOCK_SIZE]; - wire [BLOCK_SIZE-1:0] agu_result_ready; -`endif - - for (genvar bi = 0; bi < BLOCK_SIZE; ++bi) begin : g_split - `ifdef TCU_META_ENABLE - wire is_tcu_ld = (per_block_execute_if[bi].data.op_type == INST_TCU_LD); - - // To AGU when TCU_LD - assign agu_ld_valid[bi] = per_block_execute_if[bi].valid && is_tcu_ld; - assign agu_ld_data[bi] = per_block_execute_if[bi].data; - - // To tcu_core when NOT TCU_LD - assign core_execute_if[bi].valid = per_block_execute_if[bi].valid && !is_tcu_ld; - assign core_execute_if[bi].data = per_block_execute_if[bi].data; - - // Parent .ready: route to AGU on TCU_LD, otherwise to tcu_core - assign per_block_execute_if[bi].ready = is_tcu_ld - ? agu_ld_ready[bi] - : core_execute_if[bi].ready; - `else - // No sparse: pass-through to tcu_core - assign core_execute_if[bi].valid = per_block_execute_if[bi].valid; - assign core_execute_if[bi].data = per_block_execute_if[bi].data; - assign per_block_execute_if[bi].ready = core_execute_if[bi].ready; - `endif - end - - // ----------------------------------------------------------------------- - // Result_if merge: AGU result and tcu_core result are mutually exclusive - // in time per block; OR-mux with priority arbiter (AGU wins: TCU_LD is rare). - // ----------------------------------------------------------------------- -`ifdef TCU_META_ENABLE - // AGU wins same-cycle conflicts; tcu_core stalls (ready=0) and retries next cycle. - for (genvar bi = 0; bi < BLOCK_SIZE; ++bi) begin : g_result_merge - assign per_block_result_if[bi].valid = agu_result_valid[bi] || core_result_if[bi].valid; - assign per_block_result_if[bi].data = agu_result_valid[bi] - ? agu_result_data[bi] - : core_result_if[bi].data; - assign agu_result_ready[bi] = per_block_result_if[bi].ready; - assign core_result_if[bi].ready = per_block_result_if[bi].ready && !agu_result_valid[bi]; - end -`else - for (genvar bi = 0; bi < BLOCK_SIZE; ++bi) begin : g_result_passthru - assign per_block_result_if[bi].valid = core_result_if[bi].valid; - assign per_block_result_if[bi].data = core_result_if[bi].data; - assign core_result_if[bi].ready = per_block_result_if[bi].ready; - end -`endif - - // ----------------------------------------------------------------------- - // WGMMA feature (orchestrator): VX_tcu_tbuf + VX_tcu_lockstep + perf. - // ----------------------------------------------------------------------- - `ifdef VX_CFG_TCU_WGMMA_ENABLE - wire [BLOCK_SIZE-1:0] exec_valid_w; - wire [BLOCK_SIZE-1:0] exec_ready_w; - tcu_execute_t exec_data_w [BLOCK_SIZE]; - for (genvar bi = 0; bi < BLOCK_SIZE; ++bi) begin : g_obs - assign exec_valid_w[bi] = core_execute_if[bi].valid; - assign exec_ready_w[bi] = core_execute_if[bi].ready; - assign exec_data_w[bi] = core_execute_if[bi].data; - end - + wire [BLOCK_SIZE-1:0] exec_valid, exec_ready; + tcu_execute_t exec_data [BLOCK_SIZE]; wire [BLOCK_SIZE-1:0][TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tbuf_rs1_data; - wire [BLOCK_SIZE-1:0][TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tbuf_rs2_data; - wire [BLOCK_SIZE-1:0] tbuf_ready_eff; - - VX_tcu_wgmma #( - .INSTANCE_ID (`SFORMATF(("%s-wgmma", INSTANCE_ID))), - .BLOCK_SIZE (BLOCK_SIZE) - ) wgmma ( - .clk (clk), - .reset (reset), - `ifdef PERF_ENABLE - .tcu_perf (tcu_perf), - `endif - .exec_valid (exec_valid_w), - .exec_ready (exec_ready_w), - .exec_data (exec_data_w), - .tcu_lmem_if (tcu_lmem_if), - .tbuf_rs1_data (tbuf_rs1_data), - .tbuf_rs2_data (tbuf_rs2_data), - .tbuf_ready_eff (tbuf_ready_eff) - ); - -`else // !VX_CFG_TCU_WGMMA_ENABLE - -`ifdef PERF_ENABLE - assign tcu_perf.tbuf_stalls = '0; - assign tcu_perf.tbuf_cache_hits = '0; - assign tcu_perf.lmem_reads = '0; - assign tcu_perf.wgmma_instrs = '0; - assign tcu_perf.wgmma_stalls = '0; + wire [BLOCK_SIZE-1:0][TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tbuf_rs2_data; + wire [BLOCK_SIZE-1:0] tbuf_ready_eff; `endif -`endif // VX_CFG_TCU_WGMMA_ENABLE - - // ----------------------------------------------------------------------- - // VX_tcu_agu — warp-level AGU for TCU_LD instructions. - // Drives meta_wr signals broadcast to every tcu_core so wmma_sp on - // any block sees the loaded metadata. - // ----------------------------------------------------------------------- `ifdef TCU_META_ENABLE - // The AGU streams NUM_TCU_LANES operands over the LSU memory client, whose - // request/response mask and data are sized by NUM_LSU_LANES; the two lane - // counts must match for the per-lane completion tracking to be correct. - `STATIC_ASSERT (NUM_LANES == `VX_CFG_NUM_LSU_LANES, ("VX_tcu_agu requires NUM_TCU_LANES == NUM_LSU_LANES")); - - wire agu_meta_wr_en; - wire [NW_WIDTH-1:0] agu_meta_wr_wid; - wire [4:0] agu_meta_wr_idx; - wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] agu_meta_wr_data; - - VX_tcu_agu #( - .INSTANCE_ID (`SFORMATF(("%s-agu", INSTANCE_ID))), - .BLOCK_SIZE (BLOCK_SIZE), - .NUM_LANES (NUM_LANES) - ) agu ( - .clk (clk), - .reset (reset), - .per_block_ld_valid (agu_ld_valid), - .per_block_ld_data (agu_ld_data), - .per_block_ld_ready (agu_ld_ready), - .client_if (tcu_mem_if), - .meta_wr_en (agu_meta_wr_en), - .meta_wr_wid (agu_meta_wr_wid), - .meta_wr_idx (agu_meta_wr_idx), - .meta_wr_data (agu_meta_wr_data), - .result_valid (agu_result_valid), - .result_data (agu_result_data), - .result_ready (agu_result_ready) - ); + wire [BLOCK_SIZE-1:0] agu_ld_valid, agu_ld_ready; + tcu_execute_t agu_ld_data [BLOCK_SIZE]; + wire [BLOCK_SIZE-1:0] agu_result_valid, agu_result_ready; + tcu_result_t agu_result_data [BLOCK_SIZE]; + wire meta_wr_en; + wire [NW_WIDTH-1:0] meta_wr_wid; + wire [4:0] meta_wr_idx; + wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] meta_wr_data; `endif - // ----------------------------------------------------------------------- - // TCU core instances - // ----------------------------------------------------------------------- - for (genvar block_idx = 0; block_idx < BLOCK_SIZE; ++block_idx) begin : g_blocks - VX_tcu_core #( - .INSTANCE_ID (`SFORMATF(("%s-fused%0d", INSTANCE_ID, block_idx))) - ) tcu_core ( + `ifdef TCU_META_ENABLE + wire block_meta_wr_en = meta_wr_en + && (NW_WIDTH'(meta_wr_wid % `VX_CFG_NUM_TCU_BLOCKS) == NW_WIDTH'(block_idx)); + `endif + + VX_tcu_block #( + .INSTANCE_ID (INSTANCE_ID), + .BLOCK_STR (`SFORMATF(("%0d", block_idx))) + ) tcu_block ( `SCOPE_IO_BIND (block_idx) - .clk (clk), - .reset (reset), + .clk (clk), + .reset (reset), `ifdef VX_CFG_TCU_WGMMA_ENABLE - .tbuf_rs1_data (tbuf_rs1_data[block_idx]), - .tbuf_rs2_data (tbuf_rs2_data[block_idx]), - .tbuf_ready (tbuf_ready_eff[block_idx]), + .exec_valid (exec_valid[block_idx]), + .exec_ready (exec_ready[block_idx]), + .exec_data (exec_data[block_idx]), + .tbuf_rs1_data (tbuf_rs1_data[block_idx]), + .tbuf_rs2_data (tbuf_rs2_data[block_idx]), + .tbuf_ready (tbuf_ready_eff[block_idx]), `endif `ifdef TCU_META_ENABLE - .ext_meta_wr_en (agu_meta_wr_en), - .ext_meta_wr_wid (agu_meta_wr_wid), - .ext_meta_wr_idx (agu_meta_wr_idx), - .ext_meta_wr_data (agu_meta_wr_data), + .agu_ld_valid (agu_ld_valid[block_idx]), + .agu_ld_data (agu_ld_data[block_idx]), + .agu_ld_ready (agu_ld_ready[block_idx]), + .agu_result_valid (agu_result_valid[block_idx]), + .agu_result_data (agu_result_data[block_idx]), + .agu_result_ready (agu_result_ready[block_idx]), + .meta_wr_en (block_meta_wr_en), + .meta_wr_wid (meta_wr_wid), + .meta_wr_idx (meta_wr_idx), + .meta_wr_data (meta_wr_data), `endif - .execute_if (core_execute_if[block_idx]), - .result_if (core_result_if[block_idx]) + .execute_if (per_block_execute_if[block_idx]), + .result_if (per_block_result_if[block_idx]) ); end + VX_tcu_shared #( + .INSTANCE_ID (INSTANCE_ID), + .BLOCK_SIZE (BLOCK_SIZE) + ) tcu_shared ( + `ifdef PERF_ENABLE + .tcu_perf (tcu_perf), + `endif + `ifdef VX_CFG_TCU_WGMMA_ENABLE + .exec_valid (exec_valid), + .exec_ready (exec_ready), + .exec_data (exec_data), + .tbuf_rs1_data (tbuf_rs1_data), + .tbuf_rs2_data (tbuf_rs2_data), + .tbuf_ready_eff (tbuf_ready_eff), + .tcu_lmem_if (tcu_lmem_if), + `endif + `ifdef TCU_META_ENABLE + .agu_ld_valid (agu_ld_valid), + .agu_ld_data (agu_ld_data), + .agu_ld_ready (agu_ld_ready), + .agu_result_valid (agu_result_valid), + .agu_result_data (agu_result_data), + .agu_result_ready (agu_result_ready), + .meta_wr_en (meta_wr_en), + .meta_wr_wid (meta_wr_wid), + .meta_wr_idx (meta_wr_idx), + .meta_wr_data (meta_wr_data), + .tcu_mem_if (tcu_mem_if), + `endif + .clk (clk), + .reset (reset) + ); + // ----------------------------------------------------------------------- // Lane gather // ----------------------------------------------------------------------- diff --git a/hw/rtl/tex/VX_tex_core.sv b/hw/rtl/tex/VX_tex_core.sv index 7ffec13f8c..c8e427df62 100644 --- a/hw/rtl/tex/VX_tex_core.sv +++ b/hw/rtl/tex/VX_tex_core.sv @@ -48,7 +48,7 @@ module VX_tex_core import VX_gpu_pkg::*; import VX_tex_pkg::*; #( tex_dcrs_t tex_dcrs; VX_tex_dcr #( - .INSTANCE_ID ($sformatf("%s-dcr", INSTANCE_ID)), + .INSTANCE_ID (`SFORMATF(("%s-dcr", INSTANCE_ID))), .NUM_STAGES (`VX_TEX_STAGE_COUNT) ) tex_dcr ( .clk (clk), @@ -132,7 +132,7 @@ module VX_tex_core import VX_gpu_pkg::*; import VX_tex_pkg::*; #( // sampler, for the same reason the format does: the state a sample merges // under is the state it was issued under. VX_tex_addr #( - .INSTANCE_ID ($sformatf("%s-addr", INSTANCE_ID)), + .INSTANCE_ID (`SFORMATF(("%s-addr", INSTANCE_ID))), .REQ_TAGW (TAG_WIDTH + TEX_FORMAT_BITS + 32), .NUM_LANES (NUM_LANES) ) tex_addr ( @@ -176,7 +176,7 @@ module VX_tex_core import VX_gpu_pkg::*; import VX_tex_pkg::*; #( wire mem_rsp_ready; VX_tex_mem #( - .INSTANCE_ID ($sformatf("%s-mem", INSTANCE_ID)), + .INSTANCE_ID (`SFORMATF(("%s-mem", INSTANCE_ID))), .REQ_TAGW (TAG_WIDTH + TEX_FORMAT_BITS + 32 + BLEND_FRAC_W + BORDER_W + LODFRAC_W), .NUM_LANES (NUM_LANES) ) tex_mem ( @@ -218,7 +218,7 @@ module VX_tex_core import VX_gpu_pkg::*; import VX_tex_pkg::*; #( wire [TAG_WIDTH-1:0] mem_rsp_tag = mem_rsp_info[(LODFRAC_W + BORDER_W + BLEND_FRAC_W + 32 + TEX_FORMAT_BITS) +: TAG_WIDTH]; VX_tex_sampler #( - .INSTANCE_ID ($sformatf("%s-sampler", INSTANCE_ID)), + .INSTANCE_ID (`SFORMATF(("%s-sampler", INSTANCE_ID))), .REQ_TAGW (TAG_WIDTH), .NUM_LANES (NUM_LANES) ) tex_sampler ( diff --git a/hw/rtl/tex/VX_tex_mem.sv b/hw/rtl/tex/VX_tex_mem.sv index d712174204..8907ff267d 100644 --- a/hw/rtl/tex/VX_tex_mem.sv +++ b/hw/rtl/tex/VX_tex_mem.sv @@ -128,7 +128,7 @@ module VX_tex_mem import VX_gpu_pkg::*; import VX_tex_pkg::*; #( ) lsu_mem_if(); VX_mem_scheduler #( - .INSTANCE_ID ($sformatf("%s-memsched", INSTANCE_ID)), + .INSTANCE_ID (`SFORMATF(("%s-memsched", INSTANCE_ID))), .CORE_REQS (TEX_MEM_REQS), .MEM_CHANNELS(TCACHE_NUM_REQS), .WORD_SIZE (4), diff --git a/hw/rtl/tex/VX_tex_unit.sv b/hw/rtl/tex/VX_tex_unit.sv index 56c19b2be8..d49391f3e1 100644 --- a/hw/rtl/tex/VX_tex_unit.sv +++ b/hw/rtl/tex/VX_tex_unit.sv @@ -32,7 +32,6 @@ module VX_tex_unit import VX_gpu_pkg::*, VX_tex_pkg::*; #( parameter `STRING INSTANCE_ID = "", - parameter CORE_ID = 0, parameter NUM_LANES = `VX_CFG_NUM_THREADS ) ( input wire clk, @@ -46,7 +45,6 @@ module VX_tex_unit import VX_gpu_pkg::*, VX_tex_pkg::*; #( VX_tex_bus_if.master tex_bus_if ); `UNUSED_SPARAM (INSTANCE_ID) - `UNUSED_PARAM (CORE_ID) localparam REQ_QUEUE_BITS = `LOG2UP(`VX_CFG_TEX_REQ_QUEUE_SIZE); localparam PID_W = `LOG2UP(`VX_CFG_NUM_THREADS / NUM_LANES); diff --git a/hw/syn/firesim/Makefile b/hw/syn/firesim/Makefile index 10e96cd472..b945a90189 100644 --- a/hw/syn/firesim/Makefile +++ b/hw/syn/firesim/Makefile @@ -123,7 +123,7 @@ PLATFORM_DEFINES += -DPLATFORM_MEMORY_ID_WIDTH=4 # them. The AXI boundary cannot show any of that -- a core waiting on a response # it never retired looks exactly like a core with nothing left to do. They are # read over the DCR response path, which both the harness and the driver already -# drive, but VX_csr_data.sv gates them behind PERF_ENABLE. Off by default because +# drive, but VX_csr_mpm.sv gates them behind PERF_ENABLE. Off by default because # they cost area; build with PERF_DEFINES=-DPERF_ENABLE to read them, and note # that without it every counter reads zero, which is indistinguishable from an # idle core rather than an absent instrument. diff --git a/hw/syn/xilinx/aved/platforms.mk b/hw/syn/xilinx/aved/platforms.mk index 9a6b5b2a90..80ced1dd35 100644 --- a/hw/syn/xilinx/aved/platforms.mk +++ b/hw/syn/xilinx/aved/platforms.mk @@ -33,7 +33,7 @@ override CONFIGS += -DPLATFORM_MEMORY_OFFSET=40\'h4000000000 # which the compute shell maps as 1 GB at 0x40_0000_0000. Vortex's 32-bit memory # map puts the stack at the top of a 4 GB space -- vx_start.S sets sp to # VX_MEM_STACK_BASE_ADDR (0xFFFF0000) and grows down, and that lands just below -# the local-memory window (VX_lsu_slice.sv decodes LMEM *upward* from the same +# the local-memory window (VX_lsu_block.sv decodes LMEM *upward* from the same # base), so it is a global-memory access. Rebased it becomes 0x40_FFFE_Fxxx, # which a 1 GB aperture cannot answer: the read gets no response, the LSU stalls # forever, and the AFU never goes idle -- taking the shell's AXI path with it. diff --git a/hw/unittest/core/VX_core_top.sv b/hw/unittest/core/VX_core_top.sv index 1e0621d0a7..b0dbb9b44a 100644 --- a/hw/unittest/core/VX_core_top.sv +++ b/hw/unittest/core/VX_core_top.sv @@ -42,18 +42,18 @@ module VX_core_top import VX_gpu_pkg::*; input wire [DCACHE_NUM_REQS-1:0][DCACHE_TAG_WIDTH-1:0] dcache_rsp_tag, output wire [DCACHE_NUM_REQS-1:0] dcache_rsp_ready, - output wire icache_req_valid, - output wire icache_req_rw, - output wire [ICACHE_WORD_SIZE-1:0] icache_req_byteen, - output wire [ICACHE_ADDR_WIDTH-1:0] icache_req_addr, - output wire [ICACHE_WORD_SIZE*8-1:0] icache_req_data, - output wire [ICACHE_TAG_WIDTH-1:0] icache_req_tag, - input wire icache_req_ready, - - input wire icache_rsp_valid, - input wire [ICACHE_WORD_SIZE*8-1:0] icache_rsp_data, - input wire [ICACHE_TAG_WIDTH-1:0] icache_rsp_tag, - output wire icache_rsp_ready, + output wire [ICACHE_NUM_REQS-1:0] icache_req_valid, + output wire [ICACHE_NUM_REQS-1:0] icache_req_rw, + output wire [ICACHE_NUM_REQS-1:0][ICACHE_WORD_SIZE-1:0] icache_req_byteen, + output wire [ICACHE_NUM_REQS-1:0][ICACHE_ADDR_WIDTH-1:0] icache_req_addr, + output wire [ICACHE_NUM_REQS-1:0][ICACHE_WORD_SIZE*8-1:0] icache_req_data, + output wire [ICACHE_NUM_REQS-1:0][ICACHE_TAG_WIDTH-1:0] icache_req_tag, + input wire [ICACHE_NUM_REQS-1:0] icache_req_ready, + + input wire [ICACHE_NUM_REQS-1:0] icache_rsp_valid, + input wire [ICACHE_NUM_REQS-1:0][ICACHE_WORD_SIZE*8-1:0] icache_rsp_data, + input wire [ICACHE_NUM_REQS-1:0][ICACHE_TAG_WIDTH-1:0] icache_rsp_tag, + output wire [ICACHE_NUM_REQS-1:0] icache_rsp_ready, output wire gbar_req_valid, output wire [NB_WIDTH-1:0] gbar_req_id, @@ -174,21 +174,23 @@ module VX_core_top import VX_gpu_pkg::*; VX_mem_bus_if #( .DATA_SIZE (ICACHE_WORD_SIZE), .TAG_WIDTH (ICACHE_TAG_WIDTH) - ) icache_bus_if(); - - assign icache_req_valid = icache_bus_if.req_valid; - assign icache_req_rw = icache_bus_if.req_data.rw; - assign icache_req_byteen = icache_bus_if.req_data.byteen; - assign icache_req_addr = icache_bus_if.req_data.addr; - assign icache_req_data = icache_bus_if.req_data.data; - assign icache_req_tag = icache_bus_if.req_data.tag; - assign icache_bus_if.req_ready = icache_req_ready; - `UNUSED_VAR (icache_bus_if.req_data.attr) - - assign icache_bus_if.rsp_valid = icache_rsp_valid; - assign icache_bus_if.rsp_data.tag = icache_rsp_tag; - assign icache_bus_if.rsp_data.data = icache_rsp_data; - assign icache_rsp_ready = icache_bus_if.rsp_ready; + ) icache_bus_if[ICACHE_NUM_REQS](); + + for (genvar i = 0; i < ICACHE_NUM_REQS; ++i) begin : g_icache_bus_if + assign icache_req_valid[i] = icache_bus_if[i].req_valid; + assign icache_req_rw[i] = icache_bus_if[i].req_data.rw; + assign icache_req_byteen[i] = icache_bus_if[i].req_data.byteen; + assign icache_req_addr[i] = icache_bus_if[i].req_data.addr; + assign icache_req_data[i] = icache_bus_if[i].req_data.data; + assign icache_req_tag[i] = icache_bus_if[i].req_data.tag; + assign icache_bus_if[i].req_ready = icache_req_ready[i]; + `UNUSED_VAR (icache_bus_if[i].req_data.attr) + + assign icache_bus_if[i].rsp_valid = icache_rsp_valid[i]; + assign icache_bus_if[i].rsp_data.tag = icache_rsp_tag[i]; + assign icache_bus_if[i].rsp_data.data = icache_rsp_data[i]; + assign icache_rsp_ready[i] = icache_bus_if[i].rsp_ready; + end `ifdef PERF_ENABLE sysmem_perf_t mem_perf; @@ -208,12 +210,12 @@ module VX_core_top import VX_gpu_pkg::*; `endif VX_core #( - .INSTANCE_ID (`SFORMATF(("core"))), - .CORE_ID (CORE_ID) + .INSTANCE_ID (`SFORMATF(("core"))) ) core ( `SCOPE_IO_BIND (0) .clk (clk), .reset (reset), + .core_id (CORE_ID_WIDTH'(CORE_ID)), `ifdef PERF_ENABLE .sysmem_perf (sysmem_perf), diff --git a/hw/unittest/cta_dispatcher/VX_cta_dispatch_top.sv b/hw/unittest/cta_dispatcher/VX_cta_dispatch_top.sv index 564634269b..9062fd6110 100644 --- a/hw/unittest/cta_dispatcher/VX_cta_dispatch_top.sv +++ b/hw/unittest/cta_dispatcher/VX_cta_dispatch_top.sv @@ -101,21 +101,18 @@ module VX_cta_dispatch_top import VX_gpu_pkg::*; wire cta_init; wire busy; - // CTA-CSR read-back / schedule lookup ports are unexercised by this bench - // (it only checks warp dispatch via cta_fire/cta_wid), so the read indices - // are tied off and the read-back outputs are left unused. - cta_csrs_t cta_rd_csrs; - cta_lane_t [`VX_CFG_NUM_THREADS-1:0] cta_rd_lane; - wire [NCTA_WIDTH-1:0] schedule_cta_id; + // The CTA-CSR broadcast is unexercised by this bench (it only checks warp + // dispatch via cta_fire/cta_wid). + cta_bcast_t cta_bcast; + cta_lane_bcast_t cta_lane_bcast; `UNUSED_VAR (cta_PC) `UNUSED_VAR (cta_tmask) `UNUSED_VAR (cta_param) `UNUSED_VAR (cta_init) `UNUSED_VAR (busy) - `UNUSED_VAR (cta_rd_csrs) - `UNUSED_VAR (cta_rd_lane) - `UNUSED_VAR (schedule_cta_id) + `UNUSED_VAR (cta_bcast) + `UNUSED_VAR (cta_lane_bcast) VX_cta_dispatch cta_dispatch ( .clk (clk), @@ -130,12 +127,8 @@ module VX_cta_dispatch_top import VX_gpu_pkg::*; .cta_tmask (cta_tmask), .cta_param (cta_param), .cta_init (cta_init), - .csr_rd_wid ('0), - .csr_rd_cta_id ('0), - .cta_rd_csrs (cta_rd_csrs), - .cta_rd_lane (cta_rd_lane), - .schedule_wid ('0), - .schedule_cta_id (schedule_cta_id), + .cta_bcast (cta_bcast), + .cta_lane_bcast(cta_lane_bcast), .busy (busy) ); diff --git a/hw/unittest/gfx/VX_gfx_top.sv b/hw/unittest/gfx/VX_gfx_top.sv index ce4ef6876d..09159d41da 100644 --- a/hw/unittest/gfx/VX_gfx_top.sv +++ b/hw/unittest/gfx/VX_gfx_top.sv @@ -664,10 +664,11 @@ module VX_gfx_top import VX_gpu_pkg::* `endif VX_graphics #( - .CLUSTER_ID (0) + .INSTANCE_ID ("cluster0") ) graphics ( .clk (clk), .reset (reset), + .cluster_id ('0), `ifdef VX_CFG_EXT_RASTER_ENABLE .raster_kmu_bus_if (raster_kmu_bus_if), .rcache_mem_bus_if (rcache_l2_bus_if), diff --git a/hw/unittest/issue/VX_issue_top.sv b/hw/unittest/issue/VX_issue_top.sv index 0b34f25478..01de117a5e 100644 --- a/hw/unittest/issue/VX_issue_top.sv +++ b/hw/unittest/issue/VX_issue_top.sv @@ -90,7 +90,21 @@ module VX_issue_top import VX_gpu_pkg::*; #( assign decode_if.data.rs1 = decode_rs1; assign decode_if.data.rs2 = decode_rs2; assign decode_if.data.rs3 = decode_rs3; - assign decode_ready = decode_if.ready; + + // issue slice i takes the warps with wid mod ISSUE_WIDTH == i + VX_decode_if slice_decode_if[`VX_CFG_ISSUE_WIDTH](); + wire [ISSUE_ISW_W-1:0] decode_isw = wid_to_isw(decode_wid); + wire [`VX_CFG_ISSUE_WIDTH-1:0] slice_decode_ready; + for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_decode_if + assign slice_decode_if[i].valid = decode_if.valid && (decode_isw == ISSUE_ISW_W'(i)); + assign slice_decode_if[i].data = decode_if.data; + assign slice_decode_ready[i] = slice_decode_if[i].ready; + `UNUSED_VAR (slice_decode_if[i].ibuf_pop) + end + assign decode_if.ready = slice_decode_ready[decode_isw]; + assign decode_if.ibuf_pop = '0; + assign decode_ready = decode_if.ready; + `UNUSED_VAR (decode_if.ibuf_pop) for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_writeback_if assign writeback_if[i].valid = writeback_valid[i]; @@ -157,7 +171,7 @@ module VX_issue_top import VX_gpu_pkg::*; #( .issue_perf (issue_perf), `endif - .decode_if (decode_if), + .decode_if (slice_decode_if), .writeback_if (writeback_if), .dispatch_if (dispatch_if), .issue_sched_if (issue_sched_if) diff --git a/hw/unittest/raster_core/VX_raster_core_top.sv b/hw/unittest/raster_core/VX_raster_core_top.sv index 9e1665a093..fb64f41c63 100644 --- a/hw/unittest/raster_core/VX_raster_core_top.sv +++ b/hw/unittest/raster_core/VX_raster_core_top.sv @@ -108,7 +108,6 @@ module VX_raster_core_top import VX_gpu_pkg::*; import VX_raster_pkg::*; #( VX_raster_core #( .INSTANCE_ID (INSTANCE_ID), - .INSTANCE_IDX (INSTANCE_IDX), .NUM_INSTANCES (NUM_INSTANCES), .NUM_SLICES (NUM_SLICES), .TILE_LOGSIZE (TILE_LOGSIZE), @@ -120,6 +119,7 @@ module VX_raster_core_top import VX_gpu_pkg::*; import VX_raster_pkg::*; #( `SCOPE_IO_BIND (0) .clk (clk), .reset (reset), + .instance_idx (`LOG2UP(NUM_INSTANCES)'(INSTANCE_IDX)), `ifdef PERF_ENABLE .perf_raster_if(perf_raster_if), `endif diff --git a/sim/firesim/firesim_sim.cpp b/sim/firesim/firesim_sim.cpp index 1ff8723c8c..9604c3a01e 100644 --- a/sim/firesim/firesim_sim.cpp +++ b/sim/firesim/firesim_sim.cpp @@ -483,7 +483,7 @@ void firesim_sim::report_progress(const char *why) { // they are parked at a barrier, and which pipeline is holding them. // // Requires the RTL to be built with PERF_ENABLE (see hw/syn/firesim/Makefile); -// without it VX_csr_data.sv ties the counters off and every read returns zero, +// without it VX_csr_mpm.sv ties the counters off and every read returns zero, // so an all-zero line here means the instrument is absent, not that the core is // idle. That distinction is exactly the kind this project has been bitten by, // so it is called out rather than left to be inferred. diff --git a/sim/simx/constants.h b/sim/simx/constants.h index 9b3703bda0..da7a62eaa3 100644 --- a/sim/simx/constants.h +++ b/sim/simx/constants.h @@ -105,4 +105,13 @@ inline constexpr uint32_t VX_CFG_L3_NUM_REQS = VX_CFG_NUM_CLUSTERS * VX_CFG_ inline constexpr uint32_t PER_ISSUE_WARPS = VX_CFG_NUM_WARPS / VX_CFG_ISSUE_WIDTH; inline constexpr uint32_t ISSUE_WIS_BITS = log2ceil(PER_ISSUE_WARPS); +// Fetch slice f owns the warps with wid % VX_CFG_FETCH_WIDTH == f; each has its +// own warp scheduler, fetch unit, decoder and icache request port. +inline constexpr uint32_t PER_FETCH_WARPS = VX_CFG_NUM_WARPS / VX_CFG_FETCH_WIDTH; +inline constexpr uint32_t ICACHE_NUM_REQS = VX_CFG_FETCH_WIDTH; +static_assert(VX_CFG_FETCH_WIDTH != 0 && (VX_CFG_FETCH_WIDTH & (VX_CFG_FETCH_WIDTH - 1)) == 0 + && VX_CFG_FETCH_WIDTH <= VX_CFG_NUM_WARPS + && (VX_CFG_ISSUE_WIDTH % VX_CFG_FETCH_WIDTH) == 0, + "FETCH_WIDTH must be a power of two that divides ISSUE_WIDTH and is at most NUM_WARPS"); + } // namespace vortex diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index ec729d83e3..447314ce80 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -66,9 +66,6 @@ class Core::Impl { , func_units_((uint32_t)FUType::Count) , lmem_switch_(VX_CFG_NUM_LSU_BLOCKS) , mem_coalescers_(VX_CFG_NUM_LSU_BLOCKS) - , fetch_latch_(ctx, "fetch_latch", 2, 2) - , decode_latch_(ctx, "decode_latch", 1, 2) - , pending_icache_(VX_CFG_NUM_WARPS) , ibuffer_arbs_(VX_CFG_ISSUE_WIDTH, {ArbiterType::GreedyRR, PER_ISSUE_WARPS}) , issue_lock_owner_(VX_CFG_ISSUE_WIDTH, kLockOpen) , fu_credits_(VX_CFG_ISSUE_WIDTH, std::vector((uint32_t)FUType::Count, 0)) @@ -81,6 +78,16 @@ class Core::Impl { const std::string& name = simobject_->name(); char sname[100]; + // per fetch slice: scheduler-to-fetch and fetch-to-decode latches, and the + // icache tags of the slice's in-flight fetches + for (uint32_t f = 0; f < VX_CFG_FETCH_WIDTH; ++f) { + snprintf(sname, 100, "fetch_latch%d", f); + fetch_latch_.emplace_back(std::make_unique>(ctx, sname, 2, 2)); + snprintf(sname, 100, "decode_latch%d", f); + decode_latch_.emplace_back(std::make_unique>(ctx, sname, 1, 2)); + pending_icache_.emplace_back(PER_FETCH_WARPS); + } + // create scheduler (SimObject) — also creates the CTA dispatcher. snprintf(sname, 100, "%s-scheduler", name.c_str()); scheduler_ = SimPlatform::instance().create_object(sname, simobject_); @@ -92,8 +99,10 @@ class Core::Impl { #ifdef VX_CFG_EXT_C_ENABLE // create RVC fetch decompressor (SimObject) — owns per-warp half-word // buffer + refetch queue. Resets via SimObject::on_reset. - snprintf(sname, 100, "%s-decompressor", name.c_str()); - decompressor_ = SimPlatform::instance().create_object(sname, VX_CFG_NUM_WARPS); + for (uint32_t f = 0; f < VX_CFG_FETCH_WIDTH; ++f) { + snprintf(sname, 100, "%s-decompressor%d", name.c_str(), f); + decompressors_.push_back(SimPlatform::instance().create_object(sname, VX_CFG_NUM_WARPS)); + } #endif // create scoreboard (SimObject) @@ -209,11 +218,11 @@ class Core::Impl { dcache_mmu_ = Mmu::Create(sname, VX_CFG_DCACHE_NUM_REQS, VX_CFG_DTLB_SIZE, local_core * 2 + 0, false); - // Per-core icache MMU (1 port). Fetch reads/writes its upstream - // channels (ReqIn[0]/RspOut[0]) directly; the downstream side is - // bound to the Core's icache port below. + // Per-core icache MMU, one port per fetch slice. Fetch reads/writes its + // upstream channels directly; the downstream side is bound to the Core's + // icache ports below. snprintf(sname, 100, "%s-icache_mmu", name.c_str()); - icache_mmu_ = Mmu::Create(sname, 1, + icache_mmu_ = Mmu::Create(sname, ICACHE_NUM_REQS, VX_CFG_ITLB_SIZE, local_core * 2 + 1, true); // LSU memory side -> dcache MMU -> core's dcache port @@ -224,10 +233,12 @@ class Core::Impl { simobject_->dcache_rsp_in.at(p).bind(&dcache_mmu_->RspIn.at(p)); } - // icache MMU downstream side -> core's icache port. Upstream side - // (ReqIn[0]/RspOut[0]) is consumed directly by fetch() in on_tick. - icache_mmu_->ReqOut.at(0).bind(&simobject_->icache_req_out.at(0)); - simobject_->icache_rsp_in.at(0).bind(&icache_mmu_->RspIn.at(0)); + // icache MMU downstream side -> core's icache ports. Upstream side is + // consumed directly by fetch() in on_tick. + for (uint32_t f = 0; f < ICACHE_NUM_REQS; ++f) { + icache_mmu_->ReqOut.at(f).bind(&simobject_->icache_req_out.at(f)); + simobject_->icache_rsp_in.at(f).bind(&icache_mmu_->RspIn.at(f)); + } #else // No-VM: direct passthrough. for (uint32_t p = 0; p < VX_CFG_NUM_LSU_BLOCKS * DCACHE_CHANNELS; ++p) { @@ -360,61 +371,88 @@ class Core::Impl { perf_stats_.active_warps += scheduler_->active_warps().count(); perf_stats_.stalled_warps += scheduler_->stalled_warps().count(); - // stop when fetch_latch cannot accept - if (fetch_latch_.full()) - return; + scheduler_->schedule_begin(); + + // each fetch slice picks among its own warps whose ibuffer is not yet saturated + // the cycle is idle when no slice has a warp to fetch, its latch free or not + bool any_cand = false; + for (uint32_t f = 0; f < VX_CFG_FETCH_WIDTH; ++f) { + auto& fetch_latch = fetch_latch_.at(f); + WarpMask warp_mask; + for (uint32_t w = f; w < VX_CFG_NUM_WARPS; w += VX_CFG_FETCH_WIDTH) { + if (ibuf_inflight_.at(w) < VX_CFG_IBUF_SIZE) { + warp_mask.set(w); + } + } - // eligible warps: ibuffer not yet saturated - WarpMask warp_mask; - for (uint32_t w = 0, nw = VX_CFG_NUM_WARPS; w < nw; ++w) { - if (ibuf_inflight_.at(w) < VX_CFG_IBUF_SIZE) { - warp_mask.set(w); + if (fetch_latch->full()) { + any_cand |= scheduler_->has_ready(warp_mask); + continue; } - } - auto trace = scheduler_->schedule(warp_mask); - if (trace == nullptr) { + auto trace = scheduler_->schedule(warp_mask); + if (trace == nullptr) { + continue; + } + any_cand = true; + + fetch_latch->push(trace); + DT(3, simobject_->name() << "-pipeline schedule: " << *trace); + pending_instrs_.push_back(trace); + ++ibuf_inflight_.at(trace->wid); + perf_stats_.issued_warps += 1; + perf_stats_.issued_threads += trace->tmask.count(); + } + if (!any_cand) { ++perf_stats_.sched_idle; - return; } - fetch_latch_.push(trace); - DT(3, simobject_->name() << "-pipeline schedule: " << *trace); - pending_instrs_.push_back(trace); - ++ibuf_inflight_.at(trace->wid); - perf_stats_.issued_warps += 1; - perf_stats_.issued_threads += trace->tmask.count(); + scheduler_->schedule_end(); } void fetch() { perf_stats_.ifetch_latency += pending_ifetches_; + for (uint32_t f = 0; f < VX_CFG_FETCH_WIDTH; ++f) { + this->fetch_slice(f); + } + } + + // Fetch slice f: its icache response into its decode latch, then its next + // fetch request out of its fetch latch, on icache port f. + void fetch_slice(uint32_t f) { + auto& fetch_latch = *fetch_latch_.at(f); + auto& decode_latch = *decode_latch_.at(f); + auto& pending_icache = pending_icache_.at(f); + #ifdef VX_CFG_EXT_C_ENABLE + auto& decompressor = decompressors_.at(f); + #endif #ifdef VX_CFG_VM_ENABLE // With VM, fetch reads/writes the icache MMU's upstream channels. // The MMU translates and forwards to simobject_->icache_req_out // downstream (driven by SimChannel binding in the constructor). - auto& icache_rsp = icache_mmu_->RspOut.at(0); - auto& icache_req = icache_mmu_->ReqIn.at(0); + auto& icache_rsp = icache_mmu_->RspOut.at(f); + auto& icache_req = icache_mmu_->ReqIn.at(f); #else - auto& icache_rsp = simobject_->icache_rsp_in.at(0); - auto& icache_req = simobject_->icache_req_out.at(0); + auto& icache_rsp = simobject_->icache_rsp_in.at(f); + auto& icache_req = simobject_->icache_req_out.at(f); #endif // handle icache response — extract instruction word from the TLM line payload if (!icache_rsp.empty()){ auto& mem_rsp = icache_rsp.peek(); - auto trace = pending_icache_.at(mem_rsp.tag); + auto trace = pending_icache.at(mem_rsp.tag); assert(mem_rsp.data && "icache response must carry line payload"); #ifdef VX_CFG_EXT_C_ENABLE // Decompressor classifies the rsp: emit-ready (sets trace->code) or // queues a refetch internally for cross-word 32-bit completion. - bool emit = decompressor_->on_icache_rsp(trace, *mem_rsp.data); - bool progressed = !emit || decode_latch_.try_push(trace); + bool emit = decompressor->on_icache_rsp(trace, *mem_rsp.data); + bool progressed = !emit || decode_latch.try_push(trace); #else bool emit = true; bool progressed = false; - if (decode_latch_.try_push(trace)) { + if (decode_latch.try_push(trace)) { uint32_t offset = trace->PC & (VX_CFG_MEM_BLOCK_SIZE - 1); std::memcpy(&trace->code, mem_rsp.data->data() + offset, sizeof(uint32_t)); progressed = true; @@ -427,7 +465,7 @@ class Core::Impl { << ", PC=0x" << std::hex << trace->PC << " (#" << std::dec << trace->uuid << ")"); } DT(3, simobject_->name() << " icache-rsp: addr=0x" << std::hex << trace->PC << ", tag=0x" << mem_rsp.tag << std::dec << ", " << *trace); - pending_icache_.release(mem_rsp.tag); + pending_icache.release(mem_rsp.tag); icache_rsp.pop(); --pending_ifetches_; } @@ -440,15 +478,15 @@ class Core::Impl { #ifdef VX_CFG_EXT_C_ENABLE bool from_refetch = false; { - auto pick = decompressor_->pick_request(fetch_latch_.empty() ? nullptr - : fetch_latch_.peek()); + auto pick = decompressor->pick_request(fetch_latch.empty() ? nullptr + : fetch_latch.peek()); trace = pick.trace; req_addr = pick.addr; from_refetch = pick.from_refetch; } #else - if (!fetch_latch_.empty()) { - trace = fetch_latch_.peek(); + if (!fetch_latch.empty()) { + trace = fetch_latch.peek(); req_addr = trace->PC; } #endif @@ -456,7 +494,7 @@ class Core::Impl { return; // Avoid leaking icache tags when the request port back-pressures. - if (pending_icache_.full()) { + if (pending_icache.full()) { ++perf_stats_.fetch_stalls; return; } @@ -466,31 +504,39 @@ class Core::Impl { // substitutes PA before forwarding; with VM off, this is just the PA. mem_req.addr = req_addr; // op defaults to MemOp::LD — icache request is a load. - uint32_t tag = pending_icache_.allocate(trace); + uint32_t tag = pending_icache.allocate(trace); mem_req.tag = tag; mem_req.hart_id = trace->cid; mem_req.uuid = trace->uuid; if (icache_req.try_send(mem_req)) { DT(3, simobject_->name() << " icache-req: addr=0x" << std::hex << mem_req.addr << ", tag=0x" << mem_req.tag << std::dec << ", " << *trace); #ifdef VX_CFG_EXT_C_ENABLE - decompressor_->commit_request(from_refetch); - if (!from_refetch) fetch_latch_.pop(); + decompressor->commit_request(from_refetch); + if (!from_refetch) { fetch_latch.pop(); } #else - fetch_latch_.pop(); + fetch_latch.pop(); #endif ++perf_stats_.ifetches; ++pending_ifetches_; } else { - pending_icache_.release(tag); + pending_icache.release(tag); ++perf_stats_.fetch_stalls; } } void decode() { - if (decode_latch_.empty()) + for (uint32_t f = 0; f < VX_CFG_FETCH_WIDTH; ++f) { + this->decode_slice(*decode_latch_.at(f)); + } + } + + // One fetch slice's decoder: it feeds the issue slice of each decoded warp. + void decode_slice(TFifo& decode_latch) { + if (decode_latch.empty()) { return; + } - auto trace = decode_latch_.peek(); + auto trace = decode_latch.peek(); // check ibuffer capacity auto& ibuffer = ibuffers_.at(trace->wid); @@ -549,7 +595,7 @@ class Core::Impl { trace->ibuf_time = SimPlatform::instance().cycles(); ibuffer->push(trace); - decode_latch_.pop(); + decode_latch.pop(); } void issue() { @@ -1092,10 +1138,10 @@ class Core::Impl { std::vector lmem_switch_; std::vector mem_coalescers_; - TFifo fetch_latch_; - TFifo decode_latch_; + std::vector>> fetch_latch_; + std::vector>> decode_latch_; - HashTable pending_icache_; + std::vector> pending_icache_; std::list> pending_instrs_; uint64_t pending_ifetches_; @@ -1121,9 +1167,9 @@ class Core::Impl { PoolAllocator trace_pool_; #ifdef VX_CFG_EXT_C_ENABLE - // Per-core RVC fetch SimObject (per-warp halfword buffer + refetch + // RVC fetch SimObject per fetch slice (per-warp halfword buffer + refetch // queue + pick/commit_request helpers). See decompressor.h. - Decompressor::Ptr decompressor_; + std::vector decompressors_; #endif #ifdef VX_CFG_VM_ENABLE Mmu::Ptr dcache_mmu_; @@ -1139,8 +1185,8 @@ Core::Core(const SimContext& ctx, Socket* socket ) : SimObject(ctx, name) - , icache_req_out(1, this) - , icache_rsp_in(1, this) + , icache_req_out(ICACHE_NUM_REQS, this) + , icache_rsp_in(ICACHE_NUM_REQS, this) , dcache_req_out(VX_CFG_DCACHE_NUM_REQS, this) , dcache_rsp_in(VX_CFG_DCACHE_NUM_REQS, this) , branch_ctl_in(this, VX_CFG_NUM_WARPS) diff --git a/sim/simx/cta_dispatcher.cpp b/sim/simx/cta_dispatcher.cpp index cb8c10f5e0..950c972df9 100644 --- a/sim/simx/cta_dispatcher.cpp +++ b/sim/simx/cta_dispatcher.cpp @@ -39,11 +39,17 @@ CtaDispatcher::CtaDispatcher(const SimContext& ctx, const char* name, Core* core , rank_(0) , block_size_rem_(0) , lmem_addr_(0) + , lane_rr_(0) + , init_wait_(0) , cur_kernel_pc_(0) , cur_ctx_id_(0) , warp_init_lanes_(num_warps_) { thread_idx_[0] = thread_idx_[1] = thread_idx_[2] = 0; + for (uint32_t i = 0; i < 3; ++i) { + shape_step_[i] = 0; + shape_dim_[i] = 0; + } for (auto& lanes : warp_init_lanes_) { lanes.resize(num_threads_); } @@ -54,11 +60,13 @@ CtaDispatcher::~CtaDispatcher() {} void CtaDispatcher::on_reset() { has_cta_ = false; tail_slot_ = 0; + init_wait_ = 0; for (uint32_t i = 0; i < num_warps_; ++i) { slot_rem_warps_[i] = 0; wid_to_slot_[i] = num_warps_; // invalid } cur_kernel_pc_ = 0; + // lane_rr_ and the cached block shape are device state, like warp_init_lanes_. // warp_init_lanes_ and cur_ctx_id_ are excluded: which lanes of a slot have // already run the startup is keyed on the context, and a launch is not a // device reset. Clearing them here would make the context check dead and @@ -91,14 +99,14 @@ uint32_t CtaDispatcher::usable_slots(uint32_t stride) const { return fit; } -bool CtaDispatcher::step(const WarpMask& active_warps, uint32_t* wid_out, cta_warp_record_t* rec_out) { +uint32_t CtaDispatcher::step(const WarpMask& active_warps, uint32_t* wids, cta_warp_record_t* recs) { if (!has_cta_) { // The KMU pushes CTAs down a registered, credit-gated launch lane; the // endpoint holds the one CTA committed to this core. It is popped only on // admission — an unadmitted CTA keeps the lane's credit, so the KMU // cannot over-commit work to a busy core. if (bus_in.empty()) - return false; + return 0; const kmu_req_t& next = bus_in.peek(); // Fixed-stride admission. Round the per-CTA LMEM size up to a @@ -133,13 +141,13 @@ bool CtaDispatcher::step(const WarpMask& active_warps, uint32_t* wid_out, cta_wa k = max_slots; // cluster larger than co-residency: clamp (degenerate) for (uint32_t i = 0; i < k; ++i) { if (slot_rem_warps_[base + i] != 0) - return false; // window not free yet — wait + return 0; // window not free yet — wait } } else { // Standalone CTA, or a following cluster member (its slot was reserved by // the first-of-cluster window above and is guaranteed free). if (slot_rem_warps_[base] != 0) - return false; + return 0; } // A new launch is a new context: every slot owes the startup again. Keyed on @@ -169,39 +177,65 @@ bool CtaDispatcher::step(const WarpMask& active_warps, uint32_t* wid_out, cta_wa tail_slot_ = (base + 1) % max_slots; has_cta_ = true; - } - // Find lowest-index free warp slot. - int free_wid = -1; - for (int wid = 0; wid < (int)num_warps_; ++wid) { - if (!active_warps.test(wid)) { - free_wid = wid; - break; + if (VX_CFG_FETCH_WIDTH > 1) { + bool same_shape = true; + for (uint32_t d = 0; d < 3; ++d) { + same_shape = same_shape && (shape_step_[d] == cta_.warp_step[d]) + && (shape_dim_[d] == cta_.block_dim[d]); + } + if (!same_shape) { + for (uint32_t d = 0; d < 3; ++d) { + shape_step_[d] = cta_.warp_step[d]; + shape_dim_[d] = cta_.block_dim[d]; + } + init_wait_ = VX_CFG_FETCH_WIDTH; + } } } - if (free_wid < 0) return false; - // Skip the startup only if every lane this launch activates has already run it. - uint32_t active = (block_size_rem_ >= num_threads_) ? num_threads_ : block_size_rem_; - auto& inited = warp_init_lanes_[free_wid]; - bool needs_init = false; - for (uint32_t t = 0; t < active; ++t) { - if (!inited.test(t)) { - needs_init = true; - break; - } + if (init_wait_ != 0) { + --init_wait_; + return 0; } - if (!next_warp(needs_init, rec_out)) - return false; - - for (uint32_t t = 0; t < active; ++t) { - inited.set(t); + // Lanes with a free slot, round-robin from lane_rr_, up to the warps left; + // each launches its lowest free slot. + uint32_t count = 0; + for (uint32_t o = 0; o < VX_CFG_FETCH_WIDTH && has_cta_; ++o) { + uint32_t lane = (lane_rr_ + o) % VX_CFG_FETCH_WIDTH; + int free_wid = -1; + for (uint32_t wid = lane; wid < num_warps_; wid += VX_CFG_FETCH_WIDTH) { + if (!active_warps.test(wid)) { + free_wid = int(wid); + break; + } + } + if (free_wid < 0) { + continue; + } + // Skip the startup only if every lane this launch activates has already run it. + uint32_t active = (block_size_rem_ >= num_threads_) ? num_threads_ : block_size_rem_; + auto& inited = warp_init_lanes_[free_wid]; + bool needs_init = false; + for (uint32_t t = 0; t < active; ++t) { + if (!inited.test(t)) { + needs_init = true; + break; + } + } + if (!next_warp(needs_init, &recs[count])) { + break; + } + for (uint32_t t = 0; t < active; ++t) { + inited.set(t); + } + wid_to_slot_[free_wid] = cur_slot_; + wids[count] = uint32_t(free_wid); + ++count; } - - wid_to_slot_[free_wid] = cur_slot_; - *wid_out = uint32_t(free_wid); - return true; + lane_rr_ = (lane_rr_ + count) % VX_CFG_FETCH_WIDTH; + return count; } void CtaDispatcher::warp_done(uint32_t wid) { @@ -263,7 +297,9 @@ bool CtaDispatcher::next_warp(bool do_init, cta_warp_record_t* out) { out->cluster_size = cta_.cluster_dim[0] * cta_.cluster_dim[1] * cta_.cluster_dim[2]; // Thread mask: full warp or partial last warp. + // The record is reused across launches, so clear the lanes of the last one. out->tmask.resize(num_threads_); + out->tmask.reset(); uint32_t active = (block_size_rem_ >= num_threads_) ? num_threads_ : block_size_rem_; for (uint32_t t = 0; t < active; ++t) { out->tmask.set(t); diff --git a/sim/simx/cta_dispatcher.h b/sim/simx/cta_dispatcher.h index 74d266c2e3..2404f75055 100644 --- a/sim/simx/cta_dispatcher.h +++ b/sim/simx/cta_dispatcher.h @@ -40,8 +40,9 @@ struct cta_warp_record_t { uint32_t cluster_size; }; -// CTA dispatcher: consumes CTAs pushed by the KMU and admits one warp rank -// per step() into a free warp slot. +// CTA dispatcher: consumes CTAs pushed by the KMU and, per step(), launches up +// to one warp of the CTA on each launch lane, into a free warp slot of that +// lane's frontend slice. class CtaDispatcher : public SimObject { public: // Launch-lane endpoint (capacity 1): the CTA the KMU has committed to this @@ -55,9 +56,10 @@ class CtaDispatcher : public SimObject { CtaDispatcher(const SimContext& ctx, const char* name, Core* core); ~CtaDispatcher(); - // Try to dispatch one warp rank into a free warp slot. - // Returns true on success and sets *wid_out / *rec_out. - bool step(const WarpMask& active_warps, uint32_t* wid_out, cta_warp_record_t* rec_out); + // Dispatch warps into free warp slots, at most one per launch lane: lane l + // owns the slots with wid % VX_CFG_FETCH_WIDTH == l. Fills wids[] / recs[] in + // launch order and returns how many warps were dispatched. + uint32_t step(const WarpMask& active_warps, uint32_t* wids, cta_warp_record_t* recs); // Notify that a warp has exited; frees its lmem slot when all warps of the CTA are done. void warp_done(uint32_t wid); @@ -120,6 +122,14 @@ class CtaDispatcher : public SimObject { uint32_t thread_idx_[3]; uint64_t lmem_addr_; + // Launch lanes are taken round-robin from lane_rr_. Several lanes need the + // per-lane thread-index window rebuilt when the block shape changes, which + // holds the launch for one cycle per lane. + uint32_t lane_rr_; + uint32_t init_wait_; + uint32_t shape_step_[3]; + uint32_t shape_dim_[3]; + Word cur_kernel_pc_; uint8_t cur_ctx_id_; // Which lanes of each warp slot have run the startup stub. Per LANE, not per diff --git a/sim/simx/dxa/dxa_core.cpp b/sim/simx/dxa/dxa_core.cpp index 5f3c357644..abbaf45ced 100644 --- a/sim/simx/dxa/dxa_core.cpp +++ b/sim/simx/dxa/dxa_core.cpp @@ -27,6 +27,10 @@ using namespace vortex; +// response tags index the in-flight slots by masking +static_assert(VX_CFG_DXA_MAX_INFLIGHT >= 2 && (VX_CFG_DXA_MAX_INFLIGHT & (VX_CFG_DXA_MAX_INFLIGHT - 1)) == 0, + "DXA_MAX_INFLIGHT must be a power of two of at least 2"); + namespace { // One GMEM port: the DXA joins its socket's L2-facing arb as a single peer. diff --git a/sim/simx/dxa/dxa_unit.cpp b/sim/simx/dxa/dxa_unit.cpp index 42db10595f..e84f2ea0d2 100644 --- a/sim/simx/dxa/dxa_unit.cpp +++ b/sim/simx/dxa/dxa_unit.cpp @@ -62,7 +62,7 @@ instr_trace_t* DxaUnit::process(instr_trace_t* trace) { // completion; pre-registration happens explicitly per-CTA so multicast // destinations correctly wait. - req_out_.send(req); + req_out_.send(req, 1 + bus_delay_); DT(4, "dxa-unit submit: core=" << core_->id() << ", wid=" << trace->wid << ", slot=" << desc_slot << ", bar=" << raw_bar << ", cta_mask=0x" << std::hex << cta_mask << std::dec); diff --git a/sim/simx/dxa/dxa_unit.h b/sim/simx/dxa/dxa_unit.h index c076a14b53..04ab7c3435 100644 --- a/sim/simx/dxa/dxa_unit.h +++ b/sim/simx/dxa/dxa_unit.h @@ -43,8 +43,10 @@ struct DxaReq { class DxaUnit { public: // The outbound channel lives on SfuUnit (which is the SimObject). - DxaUnit(Core* core, SimChannel& req_out) - : core_(core), req_out_(req_out) {} + // bus_delay: extra cycles a request spends merging with other SFU blocks' + // requests onto the core's DXA bus. + DxaUnit(Core* core, SimChannel& req_out, uint32_t bus_delay = 0) + : core_(core), req_out_(req_out), bus_delay_(bus_delay) {} // Decode lanes 0..3 from the trace's source operands and try to push a // DxaReq on the SFU's outbound channel. Returns the trace on success @@ -55,6 +57,7 @@ class DxaUnit { private: Core* core_; SimChannel& req_out_; + uint32_t bus_delay_; }; } // namespace vortex diff --git a/sim/simx/lsu_unit.cpp b/sim/simx/lsu_unit.cpp index 78fd3fd466..2119b6d76f 100644 --- a/sim/simx/lsu_unit.cpp +++ b/sim/simx/lsu_unit.cpp @@ -176,7 +176,7 @@ void LsuUnit::compute_addrs(uint32_t b, instr_trace_t* trace) { // The datapath has no misalignment support: only a naturally aligned // access is guaranteed to sit inside one memory block, which is what // lets a lane's payload pack into a single block downstream. - // VX_lsu_slice.sv enforces the same contract in hardware, so modelling + // The hardware LSU enforces the same contract, so modelling // anything looser here would let a shader pass in simulation and fail // on the device. if ((e.addr % e.size) != 0) { diff --git a/sim/simx/rtu/rtu_unit.cpp b/sim/simx/rtu/rtu_unit.cpp index 2d8cef624f..4f8878c3ba 100644 --- a/sim/simx/rtu/rtu_unit.cpp +++ b/sim/simx/rtu/rtu_unit.cpp @@ -35,10 +35,11 @@ inline uint32_t float_to_bits(float f) { } } // namespace -RtuUnit::RtuUnit(Core* core, SimChannel& req_out, RtuWindow& window) +RtuUnit::RtuUnit(Core* core, SimChannel& req_out, RtuWindow& window, uint32_t bus_delay) : window_(window) , core_(core) , req_out_(req_out) + , bus_delay_(bus_delay) { trace_slot_.fill(-1); for (auto& s : trace_scene_) s.fill(0); @@ -252,7 +253,7 @@ instr_trace_t* RtuUnit::process_cb_ret(instr_trace_t* trace, uint32_t block_id) trace->dst_data[t].u = 0; // no writeback } req.tmask_bits = bits; - req_out_.send(req); + req_out_.send(req, 1 + bus_delay_); DT(3, "rtu-cb_ret submit: core=" << core_->id() << ", wid=" << trace->wid << ", tmask=0x" << std::hex << bits << std::dec); return trace; @@ -527,7 +528,7 @@ instr_trace_t* RtuUnit::process_trace_uop(instr_trace_t* trace, uint32_t block_i req.cull_mask[t] = trace_cull_.at(wid); } req.tmask_bits = bits; - req_out_.send(req); + req_out_.send(req, 1 + bus_delay_); trace_slot_.at(wid) = -1; DT(3, "rtu-trace arm: core=" << core_->id() << ", wid=" << wid << ", slot=" << slot << ", tmask=0x" << std::hex << bits << std::dec); diff --git a/sim/simx/rtu/rtu_unit.h b/sim/simx/rtu/rtu_unit.h index 978e7ea511..a1ad4bdc4f 100644 --- a/sim/simx/rtu/rtu_unit.h +++ b/sim/simx/rtu/rtu_unit.h @@ -69,7 +69,9 @@ class RtuUopGen { // register file. Plain (non-SimObject) helper owned by SfuUnit. class RtuUnit { public: - RtuUnit(Core* core, SimChannel& req_out, RtuWindow& window); + // bus_delay: extra cycles a request spends merging with other SFU blocks' + // requests onto the core's RTU bus. + RtuUnit(Core* core, SimChannel& req_out, RtuWindow& window, uint32_t bus_delay = 0); // Async ray pool. process_wait either: // - returns the trace with the per-lane status word written @@ -187,6 +189,7 @@ class RtuUnit { Core* core_; SimChannel& req_out_; + uint32_t bus_delay_; // Async ray pool. Borrowed from the Socket via set_rtu_core(); // null until the Socket has wired it (TRACE/WAIT paths must NEVER // dereference rtu_core_ before that — but in practice the Socket diff --git a/sim/simx/scheduler.cpp b/sim/simx/scheduler.cpp index 551adf6976..793b1c4074 100644 --- a/sim/simx/scheduler.cpp +++ b/sim/simx/scheduler.cpp @@ -89,7 +89,7 @@ void Scheduler::on_reset() { stalled_warps_next_.reset(); active_warps_.reset(); cta_active_view_.reset(); - cta_fire_ = false; + cta_fire_cnt_ = 0; // Sequencers live on Core now; Core::on_reset() resets them. wspawn_.valid = false; @@ -155,26 +155,15 @@ void Scheduler::activate_warp(uint32_t wid, const cta_warp_record_t& rec) { << ", mscratch=0x" << std::hex << warp.mscratch << std::dec); } -instr_trace_t* Scheduler::schedule(const WarpMask& warp_mask) { - int scheduled_warp = -1; - - // Dispatch one CTA warp. The dispatcher picks from the registered active - // set and its pick lands the next cycle; the warp it activates becomes - // schedulable the cycle after that. - if (cta_fire_) { - activate_warp(cta_fire_wid_, cta_fire_rec_); - stalled_warps_.set(cta_fire_wid_); - cta_fire_ = false; - } - { - uint32_t wid; - cta_warp_record_t rec; - if (cta_dispatcher_->step(cta_active_view_, &wid, &rec)) { - cta_fire_ = true; - cta_fire_wid_ = wid; - cta_fire_rec_ = rec; - } +void Scheduler::schedule_begin() { + // Dispatch CTA warps, one per launch lane. The dispatcher picks from the + // registered active set and its picks land the next cycle; a warp it + // activates becomes schedulable the cycle after that. + for (uint32_t i = 0; i < cta_fire_cnt_; ++i) { + activate_warp(cta_fire_wids_[i], cta_fire_recs_[i]); + stalled_warps_.set(cta_fire_wids_[i]); } + cta_fire_cnt_ = cta_dispatcher_->step(cta_active_view_, cta_fire_wids_.data(), cta_fire_recs_.data()); #ifdef VX_CFG_EXT_RASTER_ENABLE // Inject ready fragment waves into free warp slots. @@ -200,6 +189,14 @@ instr_trace_t* Scheduler::schedule(const WarpMask& warp_mask) { wspawn_.valid = false; this->resume(0); } +} + +bool Scheduler::has_ready(const WarpMask& warp_mask) const { + return (active_warps_ & ~stalled_warps_ & warp_mask).any(); +} + +instr_trace_t* Scheduler::schedule(const WarpMask& warp_mask) { + int scheduled_warp = -1; // pick next ready warp for (size_t wid = 0, nw = VX_CFG_NUM_WARPS; wid < nw; ++wid) { @@ -244,20 +241,23 @@ instr_trace_t* Scheduler::schedule(const WarpMask& warp_mask) { this->suspend(scheduled_warp); } + return trace; +} + +void Scheduler::schedule_end() { // Clock the registered warp-stall state. The suspend above, or a resume from // decode/commit/FU earlier this cycle, drives stalled_warps_next_ and only // becomes visible to the pick loop next cycle — so a warp released as its // instruction resolves is never re-scheduled the same cycle. stalled_warps_ = stalled_warps_next_; cta_active_view_ = active_warps_; - if (cta_fire_) { - cta_active_view_.set(cta_fire_wid_); + for (uint32_t i = 0; i < cta_fire_cnt_; ++i) { + cta_active_view_.set(cta_fire_wids_[i]); } - return trace; } bool Scheduler::running() const { - return active_warps_.any() || cta_fire_ || cta_dispatcher_->running() + return active_warps_.any() || (cta_fire_cnt_ != 0) || cta_dispatcher_->running() #ifdef VX_CFG_EXT_RASTER_ENABLE || fwd_armed_ #endif @@ -417,7 +417,11 @@ void Scheduler::fwd_try_inject() { // Find any free warp slot — there is no driver warp to skip in the push model. int wid = -1; for (uint32_t w = 0; w < VX_CFG_NUM_WARPS; ++w) { - if (!active_warps_.test(w) && !(cta_fire_ && w == cta_fire_wid_)) { wid = int(w); break; } + bool firing = false; + for (uint32_t i = 0; i < cta_fire_cnt_; ++i) { + firing = firing || (cta_fire_wids_[i] == w); + } + if (!active_warps_.test(w) && !firing) { wid = int(w); break; } } if (wid < 0) break; // no free slot this cycle diff --git a/sim/simx/scheduler.h b/sim/simx/scheduler.h index a397eb60ba..58f10378b0 100644 --- a/sim/simx/scheduler.h +++ b/sim/simx/scheduler.h @@ -118,7 +118,18 @@ class Scheduler : public SimObject { ~Scheduler(); // ----- Warp lifecycle ----- + // Once per cycle, before the fetch slices pick: activate the warps the CTA + // dispatcher launched last cycle, launch the next ones, and apply a wspawn. + void schedule_begin(); + + // One fetch slice's pick: its lowest ready warp in warp_mask. instr_trace_t* schedule(const WarpMask& warp_mask); + + // Whether warp_mask holds a ready warp, without picking it. + bool has_ready(const WarpMask& warp_mask) const; + + // Once per cycle, after the picks: clock the registered warp state. + void schedule_end(); void suspend(uint32_t wid); void resume(uint32_t wid); // Advance the warp's PC by `inc` bytes (called at decode with 2 or 4 @@ -211,12 +222,12 @@ class Scheduler : public SimObject { WarpMask active_warps_; WarpMask stalled_warps_; // registered (current) state read by schedule() WarpMask stalled_warps_next_; // next-state written by suspend()/resume() - // CTA dispatch pipeline: the active set the dispatcher sees, and the warp - // it picked last cycle. + // the active set the CTA dispatcher sees WarpMask cta_active_view_; - bool cta_fire_ = false; - uint32_t cta_fire_wid_ = 0; - cta_warp_record_t cta_fire_rec_; + // Warps the CTA dispatcher launched this cycle, one per launch lane at most. + uint32_t cta_fire_cnt_ = 0; + std::array cta_fire_wids_; + std::array cta_fire_recs_; uint32_t ipdom_size_; wspawn_t wspawn_; uint32_t mpm_class_; diff --git a/sim/simx/sfu_unit.cpp b/sim/simx/sfu_unit.cpp index de61756629..edf9311c5f 100644 --- a/sim/simx/sfu_unit.cpp +++ b/sim/simx/sfu_unit.cpp @@ -30,6 +30,10 @@ using namespace vortex; +// With several SFU blocks, their TEX/RTU/DXA requests merge onto the core's bus +// through one registered arbiter stage. +static constexpr uint32_t kBusMergeDelay = (VX_CFG_NUM_SFU_BLOCKS > 1) ? 1 : 0; + SfuUnit::SfuUnit(const SimContext& ctx, const char* name, Core* core) : FuncUnit(ctx, name, core, 6) , warp_ctl_out(this, VX_CFG_NUM_WARPS) @@ -54,16 +58,16 @@ SfuUnit::SfuUnit(const SimContext& ctx, const char* name, Core* core) , wctl_unit_(new WctlUnit(core)) , csr_unit_(new CsrUnit(core)) #ifdef VX_CFG_EXT_DXA_ENABLE - , dxa_unit_(new DxaUnit(core, dxa_req_out)) + , dxa_unit_(new DxaUnit(core, dxa_req_out, kBusMergeDelay)) #endif #ifdef VX_CFG_EXT_TEX_ENABLE - , tex_unit_(new TexUnit(core, tex_req_out)) + , tex_unit_(new TexUnit(core, tex_req_out, kBusMergeDelay)) #endif #ifdef VX_CFG_EXT_OM_ENABLE , om_unit_(new OmUnit(core, om_req_out)) #endif #ifdef VX_CFG_EXT_RTU_ENABLE - , rtu_unit_(new RtuUnit(core, rtu_req_out, rtu_window_)) + , rtu_unit_(new RtuUnit(core, rtu_req_out, rtu_window_, kBusMergeDelay)) #endif { } @@ -256,12 +260,36 @@ void SfuUnit::on_tick() { } #endif + // Grant each shared resource to one requesting block this cycle. + SharedRes want[VX_CFG_NUM_SFU_BLOCKS]; + int winner[RES_COUNT]; + instr_trace_t* winner_head[RES_COUNT] = {}; + for (uint32_t r = 0; r < RES_COUNT; ++r) { + winner[r] = -1; + } + for (uint32_t b = 0; b < VX_CFG_NUM_SFU_BLOCKS; ++b) { + want[b] = this->shared_res(b); + } + for (uint32_t r = RES_WCTL; r < RES_COUNT; ++r) { + for (uint32_t i = 0; i < VX_CFG_NUM_SFU_BLOCKS; ++i) { + uint32_t b = (res_rr_[r] + i) % VX_CFG_NUM_SFU_BLOCKS; + if (want[b] == r) { + winner[r] = b; + winner_head[r] = Inputs.at(b).peek(); + break; + } + } + } + // PE switch: peek input, route to the matching sub-unit (WCTL / CSR / // DXA / TEX / OM / RASTER) by op_type, gather to the single result port. for (uint32_t b = 0; b < VX_CFG_NUM_SFU_BLOCKS; ++b) { auto& input = Inputs.at(b); if (input.empty()) continue; + if (want[b] != RES_NONE && winner[want[b]] != int(b)) { + continue; // another block holds the shared resource this cycle + } auto& output = Outputs.at(b); auto trace = input.peek(); @@ -441,4 +469,53 @@ void SfuUnit::on_tick() { input.pop(); } + + // A granted request that went through moves its resource's pointer past it. + for (uint32_t r = RES_WCTL; r < RES_COUNT; ++r) { + if (winner[r] < 0) { + continue; + } + auto& input = Inputs.at(winner[r]); + if (input.empty() || input.peek() != winner_head[r]) { + res_rr_[r] = (winner[r] + 1) % VX_CFG_NUM_SFU_BLOCKS; + } + } +} + +SfuUnit::SharedRes SfuUnit::shared_res(uint32_t block) { + auto& input = Inputs.at(block); + if (input.empty()) { + return RES_NONE; + } + auto trace = input.peek(); +#ifdef VX_CFG_EXT_TEX_ENABLE + if (std::get_if(&trace->op_type)) { + return RES_TEX; + } +#endif +#ifdef VX_CFG_EXT_RTU_ENABLE + if (auto rtu_p = std::get_if(&trace->op_type)) { + if (*rtu_p == GfxwType::TRACE || *rtu_p == GfxwType::CB_RET) { + return RES_RTU; + } + } +#endif +#ifdef VX_CFG_EXT_DXA_ENABLE + if (std::get_if(&trace->op_type)) { + return RES_DXA; + } +#endif + if (auto wctl_p = std::get_if(&trace->op_type)) { + // a block requests only when its op could fire this cycle + if (Outputs.at(block).full()) { + return RES_NONE; + } + if (*wctl_p == WctlType::BAR && trace->eop && !core_->lsu_drained()) { + return RES_NONE; + } + if (wctl_unit_->is_shared(trace)) { + return RES_WCTL; + } + } + return RES_NONE; } diff --git a/sim/simx/sfu_unit.h b/sim/simx/sfu_unit.h index 1eb74a31f1..ffc26c22b0 100644 --- a/sim/simx/sfu_unit.h +++ b/sim/simx/sfu_unit.h @@ -123,6 +123,13 @@ class SfuUnit : public FuncUnit { private: uint32_t latency_of(const instr_trace_t* trace) const; + // Core-wide resources the SFU blocks share: the warp-control resources + // (barrier unit, wspawn, warp retirement) and the TEX/RTU/DXA buses each + // accept one request per cycle, granted round-robin per resource. + enum SharedRes { RES_NONE = 0, RES_WCTL, RES_TEX, RES_RTU, RES_DXA, RES_COUNT }; + SharedRes shared_res(uint32_t block); + uint32_t res_rr_[RES_COUNT] = {}; + std::unique_ptr wctl_unit_; std::unique_ptr csr_unit_; #ifdef VX_CFG_EXT_RTU_ENABLE diff --git a/sim/simx/socket.cpp b/sim/simx/socket.cpp index 5a1861155a..536dfe4d1e 100644 --- a/sim/simx/socket.cpp +++ b/sim/simx/socket.cpp @@ -60,9 +60,9 @@ class Socket::Impl { log2ceil(VX_CFG_L1_LINE_SIZE), // S (no sectoring) log2ceil(sizeof(uint32_t)), // W log2ceil(VX_CFG_ICACHE_NUM_WAYS),// A - log2ceil(1), // B + log2ceil(VX_CFG_ICACHE_NUM_BANKS), // B VX_CFG_XLEN, // address bits - 1, // number of inputs + ICACHE_NUM_REQS, // number of inputs (one per fetch slice) VX_CFG_ICACHE_MEM_PORTS, // memory ports false, // write-back false, // write response @@ -276,8 +276,10 @@ class Socket::Impl { // connect cores to caches for (uint32_t i = 0; i < cores_per_socket; ++i) { - cores_.at(i)->icache_req_out.at(0).bind(&icaches_->core_req_in.at(i).at(0)); - icaches_->core_rsp_out.at(i).at(0).bind(&cores_.at(i)->icache_rsp_in.at(0)); + for (uint32_t j = 0; j < ICACHE_NUM_REQS; ++j) { + cores_.at(i)->icache_req_out.at(j).bind(&icaches_->core_req_in.at(i).at(j)); + icaches_->core_rsp_out.at(i).at(j).bind(&cores_.at(i)->icache_rsp_in.at(j)); + } for (uint32_t j = 0; j < VX_CFG_DCACHE_NUM_REQS; ++j) { cores_.at(i)->dcache_req_out.at(j).bind(&dcaches_->core_req_in.at(i).at(j)); diff --git a/sim/simx/tex/tex_unit.cpp b/sim/simx/tex/tex_unit.cpp index fd88c0731b..462a8f924f 100644 --- a/sim/simx/tex/tex_unit.cpp +++ b/sim/simx/tex/tex_unit.cpp @@ -49,7 +49,7 @@ instr_trace_t* TexUnit::process(instr_trace_t* trace, uint32_t block_id) { // Combinational into the shared arbiter; the arbiter's forward is the // registered boundary. - req_out_.send(req, 0); + req_out_.send(req, bus_delay_); DT(3, "tex-unit submit: core=" << core_->id() << ", wid=" << trace->wid << ", stage=" << args.stage); return trace; diff --git a/sim/simx/tex/tex_unit.h b/sim/simx/tex/tex_unit.h index 710c011701..65b1319eb5 100644 --- a/sim/simx/tex/tex_unit.h +++ b/sim/simx/tex/tex_unit.h @@ -84,8 +84,10 @@ class TexCore; // backpressure (SfuUnit retries idempotently next cycle). class TexUnit { public: - TexUnit(Core* core, SimChannel& req_out) - : core_(core), req_out_(req_out) {} + // bus_delay: extra cycles a request spends merging with other SFU blocks' + // requests onto the core's texture bus. + TexUnit(Core* core, SimChannel& req_out, uint32_t bus_delay = 0) + : core_(core), req_out_(req_out), bus_delay_(bus_delay) {} // Submit one sample request: u/v/lod ride src_data[0..2], the texel returns in // rd. Returns the trace if accepted, or nullptr on full output channel. @@ -94,6 +96,7 @@ class TexUnit { private: Core* core_; SimChannel& req_out_; + uint32_t bus_delay_; }; } // namespace vortex diff --git a/sim/simx/wctl_unit.cpp b/sim/simx/wctl_unit.cpp index 4fbe8a531e..e989360acc 100644 --- a/sim/simx/wctl_unit.cpp +++ b/sim/simx/wctl_unit.cpp @@ -20,6 +20,43 @@ using namespace vortex; +bool WctlUnit::is_shared(const instr_trace_t* trace) const { + if (!trace->eop) { + return false; + } + auto wctl_type = std::get(trace->op_type); + if (wctl_type == WctlType::BAR || wctl_type == WctlType::WSPAWN) { + return true; + } + if (wctl_type != WctlType::TMC && wctl_type != WctlType::PRED) { + return false; + } + auto& warp = core_->scheduler().warp(trace->wid); + uint32_t num_threads = VX_CFG_NUM_THREADS; + int32_t thread_last = num_threads - 1; + for (; thread_last >= 0; --thread_last) { + if (warp.tmask.test(thread_last)) { break; } + } + auto& rs1_data = trace->src_data[0]; + auto& rs2_data = trace->src_data[1]; + ThreadMask next_tmask(num_threads); + if (wctl_type == WctlType::TMC) { + for (uint32_t t = 0; t < num_threads; ++t) { + next_tmask.set(t, rs1_data.at(thread_last).u & (1 << t)); + } + } else { + auto wctlArgs = std::get(trace->instr_ptr->get_args()); + auto not_pred = wctlArgs.is_cond_neg; + ThreadMask pred(num_threads); + for (uint32_t t = 0; t < num_threads; ++t) { + auto cond = (rs1_data.at(t).i & 0x1) ^ not_pred; + pred[t] = warp.tmask.test(t) && cond; + } + next_tmask = pred.any() ? (warp.tmask & pred) : ThreadMask(num_threads, rs2_data.at(thread_last).u); + } + return next_tmask.none(); +} + bool WctlUnit::process(instr_trace_t* trace, bool* exit) { bool release_warp = trace->fetch_stall; *exit = false; diff --git a/sim/simx/wctl_unit.h b/sim/simx/wctl_unit.h index 799c34647e..e688a59abd 100644 --- a/sim/simx/wctl_unit.h +++ b/sim/simx/wctl_unit.h @@ -34,6 +34,11 @@ class WctlUnit { // the latency. bool process(instr_trace_t* trace, bool* exit); + // Whether `trace` needs a core-wide warp-control resource (the barrier unit, + // wspawn, or warp retirement), which takes one request per cycle across all + // SFU blocks. No side effects. + bool is_shared(const instr_trace_t* trace) const; + private: Core* core_; }; From 285e94684974675a67fbb089e302c07f1f0c1725 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 18 Oct 2026 14:46:11 -0700 Subject: [PATCH 037/124] simx: model the RTL's buffer structure at every pipeline boundary SimX matched the RTL's uncontended latencies but not its structure: buffers had different capacities and ready timing, so under contention the two models waited at different points, and several end-to-end latencies matched only because errors on the request and response sides cancelled. A cycle-level trace diff of both models on targeted microbenchmarks located each difference; each change below matches the RTL element it models. SimX: - SimChannel gains an opt-in registered ready: a slot freed by a pop is offered to producers from the next cycle, independent of tick order. - Operand collector: an instruction holds its input slot until its last operand read (the 2-entry skid buffer after the scoreboard), with registered ready; bank-conflict cost is max-per-bank - 1. - Issue grants only when the collector input has room. - Arbiter copies get their own state. std::vector copies shared one implementation, so every issue slice shared grant state. - SFU: CSR result one cycle later (the hold was counted twice); registered ready on the block inputs; a barrier arrive is non-posted (request after the LSU drains, applied two cycles later, completes the cycle after) and frees the shared warp-control port once sent. - Barrier releases, local and global, go through one registered unlock path, as the RTL's unlock_valid/unlock_mask do. - LSU drained() includes pending requests, as the RTL's barrier drain check does; stores commit at queue accept. - dcache: a port arbiter grants one core port per bank per cycle with a round-robin pointer per bank (the RTL core crossbar), only when the bank's input buffer has room; the non-cacheable bypass layer adds its response register on the dcache, L2 and L3 (NC_ENABLE in the RTL). - icache port 0 carries the flush register in front of the cache. - Local memory: the request and response crossbars become one cycle deeper above 1.5x the fan-out limit (the RTL's arbiter tree). - CTA dispatch: the launch pipeline's init delay. - DXA: SMEM write timing per layout and the engine's request staging. - Functional builds (-DSIMX_FUNCTIONAL) accept the registered-ready option and ignore it. RTL: - VX_dxa_gmem_req buffers memory responses per slot, so one slow response no longer blocks the others (head-of-line). It holds one entry per busy slot, so it never overflows. Testbench: - rtlsim completes DRAM requests in per-bank arrival order, as the AXI platform does. Configuration: - NUM_SFU_BLOCKS defaults to 1, as before the split. With one SFU block the stage-sliced back end is selected for ISSUE_WIDTH > 1, so the CI cases that cover the compute-slice back end set the SFU block count explicitly. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/dxa/VX_dxa_gmem_req.sv | 50 +++++++++++- sim/common/simobject.h | 35 ++++++++- sim/common/simobject_functional.h | 4 + sim/rtlsim/processor.cpp | 40 ++++++---- sim/simx/barrier_unit.cpp | 33 +++++--- sim/simx/barrier_unit.h | 9 ++- sim/simx/cluster.cpp | 1 + sim/simx/core.cpp | 74 +++++++++++++----- sim/simx/core.h | 3 +- sim/simx/cta_dispatcher.cpp | 18 ++++- sim/simx/cta_dispatcher.h | 2 + sim/simx/dxa/dxa_core.cpp | 44 +++++++++-- sim/simx/lsu_unit.cpp | 30 ++++++-- sim/simx/mem/cache.cpp | 123 +++++++++++++++++++++++++----- sim/simx/mem/cache.h | 1 + sim/simx/mem/local_mem.cpp | 12 ++- sim/simx/opc_unit.cpp | 38 +++++---- sim/simx/opc_unit.h | 2 + sim/simx/processor.cpp | 1 + sim/simx/sfu_unit.cpp | 77 +++++++++++++++++-- sim/simx/sfu_unit.h | 12 +++ sim/simx/socket.cpp | 1 + sim/simx/types.h | 19 ++++- sim/simx/wctl_unit.cpp | 75 ++++++++---------- sim/simx/wctl_unit.h | 3 + 25 files changed, 556 insertions(+), 151 deletions(-) diff --git a/hw/rtl/dxa/VX_dxa_gmem_req.sv b/hw/rtl/dxa/VX_dxa_gmem_req.sv index 43e467ab46..929b02c3e7 100644 --- a/hw/rtl/dxa/VX_dxa_gmem_req.sv +++ b/hw/rtl/dxa/VX_dxa_gmem_req.sv @@ -15,9 +15,9 @@ // // Manages a slot-table-indexed pool of MAX_OUTSTANDING in-flight tags. On // each accept, allocates a free slot and writes per-tag metadata. GMEM -// responses (out-of-order from the L1) are streamed directly to smem_wr -// via the `sw_*` channel — no rsp_buf BRAM. OOB entries skip the bus and -// are presented to smem_wr via an oob_pending[] bitvector. +// responses (out-of-order from the L1) queue in a response buffer with one +// entry per slot and drain to smem_wr via the `sw_*` channel. OOB entries +// skip the bus and are presented to smem_wr via an oob_pending[] bitvector. // // Tag reuse is per-slot via a busy[] bitvector and outstanding counter; // release order is independent of issue order (OOO drain in smem_wr). @@ -242,6 +242,48 @@ module VX_dxa_gmem_req import VX_gpu_pkg::*, VX_dxa_pkg::*; #( end end + VX_mem_bus_if #( + .DATA_SIZE (GMEM_BYTES), + .TAG_WIDTH (GMEM_TAG_WIDTH) + ) mem_bus_raw (); + + assign mem_bus_raw.req_valid = mem_bus_w.req_valid; + assign mem_bus_raw.req_data = mem_bus_w.req_data; + assign mem_bus_w.req_ready = mem_bus_raw.req_ready; + + // smem_wr takes a scattered line one element per cycle. The memory + // response path is shared with the L1 fills, so a response must never + // wait on it: one entry per slot means the buffer is never full. A + // response passes straight through when the buffer is empty and smem_wr + // is ready. + wire rsp_buf_empty, rsp_buf_full; + wire [GMEM_DATAW + GMEM_TAG_WIDTH-1:0] rsp_buf_data; + wire rsp_bypass = rsp_buf_empty && mem_bus_w.rsp_ready; + + VX_fifo_queue #( + .DATAW (GMEM_DATAW + GMEM_TAG_WIDTH), + .DEPTH (MAX_OUTSTANDING) + ) rsp_buf ( + .clk (clk), + .reset (reset), + .push (mem_bus_raw.rsp_valid && ~rsp_bypass), + .pop (~rsp_buf_empty && mem_bus_w.rsp_ready), + .data_in (mem_bus_raw.rsp_data), + .data_out (rsp_buf_data), + .empty (rsp_buf_empty), + .full (rsp_buf_full), + `UNUSED_PIN (alm_empty), + `UNUSED_PIN (alm_full), + `UNUSED_PIN (size) + ); + + `RUNTIME_ASSERT(~(mem_bus_raw.rsp_valid && ~rsp_bypass && rsp_buf_full), + ("%t: GMEM response buffer overflow", $time)) + + assign mem_bus_raw.rsp_ready = 1'b1; + assign mem_bus_w.rsp_valid = mem_bus_raw.rsp_valid || ~rsp_buf_empty; + assign mem_bus_w.rsp_data = rsp_buf_empty ? mem_bus_raw.rsp_data : rsp_buf_data; + VX_mem_bus_slice #( .DATA_SIZE (GMEM_BYTES), .TAG_WIDTH (GMEM_TAG_WIDTH), @@ -250,7 +292,7 @@ module VX_dxa_gmem_req import VX_gpu_pkg::*, VX_dxa_pkg::*; #( ) gmem_out_slice ( .clk (clk), .reset (reset), - .bus_in_if (mem_bus_w), + .bus_in_if (mem_bus_raw), .bus_out_if (gmem_bus_if) ); diff --git a/sim/common/simobject.h b/sim/common/simobject.h index b3c9879dc8..225dab157a 100644 --- a/sim/common/simobject.h +++ b/sim/common/simobject.h @@ -245,6 +245,10 @@ class SimPlatform { return (tl_exec_domain_ == HOST_DOMAIN) ? cycles_ : tl_cycles_; } + // Counts resets: cycle numbers repeat across runs, so per-cycle state is + // tagged with the run it belongs to. + uint64_t reset_gen() const { return reset_gen_; } + // Sends issued outside any object tick (driver/host calls). static constexpr uint32_t HOST_DOMAIN = 0xffffffffu; @@ -387,6 +391,7 @@ class SimPlatform { std::vector> domains_; uint64_t cycles_; + uint64_t reset_gen_ = 0; uint32_t num_domains_ = 1; uint32_t build_domain_ = 0; @@ -651,6 +656,7 @@ class SimChannel : public SimChannelBase { : SimChannelBase(other.module_) , storage_(other.storage_.capacity()) , pending_count_(0) + , reg_ready_(other.reg_ready_) , convert_fn_(other.convert_fn_) , tx_cb_(other.tx_cb_) { sink_ = nullptr; @@ -666,6 +672,9 @@ class SimChannel : public SimChannelBase { pop_cb_ = {}; storage_ = RingQueue(other.storage_.capacity()); pending_count_ = 0; + reg_ready_ = other.reg_ready_; + freed_cycle_ = ~uint64_t(0); + freed_count_ = 0; convert_fn_ = other.convert_fn_; tx_cb_ = other.tx_cb_; } @@ -704,9 +713,14 @@ class SimChannel : public SimChannelBase { return sink_->full(); } assert(!SimPlatform::instance().mt_cross_access(module_)); - return this->occupancy() >= storage_.capacity(); + return this->occupancy() + this->freed_now() >= storage_.capacity(); } + // A registered ready: a slot freed by a pop is offered to producers from the + // next cycle, as a buffer whose full flag is a register does, whatever order + // the producer and the consumer tick in. + void set_registered_ready(bool enable) { reg_ready_ = enable; } + void send(const Pkt& pkt, uint64_t delay = 1) { auto& platform = SimPlatform::instance(); // A send whose endpoint lives in another domain is handed to that @@ -849,7 +863,21 @@ class SimChannel : public SimChannelBase { bool queue_empty() const { return storage_.empty(); } uint32_t queue_size() const { return storage_.size(); } const Pkt& queue_front() const { return storage_.front(); } + uint32_t freed_now() const { + auto& platform = SimPlatform::instance(); + return (reg_ready_ && freed_cycle_ == platform.cycles() && freed_gen_ == platform.reset_gen()) + ? freed_count_ : 0; + } void queue_pop() { + if (reg_ready_) { + auto& platform = SimPlatform::instance(); + if (freed_cycle_ != platform.cycles() || freed_gen_ != platform.reset_gen()) { + freed_cycle_ = platform.cycles(); + freed_gen_ = platform.reset_gen(); + freed_count_ = 0; + } + ++freed_count_; + } storage_.pop(); --SimChannelBase::inflight_count(); if (pop_cb_) { @@ -860,6 +888,10 @@ class SimChannel : public SimChannelBase { RingQueue storage_; uint32_t pending_count_; + bool reg_ready_ = false; + uint64_t freed_cycle_ = ~uint64_t(0); + uint64_t freed_gen_ = 0; + uint32_t freed_count_ = 0; SmallFunction convert_fn_; TxCallback tx_cb_; @@ -1198,6 +1230,7 @@ inline void SimPlatform::reset() { // Reset timing cycles_ = 0; + ++reset_gen_; } inline void SimPlatform::tick() { diff --git a/sim/common/simobject_functional.h b/sim/common/simobject_functional.h index 8074a179a1..924f09ba56 100644 --- a/sim/common/simobject_functional.h +++ b/sim/common/simobject_functional.h @@ -717,6 +717,10 @@ class SimChannel : public SimChannelBase { template void tx_callback(F&& callback) { tx_cb_ = std::forward(callback); } + // Ready timing is a timing-model property; accepted for API compatibility + // and ignored. + void set_registered_ready(bool /*enable*/) {} + // Back-pressure is disabled: producers never stall. bool full() const override { return false; } diff --git a/sim/rtlsim/processor.cpp b/sim/rtlsim/processor.cpp index 98e1eaa40d..eaef68cc0e 100644 --- a/sim/rtlsim/processor.cpp +++ b/sim/rtlsim/processor.cpp @@ -74,6 +74,7 @@ typedef uint64_t Word; using namespace vortex; +static uint64_t g_mem_done_seq = 0; static uint32_t g_mem_bank_addr_width = (VX_CFG_PLATFORM_MEMORY_ADDR_WIDTH - log2ceil(VX_CFG_PLATFORM_MEMORY_NUM_BANKS)); static uint64_t timestamp = 0; @@ -315,6 +316,7 @@ class Processor::Impl { // mark completed request as ready auto orig_req = reinterpret_cast(arg); orig_req->ready = true; + orig_req->done_seq = g_mem_done_seq++; return true; }, mem_req); dram_queue_[b].pop(); @@ -361,22 +363,29 @@ class Processor::Impl { if (device_->mem_rsp_valid[b] && mem_rd_rsp_ready_[b]) { device_->mem_rsp_valid[b] = 0; } - if (device_->mem_rsp_valid[b] == 0) { - if (!pending_mem_reqs_[b].empty()) { - auto mem_rsp_it = pending_mem_reqs_[b].begin(); - auto mem_rsp = *mem_rsp_it; - if (mem_rsp->ready) { - if (!mem_rsp->write) { - // return read responses - device_->mem_rsp_valid[b] = 1; - memcpy(VDataCast::get(device_->mem_rsp_data[b]), mem_rsp->data.data(), VX_CFG_PLATFORM_MEMORY_DATA_SIZE); - device_->mem_rsp_tag[b] = mem_rsp->tag; - } - // delete the request - pending_mem_reqs_[b].erase(mem_rsp_it); - delete mem_rsp; - } + // Reads carry distinct AXI IDs, so the memory returns them in the order + // they complete; writes are posted and never hold back a read. + auto& pending = pending_mem_reqs_[b]; + auto next_rd = pending.end(); + for (auto it = pending.begin(); it != pending.end();) { + auto mem_req = *it; + if (mem_req->ready && mem_req->write) { + it = pending.erase(it); + delete mem_req; + continue; } + if (mem_req->ready && (next_rd == pending.end() || mem_req->done_seq < (*next_rd)->done_seq)) { + next_rd = it; + } + ++it; + } + if (next_rd != pending.end() && !device_->mem_rsp_valid[b]) { + auto mem_rsp = *next_rd; + device_->mem_rsp_valid[b] = 1; + memcpy(VDataCast::get(device_->mem_rsp_data[b]), mem_rsp->data.data(), VX_CFG_PLATFORM_MEMORY_DATA_SIZE); + device_->mem_rsp_tag[b] = mem_rsp->tag; + pending.erase(next_rd); + delete mem_rsp; } // process memory requests @@ -454,6 +463,7 @@ class Processor::Impl { uint64_t tag; bool write; bool ready; + uint64_t done_seq; } mem_req_t; std::unordered_map print_bufs_; diff --git a/sim/simx/barrier_unit.cpp b/sim/simx/barrier_unit.cpp index 92349d7add..6538afc2a7 100644 --- a/sim/simx/barrier_unit.cpp +++ b/sim/simx/barrier_unit.cpp @@ -21,8 +21,17 @@ using namespace vortex; +// Cycles from an SFU barrier op to its effect in the barrier store: the store +// is read a cycle before an arrival applies, and with several SFU blocks the +// request is also registered out of its block. A transaction-barrier +// completion is registered once before it applies; a global-barrier release +// applies as it arrives. A release then reaches the warp scheduler registered. +static constexpr uint32_t kOpApplyLag = (VX_CFG_NUM_SFU_BLOCKS > 1) ? 2 : 1; +static constexpr uint32_t kEventApplyLag = 1; + BarrierUnit::BarrierUnit(const SimContext& ctx, const char* name, Core* core, Scheduler* scheduler) : SimObject(ctx, name) + , unlock_out(this, VX_CFG_NUM_WARPS) , gbar_send_next_(0) , core_(core) , scheduler_(scheduler) @@ -34,13 +43,17 @@ BarrierUnit::~BarrierUnit() {} // One request leaves the queue per cycle, so a completion that lands while the // queue is still draining is delivered on the next free cycle rather than // immediately. Mirrors the RTL request FIFO's drain rate. -void BarrierUnit::gbar_send(uint32_t bar_id, uint32_t count) { +void BarrierUnit::gbar_send(uint32_t bar_id, uint32_t count, uint32_t lag) { uint64_t now = SimPlatform::instance().cycles(); - uint64_t due = std::max(now + 1, gbar_send_next_); + uint64_t due = std::max(now + lag + 1, gbar_send_next_); gbar_send_next_ = due + 1; core_->gbar_arrive_out.send({bar_id, count, core_->id()}, due - now); } +void BarrierUnit::release(uint32_t wid, uint32_t lag) { + unlock_out.send(wid, lag + 1); +} + void BarrierUnit::on_reset() { gbar_send_next_ = 0; for (auto& barrier : barriers_) { @@ -74,7 +87,7 @@ void BarrierUnit::arrive(uint32_t bar_id, uint32_t count, uint32_t wid, bool is_ if (barrier.arrival_mask.count() == active_warps.count() && barrier.events == 0) { // wait_mask is preserved across the async hop (holds warps resumed by // global_resume) and cleared there. arrival_mask is local-only and reset here. - this->gbar_send(bar_id, count); + this->gbar_send(bar_id, count, kOpApplyLag); barrier.arrival_mask.reset(); barrier.count = 0; barrier.events = 0; @@ -90,7 +103,7 @@ void BarrierUnit::arrive(uint32_t bar_id, uint32_t count, uint32_t wid, bool is_ for (uint32_t i = 0; i < VX_CFG_NUM_WARPS; ++i) { if (barrier.wait_mask.test(i)) { DP(3, "*** Resume core #" << core_->id() << ", warp #" << i << " at barrier #" << bar_id); - scheduler_->resume(i); + this->release(i, kOpApplyLag); } } // reset barrier and advance phase @@ -106,7 +119,7 @@ void BarrierUnit::arrive(uint32_t bar_id, uint32_t count, uint32_t wid, bool is_ } } -bool BarrierUnit::wait(uint32_t bar_id, uint32_t phase, uint32_t wid) { +void BarrierUnit::wait(uint32_t bar_id, uint32_t phase, uint32_t wid) { auto bar_index = bar_id & 0x7fffffff; auto& barrier = barriers_.at(bar_index); @@ -118,9 +131,9 @@ bool BarrierUnit::wait(uint32_t bar_id, uint32_t phase, uint32_t wid) { // add warp to wait list barrier.wait_mask.set(wid); DP(3, "*** Suspend core #" << core_->id() << ", warp #" << wid << " at barrier #" << bar_id); + } else { + this->release(wid, kOpApplyLag); } - - return wait; } void BarrierUnit::global_resume(uint32_t bar_id) { @@ -132,7 +145,7 @@ void BarrierUnit::global_resume(uint32_t bar_id) { for (uint32_t i = 0; i < VX_CFG_NUM_WARPS; ++i) { if (barrier.wait_mask.test(i)) { DP(3, "*** Resume core #" << core_->id() << ", warp #" << i << " at barrier #" << bar_id); - scheduler_->resume(i); + this->release(i, 0); } } barrier.wait_mask.reset(); @@ -156,7 +169,7 @@ void BarrierUnit::event_release(uint32_t bar_id) { if (is_global) { if (barrier.arrival_mask.count() == active_warps.count()) { uint32_t num_cores = barrier.count; - this->gbar_send(bar_id, num_cores); + this->gbar_send(bar_id, num_cores, kEventApplyLag); barrier.arrival_mask.reset(); barrier.count = 0; barrier.events = 0; @@ -167,7 +180,7 @@ void BarrierUnit::event_release(uint32_t bar_id) { for (uint32_t i = 0; i < VX_CFG_NUM_WARPS; ++i) { if (barrier.wait_mask.test(i)) { DP(3, "*** Resume core #" << core_->id() << ", warp #" << i << " at barrier #" << bar_id); - scheduler_->resume(i); + this->release(i, kEventApplyLag); } } // reset barrier and advance phase diff --git a/sim/simx/barrier_unit.h b/sim/simx/barrier_unit.h index 253fb601c7..d75222612f 100644 --- a/sim/simx/barrier_unit.h +++ b/sim/simx/barrier_unit.h @@ -30,11 +30,14 @@ class BarrierUnit : public SimObject { uint32_t get_phase(uint32_t bar_id) const; void arrive(uint32_t bar_id, uint32_t count, uint32_t wid, bool is_sync_bar); - bool wait(uint32_t bar_id, uint32_t phase, uint32_t wid); + void wait(uint32_t bar_id, uint32_t phase, uint32_t wid); void global_resume(uint32_t bar_id); void event_attach(uint32_t bar_id, uint32_t count = 1); void event_release(uint32_t bar_id); + // warps a local barrier releases, to the core's warp scheduler + SimChannel unlock_out; + protected: void on_reset(); @@ -44,7 +47,9 @@ class BarrierUnit : public SimObject { // send on its own cycle so back-to-back completions serialize here too. uint64_t gbar_send_next_; - void gbar_send(uint32_t bar_id, uint32_t count); + // lag: cycles until the triggering request applies in the barrier store + void gbar_send(uint32_t bar_id, uint32_t count, uint32_t lag); + void release(uint32_t wid, uint32_t lag); Core* core_; Scheduler* scheduler_; diff --git a/sim/simx/cluster.cpp b/sim/simx/cluster.cpp index 95b3669a3d..8674f73591 100644 --- a/sim/simx/cluster.cpp +++ b/sim/simx/cluster.cpp @@ -84,6 +84,7 @@ class Cluster::Impl { VX_CFG_L2_LATENCY, // pipeline latency VX_CFG_L2_REPL_POLICY, // replacement policy (VX_CFG_L2_ENABLED != 0) && (VX_CFG_L3_ENABLED == 0), // is_llc + true, // non-cacheable bypass (VX_CFG_L2_BANK_HASH != 0) && (VX_CFG_L2_NUM_BANKS > 1), // bank_hash L2_SHARED_LLC, // shared_llc VX_CFG_L2_LAZY_FETCH != 0, // lazy_fetch diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index 447314ce80..a6fcfdff90 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -74,6 +74,7 @@ class Core::Impl { , ibuf_inflight_(VX_CFG_NUM_WARPS, 0) , last_issue_(VX_CFG_NUM_WARPS, 0) , staged_since_(VX_CFG_NUM_WARPS, 0) + , head_staged_(VX_CFG_NUM_WARPS, false) { const std::string& name = simobject_->name(); char sname[100]; @@ -258,6 +259,7 @@ class Core::Impl { func_units_.at((int)FUType::SFU) = SimPlatform::instance().create_object(sname, simobject_); std::static_pointer_cast(func_units_.at((int)FUType::ALU))->branch_ctl_out.bind(&simobject_->branch_ctl_in); std::static_pointer_cast(func_units_.at((int)FUType::SFU))->warp_ctl_out.bind(&simobject_->warp_ctl_in); + scheduler_->barrier_unit().unlock_out.bind(&simobject_->bar_unlock_in); #ifdef VX_CFG_EXT_TCU_ENABLE snprintf(sname, 100, "%s-tcu", name.c_str()); tcu_unit_ = SimPlatform::instance().create_object(sname, simobject_); @@ -338,6 +340,7 @@ class Core::Impl { std::fill(issue_lock_owner_.begin(), issue_lock_owner_.end(), kLockOpen); std::fill(last_issue_.begin(), last_issue_.end(), 0); std::fill(staged_since_.begin(), staged_since_.end(), 0); + std::fill(head_staged_.begin(), head_staged_.end(), false); pending_instrs_.clear(); pending_ifetches_ = 0; @@ -373,14 +376,20 @@ class Core::Impl { scheduler_->schedule_begin(); - // each fetch slice picks among its own warps whose ibuffer is not yet saturated + // Each fetch slice picks among its own warps whose ibuffer is not full; a + // buffer is full at exactly IBUF_SIZE entries. With an L1, once every warp's + // buffer is full any ready warp may fetch, its instruction waiting at decode. + bool all_full = (VX_CFG_ICACHE_ENABLED || VX_CFG_DCACHE_ENABLED); + for (uint32_t w = 0; w < VX_CFG_NUM_WARPS && all_full; ++w) { + all_full = (ibuf_inflight_.at(w) == VX_CFG_IBUF_SIZE); + } // the cycle is idle when no slice has a warp to fetch, its latch free or not bool any_cand = false; for (uint32_t f = 0; f < VX_CFG_FETCH_WIDTH; ++f) { auto& fetch_latch = fetch_latch_.at(f); WarpMask warp_mask; for (uint32_t w = f; w < VX_CFG_NUM_WARPS; w += VX_CFG_FETCH_WIDTH) { - if (ibuf_inflight_.at(w) < VX_CFG_IBUF_SIZE) { + if (all_full || ibuf_inflight_.at(w) != VX_CFG_IBUF_SIZE) { warp_mask.set(w); } } @@ -508,7 +517,9 @@ class Core::Impl { mem_req.tag = tag; mem_req.hart_id = trace->cid; mem_req.uuid = trace->uuid; - if (icache_req.try_send(mem_req)) { + // The first icache port also carries the cache-flush injection, which + // registers its requests once more. + if (icache_req.try_send(mem_req, (f == 0) ? 2 : 1)) { DT(3, simobject_->name() << " icache-req: addr=0x" << std::hex << mem_req.addr << ", tag=0x" << mem_req.tag << std::dec << ", " << *trace); #ifdef VX_CFG_EXT_C_ENABLE decompressor->commit_request(from_refetch); @@ -634,6 +645,15 @@ class Core::Impl { // check scoreboard against sequencer micro-op (handles uop WAW/RAW) auto trace = ibuffer->peek(); + + // The head moves into the warp's staging register, freeing its buffer + // slot, once it has arrived and the warp's previous instruction has + // issued; the scheduler sees the freed slot the cycle after. + if (!head_staged_.at(wid) + && SimPlatform::instance().cycles() > std::max(last_issue_.at(wid), trace->ibuf_time)) { + head_staged_.at(wid) = true; + this->ibuf_release(wid); + } auto seq = sequencers_.at(wid); auto uop_trace = seq->get(trace); // returns cached uop or generates next @@ -694,7 +714,8 @@ class Core::Impl { } } - if (ready_set.any()) { + // The arbiter advances only on an issue that fires. + if (ready_set.any() && !operands_.at(iw)->Input.full()) { // select one instruction from ready set auto w = ibuffer_arbs_.at(iw).grant(ready_set); uint32_t wid = w * VX_CFG_ISSUE_WIDTH + iw; @@ -749,19 +770,10 @@ class Core::Impl { trace_pool_.deallocate(trace, 1); } ibuffer->pop(); - // Release ibuffer backpressure slot (matches ++ in schedule()). - // Saturating decrement: flush_warp_pipeline only drains the - // ibuffer + resets ibuf_inflight to 0, but fetch_latch_ / - // decode-stage entries scheduled pre-trap can still arrive - // at issue() post-flush and decrement the (now-zeroed) - // counter. Async-trap users (RTU CB_YIELD path: AHS, CHS, - // MISS) routinely hit this; a non-saturating decrement - // underflows to UINT32_MAX and stalls scheduling. Proper - // fix is to extend flush_warp_pipeline to also drain - // fetch_latch_ + pending icache fetches for the warp. - if (ibuf_inflight_.at(wid) > 0) { - --ibuf_inflight_.at(wid); + if (!head_staged_.at(wid)) { + this->ibuf_release(wid); } + head_staged_.at(wid) = false; } } } @@ -773,6 +785,21 @@ class Core::Impl { } } + // Release an ibuffer backpressure slot (matches ++ in schedule()). + // Saturating decrement: flush_warp_pipeline only drains the ibuffer and + // resets ibuf_inflight to 0, but fetch_latch_ / decode-stage entries + // scheduled pre-trap can still arrive at issue() post-flush and decrement + // the (now-zeroed) counter. Async-trap users (RTU CB_YIELD path: AHS, CHS, + // MISS) routinely hit this; a non-saturating decrement underflows to + // UINT32_MAX and stalls scheduling. Proper fix is to extend + // flush_warp_pipeline to also drain fetch_latch_ + pending icache fetches + // for the warp. + void ibuf_release(uint32_t wid) { + if (ibuf_inflight_.at(wid) > 0) { + --ibuf_inflight_.at(wid); + } + } + void update_queue_full() { for (uint32_t iw = 0; iw < VX_CFG_ISSUE_WIDTH; ++iw) { fu_full_seen_.at(iw) = fu_full_.at(iw); @@ -818,6 +845,11 @@ class Core::Impl { } warp_ctl.pop(); } + auto& bar_unlock = simobject_->bar_unlock_in; + while (!bar_unlock.empty()) { + scheduler_->resume(bar_unlock.peek()); + bar_unlock.pop(); + } // Fan-in: route per-block FU outputs to per-iw commit queues by trace->wid. // Each FU has NUM_*_BLOCKS outputs; the original iw was lost during @@ -934,7 +966,8 @@ class Core::Impl { // ----- Public-facing API (called from outer Core trampolines) ----- bool running() const { - if (scheduler_->running() || !pending_instrs_.empty()) { + // committed stores wait in the LSU request queue until sent to memory + if (scheduler_->running() || !pending_instrs_.empty() || !this->lsu_drained()) { return true; } #ifdef VX_CFG_VM_ENABLE @@ -977,6 +1010,7 @@ class Core::Impl { ibuffer->pop(); } ibuf_inflight_.at(wid) = 0; + head_staged_.at(wid) = false; uint32_t iw = wid % VX_CFG_ISSUE_WIDTH; if (issue_lock_owner_.at(iw) == wid / VX_CFG_ISSUE_WIDTH) { issue_lock_owner_.at(iw) = kLockOpen; @@ -1163,6 +1197,7 @@ class Core::Impl { std::vector ibuf_inflight_; std::vector last_issue_; // [wid] cycle of the warp's last issue std::vector staged_since_; // [wid] cycle the last issued candidate was staged + std::vector head_staged_; // [wid] the buffer head has left for staging PoolAllocator trace_pool_; @@ -1191,6 +1226,7 @@ Core::Core(const SimContext& ctx, , dcache_rsp_in(VX_CFG_DCACHE_NUM_REQS, this) , branch_ctl_in(this, VX_CFG_NUM_WARPS) , warp_ctl_in(this, VX_CFG_NUM_WARPS) + , bar_unlock_in(this, VX_CFG_NUM_WARPS) , gbar_arrive_out(this) , gbar_resume_in(this) #ifdef VX_CFG_EXT_RASTER_ENABLE @@ -1243,8 +1279,8 @@ void Core::barrier_arrive(uint32_t bar_id, uint32_t count, uint32_t wid, bool is impl_->scheduler()->barrier_unit().arrive(bar_id, count, wid, is_sync_bar); } -bool Core::barrier_wait(uint32_t bar_id, uint32_t phase, uint32_t wid) { - return impl_->scheduler()->barrier_unit().wait(bar_id, phase, wid); +void Core::barrier_wait(uint32_t bar_id, uint32_t phase, uint32_t wid) { + impl_->scheduler()->barrier_unit().wait(bar_id, phase, wid); } void Core::global_barrier_resume(uint32_t bar_id) { diff --git a/sim/simx/core.h b/sim/simx/core.h index df1e117b7b..49462541bd 100644 --- a/sim/simx/core.h +++ b/sim/simx/core.h @@ -82,6 +82,7 @@ class Core : public SimObject { // A warp has at most one release in flight, so NUM_WARPS bounds each. SimChannel branch_ctl_in; SimChannel warp_ctl_in; + SimChannel bar_unlock_in; // Global-barrier event links, wired core <-> cluster at elaboration. SimEventLink gbar_arrive_out; @@ -110,7 +111,7 @@ class Core : public SimObject { void barrier_arrive(uint32_t bar_id, uint32_t count, uint32_t wid, bool is_sync_bar); - bool barrier_wait(uint32_t bar_id, uint32_t phase, uint32_t wid); + void barrier_wait(uint32_t bar_id, uint32_t phase, uint32_t wid); void global_barrier_resume(uint32_t bar_id); diff --git a/sim/simx/cta_dispatcher.cpp b/sim/simx/cta_dispatcher.cpp index 950c972df9..1d0fb6883a 100644 --- a/sim/simx/cta_dispatcher.cpp +++ b/sim/simx/cta_dispatcher.cpp @@ -34,6 +34,7 @@ CtaDispatcher::CtaDispatcher(const SimContext& ctx, const char* name, Core* core , wid_to_slot_(num_warps_, num_warps_) // num_warps_ = invalid/unassigned , tail_slot_(0) , has_cta_(false) + , cta_done_(false) , cur_slot_(0) , cta_size_(0) , rank_(0) @@ -59,6 +60,7 @@ CtaDispatcher::~CtaDispatcher() {} void CtaDispatcher::on_reset() { has_cta_ = false; + cta_done_ = false; tail_slot_ = 0; init_wait_ = 0; for (uint32_t i = 0; i < num_warps_; ++i) { @@ -100,6 +102,11 @@ uint32_t CtaDispatcher::usable_slots(uint32_t stride) const { } uint32_t CtaDispatcher::step(const WarpMask& active_warps, uint32_t* wids, cta_warp_record_t* recs) { + if (cta_done_) { + cta_done_ = false; + has_cta_ = false; + return 0; + } if (!has_cta_) { // The KMU pushes CTAs down a registered, credit-gated launch lane; the // endpoint holds the one CTA committed to this core. It is popped only on @@ -178,6 +185,9 @@ uint32_t CtaDispatcher::step(const WarpMask& active_warps, uint32_t* wids, cta_w has_cta_ = true; + // The accept moves the dispatcher to its launch state. + init_wait_ = 1; + if (VX_CFG_FETCH_WIDTH > 1) { bool same_shape = true; for (uint32_t d = 0; d < 3; ++d) { @@ -189,7 +199,7 @@ uint32_t CtaDispatcher::step(const WarpMask& active_warps, uint32_t* wids, cta_w shape_step_[d] = cta_.warp_step[d]; shape_dim_[d] = cta_.block_dim[d]; } - init_wait_ = VX_CFG_FETCH_WIDTH; + init_wait_ += VX_CFG_FETCH_WIDTH; } } } @@ -202,7 +212,7 @@ uint32_t CtaDispatcher::step(const WarpMask& active_warps, uint32_t* wids, cta_w // Lanes with a free slot, round-robin from lane_rr_, up to the warps left; // each launches its lowest free slot. uint32_t count = 0; - for (uint32_t o = 0; o < VX_CFG_FETCH_WIDTH && has_cta_; ++o) { + for (uint32_t o = 0; o < VX_CFG_FETCH_WIDTH && !cta_done_; ++o) { uint32_t lane = (lane_rr_ + o) % VX_CFG_FETCH_WIDTH; int free_wid = -1; for (uint32_t wid = lane; wid < num_warps_; wid += VX_CFG_FETCH_WIDTH) { @@ -266,7 +276,7 @@ void CtaDispatcher::mark_slot_inited(uint32_t wid, const ThreadMask& tmask) { } bool CtaDispatcher::next_warp(bool do_init, cta_warp_record_t* out) { - if (!has_cta_) return false; + if (!has_cta_ || cta_done_) return false; out->do_init = do_init; out->PC = Word(cta_.PC); @@ -320,7 +330,7 @@ bool CtaDispatcher::next_warp(bool do_init, cta_warp_record_t* out) { ++rank_; if (rank_ >= cta_size_) { - has_cta_ = false; + cta_done_ = true; } return true; diff --git a/sim/simx/cta_dispatcher.h b/sim/simx/cta_dispatcher.h index 2404f75055..7c74930292 100644 --- a/sim/simx/cta_dispatcher.h +++ b/sim/simx/cta_dispatcher.h @@ -114,6 +114,8 @@ class CtaDispatcher : public SimObject { // Currently-in-flight CTA (being dispatched warp-rank by warp-rank) bool has_cta_; + // the last warp has launched; the dispatcher returns to accepting next cycle + bool cta_done_; uint32_t cur_slot_; // slot index assigned to the current in-flight CTA kmu_req_t cta_; uint32_t cta_size_; diff --git a/sim/simx/dxa/dxa_core.cpp b/sim/simx/dxa/dxa_core.cpp index abbaf45ced..3ad602dd27 100644 --- a/sim/simx/dxa/dxa_core.cpp +++ b/sim/simx/dxa/dxa_core.cpp @@ -48,6 +48,8 @@ constexpr uint64_t kGmemLineMask = ~uint64_t(VX_CFG_L1_LINE_SIZE - 1); // Cores served by one (socket-resident) DXA engine. constexpr uint32_t kCoresPerSocket = VX_CFG_SOCKET_SIZE; +// A delivered line reaches its SMEM write through two pipeline registers. +constexpr uint32_t kSmemWrLatency = 2; } // namespace @@ -95,6 +97,8 @@ class DxaCore::Impl { struct InflightSlot { bool allocated = false; bool rsp_arrived = false; + uint64_t ready_cycle = 0; // first cycle the line can be written to SMEM + uint64_t free_cycle = 0; // first cycle the slot can carry a new read LineWork work; // captured at allocation std::shared_ptr rsp_data; }; @@ -112,6 +116,11 @@ class DxaCore::Impl { uint32_t smem_stride = 0; uint64_t issue_cycle = 0; + // setup engine: the next transfer is set up while this one drains + bool staged = false; + DxaReq staged_req; + uint64_t staged_ready = 0; + // addr_gen: pre-enumerated work list (one entry per GMEM CL + // multicast replays handled at smem_wr stage). std::vector work_list; @@ -149,7 +158,8 @@ class DxaCore::Impl { w.cta_indices.clear(); w.issued_order.clear(); w.drain_slot = UINT32_MAX; - for (auto& s : w.inflight) { s.allocated = false; s.rsp_arrived = false; s.rsp_data.reset(); } + for (auto& s : w.inflight) { s = InflightSlot(); } + w.staged = false; w.ag_idx = 0; w.mc_cta_idx = 0; w.km_elem_idx = 0; @@ -223,6 +233,7 @@ class DxaCore::Impl { auto& w = workers_[worker_id]; if (slot_id < w.inflight.size() && w.inflight[slot_id].allocated) { w.inflight[slot_id].rsp_arrived = true; + w.inflight[slot_id].ready_cycle = cycle_ + kSmemWrLatency; w.inflight[slot_id].rsp_data = rsp.data; } ch.pop(); @@ -237,10 +248,18 @@ class DxaCore::Impl { } } - // 3) Dispatch from queue to idle workers. + // 3) Start a set-up transfer on an idle worker; otherwise let the + // worker's setup engine take the next request. for (auto& w : workers_) { - if (w.state == WState::IDLE && !queue_.empty()) - start_worker(w, queue_.front()), queue_.pop_front(); + if (w.staged && w.state == WState::IDLE && cycle_ >= w.staged_ready) { + w.staged = false; + start_worker(w, w.staged_req); + } else if (!w.staged && !queue_.empty()) { + w.staged_req = queue_.front(); + w.staged_ready = cycle_ + this->setup_latency(w.staged_req); + w.staged = true; + queue_.pop_front(); + } } // 4) Drain DxaUnit channels → req queue (round-robin). @@ -323,6 +342,14 @@ class DxaCore::Impl { return (k_blk * n_steps + n_blk) * b_blk_elems + n_in * kw + r_in; } + // Parameter setup takes longer for higher-rank descriptors: each extra + // dimension adds a multiply phase. + uint32_t setup_latency(const DxaReq& req) const { + uint32_t rank = (req.desc_slot < VX_DCR_DXA_DESC_COUNT) + ? desc_rank(descriptors_.at(req.desc_slot).meta) : 1; + return 2 * std::max(rank, 2) + 2; + } + // ── Pull from per-core dxa_req_in[] into queue_ (round-robin) ──────── void drain_req_in() { auto& chs = simobject_->dxa_req_in; @@ -548,7 +575,7 @@ class DxaCore::Impl { // Find a free slot. uint32_t slot = UINT32_MAX; for (uint32_t s = 0; s < w.inflight.size(); ++s) { - if (!w.inflight[s].allocated) { + if (!w.inflight[s].allocated && cycle_ >= w.inflight[s].free_cycle) { slot = s; break; } @@ -570,6 +597,7 @@ class DxaCore::Impl { if (lw.oob) { w.inflight[slot].allocated = true; w.inflight[slot].rsp_arrived = true; + w.inflight[slot].ready_cycle = cycle_ + kSmemWrLatency; w.inflight[slot].rsp_data.reset(); w.inflight[slot].work = lw; w.issued_order.push_back(slot); @@ -614,7 +642,8 @@ class DxaCore::Impl { if (slot == UINT32_MAX) { for (uint32_t i = 0; i < w.inflight.size(); ++i) { const auto& candidate = w.inflight[i]; - if (candidate.allocated && candidate.rsp_arrived && !candidate.work.last) { + if (candidate.allocated && candidate.rsp_arrived && !candidate.work.last + && cycle_ >= candidate.ready_cycle) { slot = i; break; } @@ -623,7 +652,7 @@ class DxaCore::Impl { if (slot == UINT32_MAX && w.issued_order.size() == 1) { uint32_t last_slot = w.issued_order.front(); const auto& candidate = w.inflight[last_slot]; - if (candidate.allocated && candidate.rsp_arrived) + if (candidate.allocated && candidate.rsp_arrived && cycle_ >= candidate.ready_cycle) slot = last_slot; } @@ -787,6 +816,7 @@ class DxaCore::Impl { // Slot done — release. s.allocated = false; s.rsp_arrived = false; + s.free_cycle = cycle_ + 1; s.rsp_data.reset(); auto it = std::find(w.issued_order.begin(), w.issued_order.end(), slot); if (it != w.issued_order.end()) w.issued_order.erase(it); diff --git a/sim/simx/lsu_unit.cpp b/sim/simx/lsu_unit.cpp index 2119b6d76f..c81c3458aa 100644 --- a/sim/simx/lsu_unit.cpp +++ b/sim/simx/lsu_unit.cpp @@ -87,7 +87,9 @@ LsuUnit::LsuUnit(const SimContext& ctx, const char* name, Core* core) LsuUnit::~LsuUnit() {} -// Returns true when all blocks' input queues are empty and no reads are outstanding. +// Returns true when no block holds an instruction with requests still to send +// or a read awaiting its response; ops still in the dispatch buffer do not hold +// a barrier. bool LsuUnit::drained() const { #ifdef TCU_META_ENABLE // TcuRspOut forwards to the TCU's endpoint, so query via size() (empty() @@ -97,7 +99,6 @@ bool LsuUnit::drained() const { } #endif for (uint32_t b = 0; b < VX_CFG_NUM_LSU_BLOCKS; ++b) { - if (!Inputs.at(b).empty()) return false; if (!states_.at(b).req_queue.empty()) return false; if (!states_.at(b).pending_reqs.empty()) return false; } @@ -294,8 +295,7 @@ void LsuUnit::process_response_step(uint32_t b) { state.pending_reqs.release(lsu_rsp.tag); } if (is_final && entry.eop) { - // Load writeback crosses one more registered stage than the - // direct-commit path before reaching the commit arbiter. + // the response buffer, then the result arbiter's register output.send(trace, 2); } else { // A non-final fragment still spends a commit-port slot: it drains @@ -335,6 +335,18 @@ void LsuUnit::ingest_inputs(uint32_t b) { } ++state.staged_reads; } + if (lsu_type_tag && *lsu_type_tag == LsuType::STORE) { + // A store commits as the request queue accepts it, through the + // no-response path, and is sent to memory later. The commit stage + // retires the trace, so the queue keeps a copy of its own. + if (Outputs.at(b).full()) { + return; + } + auto copy = new (core_->trace_pool().allocate(1)) instr_trace_t(*trace); + // the no-response buffer, then the result arbiter's register + Outputs.at(b).send(trace, 2); + trace = copy; + } state.req_queue.push(trace); input.pop(); } @@ -451,9 +463,9 @@ void LsuUnit::process_request_step(uint32_t b) { state.head_staged = !is_write || is_amo; } - // AMO always returns to rd, so it is not direct-commit even though it - // carries a side-effect write. - bool direct_commit = (is_write || 0 == state.addr_list.size()) && !is_amo; + // A store committed when the queue accepted it. AMO always returns to rd, + // so it is not direct-commit even though it carries a side-effect write. + bool direct_commit = !is_write && 0 == state.addr_list.size() && !is_amo; // Skip beats whose tid group is entirely inactive — nothing to issue. while (state.remain_addrs != 0) { @@ -582,6 +594,10 @@ void LsuUnit::process_request_step(uint32_t b) { state.head_staged = false; } state.req_queue.pop(); + if (is_write) { + trace->~instr_trace_t(); + core_->trace_pool().deallocate(trace, 1); + } } } diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index 72124b8e3d..f7c9c82bc0 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -2044,6 +2044,93 @@ class CacheBank : public SimObject { /////////////////////////////////////////////////////////////////////////////// +// The cache's core-port arbiter: a port's request is accepted only when its +// bank grants it, one per bank per cycle, round-robin over the ports, and only +// while the bank's input buffer has room; non-cacheable requests are split off +// for the bypass path. It ticks ahead of the banks, so a grant reaches its bank +// in the same cycle unless the crossbar registers its output. +class CachePortArb : public SimObject { +public: + std::vector> ReqIn; // core ports + std::vector> ReqOut; // banks + std::vector> IoOut; // non-cacheable requests, per port + + CachePortArb(const SimContext& ctx, const char* name, const params_t& params, + uint32_t num_inputs, uint32_t num_banks) + : SimObject(ctx, name) + , ReqIn(num_inputs, this) + , ReqOut(num_banks, this) + , IoOut(num_inputs, this) + , params_(params) + , lg2_inputs_(log2ceil(num_inputs)) + // the request crossbar registers its output when it has more than two inputs + , delay_((num_inputs > 2) ? 1 : 0) + , rr_(num_banks, num_inputs - 1) + { + assert(num_inputs <= 64); + } + + uint64_t collisions() const { return collisions_; } + + // The cache's line initialization holds the core ports for these cycles. + void set_init_cycles(uint32_t cycles) { init_cycles_ = cycles; } + +protected: + void on_reset() { + std::fill(rr_.begin(), rr_.end(), ReqIn.size() - 1); + collisions_ = 0; + } + + void on_tick() { + if (init_cycles_ != 0) { + --init_cycles_; + return; + } + uint32_t n = ReqIn.size(); + // requesting ports per bank; non-cacheable requests leave on their own port + std::vector want(ReqOut.size(), 0); + for (uint32_t i = 0; i < n; ++i) { + auto& in = ReqIn.at(i); + if (in.empty()) { + continue; + } + auto& req = in.peek(); + if (req.flags.io) { + if (IoOut.at(i).try_send(req, 0)) { + in.pop(); + } + continue; + } + want.at(params_.addr_bank_id(req.addr)) |= 1ull << i; + } + for (uint32_t b = 0, nb = ReqOut.size(); b < nb; ++b) { + if (want[b] == 0 || ReqOut.at(b).full()) { + continue; + } + uint32_t i = rr_[b]; + do { + i = (i + 1) % n; + } while (!(want[b] & (1ull << i))); + collisions_ += __builtin_popcountll(want[b]) - 1; + auto req = ReqIn.at(i).peek(); + req.tag = (req.tag << lg2_inputs_) | i; + ReqOut.at(b).send(req, delay_); + ReqIn.at(i).pop(); + rr_[b] = i; + } + } + +private: + params_t params_; + uint32_t lg2_inputs_; + uint32_t delay_; + uint32_t init_cycles_ = 0; + std::vector rr_; + uint64_t collisions_ = 0; + + friend class SimObject; +}; + class Cache::Impl { public: Impl(Cache *simobject, const Config &config) @@ -2088,7 +2175,15 @@ class Cache::Impl { nc_mem_arbs_.at(i)->RspOut.at(0).bind(&bank_mem_arb->RspIn.at(i)); } - // Create bank's core crossbar + // Create the core-port arbiter ahead of the banks it feeds + snprintf(sname, 100, "%s-port_arb", simobject->name().c_str()); + port_arb_ = CachePortArb::Create(sname, params_, config_.num_inputs, num_banks); + for (uint32_t i = 0; i < config_.num_inputs; ++i) { + simobject->core_req_in.at(i).bind(&port_arb_->ReqIn.at(i)); + } + + // Create bank's core crossbar; it carries the bank responses back to the + // core ports snprintf(sname, 100, "%s-core_xbar", simobject->name().c_str()); bank_core_xbar_ = MemCrossBar::Create(sname, ArbiterType::RoundRobin, config_.num_inputs, num_banks, [&](const MemCrossBar::ReqType &req) { @@ -2101,7 +2196,7 @@ class Cache::Impl { banks_.at(i) = CacheBank::Create(sname, config, params_, i); // bind core ports - bank_core_xbar_->ReqOut.at(i).bind(&banks_.at(i)->core_req_in); + port_arb_->ReqOut.at(i).bind(&banks_.at(i)->core_req_in); banks_.at(i)->core_rsp_out.bind(&bank_core_xbar_->RspIn.at(i)); // bind memory ports @@ -2115,6 +2210,7 @@ class Cache::Impl { return; // calculate cache initialization cycles init_cycles_ = keep_lines_ ? 0 : params_.sets_per_bank; + port_arb_->set_init_cycles(init_cycles_); keep_lines_ = false; } @@ -2147,26 +2243,18 @@ class Cache::Impl { if (bank_rsp.empty()) continue; auto &core_rsp = bank_rsp.peek(); - if (simobject_->core_rsp_out.at(req_id).try_send(core_rsp, 0)) { + // the non-cacheable bypass layer's response arbiter registers it once more + if (simobject_->core_rsp_out.at(req_id).try_send(core_rsp, config_.nc_enable ? 1 : 0)) { DT(3, simobject_->name() << " core-rsp: " << core_rsp); bank_rsp.pop(); } } - // schedule core requests + // non-cacheable core requests split off by the port arbiter for (uint32_t req_id = 0, n = config_.num_inputs; req_id < n; ++req_id) { - auto &core_req_in = simobject_->core_req_in.at(req_id); - if (core_req_in.empty()) - continue; - auto &core_req = core_req_in.peek(); - if (core_req.flags.io) { - if (this->processBypassRequest(core_req, req_id)) { - core_req_in.pop(); - } - } else { - if (bank_core_xbar_->ReqIn.at(req_id).try_send(core_req, 0)) { - core_req_in.pop(); - } + auto &io_req = port_arb_->IoOut.at(req_id); + if (!io_req.empty() && this->processBypassRequest(io_req.peek(), req_id)) { + io_req.pop(); } } } @@ -2177,7 +2265,7 @@ class Cache::Impl { for (const auto &bank : banks_) { perf_stats += bank->perf_stats(); } - perf_stats.bank_stalls = bank_core_xbar_->collisions(); + perf_stats.bank_stalls = port_arb_->collisions(); } return perf_stats; } @@ -2233,6 +2321,7 @@ class Cache::Impl { std::vector banks_; MemArbiter::Ptr bank_arb_; std::vector nc_mem_arbs_; + CachePortArb::Ptr port_arb_; MemCrossBar::Ptr bank_core_xbar_; uint32_t init_cycles_; bool keep_lines_ = false; diff --git a/sim/simx/mem/cache.h b/sim/simx/mem/cache.h index 5dd4c9bd08..40d7890501 100644 --- a/sim/simx/mem/cache.h +++ b/sim/simx/mem/cache.h @@ -43,6 +43,7 @@ class Cache : public SimObject { uint16_t latency; // pipeline latency uint8_t repl_policy; // replacement policy (ReplPolicy) bool is_llc; // is last-level cache + bool nc_enable = false; // non-cacheable bypass layer in front of the cache bool bank_hash = false; // hashed bank select bool shared_llc = false; // the LLC serves every memory client alone bool lazy_fetch = false; // write miss allocates without a fill (write-back LLC only) diff --git a/sim/simx/mem/local_mem.cpp b/sim/simx/mem/local_mem.cpp index d2b6d60598..9e3dc16884 100644 --- a/sim/simx/mem/local_mem.cpp +++ b/sim/simx/mem/local_mem.cpp @@ -85,11 +85,21 @@ class LocalMem::Impl { snprintf(sname, 100, "%s-xbar", simobject->name().c_str()); uint32_t lg2_line_size = log2ceil(config_.line_size); uint32_t num_banks = 1 << config.B; + // The crossbar's registered request and response outputs are taken by + // the switch and bank stages around it, so its delays here are only the + // extra tree level: an arbiter over more than 1.5x the fan-out limit is + // split into a buffered tree, one cycle deeper. The request side + // arbitrates the lanes at each bank, the response side the banks at + // each lane. + auto tree_delay = [](uint32_t requesters) -> uint32_t { + constexpr uint32_t kMaxFanout = 8; + return (requesters > kMaxFanout + kMaxFanout / 2) ? 1 : 0; + }; mem_xbar_ = MemCrossBar::Create(sname, ArbiterType::Priority, config.num_reqs, num_banks, [lg2_line_size, num_banks](const MemCrossBar::ReqType& req) { // Custom logic to calculate the output index using bank interleaving return (uint32_t)((req.addr >> lg2_line_size) & (num_banks-1)); - }); + }, tree_delay(config.num_reqs), tree_delay(num_banks)); for (uint32_t i = 0; i < config.num_reqs; ++i) { simobject->Inputs.at(i).bind(&mem_xbar_->ReqIn.at(i)); mem_xbar_->RspOut.at(i).bind(&simobject->Outputs.at(i)); diff --git a/sim/simx/opc_unit.cpp b/sim/simx/opc_unit.cpp index e5b1f8aac1..9585ba21fb 100644 --- a/sim/simx/opc_unit.cpp +++ b/sim/simx/opc_unit.cpp @@ -13,6 +13,7 @@ #include "opc_unit.h" #include "core.h" +#include #include #include #include "types.h" @@ -30,6 +31,9 @@ OpcUnit::OpcUnit(const SimContext &ctx, const char* name, , Input(this) , Output(this) , num_threads_(num_threads) { + // The scoreboard's output skid buffer: two entries, the one being collected + // among them, with a registered ready. + Input.set_registered_ready(true); regs_.reserve(num_warp_slots); for (uint32_t i = 0; i < num_warp_slots; ++i) { regs_.emplace_back(num_threads); @@ -43,29 +47,29 @@ void OpcUnit::on_reset() { total_stalls_ = 0; cur_trace_ = nullptr; release_cycle_ = 0; + consume_cycle_ = 0; + consumed_ = false; // The register files are excluded: they are storage, not pipeline state, and // a launch is not a device reset. A slot that skips its startup relies on the // stack pointer that startup left behind, so clearing them here would strand // it at zero. They are zero-initialized at construction. } +// Each bank serves one operand per cycle. static uint32_t compute_bank_conflicts(const instr_trace_t* trace) { - uint32_t stalls = 0; + uint32_t per_bank[VX_CFG_NUM_GPR_BANKS] = {}; + uint32_t most = 0; for (uint32_t i = 0; i < NUM_SRC_REGS; ++i) { - for (uint32_t j = i + 1; j < NUM_SRC_REGS; ++j) { - if ((trace->src_regs[i].type == RegType::None) - || (trace->src_regs[j].type == RegType::None)) - continue; - if ((trace->src_regs[i].type == RegType::Integer && trace->src_regs[i].id() == 0) - || (trace->src_regs[j].type == RegType::Integer && trace->src_regs[j].id() == 0)) - continue; // skip x0 - uint32_t bank_i = trace->src_regs[i].idx % VX_CFG_NUM_GPR_BANKS; - uint32_t bank_j = trace->src_regs[j].idx % VX_CFG_NUM_GPR_BANKS; - if (bank_i == bank_j) - ++stalls; + auto& reg = trace->src_regs[i]; + if (reg.type == RegType::None) { + continue; + } + if (reg.type == RegType::Integer && reg.id() == 0) { + continue; // skip x0 } + most = std::max(most, ++per_bank[reg.idx % VX_CFG_NUM_GPR_BANKS]); } - return stalls; + return most ? (most - 1) : 0; } void OpcUnit::on_tick() { @@ -79,14 +83,20 @@ void OpcUnit::on_tick() { cur_trace_ = nullptr; } - // accept next uop into the holding slot + // accept next uop into the holding slot; it stays in the input buffer + // until its last operand read if (cur_trace_ == nullptr && !Input.empty()) { auto trace = Input.peek(); uint32_t stalls = compute_bank_conflicts(trace); total_stalls_ += stalls; cur_trace_ = trace; + consume_cycle_ = cur_cycle + stalls; release_cycle_ = cur_cycle + 1 + stalls; + consumed_ = false; + } + if (cur_trace_ != nullptr && !consumed_ && cur_cycle >= consume_cycle_) { Input.pop(); + consumed_ = true; } } diff --git a/sim/simx/opc_unit.h b/sim/simx/opc_unit.h index af8f201468..ed15f26819 100644 --- a/sim/simx/opc_unit.h +++ b/sim/simx/opc_unit.h @@ -76,6 +76,8 @@ class OpcUnit : public SimObject { uint32_t total_stalls_ = 0; instr_trace_t* cur_trace_ = nullptr; uint64_t release_cycle_ = 0; + uint64_t consume_cycle_ = 0; + bool consumed_ = false; friend class SimObject; friend class Operands; // Operands::get_exit_code reads regs_ directly diff --git a/sim/simx/processor.cpp b/sim/simx/processor.cpp index aef1822cf6..b7d31c1705 100644 --- a/sim/simx/processor.cpp +++ b/sim/simx/processor.cpp @@ -104,6 +104,7 @@ ProcessorImpl::ProcessorImpl() VX_CFG_L3_LATENCY, // pipeline latency VX_CFG_L3_REPL_POLICY, // replacement policy VX_CFG_L3_ENABLED != 0, // is_llc when L3 is the LLC + true, // non-cacheable bypass false, // bank_hash L3_SHARED_LLC, // shared_llc VX_CFG_L3_LAZY_FETCH != 0, // lazy_fetch diff --git a/sim/simx/sfu_unit.cpp b/sim/simx/sfu_unit.cpp index edf9311c5f..4c57d1ab8b 100644 --- a/sim/simx/sfu_unit.cpp +++ b/sim/simx/sfu_unit.cpp @@ -33,6 +33,9 @@ using namespace vortex; // With several SFU blocks, their TEX/RTU/DXA requests merge onto the core's bus // through one registered arbiter stage. static constexpr uint32_t kBusMergeDelay = (VX_CFG_NUM_SFU_BLOCKS > 1) ? 1 : 0; +// With several SFU blocks, warp-control requests are registered toward the +// core-wide warp-control unit; a sole block reaches it directly. +static constexpr bool kWctlBuffered = (VX_CFG_NUM_SFU_BLOCKS > 1); SfuUnit::SfuUnit(const SimContext& ctx, const char* name, Core* core) : FuncUnit(ctx, name, core, 6) @@ -70,6 +73,10 @@ SfuUnit::SfuUnit(const SimContext& ctx, const char* name, Core* core) , rtu_unit_(new RtuUnit(core, rtu_req_out, rtu_window_, kBusMergeDelay)) #endif { + // the dispatch skid buffer's ready is registered + for (auto& input : Inputs) { + input.set_registered_ready(true); + } } // Cycles from processing to the output: the result buffer and the PE @@ -418,6 +425,43 @@ void SfuUnit::on_tick() { // The fragment work distributor (above + scheduler) launches fragment // warps directly from the autonomously-pulled covered-quad waves. + // A CSR op is held a cycle before it may fire: the CTA context RAMs are + // read and the CSR decode registered in it. The wait restarts per op, so + // the CSR path takes one op every two cycles. + const bool is_csr = std::get_if(&trace->op_type) != nullptr; + if (is_csr && csr_read_done_.at(b) != trace) { + csr_read_done_.at(b) = trace; + continue; + } + + // A barrier arrive is the one shared request with a response. With + // several SFU blocks its request is registered toward the shared unit, + // which answers it two cycles after it is sent; it is sent once the LSU + // has drained, and the arrival takes effect as it is sent, as the + // unit's one grant per cycle orders it among the other arrivals. A sole + // block is answered in the cycle it asks, like any other op. + bool bar_arrive = false; + if (auto wctl_p = std::get_if(&trace->op_type); + kWctlBuffered && wctl_p && *wctl_p == WctlType::BAR && trace->eop) { + auto args = std::get(trace->instr_ptr->get_args()); + bar_arrive = args.is_bar_arrive; + } + if (bar_arrive) { + auto& ba = bar_arrive_.at(b); + uint64_t now = SimPlatform::instance().cycles(); + if (ba.trace != trace) { + if (!core_->lsu_drained()) { + continue; // drain LSU before the barrier + } + ba = BarArrive{trace, now, false, false}; + ba.release_warp = wctl_unit_->process(trace, &ba.warp_exit); + continue; + } + if (now < ba.sent + 2 || output.full()) { + continue; + } + } + if (output.full()) continue; // stall — no side effects this tick @@ -429,7 +473,7 @@ void SfuUnit::on_tick() { if (*wctl_p == WctlType::WSYNC) { if (core_->has_pending_instrs(trace->wid)) continue; // wait for the warp's prior instrs to retire - } else if (*wctl_p == WctlType::BAR) { + } else if (*wctl_p == WctlType::BAR && !bar_arrive) { if (!core_->lsu_drained()) continue; // drain LSU before the barrier } @@ -438,10 +482,15 @@ void SfuUnit::on_tick() { bool release_warp = trace->fetch_stall; bool warp_exit = false; - if (std::get_if(&trace->op_type)) { + if (bar_arrive) { + release_warp = bar_arrive_.at(b).release_warp; + warp_exit = bar_arrive_.at(b).warp_exit; + bar_arrive_.at(b).trace = nullptr; + } else if (std::get_if(&trace->op_type)) { release_warp = wctl_unit_->process(trace, &warp_exit); - } else if (std::get_if(&trace->op_type)) { + } else if (is_csr) { csr_unit_->process(trace); + csr_read_done_.at(b) = nullptr; #ifdef VX_CFG_EXT_DXA_ENABLE } else if (std::get_if(&trace->op_type)) { // process() returns nullptr on backpressure (idempotent retry next @@ -476,12 +525,19 @@ void SfuUnit::on_tick() { continue; } auto& input = Inputs.at(winner[r]); - if (input.empty() || input.peek() != winner_head[r]) { + if (input.empty() || input.peek() != winner_head[r] + || bar_arrive_.at(winner[r]).trace == winner_head[r]) { res_rr_[r] = (winner[r] + 1) % VX_CFG_NUM_SFU_BLOCKS; } } } +void SfuUnit::on_reset() { + std::fill(std::begin(res_rr_), std::end(res_rr_), 0); + csr_read_done_.fill(nullptr); + bar_arrive_.fill(BarArrive{}); +} + SfuUnit::SharedRes SfuUnit::shared_res(uint32_t block) { auto& input = Inputs.at(block); if (input.empty()) { @@ -506,8 +562,17 @@ SfuUnit::SharedRes SfuUnit::shared_res(uint32_t block) { } #endif if (auto wctl_p = std::get_if(&trace->op_type)) { - // a block requests only when its op could fire this cycle - if (Outputs.at(block).full()) { + // a barrier arrive whose request has left no longer needs the port + if (bar_arrive_.at(block).trace == trace) { + return RES_NONE; + } + // A block requests only when its op could fire this cycle. A barrier + // arrive's request does not wait on the output: it completes later. + bool bar_arrive = false; + if (*wctl_p == WctlType::BAR && trace->eop) { + bar_arrive = std::get(trace->instr_ptr->get_args()).is_bar_arrive; + } + if (!bar_arrive && Outputs.at(block).full()) { return RES_NONE; } if (*wctl_p == WctlType::BAR && trace->eop && !core_->lsu_drained()) { diff --git a/sim/simx/sfu_unit.h b/sim/simx/sfu_unit.h index ffc26c22b0..5e208b614e 100644 --- a/sim/simx/sfu_unit.h +++ b/sim/simx/sfu_unit.h @@ -118,6 +118,7 @@ class SfuUnit : public FuncUnit { #endif protected: + void on_reset() override; void on_tick() override; private: @@ -132,6 +133,17 @@ class SfuUnit : public FuncUnit { std::unique_ptr wctl_unit_; std::unique_ptr csr_unit_; + // Per block, the CSR op that has spent its one-cycle CTA-context read. + std::array csr_read_done_{}; + // Per block, the barrier arrive waiting for the shared barrier unit's + // response, which carries the phase it writes back. + struct BarArrive { + const instr_trace_t* trace = nullptr; + uint64_t sent = 0; + bool release_warp = false; + bool warp_exit = false; + }; + std::array bar_arrive_{}; #ifdef VX_CFG_EXT_RTU_ENABLE // Shared graphics register window (SETW/GETW/GETWF slot file) used by the RTU // ray/hit stream, TEX (vx_tex4) u,v payload + texel, and OM (vx_om4) payload. diff --git a/sim/simx/socket.cpp b/sim/simx/socket.cpp index 536dfe4d1e..83eac594da 100644 --- a/sim/simx/socket.cpp +++ b/sim/simx/socket.cpp @@ -91,6 +91,7 @@ class Socket::Impl { VX_CFG_DCACHE_LATENCY, // pipeline latency (capacity-scaled, matches hardware) VX_CFG_DCACHE_REPL_POLICY, // replacement policy (VX_CFG_DCACHE_ENABLED != 0) && (VX_CFG_L2_ENABLED == 0) && (VX_CFG_L3_ENABLED == 0), // is_llc + true, // non-cacheable bypass (VX_CFG_DCACHE_BANK_HASH != 0) && (VX_CFG_DCACHE_NUM_BANKS > 1), // bank_hash DCACHE_SHARED_LLC, // shared_llc VX_CFG_DCACHE_LAZY_FETCH != 0, // lazy_fetch diff --git a/sim/simx/types.h b/sim/simx/types.h index 1658771c6e..85b248ade7 100644 --- a/sim/simx/types.h +++ b/sim/simx/types.h @@ -1052,7 +1052,8 @@ class GreedyRRArbiter : public IArbiterImpl { class Arbiter { public: - Arbiter(ArbiterType type = ArbiterType::Priority, uint32_t size = 0) { + Arbiter(ArbiterType type = ArbiterType::Priority, uint32_t size = 0) + : type_(type), size_(size) { switch (type) { case ArbiterType::Priority: impl_ = std::make_shared(size); @@ -1071,6 +1072,20 @@ class Arbiter { } } + // A copy is a separate arbiter with its own grant state, so a vector of + // arbiters built from one prototype does not share a single state. + Arbiter(const Arbiter& other) : Arbiter(other.type_, other.size_) {} + + Arbiter& operator=(const Arbiter& other) { + if (this != &other) { + *this = Arbiter(other.type_, other.size_); + } + return *this; + } + + Arbiter(Arbiter&&) = default; + Arbiter& operator=(Arbiter&&) = default; + virtual ~Arbiter() {} uint32_t grant(const BitVector<>& requests) { @@ -1082,6 +1097,8 @@ class Arbiter { } private: + ArbiterType type_; + uint32_t size_; std::shared_ptr impl_; }; diff --git a/sim/simx/wctl_unit.cpp b/sim/simx/wctl_unit.cpp index e989360acc..e31d612e72 100644 --- a/sim/simx/wctl_unit.cpp +++ b/sim/simx/wctl_unit.cpp @@ -20,17 +20,8 @@ using namespace vortex; -bool WctlUnit::is_shared(const instr_trace_t* trace) const { - if (!trace->eop) { - return false; - } +ThreadMask WctlUnit::next_tmask(const instr_trace_t* trace) const { auto wctl_type = std::get(trace->op_type); - if (wctl_type == WctlType::BAR || wctl_type == WctlType::WSPAWN) { - return true; - } - if (wctl_type != WctlType::TMC && wctl_type != WctlType::PRED) { - return false; - } auto& warp = core_->scheduler().warp(trace->wid); uint32_t num_threads = VX_CFG_NUM_THREADS; int32_t thread_last = num_threads - 1; @@ -39,22 +30,35 @@ bool WctlUnit::is_shared(const instr_trace_t* trace) const { } auto& rs1_data = trace->src_data[0]; auto& rs2_data = trace->src_data[1]; - ThreadMask next_tmask(num_threads); if (wctl_type == WctlType::TMC) { + ThreadMask tmask(num_threads); for (uint32_t t = 0; t < num_threads; ++t) { - next_tmask.set(t, rs1_data.at(thread_last).u & (1 << t)); + tmask.set(t, rs1_data.at(thread_last).u & (1 << t)); } - } else { - auto wctlArgs = std::get(trace->instr_ptr->get_args()); - auto not_pred = wctlArgs.is_cond_neg; - ThreadMask pred(num_threads); - for (uint32_t t = 0; t < num_threads; ++t) { - auto cond = (rs1_data.at(t).i & 0x1) ^ not_pred; - pred[t] = warp.tmask.test(t) && cond; - } - next_tmask = pred.any() ? (warp.tmask & pred) : ThreadMask(num_threads, rs2_data.at(thread_last).u); + return tmask; + } + auto wctlArgs = std::get(trace->instr_ptr->get_args()); + auto not_pred = wctlArgs.is_cond_neg; + ThreadMask pred(num_threads); + for (uint32_t t = 0; t < num_threads; ++t) { + auto cond = (rs1_data.at(t).i & 0x1) ^ not_pred; + pred[t] = warp.tmask.test(t) && cond; + } + return pred.any() ? (warp.tmask & pred) : ThreadMask(num_threads, rs2_data.at(thread_last).u); +} + +bool WctlUnit::is_shared(const instr_trace_t* trace) const { + if (!trace->eop) { + return false; + } + auto wctl_type = std::get(trace->op_type); + if (wctl_type == WctlType::BAR || wctl_type == WctlType::WSPAWN) { + return true; } - return next_tmask.none(); + if (wctl_type != WctlType::TMC && wctl_type != WctlType::PRED) { + return false; + } + return this->next_tmask(trace).none(); } bool WctlUnit::process(instr_trace_t* trace, bool* exit) { @@ -80,16 +84,13 @@ bool WctlUnit::process(instr_trace_t* trace, bool* exit) { switch (wctl_type) { case WctlType::TMC: { - ThreadMask next_tmask(num_threads); - for (uint32_t t = 0; t < num_threads; ++t) { - next_tmask.set(t, rs1_data.at(thread_last).u & (1 << t)); - } if (trace->eop) { - if (next_tmask.none()) { + auto tmask = this->next_tmask(trace); + if (tmask.none()) { *exit = true; release_warp = false; } else { - release_warp = core_->setTmask(trace->wid, next_tmask); + release_warp = core_->setTmask(trace->wid, tmask); } } } break; @@ -180,25 +181,15 @@ bool WctlUnit::process(instr_trace_t* trace, bool* exit) { release_warp = false; } } else { - release_warp = !core_->barrier_wait(bar_id, arg2, trace->wid); + // the barrier unit releases the warp once its phase has passed + core_->barrier_wait(bar_id, arg2, trace->wid); + release_warp = false; } } } break; case WctlType::PRED: { - ThreadMask pred(num_threads); - auto not_pred = wctlArgs.is_cond_neg; - for (uint32_t t = 0; t < num_threads; ++t) { - auto cond = (rs1_data.at(t).i & 0x1) ^ not_pred; - pred[t] = warp.tmask.test(t) && cond; - } - ThreadMask next_tmask = warp.tmask; - if (pred.any()) { - next_tmask &= pred; - } else { - next_tmask = ThreadMask(num_threads, rs2_data.at(thread_last).u); - } if (trace->eop) { - release_warp = core_->setTmask(trace->wid, next_tmask); + release_warp = core_->setTmask(trace->wid, this->next_tmask(trace)); } } break; case WctlType::WSYNC: diff --git a/sim/simx/wctl_unit.h b/sim/simx/wctl_unit.h index e688a59abd..d1ab897a28 100644 --- a/sim/simx/wctl_unit.h +++ b/sim/simx/wctl_unit.h @@ -40,6 +40,9 @@ class WctlUnit { bool is_shared(const instr_trace_t* trace) const; private: + // Thread mask a TMC or PRED installs. + ThreadMask next_tmask(const instr_trace_t* trace) const; + Core* core_; }; From 397d7d24836af24024aaab8359c0e8a65e0ed3d0 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 18 Oct 2026 14:46:12 -0700 Subject: [PATCH 038/124] rtl: fix -DSCOPE builds of fetch, issue, LSU, TCU and graphics SCOPE_IO_SWITCH was declared inside the DBG_SCOPE_* blocks of VX_fetch, VX_issue_slice and VX_lsu_block, so a -DSCOPE build without the matching DBG_SCOPE_* tap left the module's scope bus undeclared and failed to elaborate. Declare the switch whenever SCOPE is set, and keep only the taps under DBG_SCOPE_*. With the TCU enabled, VX_execute never connected tcu_unit's scope port and VX_tcu_core never drove its scope output. VX_execute now switches 1 + EXT_TCU_ENABLED scope buses and binds the TCU to the second one; VX_tcu_core terminates its bus with SCOPE_IO_UNUSED. The texture, raster and OM cores terminated their scope bus with an empty SCOPE_IO_UNUSED(), which does not parse; they now pass the bus index. Their parents never bound those buses: VX_graphics now switches one bus per raster and OM core, VX_socket adds one per texture core, and VX_cluster gives VX_graphics its own slot. A -DSCOPE build with TEX, RASTER and OM enabled now lints clean. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/VX_cluster.sv | 6 ++++++ hw/rtl/VX_graphics.sv | 18 ++++++++++++++++++ hw/rtl/VX_socket.sv | 6 ++++++ hw/rtl/core/VX_execute.sv | 6 +++++- hw/rtl/core/VX_fetch.sv | 2 +- hw/rtl/core/VX_issue_slice.sv | 2 +- hw/rtl/core/VX_lsu_block.sv | 2 +- hw/rtl/om/VX_om_core.sv | 4 ++-- hw/rtl/raster/VX_raster_core.sv | 4 ++-- hw/rtl/tcu/VX_tcu_core.sv | 5 +++++ hw/rtl/tex/VX_tex_core.sv | 4 ++-- 11 files changed, 49 insertions(+), 10 deletions(-) diff --git a/hw/rtl/VX_cluster.sv b/hw/rtl/VX_cluster.sv index 6204d4ee88..79f625eda4 100644 --- a/hw/rtl/VX_cluster.sv +++ b/hw/rtl/VX_cluster.sv @@ -59,8 +59,13 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; `ifdef SCOPE localparam scope_socket = 0; +`ifdef EXT_GFX_CLUSTER_ENABLE + localparam scope_gfx = NUM_SOCKETS; + `SCOPE_IO_SWITCH (NUM_SOCKETS + 1); +`else `SCOPE_IO_SWITCH (NUM_SOCKETS); `endif +`endif `ifdef PERF_ENABLE cache_perf_t l2_perf; @@ -429,6 +434,7 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; VX_graphics #( .INSTANCE_ID (INSTANCE_ID) ) graphics ( + `SCOPE_IO_BIND (scope_gfx) .clk (clk), .reset (reset), .cluster_id (cluster_id), diff --git a/hw/rtl/VX_graphics.sv b/hw/rtl/VX_graphics.sv index 329dd7fc67..37a035506d 100644 --- a/hw/rtl/VX_graphics.sv +++ b/hw/rtl/VX_graphics.sv @@ -23,6 +23,8 @@ module VX_graphics import VX_gpu_pkg::* ; #( parameter `STRING INSTANCE_ID = "" ) ( + `SCOPE_IO_DECL + input wire clk, input wire reset, @@ -67,6 +69,20 @@ module VX_graphics import VX_gpu_pkg::* // no in-band `done`); the cluster ORs this into the device busy aggregation. output wire busy ); + +`ifdef SCOPE +`ifdef VX_CFG_EXT_RASTER_ENABLE + localparam scope_raster_cnt = `VX_CFG_NUM_RASTER_CORES; +`else + localparam scope_raster_cnt = 0; +`endif +`ifdef VX_CFG_EXT_OM_ENABLE + localparam scope_om_cnt = `VX_CFG_NUM_OM_CORES; +`else + localparam scope_om_cnt = 0; +`endif + `SCOPE_IO_SWITCH (scope_raster_cnt + scope_om_cnt); +`endif `UNUSED_SPARAM (INSTANCE_ID) `ifndef VX_CFG_EXT_RASTER_ENABLE `UNUSED_VAR (cluster_id) @@ -156,6 +172,7 @@ module VX_graphics import VX_gpu_pkg::* .QUAD_FIFO_DEPTH (`VX_CFG_RASTER_QUAD_FIFO_DEPTH), .OUTPUT_QUADS (`VX_CFG_NUM_THREADS) ) raster_core ( + `SCOPE_IO_BIND (i) .clk (clk), .reset (reset), .instance_idx (`LOG2UP(`VX_CFG_NUM_CLUSTERS * `VX_CFG_NUM_RASTER_CORES)'(cluster_id) * `LOG2UP(`VX_CFG_NUM_CLUSTERS * `VX_CFG_NUM_RASTER_CORES)'(`VX_CFG_NUM_RASTER_CORES) @@ -417,6 +434,7 @@ module VX_graphics import VX_gpu_pkg::* .INSTANCE_ID (`SFORMATF(("%s-om%0d", INSTANCE_ID, i))), .NUM_LANES (OM_CORE_LANES) ) om_core ( + `SCOPE_IO_BIND (scope_raster_cnt + i) .clk (clk), .reset (reset), `ifdef PERF_ENABLE diff --git a/hw/rtl/VX_socket.sv b/hw/rtl/VX_socket.sv index e8dd0c8248..08aa35fc86 100644 --- a/hw/rtl/VX_socket.sv +++ b/hw/rtl/VX_socket.sv @@ -65,7 +65,12 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; `ifdef SCOPE localparam scope_core = 0; +`ifdef VX_CFG_EXT_TEX_ENABLE + localparam scope_tex = `VX_CFG_SOCKET_SIZE; + `SCOPE_IO_SWITCH (`VX_CFG_SOCKET_SIZE + `VX_CFG_NUM_TEX_CORES); +`else `SCOPE_IO_SWITCH (`VX_CFG_SOCKET_SIZE); +`endif `endif VX_kmu_bus_if per_core_kmu_bus_if[`VX_CFG_SOCKET_SIZE](); @@ -442,6 +447,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .NUM_LANES (`VX_CFG_NUM_SFU_LANES), .TAG_WIDTH (TEX_REQ_ARB1_TAG_WIDTH) ) tex_core ( + `SCOPE_IO_BIND (scope_tex + i) .clk (clk), .reset (reset), `ifdef PERF_ENABLE diff --git a/hw/rtl/core/VX_execute.sv b/hw/rtl/core/VX_execute.sv index b1601ff6da..7454c4d92e 100644 --- a/hw/rtl/core/VX_execute.sv +++ b/hw/rtl/core/VX_execute.sv @@ -92,7 +92,10 @@ module VX_execute import VX_gpu_pkg::*; #( .branch_ctl_if (branch_ctl_if) ); - `SCOPE_IO_SWITCH (1); +`ifdef SCOPE + localparam SCOPE_COUNT = 1 + `VX_CFG_EXT_TCU_ENABLED; +`endif + `SCOPE_IO_SWITCH (SCOPE_COUNT); VX_lsu_unit #( .INSTANCE_ID (`SFORMATF(("%s-lsu", INSTANCE_ID))) @@ -122,6 +125,7 @@ module VX_execute import VX_gpu_pkg::*; #( VX_tcu_unit #( .INSTANCE_ID (`SFORMATF(("%s-tcu", INSTANCE_ID))) ) tcu_unit ( + `SCOPE_IO_BIND (1) .clk (clk), .reset (reset), `ifdef PERF_ENABLE diff --git a/hw/rtl/core/VX_fetch.sv b/hw/rtl/core/VX_fetch.sv index a4f8847e7f..43442d9bb0 100644 --- a/hw/rtl/core/VX_fetch.sv +++ b/hw/rtl/core/VX_fetch.sv @@ -260,8 +260,8 @@ module VX_fetch import VX_gpu_pkg::*; #( `endif `ifdef SCOPE -`ifdef DBG_SCOPE_FETCH `SCOPE_IO_SWITCH (1); +`ifdef DBG_SCOPE_FETCH wire schedule_fire = schedule_if.valid && schedule_if.ready; wire icache_bus_req_fire = icache_bus_if.req_valid && icache_bus_if.req_ready; wire icache_bus_rsp_fire = icache_bus_if.rsp_valid && icache_bus_if.rsp_ready; diff --git a/hw/rtl/core/VX_issue_slice.sv b/hw/rtl/core/VX_issue_slice.sv index 4f147c68e2..2ad54391fd 100644 --- a/hw/rtl/core/VX_issue_slice.sv +++ b/hw/rtl/core/VX_issue_slice.sv @@ -99,8 +99,8 @@ module VX_issue_slice import VX_gpu_pkg::*; #( assign warp_issued_wis = scoreboard_if.data.wis; `ifdef SCOPE -`ifdef DBG_SCOPE_ISSUE `SCOPE_IO_SWITCH (1); +`ifdef DBG_SCOPE_ISSUE wire decode_fire = decode_if.valid && decode_if.ready; wire operands_fire = operands_if.valid && operands_if.ready; wire reset_negedge; diff --git a/hw/rtl/core/VX_lsu_block.sv b/hw/rtl/core/VX_lsu_block.sv index f127d9f260..3d9406851a 100644 --- a/hw/rtl/core/VX_lsu_block.sv +++ b/hw/rtl/core/VX_lsu_block.sv @@ -543,8 +543,8 @@ module VX_lsu_block import VX_gpu_pkg::*; #( `ifdef SCOPE -`ifdef DBG_SCOPE_LSU `SCOPE_IO_SWITCH (1); +`ifdef DBG_SCOPE_LSU wire reset_negedge; `NEG_EDGE (reset_negedge, reset); `SCOPE_TAP_EX (0, 3, 4, 2, ( diff --git a/hw/rtl/om/VX_om_core.sv b/hw/rtl/om/VX_om_core.sv index f8f610d39f..e1821560c3 100644 --- a/hw/rtl/om/VX_om_core.sv +++ b/hw/rtl/om/VX_om_core.sv @@ -575,8 +575,8 @@ module VX_om_core import VX_gpu_pkg::*; import VX_om_pkg::*; #( || ~pending_reads_empty || mem_rsp_valid || mem_unit_busy; `ifdef SCOPE -`ifdef DBG_SCOPE_OM `SCOPE_IO_SWITCH (1); +`ifdef DBG_SCOPE_OM wire cache_bus_req_fire_0 = cache_bus_if[0].req_valid && cache_bus_if[0].req_ready; wire cache_bus_rsp_fire_0 = cache_bus_if[0].rsp_valid && cache_bus_if[0].rsp_ready; wire [OCACHE_TAG_WIDTH-UUID_WIDTH-1:0] cache_bus_req_tag = cache_bus_if[0].req_data.tag[OCACHE_TAG_WIDTH-UUID_WIDTH-1:0]; @@ -628,7 +628,7 @@ module VX_om_core import VX_gpu_pkg::*; import VX_om_pkg::*; #( reset_negedge, 1'b0, 4096 ); `else - `SCOPE_IO_UNUSED() + `SCOPE_IO_UNUSED(0) `endif `endif `ifdef CHIPSCOPE diff --git a/hw/rtl/raster/VX_raster_core.sv b/hw/rtl/raster/VX_raster_core.sv index be08681dbc..372dd007a0 100644 --- a/hw/rtl/raster/VX_raster_core.sv +++ b/hw/rtl/raster/VX_raster_core.sv @@ -560,8 +560,8 @@ module VX_raster_core import VX_gpu_pkg::*; import VX_raster_pkg::*; #( assign busy = armed_r | frame_kick; `ifdef SCOPE -`ifdef DBG_SCOPE_RASTER `SCOPE_IO_SWITCH (1); +`ifdef DBG_SCOPE_RASTER wire cache_bus_req_fire_0 = cache_bus_if[0].req_valid && cache_bus_if[0].req_ready; wire cache_bus_rsp_fire_0 = cache_bus_if[0].rsp_valid && cache_bus_if[0].rsp_ready; wire raster_bus_fire = raster_bus_tmp_if[0].req_valid && raster_bus_tmp_if[0].req_ready; @@ -615,7 +615,7 @@ module VX_raster_core import VX_gpu_pkg::*; import VX_raster_pkg::*; #( reset_negedge, 1'b0, 4096 ); `else - `SCOPE_IO_UNUSED() + `SCOPE_IO_UNUSED(0) `endif `endif `ifdef CHIPSCOPE diff --git a/hw/rtl/tcu/VX_tcu_core.sv b/hw/rtl/tcu/VX_tcu_core.sv index 1762caafd7..e4fc89786f 100644 --- a/hw/rtl/tcu/VX_tcu_core.sv +++ b/hw/rtl/tcu/VX_tcu_core.sv @@ -41,6 +41,11 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( // Outputs VX_result_if.master result_if ); + +`ifdef SCOPE + `SCOPE_IO_SWITCH (1); + `SCOPE_IO_UNUSED (0) +`endif `UNUSED_SPARAM (INSTANCE_ID); // FEDP dot-product width in 32-bit words (doubles under FEDP2K). diff --git a/hw/rtl/tex/VX_tex_core.sv b/hw/rtl/tex/VX_tex_core.sv index c8e427df62..490b48965f 100644 --- a/hw/rtl/tex/VX_tex_core.sv +++ b/hw/rtl/tex/VX_tex_core.sv @@ -259,8 +259,8 @@ module VX_tex_core import VX_gpu_pkg::*; import VX_tex_pkg::*; #( ); `ifdef SCOPE -`ifdef DBG_SCOPE_TEX `SCOPE_IO_SWITCH (1); +`ifdef DBG_SCOPE_TEX wire cache_bus_req_fire_0 = cache_bus_if[0].req_valid && cache_bus_if[0].req_ready; wire cache_bus_rsp_fire_0 = cache_bus_if[0].rsp_valid && cache_bus_if[0].rsp_ready; wire [TCACHE_TAG_WIDTH-UUID_WIDTH-1:0] cache_bus_req_tag = cache_bus_if[0].req_data.tag[TCACHE_TAG_WIDTH-UUID_WIDTH-1:0]; @@ -324,7 +324,7 @@ module VX_tex_core import VX_gpu_pkg::*; import VX_tex_pkg::*; #( reset_negedge, 1'b0, 4096 ); `else - `SCOPE_IO_UNUSED() + `SCOPE_IO_UNUSED(0) `endif `endif `ifdef CHIPSCOPE From 8705e2b7b666e3ed2664b7254257bfa26f6c889a Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 18 Oct 2026 14:46:13 -0700 Subject: [PATCH 039/124] core: derive the core partitioning from the widths The core now has three structures, selected by VX_gpu_pkg from the fetch width (FW), issue width (IW) and per-unit block counts, with no config switch: - some unit has blocks != IW: FW frontend slices, the frontend crossbar, and the stage-sliced VX_issue / VX_execute / VX_commit; - every unit has blocks == IW, FW < IW (BACKEND_SLICES): FW frontend slices, the frontend crossbar, and IW backend slices; - every unit has blocks == IW == FW (COMPUTE_SLICES): IW compute slices, each a frontend slice joined directly to its backend slice, with no crossbar. This is the NVIDIA processing-block organization. Modules: - VX_frontend_slice (was VX_fetch_slice): warp scheduler, fetch, decode. - VX_frontend_xbar (was VX_fetch_xbar): routes instructions, branch and warp-control events and CSR traffic by warp id in the first two structures. - VX_backend_slice (was VX_compute_slice): issue, one block of every unit, and commit for one issue slice. - VX_compute_slice (new): VX_frontend_slice + VX_backend_slice. VX_CFG_COMPUTE_SLICES_ENABLE is removed: it could only turn off a structure that the widths already select. VX_sfu_shared (DXA/TEX/RTU request arbitration) and VX_tcu_shared (WGMMA tile buffers, AGU and sparse metadata) serve every block of the core, so they move from VX_sfu_unit / VX_tcu_unit to VX_core and are shared by all three structures. The units expose per-block ports instead. The tcu_unit and tensor unit-test wrappers instantiate VX_tcu_shared themselves, so the measured design still includes it. SimX already models this timing, so only its comments change to the new terminology. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/tensor_wg.yaml | 2 +- docs/designs/microarchitecture.md | 35 + docs/designs/tensor_core_wgmma_engine.md | 3 +- hw/rtl/VX_gpu_pkg.sv | 31 +- hw/rtl/core/VX_backend_slice.sv | 192 +++++ hw/rtl/core/VX_compute_slice.sv | 221 ++++-- hw/rtl/core/VX_core.sv | 688 ++++++++++-------- hw/rtl/core/VX_execute.sv | 74 +- ...VX_fetch_slice.sv => VX_frontend_slice.sv} | 4 +- .../{VX_fetch_xbar.sv => VX_frontend_xbar.sv} | 14 +- hw/rtl/core/VX_scheduler.sv | 4 +- hw/rtl/core/VX_sfu_unit.sv | 47 +- hw/rtl/core/VX_uuid_gen.sv | 2 +- hw/rtl/core/VX_warp_sched.sv | 2 +- hw/rtl/interfaces/VX_slice_sched_if.sv | 2 +- hw/rtl/tcu/VX_tcu_bbuf.sv | 2 +- hw/rtl/tcu/VX_tcu_unit.sv | 80 +- hw/rtl/tcu/VX_tcu_wgmma.sv | 4 +- hw/unittest/tcu_unit/VX_tcu_unit_top.sv | 77 +- hw/unittest/tensor/VX_tensor_top.sv | 76 +- sim/simx/constants.h | 2 +- sim/simx/core.cpp | 12 +- sim/simx/scheduler.h | 4 +- sim/simx/socket.cpp | 2 +- 24 files changed, 1045 insertions(+), 535 deletions(-) create mode 100644 hw/rtl/core/VX_backend_slice.sv rename hw/rtl/core/{VX_fetch_slice.sv => VX_frontend_slice.sv} (97%) rename hw/rtl/core/{VX_fetch_xbar.sv => VX_frontend_xbar.sv} (95%) diff --git a/ci/testcases/tensor_wg.yaml b/ci/testcases/tensor_wg.yaml index b9bb956479..937d898bff 100644 --- a/ci/testcases/tensor_wg.yaml +++ b/ci/testcases/tensor_wg.yaml @@ -232,7 +232,7 @@ tests: args: -m64 -n64 -k64 # One execution block per unit: the four-block model ISSUE_WIDTH=4 implies is # several times the size, and a hosted CI runner cannot build it in a job. - # Two fetch slices: covers the fetch-slice front end on both models. + # Two frontend slices: covers the frontend crossbar on both models. configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_TCU_FEDP2K -DVX_CFG_TCU_TYPE_DPI -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DWGMMA_RS -DVX_CFG_FETCH_WIDTH=2 diff --git a/docs/designs/microarchitecture.md b/docs/designs/microarchitecture.md index 3a9913d82d..10516946c5 100644 --- a/docs/designs/microarchitecture.md +++ b/docs/designs/microarchitecture.md @@ -75,6 +75,41 @@ Vortex has a 6-stage pipeline: - **Commit** - Write result back to the register file and update the Scoreboard. +### Core partitioning + +The core splits its warps into slices, each a constant bit-slice of the warp id: +`VX_CFG_FETCH_WIDTH` (FW) frontend slices (`wid mod FW`), `VX_CFG_ISSUE_WIDTH` (IW) +issue slices (`wid mod IW`), and B blocks per execution unit (`wid mod B`), with +FW and B dividing IW. A slice owns its warps' pipeline state; the core scheduler +(CTA launch, barriers, wspawn, retirement), the shared SFU/TCU services and the +memory unit stay core-wide. + +The structure follows from the widths: + +| Widths | Structure | +|---|---| +| every B = IW = FW | IW compute slices (`VX_compute_slice`), no frontend crossbar | +| every B = IW, FW < IW | FW frontend slices, the frontend crossbar, IW backend slices (`VX_backend_slice`) | +| some unit has B ≠ IW | FW frontend slices, the frontend crossbar, stage-sliced `VX_issue` / `VX_execute` / `VX_commit` | + +- **Frontend slice** (`VX_frontend_slice`): warp scheduler, fetch and decode for + `wid mod FW`, on its own icache port. +- **Backend slice** (`VX_backend_slice`): issue, execute (one block of every + unit) and commit for one issue slice. +- **Compute slice** (`VX_compute_slice`): a frontend slice and its backend slice, + joined directly: one end-to-end pipeline for `wid mod IW`. This is the + counterpart of an NVIDIA SM processing block. +- **Frontend crossbar** (`VX_frontend_xbar`): combinational routing between FW + frontend slices and the back end whenever the core is not built from + compute slices. It sends each instruction to its issue slice, and each + branch, warp-control and CSR event to the frontend slice that owns the warp. +- **Core-wide services**, reached through one port per block in every structure: + - `VX_sfu_shared` arbitrates the SFU blocks onto the core's single DXA, TEX + and RTU buses. + - `VX_tcu_shared` holds the state that spans TCU blocks: the WGMMA tile + buffers, whose B operand is common to a warpgroup's blocks, and the sparse + metadata AGU. + ### Vortex clustering architecture - Sockets - Grouping multiple cores sharing L1 cache diff --git a/docs/designs/tensor_core_wgmma_engine.md b/docs/designs/tensor_core_wgmma_engine.md index d6eefb36f1..ad890708d8 100644 --- a/docs/designs/tensor_core_wgmma_engine.md +++ b/docs/designs/tensor_core_wgmma_engine.md @@ -192,7 +192,8 @@ is muxed by `op_type`: `VX_tcu_wgmma` is a third party that only **observes** the dispatch path. It builds the tile-buffer request from each WGMMA micro-op and returns operand data and a ready signal, but never drives the handshake itself; the core -does. +does. `VX_tcu_agu` and `VX_tcu_wgmma` serve every block, so they live in +the core-level `VX_tcu_shared` and reach each block through per-block ports. --- diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index 114b2ef153..ec92eca60a 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -630,7 +630,7 @@ package VX_gpu_pkg; end endfunction - // Fetch slice f schedules and fetches the warps with wid mod FETCH_WIDTH == f, + // Frontend slice f schedules and fetches the warps with wid mod FETCH_WIDTH == f, // naming them by their local id wfs = wid / FETCH_WIDTH. localparam FETCH_FSW_BITS = `CLOG2(`VX_CFG_FETCH_WIDTH); localparam FETCH_FSW_W = `UP(FETCH_FSW_BITS); @@ -661,8 +661,8 @@ package VX_gpu_pkg; end endfunction - // FU block ports per fetch slice: a unit with B blocks serves warps with - // wid mod B == b, so B >= FETCH_WIDTH gives each fetch slice B / FETCH_WIDTH + // FU block ports per frontend slice: a unit with B blocks serves warps with + // wid mod B == b, so B >= FETCH_WIDTH gives each frontend slice B / FETCH_WIDTH // blocks and B < FETCH_WIDTH shares each block among FETCH_WIDTH / B slices. localparam FETCH_ALU_PORTS = (`VX_CFG_NUM_ALU_BLOCKS > `VX_CFG_FETCH_WIDTH) ? (`VX_CFG_NUM_ALU_BLOCKS / `VX_CFG_FETCH_WIDTH) : 1; localparam FETCH_SFU_PORTS = (`VX_CFG_NUM_SFU_BLOCKS > `VX_CFG_FETCH_WIDTH) ? (`VX_CFG_NUM_SFU_BLOCKS / `VX_CFG_FETCH_WIDTH) : 1; @@ -719,14 +719,17 @@ package VX_gpu_pkg; end endfunction - // The core is built from ISSUE_WIDTH compute slices (VX_compute_slice) when - // every execution unit has one block per issue slice; otherwise it keeps the - // stage-sliced issue / execute / commit structure. - localparam COMPUTE_SLICES_ENABLE = (`VX_CFG_NUM_ALU_BLOCKS == `VX_CFG_ISSUE_WIDTH) - && (`VX_CFG_NUM_LSU_BLOCKS == `VX_CFG_ISSUE_WIDTH) - && (`VX_CFG_NUM_SFU_BLOCKS == `VX_CFG_ISSUE_WIDTH) - && (!`VX_CFG_EXT_F_ENABLED || (`VX_CFG_NUM_FPU_BLOCKS == `VX_CFG_ISSUE_WIDTH)) - && (!`VX_CFG_EXT_TCU_ENABLED || (`VX_CFG_NUM_TCU_BLOCKS == `VX_CFG_ISSUE_WIDTH)); + // The core structure follows from the widths. When every execution unit has + // one block per issue slice, the back end is ISSUE_WIDTH backend slices + // (VX_backend_slice: issue, execute, commit); otherwise it is stage-sliced. + // When the fetch width matches too, each backend slice and its frontend slice + // form one compute slice (VX_compute_slice), with no frontend crossbar. + localparam BACKEND_SLICES = (`VX_CFG_NUM_ALU_BLOCKS == `VX_CFG_ISSUE_WIDTH) + && (`VX_CFG_NUM_LSU_BLOCKS == `VX_CFG_ISSUE_WIDTH) + && (`VX_CFG_NUM_SFU_BLOCKS == `VX_CFG_ISSUE_WIDTH) + && (!`VX_CFG_EXT_F_ENABLED || (`VX_CFG_NUM_FPU_BLOCKS == `VX_CFG_ISSUE_WIDTH)) + && (!`VX_CFG_EXT_TCU_ENABLED || (`VX_CFG_NUM_TCU_BLOCKS == `VX_CFG_ISSUE_WIDTH)); + localparam COMPUTE_SLICES = BACKEND_SLICES && (`VX_CFG_FETCH_WIDTH == `VX_CFG_ISSUE_WIDTH); /////////////////////////////// TENSOR UNIT /////////////////////////////// @@ -1022,7 +1025,7 @@ package VX_gpu_pkg; logic [NCTA_WIDTH-1:0] id_cta; } cta_lane_bcast_t; - // Scheduler traffic between an FU block and the fetch slice owning the warp, + // Scheduler traffic between an FU block and the frontend slice owning the warp, // addressed by core-wide wid. // ALU branch resolution @@ -1648,11 +1651,11 @@ package VX_gpu_pkg; // Block size in bytes localparam ICACHE_LINE_SIZE = `VX_CFG_L1_LINE_SIZE; - // Request ports per core: one per fetch slice, each fetching its own warps + // Request ports per core: one per frontend slice, each fetching its own warps localparam ICACHE_NUM_REQS = `VX_CFG_FETCH_WIDTH; localparam ICACHE_NUM_BANKS = `VX_CFG_ICACHE_NUM_BANKS; - // Core request tag Id bits: the warp id local to the requesting fetch slice + // Core request tag Id bits: the warp id local to the requesting frontend slice localparam ICACHE_TAG_ID_BITS = FETCH_WFS_W; // Core request tag bits as VX_fetch writes them (UUID + warp id). diff --git a/hw/rtl/core/VX_backend_slice.sv b/hw/rtl/core/VX_backend_slice.sv new file mode 100644 index 0000000000..93732a84cd --- /dev/null +++ b/hw/rtl/core/VX_backend_slice.sv @@ -0,0 +1,192 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// One issue slice carried through issue, execute and commit, serving the warps +// with wid mod ISSUE_WIDTH == slice_id with one block of every execution unit. +// Every slice is the same module: its identity arrives on slice_id/core_id, and +// only the core-wide resources (frontend slices, warp-control resources, memory +// system, SFU buses, WGMMA tile buffers and the metadata AGU) are reached through +// its ports. +module VX_backend_slice import VX_gpu_pkg::*; +`ifdef VX_CFG_EXT_TCU_ENABLE +import VX_tcu_pkg::*; +`endif +#( + parameter `STRING INSTANCE_ID = "", + parameter `STRING SLICE_STR = "" +) ( + `SCOPE_IO_DECL + + input wire clk, + input wire reset, + + input wire [CORE_ID_WIDTH-1:0] core_id, + input wire [ISSUE_ISW_W-1:0] slice_id, + +`ifdef PERF_ENABLE + output issue_perf_t issue_perf, +`endif + + // decoded instructions of the slice's warps + VX_decode_if.slave decode_if, + + // issued and retired warps (registered), indexed by wid / ISSUE_WIDTH + output wire warp_issued, + output wire [ISSUE_WIS_W-1:0] warp_issued_wis, + output wire [PER_ISSUE_WARPS-1:0] committed_wis, + + // scheduler traffic of the slice's ALU and SFU blocks + VX_branch_ctl_if.master branch_ctl_if, + VX_warp_ctl_if.master warp_ctl_if, + VX_sched_csr_if.slave sched_csr_if, +`ifdef VX_CFG_EXT_RTU_ENABLE + VX_sched_unlock_if.master sched_unlock_if, +`endif + + // LSU + VX_lsu_sched_if.master lsu_client_if, + + // SFU +`ifdef VX_CFG_EXT_DXA_ENABLE + VX_dxa_req_bus_if.master dxa_req_bus_if, +`endif +`ifdef VX_CFG_EXT_TEX_ENABLE + VX_tex_bus_if.master tex_bus_if, +`endif +`ifdef VX_CFG_EXT_RTU_ENABLE + VX_rtu_bus_if.master rtu_bus_if, +`endif + + // TCU +`ifdef VX_CFG_TCU_WGMMA_ENABLE + output wire tcu_exec_valid, + output wire tcu_exec_ready, + output tcu_execute_t tcu_exec_data, + input wire [TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs1_data, + input wire [TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs2_data, + input wire tcu_tbuf_ready, +`endif +`ifdef TCU_META_ENABLE + output wire tcu_agu_ld_valid, + output tcu_execute_t tcu_agu_ld_data, + input wire tcu_agu_ld_ready, + input wire tcu_agu_result_valid, + input tcu_result_t tcu_agu_result_data, + output wire tcu_agu_result_ready, + input wire tcu_meta_wr_en, + input wire [NW_WIDTH-1:0] tcu_meta_wr_wid, + input wire [4:0] tcu_meta_wr_idx, + input wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] tcu_meta_wr_data, +`endif + + // core-wide warp control + VX_warp_shared_if.master warp_shared_if +); + `STATIC_ASSERT (BACKEND_SLICES, ("invalid parameter")) + + `SCOPE_IO_SWITCH (2); + + VX_dispatch_if dispatch_if [NUM_EX_UNITS](); + VX_commit_if commit_if [NUM_EX_UNITS](); + VX_writeback_if writeback_if(); + + // issue + + wire issue_fire; + wire [ISSUE_WIS_W-1:0] issue_wis; + + VX_issue_slice #( + .INSTANCE_ID (`SFORMATF(("%s-issue%s", INSTANCE_ID, SLICE_STR))) + ) issue_slice ( + `SCOPE_IO_BIND (0) + .clk (clk), + .reset (reset), + .slice_id (slice_id), + `ifdef PERF_ENABLE + .issue_perf (issue_perf), + `endif + .decode_if (decode_if), + .writeback_if (writeback_if), + .dispatch_if (dispatch_if), + .warp_issued (issue_fire), + .warp_issued_wis (issue_wis) + ); + + `BUFFER(warp_issued, issue_fire); + `BUFFER(warp_issued_wis, issue_wis); + + // execute + + VX_execute_slice #( + .INSTANCE_ID (`SFORMATF(("%s-execute", INSTANCE_ID))), + .SLICE_STR (SLICE_STR) + ) execute_slice ( + `SCOPE_IO_BIND (1) + .clk (clk), + .reset (reset), + .core_id (core_id), + .slice_id (slice_id), + .dispatch_if (dispatch_if), + .commit_if (commit_if), + .branch_ctl_if (branch_ctl_if), + .lsu_client_if (lsu_client_if), + `ifdef VX_CFG_EXT_DXA_ENABLE + .dxa_req_bus_if (dxa_req_bus_if), + `endif + `ifdef VX_CFG_EXT_TEX_ENABLE + .tex_bus_if (tex_bus_if), + `endif + `ifdef VX_CFG_EXT_RTU_ENABLE + .rtu_bus_if (rtu_bus_if), + .sched_unlock_if (sched_unlock_if), + `endif + `ifdef VX_CFG_TCU_WGMMA_ENABLE + .tcu_exec_valid (tcu_exec_valid), + .tcu_exec_ready (tcu_exec_ready), + .tcu_exec_data (tcu_exec_data), + .tcu_tbuf_rs1_data (tcu_tbuf_rs1_data), + .tcu_tbuf_rs2_data (tcu_tbuf_rs2_data), + .tcu_tbuf_ready (tcu_tbuf_ready), + `endif + `ifdef TCU_META_ENABLE + .tcu_agu_ld_valid (tcu_agu_ld_valid), + .tcu_agu_ld_data (tcu_agu_ld_data), + .tcu_agu_ld_ready (tcu_agu_ld_ready), + .tcu_agu_result_valid (tcu_agu_result_valid), + .tcu_agu_result_data (tcu_agu_result_data), + .tcu_agu_result_ready (tcu_agu_result_ready), + .tcu_meta_wr_en (tcu_meta_wr_en), + .tcu_meta_wr_wid (tcu_meta_wr_wid), + .tcu_meta_wr_idx (tcu_meta_wr_idx), + .tcu_meta_wr_data (tcu_meta_wr_data), + `endif + .warp_shared_if (warp_shared_if), + .warp_ctl_if (warp_ctl_if), + .sched_csr_if (sched_csr_if) + ); + + // commit + + VX_commit_slice #( + .INSTANCE_ID (`SFORMATF(("%s-commit", INSTANCE_ID))) + ) commit_slice ( + .clk (clk), + .reset (reset), + .commit_if (commit_if), + .writeback_if (writeback_if), + .committed_wis (committed_wis) + ); + +endmodule diff --git a/hw/rtl/core/VX_compute_slice.sv b/hw/rtl/core/VX_compute_slice.sv index 33c771fdc1..0111b36d12 100644 --- a/hw/rtl/core/VX_compute_slice.sv +++ b/hw/rtl/core/VX_compute_slice.sv @@ -13,12 +13,12 @@ `include "VX_define.vh" -// One issue slice carried through issue, execute and commit, serving the warps -// with wid mod ISSUE_WIDTH == slice_id with one block of every execution unit. -// Every slice is the same module: its identity arrives on slice_id/core_id, and -// only the core-wide resources (fetch slices, warp-control resources, memory -// system, SFU buses, WGMMA tile buffers and the metadata AGU) are reached through -// its ports. +// A frontend slice and its backend slice: the warps with wid mod ISSUE_WIDTH == +// slice_id carried through fetch, decode, issue, execute and commit. Used when +// the fetch, issue and every execution-unit width are equal, so frontend slice i +// serves only issue slice i and the blocks of slice i, and the two meet +// directly. Only the core scheduler, the icache, the memory system and the +// shared SFU/TCU services are reached through the ports. module VX_compute_slice import VX_gpu_pkg::*; `ifdef VX_CFG_EXT_TCU_ENABLE import VX_tcu_pkg::*; @@ -36,23 +36,26 @@ import VX_tcu_pkg::*; input wire [ISSUE_ISW_W-1:0] slice_id, `ifdef PERF_ENABLE + output fetch_perf_t fetch_perf, output issue_perf_t issue_perf, `endif - // decoded instructions of the slice's warps - VX_decode_if.slave decode_if, - - // issued and retired warps (registered), indexed by wid / ISSUE_WIDTH - output wire warp_issued, - output wire [ISSUE_WIS_W-1:0] warp_issued_wis, + // core scheduler and icache + VX_slice_sched_if.slave slice_sched_if, + VX_mem_bus_if.master icache_bus_if, output wire [PER_ISSUE_WARPS-1:0] committed_wis, - // scheduler traffic of the slice's ALU and SFU blocks - VX_branch_ctl_if.master branch_ctl_if, - VX_warp_ctl_if.master warp_ctl_if, - VX_sched_csr_if.slave sched_csr_if, -`ifdef VX_CFG_EXT_RTU_ENABLE - VX_sched_unlock_if.master sched_unlock_if, + // core scheduler state + input wire [PERF_CTR_BITS-1:0] cycles, + input wire [PERF_CTR_BITS-1:0] instret, + input wire [`VX_CFG_NUM_WARPS-1:0] active_warps, + input wire lsu_sched_drained, + output wire [NCTA_WIDTH-1:0] csr_rd_cta_id, + input cta_ctx_t csr_rd_ctx, +`ifdef VX_CFG_VM_ENABLE + input wire [`VX_CFG_XLEN-1:0] satp, + output wire satp_wr_valid, + output wire [`VX_CFG_XLEN-1:0] satp_wr_data, `endif // LSU @@ -94,53 +97,160 @@ import VX_tcu_pkg::*; // core-wide warp control VX_warp_shared_if.master warp_shared_if ); - `STATIC_ASSERT (COMPUTE_SLICES_ENABLE, ("invalid parameter")) + `STATIC_ASSERT (COMPUTE_SLICES, ("invalid parameter")) + + localparam SW = PER_FETCH_WARPS; `SCOPE_IO_SWITCH (2); - VX_dispatch_if dispatch_if [NUM_EX_UNITS](); - VX_commit_if commit_if [NUM_EX_UNITS](); - VX_writeback_if writeback_if(); + VX_decode_if decode_if(); + VX_branch_ctl_if branch_ctl_if(); + VX_warp_ctl_if warp_ctl_if(); + VX_sched_csr_if sched_csr_if(); +`ifdef VX_CFG_EXT_RTU_ENABLE + VX_sched_unlock_if sched_unlock_if(); +`endif - // issue + wire warp_issued; + wire [ISSUE_WIS_W-1:0] warp_issued_wis; - wire issue_fire; - wire [ISSUE_WIS_W-1:0] issue_wis; + // slice-local view of the frontend slice's ports: block i serves only frontend slice + // i, and a warp's fetch-local and issue-local ids are the same + sched_branch_t [0:0] branch; + sched_wctl_t [0:0] wctl; + wire [0:0][NW_WIDTH-1:0] dvstack_wid; + wire [0:0][DV_STACK_SIZEW-1:0] dvstack_ptr; + wire [0:0][`VX_CFG_MEM_ADDR_WIDTH-1:0] dvstack_mscratch; + wire [0:0][NW_WIDTH-1:0] csr_rd_wid; + sched_csr_rd_t [0:0] csr_rd; + sched_csr_wr_t [0:0] csr_wr; +`ifdef VX_CFG_EXT_RTU_ENABLE + wire [0:0] rtu_unlock_valid; + wire [0:0][NW_WIDTH-1:0] rtu_unlock_wid; +`endif + wire [SW-1:0] issued; + wire [SW-1:0] pending_alm_empty; - VX_issue_slice #( - .INSTANCE_ID (`SFORMATF(("%s-issue%s", INSTANCE_ID, SLICE_STR))) - ) issue_slice ( - `SCOPE_IO_BIND (0) - .clk (clk), - .reset (reset), - .slice_id (slice_id), + VX_frontend_slice #( + .INSTANCE_ID (INSTANCE_ID), + .SLICE_STR ((`VX_CFG_FETCH_WIDTH > 1) ? SLICE_STR : "") + ) frontend_slice ( + `SCOPE_IO_BIND (0) + .clk (clk), + .reset (reset), + .core_id (core_id), + .slice_id (FETCH_FSW_W'(slice_id)), `ifdef PERF_ENABLE - .issue_perf (issue_perf), + .fetch_perf (fetch_perf), `endif - .decode_if (decode_if), - .writeback_if (writeback_if), - .dispatch_if (dispatch_if), - .warp_issued (issue_fire), - .warp_issued_wis (issue_wis) + .slice_sched_if (slice_sched_if), + .icache_bus_if (icache_bus_if), + .decode_if (decode_if), + .branch (branch), + .wctl (wctl), + .dvstack_wid (dvstack_wid), + .dvstack_ptr (dvstack_ptr), + .dvstack_mscratch (dvstack_mscratch), + .csr_rd_wid (csr_rd_wid), + .csr_rd (csr_rd), + .csr_wr (csr_wr), + `ifdef VX_CFG_EXT_RTU_ENABLE + .rtu_unlock_valid (rtu_unlock_valid), + .rtu_unlock_wid (rtu_unlock_wid), + `endif + .issued (issued), + .committed (committed_wis), + .pending_alm_empty (pending_alm_empty) ); - `BUFFER(warp_issued, issue_fire); - `BUFFER(warp_issued_wis, issue_wis); + for (genvar l = 0; l < SW; ++l) begin : g_issued + assign issued[l] = warp_issued && (warp_issued_wis == ISSUE_WIS_W'(l)); + end + + assign branch[0].valid = branch_ctl_if.valid; + assign branch[0].wid = branch_ctl_if.wid; + assign branch[0].taken = branch_ctl_if.taken; + assign branch[0].dest = branch_ctl_if.dest; + assign branch[0].is_trap = branch_ctl_if.is_trap; + assign branch[0].is_mret = branch_ctl_if.is_mret; + assign branch[0].trap_cause = branch_ctl_if.trap_cause; + + assign wctl[0].tmc_valid = warp_ctl_if.tmc_valid; + assign wctl[0].split_valid = warp_ctl_if.split_valid; + assign wctl[0].sjoin_valid = warp_ctl_if.sjoin_valid; + assign wctl[0].wsync_valid = warp_ctl_if.wsync_valid; + assign wctl[0].wid = warp_ctl_if.wid; + assign wctl[0].tmc = warp_ctl_if.tmc; + assign wctl[0].split = warp_ctl_if.split; + assign wctl[0].sjoin = warp_ctl_if.sjoin; + assign dvstack_wid[0] = warp_ctl_if.dvstack_wid; + assign warp_ctl_if.dvstack_ptr = dvstack_ptr[0]; + assign warp_ctl_if.mscratch = dvstack_mscratch[0]; + assign warp_ctl_if.warp_pending_alm_empty = pending_alm_empty; + assign warp_ctl_if.lsu_sched_drained = lsu_sched_drained; - // execute + assign csr_rd_wid[0] = sched_csr_if.csr_rd_wid; + assign csr_rd_cta_id = sched_csr_if.csr_rd_cta_id; + assign csr_wr[0].valid = sched_csr_if.csr_wr_valid; + assign csr_wr[0].wid = sched_csr_if.csr_wr_wid; + assign csr_wr[0].data = sched_csr_if.csr_wr_data; + assign csr_wr[0].trap_valid = sched_csr_if.trap_csr_wr_valid; + assign csr_wr[0].trap_data = sched_csr_if.trap_csr_wr_data; - VX_execute_slice #( - .INSTANCE_ID (`SFORMATF(("%s-execute", INSTANCE_ID))), + assign sched_csr_if.cycles = cycles; + assign sched_csr_if.instret = instret; + assign sched_csr_if.active_warps = active_warps; + assign sched_csr_if.thread_mask = csr_rd[0].thread_mask; + assign sched_csr_if.mscratch = csr_rd[0].mscratch; + assign sched_csr_if.csr_mstatus = csr_rd[0].mstatus; + assign sched_csr_if.csr_mtvec = csr_rd[0].mtvec; + assign sched_csr_if.csr_mepc = csr_rd[0].mepc; + assign sched_csr_if.csr_mcause = csr_rd[0].mcause; + assign sched_csr_if.csr_mtval = csr_rd[0].mtval; + assign sched_csr_if.cta_csrs.cta_id = sched_csr_if.csr_rd_cta_id; + assign sched_csr_if.cta_csrs.cta_rank = csr_rd[0].cta_rank; + assign sched_csr_if.cta_csrs.cta_size = csr_rd_ctx.cta_size; + assign sched_csr_if.cta_csrs.block_idx = csr_rd_ctx.block_idx; + assign sched_csr_if.cta_csrs.block_dim = csr_rd_ctx.block_dim; + assign sched_csr_if.cta_csrs.grid_dim = csr_rd_ctx.grid_dim; + assign sched_csr_if.cta_csrs.entry = csr_rd_ctx.entry; + assign sched_csr_if.cta_csrs.param = csr_rd_ctx.param; + assign sched_csr_if.cta_csrs.lmem_addr = csr_rd_ctx.lmem_addr; + assign sched_csr_if.cta_csrs.cluster_size = csr_rd_ctx.cluster_size; + assign sched_csr_if.cta_lane = csr_rd[0].cta_lane; +`ifdef VX_CFG_VM_ENABLE + assign sched_csr_if.csr_satp = satp; + assign satp_wr_valid = sched_csr_if.satp_wr_valid; + assign satp_wr_data = sched_csr_if.satp_wr_data; +`endif + +`ifdef VX_CFG_EXT_RTU_ENABLE + assign rtu_unlock_valid[0] = sched_unlock_if.valid; + assign rtu_unlock_wid[0] = sched_unlock_if.wid; +`endif + + VX_backend_slice #( + .INSTANCE_ID (INSTANCE_ID), .SLICE_STR (SLICE_STR) - ) execute_slice ( - `SCOPE_IO_BIND (1) + ) backend_slice ( + `SCOPE_IO_BIND (1) .clk (clk), .reset (reset), .core_id (core_id), .slice_id (slice_id), - .dispatch_if (dispatch_if), - .commit_if (commit_if), + `ifdef PERF_ENABLE + .issue_perf (issue_perf), + `endif + .decode_if (decode_if), + .warp_issued (warp_issued), + .warp_issued_wis (warp_issued_wis), + .committed_wis (committed_wis), .branch_ctl_if (branch_ctl_if), + .warp_ctl_if (warp_ctl_if), + .sched_csr_if (sched_csr_if), + `ifdef VX_CFG_EXT_RTU_ENABLE + .sched_unlock_if (sched_unlock_if), + `endif .lsu_client_if (lsu_client_if), `ifdef VX_CFG_EXT_DXA_ENABLE .dxa_req_bus_if (dxa_req_bus_if), @@ -150,7 +260,6 @@ import VX_tcu_pkg::*; `endif `ifdef VX_CFG_EXT_RTU_ENABLE .rtu_bus_if (rtu_bus_if), - .sched_unlock_if (sched_unlock_if), `endif `ifdef VX_CFG_TCU_WGMMA_ENABLE .tcu_exec_valid (tcu_exec_valid), @@ -172,21 +281,7 @@ import VX_tcu_pkg::*; .tcu_meta_wr_idx (tcu_meta_wr_idx), .tcu_meta_wr_data (tcu_meta_wr_data), `endif - .warp_shared_if (warp_shared_if), - .warp_ctl_if (warp_ctl_if), - .sched_csr_if (sched_csr_if) - ); - - // commit - - VX_commit_slice #( - .INSTANCE_ID (`SFORMATF(("%s-commit", INSTANCE_ID))) - ) commit_slice ( - .clk (clk), - .reset (reset), - .commit_if (commit_if), - .writeback_if (writeback_if), - .committed_wis (committed_wis) + .warp_shared_if (warp_shared_if) ); endmodule diff --git a/hw/rtl/core/VX_core.sv b/hw/rtl/core/VX_core.sv index b8aaf1d528..4b1b9c6e42 100644 --- a/hw/rtl/core/VX_core.sv +++ b/hw/rtl/core/VX_core.sv @@ -94,7 +94,7 @@ import VX_tcu_pkg::*; VX_lsu_sched_if tcu_mem_if(); `endif - // fetch-slice icache requests; port 0 also carries the injected flush + // frontend-slice icache requests; port 0 also carries the injected flush VX_mem_bus_if #( .DATA_SIZE (ICACHE_WORD_SIZE), .TAG_WIDTH (ICACHE_FETCH_TAG_WIDTH) @@ -200,7 +200,7 @@ import VX_tcu_pkg::*; .cache_bus_if (icache_bus_if[0]) ); - // The other ports carry their fetch slice's registered requests, with the + // The other ports carry their frontend slice's registered requests, with the // flush arbiter's select bit, the tag's LSB, clear. for (genvar i = 1; i < ICACHE_NUM_REQS; ++i) begin : g_icache_ports assign icache_bus_if[i].req_valid = fetch_icache_if[i].req_valid; @@ -219,25 +219,12 @@ import VX_tcu_pkg::*; `UNUSED_VAR (icache_bus_if[i].rsp_data.tag[0]) end - // scheduling and the front end: the core scheduler, FETCH_WIDTH fetch slices - // and the crossbar to the issue slices and ALU/SFU blocks + // scheduling: the core scheduler and its per-frontend-slice interfaces - localparam FW = `VX_CFG_FETCH_WIDTH; - localparam NPA = FETCH_ALU_PORTS; - localparam NPS = FETCH_SFU_PORTS; + localparam FW = `VX_CFG_FETCH_WIDTH; VX_slice_sched_if slice_sched_if[FW](); - VX_decode_if fs_decode_if[FW](); - VX_decode_if is_decode_if[`VX_CFG_ISSUE_WIDTH](); - VX_branch_ctl_if branch_ctl_if[`VX_CFG_NUM_ALU_BLOCKS](); - VX_warp_ctl_if warp_ctl_if[`VX_CFG_NUM_SFU_BLOCKS](); - VX_sched_csr_if sched_csr_if[`VX_CFG_NUM_SFU_BLOCKS](); -`ifdef VX_CFG_EXT_RTU_ENABLE - VX_sched_unlock_if sched_unlock_if[`VX_CFG_NUM_SFU_BLOCKS](); -`endif - wire [`VX_CFG_ISSUE_WIDTH-1:0] issue_valid; - wire [`VX_CFG_ISSUE_WIDTH-1:0][ISSUE_WIS_W-1:0] issue_wis; wire [`VX_CFG_NUM_WARPS-1:0] committed_warps; wire [`VX_CFG_NUM_WARPS-1:0] sched_active_warps; `ifdef VX_CFG_VM_ENABLE @@ -246,27 +233,8 @@ import VX_tcu_pkg::*; wire [`VX_CFG_NUM_SFU_BLOCKS-1:0][`VX_CFG_XLEN-1:0] satp_wr_data; `endif - sched_branch_t [FW-1:0][NPA-1:0] fs_branch; - sched_wctl_t [FW-1:0][NPS-1:0] fs_wctl; - wire [FW-1:0][NPS-1:0][NW_WIDTH-1:0] fs_dvstack_wid; - wire [FW-1:0][NPS-1:0][DV_STACK_SIZEW-1:0] fs_dvstack_ptr; - wire [FW-1:0][NPS-1:0][`VX_CFG_MEM_ADDR_WIDTH-1:0] fs_dvstack_mscratch; - wire [FW-1:0][NPS-1:0][NW_WIDTH-1:0] fs_csr_rd_wid; - sched_csr_rd_t [FW-1:0][NPS-1:0] fs_csr_rd; - sched_csr_wr_t [FW-1:0][NPS-1:0] fs_csr_wr; -`ifdef VX_CFG_EXT_RTU_ENABLE - wire [FW-1:0][NPS-1:0] fs_rtu_unlock_valid; - wire [FW-1:0][NPS-1:0][NW_WIDTH-1:0] fs_rtu_unlock_wid; -`endif - wire [FW-1:0][PER_FETCH_WARPS-1:0] fs_issued; - wire [FW-1:0][PER_FETCH_WARPS-1:0] fs_committed; - wire [FW-1:0][PER_FETCH_WARPS-1:0] fs_pending_alm_empty; - wire [`VX_CFG_NUM_SFU_BLOCKS-1:0][NCTA_WIDTH-1:0] csr_rd_cta_id; cta_ctx_t [`VX_CFG_NUM_SFU_BLOCKS-1:0] csr_rd_ctx; - for (genvar b = 0; b < `VX_CFG_NUM_SFU_BLOCKS; ++b) begin : g_csr_rd_cta_id - assign csr_rd_cta_id[b] = sched_csr_if[b].csr_rd_cta_id; - end VX_scheduler #( .INSTANCE_ID (`SFORMATF(("%s-scheduler", INSTANCE_ID))) @@ -298,105 +266,103 @@ import VX_tcu_pkg::*; .busy (sched_busy) ); - VX_fetch_xbar fetch_xbar ( - .fs_decode_if (fs_decode_if), - .is_decode_if (is_decode_if), - .issue_valid (issue_valid), - .issue_wis (issue_wis), - .committed_warps (committed_warps), - .fs_issued (fs_issued), - .fs_committed (fs_committed), - .fs_pending_alm_empty (fs_pending_alm_empty), - .branch_ctl_if (branch_ctl_if), - .warp_ctl_if (warp_ctl_if), - .sched_csr_if (sched_csr_if), - `ifdef VX_CFG_EXT_RTU_ENABLE - .sched_unlock_if (sched_unlock_if), +`ifdef PERF_ENABLE + fetch_perf_t per_slice_fetch_perf [FW]; + `PERF_COUNTER_ADD (pipeline_perf.fetch, per_slice_fetch_perf, stalls, PERF_CTR_BITS, FW, (FW > 2)) +`endif + + // core-wide services, reached through one port per SFU or TCU block + +`ifdef VX_CFG_EXT_DXA_ENABLE + VX_dxa_req_bus_if block_dxa_req_bus_if[`VX_CFG_NUM_SFU_BLOCKS](); +`endif +`ifdef VX_CFG_EXT_TEX_ENABLE + VX_tex_bus_if #( + .NUM_LANES (`VX_CFG_NUM_SFU_LANES), + .TAG_WIDTH (TEX_REQ_TAG_WIDTH) + ) block_tex_bus_if[`VX_CFG_NUM_SFU_BLOCKS](); +`endif +`ifdef VX_CFG_EXT_RTU_ENABLE + VX_rtu_bus_if #( + .NUM_LANES (`VX_CFG_NUM_SFU_LANES), + .TAG_WIDTH (RTU_REQ_TAG_WIDTH) + ) block_rtu_bus_if[`VX_CFG_NUM_SFU_BLOCKS](); +`endif +`ifdef VX_CFG_TCU_WGMMA_ENABLE + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_exec_valid, tcu_exec_ready; + tcu_execute_t tcu_exec_data [`VX_CFG_NUM_TCU_BLOCKS]; + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0][TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs1_data; + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0][TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs2_data; + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_tbuf_ready; +`endif +`ifdef TCU_META_ENABLE + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_agu_ld_valid, tcu_agu_ld_ready; + tcu_execute_t tcu_agu_ld_data [`VX_CFG_NUM_TCU_BLOCKS]; + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_agu_result_valid, tcu_agu_result_ready; + tcu_result_t tcu_agu_result_data [`VX_CFG_NUM_TCU_BLOCKS]; + wire tcu_meta_wr_en; + wire [NW_WIDTH-1:0] tcu_meta_wr_wid; + wire [4:0] tcu_meta_wr_idx; + wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] tcu_meta_wr_data; +`endif + + VX_sfu_shared #( + .INSTANCE_ID (`SFORMATF(("%s-sfu-shared", INSTANCE_ID))) + ) sfu_shared ( + `ifdef VX_CFG_EXT_DXA_ENABLE + .block_dxa_req_bus_if (block_dxa_req_bus_if), + .dxa_req_bus_if (dxa_req_bus_if), `endif - .cycles (sched_cycles), - .instret (sched_instret), - .active_warps (sched_active_warps), - .lsu_sched_drained (lsu_drained), - .csr_rd_ctx (csr_rd_ctx), - `ifdef VX_CFG_VM_ENABLE - .satp (sched_satp), - .satp_wr_valid (satp_wr_valid), - .satp_wr_data (satp_wr_data), + `ifdef VX_CFG_EXT_TEX_ENABLE + .block_tex_bus_if (block_tex_bus_if), + .tex_bus_if (tex_bus_if), `endif - .fs_branch (fs_branch), - .fs_wctl (fs_wctl), - .fs_dvstack_wid (fs_dvstack_wid), - .fs_dvstack_ptr (fs_dvstack_ptr), - .fs_dvstack_mscratch (fs_dvstack_mscratch), - .fs_csr_rd_wid (fs_csr_rd_wid), - .fs_csr_rd (fs_csr_rd), `ifdef VX_CFG_EXT_RTU_ENABLE - .fs_rtu_unlock_valid (fs_rtu_unlock_valid), - .fs_rtu_unlock_wid (fs_rtu_unlock_wid), + .block_rtu_bus_if (block_rtu_bus_if), + .rtu_bus_if (rtu_bus_if), `endif - .fs_csr_wr (fs_csr_wr) + .clk (clk), + .reset (reset) ); -`ifdef PERF_ENABLE - fetch_perf_t per_slice_fetch_perf [FW]; - `PERF_COUNTER_ADD (pipeline_perf.fetch, per_slice_fetch_perf, stalls, PERF_CTR_BITS, FW, (FW > 2)) -`endif - -`ifdef SCOPE - wire [FW-1:0] fetch_scope_bus_in, fetch_scope_bus_out; - VX_scope_switch #( - .N (FW) - ) fetch_scope_switch ( - .clk (clk), - .reset (scope_reset_w[0]), - .req_in (scope_bus_in_w[0]), - .rsp_out (scope_bus_out_w[0]), - .req_out (fetch_scope_bus_in), - .rsp_in (fetch_scope_bus_out) +`ifdef VX_CFG_EXT_TCU_ENABLE + VX_tcu_shared #( + .INSTANCE_ID (`SFORMATF(("%s-tcu-shared", INSTANCE_ID))), + .BLOCK_SIZE (`VX_CFG_NUM_TCU_BLOCKS) + ) tcu_shared ( + `ifdef PERF_ENABLE + .tcu_perf (tcu_perf), + `endif + `ifdef VX_CFG_TCU_WGMMA_ENABLE + .exec_valid (tcu_exec_valid), + .exec_ready (tcu_exec_ready), + .exec_data (tcu_exec_data), + .tbuf_rs1_data (tcu_tbuf_rs1_data), + .tbuf_rs2_data (tcu_tbuf_rs2_data), + .tbuf_ready_eff (tcu_tbuf_ready), + .tcu_lmem_if (tcu_lmem_if), + `endif + `ifdef TCU_META_ENABLE + .agu_ld_valid (tcu_agu_ld_valid), + .agu_ld_data (tcu_agu_ld_data), + .agu_ld_ready (tcu_agu_ld_ready), + .agu_result_valid (tcu_agu_result_valid), + .agu_result_data (tcu_agu_result_data), + .agu_result_ready (tcu_agu_result_ready), + .meta_wr_en (tcu_meta_wr_en), + .meta_wr_wid (tcu_meta_wr_wid), + .meta_wr_idx (tcu_meta_wr_idx), + .meta_wr_data (tcu_meta_wr_data), + .tcu_mem_if (tcu_mem_if), + `endif + .clk (clk), + .reset (reset) ); `endif - for (genvar f = 0; f < FW; ++f) begin : g_fetch_slices - VX_fetch_slice #( - .INSTANCE_ID (INSTANCE_ID), - .SLICE_STR ((FW > 1) ? `SFORMATF(("%0d", f)) : "") - ) fetch_slice ( - `ifdef SCOPE - .scope_reset (scope_reset_w[0]), - .scope_bus_in (fetch_scope_bus_in[f]), - .scope_bus_out (fetch_scope_bus_out[f]), - `endif - .clk (clk), - .reset (reset), - .core_id (core_id), - .slice_id (FETCH_FSW_W'(f)), - `ifdef PERF_ENABLE - .fetch_perf (per_slice_fetch_perf[f]), - `endif - .slice_sched_if (slice_sched_if[f]), - .icache_bus_if (fetch_icache_if[f]), - .decode_if (fs_decode_if[f]), - .branch (fs_branch[f]), - .wctl (fs_wctl[f]), - .dvstack_wid (fs_dvstack_wid[f]), - .dvstack_ptr (fs_dvstack_ptr[f]), - .dvstack_mscratch (fs_dvstack_mscratch[f]), - .csr_rd_wid (fs_csr_rd_wid[f]), - .csr_rd (fs_csr_rd[f]), - .csr_wr (fs_csr_wr[f]), - `ifdef VX_CFG_EXT_RTU_ENABLE - .rtu_unlock_valid (fs_rtu_unlock_valid[f]), - .rtu_unlock_wid (fs_rtu_unlock_wid[f]), - `endif - .issued (fs_issued[f]), - .committed (fs_committed[f]), - .pending_alm_empty (fs_pending_alm_empty[f]) - ); - end - - // issue, execute and commit + // fetch, issue, execute and commit, structured by COMPUTE_SLICES / BACKEND_SLICES - if (COMPUTE_SLICES_ENABLE) begin : g_compute_slices + if (COMPUTE_SLICES) begin : g_compute_slices `ifdef PERF_ENABLE issue_perf_t per_slice_issue_perf [`VX_CFG_ISSUE_WIDTH]; @@ -409,40 +375,6 @@ import VX_tcu_pkg::*; end `endif - `ifdef VX_CFG_EXT_DXA_ENABLE - VX_dxa_req_bus_if slice_dxa_req_bus_if[`VX_CFG_ISSUE_WIDTH](); - `endif - `ifdef VX_CFG_EXT_TEX_ENABLE - VX_tex_bus_if #( - .NUM_LANES (`VX_CFG_NUM_SFU_LANES), - .TAG_WIDTH (TEX_REQ_TAG_WIDTH) - ) slice_tex_bus_if[`VX_CFG_ISSUE_WIDTH](); - `endif - `ifdef VX_CFG_EXT_RTU_ENABLE - VX_rtu_bus_if #( - .NUM_LANES (`VX_CFG_NUM_SFU_LANES), - .TAG_WIDTH (RTU_REQ_TAG_WIDTH) - ) slice_rtu_bus_if[`VX_CFG_ISSUE_WIDTH](); - `endif - - `ifdef VX_CFG_TCU_WGMMA_ENABLE - wire [`VX_CFG_ISSUE_WIDTH-1:0] tcu_exec_valid, tcu_exec_ready; - tcu_execute_t tcu_exec_data [`VX_CFG_ISSUE_WIDTH]; - wire [`VX_CFG_ISSUE_WIDTH-1:0][TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs1_data; - wire [`VX_CFG_ISSUE_WIDTH-1:0][TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs2_data; - wire [`VX_CFG_ISSUE_WIDTH-1:0] tcu_tbuf_ready; - `endif - `ifdef TCU_META_ENABLE - wire [`VX_CFG_ISSUE_WIDTH-1:0] tcu_agu_ld_valid, tcu_agu_ld_ready; - tcu_execute_t tcu_agu_ld_data [`VX_CFG_ISSUE_WIDTH]; - wire [`VX_CFG_ISSUE_WIDTH-1:0] tcu_agu_result_valid, tcu_agu_result_ready; - tcu_result_t tcu_agu_result_data [`VX_CFG_ISSUE_WIDTH]; - wire tcu_meta_wr_en; - wire [NW_WIDTH-1:0] tcu_meta_wr_wid; - wire [4:0] tcu_meta_wr_idx; - wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] tcu_meta_wr_data; - `endif - `ifdef SCOPE wire [`VX_CFG_ISSUE_WIDTH-1:0] slice_scope_bus_in, slice_scope_bus_out; VX_scope_switch #( @@ -456,6 +388,7 @@ import VX_tcu_pkg::*; .rsp_in (slice_scope_bus_out) ); `endif + `SCOPE_IO_UNUSED (0) `SCOPE_IO_UNUSED (2) for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_slices @@ -475,27 +408,32 @@ import VX_tcu_pkg::*; .core_id (core_id), .slice_id (ISSUE_ISW_W'(i)), `ifdef PERF_ENABLE + .fetch_perf (per_slice_fetch_perf[i]), .issue_perf (per_slice_issue_perf[i]), `endif - .decode_if (is_decode_if[i]), - .warp_issued (issue_valid[i]), - .warp_issued_wis (issue_wis[i]), + .slice_sched_if (slice_sched_if[i]), + .icache_bus_if (fetch_icache_if[i]), .committed_wis (committed_wis), - .branch_ctl_if (branch_ctl_if[i]), - .warp_ctl_if (warp_ctl_if[i]), - .sched_csr_if (sched_csr_if[i]), - `ifdef VX_CFG_EXT_RTU_ENABLE - .sched_unlock_if (sched_unlock_if[i]), + .cycles (sched_cycles), + .instret (sched_instret), + .active_warps (sched_active_warps), + .lsu_sched_drained (lsu_drained), + .csr_rd_cta_id (csr_rd_cta_id[i]), + .csr_rd_ctx (csr_rd_ctx[i]), + `ifdef VX_CFG_VM_ENABLE + .satp (sched_satp[i]), + .satp_wr_valid (satp_wr_valid[i]), + .satp_wr_data (satp_wr_data[i]), `endif .lsu_client_if (lsu_client_if[i]), `ifdef VX_CFG_EXT_DXA_ENABLE - .dxa_req_bus_if (slice_dxa_req_bus_if[i]), + .dxa_req_bus_if (block_dxa_req_bus_if[i]), `endif `ifdef VX_CFG_EXT_TEX_ENABLE - .tex_bus_if (slice_tex_bus_if[i]), + .tex_bus_if (block_tex_bus_if[i]), `endif `ifdef VX_CFG_EXT_RTU_ENABLE - .rtu_bus_if (slice_rtu_bus_if[i]), + .rtu_bus_if (block_rtu_bus_if[i]), `endif `ifdef VX_CFG_TCU_WGMMA_ENABLE .tcu_exec_valid (tcu_exec_valid[i]), @@ -526,155 +464,331 @@ import VX_tcu_pkg::*; end end - VX_sfu_shared #( - .INSTANCE_ID (`SFORMATF(("%s-execute-sfu-shared", INSTANCE_ID))) - ) sfu_shared ( - `ifdef VX_CFG_EXT_DXA_ENABLE - .block_dxa_req_bus_if (slice_dxa_req_bus_if), - .dxa_req_bus_if (dxa_req_bus_if), + end else begin : g_crossbar + + VX_decode_if fs_decode_if[FW](); + VX_decode_if is_decode_if[`VX_CFG_ISSUE_WIDTH](); + VX_branch_ctl_if branch_ctl_if[`VX_CFG_NUM_ALU_BLOCKS](); + VX_warp_ctl_if warp_ctl_if[`VX_CFG_NUM_SFU_BLOCKS](); + VX_sched_csr_if sched_csr_if[`VX_CFG_NUM_SFU_BLOCKS](); + for (genvar b = 0; b < `VX_CFG_NUM_SFU_BLOCKS; ++b) begin : g_csr_rd_cta_id + assign csr_rd_cta_id[b] = sched_csr_if[b].csr_rd_cta_id; + end + `ifdef VX_CFG_EXT_RTU_ENABLE + VX_sched_unlock_if sched_unlock_if[`VX_CFG_NUM_SFU_BLOCKS](); + `endif + wire [`VX_CFG_ISSUE_WIDTH-1:0] issue_valid; + wire [`VX_CFG_ISSUE_WIDTH-1:0][ISSUE_WIS_W-1:0] issue_wis; + + localparam NPA = FETCH_ALU_PORTS; + localparam NPS = FETCH_SFU_PORTS; + + sched_branch_t [FW-1:0][NPA-1:0] fs_branch; + sched_wctl_t [FW-1:0][NPS-1:0] fs_wctl; + wire [FW-1:0][NPS-1:0][NW_WIDTH-1:0] fs_dvstack_wid; + wire [FW-1:0][NPS-1:0][DV_STACK_SIZEW-1:0] fs_dvstack_ptr; + wire [FW-1:0][NPS-1:0][`VX_CFG_MEM_ADDR_WIDTH-1:0] fs_dvstack_mscratch; + wire [FW-1:0][NPS-1:0][NW_WIDTH-1:0] fs_csr_rd_wid; + sched_csr_rd_t [FW-1:0][NPS-1:0] fs_csr_rd; + sched_csr_wr_t [FW-1:0][NPS-1:0] fs_csr_wr; + `ifdef VX_CFG_EXT_RTU_ENABLE + wire [FW-1:0][NPS-1:0] fs_rtu_unlock_valid; + wire [FW-1:0][NPS-1:0][NW_WIDTH-1:0] fs_rtu_unlock_wid; + `endif + wire [FW-1:0][PER_FETCH_WARPS-1:0] fs_issued; + wire [FW-1:0][PER_FETCH_WARPS-1:0] fs_committed; + wire [FW-1:0][PER_FETCH_WARPS-1:0] fs_pending_alm_empty; + + VX_frontend_xbar frontend_xbar ( + .fs_decode_if (fs_decode_if), + .is_decode_if (is_decode_if), + .issue_valid (issue_valid), + .issue_wis (issue_wis), + .committed_warps (committed_warps), + .fs_issued (fs_issued), + .fs_committed (fs_committed), + .fs_pending_alm_empty (fs_pending_alm_empty), + .branch_ctl_if (branch_ctl_if), + .warp_ctl_if (warp_ctl_if), + .sched_csr_if (sched_csr_if), + `ifdef VX_CFG_EXT_RTU_ENABLE + .sched_unlock_if (sched_unlock_if), `endif - `ifdef VX_CFG_EXT_TEX_ENABLE - .block_tex_bus_if (slice_tex_bus_if), - .tex_bus_if (tex_bus_if), + .cycles (sched_cycles), + .instret (sched_instret), + .active_warps (sched_active_warps), + .lsu_sched_drained (lsu_drained), + .csr_rd_ctx (csr_rd_ctx), + `ifdef VX_CFG_VM_ENABLE + .satp (sched_satp), + .satp_wr_valid (satp_wr_valid), + .satp_wr_data (satp_wr_data), `endif + .fs_branch (fs_branch), + .fs_wctl (fs_wctl), + .fs_dvstack_wid (fs_dvstack_wid), + .fs_dvstack_ptr (fs_dvstack_ptr), + .fs_dvstack_mscratch (fs_dvstack_mscratch), + .fs_csr_rd_wid (fs_csr_rd_wid), + .fs_csr_rd (fs_csr_rd), `ifdef VX_CFG_EXT_RTU_ENABLE - .block_rtu_bus_if (slice_rtu_bus_if), - .rtu_bus_if (rtu_bus_if), + .fs_rtu_unlock_valid (fs_rtu_unlock_valid), + .fs_rtu_unlock_wid (fs_rtu_unlock_wid), `endif - .clk (clk), - .reset (reset) + .fs_csr_wr (fs_csr_wr) ); - `ifdef VX_CFG_EXT_TCU_ENABLE - VX_tcu_shared #( - .INSTANCE_ID (`SFORMATF(("%s-execute-tcu", INSTANCE_ID))), - .BLOCK_SIZE (`VX_CFG_ISSUE_WIDTH) - ) tcu_shared ( - `ifdef PERF_ENABLE - .tcu_perf (tcu_perf), - `endif - `ifdef VX_CFG_TCU_WGMMA_ENABLE - .exec_valid (tcu_exec_valid), - .exec_ready (tcu_exec_ready), - .exec_data (tcu_exec_data), - .tbuf_rs1_data (tcu_tbuf_rs1_data), - .tbuf_rs2_data (tcu_tbuf_rs2_data), - .tbuf_ready_eff (tcu_tbuf_ready), - .tcu_lmem_if (tcu_lmem_if), - `endif - `ifdef TCU_META_ENABLE - .agu_ld_valid (tcu_agu_ld_valid), - .agu_ld_data (tcu_agu_ld_data), - .agu_ld_ready (tcu_agu_ld_ready), - .agu_result_valid (tcu_agu_result_valid), - .agu_result_data (tcu_agu_result_data), - .agu_result_ready (tcu_agu_result_ready), - .meta_wr_en (tcu_meta_wr_en), - .meta_wr_wid (tcu_meta_wr_wid), - .meta_wr_idx (tcu_meta_wr_idx), - .meta_wr_data (tcu_meta_wr_data), - .tcu_mem_if (tcu_mem_if), - `endif - .clk (clk), - .reset (reset) + `ifdef SCOPE + wire [FW-1:0] frontend_scope_bus_in, frontend_scope_bus_out; + VX_scope_switch #( + .N (FW) + ) frontend_scope_switch ( + .clk (clk), + .reset (scope_reset_w[0]), + .req_in (scope_bus_in_w[0]), + .rsp_out (scope_bus_out_w[0]), + .req_out (frontend_scope_bus_in), + .rsp_in (frontend_scope_bus_out) ); `endif - end else begin : g_stage_sliced + for (genvar f = 0; f < FW; ++f) begin : g_frontend_slices + VX_frontend_slice #( + .INSTANCE_ID (INSTANCE_ID), + .SLICE_STR ((FW > 1) ? `SFORMATF(("%0d", f)) : "") + ) frontend_slice ( + `ifdef SCOPE + .scope_reset (scope_reset_w[0]), + .scope_bus_in (frontend_scope_bus_in[f]), + .scope_bus_out (frontend_scope_bus_out[f]), + `endif + .clk (clk), + .reset (reset), + .core_id (core_id), + .slice_id (FETCH_FSW_W'(f)), + `ifdef PERF_ENABLE + .fetch_perf (per_slice_fetch_perf[f]), + `endif + .slice_sched_if (slice_sched_if[f]), + .icache_bus_if (fetch_icache_if[f]), + .decode_if (fs_decode_if[f]), + .branch (fs_branch[f]), + .wctl (fs_wctl[f]), + .dvstack_wid (fs_dvstack_wid[f]), + .dvstack_ptr (fs_dvstack_ptr[f]), + .dvstack_mscratch (fs_dvstack_mscratch[f]), + .csr_rd_wid (fs_csr_rd_wid[f]), + .csr_rd (fs_csr_rd[f]), + .csr_wr (fs_csr_wr[f]), + `ifdef VX_CFG_EXT_RTU_ENABLE + .rtu_unlock_valid (fs_rtu_unlock_valid[f]), + .rtu_unlock_wid (fs_rtu_unlock_wid[f]), + `endif + .issued (fs_issued[f]), + .committed (fs_committed[f]), + .pending_alm_empty (fs_pending_alm_empty[f]) + ); + end - VX_issue_sched_if issue_sched_if[`VX_CFG_ISSUE_WIDTH](); - VX_commit_sched_if commit_sched_if(); + if (BACKEND_SLICES) begin : g_backend_slices - VX_dispatch_if dispatch_if[NUM_EX_UNITS * `VX_CFG_ISSUE_WIDTH](); - VX_commit_if commit_if[NUM_EX_UNITS * `VX_CFG_ISSUE_WIDTH](); - VX_writeback_if writeback_if[`VX_CFG_ISSUE_WIDTH](); + `ifdef PERF_ENABLE + issue_perf_t per_slice_issue_perf [`VX_CFG_ISSUE_WIDTH]; + `PERF_COUNTER_ADD (pipeline_perf.issue, per_slice_issue_perf, ibf_stalls, PERF_CTR_BITS, `VX_CFG_ISSUE_WIDTH, (`VX_CFG_ISSUE_WIDTH > 2)) + `PERF_COUNTER_ADD (pipeline_perf.issue, per_slice_issue_perf, scb_stalls, PERF_CTR_BITS, `VX_CFG_ISSUE_WIDTH, (`VX_CFG_ISSUE_WIDTH > 2)) + `PERF_COUNTER_ADD (pipeline_perf.issue, per_slice_issue_perf, opd_stalls, PERF_CTR_BITS, `VX_CFG_ISSUE_WIDTH, (`VX_CFG_ISSUE_WIDTH > 2)) + for (genvar i = 0; i < NUM_EX_UNITS; ++i) begin : g_issue_perf_units_uses + `PERF_COUNTER_ADD (pipeline_perf.issue, per_slice_issue_perf, dispatch_stalls[i], PERF_CTR_BITS, `VX_CFG_ISSUE_WIDTH, (`VX_CFG_ISSUE_WIDTH > 2)) + `PERF_COUNTER_ADD (pipeline_perf.issue, per_slice_issue_perf, dispatch_instrs[i], PERF_CTR_BITS, `VX_CFG_ISSUE_WIDTH, (`VX_CFG_ISSUE_WIDTH > 2)) + end + `endif - VX_issue #( - .INSTANCE_ID (`SFORMATF(("%s-issue", INSTANCE_ID))) - ) issue ( - `SCOPE_IO_BIND (1) + `ifdef SCOPE + wire [`VX_CFG_ISSUE_WIDTH-1:0] slice_scope_bus_in, slice_scope_bus_out; + VX_scope_switch #( + .N (`VX_CFG_ISSUE_WIDTH) + ) slice_scope_switch ( + .clk (clk), + .reset (scope_reset_w[1]), + .req_in (scope_bus_in_w[1]), + .rsp_out (scope_bus_out_w[1]), + .req_out (slice_scope_bus_in), + .rsp_in (slice_scope_bus_out) + ); + `endif + `SCOPE_IO_UNUSED (2) + + for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_slices + wire [PER_ISSUE_WARPS-1:0] committed_wis; + + VX_backend_slice #( + .INSTANCE_ID (INSTANCE_ID), + .SLICE_STR (`SFORMATF(("%0d", i))) + ) backend_slice ( + `ifdef SCOPE + .scope_reset (scope_reset_w[1]), + .scope_bus_in (slice_scope_bus_in[i]), + .scope_bus_out (slice_scope_bus_out[i]), + `endif + .clk (clk), + .reset (reset), + .core_id (core_id), + .slice_id (ISSUE_ISW_W'(i)), + `ifdef PERF_ENABLE + .issue_perf (per_slice_issue_perf[i]), + `endif + .decode_if (is_decode_if[i]), + .warp_issued (issue_valid[i]), + .warp_issued_wis (issue_wis[i]), + .committed_wis (committed_wis), + .branch_ctl_if (branch_ctl_if[i]), + .warp_ctl_if (warp_ctl_if[i]), + .sched_csr_if (sched_csr_if[i]), + `ifdef VX_CFG_EXT_RTU_ENABLE + .sched_unlock_if (sched_unlock_if[i]), + `endif + .lsu_client_if (lsu_client_if[i]), + `ifdef VX_CFG_EXT_DXA_ENABLE + .dxa_req_bus_if (block_dxa_req_bus_if[i]), + `endif + `ifdef VX_CFG_EXT_TEX_ENABLE + .tex_bus_if (block_tex_bus_if[i]), + `endif + `ifdef VX_CFG_EXT_RTU_ENABLE + .rtu_bus_if (block_rtu_bus_if[i]), + `endif + `ifdef VX_CFG_TCU_WGMMA_ENABLE + .tcu_exec_valid (tcu_exec_valid[i]), + .tcu_exec_ready (tcu_exec_ready[i]), + .tcu_exec_data (tcu_exec_data[i]), + .tcu_tbuf_rs1_data (tcu_tbuf_rs1_data[i]), + .tcu_tbuf_rs2_data (tcu_tbuf_rs2_data[i]), + .tcu_tbuf_ready (tcu_tbuf_ready[i]), + `endif + `ifdef TCU_META_ENABLE + .tcu_agu_ld_valid (tcu_agu_ld_valid[i]), + .tcu_agu_ld_data (tcu_agu_ld_data[i]), + .tcu_agu_ld_ready (tcu_agu_ld_ready[i]), + .tcu_agu_result_valid (tcu_agu_result_valid[i]), + .tcu_agu_result_data (tcu_agu_result_data[i]), + .tcu_agu_result_ready (tcu_agu_result_ready[i]), + .tcu_meta_wr_en (tcu_meta_wr_en && (NW_WIDTH'(tcu_meta_wr_wid % `VX_CFG_NUM_TCU_BLOCKS) == NW_WIDTH'(i))), + .tcu_meta_wr_wid (tcu_meta_wr_wid), + .tcu_meta_wr_idx (tcu_meta_wr_idx), + .tcu_meta_wr_data (tcu_meta_wr_data), + `endif + .warp_shared_if (warp_shared_if[i]) + ); + + // wid = {wis, isw}: slice i owns every ISSUE_WIDTH-th warp starting at i + for (genvar w = 0; w < PER_ISSUE_WARPS; ++w) begin : g_committed + assign committed_warps[w * `VX_CFG_ISSUE_WIDTH + i] = committed_wis[w]; + end + end - .clk (clk), - .reset (reset), + end else begin : g_stage_sliced - `ifdef PERF_ENABLE - .issue_perf (pipeline_perf.issue), - `endif + VX_issue_sched_if issue_sched_if[`VX_CFG_ISSUE_WIDTH](); + VX_commit_sched_if commit_sched_if(); - .decode_if (is_decode_if), - .writeback_if (writeback_if), - .dispatch_if (dispatch_if), - .issue_sched_if (issue_sched_if) - ); + VX_dispatch_if dispatch_if[NUM_EX_UNITS * `VX_CFG_ISSUE_WIDTH](); + VX_commit_if commit_if[NUM_EX_UNITS * `VX_CFG_ISSUE_WIDTH](); + VX_writeback_if writeback_if[`VX_CFG_ISSUE_WIDTH](); - for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_issue_sched - assign issue_valid[i] = issue_sched_if[i].valid; - assign issue_wis[i] = issue_sched_if[i].wis; - end + VX_issue #( + .INSTANCE_ID (`SFORMATF(("%s-issue", INSTANCE_ID))) + ) issue ( + `SCOPE_IO_BIND (1) - VX_execute #( - .INSTANCE_ID (`SFORMATF(("%s-execute", INSTANCE_ID))) - ) execute ( - `SCOPE_IO_BIND (2) + .clk (clk), + .reset (reset), - .clk (clk), - .reset (reset), - .core_id (core_id), + `ifdef PERF_ENABLE + .issue_perf (pipeline_perf.issue), + `endif - `ifdef PERF_ENABLE - `ifdef VX_CFG_EXT_TCU_ENABLE - .tcu_perf (tcu_perf), - `endif - `endif + .decode_if (is_decode_if), + .writeback_if (writeback_if), + .dispatch_if (dispatch_if), + .issue_sched_if (issue_sched_if) + ); - // execute exposes LSU client interfaces; lsu_scheduler sits between - // execute and lsu_mem_if (which connects to mem_unit). - .lsu_client_if (lsu_client_if), + for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_issue_sched + assign issue_valid[i] = issue_sched_if[i].valid; + assign issue_wis[i] = issue_sched_if[i].wis; + end - `ifdef TCU_META_ENABLE - .tcu_mem_if (tcu_mem_if), - `endif + VX_execute #( + .INSTANCE_ID (`SFORMATF(("%s-execute", INSTANCE_ID))) + ) execute ( + `SCOPE_IO_BIND (2) - .dispatch_if (dispatch_if), - .commit_if (commit_if), + .clk (clk), + .reset (reset), + .core_id (core_id), - .sched_csr_if (sched_csr_if), + // execute exposes LSU client interfaces; lsu_scheduler sits between + // execute and lsu_mem_if (which connects to mem_unit). + .lsu_client_if (lsu_client_if), - `ifdef VX_CFG_TCU_WGMMA_ENABLE - .tcu_lmem_if (tcu_lmem_if), - `endif - `ifdef VX_CFG_EXT_DXA_ENABLE - .dxa_req_bus_if (dxa_req_bus_if), - `endif - `ifdef VX_CFG_EXT_TEX_ENABLE - .tex_bus_if (tex_bus_if), - `endif - `ifdef VX_CFG_EXT_RTU_ENABLE - .rtu_bus_if (rtu_bus_if), - .sched_unlock_if(sched_unlock_if), - `endif + .dispatch_if (dispatch_if), + .commit_if (commit_if), - .warp_shared_if (warp_shared_if), + .sched_csr_if (sched_csr_if), - .warp_ctl_if (warp_ctl_if), - .branch_ctl_if (branch_ctl_if) - ); + `ifdef VX_CFG_TCU_WGMMA_ENABLE + .tcu_exec_valid (tcu_exec_valid), + .tcu_exec_ready (tcu_exec_ready), + .tcu_exec_data (tcu_exec_data), + .tcu_tbuf_rs1_data (tcu_tbuf_rs1_data), + .tcu_tbuf_rs2_data (tcu_tbuf_rs2_data), + .tcu_tbuf_ready (tcu_tbuf_ready), + `endif + `ifdef TCU_META_ENABLE + .tcu_agu_ld_valid (tcu_agu_ld_valid), + .tcu_agu_ld_data (tcu_agu_ld_data), + .tcu_agu_ld_ready (tcu_agu_ld_ready), + .tcu_agu_result_valid (tcu_agu_result_valid), + .tcu_agu_result_data (tcu_agu_result_data), + .tcu_agu_result_ready (tcu_agu_result_ready), + .tcu_meta_wr_en (tcu_meta_wr_en), + .tcu_meta_wr_wid (tcu_meta_wr_wid), + .tcu_meta_wr_idx (tcu_meta_wr_idx), + .tcu_meta_wr_data (tcu_meta_wr_data), + `endif + `ifdef VX_CFG_EXT_DXA_ENABLE + .dxa_req_bus_if (block_dxa_req_bus_if), + `endif + `ifdef VX_CFG_EXT_TEX_ENABLE + .tex_bus_if (block_tex_bus_if), + `endif + `ifdef VX_CFG_EXT_RTU_ENABLE + .rtu_bus_if (block_rtu_bus_if), + .sched_unlock_if (sched_unlock_if), + `endif - VX_commit #( - .INSTANCE_ID (`SFORMATF(("%s-commit", INSTANCE_ID))) - ) commit ( - .clk (clk), - .reset (reset), + .warp_shared_if (warp_shared_if), - .commit_if (commit_if), + .warp_ctl_if (warp_ctl_if), + .branch_ctl_if (branch_ctl_if) + ); - .writeback_if (writeback_if), + VX_commit #( + .INSTANCE_ID (`SFORMATF(("%s-commit", INSTANCE_ID))) + ) commit ( + .clk (clk), + .reset (reset), - .commit_sched_if(commit_sched_if) - ); + .commit_if (commit_if), - assign committed_warps = commit_sched_if.committed_warps; + .writeback_if (writeback_if), + .commit_sched_if(commit_sched_if) + ); + + assign committed_warps = commit_sched_if.committed_warps; + + end end // One memory scheduler per LSU block. With the TCU metadata AGU, every block diff --git a/hw/rtl/core/VX_execute.sv b/hw/rtl/core/VX_execute.sv index 7454c4d92e..94e33c191a 100644 --- a/hw/rtl/core/VX_execute.sv +++ b/hw/rtl/core/VX_execute.sv @@ -13,7 +13,11 @@ `include "VX_define.vh" -module VX_execute import VX_gpu_pkg::*; #( +module VX_execute import VX_gpu_pkg::*; +`ifdef VX_CFG_EXT_TCU_ENABLE +import VX_tcu_pkg::*; +`endif +#( parameter `STRING INSTANCE_ID = "" ) ( `SCOPE_IO_DECL @@ -23,35 +27,24 @@ module VX_execute import VX_gpu_pkg::*; #( input wire [CORE_ID_WIDTH-1:0] core_id, -`ifdef PERF_ENABLE -`ifdef VX_CFG_EXT_TCU_ENABLE - output tcu_perf_t tcu_perf, -`endif -`endif - // Per-block LSU client connections to VX_lsu_scheduler (at VX_core). // The dcache port (lsu_mem_if) lives at VX_core; execute exposes // only the LSU client interfaces. VX_lsu_sched_if.master lsu_client_if [`VX_CFG_NUM_LSU_BLOCKS], -`ifdef TCU_META_ENABLE - // TCU AGU memory client (single warp-level AGU shared across blocks). - // VX_core wires this to client 1 of block 0's lsu_scheduler. - VX_lsu_sched_if.master tcu_mem_if, -`endif - // dispatch interface VX_dispatch_if.slave dispatch_if [NUM_EX_UNITS * `VX_CFG_ISSUE_WIDTH], // commit interface VX_commit_if.master commit_if [NUM_EX_UNITS * `VX_CFG_ISSUE_WIDTH], + // per-block ports of the core-wide SFU and TCU services `ifdef VX_CFG_EXT_DXA_ENABLE - VX_dxa_req_bus_if.master dxa_req_bus_if, + VX_dxa_req_bus_if.master dxa_req_bus_if [`VX_CFG_NUM_SFU_BLOCKS], `endif `ifdef VX_CFG_EXT_TEX_ENABLE - VX_tex_bus_if.master tex_bus_if, + VX_tex_bus_if.master tex_bus_if [`VX_CFG_NUM_SFU_BLOCKS], `endif `ifdef VX_CFG_EXT_OM_ENABLE @@ -62,13 +55,29 @@ module VX_execute import VX_gpu_pkg::*; #( `endif `ifdef VX_CFG_EXT_RTU_ENABLE - VX_rtu_bus_if.master rtu_bus_if, + VX_rtu_bus_if.master rtu_bus_if [`VX_CFG_NUM_SFU_BLOCKS], VX_sched_unlock_if.master sched_unlock_if [`VX_CFG_NUM_SFU_BLOCKS], `endif `ifdef VX_CFG_TCU_WGMMA_ENABLE - // TCU tile-buffer local-memory read port - VX_mem_bus_if.master tcu_lmem_if, + output wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_exec_valid, + output wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_exec_ready, + output tcu_execute_t tcu_exec_data [`VX_CFG_NUM_TCU_BLOCKS], + input wire [`VX_CFG_NUM_TCU_BLOCKS-1:0][TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs1_data, + input wire [`VX_CFG_NUM_TCU_BLOCKS-1:0][TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs2_data, + input wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_tbuf_ready, +`endif +`ifdef TCU_META_ENABLE + output wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_agu_ld_valid, + output tcu_execute_t tcu_agu_ld_data [`VX_CFG_NUM_TCU_BLOCKS], + input wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_agu_ld_ready, + input wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_agu_result_valid, + input tcu_result_t tcu_agu_result_data [`VX_CFG_NUM_TCU_BLOCKS], + output wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_agu_result_ready, + input wire tcu_meta_wr_en, + input wire [NW_WIDTH-1:0] tcu_meta_wr_wid, + input wire [4:0] tcu_meta_wr_idx, + input wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] tcu_meta_wr_data, `endif // scheduler interfaces @@ -126,19 +135,30 @@ module VX_execute import VX_gpu_pkg::*; #( .INSTANCE_ID (`SFORMATF(("%s-tcu", INSTANCE_ID))) ) tcu_unit ( `SCOPE_IO_BIND (1) - .clk (clk), - .reset (reset), - `ifdef PERF_ENABLE - .tcu_perf (tcu_perf), - `endif + .clk (clk), + .reset (reset), `ifdef VX_CFG_TCU_WGMMA_ENABLE - .tcu_lmem_if (tcu_lmem_if), + .exec_valid (tcu_exec_valid), + .exec_ready (tcu_exec_ready), + .exec_data (tcu_exec_data), + .tbuf_rs1_data (tcu_tbuf_rs1_data), + .tbuf_rs2_data (tcu_tbuf_rs2_data), + .tbuf_ready (tcu_tbuf_ready), `endif `ifdef TCU_META_ENABLE - .tcu_mem_if (tcu_mem_if), + .agu_ld_valid (tcu_agu_ld_valid), + .agu_ld_data (tcu_agu_ld_data), + .agu_ld_ready (tcu_agu_ld_ready), + .agu_result_valid (tcu_agu_result_valid), + .agu_result_data (tcu_agu_result_data), + .agu_result_ready (tcu_agu_result_ready), + .meta_wr_en (tcu_meta_wr_en), + .meta_wr_wid (tcu_meta_wr_wid), + .meta_wr_idx (tcu_meta_wr_idx), + .meta_wr_data (tcu_meta_wr_data), `endif - .dispatch_if (dispatch_if[EX_TCU * `VX_CFG_ISSUE_WIDTH +: `VX_CFG_ISSUE_WIDTH]), - .commit_if (commit_if[EX_TCU * `VX_CFG_ISSUE_WIDTH +: `VX_CFG_ISSUE_WIDTH]) + .dispatch_if (dispatch_if[EX_TCU * `VX_CFG_ISSUE_WIDTH +: `VX_CFG_ISSUE_WIDTH]), + .commit_if (commit_if[EX_TCU * `VX_CFG_ISSUE_WIDTH +: `VX_CFG_ISSUE_WIDTH]) ); `endif diff --git a/hw/rtl/core/VX_fetch_slice.sv b/hw/rtl/core/VX_frontend_slice.sv similarity index 97% rename from hw/rtl/core/VX_fetch_slice.sv rename to hw/rtl/core/VX_frontend_slice.sv index 41d788c064..9bc2eed602 100644 --- a/hw/rtl/core/VX_fetch_slice.sv +++ b/hw/rtl/core/VX_frontend_slice.sv @@ -13,11 +13,11 @@ `include "VX_define.vh" -// One fetch slice: warp scheduling, fetch and decode of the warps with +// One frontend slice: warp scheduling, fetch and decode of the warps with // wid mod FETCH_WIDTH == slice_id, on the slice's own icache port. The front end // names the slice's warps by wfs; decoded instructions leave with their // core-wide wid, and ibuffer pops return indexed by wfs. -module VX_fetch_slice import VX_gpu_pkg::*; #( +module VX_frontend_slice import VX_gpu_pkg::*; #( parameter `STRING INSTANCE_ID = "", parameter `STRING SLICE_STR = "" ) ( diff --git a/hw/rtl/core/VX_fetch_xbar.sv b/hw/rtl/core/VX_frontend_xbar.sv similarity index 95% rename from hw/rtl/core/VX_fetch_xbar.sv rename to hw/rtl/core/VX_frontend_xbar.sv index 85c7babd07..8f4d7cf3c7 100644 --- a/hw/rtl/core/VX_fetch_xbar.sv +++ b/hw/rtl/core/VX_frontend_xbar.sv @@ -13,12 +13,12 @@ `include "VX_define.vh" -// Connects the fetch slices (fetch slice f owns wid mod FETCH_WIDTH == f) to the +// Connects the frontend slices (frontend slice f owns wid mod FETCH_WIDTH == f) to the // issue slices (wid mod ISSUE_WIDTH) and the ALU/SFU blocks (wid mod B). Issue -// slice i takes its instructions from fetch slice i mod FETCH_WIDTH. A block -// serving several fetch slices sends each its own warps' events and answers its +// slice i takes its instructions from frontend slice i mod FETCH_WIDTH. A block +// serving several frontend slices sends each its own warps' events and answers its // reads from the slice owning the read wid. Purely combinational. -module VX_fetch_xbar import VX_gpu_pkg::*; ( +module VX_frontend_xbar import VX_gpu_pkg::*; ( // instruction path VX_decode_if.slave fs_decode_if [`VX_CFG_FETCH_WIDTH], VX_decode_if.master is_decode_if [`VX_CFG_ISSUE_WIDTH], @@ -51,7 +51,7 @@ module VX_fetch_xbar import VX_gpu_pkg::*; ( output wire [`VX_CFG_NUM_SFU_BLOCKS-1:0][`VX_CFG_XLEN-1:0] satp_wr_data, `endif - // fetch slices + // frontend slices output sched_branch_t [`VX_CFG_FETCH_WIDTH-1:0][FETCH_ALU_PORTS-1:0] fs_branch, output sched_wctl_t [`VX_CFG_FETCH_WIDTH-1:0][FETCH_SFU_PORTS-1:0] fs_wctl, output wire [`VX_CFG_FETCH_WIDTH-1:0][FETCH_SFU_PORTS-1:0][NW_WIDTH-1:0] fs_dvstack_wid, @@ -89,7 +89,7 @@ module VX_fetch_xbar import VX_gpu_pkg::*; ( end end - for (genvar f = 0; f < FW; ++f) begin : g_fetch_slices + for (genvar f = 0; f < FW; ++f) begin : g_frontend_slices assign fs_decode_if[f].ready = is_ready[wid_to_isw(fs_decode_if[f].data.wid)]; for (genvar l = 0; l < SW; ++l) begin : g_ibuf_pop localparam W = l * FW + f; @@ -111,7 +111,7 @@ module VX_fetch_xbar import VX_gpu_pkg::*; ( assign pending_alm_empty[w] = fs_pending_alm_empty[w % FW][w / FW]; end - // ALU blocks: fetch slice f port q is block q * FW + f, or block f mod BA when shared + // ALU blocks: frontend slice f port q is block q * FW + f, or block f mod BA when shared for (genvar f = 0; f < FW; ++f) begin : g_alu_slices for (genvar q = 0; q < NPA; ++q) begin : g_ports diff --git a/hw/rtl/core/VX_scheduler.sv b/hw/rtl/core/VX_scheduler.sv index c2d36fda22..246274c7ca 100644 --- a/hw/rtl/core/VX_scheduler.sv +++ b/hw/rtl/core/VX_scheduler.sv @@ -14,7 +14,7 @@ `include "VX_define.vh" // The core-level part of scheduling: CTA dispatch and the core-wide warp-control -// resources. Each fetch slice schedules and fetches its own warps (VX_warp_sched, +// resources. Each frontend slice schedules and fetches its own warps (VX_warp_sched, // VX_fetch); core-wide events are handed to it with slice-local warp ids. module VX_scheduler import VX_gpu_pkg::*; #( parameter `STRING INSTANCE_ID = "" @@ -70,7 +70,7 @@ module VX_scheduler import VX_gpu_pkg::*; #( assign pending_warp_empty[w] = slice_if[w % FW].pending_empty[w / FW]; end - // CTA dispatcher: lane p launches fetch slice p's warps + // CTA dispatcher: lane p launches frontend slice p's warps wire [FW-1:0] cta_fire; wire [FW-1:0][NW_WIDTH-1:0] cta_wid; wire [PC_BITS-1:0] cta_PC; diff --git a/hw/rtl/core/VX_sfu_unit.sv b/hw/rtl/core/VX_sfu_unit.sv index ecbbf46146..a7f14fe32d 100644 --- a/hw/rtl/core/VX_sfu_unit.sv +++ b/hw/rtl/core/VX_sfu_unit.sv @@ -33,15 +33,15 @@ module VX_sfu_unit import VX_gpu_pkg::*; #( `endif `ifdef VX_CFG_EXT_DXA_ENABLE - VX_dxa_req_bus_if.master dxa_req_bus_if, + VX_dxa_req_bus_if.master dxa_req_bus_if [`VX_CFG_NUM_SFU_BLOCKS], `endif `ifdef VX_CFG_EXT_TEX_ENABLE - VX_tex_bus_if.master tex_bus_if, + VX_tex_bus_if.master tex_bus_if [`VX_CFG_NUM_SFU_BLOCKS], `endif `ifdef VX_CFG_EXT_RTU_ENABLE - VX_rtu_bus_if.master rtu_bus_if, + VX_rtu_bus_if.master rtu_bus_if [`VX_CFG_NUM_SFU_BLOCKS], VX_sched_unlock_if.master sched_unlock_if [`VX_CFG_NUM_SFU_BLOCKS], `endif @@ -82,22 +82,6 @@ module VX_sfu_unit import VX_gpu_pkg::*; #( wire [BLOCK_SIZE-1:0][INST_FRM_BITS+`FP_FLAGS_BITS-1:0] fcsr_wr_data; `endif -`ifdef VX_CFG_EXT_DXA_ENABLE - VX_dxa_req_bus_if block_dxa_req_bus_if[BLOCK_SIZE](); -`endif -`ifdef VX_CFG_EXT_TEX_ENABLE - VX_tex_bus_if #( - .NUM_LANES (NUM_LANES), - .TAG_WIDTH (TEX_REQ_TAG_WIDTH) - ) block_tex_bus_if[BLOCK_SIZE](); -`endif -`ifdef VX_CFG_EXT_RTU_ENABLE - VX_rtu_bus_if #( - .NUM_LANES (NUM_LANES), - .TAG_WIDTH (RTU_REQ_TAG_WIDTH) - ) block_rtu_bus_if[BLOCK_SIZE](); -`endif - for (genvar block_idx = 0; block_idx < BLOCK_SIZE; ++block_idx) begin : g_blocks VX_sfu_block #( .INSTANCE_ID ((BLOCK_SIZE > 1) ? `SFORMATF(("%s%0d", INSTANCE_ID, block_idx)) : INSTANCE_ID) @@ -115,13 +99,13 @@ module VX_sfu_unit import VX_gpu_pkg::*; #( .fcsr_wr_data (fcsr_wr_data[block_idx]), `endif `ifdef VX_CFG_EXT_DXA_ENABLE - .dxa_req_bus_if (block_dxa_req_bus_if[block_idx]), + .dxa_req_bus_if (dxa_req_bus_if[block_idx]), `endif `ifdef VX_CFG_EXT_TEX_ENABLE - .tex_bus_if (block_tex_bus_if[block_idx]), + .tex_bus_if (tex_bus_if[block_idx]), `endif `ifdef VX_CFG_EXT_RTU_ENABLE - .rtu_bus_if (block_rtu_bus_if[block_idx]), + .rtu_bus_if (rtu_bus_if[block_idx]), .sched_unlock_if(sched_unlock_if[block_idx]), `endif .sched_csr_if (sched_csr_if[block_idx]), @@ -145,25 +129,6 @@ module VX_sfu_unit import VX_gpu_pkg::*; #( ); `endif - VX_sfu_shared #( - .INSTANCE_ID (`SFORMATF(("%s-shared", INSTANCE_ID))) - ) sfu_shared ( - `ifdef VX_CFG_EXT_DXA_ENABLE - .block_dxa_req_bus_if (block_dxa_req_bus_if), - .dxa_req_bus_if (dxa_req_bus_if), - `endif - `ifdef VX_CFG_EXT_TEX_ENABLE - .block_tex_bus_if (block_tex_bus_if), - .tex_bus_if (tex_bus_if), - `endif - `ifdef VX_CFG_EXT_RTU_ENABLE - .block_rtu_bus_if (block_rtu_bus_if), - .rtu_bus_if (rtu_bus_if), - `endif - .clk (clk), - .reset (reset) - ); - VX_lane_gather #( .BLOCK_SIZE (BLOCK_SIZE), .NUM_LANES (NUM_LANES), diff --git a/hw/rtl/core/VX_uuid_gen.sv b/hw/rtl/core/VX_uuid_gen.sv index 34ea3aab1d..5e2da82927 100644 --- a/hw/rtl/core/VX_uuid_gen.sv +++ b/hw/rtl/core/VX_uuid_gen.sv @@ -14,7 +14,7 @@ `include "VX_define.vh" // Per-warp instruction ids. NUM_WARPS is the number of warps counted here: all -// of the core's, or the PER_FETCH_WARPS of one fetch slice (wid = {wfs, slice}). +// of the core's, or the PER_FETCH_WARPS of one frontend slice (wid = {wfs, slice}). module VX_uuid_gen import VX_gpu_pkg::*; #( parameter NUM_WARPS = `VX_CFG_NUM_WARPS ) ( diff --git a/hw/rtl/core/VX_warp_sched.sv b/hw/rtl/core/VX_warp_sched.sv index eeeb663bc4..70cdac2baa 100644 --- a/hw/rtl/core/VX_warp_sched.sv +++ b/hw/rtl/core/VX_warp_sched.sv @@ -13,7 +13,7 @@ `include "VX_define.vh" -// The warp scheduler of one fetch slice: the state of the slice's warps +// The warp scheduler of one frontend slice: the state of the slice's warps // (wid = {wfs, slice_id}), its warp selection and a registered fetch request to // the slice's fetch unit, which names warps by wfs. FU block port q carries the // traffic of the local warps with wfs mod PORTS == q, addressed by core-wide wid; diff --git a/hw/rtl/interfaces/VX_slice_sched_if.sv b/hw/rtl/interfaces/VX_slice_sched_if.sv index c5e056ab43..4f66b97f24 100644 --- a/hw/rtl/interfaces/VX_slice_sched_if.sv +++ b/hw/rtl/interfaces/VX_slice_sched_if.sv @@ -13,7 +13,7 @@ `include "VX_define.vh" -// Core scheduler state exchanged with one fetch slice's warp scheduler. +// Core scheduler state exchanged with one frontend slice's warp scheduler. // Core-wide events arrive already addressed to the slice: warp ids are // slice-local (wid = {wfs, fsw}) and per-warp masks hold only its warps. interface VX_slice_sched_if import VX_gpu_pkg::*; (); diff --git a/hw/rtl/tcu/VX_tcu_bbuf.sv b/hw/rtl/tcu/VX_tcu_bbuf.sv index 11a4624a26..e87fb7e9c6 100644 --- a/hw/rtl/tcu/VX_tcu_bbuf.sv +++ b/hw/rtl/tcu/VX_tcu_bbuf.sv @@ -18,7 +18,7 @@ // // TB-shared B buffer (block-major SMEM, 1 bank-row storage). // -// Single instance per VX_tcu_unit. Holds the bank-row of B that contains +// Single instance per core (in VX_tcu_shared). Holds the bank-row of B that contains // the current (step_k, step_n) block. All Q tcu_cores read from the // same buffer (structural fan-out, not arbitrated). // diff --git a/hw/rtl/tcu/VX_tcu_unit.sv b/hw/rtl/tcu/VX_tcu_unit.sv index fe0e4838e7..91dd5fd292 100644 --- a/hw/rtl/tcu/VX_tcu_unit.sv +++ b/hw/rtl/tcu/VX_tcu_unit.sv @@ -21,18 +21,28 @@ module VX_tcu_unit import VX_gpu_pkg::*, VX_tcu_pkg::*; #( input wire clk, input wire reset, -`ifdef PERF_ENABLE - output tcu_perf_t tcu_perf, -`endif - `ifdef VX_CFG_TCU_WGMMA_ENABLE - // Bank-parallel LMEM read port - VX_mem_bus_if.master tcu_lmem_if, + // per-block ports of the core-wide WGMMA tile buffers (VX_tcu_shared) + output wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] exec_valid, + output wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] exec_ready, + output tcu_execute_t exec_data [`VX_CFG_NUM_TCU_BLOCKS], + input wire [`VX_CFG_NUM_TCU_BLOCKS-1:0][TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tbuf_rs1_data, + input wire [`VX_CFG_NUM_TCU_BLOCKS-1:0][TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tbuf_rs2_data, + input wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tbuf_ready, `endif `ifdef TCU_META_ENABLE - // TCU_LD memory client connection to VX_lsu_scheduler at VX_core. - VX_lsu_sched_if.master tcu_mem_if, + // per-block ports of the core-wide metadata AGU (VX_tcu_shared) + output wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] agu_ld_valid, + output tcu_execute_t agu_ld_data [`VX_CFG_NUM_TCU_BLOCKS], + input wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] agu_ld_ready, + input wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] agu_result_valid, + input tcu_result_t agu_result_data [`VX_CFG_NUM_TCU_BLOCKS], + output wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] agu_result_ready, + input wire meta_wr_en, + input wire [NW_WIDTH-1:0] meta_wr_wid, + input wire [4:0] meta_wr_idx, + input wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] meta_wr_data, `endif // Inputs @@ -68,25 +78,6 @@ module VX_tcu_unit import VX_gpu_pkg::*, VX_tcu_pkg::*; #( .data_t (tcu_result_t) ) per_block_result_if[BLOCK_SIZE](); -`ifdef VX_CFG_TCU_WGMMA_ENABLE - wire [BLOCK_SIZE-1:0] exec_valid, exec_ready; - tcu_execute_t exec_data [BLOCK_SIZE]; - wire [BLOCK_SIZE-1:0][TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tbuf_rs1_data; - wire [BLOCK_SIZE-1:0][TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tbuf_rs2_data; - wire [BLOCK_SIZE-1:0] tbuf_ready_eff; -`endif - -`ifdef TCU_META_ENABLE - wire [BLOCK_SIZE-1:0] agu_ld_valid, agu_ld_ready; - tcu_execute_t agu_ld_data [BLOCK_SIZE]; - wire [BLOCK_SIZE-1:0] agu_result_valid, agu_result_ready; - tcu_result_t agu_result_data [BLOCK_SIZE]; - wire meta_wr_en; - wire [NW_WIDTH-1:0] meta_wr_wid; - wire [4:0] meta_wr_idx; - wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] meta_wr_data; -`endif - for (genvar block_idx = 0; block_idx < BLOCK_SIZE; ++block_idx) begin : g_blocks `ifdef TCU_META_ENABLE wire block_meta_wr_en = meta_wr_en @@ -106,7 +97,7 @@ module VX_tcu_unit import VX_gpu_pkg::*, VX_tcu_pkg::*; #( .exec_data (exec_data[block_idx]), .tbuf_rs1_data (tbuf_rs1_data[block_idx]), .tbuf_rs2_data (tbuf_rs2_data[block_idx]), - .tbuf_ready (tbuf_ready_eff[block_idx]), + .tbuf_ready (tbuf_ready[block_idx]), `endif `ifdef TCU_META_ENABLE .agu_ld_valid (agu_ld_valid[block_idx]), @@ -125,39 +116,6 @@ module VX_tcu_unit import VX_gpu_pkg::*, VX_tcu_pkg::*; #( ); end - VX_tcu_shared #( - .INSTANCE_ID (INSTANCE_ID), - .BLOCK_SIZE (BLOCK_SIZE) - ) tcu_shared ( - `ifdef PERF_ENABLE - .tcu_perf (tcu_perf), - `endif - `ifdef VX_CFG_TCU_WGMMA_ENABLE - .exec_valid (exec_valid), - .exec_ready (exec_ready), - .exec_data (exec_data), - .tbuf_rs1_data (tbuf_rs1_data), - .tbuf_rs2_data (tbuf_rs2_data), - .tbuf_ready_eff (tbuf_ready_eff), - .tcu_lmem_if (tcu_lmem_if), - `endif - `ifdef TCU_META_ENABLE - .agu_ld_valid (agu_ld_valid), - .agu_ld_data (agu_ld_data), - .agu_ld_ready (agu_ld_ready), - .agu_result_valid (agu_result_valid), - .agu_result_data (agu_result_data), - .agu_result_ready (agu_result_ready), - .meta_wr_en (meta_wr_en), - .meta_wr_wid (meta_wr_wid), - .meta_wr_idx (meta_wr_idx), - .meta_wr_data (meta_wr_data), - .tcu_mem_if (tcu_mem_if), - `endif - .clk (clk), - .reset (reset) - ); - // ----------------------------------------------------------------------- // Lane gather // ----------------------------------------------------------------------- diff --git a/hw/rtl/tcu/VX_tcu_wgmma.sv b/hw/rtl/tcu/VX_tcu_wgmma.sv index 671a1a8d61..4df2e33579 100644 --- a/hw/rtl/tcu/VX_tcu_wgmma.sv +++ b/hw/rtl/tcu/VX_tcu_wgmma.sv @@ -20,8 +20,8 @@ // CTA lockstep gate, and WGMMA-specific perf counters. // // Observation-only on the dispatch path: this module does not drive -// .ready. The wrapper (VX_tcu_unit) feeds tbuf_ready_eff into tcu_core, -// which produces the actual handshake .ready going back to dispatch. +// .ready. VX_tcu_shared exports tbuf_ready_eff per block; the block's +// tcu_core produces the actual handshake .ready going back to dispatch. module VX_tcu_wgmma import VX_gpu_pkg::*, VX_tcu_pkg::*; #( parameter `STRING INSTANCE_ID = "", diff --git a/hw/unittest/tcu_unit/VX_tcu_unit_top.sv b/hw/unittest/tcu_unit/VX_tcu_unit_top.sv index 192fb19b5a..637ba9bd9a 100644 --- a/hw/unittest/tcu_unit/VX_tcu_unit_top.sv +++ b/hw/unittest/tcu_unit/VX_tcu_unit_top.sv @@ -13,7 +13,7 @@ `include "VX_define.vh" -// Flat-port testbench wrapper around VX_tcu_unit. +// Flat-port testbench wrapper around VX_tcu_unit and VX_tcu_shared. // All VX_dispatch_if and VX_commit_if signals are exposed as individual // ports so that the Verilator C++ testbench can drive them directly. // Only ISSUE_WIDTH == 1 is supported by this wrapper. @@ -164,20 +164,83 @@ module VX_tcu_unit_top import VX_gpu_pkg::*, VX_tcu_pkg::*; ( `UNUSED_VAR (commit_if[0].data.cta_id) // ---- DUT ------------------------------------------------------------- +`ifdef VX_CFG_TCU_WGMMA_ENABLE + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_exec_valid, tcu_exec_ready; + tcu_execute_t tcu_exec_data [`VX_CFG_NUM_TCU_BLOCKS]; + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0][TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs1_data; + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0][TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs2_data; + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_tbuf_ready; +`endif +`ifdef TCU_META_ENABLE + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_agu_ld_valid, tcu_agu_ld_ready; + tcu_execute_t tcu_agu_ld_data [`VX_CFG_NUM_TCU_BLOCKS]; + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_agu_result_valid, tcu_agu_result_ready; + tcu_result_t tcu_agu_result_data [`VX_CFG_NUM_TCU_BLOCKS]; + wire tcu_meta_wr_en; + wire [NW_WIDTH-1:0] tcu_meta_wr_wid; + wire [4:0] tcu_meta_wr_idx; + wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] tcu_meta_wr_data; +`endif + VX_tcu_unit #( .INSTANCE_ID ("tcu_unit_top") ) tcu_unit ( `SCOPE_IO_BIND (0) - .clk (clk), - .reset (reset), + .clk (clk), + .reset (reset), + `ifdef VX_CFG_TCU_WGMMA_ENABLE + .exec_valid (tcu_exec_valid), + .exec_ready (tcu_exec_ready), + .exec_data (tcu_exec_data), + .tbuf_rs1_data (tcu_tbuf_rs1_data), + .tbuf_rs2_data (tcu_tbuf_rs2_data), + .tbuf_ready (tcu_tbuf_ready), + `endif + `ifdef TCU_META_ENABLE + .agu_ld_valid (tcu_agu_ld_valid), + .agu_ld_data (tcu_agu_ld_data), + .agu_ld_ready (tcu_agu_ld_ready), + .agu_result_valid (tcu_agu_result_valid), + .agu_result_data (tcu_agu_result_data), + .agu_result_ready (tcu_agu_result_ready), + .meta_wr_en (tcu_meta_wr_en), + .meta_wr_wid (tcu_meta_wr_wid), + .meta_wr_idx (tcu_meta_wr_idx), + .meta_wr_data (tcu_meta_wr_data), + `endif + .dispatch_if (dispatch_if), + .commit_if (commit_if) + ); + + // the core-wide TCU state the blocks share: WGMMA tile buffers, metadata AGU + VX_tcu_shared #( + .INSTANCE_ID ("tcu_unit_top"), + .BLOCK_SIZE (`VX_CFG_NUM_TCU_BLOCKS) + ) tcu_shared ( `ifdef VX_CFG_TCU_WGMMA_ENABLE - .tcu_lmem_if (tcu_lmem_if), + .exec_valid (tcu_exec_valid), + .exec_ready (tcu_exec_ready), + .exec_data (tcu_exec_data), + .tbuf_rs1_data (tcu_tbuf_rs1_data), + .tbuf_rs2_data (tcu_tbuf_rs2_data), + .tbuf_ready_eff (tcu_tbuf_ready), + .tcu_lmem_if (tcu_lmem_if), `endif `ifdef TCU_META_ENABLE - .tcu_mem_if (tcu_mem_if), + .agu_ld_valid (tcu_agu_ld_valid), + .agu_ld_data (tcu_agu_ld_data), + .agu_ld_ready (tcu_agu_ld_ready), + .agu_result_valid (tcu_agu_result_valid), + .agu_result_data (tcu_agu_result_data), + .agu_result_ready (tcu_agu_result_ready), + .meta_wr_en (tcu_meta_wr_en), + .meta_wr_wid (tcu_meta_wr_wid), + .meta_wr_idx (tcu_meta_wr_idx), + .meta_wr_data (tcu_meta_wr_data), + .tcu_mem_if (tcu_mem_if), `endif - .dispatch_if (dispatch_if), - .commit_if (commit_if) + .clk (clk), + .reset (reset) ); endmodule diff --git a/hw/unittest/tensor/VX_tensor_top.sv b/hw/unittest/tensor/VX_tensor_top.sv index abd8b0b526..e56e5c14b6 100644 --- a/hw/unittest/tensor/VX_tensor_top.sv +++ b/hw/unittest/tensor/VX_tensor_top.sv @@ -20,6 +20,7 @@ // // per core (VX_CFG_SOCKET_SIZE): // VX_tcu_unit : tensor-core unit (dispatch/commit from top ports) +// VX_tcu_shared : WGMMA tile buffers + metadata AGU // VX_lsu_scheduler : per-LSU-block, 2 clients — the ordinary LSU stream // (top ports) and the TCU AGU metadata-load port // VX_mem_unit : LMEM + coalescer + LMEM-DMA arb (TCU WGMMA + DXA @@ -175,20 +176,83 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( VX_lsu_sched_if tcu_mem_if(); `endif + `ifdef VX_CFG_TCU_WGMMA_ENABLE + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_exec_valid, tcu_exec_ready; + tcu_execute_t tcu_exec_data [`VX_CFG_NUM_TCU_BLOCKS]; + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0][TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs1_data; + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0][TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs2_data; + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_tbuf_ready; + `endif + `ifdef TCU_META_ENABLE + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_agu_ld_valid, tcu_agu_ld_ready; + tcu_execute_t tcu_agu_ld_data [`VX_CFG_NUM_TCU_BLOCKS]; + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_agu_result_valid, tcu_agu_result_ready; + tcu_result_t tcu_agu_result_data [`VX_CFG_NUM_TCU_BLOCKS]; + wire tcu_meta_wr_en; + wire [NW_WIDTH-1:0] tcu_meta_wr_wid; + wire [4:0] tcu_meta_wr_idx; + wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] tcu_meta_wr_data; + `endif + VX_tcu_unit #( .INSTANCE_ID (`SFORMATF(("%s-tcu%0d", INSTANCE_ID, c))) ) tcu_unit ( `SCOPE_IO_BIND (c) - .clk (clk), - .reset (reset), + .clk (clk), + .reset (reset), + `ifdef VX_CFG_TCU_WGMMA_ENABLE + .exec_valid (tcu_exec_valid), + .exec_ready (tcu_exec_ready), + .exec_data (tcu_exec_data), + .tbuf_rs1_data (tcu_tbuf_rs1_data), + .tbuf_rs2_data (tcu_tbuf_rs2_data), + .tbuf_ready (tcu_tbuf_ready), + `endif + `ifdef TCU_META_ENABLE + .agu_ld_valid (tcu_agu_ld_valid), + .agu_ld_data (tcu_agu_ld_data), + .agu_ld_ready (tcu_agu_ld_ready), + .agu_result_valid (tcu_agu_result_valid), + .agu_result_data (tcu_agu_result_data), + .agu_result_ready (tcu_agu_result_ready), + .meta_wr_en (tcu_meta_wr_en), + .meta_wr_wid (tcu_meta_wr_wid), + .meta_wr_idx (tcu_meta_wr_idx), + .meta_wr_data (tcu_meta_wr_data), + `endif + .dispatch_if (dispatch_if), + .commit_if (commit_if) + ); + + // the core-wide TCU state the blocks share: WGMMA tile buffers, metadata AGU + VX_tcu_shared #( + .INSTANCE_ID (`SFORMATF(("%s-tcu%0d", INSTANCE_ID, c))), + .BLOCK_SIZE (`VX_CFG_NUM_TCU_BLOCKS) + ) tcu_shared ( `ifdef VX_CFG_TCU_WGMMA_ENABLE - .tcu_lmem_if (tcu_lmem_if), + .exec_valid (tcu_exec_valid), + .exec_ready (tcu_exec_ready), + .exec_data (tcu_exec_data), + .tbuf_rs1_data (tcu_tbuf_rs1_data), + .tbuf_rs2_data (tcu_tbuf_rs2_data), + .tbuf_ready_eff (tcu_tbuf_ready), + .tcu_lmem_if (tcu_lmem_if), `endif `ifdef TCU_META_ENABLE - .tcu_mem_if (tcu_mem_if), + .agu_ld_valid (tcu_agu_ld_valid), + .agu_ld_data (tcu_agu_ld_data), + .agu_ld_ready (tcu_agu_ld_ready), + .agu_result_valid (tcu_agu_result_valid), + .agu_result_data (tcu_agu_result_data), + .agu_result_ready (tcu_agu_result_ready), + .meta_wr_en (tcu_meta_wr_en), + .meta_wr_wid (tcu_meta_wr_wid), + .meta_wr_idx (tcu_meta_wr_idx), + .meta_wr_data (tcu_meta_wr_data), + .tcu_mem_if (tcu_mem_if), `endif - .dispatch_if (dispatch_if), - .commit_if (commit_if) + .clk (clk), + .reset (reset) ); // --- LSU schedulers: client 0 from top ports, client 1 = TCU AGU --- diff --git a/sim/simx/constants.h b/sim/simx/constants.h index da7a62eaa3..64a61c14d2 100644 --- a/sim/simx/constants.h +++ b/sim/simx/constants.h @@ -105,7 +105,7 @@ inline constexpr uint32_t VX_CFG_L3_NUM_REQS = VX_CFG_NUM_CLUSTERS * VX_CFG_ inline constexpr uint32_t PER_ISSUE_WARPS = VX_CFG_NUM_WARPS / VX_CFG_ISSUE_WIDTH; inline constexpr uint32_t ISSUE_WIS_BITS = log2ceil(PER_ISSUE_WARPS); -// Fetch slice f owns the warps with wid % VX_CFG_FETCH_WIDTH == f; each has its +// Frontend slice f owns the warps with wid % VX_CFG_FETCH_WIDTH == f; each has its // own warp scheduler, fetch unit, decoder and icache request port. inline constexpr uint32_t PER_FETCH_WARPS = VX_CFG_NUM_WARPS / VX_CFG_FETCH_WIDTH; inline constexpr uint32_t ICACHE_NUM_REQS = VX_CFG_FETCH_WIDTH; diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index a6fcfdff90..c81fc2709b 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -79,7 +79,7 @@ class Core::Impl { const std::string& name = simobject_->name(); char sname[100]; - // per fetch slice: scheduler-to-fetch and fetch-to-decode latches, and the + // per frontend slice: scheduler-to-fetch and fetch-to-decode latches, and the // icache tags of the slice's in-flight fetches for (uint32_t f = 0; f < VX_CFG_FETCH_WIDTH; ++f) { snprintf(sname, 100, "fetch_latch%d", f); @@ -219,7 +219,7 @@ class Core::Impl { dcache_mmu_ = Mmu::Create(sname, VX_CFG_DCACHE_NUM_REQS, VX_CFG_DTLB_SIZE, local_core * 2 + 0, false); - // Per-core icache MMU, one port per fetch slice. Fetch reads/writes its + // Per-core icache MMU, one port per frontend slice. Fetch reads/writes its // upstream channels directly; the downstream side is bound to the Core's // icache ports below. snprintf(sname, 100, "%s-icache_mmu", name.c_str()); @@ -376,7 +376,7 @@ class Core::Impl { scheduler_->schedule_begin(); - // Each fetch slice picks among its own warps whose ibuffer is not full; a + // Each frontend slice picks among its own warps whose ibuffer is not full; a // buffer is full at exactly IBUF_SIZE entries. With an L1, once every warp's // buffer is full any ready warp may fetch, its instruction waiting at decode. bool all_full = (VX_CFG_ICACHE_ENABLED || VX_CFG_DCACHE_ENABLED); @@ -426,7 +426,7 @@ class Core::Impl { } } - // Fetch slice f: its icache response into its decode latch, then its next + // Frontend slice f: its icache response into its decode latch, then its next // fetch request out of its fetch latch, on icache port f. void fetch_slice(uint32_t f) { auto& fetch_latch = *fetch_latch_.at(f); @@ -541,7 +541,7 @@ class Core::Impl { } } - // One fetch slice's decoder: it feeds the issue slice of each decoded warp. + // One frontend slice's decoder: it feeds the issue slice of each decoded warp. void decode_slice(TFifo& decode_latch) { if (decode_latch.empty()) { return; @@ -1202,7 +1202,7 @@ class Core::Impl { PoolAllocator trace_pool_; #ifdef VX_CFG_EXT_C_ENABLE - // RVC fetch SimObject per fetch slice (per-warp halfword buffer + refetch + // RVC fetch SimObject per frontend slice (per-warp halfword buffer + refetch // queue + pick/commit_request helpers). See decompressor.h. std::vector decompressors_; #endif diff --git a/sim/simx/scheduler.h b/sim/simx/scheduler.h index 58f10378b0..7a3fd6dee2 100644 --- a/sim/simx/scheduler.h +++ b/sim/simx/scheduler.h @@ -118,11 +118,11 @@ class Scheduler : public SimObject { ~Scheduler(); // ----- Warp lifecycle ----- - // Once per cycle, before the fetch slices pick: activate the warps the CTA + // Once per cycle, before the frontend slices pick: activate the warps the CTA // dispatcher launched last cycle, launch the next ones, and apply a wspawn. void schedule_begin(); - // One fetch slice's pick: its lowest ready warp in warp_mask. + // One frontend slice's pick: its lowest ready warp in warp_mask. instr_trace_t* schedule(const WarpMask& warp_mask); // Whether warp_mask holds a ready warp, without picking it. diff --git a/sim/simx/socket.cpp b/sim/simx/socket.cpp index 83eac594da..c919ce378f 100644 --- a/sim/simx/socket.cpp +++ b/sim/simx/socket.cpp @@ -62,7 +62,7 @@ class Socket::Impl { log2ceil(VX_CFG_ICACHE_NUM_WAYS),// A log2ceil(VX_CFG_ICACHE_NUM_BANKS), // B VX_CFG_XLEN, // address bits - ICACHE_NUM_REQS, // number of inputs (one per fetch slice) + ICACHE_NUM_REQS, // number of inputs (one per frontend slice) VX_CFG_ICACHE_MEM_PORTS, // memory ports false, // write-back false, // write response From e849c567cf580ba238172995c8c1a6b938b7b976 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 18 Oct 2026 14:46:14 -0700 Subject: [PATCH 040/124] simx: register the cache core-response port as the RTL does The RTL caches register their core-response port (CORE_OUT_BUF) and the LSU-side coalescer and lane adapter answer combinationally, while SimX put the register in the LSU adapters and the memory coalescer instead. The total load latency matched, but under contention the models stalled at different points. The cache config gains core_out_buf. The dcache, L2 and L3 set it, and a cache response is delayed by that register plus the non-cacheable bypass layer's response register. The memory coalescer and the LSU lane adapter now send responses with no delay, and both tick ahead of the local-memory switch. Co-Authored-By: Claude Opus 5.5 --- sim/simx/cluster.cpp | 1 + sim/simx/core.cpp | 15 ++++++++++----- sim/simx/mem/cache.cpp | 6 ++++-- sim/simx/mem/cache.h | 1 + sim/simx/mem/lsu_mem_adapter.cpp | 6 ++++-- sim/simx/mem/lsu_mem_adapter.h | 4 +++- sim/simx/mem/mem_coalescer.cpp | 4 ++-- sim/simx/processor.cpp | 1 + sim/simx/socket.cpp | 1 + 9 files changed, 27 insertions(+), 12 deletions(-) diff --git a/sim/simx/cluster.cpp b/sim/simx/cluster.cpp index 8674f73591..88ab5dbb20 100644 --- a/sim/simx/cluster.cpp +++ b/sim/simx/cluster.cpp @@ -88,6 +88,7 @@ class Cluster::Impl { (VX_CFG_L2_BANK_HASH != 0) && (VX_CFG_L2_NUM_BANKS > 1), // bank_hash L2_SHARED_LLC, // shared_llc VX_CFG_L2_LAZY_FETCH != 0, // lazy_fetch + true, // core-response register (behind the bypass layer) }); diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index c81fc2709b..558bf54a6e 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -128,10 +128,18 @@ class Core::Impl { ibuffers_.at(i) = TFifo::Create(sname, 1, VX_CFG_IBUF_SIZE); } - // create the memory coalescer + // create the memory coalescer, or the lane adapter that replaces it. + // Both tick ahead of the lmem switch: their response paths are + // combinational, and the dcache registers its output. + const bool dcache_coalescing = (VX_CFG_NUM_LSU_LANES > 1) && (DCACHE_WORD_SIZE > LSU_WORD_SIZE); + std::vector lsu_dcache_adapter(VX_CFG_NUM_LSU_BLOCKS); for (uint32_t b = 0; b < VX_CFG_NUM_LSU_BLOCKS; ++b) { snprintf(sname, 100, "%s-coalescer%d", name.c_str(), b); mem_coalescers_.at(b) = MemCoalescer::Create(sname, LSU_CHANNELS, DCACHE_CHANNELS, DCACHE_WORD_SIZE, LSU_QUEUE_OUT_SIZE, 1); + if (!dcache_coalescing) { + snprintf(sname, 100, "%s-lsu_dcache_adapter%d", name.c_str(), b); + lsu_dcache_adapter.at(b) = LsuMemAdapter::Create(sname, DCACHE_CHANNELS, 1, 0); + } } // create local memory. @@ -180,7 +188,7 @@ class Core::Impl { // The first data-cache port also carries the cache-flush injection, which // registers its requests once more. - if ((VX_CFG_NUM_LSU_LANES > 1) && (DCACHE_WORD_SIZE > LSU_WORD_SIZE)) { + if (dcache_coalescing) { // connect memory coalescer; its memory side drives the dcache // channels directly (combinational lane fan-out/fan-in). for (uint32_t b = 0; b < VX_CFG_NUM_LSU_BLOCKS; ++b) { @@ -195,10 +203,7 @@ class Core::Impl { } else { // bypass memory coalescer: per-block lane adapter (channel-fused // pass-through when DCACHE_CHANNELS == 1) - std::vector lsu_dcache_adapter(VX_CFG_NUM_LSU_BLOCKS); for (uint32_t b = 0; b < VX_CFG_NUM_LSU_BLOCKS; ++b) { - snprintf(sname, 100, "%s-lsu_dcache_adapter%d", name.c_str(), b); - lsu_dcache_adapter.at(b) = LsuMemAdapter::Create(sname, DCACHE_CHANNELS, 1); lmem_switch_.at(b)->ReqOutDC.bind(&lsu_dcache_adapter.at(b)->ReqIn); lsu_dcache_adapter.at(b)->RspOut.bind(&lmem_switch_.at(b)->RspInDC); for (uint32_t c = 0; c < DCACHE_CHANNELS; ++c) { diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index f7c9c82bc0..ecd6b16448 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -2243,8 +2243,10 @@ class Cache::Impl { if (bank_rsp.empty()) continue; auto &core_rsp = bank_rsp.peek(); - // the non-cacheable bypass layer's response arbiter registers it once more - if (simobject_->core_rsp_out.at(req_id).try_send(core_rsp, config_.nc_enable ? 1 : 0)) { + // the core-response port's register, then the non-cacheable bypass + // layer's response arbiter + uint32_t rsp_delay = (config_.core_out_buf ? 1 : 0) + (config_.nc_enable ? 1 : 0); + if (simobject_->core_rsp_out.at(req_id).try_send(core_rsp, rsp_delay)) { DT(3, simobject_->name() << " core-rsp: " << core_rsp); bank_rsp.pop(); } diff --git a/sim/simx/mem/cache.h b/sim/simx/mem/cache.h index 40d7890501..6371f75d09 100644 --- a/sim/simx/mem/cache.h +++ b/sim/simx/mem/cache.h @@ -47,6 +47,7 @@ class Cache : public SimObject { bool bank_hash = false; // hashed bank select bool shared_llc = false; // the LLC serves every memory client alone bool lazy_fetch = false; // write miss allocates without a fill (write-back LLC only) + bool core_out_buf = false; // registered core-response port }; struct PerfStats { diff --git a/sim/simx/mem/lsu_mem_adapter.cpp b/sim/simx/mem/lsu_mem_adapter.cpp index 7e4460c1c5..a34aacfe6c 100644 --- a/sim/simx/mem/lsu_mem_adapter.cpp +++ b/sim/simx/mem/lsu_mem_adapter.cpp @@ -19,13 +19,15 @@ LsuMemAdapter::LsuMemAdapter( const SimContext& ctx, const char* name, uint32_t num_inputs, - uint32_t delay + uint32_t delay, + uint32_t rsp_delay ) : SimObject(ctx, name) , ReqIn(this) , RspOut(this) , ReqOut(num_inputs, this) , RspIn(num_inputs, this) , delay_(delay) + , rsp_delay_(rsp_delay) , port_delay_(num_inputs, 0) , pending_mask_(num_inputs) { @@ -106,7 +108,7 @@ void LsuMemAdapter::on_tick() { } // send memory response - RspOut.send(out_rsp, 1); + RspOut.send(out_rsp, rsp_delay_); // remove input DT(4, this->name() << "-rsp" << i << ": " << rsp_in); diff --git a/sim/simx/mem/lsu_mem_adapter.h b/sim/simx/mem/lsu_mem_adapter.h index 0d0e05ebe6..6f6499716f 100644 --- a/sim/simx/mem/lsu_mem_adapter.h +++ b/sim/simx/mem/lsu_mem_adapter.h @@ -31,7 +31,8 @@ class LsuMemAdapter : public SimObject { const SimContext& ctx, const char* name, uint32_t num_inputs, - uint32_t delay + uint32_t delay, + uint32_t rsp_delay = 1 ); LsuMemAdapter( @@ -50,6 +51,7 @@ class LsuMemAdapter : public SimObject { private: uint32_t delay_; + uint32_t rsp_delay_; std::vector port_delay_; BitVector pending_mask_; diff --git a/sim/simx/mem/mem_coalescer.cpp b/sim/simx/mem/mem_coalescer.cpp index 533e236b13..51a0b95a12 100644 --- a/sim/simx/mem/mem_coalescer.cpp +++ b/sim/simx/mem/mem_coalescer.cpp @@ -108,8 +108,8 @@ void MemCoalescer::on_tick() { } } - // send memory response - RspOut.send(out_rsp, 1); + // the response path is combinational; the cache registers its output + RspOut.send(out_rsp, 0); DT(4, this->name() << " mem-rsp: " << out_rsp); // track remaining responses diff --git a/sim/simx/processor.cpp b/sim/simx/processor.cpp index b7d31c1705..77163fe4df 100644 --- a/sim/simx/processor.cpp +++ b/sim/simx/processor.cpp @@ -108,6 +108,7 @@ ProcessorImpl::ProcessorImpl() false, // bank_hash L3_SHARED_LLC, // shared_llc VX_CFG_L3_LAZY_FETCH != 0, // lazy_fetch + true, // core-response register (behind the bypass layer) } ); diff --git a/sim/simx/socket.cpp b/sim/simx/socket.cpp index c919ce378f..064eaba4e2 100644 --- a/sim/simx/socket.cpp +++ b/sim/simx/socket.cpp @@ -95,6 +95,7 @@ class Socket::Impl { (VX_CFG_DCACHE_BANK_HASH != 0) && (VX_CFG_DCACHE_NUM_BANKS > 1), // bank_hash DCACHE_SHARED_LLC, // shared_llc VX_CFG_DCACHE_LAZY_FETCH != 0, // lazy_fetch + true, // core-response register }); #ifdef VX_CFG_EXT_TEX_ENABLE From 6d7df03f8baf09b63c5103500961fee0477e5c1f Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 18 Oct 2026 14:46:15 -0700 Subject: [PATCH 041/124] simx: never launch two warps of one CTA into the same slot The CTA dispatcher picked any warp slot that was not active. A short warp of a CTA could exit before the CTA finished launching, and the dispatcher then launched a later warp of the same CTA into that slot. The RTL never reuses a slot within a CTA. The dispatcher tracks the slots the current CTA has launched into and skips them until the next CTA starts. Co-Authored-By: Claude Opus 5.5 --- sim/simx/cta_dispatcher.cpp | 5 ++++- sim/simx/cta_dispatcher.h | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/sim/simx/cta_dispatcher.cpp b/sim/simx/cta_dispatcher.cpp index 1d0fb6883a..67c82eca72 100644 --- a/sim/simx/cta_dispatcher.cpp +++ b/sim/simx/cta_dispatcher.cpp @@ -61,6 +61,7 @@ CtaDispatcher::~CtaDispatcher() {} void CtaDispatcher::on_reset() { has_cta_ = false; cta_done_ = false; + cta_wids_.reset(); tail_slot_ = 0; init_wait_ = 0; for (uint32_t i = 0; i < num_warps_; ++i) { @@ -176,6 +177,7 @@ uint32_t CtaDispatcher::step(const WarpMask& active_warps, uint32_t* wids, cta_w cta_size_ = (cta_.block_size + num_threads_ - 1) / num_threads_; rank_ = 0; block_size_rem_ = cta_.block_size; + cta_wids_.reset(); thread_idx_[0] = thread_idx_[1] = thread_idx_[2] = 0; cur_slot_ = base; @@ -216,7 +218,7 @@ uint32_t CtaDispatcher::step(const WarpMask& active_warps, uint32_t* wids, cta_w uint32_t lane = (lane_rr_ + o) % VX_CFG_FETCH_WIDTH; int free_wid = -1; for (uint32_t wid = lane; wid < num_warps_; wid += VX_CFG_FETCH_WIDTH) { - if (!active_warps.test(wid)) { + if (!active_warps.test(wid) && !cta_wids_.test(wid)) { free_wid = int(wid); break; } @@ -241,6 +243,7 @@ uint32_t CtaDispatcher::step(const WarpMask& active_warps, uint32_t* wids, cta_w inited.set(t); } wid_to_slot_[free_wid] = cur_slot_; + cta_wids_.set(free_wid); wids[count] = uint32_t(free_wid); ++count; } diff --git a/sim/simx/cta_dispatcher.h b/sim/simx/cta_dispatcher.h index 7c74930292..314fd1d4ce 100644 --- a/sim/simx/cta_dispatcher.h +++ b/sim/simx/cta_dispatcher.h @@ -123,6 +123,9 @@ class CtaDispatcher : public SimObject { uint32_t block_size_rem_; uint32_t thread_idx_[3]; uint64_t lmem_addr_; + // Slots this CTA has launched into: a CTA never reuses one, even after the + // warp it launched there has exited. + WarpMask cta_wids_; // Launch lanes are taken round-robin from lane_rr_. Several lanes need the // per-lane thread-index window rebuilt when the block shape changes, which From 64c2039a835834a78aa5ac15795f62438040ed62 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 18 Oct 2026 14:46:16 -0700 Subject: [PATCH 042/124] simx: model the depth of the cache networks SimX treated every cache network as one registered crossbar. The RTL builds them from smaller pieces, and each added level is another registered stage: - an arbiter over more than 1.5x the fan-out limit (8) is a buffered tree, one cycle deeper, so the core-side request arbiter gains a stage ahead of each bank when it has more than 12 inputs; - the bank-to-memory-port network is a radix-2 omega network, one registered stage per level, and its arbiter registers once when banks and ports differ in number and either side exceeds two; - a registered core-response port is a network registered at every level, applied as the response leaves the cache. These delays are charged on the request and response paths of each cache. Co-Authored-By: Claude Opus 5.5 --- sim/simx/mem/cache.cpp | 92 +++++++++++++++++++++++++++++++++++------- 1 file changed, 78 insertions(+), 14 deletions(-) diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index ecd6b16448..efd5979a7e 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -29,6 +29,24 @@ using namespace vortex; +namespace { + +// An arbiter over more than 1.5x the fan-out limit is split into a buffered +// tree, one cycle deeper. +uint32_t arb_tree_delay(uint32_t requesters) { + constexpr uint32_t kMaxFanout = 8; + return (requesters > kMaxFanout + kMaxFanout / 2) ? 1 : 0; +} + +// Registered stages of a radix-2 omega network between a and b ports; up to +// two ports on each side it is a single crossbar. +uint32_t omega_stages(uint32_t a, uint32_t b) { + uint32_t n = std::max(a, b); + return (n <= 2) ? 1 : log2ceil(n); +} + +} + struct params_t { uint32_t sets_per_bank; uint32_t lines_per_set; @@ -2066,6 +2084,7 @@ class CachePortArb : public SimObject { // the request crossbar registers its output when it has more than two inputs , delay_((num_inputs > 2) ? 1 : 0) , rr_(num_banks, num_inputs - 1) + , tree_stage_((arb_tree_delay(num_inputs) != 0) ? num_banks : 0, this) { assert(num_inputs <= 64); } @@ -2086,6 +2105,14 @@ class CachePortArb : public SimObject { --init_cycles_; return; } + // the arbiter tree's buffered level ahead of each bank + for (uint32_t b = 0, nb = tree_stage_.size(); b < nb; ++b) { + auto& stage = tree_stage_.at(b); + if (!stage.empty() && !ReqOut.at(b).full()) { + ReqOut.at(b).send(stage.peek(), delay_); + stage.pop(); + } + } uint32_t n = ReqIn.size(); // requesting ports per bank; non-cacheable requests leave on their own port std::vector want(ReqOut.size(), 0); @@ -2104,7 +2131,8 @@ class CachePortArb : public SimObject { want.at(params_.addr_bank_id(req.addr)) |= 1ull << i; } for (uint32_t b = 0, nb = ReqOut.size(); b < nb; ++b) { - if (want[b] == 0 || ReqOut.at(b).full()) { + auto& out = tree_stage_.empty() ? ReqOut.at(b) : tree_stage_.at(b); + if (want[b] == 0 || out.full()) { continue; } uint32_t i = rr_[b]; @@ -2114,7 +2142,7 @@ class CachePortArb : public SimObject { collisions_ += __builtin_popcountll(want[b]) - 1; auto req = ReqIn.at(i).peek(); req.tag = (req.tag << lg2_inputs_) | i; - ReqOut.at(b).send(req, delay_); + out.send(req, tree_stage_.empty() ? delay_ : 1); ReqIn.at(i).pop(); rr_[b] = i; } @@ -2126,6 +2154,7 @@ class CachePortArb : public SimObject { uint32_t delay_; uint32_t init_cycles_ = 0; std::vector rr_; + std::vector> tree_stage_; uint64_t collisions_ = 0; friend class SimObject; @@ -2165,13 +2194,30 @@ class Cache::Impl { simobject->mem_rsp_in.at(i).bind(&nc_mem_arbs_.at(i)->RspIn.at(0)); } - // Create bank's memory arbiter + // Create bank's memory arbiter. Each level of the memory-side networks + // beyond the first is a registered stage; the request path buffers the + // levels the arbiter's own register does not cover in mem_req_net_, the + // response path in the bank's memory-response queue. snprintf(sname, 100, "%s-mem_arb", simobject->name().c_str()); - auto bank_mem_arb = MemArbiter::Create(sname, ArbiterType::RoundRobin, num_banks, config_.mem_ports); + uint32_t mem_arb_delay = ((num_banks != config_.mem_ports) && (num_banks > 2 || config_.mem_ports > 2)) ? 1 : 0; + uint32_t mem_net_delay = omega_stages(num_banks, config_.mem_ports) - 1; + uint32_t mem_net_extra = (mem_net_delay > mem_arb_delay) ? (mem_net_delay - mem_arb_delay) : 0; + auto bank_mem_arb = MemArbiter::Create(sname, ArbiterType::RoundRobin, num_banks, config_.mem_ports, + mem_arb_delay + mem_net_extra, mem_arb_delay + mem_net_extra); // Connect bank's memory arbiter to non-cacheable arbiter's input port 0 + if (mem_net_extra != 0) { + mem_req_net_.reserve(config_.mem_ports); + for (uint32_t i = 0; i < config_.mem_ports; ++i) { + mem_req_net_.emplace_back(simobject, mem_arb_delay + mem_net_extra + 1); + } + } for (uint32_t i = 0; i < config_.mem_ports; ++i) { - bank_mem_arb->ReqOut.at(i).bind(&nc_mem_arbs_.at(i)->ReqIn.at(0)); + if (mem_net_extra != 0) { + bank_mem_arb->ReqOut.at(i).bind(&mem_req_net_.at(i)); + } else { + bank_mem_arb->ReqOut.at(i).bind(&nc_mem_arbs_.at(i)->ReqIn.at(0)); + } nc_mem_arbs_.at(i)->RspOut.at(0).bind(&bank_mem_arb->RspIn.at(i)); } @@ -2183,12 +2229,21 @@ class Cache::Impl { } // Create bank's core crossbar; it carries the bank responses back to the - // core ports + // core ports. A registered core-response port is a network registered at + // every level, applied as the response leaves the cache. snprintf(sname, 100, "%s-core_xbar", simobject->name().c_str()); - bank_core_xbar_ = MemCrossBar::Create(sname, ArbiterType::RoundRobin, config_.num_inputs, num_banks, - [&](const MemCrossBar::ReqType &req) { - return params_.addr_bank_id(req.addr); - }); + auto bank_sel = [&](const MemCrossBar::ReqType &req) { + return params_.addr_bank_id(req.addr); + }; + if (config_.core_out_buf) { + bank_core_xbar_ = MemCrossBar::Create(sname, ArbiterType::RoundRobin, config_.num_inputs, num_banks, bank_sel, 0, 0); + core_rsp_delay_ = omega_stages(num_banks, config_.num_inputs); + } else { + bank_core_xbar_ = MemCrossBar::Create(sname, ArbiterType::RoundRobin, config_.num_inputs, num_banks, bank_sel); + } + if (config_.nc_enable) { + ++core_rsp_delay_; + } // Create cache banks for (uint32_t i = 0, n = num_banks; i < n; ++i) { @@ -2237,16 +2292,21 @@ class Cache::Impl { } } + // bank memory requests through the memory-side network's deeper levels + for (uint32_t i = 0, n = mem_req_net_.size(); i < n; ++i) { + auto &net = mem_req_net_.at(i); + if (!net.empty() && nc_mem_arbs_.at(i)->ReqIn.at(0).try_send(net.peek(), 0)) { + net.pop(); + } + } + // schedule core responses for (uint32_t req_id = 0, n = config_.num_inputs; req_id < n; ++req_id) { auto &bank_rsp = bank_core_xbar_->RspOut.at(req_id); if (bank_rsp.empty()) continue; auto &core_rsp = bank_rsp.peek(); - // the core-response port's register, then the non-cacheable bypass - // layer's response arbiter - uint32_t rsp_delay = (config_.core_out_buf ? 1 : 0) + (config_.nc_enable ? 1 : 0); - if (simobject_->core_rsp_out.at(req_id).try_send(core_rsp, rsp_delay)) { + if (simobject_->core_rsp_out.at(req_id).try_send(core_rsp, core_rsp_delay_)) { DT(3, simobject_->name() << " core-rsp: " << core_rsp); bank_rsp.pop(); } @@ -2325,6 +2385,10 @@ class Cache::Impl { std::vector nc_mem_arbs_; CachePortArb::Ptr port_arb_; MemCrossBar::Ptr bank_core_xbar_; + // the core-response network's levels, then the non-cacheable bypass + // layer's response arbiter + uint32_t core_rsp_delay_ = 0; + std::vector> mem_req_net_; uint32_t init_cycles_; bool keep_lines_ = false; }; From f367237417e464ff6d1f50ebd495bc4046373f6b Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 18 Oct 2026 14:46:17 -0700 Subject: [PATCH 043/124] simx: register the icache core-response port as the RTL does The icache config now sets the core-response register, as the RTL's icache does, and the fetch latch it fed holds one instruction per slice instead of two. The extra entry stood in for the register and let a fetch slice run one instruction further ahead than the RTL. Co-Authored-By: Claude Opus 5.5 --- sim/simx/core.cpp | 2 +- sim/simx/socket.cpp | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index 558bf54a6e..4fdf2f54ce 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -83,7 +83,7 @@ class Core::Impl { // icache tags of the slice's in-flight fetches for (uint32_t f = 0; f < VX_CFG_FETCH_WIDTH; ++f) { snprintf(sname, 100, "fetch_latch%d", f); - fetch_latch_.emplace_back(std::make_unique>(ctx, sname, 2, 2)); + fetch_latch_.emplace_back(std::make_unique>(ctx, sname, 1, 2)); snprintf(sname, 100, "decode_latch%d", f); decode_latch_.emplace_back(std::make_unique>(ctx, sname, 1, 2)); pending_icache_.emplace_back(PER_FETCH_WARPS); diff --git a/sim/simx/socket.cpp b/sim/simx/socket.cpp index 064eaba4e2..7c8a76942f 100644 --- a/sim/simx/socket.cpp +++ b/sim/simx/socket.cpp @@ -70,6 +70,11 @@ class Socket::Impl { VX_CFG_ICACHE_LATENCY, // pipeline latency (capacity-scaled, matches hardware) VX_CFG_ICACHE_REPL_POLICY, // replacement policy false, // is_llc (icache never carries AMO state) + false, // non-cacheable bypass + false, // bank_hash + false, // shared_llc + false, // lazy_fetch + true, // core-response register }); snprintf(sname, 100, "%s-dcache", name.c_str()); From 6f3ea117ec57114c467c2cddb05346859d748a71 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 18 Oct 2026 14:46:18 -0700 Subject: [PATCH 044/124] ci: add a model_parity case for SGEMM on two frontend slices parity-sgemm-fw2 runs sgemm -n128 with 16 threads x 16 warps, ISSUE_WIDTH = 4 and FETCH_WIDTH = 2: per-slice fetch on a two-port icache, CTAs as wide as the core, and a 16-port dcache. It covers the per-partition front end on both models. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/core.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ci/testcases/core.yaml b/ci/testcases/core.yaml index 4f3f4c657e..6f9187911d 100644 --- a/ci/testcases/core.yaml +++ b/ci/testcases/core.yaml @@ -176,6 +176,14 @@ tests: args: -n128 configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 shape: {cores: 2, l2cache: true} +# SGEMM on two frontend slices feeding four issue slices: per-slice fetch on a +# two-port icache, CTAs as wide as the core, a 16-port dcache. +- id: parity-sgemm-fw2 + check: model_parity + via: blackbox + app: sgemm + args: -n128 + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_FETCH_WIDTH=2 # SGEMM with local-memory tiling (kernel-managed shared memory). - id: parity-sgemmx check: model_parity From 5f1ef4fad9487b304dcf83bbcdfac487d954b5fe Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 18 Oct 2026 14:53:11 -0700 Subject: [PATCH 045/124] Count global barriers across the whole device, not per cluster vx_num_cores() returns the device total (cores x clusters), and vortex::gbarrier passes it as the barrier count, but every cluster ran its own global-barrier unit that only ever saw its own cores: - SimX's per-cluster count could never reach the total, so any kernel using a global barrier hung on more than one cluster. - In the RTL the count and core id were NC_WIDTH (cores per cluster) wide. At 2 clusters x 1 core the global path was not even built and the scheduler timed out. At 2 clusters x 2 cores the truncated count released each cluster after its own two arrivals, so the barrier gave no ordering between clusters. One global-barrier unit now serves the device. Each cluster forwards its cores' arrivals through an arbiter in Vortex.sv (OUT_BUF 3 with several clusters, none with one) to a single VX_gbar_unit. The unit keeps one arrival counter per barrier, compared at full width, in place of the per-core mask and popcount; a core arrives once per barrier generation, so the count identifies the release. The request carries the device-global core id, and the global path is built whenever the device has more than one core. SimX adds the matching device-level GbarUnit; with one cluster both models keep their existing path and timing. rtlsim and SimX: async_gbarrier and gbar_phase pass at 2 clusters x 1 core, 2 clusters x 2 cores with L2 and L3, 2 clusters with L2 only, and 4 clusters with L3. Before the fix, gbar_phase at 2 clusters x 1 core hits the scheduler timeout on rtlsim, and every case above hangs on SimX. sgemm, raycast and gbar_phase at one cluster are cycle-identical on both models. CI adds the four 2-cluster cases on simx and rtlsim. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/regression.yaml | 43 +++++++++++++++++++++++ hw/rtl/VX_cluster.sv | 12 ++----- hw/rtl/VX_gpu_pkg.sv | 6 ++-- hw/rtl/Vortex.sv | 26 ++++++++++++++ hw/rtl/core/VX_bar_unit.sv | 24 +++++++------ hw/rtl/mem/VX_gbar_unit.sv | 25 +++++++------ hw/unittest/core/VX_core_top.sv | 4 +-- sim/simx/Makefile | 2 +- sim/simx/cluster.cpp | 58 +++++++++++++++++++++++-------- sim/simx/cluster.h | 6 ++++ sim/simx/gbar_unit.cpp | 50 ++++++++++++++++++++++++++ sim/simx/gbar_unit.h | 45 ++++++++++++++++++++++++ sim/simx/processor.cpp | 8 +++++ sim/simx/processor_impl.h | 2 ++ sw/kernel/include/vx_intrinsics.h | 2 +- 15 files changed, 259 insertions(+), 54 deletions(-) create mode 100644 sim/simx/gbar_unit.cpp create mode 100644 sim/simx/gbar_unit.h diff --git a/ci/testcases/regression.yaml b/ci/testcases/regression.yaml index af35816d28..4c16b29ba2 100644 --- a/ci/testcases/regression.yaml +++ b/ci/testcases/regression.yaml @@ -98,6 +98,49 @@ tests: app: gbar_phase shape: cores: 4 +# A global barrier spans clusters: the device-level unit counts every core, so +# one core per cluster still meets the count and two clusters of two cores do +# not release on a cluster's own arrivals. +- id: gbar-phase-cl2-1 + via: blackbox + drivers: + - simx + - rtlsim + app: gbar_phase + shape: + clusters: 2 + cores: 1 +- id: gbar-phase-cl2-2 + via: blackbox + drivers: + - simx + - rtlsim + app: gbar_phase + shape: + clusters: 2 + cores: 2 + l2cache: true + l3cache: true +- id: async-gbarrier-cl2-1 + via: blackbox + drivers: + - simx + - rtlsim + app: async_gbarrier + shape: + clusters: 2 + cores: 1 +- id: async-gbarrier-cl2-2 + via: blackbox + drivers: + - simx + - rtlsim + app: async_gbarrier + shape: + clusters: 2 + cores: 2 + l2cache: true + l3cache: true # SimX<->rtlsim cycle parity on the multi-core global barrier. Until this case # existed the gate could not observe the gbar path at all: every other parity # case is single-core, so USE_GBAR is 0 and the request queue is not built. diff --git a/hw/rtl/VX_cluster.sv b/hw/rtl/VX_cluster.sv index 79f625eda4..05fed84daa 100644 --- a/hw/rtl/VX_cluster.sv +++ b/hw/rtl/VX_cluster.sv @@ -38,6 +38,9 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; // KMU bus VX_kmu_bus_if.slave kmu_bus_if[1], + // Global barrier bus to the device-level barrier unit + VX_gbar_bus_if.master gbar_bus_if, + `ifdef VX_CFG_EXT_RASTER_ENABLE // Delegated draw launch (device KMU → raster engines) VX_raster_launch_if.slave raster_launch_if[1], @@ -161,7 +164,6 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; `endif VX_gbar_bus_if per_socket_gbar_bus_if[NUM_SOCKETS](); - VX_gbar_bus_if gbar_bus_if(); VX_gbar_arb #( .NUM_REQS (NUM_SOCKETS), @@ -174,14 +176,6 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; .bus_out_if (gbar_bus_if) ); - VX_gbar_unit #( - .INSTANCE_ID (`SFORMATF(("%s-gbar", INSTANCE_ID))) - ) gbar_unit ( - .clk (clk), - .reset (reset), - .gbar_bus_if (gbar_bus_if) - ); - // L2 input buses (socket ports + cluster-resident gfx cache ports) VX_mem_bus_if #( .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index ec92eca60a..db202a7779 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -88,7 +88,7 @@ package VX_gpu_pkg; localparam BAR_ADDR_W = `UP(BAR_ADDR_BITS); localparam BAR_ID_SHIFT = 8; - localparam BAR_SIZE_W = `MAX(`MAX(NW_WIDTH, NC_WIDTH), `CLOG2(`VX_CFG_MAX_BAR_EVENTS)); + localparam BAR_SIZE_W = `MAX(`MAX(NW_WIDTH, CORE_ID_WIDTH), `CLOG2(`VX_CFG_MAX_BAR_EVENTS)); localparam UOP_PACKLD = 0; localparam UOP_TCU = UOP_PACKLD + 1; @@ -802,8 +802,8 @@ package VX_gpu_pkg; typedef struct packed { logic [NB_WIDTH-1:0] id; - logic [NC_WIDTH-1:0] size_m1; - logic [NC_WIDTH-1:0] core_id; + logic [CORE_ID_WIDTH-1:0] size_m1; + logic [CORE_ID_WIDTH-1:0] core_id; } gbar_req_t; typedef struct packed { diff --git a/hw/rtl/Vortex.sv b/hw/rtl/Vortex.sv index e536ba88b8..b3bb5010d4 100644 --- a/hw/rtl/Vortex.sv +++ b/hw/rtl/Vortex.sv @@ -331,6 +331,30 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( .bus_out_if (per_cluster_dcr_bus_if) ); + // A global barrier's participants can span clusters, so its arrivals are + // counted once for the whole device. + VX_gbar_bus_if per_cluster_gbar_bus_if[`VX_CFG_NUM_CLUSTERS](); + VX_gbar_bus_if gbar_bus_if(); + + VX_gbar_arb #( + .NUM_REQS (`VX_CFG_NUM_CLUSTERS), + .REQ_OUT_BUF ((`VX_CFG_NUM_CLUSTERS > 1) ? 3 : 0), + .RSP_OUT_BUF ((`VX_CFG_NUM_CLUSTERS > 1) ? 3 : 0) + ) gbar_arb ( + .clk (clk), + .reset (reset), + .bus_in_if (per_cluster_gbar_bus_if), + .bus_out_if (gbar_bus_if) + ); + + VX_gbar_unit #( + .INSTANCE_ID ("gbar") + ) gbar_unit ( + .clk (clk), + .reset (reset), + .gbar_bus_if (gbar_bus_if) + ); + // Generate all clusters for (genvar cluster_id = 0; cluster_id < `VX_CFG_NUM_CLUSTERS; ++cluster_id) begin : g_clusters @@ -353,6 +377,8 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( .kmu_bus_if (per_cluster_kmu_bus_if[cluster_id +: 1]), + .gbar_bus_if (per_cluster_gbar_bus_if[cluster_id]), + `ifdef VX_CFG_EXT_RASTER_ENABLE .raster_launch_if (per_cluster_raster_launch_if[cluster_id +: 1]), `endif diff --git a/hw/rtl/core/VX_bar_unit.sv b/hw/rtl/core/VX_bar_unit.sv index f608b71e79..1ab924331a 100644 --- a/hw/rtl/core/VX_bar_unit.sv +++ b/hw/rtl/core/VX_bar_unit.sv @@ -40,12 +40,14 @@ module VX_bar_unit import VX_gpu_pkg::*; #( // warp mask + warp count + event count localparam EVENT_WIDTH = `CLOG2(`VX_CFG_MAX_BAR_EVENTS + 1); - localparam BAR_STATEW = `VX_CFG_NUM_WARPS + NW_WIDTH + EVENT_WIDTH; - localparam GBAR_REQW = NB_WIDTH + NC_WIDTH; - localparam USE_GBAR = (`VX_CFG_NUM_CORES > 1); + // the count holds warps for a local barrier, cores for a global one + localparam COUNT_W = `MAX(NW_WIDTH, CORE_ID_WIDTH); + localparam BAR_STATEW = `VX_CFG_NUM_WARPS + COUNT_W + EVENT_WIDTH; + localparam GBAR_REQW = NB_WIDTH + CORE_ID_WIDTH; + localparam USE_GBAR = ((`VX_CFG_NUM_CORES * `VX_CFG_NUM_CLUSTERS) > 1); logic [`VX_CFG_NUM_WARPS-1:0] mask_r, mask_n; - logic [NW_WIDTH-1:0] count_r, count_n; + logic [COUNT_W-1:0] count_r, count_n; logic [EVENT_WIDTH-1:0] events_r, events_n; logic phase_r, phase_n; @@ -59,14 +61,14 @@ module VX_bar_unit import VX_gpu_pkg::*; #( logic gbar_enqueue; logic [NB_WIDTH-1:0] gbar_enqueue_id; - logic [NC_WIDTH-1:0] gbar_enqueue_size_m1; + logic [CORE_ID_WIDTH-1:0] gbar_enqueue_size_m1; wire gbar_rsp_ready = ~req_valid; wire gbar_rsp_fire = gbar_bus_if.rsp_valid && gbar_rsp_ready; wire gbar_rsp_apply = USE_GBAR && gbar_rsp_fire && gbar_pending_r[gbar_bus_if.rsp_data.id]; wire [`VX_CFG_NUM_WARPS-1:0] wait_mask = ((`VX_CFG_NUM_WARPS)'(1) << req_wid) | mask_r; - wire [NW_WIDTH-1:0] next_count = count_r + NW_WIDTH'(1); + wire [COUNT_W-1:0] next_count = count_r + COUNT_W'(1); wire next_phase = ~phase_r; always @(*) begin @@ -104,7 +106,7 @@ module VX_bar_unit import VX_gpu_pkg::*; #( end end else if (req_data.is_arrive) begin // barrier arrival - if (count_r == NW_WIDTH'(req_data.size_m1)) begin + if (count_r == COUNT_W'(NW_WIDTH'(req_data.size_m1))) begin count_n = '0; if (events_r == 0) begin mask_n = '0; @@ -148,11 +150,11 @@ module VX_bar_unit import VX_gpu_pkg::*; #( mask_n = '0; gbar_enqueue = 1; // notify global barrier gbar_enqueue_id = req_data.id; - gbar_enqueue_size_m1 = NC_WIDTH'(count_r); // was saved in barrier_arrive + gbar_enqueue_size_m1 = CORE_ID_WIDTH'(count_r); // was saved in barrier_arrive end end else if (req_data.is_arrive) begin // barrier arrival - count_n = NW_WIDTH'(req_data.size_m1); // store participating number of cores + count_n = COUNT_W'(req_data.size_m1); // store participating number of cores if (req_data.is_sync) begin gbar_waiters_n[req_data.id][req_wid] = 1; end @@ -160,7 +162,7 @@ module VX_bar_unit import VX_gpu_pkg::*; #( mask_n = '0; gbar_enqueue = 1; // notify global barrier gbar_enqueue_id = req_data.id; - gbar_enqueue_size_m1 = NC_WIDTH'(req_data.size_m1); + gbar_enqueue_size_m1 = CORE_ID_WIDTH'(req_data.size_m1); end else begin // Add arriving warp to arrival mask mask_n = wait_mask; @@ -329,7 +331,7 @@ module VX_bar_unit import VX_gpu_pkg::*; #( assign gbar_bus_if.req_valid = gbar_req_valid_r; assign {gbar_bus_if.req_data.id, gbar_bus_if.req_data.size_m1} = req_queue_data; - assign gbar_bus_if.req_data.core_id = NC_WIDTH'(32'(core_id) % `VX_CFG_NUM_CORES); + assign gbar_bus_if.req_data.core_id = core_id; assign gbar_bus_if.rsp_ready = gbar_rsp_ready; `RUNTIME_ASSERT(~gbar_enqueue || ~gbar_pending_r[gbar_enqueue_id], ("%s duplicate global barrier generation: id=%0d", INSTANCE_ID, gbar_enqueue_id)) diff --git a/hw/rtl/mem/VX_gbar_unit.sv b/hw/rtl/mem/VX_gbar_unit.sv index 9af6345ef9..072105d63a 100644 --- a/hw/rtl/mem/VX_gbar_unit.sv +++ b/hw/rtl/mem/VX_gbar_unit.sv @@ -23,36 +23,35 @@ module VX_gbar_unit import VX_gpu_pkg::*; #( ); `UNUSED_SPARAM (INSTANCE_ID) - reg [`VX_CFG_NUM_BARRIERS-1:0][`VX_CFG_NUM_CORES-1:0] barrier_masks; - wire [`CLOG2(`VX_CFG_NUM_CORES+1)-1:0] active_barrier_count; + // A core sends one request per barrier generation, so counting requests + // tracks the participants without a per-core mask over the whole device. + reg [`VX_CFG_NUM_BARRIERS-1:0][CORE_ID_WIDTH-1:0] arrive_counts; wire [NB_WIDTH-1:0] req_id = gbar_bus_if.req_data.id; - wire [`VX_CFG_NUM_CORES-1:0] curr_barrier_mask = barrier_masks[req_id]; - - `POP_COUNT(active_barrier_count, curr_barrier_mask); - `UNUSED_VAR (active_barrier_count) + wire [CORE_ID_WIDTH-1:0] curr_arrive_count = arrive_counts[req_id]; + `UNUSED_VAR (gbar_bus_if.req_data.core_id) reg rsp_valid; reg [NB_WIDTH-1:0] rsp_bar_id; - wire req_file = gbar_bus_if.req_valid && gbar_bus_if.req_ready; + wire req_fire = gbar_bus_if.req_valid && gbar_bus_if.req_ready; wire rsp_fire = gbar_bus_if.rsp_valid && gbar_bus_if.rsp_ready; wire rsp_stall = gbar_bus_if.rsp_valid && ~gbar_bus_if.rsp_ready; always @(posedge clk) begin if (reset) begin - barrier_masks <= '0; + arrive_counts <= '0; rsp_valid <= 0; end else begin if (rsp_fire) begin rsp_valid <= 0; end - if (req_file) begin - if (active_barrier_count[NC_WIDTH-1:0] == gbar_bus_if.req_data.size_m1) begin - barrier_masks[req_id] <= '0; + if (req_fire) begin + if (curr_arrive_count == gbar_bus_if.req_data.size_m1) begin + arrive_counts[req_id] <= '0; rsp_valid <= 1; rsp_bar_id <= req_id; end else begin - barrier_masks[req_id][gbar_bus_if.req_data.core_id] <= 1; + arrive_counts[req_id] <= curr_arrive_count + CORE_ID_WIDTH'(1); end end end @@ -67,7 +66,7 @@ module VX_gbar_unit import VX_gpu_pkg::*; #( `ifdef DBG_TRACE_GBAR always @(posedge clk) begin - if (req_file) begin + if (req_fire) begin `TRACE(2, ("%t: %s arrive: bar_id=%0d, size=%0d, core_id=%0d\n", $time, INSTANCE_ID, gbar_bus_if.req_data.id, gbar_bus_if.req_data.size_m1, gbar_bus_if.req_data.core_id)) end diff --git a/hw/unittest/core/VX_core_top.sv b/hw/unittest/core/VX_core_top.sv index b0dbb9b44a..d47bfad744 100644 --- a/hw/unittest/core/VX_core_top.sv +++ b/hw/unittest/core/VX_core_top.sv @@ -57,8 +57,8 @@ module VX_core_top import VX_gpu_pkg::*; output wire gbar_req_valid, output wire [NB_WIDTH-1:0] gbar_req_id, - output wire [NC_WIDTH-1:0] gbar_req_size_m1, - output wire [NC_WIDTH-1:0] gbar_req_core_id, + output wire [CORE_ID_WIDTH-1:0] gbar_req_size_m1, + output wire [CORE_ID_WIDTH-1:0] gbar_req_core_id, input wire gbar_req_ready, input wire gbar_rsp_valid, input wire [NB_WIDTH-1:0] gbar_rsp_id, diff --git a/sim/simx/Makefile b/sim/simx/Makefile index 746bd6f7f8..75a9caa738 100644 --- a/sim/simx/Makefile +++ b/sim/simx/Makefile @@ -33,7 +33,7 @@ XCONFIGS := $(shell python3 $(ROOT_DIR)/ci/gen_config.py --config=$(VORTEX_HOME) # Source files definition SRCS = $(SW_COMMON_DIR)/util.cpp $(SIM_COMMON_DIR)/mem.cpp $(SW_COMMON_DIR)/softfloat_ext.cpp $(SW_COMMON_DIR)/rvfloats.cpp $(SIM_COMMON_DIR)/dram_sim.cpp $(SIM_COMMON_DIR)/cmd_processor.cpp $(SIM_COMMON_DIR)/elf_loader.cpp $(SIM_COMMON_DIR)/host_monitor.cpp $(SIM_COMMON_DIR)/cout_drainer.cpp -SRCS += $(SRC_DIR)/processor.cpp $(SRC_DIR)/cluster.cpp $(SRC_DIR)/socket.cpp $(SRC_DIR)/core.cpp $(SRC_DIR)/scheduler.cpp $(SRC_DIR)/cta_dispatcher.cpp $(SRC_DIR)/barrier_unit.cpp +SRCS += $(SRC_DIR)/processor.cpp $(SRC_DIR)/cluster.cpp $(SRC_DIR)/socket.cpp $(SRC_DIR)/core.cpp $(SRC_DIR)/scheduler.cpp $(SRC_DIR)/cta_dispatcher.cpp $(SRC_DIR)/barrier_unit.cpp $(SRC_DIR)/gbar_unit.cpp SRCS += $(SRC_DIR)/kmu/kmu.cpp SRCS += $(SRC_DIR)/decode.cpp $(SRC_DIR)/decompressor.cpp $(SRC_DIR)/scoreboard.cpp $(SRC_DIR)/sequencer.cpp $(SRC_DIR)/opc_unit.cpp $(SRC_DIR)/dispatcher.cpp SRCS += $(SRC_DIR)/alu_unit.cpp $(SRC_DIR)/fpu_unit.cpp $(SRC_DIR)/lsu_unit.cpp $(SRC_DIR)/sfu_unit.cpp $(SRC_DIR)/wctl_unit.cpp $(SRC_DIR)/csr_unit.cpp diff --git a/sim/simx/cluster.cpp b/sim/simx/cluster.cpp index 88ab5dbb20..18fa87ad96 100644 --- a/sim/simx/cluster.cpp +++ b/sim/simx/cluster.cpp @@ -12,6 +12,7 @@ // limitations under the License. #include "cluster.h" +#include #include "socket.h" #include "core.h" #include "local_mem.h" @@ -31,6 +32,11 @@ using namespace vortex; +// Output-buffer stages of the socket and cluster gbar arbiters (OUT_BUF 3, one +// cycle each), present when the arbiter has more than one input. +static constexpr uint32_t kSocketArbLag = (VX_CFG_SOCKET_SIZE > 1) ? 1 : 0; +static constexpr uint32_t kClusterArbLag = (NUM_SOCKETS > 1) ? 1 : 0; + class Cluster::Impl { public: Impl(Cluster* simobject) @@ -304,6 +310,7 @@ class Cluster::Impl { for (auto& gbar : gbarriers_) { gbar.reset(); } + gbar_fwd_next_ = 0; // Sockets are SimObjects; reset by SimPlatform. } @@ -334,10 +341,7 @@ class Cluster::Impl { auto bar_index = bar_id % gbarriers_.size(); auto& gbar = gbarriers_.at(bar_index); - auto sockets_per_cluster = sockets_.size(); - auto cores_per_socket = cores_per_socket_; - - uint32_t cores_per_cluster = sockets_per_cluster * cores_per_socket; + uint32_t cores_per_cluster = sockets_.size() * cores_per_socket_; uint32_t local_core_id = core_id % cores_per_cluster; // set core arrival bit @@ -345,19 +349,37 @@ class Cluster::Impl { DT(4, "*** Global barrier arrive: cluster #" << simobject_->id() << ", core #" << core_id << " at barrier #" << bar_id << ", arrived=" << gbar.mask.count()); - if (gbar.mask.count() == (size_t)count) { - // resume all suspended cores - for (uint32_t s = 0; s < sockets_per_cluster; ++s) { - for (uint32_t c = 0; c < cores_per_socket; ++c) { - uint32_t i = s * cores_per_socket + c; - if (gbar.mask.test(i)) { - simobject_->gbar_resume_out.at(i).send({bar_id}); - } + if constexpr (VX_CFG_NUM_CLUSTERS > 1) { + // Up to the device unit: the socket arbiter's output buffer, the cluster + // arbiter (one arrival per cycle) and its output buffer, then the device + // arbiter's output buffer. + uint64_t now = SimPlatform::instance().cycles(); + uint64_t out = std::max(now + kSocketArbLag, gbar_fwd_next_); + gbar_fwd_next_ = out + 1; + simobject_->gbar_fwd_out.send({bar_id, count, core_id}, out + kClusterArbLag + 1 - now); + } else if (gbar.mask.count() == (size_t)count) { + this->global_barrier_resume(bar_id, 1); + } + } + + // The device unit's release crosses the cluster's and the socket's response + // forks on its way down. + void global_barrier_fwd_resume(uint32_t bar_id) { + this->global_barrier_resume(bar_id, 1 + kClusterArbLag + kSocketArbLag); + } + + // Resume the cores of this cluster that arrived, then clear the barrier. + void global_barrier_resume(uint32_t bar_id, uint32_t delay) { + auto& gbar = gbarriers_.at(bar_id % gbarriers_.size()); + for (uint32_t s = 0; s < sockets_.size(); ++s) { + for (uint32_t c = 0; c < cores_per_socket_; ++c) { + uint32_t i = s * cores_per_socket_ + c; + if (gbar.mask.test(i)) { + simobject_->gbar_resume_out.at(i).send({bar_id}, delay); } } - // reset mask and advance phase - gbar.mask.reset(); } + gbar.mask.reset(); } Cluster::PerfStats perf_stats() const { @@ -568,6 +590,7 @@ class Cluster::Impl { Cluster* simobject_; std::vector sockets_; std::vector gbarriers_; + uint64_t gbar_fwd_next_ = 0; // next free cycle at the cluster's gbar arbiter Cache::Ptr l2cache_; uint32_t cores_per_socket_; #ifdef VX_CFG_VM_ENABLE @@ -598,11 +621,14 @@ Cluster::Cluster(const SimContext& ctx, , mem_rsp_in(VX_CFG_L2_MEM_PORTS, this) , gbar_arrive_in(this) , gbar_resume_out(NUM_SOCKETS * VX_CFG_SOCKET_SIZE, this) + , gbar_fwd_out(this) + , gbar_fwd_resume_in(this) , cluster_id_(cluster_id) , processor_(processor) , impl_(new Impl(this)) { gbar_arrive_in.bind(this, &Cluster::on_gbar_arrive); + gbar_fwd_resume_in.bind(this, &Cluster::on_gbar_fwd_resume); } Cluster::~Cluster() { @@ -625,6 +651,10 @@ void Cluster::on_gbar_arrive(const GbarArrive& msg) { impl_->global_barrier_arrive(msg.bar_id, msg.count, msg.core_id); } +void Cluster::on_gbar_fwd_resume(const GbarResume& msg) { + impl_->global_barrier_fwd_resume(msg.bar_id); +} + Cluster::PerfStats Cluster::perf_stats() const { return impl_->perf_stats(); } diff --git a/sim/simx/cluster.h b/sim/simx/cluster.h index bc7fa210a4..1bcc518427 100644 --- a/sim/simx/cluster.h +++ b/sim/simx/cluster.h @@ -77,6 +77,11 @@ class Cluster : public SimObject { SimEventLink gbar_arrive_in; std::vector> gbar_resume_out; + // With more than one cluster the barrier spans clusters: arrivals go up to + // the device-level GbarUnit and its resume comes back down here. + SimEventLink gbar_fwd_out; + SimEventLink gbar_fwd_resume_in; + Cluster(const SimContext& ctx, const char* name, uint32_t cluster_id, @@ -151,6 +156,7 @@ class Cluster : public SimObject { private: void on_gbar_arrive(const GbarArrive& msg); + void on_gbar_fwd_resume(const GbarResume& msg); uint32_t cluster_id_; ProcessorImpl* processor_; diff --git a/sim/simx/gbar_unit.cpp b/sim/simx/gbar_unit.cpp new file mode 100644 index 0000000000..f331c6272c --- /dev/null +++ b/sim/simx/gbar_unit.cpp @@ -0,0 +1,50 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "gbar_unit.h" +#include +#include "debug.h" + +using namespace vortex; + +GbarUnit::GbarUnit(const SimContext& ctx, const char* name, uint32_t num_clusters) + : SimObject(ctx, name) + , arrive_in(this) + , resume_out(num_clusters, this) + , arrived_(VX_CFG_NUM_BARRIERS, 0) + , accept_next_(0) +{ + arrive_in.bind(this, &GbarUnit::on_arrive); +} + +void GbarUnit::on_reset() { + std::fill(arrived_.begin(), arrived_.end(), 0); + accept_next_ = 0; +} + +// A core arrives once per generation, so a count of arrivals tracks the +// participants without a mask over every core of the device. +void GbarUnit::on_arrive(const GbarArrive& msg) { + uint64_t now = SimPlatform::instance().cycles(); + uint64_t accept = std::max(now, accept_next_); + accept_next_ = accept + 1; + auto& arrived = arrived_.at(msg.bar_id % arrived_.size()); + ++arrived; + DT(4, "*** Device barrier arrive: core #" << msg.core_id << " at barrier #" << msg.bar_id << ", arrived=" << arrived); + if (arrived == msg.count) { + for (auto& link : resume_out) { + link.send({msg.bar_id}, accept - now + 1); + } + arrived = 0; + } +} diff --git a/sim/simx/gbar_unit.h b/sim/simx/gbar_unit.h new file mode 100644 index 0000000000..fc06fe8fe8 --- /dev/null +++ b/sim/simx/gbar_unit.h @@ -0,0 +1,45 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include "types.h" + +namespace vortex { + +// Device-level global barrier: with more than one cluster, a barrier's +// participants span clusters, so each cluster forwards its cores' arrivals +// here and every cluster is resumed once the count is met. Arrivals enter +// through the device arbiter's output buffer, one per cycle; the release is +// registered in the unit and again in the response fork to the clusters. +class GbarUnit : public SimObject { +public: + SimEventLink arrive_in; + std::vector> resume_out; + + GbarUnit(const SimContext& ctx, const char* name, uint32_t num_clusters); + +protected: + void on_reset(); + +private: + void on_arrive(const GbarArrive& msg); + + std::vector arrived_; + uint64_t accept_next_; // first cycle the unit can take another arrival + + friend class SimObject; +}; + +} // namespace vortex diff --git a/sim/simx/processor.cpp b/sim/simx/processor.cpp index 77163fe4df..268773e2d8 100644 --- a/sim/simx/processor.cpp +++ b/sim/simx/processor.cpp @@ -63,6 +63,14 @@ ProcessorImpl::ProcessorImpl() clusters_.at(i) = Cluster::Create(sname, i, this); } + if constexpr (VX_CFG_NUM_CLUSTERS > 1) { + gbar_unit_ = GbarUnit::Create("gbar", VX_CFG_NUM_CLUSTERS); + for (uint32_t i = 0; i < VX_CFG_NUM_CLUSTERS; ++i) { + clusters_.at(i)->gbar_fwd_out.bind(&gbar_unit_->arrive_in); + gbar_unit_->resume_out.at(i).bind(&clusters_.at(i)->gbar_fwd_resume_in); + } + } + // Launch bus: a registered, credit-gated lane from the KMU to each core's // dispatcher; CTAs cross into the core's domain through the slice, and // admission credits return to the KMU through a slice of their own. diff --git a/sim/simx/processor_impl.h b/sim/simx/processor_impl.h index 3e2f697b87..50ecc05106 100644 --- a/sim/simx/processor_impl.h +++ b/sim/simx/processor_impl.h @@ -18,6 +18,7 @@ #include "constants.h" #include "cluster.h" #include "kmu.h" +#include "gbar_unit.h" namespace vortex { @@ -85,6 +86,7 @@ class ProcessorImpl { Kmu::Ptr kmu_; std::vector clusters_; + GbarUnit::Ptr gbar_unit_; // device-level global barrier (multi-cluster) Memory::Ptr memsim_; #ifdef VX_CFG_VM_ENABLE uint64_t mmu_satp_ = 0; // assembled from the two DCR halves diff --git a/sw/kernel/include/vx_intrinsics.h b/sw/kernel/include/vx_intrinsics.h index 78468f7744..b4ed86b144 100644 --- a/sw/kernel/include/vx_intrinsics.h +++ b/sw/kernel/include/vx_intrinsics.h @@ -204,7 +204,7 @@ inline __attribute__((const)) int vx_num_warps() { return (int)csr_read_nv(VX_CSR_NUM_WARPS); } -// Return the number of cores per cluster +// Return the number of cores across all clusters inline __attribute__((const)) int vx_num_cores() { return (int)csr_read_nv(VX_CSR_NUM_CORES); } From 3f27ba402b36c1b59786e47d82591ab28488f670 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 18 Oct 2026 17:13:31 -0700 Subject: [PATCH 046/124] Stop fences at the point of coherence and skip the walk for IO fences Every RISC-V fence went to the data cache as a full flush token: each cache walked all its lines and forwarded the token, so a fence walked the L1, the L2 and the L3, and locked each cache's inputs for the walk. On printf-style kernels that fence after every device write, the walks dominated run time. Fences now take one of two lighter forms: - A memory fence (any R or W in its predecessor or successor set) sends an L1-only flush token. Each private level walks and forwards it, and the shared LLC, the point of coherence, answers without a walk. A replicated LLC still walks, since data crosses it between cores. - A fence ordering only device accesses (no R or W) decodes as a new LSU op and sends an IO token (is_flush with the io attribute). No level walks. Each cache waits until nothing is held between its inputs and its memory ports, on the cacheable and the non-cacheable path, then sends the token down every memory port, so its answer proves the earlier IO writes reached memory. The non-cacheable bypass always routes a flush token to the cache, whatever its io bit. Tokens of different kinds that meet in one cache combine to the weakest kind that serves them all. fence_mp is a new message-passing test across cores, with one case per topology: 1 core, 2 cores + L2, 2 clusters x 1 core + L2, and 2 clusters x 2 cores + L2 + L3 (rtlsim). It fails when the L2 is wrongly treated as the shared LLC on 2 clusters. rtlsim before -> after: default 2 cores + L2 printf 437,106 -> 249,404 4,006,822 -> 295,165 (13.6x) fence 6,888 -> 6,263 15,163 -> 6,798 async_gbar 67,262 -> 67,262 77,779 -> 67,402 sgemm, raycast and dogfood are cycle-identical. At 2 clusters x 2 cores + L2 + L3 the four apps above do not change. SimX does not model the tokens yet: there a fence is an ordering barrier in the LSU. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/regression.yaml | 36 ++++++ docs/designs/cache_subsystem.md | 8 +- docs/designs/lsu_pipeline_design.md | 4 +- docs/designs/multicache_amo_coherence.md | 35 +++--- hw/rtl/VX_gpu_pkg.sv | 7 +- hw/rtl/VX_trace_pkg.sv | 1 + hw/rtl/cache/VX_cache.sv | 15 ++- hw/rtl/cache/VX_cache_bank.sv | 2 + hw/rtl/cache/VX_cache_bypass.sv | 4 +- hw/rtl/cache/VX_cache_flush.sv | 7 +- hw/rtl/cache/VX_cache_init.sv | 53 +++++++-- hw/rtl/cache/VX_cache_wrap.sv | 3 +- hw/rtl/core/VX_decode.sv | 5 +- hw/rtl/core/VX_lsu_block.sv | 13 ++- tests/regression/Makefile | 2 +- tests/regression/fence_mp/Makefile | 16 +++ tests/regression/fence_mp/common.h | 18 +++ tests/regression/fence_mp/kernel.cpp | 45 ++++++++ tests/regression/fence_mp/main.cpp | 135 +++++++++++++++++++++++ 19 files changed, 369 insertions(+), 40 deletions(-) create mode 100644 tests/regression/fence_mp/Makefile create mode 100644 tests/regression/fence_mp/common.h create mode 100644 tests/regression/fence_mp/kernel.cpp create mode 100644 tests/regression/fence_mp/main.cpp diff --git a/ci/testcases/regression.yaml b/ci/testcases/regression.yaml index 4c16b29ba2..b59f66d3a7 100644 --- a/ci/testcases/regression.yaml +++ b/ci/testcases/regression.yaml @@ -238,6 +238,42 @@ tests: cores: 2 l2cache: true l3cache: true +# A memory fence carries data from one core to another across the caches they +# do not share: the producer's token reaches the point of coherence, the +# consumer's invalidates its copies above it. Two clusters with an L2 each +# replicate the last level, so the fence must walk both L2s there. +- id: fence_mp-1 + via: blackbox + drivers: + - rtlsim + app: fence_mp +- id: fence_mp-2 + via: blackbox + drivers: + - rtlsim + app: fence_mp + shape: + cores: 2 + l2cache: true +- id: fence_mp-3 + via: blackbox + drivers: + - rtlsim + app: fence_mp + shape: + clusters: 2 + cores: 1 + l2cache: true +- id: fence_mp-4 + via: blackbox + drivers: + - rtlsim + app: fence_mp + shape: + clusters: 2 + cores: 2 + l2cache: true + l3cache: true # Kernel to kernel with no host transfer between (one command batch): the # launch flushes are L1-only, so the last-level cache alone carries each # round's writes to the next round, and the full flush runs before readback. diff --git a/docs/designs/cache_subsystem.md b/docs/designs/cache_subsystem.md index 7f7c26e390..80c5038263 100644 --- a/docs/designs/cache_subsystem.md +++ b/docs/designs/cache_subsystem.md @@ -615,8 +615,12 @@ write's `VX_CACHE_FLUSH_L1_ONLY` bit, and the dcache and icache injectors carry it. The ocache, whose tokens reach the L2, sends an L1-only token only when every requester of its flush round is L1-only: the cluster flush (kind latched from the same broadcast write) and the dcache tokens held at the OM steers. -The other graphics caches do not forward and send full tokens. A fence -carries no byte enables, so a fence token is always a full one. +The other graphics caches do not forward and send full tokens. A memory fence +sends an L1-only token. A fence ordering only device accesses sends an **IO** +token (the `io` attribute): no level walks, each quiesces its banks, waits +until nothing is held on its uncached path either, and forwards an IO token +down every memory port, so the answer proves the earlier IO writes reached +memory. Kinds visible together combine to the weakest that serves them all. **Core traffic sits on the priority input, not the flush.** The flush waits until upstream is quiescent — which it reliably is, because the host only issues diff --git a/docs/designs/lsu_pipeline_design.md b/docs/designs/lsu_pipeline_design.md index 025c834d9f..1e3531a12d 100644 --- a/docs/designs/lsu_pipeline_design.md +++ b/docs/designs/lsu_pipeline_design.md @@ -148,7 +148,9 @@ A fence is a **total barrier on its slice**. The slice sets `fence_lock` when a fence's last packet is accepted and clears it when that fence's response returns. While locked, the slice accepts nothing. -The fence itself travels to memory as a request with `is_flush`, behind every +The fence itself travels to memory as a request with `is_flush` (an IO token, +the `io` attribute set, for a fence ordering only device accesses; otherwise an +L1-only token, byte enables set), behind every request the slice issued before it, so its response proves they have drained. Only the last packet of a multi-packet fence is sent; the earlier packets complete immediately. diff --git a/docs/designs/multicache_amo_coherence.md b/docs/designs/multicache_amo_coherence.md index e6ad6901ea..b0233a0252 100644 --- a/docs/designs/multicache_amo_coherence.md +++ b/docs/designs/multicache_amo_coherence.md @@ -240,19 +240,28 @@ updates it, core 0's copy is stale. The LLC never reaches up. The weak model resolves this the way a GPU does: the **consumer** discards its own copies at an acquire point and refetches. -There is no dedicated acquire-invalidate. What exists today: - -| model | what a `fence` does to the issuing core's data cache | -|---|---| -| RTL | the LSU sends the fence as a request with `is_flush` set, which starts the **whole-cache flush**; on a write-through cache the walk clears every valid bit | -| SimX | the fence is an ordering barrier in the LSU only; a flush of a write-through cache is a no-op and no line is invalidated | - -So an RTL `fence` happens to give the consumer-side invalidate, at the cost -of a full flush walk that locks the cache's inputs for its duration, and -SimX does not model it. **The two models disagree, and no regression case -exercises fence-managed cross-core visibility** — `atomic_critical`, which -runs plain loads and stores inside a lock, is skipped on multi-core -configurations for exactly this reason. +There is no dedicated acquire-invalidate. In the RTL the LSU sends a memory +`fence` (any R or W in its predecessor or successor set) to the data cache as +an **L1-only** flush token (`is_flush` with byte enables set). Each cache above +the point of coherence holds its inputs, drains, walks every set (every way of +a write-back cache), writing back dirty lines and clearing every valid bit, and +forwards the token. The shared LLC, the point of coherence, answers it without +a walk; a replicated LLC still walks. The fence holds its LSU block until the +answer. + +So a memory `fence` gives the consumer-side invalidate above the point of +coherence, at the cost of a flush walk of each private level. `fence_mp` +passes data between cores this way on shared and replicated last levels. +`atomic_critical`, which runs plain loads and stores inside a lock, is still +skipped on multi-core configurations. + +A fence ordering only device accesses (`fence ow, ow`, no R or W) sends an +**IO** token (`is_flush` with `io`): no level walks; each quiesces and waits +until its uncached path is empty, then forwards, so the answer proves the +earlier IO writes reached memory. + +SimX does not model the tokens yet: there the fence is an ordering barrier in +the LSU only. Programs that share data across cores through plain memory are outside what is validated. Share through atomics, or through the single shared level. diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index db202a7779..c18e3050cc 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -488,6 +488,7 @@ package VX_gpu_pkg; localparam INST_LSU_SH = 4'b1001; localparam INST_LSU_SW = 4'b1010; localparam INST_LSU_SD = 4'b1011; // new for RV64I SD + localparam INST_LSU_FENCE_IO = 4'b1110; // a fence ordering only device (IO) accesses localparam INST_LSU_FENCE = 4'b1111; localparam INST_LSU_BITS = 4; @@ -504,7 +505,11 @@ package VX_gpu_pkg; endfunction function automatic logic inst_lsu_is_fence(input logic [INST_LSU_BITS-1:0] op); - return (op == INST_LSU_FENCE); + return (op[3:1] == 3'b111); + endfunction + + function automatic logic inst_lsu_is_fence_io(input logic [INST_LSU_BITS-1:0] op); + return (op == INST_LSU_FENCE_IO); endfunction /////////////////////////////////////////////////////////////////////////// diff --git a/hw/rtl/VX_trace_pkg.sv b/hw/rtl/VX_trace_pkg.sv index dd16cd3dc7..68139de7cc 100644 --- a/hw/rtl/VX_trace_pkg.sv +++ b/hw/rtl/VX_trace_pkg.sv @@ -202,6 +202,7 @@ package VX_trace_pkg; INST_LSU_SW: `TRACE(level, ("SW")) INST_LSU_SD: `TRACE(level, ("SD")) INST_LSU_FENCE:`TRACE(level,("FENCE")) + INST_LSU_FENCE_IO:`TRACE(level,("FENCE.IO")) default: `TRACE(level, ("?")) endcase end diff --git a/hw/rtl/cache/VX_cache.sv b/hw/rtl/cache/VX_cache.sv index 3ee2bae7aa..dce6f187a7 100644 --- a/hw/rtl/cache/VX_cache.sv +++ b/hw/rtl/cache/VX_cache.sv @@ -64,7 +64,9 @@ module VX_cache import VX_gpu_pkg::*; #( input wire clk, input wire reset, VX_mem_bus_if.slave core_bus_if [NUM_REQS], - VX_mem_bus_if.master mem_bus_if [MEM_PORTS] + VX_mem_bus_if.master mem_bus_if [MEM_PORTS], + // nothing held between this level's inputs and its memory side, on either path + input wire nc_req_empty ); `STATIC_ASSERT(NUM_BANKS == (1 << `CLOG2(NUM_BANKS)), ("invalid parameter: number of banks must be power of 2")) `STATIC_ASSERT(WRITE_ENABLE || !WRITEBACK, ("invalid parameter: writeback requires write enable")) @@ -104,12 +106,14 @@ module VX_cache import VX_gpu_pkg::*; #( .TAG_WIDTH (TAG_WIDTH) ) core_bus2_if[NUM_REQS](); wire [NUM_BANKS-1:0] per_bank_flush_begin; + wire flush_skip; wire [`UP(UUID_WIDTH)-1:0] flush_uuid; wire [NUM_BANKS-1:0] per_bank_flush_end; wire [NUM_BANKS-1:0] per_bank_core_req_fire; wire mem_req_empty; wire [MEM_PORTS-1:0] mem_flush_valid; wire mem_flush_l1_only; + wire mem_flush_io; wire [MEM_PORTS-1:0] mem_flush_ready; wire mem_flush_rsp_phase; wire [MEM_PORTS-1:0] mem_flush_rsp_valid; @@ -133,11 +137,14 @@ module VX_cache import VX_gpu_pkg::*; #( .core_bus_out_if (core_bus2_if), .bank_req_fire (per_bank_core_req_fire), .flush_begin (per_bank_flush_begin), + .flush_skip (flush_skip), .flush_uuid (flush_uuid), .flush_end (per_bank_flush_end), .mem_req_empty (mem_req_empty), + .nc_req_empty (nc_req_empty), .mem_flush_valid (mem_flush_valid), .mem_flush_l1_only (mem_flush_l1_only), + .mem_flush_io (mem_flush_io), .mem_flush_ready (mem_flush_ready), .mem_flush_rsp_phase (mem_flush_rsp_phase), .mem_flush_rsp_valid (mem_flush_rsp_valid), @@ -417,6 +424,7 @@ module VX_cache import VX_gpu_pkg::*; #( .mem_rsp_tag (per_bank_mem_rsp_tag[bank_id]), .mem_rsp_ready (per_bank_mem_rsp_ready[bank_id]), .flush_begin (per_bank_flush_begin[bank_id]), + .flush_skip (flush_skip), .flush_uuid (flush_uuid), .flush_end (per_bank_flush_end[bank_id]) ); @@ -507,7 +515,8 @@ module VX_cache import VX_gpu_pkg::*; #( wire [MEM_PORTS-1:0][`UP(`LOG2UP(NUM_BANKS))-1:0] mem_req_tmp_idx; wire [MEM_PORTS-1:0] mem_req_tmp_ready; // The flush token for port i is a read of the port's first line with the - // flush attribute; byteen bit 0 carries the L1-only kind. It takes bank + // flush attribute; byteen bit 0 carries the L1-only kind and the io + // attribute the IO kind. It takes bank // i's lane into the request crossbar (bank i maps to port i), so the // crossbar forms its address and it leaves through the same registered // outputs as the write-backs ahead of it. @@ -524,7 +533,7 @@ module VX_cache import VX_gpu_pkg::*; #( SECTOR_SIZE'(mem_flush_l1_only), `CS_SECTOR_WIDTH'(0), BANK_MEM_TAG_WIDTH'(0), - `UP(MEM_ATTR_WIDTH)'(1 << MEM_ATTR_FLUSH_OFFS) + `UP(MEM_ATTR_WIDTH)'((1 << MEM_ATTR_FLUSH_OFFS) | (32'(mem_flush_io) << MEM_ATTR_IO_OFFS)) }; assign mem_req_xbar_valid[i] = per_bank_mem_req_valid[i] || mem_flush_valid[i]; assign mem_req_xbar_pdata[i] = per_bank_mem_req_valid[i] ? per_bank_mem_req_pdata[i] : mem_flush_pdata; diff --git a/hw/rtl/cache/VX_cache_bank.sv b/hw/rtl/cache/VX_cache_bank.sv index 90d8f1ddcb..6137bff9fc 100644 --- a/hw/rtl/cache/VX_cache_bank.sv +++ b/hw/rtl/cache/VX_cache_bank.sv @@ -91,6 +91,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // Flush input wire flush_begin, + input wire flush_skip, input wire [`UP(UUID_WIDTH)-1:0] flush_uuid, output wire flush_end ); @@ -308,6 +309,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .clk (clk), .reset (reset), .flush_begin (flush_begin), + .flush_skip (flush_skip), .flush_end (flush_end), .flush_init (init_valid), .flush_valid (flush_valid), diff --git a/hw/rtl/cache/VX_cache_bypass.sv b/hw/rtl/cache/VX_cache_bypass.sv index 29232bc2fa..950f434392 100644 --- a/hw/rtl/cache/VX_cache_bypass.sv +++ b/hw/rtl/cache/VX_cache_bypass.sv @@ -61,7 +61,9 @@ module VX_cache_bypass import VX_gpu_pkg::*; #( wire [NUM_REQS-1:0] core_req_nc_sel; for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_req_is_nc if (CACHE_ENABLE) begin : g_cache - assign core_req_nc_sel[i] = ~core_bus_in_if[i].req_data.attr[MEM_ATTR_IO_OFFS]; + // a flush token always reaches the cache, whatever its io kind bit + assign core_req_nc_sel[i] = ~core_bus_in_if[i].req_data.attr[MEM_ATTR_IO_OFFS] + | core_bus_in_if[i].req_data.attr[MEM_ATTR_FLUSH_OFFS]; end else begin : g_no_cache assign core_req_nc_sel[i] = 1'b0; end diff --git a/hw/rtl/cache/VX_cache_flush.sv b/hw/rtl/cache/VX_cache_flush.sv index c51a5aa2cc..d8227664e6 100644 --- a/hw/rtl/cache/VX_cache_flush.sv +++ b/hw/rtl/cache/VX_cache_flush.sv @@ -23,6 +23,7 @@ module VX_cache_flush import VX_gpu_pkg::*; #( input wire clk, input wire reset, input wire flush_begin, + input wire flush_skip, // the flush only quiesces the bank: an IO token output wire flush_end, output wire flush_init, output wire flush_valid, @@ -48,6 +49,7 @@ module VX_cache_flush import VX_gpu_pkg::*; #( // latch flush_begin that arrives while init is in progress reg flush_pending_r, flush_pending_n; + reg skip_r; always @(*) begin state_n = state; @@ -74,7 +76,7 @@ module VX_cache_flush import VX_gpu_pkg::*; #( // Wait for the bank to fully quiesce before evicting: // both MSHR must drain, and the bank pipeline as well to ensure no inflight misses. if (mshr_empty && bank_empty) begin - state_n = STATE_FLUSH; + state_n = skip_r ? STATE_WAIT2 : STATE_FLUSH; end end STATE_FLUSH: begin @@ -104,6 +106,9 @@ module VX_cache_flush import VX_gpu_pkg::*; #( end else begin state <= state_n; flush_pending_r <= flush_pending_n; + if (flush_begin) begin + skip_r <= flush_skip; + end if (state != STATE_IDLE) begin if ((state == STATE_INIT) || ((state == STATE_FLUSH) && flush_ready)) begin diff --git a/hw/rtl/cache/VX_cache_init.sv b/hw/rtl/cache/VX_cache_init.sv index 9dc988ff3a..ce44844a53 100644 --- a/hw/rtl/cache/VX_cache_init.sv +++ b/hw/rtl/cache/VX_cache_init.sv @@ -31,6 +31,11 @@ // the token without a walk. Any other last-level cache still walks, since a // replicated instance or a client reaching memory beside it may hold or need // lines it has not written back. +// An IO token (the io attribute: a fence ordering only device accesses) walks +// nowhere. Once nothing is held between the cache's inputs and its memory +// ports, on the cacheable and the non-cacheable path alike, it goes down every +// memory port, so its answer proves the earlier IO writes reached memory. +// Kinds visible together combine to the weakest one that serves them all. module VX_cache_init import VX_gpu_pkg::*; #( parameter NUM_REQS = 4, // Number of Word requests per cycle parameter NUM_BANKS = 1, // Number of banks @@ -47,12 +52,15 @@ module VX_cache_init import VX_gpu_pkg::*; #( VX_mem_bus_if.master core_bus_out_if [NUM_REQS], input wire [NUM_BANKS-1:0] bank_req_fire, output wire [NUM_BANKS-1:0] flush_begin, + output wire flush_skip, // the banks only quiesce: no walk is owed output wire [`UP(UUID_WIDTH)-1:0] flush_uuid, input wire [NUM_BANKS-1:0] flush_end, // flush tokens to the next level input wire mem_req_empty, // no write-back left between the banks and the ports + input wire nc_req_empty, // no request held between the inputs and the memory side output wire [MEM_PORTS-1:0] mem_flush_valid, output wire mem_flush_l1_only, + output wire mem_flush_io, input wire [MEM_PORTS-1:0] mem_flush_ready, output wire mem_flush_rsp_phase, input wire [MEM_PORTS-1:0] mem_flush_rsp_valid, @@ -119,12 +127,18 @@ module VX_cache_init import VX_gpu_pkg::*; #( end wire flush_req_enable = (| flush_req_mask); - wire [NUM_REQS-1:0] flush_l1_mask; - for (genvar i = 0; i < NUM_REQS; ++i) begin : g_flush_l1_mask - assign flush_l1_mask[i] = flush_req_mask[i] && (| core_bus_in_if[i].req_data.byteen); + wire [NUM_REQS-1:0] flush_l1_mask, flush_io_mask; + for (genvar i = 0; i < NUM_REQS; ++i) begin : g_flush_kind_mask + assign flush_io_mask[i] = flush_req_mask[i] && core_bus_in_if[i].req_data.attr[MEM_ATTR_IO_OFFS]; + assign flush_l1_mask[i] = flush_req_mask[i] && ~flush_io_mask[i] && (| core_bus_in_if[i].req_data.byteen); end + wire any_full = (| (flush_req_mask & ~flush_l1_mask & ~flush_io_mask)); + wire any_l1 = (| flush_l1_mask); + wire any_io = (| flush_io_mask); // every visible token is L1-only - wire flush_l1_only = (flush_l1_mask == flush_req_mask); + wire flush_l1_only = any_l1 && ~any_full && ~any_io; + // a walk is owed to a full token, and to an L1-only one above the point of coherence + wire flush_walk = any_full || (any_l1 && ~SHARED_LLC); reg [NUM_BANKS-1:0] flush_done, flush_done_n; reg fwd_go, fwd_go_n; // the crossbar drained; only tokens enter it now @@ -133,6 +147,8 @@ module VX_cache_init import VX_gpu_pkg::*; #( reg [NUM_REQS-1:0] rsp_pending, rsp_pending_n; // token still to be answered reg [`UP(UUID_WIDTH)-1:0] flush_uuid_r, flush_uuid_n; reg l1_only_r, l1_only_n; + reg io_r, io_n; + reg skip_r, skip_n; // answer the lowest pending input; its token stays at the input until then wire [NUM_REQS-1:0] rsp_sel_onehot; @@ -213,6 +229,10 @@ module VX_cache_init import VX_gpu_pkg::*; #( end end + // an IO token also waits for the non-cacheable path, which carries the IO + // writes it orders + wire fwd_ready = mem_req_empty && (~io_r || nc_req_empty); + always @(*) begin state_n = state; flush_done_n = flush_done; @@ -222,16 +242,19 @@ module VX_cache_init import VX_gpu_pkg::*; #( rsp_pending_n = rsp_pending & ~(rsp_sel_onehot & {NUM_REQS{flush_rsp_fire}}); flush_uuid_n = flush_uuid_r; l1_only_n = l1_only_r; + io_n = io_r; + skip_n = skip_r; case (state) // STATE_IDLE: default: begin if (flush_req_enable) begin if (BANK_SEL_LATENCY != 0) begin state_n = STATE_WAIT1; - end else if (SHARED_LLC && flush_l1_only) begin + end else if (~flush_walk && ~any_io) begin rsp_pending_n = flush_req_mask; state_n = STATE_RESPOND; end else begin + skip_n = ~flush_walk; state_n = STATE_FLUSH; end for (integer i = NUM_REQS-1; i >= 0; --i) begin @@ -243,10 +266,11 @@ module VX_cache_init import VX_gpu_pkg::*; #( end STATE_WAIT1: begin if (no_inflight_reqs) begin - if (SHARED_LLC && flush_l1_only) begin + if (~flush_walk && ~any_io) begin rsp_pending_n = flush_req_mask; state_n = STATE_RESPOND; end else begin + skip_n = ~flush_walk; state_n = STATE_FLUSH; end end @@ -260,14 +284,17 @@ module VX_cache_init import VX_gpu_pkg::*; #( flush_done_n = flush_done | flush_end; if (flush_done_n == {NUM_BANKS{1'b1}}) begin flush_done_n = '0; - // answer the tokens visible now; later ones start a new walk - rsp_pending_n = flush_req_mask; - l1_only_n = flush_l1_only; + // answer the tokens visible now; later ones start a new walk. + // A quiesce without a walk serves only the tokens owing none. + rsp_pending_n = skip_r ? (flush_io_mask | (SHARED_LLC ? flush_l1_mask : '0)) + : flush_req_mask; + l1_only_n = flush_l1_only && ~skip_r; + io_n = skip_r; state_n = FLUSH_FORWARD ? STATE_FORWARD : STATE_RESPOND; end end STATE_FORWARD: begin - fwd_go_n = fwd_go | mem_req_empty; + fwd_go_n = fwd_go | fwd_ready; fwd_sent_n = fwd_sent | (mem_flush_valid & mem_flush_ready); fwd_done_n = fwd_done | mem_flush_rsp_valid; if (fwd_done_n == {MEM_PORTS{1'b1}}) begin @@ -303,15 +330,19 @@ module VX_cache_init import VX_gpu_pkg::*; #( end flush_uuid_r <= flush_uuid_n; l1_only_r <= l1_only_n; + io_r <= io_n; + skip_r <= skip_n; end assign flush_begin = {NUM_BANKS{state == STATE_FLUSH}}; + assign flush_skip = skip_r; assign flush_uuid = flush_uuid_r; // The drain counter counts the tokens too, so the first one sent must not // take the others' valid away. - assign mem_flush_valid = {MEM_PORTS{(state == STATE_FORWARD) && (fwd_go || mem_req_empty)}} & ~fwd_sent; + assign mem_flush_valid = {MEM_PORTS{(state == STATE_FORWARD) && (fwd_go || fwd_ready)}} & ~fwd_sent; assign mem_flush_rsp_phase = (state == STATE_FORWARD); assign mem_flush_l1_only = l1_only_r; + assign mem_flush_io = io_r; endmodule diff --git a/hw/rtl/cache/VX_cache_wrap.sv b/hw/rtl/cache/VX_cache_wrap.sv index 0f87a80e45..2471155263 100644 --- a/hw/rtl/cache/VX_cache_wrap.sv +++ b/hw/rtl/cache/VX_cache_wrap.sv @@ -169,7 +169,8 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( .cache_perf (cache_perf), `endif .core_bus_if (core_bus_cache_if), - .mem_bus_if (mem_bus_cache_if) + .mem_bus_if (mem_bus_cache_if), + .nc_req_empty (req_empty) ); end else begin : g_passthru for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_bus_cache_if diff --git a/hw/rtl/core/VX_decode.sv b/hw/rtl/core/VX_decode.sv index 9c2a3fa44d..538164a70a 100644 --- a/hw/rtl/core/VX_decode.sv +++ b/hw/rtl/core/VX_decode.sv @@ -349,7 +349,10 @@ module VX_decode import end INST_FENCE: begin ex_type = EX_LSU; - op_type = INST_LSU_FENCE; + // neither the predecessor nor the successor set holds a memory + // read or write (pred = instr[27:24], succ = instr[23:20], IORW) + op_type = ((funct3 == 3'b000) && (instr[25:24] == 2'b00) && (instr[21:20] == 2'b00)) + ? INST_LSU_FENCE_IO : INST_LSU_FENCE; op_args.lsu.is_store = 0; op_args.lsu.is_float = 0; op_args.lsu.pack = 0; diff --git a/hw/rtl/core/VX_lsu_block.sv b/hw/rtl/core/VX_lsu_block.sv index 3d9406851a..56aeda0455 100644 --- a/hw/rtl/core/VX_lsu_block.sv +++ b/hw/rtl/core/VX_lsu_block.sv @@ -79,6 +79,7 @@ module VX_lsu_block import VX_gpu_pkg::*; #( // VX_lsu_agu; this slice contains no address arithmetic). wire req_is_fence, rsp_is_fence; + wire req_fence_io = inst_lsu_is_fence_io(execute_if.data.op_type); wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] full_addr; for (genvar i = 0; i < NUM_LANES; ++i) begin : g_full_addr @@ -121,8 +122,11 @@ module VX_lsu_block import VX_gpu_pkg::*; #( wire lane_is_om = 1'b0; assign mem_req_attr_struct[i].is_addr_om = 1'b0; `endif - assign mem_req_attr_struct[i].is_addr_io = ((block_addr >= io_addr_start) && (block_addr < io_addr_end)) - || lane_is_om; + // a fence's token carries its kind: io for a fence ordering only device + // accesses + assign mem_req_attr_struct[i].is_addr_io = req_is_fence ? req_fence_io + : (((block_addr >= io_addr_start) && (block_addr < io_addr_end)) + || lane_is_om); `ifdef VX_CFG_LMEM_ENABLE // is local memory address assign mem_req_attr_struct[i].is_addr_local = @@ -264,8 +268,9 @@ module VX_lsu_block import VX_gpu_pkg::*; #( default : mem_req_byteen_w = {LSU_WORD_SIZE{1'b1}}; endcase end - // a fence is a flush token: it carries no data, so no byte enables - assign mem_req_byteen[i] = req_is_fence ? '0 : mem_req_byteen_w; + // A fence is a flush token and carries no data. A memory fence is an + // L1-only token (byte enables set): it stops at the shared LLC. + assign mem_req_byteen[i] = req_is_fence ? (req_fence_io ? '0 : '1) : mem_req_byteen_w; end // memory misalignment not supported! diff --git a/tests/regression/Makefile b/tests/regression/Makefile index bb82626739..15db9427dd 100644 --- a/tests/regression/Makefile +++ b/tests/regression/Makefile @@ -5,7 +5,7 @@ include $(ROOT_DIR)/config.mk TESTS := \ minimal \ basic demo dogfood dropout dotproduct dotproduct2 mstress io_addr \ - arith printf diverge diverge_large sort fence ldst_order \ + arith printf diverge diverge_large sort fence fence_mp ldst_order \ vecadd vecadd_v1 sgemm sgemmx sgemm_v1 conv3 relu sgemv \ sgemm2 sgemm2_v1 madmax stencil3d raycast bfs jacobi pathfinder \ softmax \ diff --git a/tests/regression/fence_mp/Makefile b/tests/regression/fence_mp/Makefile new file mode 100644 index 0000000000..84b07d28fe --- /dev/null +++ b/tests/regression/fence_mp/Makefile @@ -0,0 +1,16 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := fence_mp + +SRC_DIR := $(VORTEX_HOME)/tests/regression/$(PROJECT) + +SRCS := $(SRC_DIR)/main.cpp + +VX_SRCS := $(SRC_DIR)/kernel.cpp + +OPTS ?= -n64 + +KERNEL_LIB := vortex2 + +include ../common.mk diff --git a/tests/regression/fence_mp/common.h b/tests/regression/fence_mp/common.h new file mode 100644 index 0000000000..634f577e13 --- /dev/null +++ b/tests/regression/fence_mp/common.h @@ -0,0 +1,18 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include + +#define FENCE_MP_MAGIC 0x5a5a0000u + +typedef struct { + uint32_t num_tasks; // threads launched + uint32_t count; // data words passed from producer to consumer + uint32_t spin_limit; // consumer polls before giving up + uint32_t pad; + uint64_t data_addr; // count words, zero on entry + uint64_t flag_addr; // one word, zero on entry + uint64_t out_addr; // [0] mismatches, [1] 1 if the consumer timed out, [2] pre-read sum +} kernel_arg_t; + +#endif diff --git a/tests/regression/fence_mp/kernel.cpp b/tests/regression/fence_mp/kernel.cpp new file mode 100644 index 0000000000..80b47e7909 --- /dev/null +++ b/tests/regression/fence_mp/kernel.cpp @@ -0,0 +1,45 @@ +#include +#include +#include "common.h" + +// Message passing across cores through plain memory: the producer (the first +// thread) writes data, fences, then raises a flag; the consumer (the last +// thread, on the last core) reads the data once beforehand so stale copies sit +// in its caches, polls the flag behind a fence, fences again and checks the +// data. A memory fence must make the data visible across every cache the two +// cores do not share. +__kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { + uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; + auto data = reinterpret_cast(arg->data_addr); + auto flag = reinterpret_cast(arg->flag_addr); + auto out = reinterpret_cast(arg->out_addr); + if (gid == 0) { + for (uint32_t k = 0; k < arg->count; ++k) { + data[k] = FENCE_MP_MAGIC + k; + } + vx_fence(); + *flag = 1; + vx_fence(); + } else if (gid == arg->num_tasks - 1) { + uint32_t sum = 0; + for (uint32_t k = 0; k < arg->count; ++k) { + sum += data[k]; + } + out[2] = sum; + uint32_t timeout = 1; + for (uint32_t s = 0; s < arg->spin_limit; ++s) { + vx_fence(); + if (*flag != 0) { + timeout = 0; + break; + } + } + vx_fence(); + uint32_t errors = 0; + for (uint32_t k = 0; k < arg->count; ++k) { + errors += (data[k] != FENCE_MP_MAGIC + k); + } + out[0] = errors; + out[1] = timeout; + } +} diff --git a/tests/regression/fence_mp/main.cpp b/tests/regression/fence_mp/main.cpp new file mode 100644 index 0000000000..a8b2c05393 --- /dev/null +++ b/tests/regression/fence_mp/main.cpp @@ -0,0 +1,135 @@ +#include +#include +#include +#include +#include +#include "common.h" + +#define RT_CHECK(_expr) \ + do { \ + int _ret = _expr; \ + if (0 == _ret) \ + break; \ + printf("Error: '%s' returned %d!\n", #_expr, (int)_ret); \ + cleanup(); \ + exit(-1); \ + } while (false) + +const char* kernel_file = "kernel.vxbin"; +uint32_t count = 64; +uint32_t spin_limit = 4096; + +vx_device_h device = nullptr; +vx_buffer_h data_buffer = nullptr; +vx_buffer_h flag_buffer = nullptr; +vx_buffer_h out_buffer = nullptr; +vx_queue_h queue = nullptr; +vx_module_h module_ = nullptr; +vx_kernel_h kernel = nullptr; +kernel_arg_t kernel_arg = {}; + +static void show_usage() { + std::cout << "Vortex Test." << std::endl; + std::cout << "Usage: [-k: kernel] [-n words] [-s spin limit] [-h: help]" << std::endl; +} + +static void parse_args(int argc, char **argv) { + int c; + while ((c = getopt(argc, argv, "n:s:k:h")) != -1) { + switch (c) { + case 'n': count = atoi(optarg); break; + case 's': spin_limit = atoi(optarg); break; + case 'k': kernel_file = optarg; break; + case 'h': show_usage(); exit(0); break; + default: show_usage(); exit(-1); + } + } +} + +void cleanup() { + if (device) { + if (data_buffer) vx_buffer_release(data_buffer); + if (flag_buffer) vx_buffer_release(flag_buffer); + if (out_buffer) vx_buffer_release(out_buffer); + if (kernel) vx_kernel_release(kernel); + if (module_) vx_module_release(module_); + if (queue) vx_queue_release(queue); + vx_device_release(device); + } +} + +int main(int argc, char *argv[]) { + parse_args(argc, argv); + + RT_CHECK(vx_device_open(0, &device)); + vx_queue_info_t qi = { sizeof(qi), nullptr, VX_QUEUE_PRIORITY_NORMAL, 0 }; + RT_CHECK(vx_queue_create(device, &qi, &queue)); + + uint64_t num_cores, num_warps, num_threads; + RT_CHECK(vx_device_query(device, VX_CAPS_NUM_CORES, &num_cores)); + RT_CHECK(vx_device_query(device, VX_CAPS_NUM_WARPS, &num_warps)); + RT_CHECK(vx_device_query(device, VX_CAPS_NUM_THREADS, &num_threads)); + // one thread per hardware slot, so producer and consumer are resident together + uint32_t total_threads = num_cores * num_warps * num_threads; + + kernel_arg.num_tasks = total_threads; + kernel_arg.count = count; + kernel_arg.spin_limit = spin_limit; + + std::vector zeros(count, 0); + std::vector out(4, 0); + uint32_t data_size = count * sizeof(uint32_t); + uint32_t out_size = out.size() * sizeof(uint32_t); + RT_CHECK(vx_buffer_create(device, data_size, VX_MEM_READ_WRITE, &data_buffer)); + RT_CHECK(vx_buffer_address(data_buffer, &kernel_arg.data_addr)); + RT_CHECK(vx_buffer_create(device, sizeof(uint32_t), VX_MEM_READ_WRITE, &flag_buffer)); + RT_CHECK(vx_buffer_address(flag_buffer, &kernel_arg.flag_addr)); + RT_CHECK(vx_buffer_create(device, out_size, VX_MEM_READ_WRITE, &out_buffer)); + RT_CHECK(vx_buffer_address(out_buffer, &kernel_arg.out_addr)); + RT_CHECK(vx_enqueue_write(queue, data_buffer, 0, zeros.data(), data_size, 0, nullptr, nullptr)); + RT_CHECK(vx_enqueue_write(queue, flag_buffer, 0, zeros.data(), sizeof(uint32_t), 0, nullptr, nullptr)); + RT_CHECK(vx_enqueue_write(queue, out_buffer, 0, out.data(), out_size, 0, nullptr, nullptr)); + + RT_CHECK(vx_module_load_file(device, kernel_file, &module_)); + RT_CHECK(vx_module_get_kernel(module_, "main", &kernel)); + + vx_event_h launch_ev = nullptr, read_ev = nullptr; + { + uint32_t n = total_threads; + uint32_t grid_dim[1], block_dim[1]; + RT_CHECK(vx_device_max_occupancy_grid(device, 1, &n, grid_dim, block_dim)); + vx_launch_info_t li = {}; + li.struct_size = sizeof(li); + li.kernel = kernel; + li.args_host = &kernel_arg; + li.args_size = sizeof(kernel_arg); + li.ndim = 1; + li.grid_dim[0] = grid_dim[0]; + li.block_dim[0] = block_dim[0]; + RT_CHECK(vx_enqueue_launch(queue, &li, 0, nullptr, &launch_ev)); + } + RT_CHECK(vx_enqueue_read(queue, out.data(), out_buffer, 0, out_size, 1, &launch_ev, &read_ev)); + RT_CHECK(vx_event_wait_value(read_ev, 1, VX_TIMEOUT_INFINITE)); + vx_event_release(read_ev); + vx_event_release(launch_ev); + + int errors = 0; + if (out[1] != 0) { + std::cout << "consumer timed out waiting for the flag" << std::endl; + ++errors; + } + if (out[0] != 0) { + std::cout << "consumer saw " << std::dec << out[0] << " stale data words" << std::endl; + errors += out[0]; + } + + cleanup(); + + if (errors != 0) { + std::cout << "Found " << std::dec << errors << " errors!" << std::endl; + std::cout << "FAILED!" << std::endl; + return 1; + } + std::cout << "PASSED!" << std::endl; + return 0; +} From 9473f26e94864e0e3d61fe51200a6d9e86d29b7a Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 18 Oct 2026 20:52:08 -0700 Subject: [PATCH 047/124] Let a shared L3 answer L1-only flush tokens without a walk The walk condition negated SHARED_LLC bitwise. SHARED_LLC is an untyped parameter that takes the width of the value its parent passes: the L2 gets a 1-bit expression, but the L3 gets VX_CFG_L3_ENABLED, a 32-bit 1, whose bitwise negation is nonzero. A shared L3 therefore walked on every L1-only token, from fences and from L1-only launch flushes alike, and the fence gain stopped at the L3. Compare the parameter with zero instead. rtlsim at 2 clusters x 2 cores + L2 + L3, before -> after: printf 8,566,931 -> 724,024 (11.8x) fence 34,264 -> 9,078 (3.8x) async_gbarrier 89,997 -> 69,231 (-23%) dogfood is unchanged. At 2 cores + L2 and at the default config every app is cycle-identical, as are sgemm and raycast. fence_mp passes at all four topologies. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/cache/VX_cache_init.sv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hw/rtl/cache/VX_cache_init.sv b/hw/rtl/cache/VX_cache_init.sv index ce44844a53..75c0aae526 100644 --- a/hw/rtl/cache/VX_cache_init.sv +++ b/hw/rtl/cache/VX_cache_init.sv @@ -138,7 +138,7 @@ module VX_cache_init import VX_gpu_pkg::*; #( // every visible token is L1-only wire flush_l1_only = any_l1 && ~any_full && ~any_io; // a walk is owed to a full token, and to an L1-only one above the point of coherence - wire flush_walk = any_full || (any_l1 && ~SHARED_LLC); + wire flush_walk = any_full || (any_l1 && (SHARED_LLC == 0)); reg [NUM_BANKS-1:0] flush_done, flush_done_n; reg fwd_go, fwd_go_n; // the crossbar drained; only tokens enter it now From 1d3db49b626210df9c170371c36c2aa7b74950e5 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 19 Oct 2026 02:02:57 -0700 Subject: [PATCH 048/124] Let the command processor's DMA use the shared last-level cache Host transfers went straight to memory, so the runtime fully flushed the last-level cache before every upload and read-back: a walk of the whole LLC that grows with its size, and kernels that started cold after each transfer. On kernels launched between transfers, those flushes dominated run time. With VX_CFG_CP_LLC_CLIENT set, the CP's DMA becomes a client of the LLC when one LLC alone serves every core (an L2 with one cluster, or an L3). The LLC is then the point of coherence for host transfers, which need only the L1-only flush. VX_CFG_CP_ON_LLC records where the option took effect. The default is off. - VX_cp_llc_port takes the CP's slot on the LLC input, beside the cores, without changing the bank count. It marks IO-aperture addresses with the io attribute, so console output still bypasses the cache. A launch that follows DMA writes is held until an L1-only flush token sent on the slot is answered, so the kernel sees the data. - Write-around: transfers of at least half the LLC carry a new no-alloc attribute. A write-back bank sends a no-alloc write miss with no fill pending straight to memory and updates a hit in place, so a large upload does not fill the LLC with dirty lines that write back during the kernel. Only the rtlsim CP sets it; the xrt bridge passes 0. - The runtime sees a new CP capability bit (DMA_LLC_CLIENT) and then issues only L1-only flushes around transfers. - On the multi-bank xrt AFU, the CP's memory bridge feeds the new Vortex port, which frees one fabric port. The single-bank AFU and OPAE tie it off. rtlsim total cycles, off -> on (the off path's transfers cost no device cycles on rtlsim; the on path's are timed and included): 2 clusters + L2 + L3 dogfood 267,131 -> 186,870 (-30.0%) multikernel 33,443 -> 19,152 (-42.7%) sgemm -n32 35,657 -> 30,634 (-14.1%) 4 cores + L2 dogfood 192,764 -> 175,218 (-9.1%) multikernel 21,835 -> 16,715 (-23.4%) With an 8 MB L3, dogfood drops 64%. Off, every config is cycle-identical. flush_coherence (chain-2/3, stress-1/2 under DBG_FLUSH_STALL) and fence_mp pass with the option on, including with every DMA write forced around the cache. The coherence test fails when CP traffic bypasses the LLC while the runtime skips the full flush, and when write-around data is dropped. The launch fence protects the xrt CP; rtlsim's CP model already orders its writes, so removing the fence is not caught there. Co-Authored-By: Claude Opus 5.5 --- VX_config.toml | 7 ++ hw/rtl/VX_cluster.sv | 15 ++- hw/rtl/VX_gpu_pkg.sv | 36 ++++++- hw/rtl/Vortex.sv | 91 +++++++++++++++- hw/rtl/Vortex_axi.sv | 13 +++ hw/rtl/afu/common/VX_afu_wrap.sv | 117 ++++++++++++++++---- hw/rtl/afu/opae/vortex_afu.sv | 13 +++ hw/rtl/cache/VX_cache.sv | 4 + hw/rtl/cache/VX_cache_bank.sv | 53 ++++++--- hw/rtl/cache/VX_cache_wrap.sv | 3 + hw/rtl/core/VX_lsu_block.sv | 1 + hw/rtl/cp/VX_cp_axil_regfile.sv | 10 +- hw/rtl/cp/VX_cp_core.sv | 9 +- hw/rtl/mem/VX_cp_llc_port.sv | 154 +++++++++++++++++++++++++++ sim/common/cmd_processor.cpp | 14 ++- sim/common/cmd_processor.h | 11 ++ sim/rtlsim/processor.cpp | 143 +++++++++++++++++++++++++ sim/rtlsim/processor.h | 7 ++ sim/rtlsim/rtlsim_shim.sv | 28 +++++ sw/runtime/common/device.cpp | 17 +-- sw/runtime/common/vortex2_internal.h | 9 +- sw/runtime/rtlsim/vortex.cpp | 32 ++++++ 22 files changed, 725 insertions(+), 62 deletions(-) create mode 100644 hw/rtl/mem/VX_cp_llc_port.sv diff --git a/VX_config.toml b/VX_config.toml index 2646ef048a..51115ae941 100644 --- a/VX_config.toml +++ b/VX_config.toml @@ -199,6 +199,13 @@ l3_is_llc = "expr: $VX_CFG_L3_ENABLED == 1" single_core = "expr: $VX_CFG_NUM_CORES == 1 and $VX_CFG_NUM_CLUSTERS == 1" single_cluster = "expr: $VX_CFG_NUM_CLUSTERS == 1" +# The command processor's DMA becomes a client of the last-level cache, which +# is then the point of coherence for host transfers: they need no cache flush. +# Only a last-level cache that alone serves every core can take it (an L2 with +# one cluster, or an L3); VX_CFG_CP_ON_LLC is set where it took effect. +VX_CFG_CP_LLC_CLIENT = 0 +VX_CFG_CP_ON_LLC = "expr: int($VX_CFG_CP_LLC_CLIENT != 0 and (($l2_is_llc and $single_cluster) or $l3_is_llc))" + [l1cache] VX_CFG_ICACHE_SIZE = 16384 VX_CFG_ICACHE_NUM_WAYS = 4 diff --git a/hw/rtl/VX_cluster.sv b/hw/rtl/VX_cluster.sv index 05fed84daa..7e42f65be8 100644 --- a/hw/rtl/VX_cluster.sv +++ b/hw/rtl/VX_cluster.sv @@ -56,6 +56,10 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; output wire mmu_flush_done, `endif + // Command-processor DMA, entering as an L2 client when this cluster's L2 + // is the LLC (CP_ON_L2); tied off otherwise. + VX_mem_bus_if.slave cp_mem_if, + // Status output wire busy ); @@ -180,7 +184,7 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; VX_mem_bus_if #( .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), .TAG_WIDTH (L2_TAG_WIDTH) - ) per_socket_mem_bus_if[L2_NUM_REQS](); + ) per_socket_mem_bus_if[L2_CACHE_REQS](); // Socket memory output buses (socket arb output tag width) VX_mem_bus_if #( @@ -216,7 +220,7 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; .NUM_BANKS (L2_NUM_BANKS), .NUM_WAYS (`VX_CFG_L2_NUM_WAYS), .WORD_SIZE (L2_WORD_SIZE), - .NUM_REQS (L2_NUM_REQS), + .NUM_REQS (L2_CACHE_REQS), .MEM_PORTS (L2_MEM_PORTS), .CRSQ_SIZE (`VX_CFG_L2_CRSQ_SIZE), .MSHR_SIZE (`VX_CFG_L2_MSHR_SIZE), @@ -228,6 +232,7 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; .WRITEBACK (`VX_CFG_L2_WRITEBACK), .DIRTY_BYTES (`VX_CFG_L2_DIRTYBYTES), .LAZY_FETCH (`VX_CFG_L2_LAZY_FETCH), + .WRITE_AROUND (CP_ON_L2), .REPL_POLICY (`VX_CFG_L2_REPL_POLICY), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3), @@ -291,6 +296,12 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; `ASSIGN_VX_MEM_BUS_IF (mem_bus_if[i], l2_mem_bus_if[i]); end + if (CP_ON_L2 != 0) begin : g_cp_l2_client + `ASSIGN_VX_MEM_BUS_IF (per_socket_mem_bus_if[L2_CP_IDX], cp_mem_if); + end else begin : g_no_cp_l2_client + `UNUSED_VX_MEM_BUS_IF (cp_mem_if) + end + /////////////////////////////////////////////////////////////////////////// wire [NUM_SOCKETS-1:0] per_socket_busy; diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index c18e3050cc..7acc4b6daf 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -209,8 +209,11 @@ package VX_gpu_pkg; // is_flush at LSB so dcr_flush can drive it deterministically. // is_addr_om is appended ABOVE amo, deliberately: the amo field is cast by // offset (VX_cache_bank), so inserting a bit below it would silently shift - // MEM_ATTR_AMO_OFFS and reinterpret the AMO sideband. + // MEM_ATTR_AMO_OFFS and reinterpret the AMO sideband. is_noalloc sits above + // amo for the same reason: a write that misses a write-around cache goes to + // memory instead of allocating. typedef struct packed { + logic is_noalloc; // MEM_ATTR_NOALLOC_OFFS amo_req_t amo; // MEM_ATTR_AMO_OFFS = 4 logic is_addr_om; // MEM_ATTR_OM_OFFS = 3 logic is_addr_local; // MEM_ATTR_LOCAL_OFFS = 2 @@ -223,6 +226,7 @@ package VX_gpu_pkg; localparam MEM_ATTR_LOCAL_OFFS = 2; localparam MEM_ATTR_OM_OFFS = 3; localparam MEM_ATTR_AMO_OFFS = 4; + localparam MEM_ATTR_NOALLOC_OFFS = MEM_ATTR_AMO_OFFS + AMO_REQ_BITS; // Total width of the mem-bus attr field. Use this as the parameter // default for VX_mem_bus_if's ATTR_WIDTH parameter and as the @@ -1952,6 +1956,19 @@ package VX_gpu_pkg; localparam L2_PTW_IDX = L2_SOCKET_REQS + L2_GFX_REQS; localparam L2_NUM_REQS = L2_SOCKET_REQS + L2_GFX_REQS + L2_PTW_REQS; + + // The command processor's DMA enters the shared LLC on one more client + // slot (VX_CFG_CP_ON_LLC): the cluster L2 when it is the LLC of a single + // cluster, the L3 otherwise. The slot does not size the banks, so + // L2_NUM_REQS / L3_NUM_REQS exclude it and *_CACHE_REQS count it. + localparam CP_ON_L2 = ((`VX_CFG_CP_ON_LLC != 0) + && (`VX_CFG_NUM_CLUSTERS == 1) + && (`VX_CFG_L2_ENABLED != 0) + && (`VX_CFG_L3_ENABLED == 0)) ? 1 : 0; + localparam CP_ON_L3 = ((`VX_CFG_CP_ON_LLC != 0) && (`VX_CFG_L3_ENABLED != 0)) ? 1 : 0; + localparam L2_CP_IDX = L2_NUM_REQS; + localparam L2_CACHE_REQS = L2_NUM_REQS + CP_ON_L2; + localparam L2_NUM_BANKS = `VX_CFG_L2_NUM_BANKS; localparam L2_MEM_PORTS = `VX_CFG_L2_MEM_PORTS; @@ -1974,9 +1991,9 @@ package VX_gpu_pkg; // Memory request tag bits `ifdef VX_CFG_L2_ENABLE - localparam L2_MEM_TAG_WIDTH = `CACHE_NC_MEM_TAG_WIDTH(`VX_CFG_L2_MSHR_SIZE, L2_NUM_BANKS, L2_NUM_REQS, L2_MEM_PORTS, L2_SECTOR_SIZE, L2_WORD_SIZE, L2_TAG_WIDTH, UUID_WIDTH); + localparam L2_MEM_TAG_WIDTH = `CACHE_NC_MEM_TAG_WIDTH(`VX_CFG_L2_MSHR_SIZE, L2_NUM_BANKS, L2_CACHE_REQS, L2_MEM_PORTS, L2_SECTOR_SIZE, L2_WORD_SIZE, L2_TAG_WIDTH, UUID_WIDTH); `else - localparam L2_MEM_TAG_WIDTH = `CACHE_BYPASS_TAG_WIDTH(L2_NUM_REQS, L2_MEM_PORTS, L2_SECTOR_SIZE, L2_WORD_SIZE, L2_TAG_WIDTH); + localparam L2_MEM_TAG_WIDTH = `CACHE_BYPASS_TAG_WIDTH(L2_CACHE_REQS, L2_MEM_PORTS, L2_SECTOR_SIZE, L2_WORD_SIZE, L2_TAG_WIDTH); `endif // L2 is the LLC iff L2 is enabled and L3 is not. @@ -1999,6 +2016,8 @@ package VX_gpu_pkg; localparam L3_PTW_IDX = `VX_CFG_NUM_CLUSTERS * L2_MEM_PORTS; localparam L3_NUM_REQS = `VX_CFG_NUM_CLUSTERS * L2_MEM_PORTS + ((`VX_CFG_VM_ENABLED != 0) && (PTW_ON_L2 == 0) ? 1 : 0); + localparam L3_CP_IDX = L3_NUM_REQS; + localparam L3_CACHE_REQS = L3_NUM_REQS + CP_ON_L3; localparam L3_NUM_BANKS = `VX_CFG_L3_NUM_BANKS; localparam L3_MEM_PORTS = `VX_CFG_L3_MEM_PORTS; @@ -2010,14 +2029,21 @@ package VX_gpu_pkg; // Memory request tag bits `ifdef VX_CFG_L3_ENABLE - localparam L3_MEM_TAG_WIDTH = `CACHE_NC_MEM_TAG_WIDTH(`VX_CFG_L3_MSHR_SIZE, L3_NUM_BANKS, L3_NUM_REQS, L3_MEM_PORTS, L3_SECTOR_SIZE, L3_WORD_SIZE, L3_TAG_WIDTH, UUID_WIDTH); + localparam L3_MEM_TAG_WIDTH = `CACHE_NC_MEM_TAG_WIDTH(`VX_CFG_L3_MSHR_SIZE, L3_NUM_BANKS, L3_CACHE_REQS, L3_MEM_PORTS, L3_SECTOR_SIZE, L3_WORD_SIZE, L3_TAG_WIDTH, UUID_WIDTH); `else - localparam L3_MEM_TAG_WIDTH = `CACHE_BYPASS_TAG_WIDTH(L3_NUM_REQS, L3_MEM_PORTS, L3_SECTOR_SIZE, L3_WORD_SIZE, L3_TAG_WIDTH); + localparam L3_MEM_TAG_WIDTH = `CACHE_BYPASS_TAG_WIDTH(L3_CACHE_REQS, L3_MEM_PORTS, L3_SECTOR_SIZE, L3_WORD_SIZE, L3_TAG_WIDTH); `endif // L3 is the LLC whenever it is enabled. localparam L3_IS_LLC = `VX_CFG_L3_ENABLED; localparam L3_SHARED_LLC = L3_IS_LLC; + // CP DMA port into the LLC client slot: one LLC word per request. The + // slot tag carries the port's tag plus a fence marker. + localparam CP_MEM_DATA_SIZE = (CP_ON_L2 != 0) ? L2_WORD_SIZE : L3_WORD_SIZE; + localparam CP_MEM_ADDR_WIDTH = `VX_CFG_MEM_ADDR_WIDTH - `CLOG2(CP_MEM_DATA_SIZE); + localparam CP_SLOT_TAG_WIDTH = (CP_ON_L2 != 0) ? L2_TAG_WIDTH : L3_TAG_WIDTH; + localparam CP_MEM_TAG_WIDTH = CP_SLOT_TAG_WIDTH - UUID_WIDTH - 1; + /////////////////////////////////////////////////////////////////////////// localparam VX_MEM_PORTS = L3_MEM_PORTS; diff --git a/hw/rtl/Vortex.sv b/hw/rtl/Vortex.sv index b3bb5010d4..434d9ab4ab 100644 --- a/hw/rtl/Vortex.sv +++ b/hw/rtl/Vortex.sv @@ -45,6 +45,21 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( output wire dcr_rsp_valid, output wire [VX_DCR_DATA_WIDTH-1:0] dcr_rsp_data, + // Command-processor DMA into the last-level cache (VX_CFG_CP_ON_LLC) + input wire cp_mem_req_valid, + input wire cp_mem_req_rw, + input wire cp_mem_req_noalloc, + input wire [CP_MEM_ADDR_WIDTH-1:0] cp_mem_req_addr, + input wire [CP_MEM_DATA_SIZE*8-1:0] cp_mem_req_data, + input wire [CP_MEM_DATA_SIZE-1:0] cp_mem_req_byteen, + input wire [CP_MEM_TAG_WIDTH-1:0] cp_mem_req_tag, + output wire cp_mem_req_ready, + + output wire cp_mem_rsp_valid, + output wire [CP_MEM_DATA_SIZE*8-1:0] cp_mem_rsp_data, + output wire [CP_MEM_TAG_WIDTH-1:0] cp_mem_rsp_tag, + input wire cp_mem_rsp_ready, + // ctrl/status input wire start, output wire busy @@ -82,6 +97,9 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( assign dcr_rsp_valid = dcr_bus_if.rsp_valid; assign dcr_rsp_data = dcr_bus_if.rsp_data; + wire kmu_start; + wire cp_mem_busy; + // Kernel Management Unit VX_kmu_bus_if kmu_bus_in[1](); wire kmu_busy; @@ -93,7 +111,7 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( ) kmu ( .clk (clk), .reset (reset), - .start (start), + .start (kmu_start), .busy (kmu_busy), .dcr_req_valid (dcr_req_valid), .dcr_req_rw (dcr_req_rw), @@ -124,7 +142,7 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( VX_mem_bus_if #( .DATA_SIZE (L2_SECTOR_SIZE), .TAG_WIDTH (L3_TAG_WIDTH) - ) per_cluster_mem_bus_if[L3_NUM_REQS](); + ) per_cluster_mem_bus_if[L3_CACHE_REQS](); VX_mem_bus_if #( .DATA_SIZE (L3_SECTOR_SIZE), @@ -141,7 +159,7 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( .NUM_BANKS (L3_NUM_BANKS), .NUM_WAYS (`VX_CFG_L3_NUM_WAYS), .WORD_SIZE (L3_WORD_SIZE), - .NUM_REQS (L3_NUM_REQS), + .NUM_REQS (L3_CACHE_REQS), .MEM_PORTS (L3_MEM_PORTS), .CRSQ_SIZE (`VX_CFG_L3_CRSQ_SIZE), .MSHR_SIZE (`VX_CFG_L3_MSHR_SIZE), @@ -153,6 +171,7 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( .WRITEBACK (`VX_CFG_L3_WRITEBACK), .DIRTY_BYTES (`VX_CFG_L3_DIRTYBYTES), .LAZY_FETCH (`VX_CFG_L3_LAZY_FETCH), + .WRITE_AROUND (CP_ON_L3), .REPL_POLICY (`VX_CFG_L3_REPL_POLICY), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3), @@ -190,6 +209,68 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( assign mem_rsp_ready[i] = mem_bus_if[i].rsp_ready; end + // Command-processor DMA: one leg per cluster so every instance owns its + // port's drivers; only cluster 0's leg carries traffic, and only under + // CP_ON_L2. Under CP_ON_L3 the DMA is the L3's last client. + VX_mem_bus_if #( + .DATA_SIZE (CP_MEM_DATA_SIZE), + .TAG_WIDTH (CP_SLOT_TAG_WIDTH) + ) per_cluster_cp_mem_if [`VX_CFG_NUM_CLUSTERS] (); + + if ((CP_ON_L2 != 0) || (CP_ON_L3 != 0)) begin : g_cp_mem + VX_mem_bus_if #( + .DATA_SIZE (CP_MEM_DATA_SIZE), + .TAG_WIDTH (CP_SLOT_TAG_WIDTH) + ) cp_mem_if (); + + VX_cp_llc_port #( + .DATA_SIZE (CP_MEM_DATA_SIZE), + .SLOT_TAG_WIDTH (CP_SLOT_TAG_WIDTH), + .TAG_WIDTH (CP_MEM_TAG_WIDTH), + .ADDR_WIDTH (CP_MEM_ADDR_WIDTH) + ) cp_llc_port ( + .clk (clk), + .reset (reset), + .req_valid (cp_mem_req_valid), + .req_rw (cp_mem_req_rw), + .req_noalloc (cp_mem_req_noalloc), + .req_addr (cp_mem_req_addr), + .req_data (cp_mem_req_data), + .req_byteen (cp_mem_req_byteen), + .req_tag (cp_mem_req_tag), + .req_ready (cp_mem_req_ready), + .rsp_valid (cp_mem_rsp_valid), + .rsp_data (cp_mem_rsp_data), + .rsp_tag (cp_mem_rsp_tag), + .rsp_ready (cp_mem_rsp_ready), + .start_in (start), + .start_out (kmu_start), + .busy (cp_mem_busy), + .mem_bus_if (cp_mem_if) + ); + + if (CP_ON_L2 != 0) begin : g_l2 + `ASSIGN_VX_MEM_BUS_IF (per_cluster_cp_mem_if[0], cp_mem_if); + end else begin : g_l3 + `ASSIGN_VX_MEM_BUS_IF (per_cluster_mem_bus_if[L3_CP_IDX], cp_mem_if); + `INIT_VX_MEM_BUS_IF (per_cluster_cp_mem_if[0]) + end + for (genvar c = 1; c < `VX_CFG_NUM_CLUSTERS; ++c) begin : g_cp_mem_legs + `INIT_VX_MEM_BUS_IF (per_cluster_cp_mem_if[c]) + end + end else begin : g_no_cp_mem + assign cp_mem_req_ready = 1'b0; + assign cp_mem_rsp_valid = 1'b0; + assign cp_mem_rsp_data = '0; + assign cp_mem_rsp_tag = '0; + `UNUSED_VAR ({cp_mem_req_valid, cp_mem_req_rw, cp_mem_req_noalloc, cp_mem_req_addr, cp_mem_req_data, cp_mem_req_byteen, cp_mem_req_tag, cp_mem_rsp_ready}) + assign kmu_start = start; + assign cp_mem_busy = 1'b0; + for (genvar c = 0; c < `VX_CFG_NUM_CLUSTERS; ++c) begin : g_cp_mem_legs + `INIT_VX_MEM_BUS_IF (per_cluster_cp_mem_if[c]) + end + end + wire [`VX_CFG_NUM_CLUSTERS-1:0] per_cluster_busy; VX_kmu_bus_if per_cluster_kmu_bus_if[`VX_CFG_NUM_CLUSTERS](); @@ -390,6 +471,8 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( .mmu_flush_done (cl_mmu_flush_done[cluster_id]), `endif + .cp_mem_if (per_cluster_cp_mem_if[cluster_id]), + .busy (per_cluster_busy[cluster_id]) ); end @@ -403,7 +486,7 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( end wire busy_r; `BUFFER_EX(busy_r, kmu_busy | dcr_bus_if.req_valid | (|per_cluster_busy) | dev_mmu_busy | ~l3_req_empty, 1'b1, 1, (`VX_CFG_NUM_CLUSTERS > 1)); - assign busy = busy_r | kmu_busy | dcr_bus_if.req_valid | (|per_cluster_kmu_valid); + assign busy = busy_r | kmu_busy | dcr_bus_if.req_valid | (|per_cluster_kmu_valid) | cp_mem_busy; `ifdef PERF_ENABLE diff --git a/hw/rtl/Vortex_axi.sv b/hw/rtl/Vortex_axi.sv index 5a159c9d2a..109f2b4898 100644 --- a/hw/rtl/Vortex_axi.sv +++ b/hw/rtl/Vortex_axi.sv @@ -138,6 +138,19 @@ module Vortex_axi import VX_gpu_pkg::*; #( .dcr_rsp_valid (dcr_rsp_valid), .dcr_rsp_data (dcr_rsp_data), + .cp_mem_req_valid (1'b0), + .cp_mem_req_rw (1'b0), + .cp_mem_req_noalloc (1'b0), + .cp_mem_req_addr ('0), + .cp_mem_req_data ('0), + .cp_mem_req_byteen ('0), + .cp_mem_req_tag ('0), + `UNUSED_PIN (cp_mem_req_ready), + `UNUSED_PIN (cp_mem_rsp_valid), + `UNUSED_PIN (cp_mem_rsp_data), + `UNUSED_PIN (cp_mem_rsp_tag), + .cp_mem_rsp_ready (1'b1), + .start (start), .busy (vx_busy) ); diff --git a/hw/rtl/afu/common/VX_afu_wrap.sv b/hw/rtl/afu/common/VX_afu_wrap.sv index 5f21e4842d..bea895dfe0 100644 --- a/hw/rtl/afu/common/VX_afu_wrap.sv +++ b/hw/rtl/afu/common/VX_afu_wrap.sv @@ -338,7 +338,14 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( wire cp_interrupt; - VX_cp_core cp_core ( + // With several memory ports the CP's device master enters the + // last-level cache's client slot when the configuration has one; with a + // single port it keeps sharing that port below the caches. + localparam CP_DMA_LLC = ((C_M_AXI_MEM_NUM_BANKS > 1) && ((CP_ON_L2 != 0) || (CP_ON_L3 != 0))) ? 1 : 0; + + VX_cp_core #( + .DMA_LLC_CLIENT (CP_DMA_LLC) + ) cp_core ( .clk (clk), .reset (subsys_reset), .axil_s (cp_axil), @@ -714,7 +721,7 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( localparam VX_MEM_TAG_A_WIDTH = VX_MEM_TAG_WIDTH + `MAX(SUB_LDATAW, 0); localparam VX_MEM_ADDR_A_WIDTH = VX_MEM_ADDR_WIDTH - SUB_LDATAW; - localparam FAB_NUM_PORTS = VX_MEM_PORTS + 1; + localparam FAB_NUM_PORTS = VX_MEM_PORTS + (CP_DMA_LLC ? 0 : 1); localparam FAB_CP_PORT = VX_MEM_PORTS; localparam FAB_TAG_WIDTH = `MAX(VX_MEM_TAG_A_WIDTH, `VX_CP_AXI_TID_WIDTH); localparam FAB_DATA_SIZE = C_M_AXI_MEM_DATA_WIDTH / 8; @@ -733,6 +740,18 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( wire [VX_MEM_TAG_WIDTH-1:0] vx_mem_rsp_tag [VX_MEM_PORTS]; wire vx_mem_rsp_ready [VX_MEM_PORTS]; + wire cp_llc_req_valid; + wire cp_llc_req_rw; + wire [CP_MEM_ADDR_WIDTH-1:0] cp_llc_req_addr; + wire [CP_MEM_DATA_SIZE*8-1:0] cp_llc_req_data; + wire [CP_MEM_DATA_SIZE-1:0] cp_llc_req_byteen; + wire [CP_MEM_TAG_WIDTH-1:0] cp_llc_req_tag; + wire cp_llc_req_ready; + wire cp_llc_rsp_valid; + wire [CP_MEM_DATA_SIZE*8-1:0] cp_llc_rsp_data; + wire [CP_MEM_TAG_WIDTH-1:0] cp_llc_rsp_tag; + wire cp_llc_rsp_ready; + Vortex vortex ( `SCOPE_IO_BIND (1) @@ -760,6 +779,19 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( .dcr_rsp_valid (dcr_rsp_valid), .dcr_rsp_data (dcr_rsp_data), + .cp_mem_req_valid (cp_llc_req_valid), + .cp_mem_req_rw (cp_llc_req_rw), + .cp_mem_req_noalloc (1'b0), + .cp_mem_req_addr (cp_llc_req_addr), + .cp_mem_req_data (cp_llc_req_data), + .cp_mem_req_byteen (cp_llc_req_byteen), + .cp_mem_req_tag (cp_llc_req_tag), + .cp_mem_req_ready (cp_llc_req_ready), + .cp_mem_rsp_valid (cp_llc_rsp_valid), + .cp_mem_rsp_data (cp_llc_rsp_data), + .cp_mem_rsp_tag (cp_llc_rsp_tag), + .cp_mem_rsp_ready (cp_llc_rsp_ready), + .start (vx_start), .busy (vx_busy) ); @@ -822,11 +854,20 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( // The CP's device addresses come from the same host-side allocator as the // pointers handed to the cores -- Device::global_mem_, based at // VX_MEM_USER_BASE_ADDR -- so they are offset-relative already, exactly - // like Vortex's own. They enter the fabric unchanged and - // PLATFORM_MEMORY_OFFSET is applied once, at the bank port, for both - // masters. - wire [CP_LINE_ADDRW-1:0] cp_mem_req_addr; - wire [`VX_CP_AXI_TID_WIDTH-1:0] cp_mem_req_tag; + // like Vortex's own. They enter the fabric (or the last-level cache) + // unchanged and PLATFORM_MEMORY_OFFSET is applied once, at the bank + // port, for both masters. + wire cp_mem_req_valid; + wire cp_mem_req_rw; + wire [CP_LINE_ADDRW-1:0] cp_mem_req_addr; + wire [C_M_AXI_MEM_DATA_WIDTH-1:0] cp_mem_req_data; + wire [FAB_DATA_SIZE-1:0] cp_mem_req_byteen; + wire [`VX_CP_AXI_TID_WIDTH-1:0] cp_mem_req_tag; + wire cp_mem_req_ready; + wire cp_mem_rsp_valid; + wire [C_M_AXI_MEM_DATA_WIDTH-1:0] cp_mem_rsp_data; + wire [`VX_CP_AXI_TID_WIDTH-1:0] cp_mem_rsp_tag; + wire cp_mem_rsp_ready; VX_membus_from_axi #( .ADDR_W (64), @@ -836,23 +877,59 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( .clk (clk), .reset (subsys_reset), .axi_s (cp_axi_dev), - .mem_req_valid (fab_req_valid[FAB_CP_PORT]), - .mem_req_rw (fab_req_rw[FAB_CP_PORT]), + .mem_req_valid (cp_mem_req_valid), + .mem_req_rw (cp_mem_req_rw), .mem_req_addr (cp_mem_req_addr), - .mem_req_data (fab_req_data[FAB_CP_PORT]), - .mem_req_byteen (fab_req_byteen[FAB_CP_PORT]), + .mem_req_data (cp_mem_req_data), + .mem_req_byteen (cp_mem_req_byteen), .mem_req_tag (cp_mem_req_tag), - .mem_req_ready (fab_req_ready[FAB_CP_PORT]), - .mem_rsp_valid (fab_rsp_valid[FAB_CP_PORT]), - .mem_rsp_data (fab_rsp_data[FAB_CP_PORT]), - .mem_rsp_tag (fab_rsp_tag[FAB_CP_PORT][`VX_CP_AXI_TID_WIDTH-1:0]), - .mem_rsp_ready (fab_rsp_ready[FAB_CP_PORT]) + .mem_req_ready (cp_mem_req_ready), + .mem_rsp_valid (cp_mem_rsp_valid), + .mem_rsp_data (cp_mem_rsp_data), + .mem_rsp_tag (cp_mem_rsp_tag), + .mem_rsp_ready (cp_mem_rsp_ready) ); - assign fab_req_addr[FAB_CP_PORT] = cp_mem_req_addr[VX_MEM_ADDR_A_WIDTH-1:0]; - assign fab_req_tag[FAB_CP_PORT] = FAB_TAG_WIDTH'(cp_mem_req_tag); - `UNUSED_VAR (cp_mem_req_addr[CP_LINE_ADDRW-1:VX_MEM_ADDR_A_WIDTH]) - `UNUSED_VAR (fab_rsp_tag[FAB_CP_PORT]) + if (CP_DMA_LLC != 0) begin : g_cp_llc + `STATIC_ASSERT(CP_MEM_DATA_SIZE == FAB_DATA_SIZE, ("CP DMA beat must match the last-level cache word")) + `STATIC_ASSERT(CP_MEM_TAG_WIDTH >= `VX_CP_AXI_TID_WIDTH, ("CP DMA id does not fit the last-level cache slot tag")) + assign cp_llc_req_valid = cp_mem_req_valid; + assign cp_llc_req_rw = cp_mem_req_rw; + assign cp_llc_req_addr = cp_mem_req_addr[CP_MEM_ADDR_WIDTH-1:0]; + assign cp_llc_req_data = cp_mem_req_data; + assign cp_llc_req_byteen = cp_mem_req_byteen; + assign cp_llc_req_tag = CP_MEM_TAG_WIDTH'(cp_mem_req_tag); + assign cp_mem_req_ready = cp_llc_req_ready; + assign cp_mem_rsp_valid = cp_llc_rsp_valid; + assign cp_mem_rsp_data = cp_llc_rsp_data; + assign cp_mem_rsp_tag = cp_llc_rsp_tag[`VX_CP_AXI_TID_WIDTH-1:0]; + assign cp_llc_rsp_ready = cp_mem_rsp_ready; + `UNUSED_VAR (cp_mem_req_addr[CP_LINE_ADDRW-1:CP_MEM_ADDR_WIDTH]) + `UNUSED_VAR (cp_llc_rsp_tag) + end else begin : g_cp_fabric + assign fab_req_valid[FAB_CP_PORT] = cp_mem_req_valid; + assign fab_req_rw[FAB_CP_PORT] = cp_mem_req_rw; + assign fab_req_addr[FAB_CP_PORT] = cp_mem_req_addr[VX_MEM_ADDR_A_WIDTH-1:0]; + assign fab_req_data[FAB_CP_PORT] = cp_mem_req_data; + assign fab_req_byteen[FAB_CP_PORT] = cp_mem_req_byteen; + assign fab_req_tag[FAB_CP_PORT] = FAB_TAG_WIDTH'(cp_mem_req_tag); + assign cp_mem_req_ready = fab_req_ready[FAB_CP_PORT]; + assign cp_mem_rsp_valid = fab_rsp_valid[FAB_CP_PORT]; + assign cp_mem_rsp_data = fab_rsp_data[FAB_CP_PORT]; + assign cp_mem_rsp_tag = fab_rsp_tag[FAB_CP_PORT][`VX_CP_AXI_TID_WIDTH-1:0]; + assign fab_rsp_ready[FAB_CP_PORT] = cp_mem_rsp_ready; + `UNUSED_VAR (cp_mem_req_addr[CP_LINE_ADDRW-1:VX_MEM_ADDR_A_WIDTH]) + `UNUSED_VAR (fab_rsp_tag[FAB_CP_PORT]) + + assign cp_llc_req_valid = 1'b0; + assign cp_llc_req_rw = 1'b0; + assign cp_llc_req_addr = '0; + assign cp_llc_req_data = '0; + assign cp_llc_req_byteen = '0; + assign cp_llc_req_tag = '0; + assign cp_llc_rsp_ready = 1'b1; + `UNUSED_VAR ({cp_llc_req_ready, cp_llc_rsp_valid, cp_llc_rsp_data, cp_llc_rsp_tag}) + end // The optional AXI4 sideband signals (size/burst) are unused by the // bridge's reduced view — pin them sink-side so lint stays clean. diff --git a/hw/rtl/afu/opae/vortex_afu.sv b/hw/rtl/afu/opae/vortex_afu.sv index 61ad02a8e4..5e7ca4cecb 100644 --- a/hw/rtl/afu/opae/vortex_afu.sv +++ b/hw/rtl/afu/opae/vortex_afu.sv @@ -740,6 +740,19 @@ module vortex_afu import ccip_if_pkg::*; import local_mem_cfg_pkg::*; import VX_ .dcr_rsp_valid (vx_dcr_rsp_valid), .dcr_rsp_data (vx_dcr_rsp_data), + .cp_mem_req_valid (1'b0), + .cp_mem_req_rw (1'b0), + .cp_mem_req_noalloc (1'b0), + .cp_mem_req_addr ('0), + .cp_mem_req_data ('0), + .cp_mem_req_byteen ('0), + .cp_mem_req_tag ('0), + `UNUSED_PIN (cp_mem_req_ready), + `UNUSED_PIN (cp_mem_rsp_valid), + `UNUSED_PIN (cp_mem_rsp_data), + `UNUSED_PIN (cp_mem_rsp_tag), + .cp_mem_rsp_ready (1'b1), + .start (vx_start), .busy (vx_busy) ); diff --git a/hw/rtl/cache/VX_cache.sv b/hw/rtl/cache/VX_cache.sv index dce6f187a7..c5e7989c83 100644 --- a/hw/rtl/cache/VX_cache.sv +++ b/hw/rtl/cache/VX_cache.sv @@ -42,6 +42,8 @@ module VX_cache import VX_gpu_pkg::*; #( parameter LAZY_FETCH = 0, // Write misses allocate without a fill (write-validate) + parameter WRITE_AROUND = 0, // Write misses flagged no-allocate go to memory (write-back only) + parameter REPL_POLICY = `CS_REPL_FIFO, // Replacement policy parameter TAG_WIDTH = UUID_WIDTH + 1, // core request tag size @@ -72,6 +74,7 @@ module VX_cache import VX_gpu_pkg::*; #( `STATIC_ASSERT(WRITE_ENABLE || !WRITEBACK, ("invalid parameter: writeback requires write enable")) `STATIC_ASSERT(WRITEBACK || !DIRTY_BYTES, ("invalid parameter: dirty bytes require writeback")) `STATIC_ASSERT(!LAZY_FETCH || (WRITEBACK && DIRTY_BYTES && IS_LLC), ("invalid parameter: lazy fetch requires a write-back LLC with dirty bytes")) + `STATIC_ASSERT(!WRITE_AROUND || WRITEBACK, ("invalid parameter: write-around requires a write-back cache")) `STATIC_ASSERT(NUM_BANKS >= MEM_PORTS, ("invalid parameter: number of banks must be greater or equal to number of memory ports")) `STATIC_ASSERT(!BANK_HASH || NUM_BANKS <= 256, ("invalid parameter: bank hash supports up to 256 banks")) localparam REQ_SEL_WIDTH = `UP(`CS_REQ_SEL_BITS); @@ -376,6 +379,7 @@ module VX_cache import VX_gpu_pkg::*; #( .WRITEBACK (WRITEBACK), .DIRTY_BYTES (DIRTY_BYTES), .LAZY_FETCH (LAZY_FETCH), + .WRITE_AROUND (WRITE_AROUND), .REPL_POLICY (REPL_POLICY), .CRSQ_SIZE (CRSQ_SIZE), .MSHR_SIZE (MSHR_SIZE), diff --git a/hw/rtl/cache/VX_cache_bank.sv b/hw/rtl/cache/VX_cache_bank.sv index 6137bff9fc..070cdadcdf 100644 --- a/hw/rtl/cache/VX_cache_bank.sv +++ b/hw/rtl/cache/VX_cache_bank.sv @@ -32,6 +32,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( parameter WRITEBACK = 0, parameter DIRTY_BYTES = 0, parameter LAZY_FETCH = 0, // write miss allocates without a fill; a load to a partial sector refetches + parameter WRITE_AROUND = 0, // a write miss flagged no-allocate is sent to memory instead parameter REPL_POLICY = `CS_REPL_FIFO, parameter TAG_WIDTH = UUID_WIDTH + 1, parameter CORE_OUT_BUF = 0, @@ -105,6 +106,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( localparam CRSQ_QUEUE_SIZE = 1 << `CLOG2(2 + CRSQ_SIZE); `STATIC_ASSERT(LATENCY >= 2, ("invalid parameter: cache bank LATENCY must be >= 2")) `STATIC_ASSERT(!LAZY_FETCH || (WRITEBACK && DIRTY_BYTES && IS_LLC), ("invalid parameter: lazy fetch requires a write-back LLC with dirty bytes")) + `STATIC_ASSERT(!WRITE_AROUND || (WRITEBACK && WRITE_ENABLE), ("invalid parameter: write-around requires a write-back cache")) `UNUSED_PARAM (MRSQ_SIZE) // ======================================================================== @@ -940,10 +942,10 @@ module VX_cache_bank import VX_gpu_pkg::*; #( ); // stD write word delayed to stC for its consumers there (the AMO RMW - // operand and the write-through memory payload); elided when no stC - // consumer exists (write-through banks, or writeback banks with LLC AMO). + // operand and the write-through or write-around memory payload); elided + // when no stC consumer exists. wire [`CS_WORD_WIDTH-1:0] word_stc; - if ((WRITE_ENABLE != 0) && (!WRITEBACK || (AMO_ENABLE != 0 && IS_LLC != 0))) begin : g_word_stc + if ((WRITE_ENABLE != 0) && (!WRITEBACK || (AMO_ENABLE != 0 && IS_LLC != 0) || (WRITE_AROUND != 0))) begin : g_word_stc VX_pipe_register #( .DATAW (`CS_WORD_WIDTH) ) reg_word_stc ( @@ -977,7 +979,10 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // release the entry on a hit. wire mshr_release_st1; if (WRITEBACK) begin : g_mshr_release - assign mshr_release_st1 = st1.lk.is_hit && ~is_amo_fwd_rel; + // a write-around miss keeps nothing: it leaves for memory at stC. + wire write_around_st1 = (WRITE_AROUND != 0) && st1.req.rw && st1.req.attr[MEM_ATTR_NOALLOC_OFFS] + && ~st1.lk.is_hit && ~st1.lk.mshr_pending; + assign mshr_release_st1 = (st1.lk.is_hit || write_around_st1) && ~is_amo_fwd_rel; end else begin : g_mshr_release_ro // keep missed writes in MSHR if a pending entry exists for the line, so a // pending fill arriving without the write content replays them locally. @@ -1331,8 +1336,13 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire mreq_queue_rw; wire is_fill_or_flush_stc = stC.req.is_fill || (stC.req.is_flush && WRITEBACK); + // a no-allocate write miss with no fill pending on its line is sent to + // memory as is: nothing is allocated or fetched. + wire write_around_stc = (WRITE_AROUND != 0) && do_write_stc && stC.req.attr[MEM_ATTR_NOALLOC_OFFS] + && ~stC.lk.is_hit && ~stC.lk.mshr_pending; // a write-miss root under lazy fetch loops its fill back instead. - wire lazy_alloc_stc = (LAZY_FETCH != 0) && do_write_stc && ~stC.lk.is_hit && ~stC.lk.mshr_pending; + wire lazy_alloc_stc = (LAZY_FETCH != 0) && do_write_stc && ~stC.lk.is_hit && ~stC.lk.mshr_pending + && ~write_around_stc; wire do_fill_or_flush_stc = stC.req.valid && is_fill_or_flush_stc; // a sector refill keeps the resident line (no eviction), so never writes back. wire do_writeback_stc = do_fill_or_flush_stc && stC.lk.is_dirty && ~stC.lk.is_refill && ~refetch_first_stc; @@ -1421,13 +1431,30 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // on a dirty eviction. The two are mutually exclusive (a writeback is a // fill/flush commit; a fill request is a creq commit). wire fill_req_push = (do_read_stc || do_write_stc) && ~stC.lk.is_hit && ~stC.lk.mshr_pending - && ~lazy_alloc_stc && ~pipe_stall; - assign mreq_queue_push = fill_req_push || wb_beat_accept; + && ~lazy_alloc_stc && ~write_around_stc && ~pipe_stall; + wire wa_req_push = write_around_stc && ~pipe_stall; + assign mreq_queue_push = fill_req_push || wa_req_push || wb_beat_accept; assign mreq_queue_addr = is_wb_beat ? wb_mreq_addr : rd_mreq_addr; - assign mreq_queue_rw = is_wb_beat; - assign mreq_queue_data = wb_data_sec; // read fill request: data unused - assign mreq_queue_byteen = is_wb_beat ? wb_byteen_sec : {SECTOR_SIZE{1'b1}}; - `UNUSED_VAR ({stC.req.byteen, stC.req.is_replay}) + assign mreq_queue_rw = is_wb_beat || write_around_stc; + if (WRITE_AROUND != 0) begin : g_wa_data + wire [LINE_SIZE-1:0] full_byteen; + VX_demux #( + .DATAW (WORD_SIZE), + .N (`CS_WORDS_PER_LINE) + ) byteen_demux ( + .sel_in (stC.req.word_idx), + .data_in (stC.req.byteen), + .data_out (full_byteen) + ); + wire [SECTOR_SIZE-1:0] sec_byteen = full_byteen[sector_idx_stc*SECTOR_SIZE +: SECTOR_SIZE]; + assign mreq_queue_data = is_wb_beat ? wb_data_sec : {`CS_WORDS_PER_SECTOR{word_stc}}; + assign mreq_queue_byteen = is_wb_beat ? wb_byteen_sec : (write_around_stc ? sec_byteen : {SECTOR_SIZE{1'b1}}); + end else begin : g_no_wa_data + assign mreq_queue_data = wb_data_sec; // read fill request: data unused + assign mreq_queue_byteen = is_wb_beat ? wb_byteen_sec : {SECTOR_SIZE{1'b1}}; + `UNUSED_VAR (stC.req.byteen) + end + `UNUSED_VAR (stC.req.is_replay) end else begin : g_wt // word byte-enable demuxed over the line, then sliced to the word's sector. wire [LINE_SIZE-1:0] full_byteen; @@ -1449,7 +1476,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( assign mreq_queue_rw = stC.req.rw; assign mreq_queue_data = {`CS_WORDS_PER_SECTOR{word_stc}}; assign mreq_queue_byteen = (stC.req.rw || is_amo_fwd_st1) ? sec_byteen : {SECTOR_SIZE{1'b1}}; - `UNUSED_VAR ({is_wb_beat, wb_beat_accept, wb_mreq_addr, wb_data_sec, wb_byteen_sec, wb_sector, evict_addr_stc, stC.lk.evict_tag, stC.lk.is_dirty, stC.lk.evict_dirty_mask}) + `UNUSED_VAR ({is_wb_beat, wb_beat_accept, wb_mreq_addr, wb_data_sec, wb_byteen_sec, wb_sector, evict_addr_stc, stC.lk.evict_tag, stC.lk.is_dirty, stC.lk.evict_dirty_mask, write_around_stc}) end end else begin : g_mreq_queue_ro assign mreq_queue_push = (do_read_stc && ~stC.lk.is_hit && ~stC.lk.mshr_pending) && ~pipe_stall; @@ -1457,7 +1484,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( assign mreq_queue_rw = 0; assign mreq_queue_data = '0; assign mreq_queue_byteen = {SECTOR_SIZE{1'b1}}; - `UNUSED_VAR ({is_wb_beat, wb_beat_accept, wb_mreq_addr, wb_data_sec, wb_byteen_sec, wb_sector, do_writeback_stc, evict_addr_stc, evict_byteen_stc, stC.lk.evict_tag, stC.lk.is_dirty, stC.lk.evict_dirty_mask, stC.req.byteen, stC.req.word_idx, stC.req.is_replay, do_write_stc}) + `UNUSED_VAR ({is_wb_beat, wb_beat_accept, wb_mreq_addr, wb_data_sec, wb_byteen_sec, wb_sector, do_writeback_stc, evict_addr_stc, evict_byteen_stc, stC.lk.evict_tag, stC.lk.is_dirty, stC.lk.evict_dirty_mask, stC.req.byteen, stC.req.word_idx, stC.req.is_replay, do_write_stc, write_around_stc}) end if (UUID_WIDTH != 0) begin : g_mreq_queue_tag_uuid diff --git a/hw/rtl/cache/VX_cache_wrap.sv b/hw/rtl/cache/VX_cache_wrap.sv index 2471155263..2f04c76436 100644 --- a/hw/rtl/cache/VX_cache_wrap.sv +++ b/hw/rtl/cache/VX_cache_wrap.sv @@ -44,6 +44,8 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( parameter LAZY_FETCH = 0, // Write misses allocate without a fill (write-validate) + parameter WRITE_AROUND = 0, // Write misses flagged no-allocate go to memory (write-back only) + parameter REPL_POLICY = `CS_REPL_FIFO, // Replacement policy parameter TAG_WIDTH = UUID_WIDTH + 1, // core request tag size @@ -148,6 +150,7 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( .WRITEBACK (WRITEBACK), .DIRTY_BYTES (DIRTY_BYTES), .LAZY_FETCH (LAZY_FETCH), + .WRITE_AROUND (WRITE_AROUND), .REPL_POLICY (REPL_POLICY), .CRSQ_SIZE (CRSQ_SIZE), .MSHR_SIZE (MSHR_SIZE), diff --git a/hw/rtl/core/VX_lsu_block.sv b/hw/rtl/core/VX_lsu_block.sv index 56aeda0455..eebfad60a4 100644 --- a/hw/rtl/core/VX_lsu_block.sv +++ b/hw/rtl/core/VX_lsu_block.sv @@ -104,6 +104,7 @@ module VX_lsu_block import VX_gpu_pkg::*; #( wire [MEM_ADDRW-1:0] io_addr_start = MEM_ADDRW'(`VX_CFG_XLEN'(`VX_MEM_IO_BASE_ADDR) >> MEM_ASHIFT); wire [MEM_ADDRW-1:0] io_addr_end = MEM_ADDRW'(`VX_CFG_XLEN'(`VX_MEM_IO_END_ADDR) >> MEM_ASHIFT); assign mem_req_attr_struct[i].is_flush = req_is_fence; + assign mem_req_attr_struct[i].is_noalloc = 1'b0; `ifdef VX_CFG_EXT_OM_ENABLE // OM fragment-export aperture. // diff --git a/hw/rtl/cp/VX_cp_axil_regfile.sv b/hw/rtl/cp/VX_cp_axil_regfile.sv index 806a996da9..531e5e064d 100644 --- a/hw/rtl/cp/VX_cp_axil_regfile.sv +++ b/hw/rtl/cp/VX_cp_axil_regfile.sv @@ -15,7 +15,7 @@ // 0x000 CP_CTRL RW bit0=enable_global, bit1=reset_all // 0x004 CP_STATUS RO bit0=busy, bit1=error // 0x008 CP_DEV_CAPS RO [7:0]NUM_QUEUES | [15:8]RING_SIZE_LOG2_MAX -// [23:16]AXI_TID_WIDTH +// [23:16]AXI_TID_WIDTH | [28]DMA_LLC_CLIENT // 0x010 CP_CYCLE_LO RO free-running cycle counter low 32 bits // 0x014 CP_CYCLE_HI RO high 32 bits // 0x018 GPU_DEV_CAPS_LO RO device-config caps, low 32 bits @@ -51,7 +51,9 @@ module VX_cp_axil_regfile parameter int ADDR_W = 16, // Static device-caps fields (set at synthesis time from VX_cp_pkg). parameter int RING_SIZE_LOG2_MAX = VX_CP_RING_SIZE_LOG2_C, - parameter int AXI_TID_W = VX_CP_AXI_TID_WIDTH_C + parameter int AXI_TID_W = VX_CP_AXI_TID_WIDTH_C, + // The device-side DMA reaches memory through the last-level cache. + parameter int DMA_LLC_CLIENT = 0 )( input wire clk, input wire reset, @@ -204,7 +206,9 @@ module VX_cp_axil_regfile case (addr[5:0]) 6'h00: return r_cp_ctrl; 6'h04: return {30'd0, cp_error, cp_busy}; - 6'h08: return {8'd0, + 6'h08: return {3'd0, + 1'(DMA_LLC_CLIENT), + 4'd0, 8'(AXI_TID_W), 8'(RING_SIZE_LOG2_MAX), 8'(NUM_QUEUES)}; diff --git a/hw/rtl/cp/VX_cp_core.sv b/hw/rtl/cp/VX_cp_core.sv index ee0849b4e7..417a3584f9 100644 --- a/hw/rtl/cp/VX_cp_core.sv +++ b/hw/rtl/cp/VX_cp_core.sv @@ -81,7 +81,9 @@ module VX_cp_core parameter int ADDR_W = 64, parameter int DATA_W = 512, parameter int ID_W = VX_CP_AXI_TID_WIDTH_C, - parameter int AXIL_AW = 16 + parameter int AXIL_AW = 16, + // axi_dev enters the last-level cache's client slot instead of memory. + parameter int DMA_LLC_CLIENT = 0 )( input wire clk, input wire reset, @@ -148,8 +150,9 @@ module VX_cp_core logic engine_idle [NUM_QUEUES]; VX_cp_axil_regfile #( - .NUM_QUEUES (NUM_QUEUES), - .ADDR_W (AXIL_AW) + .NUM_QUEUES (NUM_QUEUES), + .ADDR_W (AXIL_AW), + .DMA_LLC_CLIENT (DMA_LLC_CLIENT) ) u_regfile ( .clk (clk), .reset (reset), diff --git a/hw/rtl/mem/VX_cp_llc_port.sv b/hw/rtl/mem/VX_cp_llc_port.sv new file mode 100644 index 0000000000..5542758f62 --- /dev/null +++ b/hw/rtl/mem/VX_cp_llc_port.sv @@ -0,0 +1,154 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// Command-processor DMA port into the last-level cache's client slot. +// Requests in the IO aperture carry the IO attribute and bypass the cache; +// a write flagged no-allocate writes around it on a miss. +// A launch that follows DMA writes is held until those writes are inside the +// cache banks: the port sends an L1-only flush token behind them, which the +// last-level cache answers once its input crossbar has drained, and only then +// passes the start pulse on. Reads need no such fence, since the DMA waits for +// their data. +module VX_cp_llc_port import VX_gpu_pkg::*; #( + parameter DATA_SIZE = 64, + parameter SLOT_TAG_WIDTH = UUID_WIDTH + 2, + parameter TAG_WIDTH = SLOT_TAG_WIDTH - UUID_WIDTH - 1, + parameter ADDR_WIDTH = `VX_CFG_MEM_ADDR_WIDTH - `CLOG2(DATA_SIZE) +) ( + input wire clk, + input wire reset, + + input wire req_valid, + input wire req_rw, + input wire req_noalloc, + input wire [ADDR_WIDTH-1:0] req_addr, + input wire [DATA_SIZE*8-1:0] req_data, + input wire [DATA_SIZE-1:0] req_byteen, + input wire [TAG_WIDTH-1:0] req_tag, + output wire req_ready, + + output wire rsp_valid, + output wire [DATA_SIZE*8-1:0] rsp_data, + output wire [TAG_WIDTH-1:0] rsp_tag, + input wire rsp_ready, + + input wire start_in, + output wire start_out, + output wire busy, + + VX_mem_bus_if.master mem_bus_if +); + `STATIC_ASSERT(TAG_WIDTH >= 1, ("invalid parameter: TAG_WIDTH=%0d", TAG_WIDTH)) + `STATIC_ASSERT(0 == (`VX_MEM_IO_BASE_ADDR % DATA_SIZE), ("invalid parameter")) + + localparam ASHIFT = `CLOG2(DATA_SIZE); + localparam REQ_DATAW = 1 + ADDR_WIDTH + DATA_SIZE * 8 + DATA_SIZE + `UP(MEM_ATTR_WIDTH) + SLOT_TAG_WIDTH; + + localparam STATE_IDLE = 0; + localparam STATE_FENCE = 1; + localparam STATE_WAIT = 2; + + reg [1:0] state; + reg wr_pending; + + wire fence_rsp = mem_bus_if.rsp_valid && mem_bus_if.rsp_data.tag.value[0]; + + wire [ADDR_WIDTH-1:0] io_addr_start = ADDR_WIDTH'(`VX_CFG_XLEN'(`VX_MEM_IO_BASE_ADDR) >> ASHIFT); + wire [ADDR_WIDTH-1:0] io_addr_end = ADDR_WIDTH'(`VX_CFG_XLEN'(`VX_MEM_IO_END_ADDR) >> ASHIFT); + + mem_bus_attr_t req_attr; + always @(*) begin + req_attr = '0; + req_attr.is_addr_io = (req_addr >= io_addr_start) && (req_addr < io_addr_end); + req_attr.is_noalloc = req_rw && req_noalloc; + end + + wire [`UP(MEM_ATTR_WIDTH)-1:0] fence_attr = `UP(MEM_ATTR_WIDTH)'(1 << MEM_ATTR_FLUSH_OFFS); + + wire [SLOT_TAG_WIDTH-1:0] req_slot_tag = SLOT_TAG_WIDTH'({req_tag, 1'b0}); + wire [SLOT_TAG_WIDTH-1:0] fence_slot_tag = SLOT_TAG_WIDTH'(1); + + wire fence_req = (state == STATE_FENCE); + + wire buf_valid_in = fence_req || ((state == STATE_IDLE) && req_valid); + wire buf_ready_in; + wire [REQ_DATAW-1:0] buf_data_in = fence_req ? + {1'b0, ADDR_WIDTH'(0), (DATA_SIZE*8)'(0), DATA_SIZE'(1), fence_attr, fence_slot_tag} : + {req_rw, req_addr, req_data, req_byteen, `UP(MEM_ATTR_WIDTH)'(req_attr), req_slot_tag}; + + assign req_ready = (state == STATE_IDLE) && buf_ready_in; + + wire req_fire = req_valid && req_ready; + wire fence_fire = fence_req && buf_ready_in; + + VX_elastic_buffer #( + .DATAW (REQ_DATAW), + .SIZE (1) + ) req_buf ( + .clk (clk), + .reset (reset), + .valid_in (buf_valid_in), + .ready_in (buf_ready_in), + .data_in (buf_data_in), + .data_out ({mem_bus_if.req_data.rw, + mem_bus_if.req_data.addr, + mem_bus_if.req_data.data, + mem_bus_if.req_data.byteen, + mem_bus_if.req_data.attr, + mem_bus_if.req_data.tag}), + .ready_out (mem_bus_if.req_ready), + .valid_out (mem_bus_if.req_valid) + ); + + always @(posedge clk) begin + if (reset) begin + state <= STATE_IDLE; + wr_pending <= 1'b0; + end else begin + case (state) + STATE_FENCE: begin + if (fence_fire) begin + state <= STATE_WAIT; + end + end + STATE_WAIT: begin + if (fence_rsp) begin + state <= STATE_IDLE; + wr_pending <= 1'b0; + end + end + default: begin + if (start_in && wr_pending) begin + state <= STATE_FENCE; + end else if (req_fire && req_rw) begin + wr_pending <= 1'b1; + end + end + endcase + end + end + + assign start_out = ((state == STATE_IDLE) && start_in && ~wr_pending) + || ((state == STATE_WAIT) && fence_rsp); + assign busy = (state != STATE_IDLE); + + assign rsp_valid = mem_bus_if.rsp_valid && ~fence_rsp; + assign rsp_data = mem_bus_if.rsp_data.data; + assign rsp_tag = mem_bus_if.rsp_data.tag.value[1 +: TAG_WIDTH]; + assign mem_bus_if.rsp_ready = fence_rsp || rsp_ready; + + `RUNTIME_ASSERT (~(start_in && (state != STATE_IDLE)), ("%t: launch while a DMA fence is pending", $time)) + +endmodule diff --git a/sim/common/cmd_processor.cpp b/sim/common/cmd_processor.cpp index 2ace286517..0f5d3a80f2 100644 --- a/sim/common/cmd_processor.cpp +++ b/sim/common/cmd_processor.cpp @@ -110,7 +110,8 @@ uint32_t CommandProcessor::mmio_read(uint32_t off) const { case 0x000: return cp_ctrl_; case 0x004: return uint32_t(busy() ? 1 : 0); // CP_STATUS bit0 case 0x008: { - // CP_DEV_CAPS: {SUPPORTS_QMD:1 @bit26 | SUPPORTS_DRAW:1 @bit25 | + // CP_DEV_CAPS: {DMA_LLC_CLIENT:1 @bit28 | MMU_FAULT_REPORT:1 @bit27 | + // SUPPORTS_QMD:1 @bit26 | SUPPORTS_DRAW:1 @bit25 | // VM_ENABLED:1 @bit24 | AXI_TID_W:8 | RING_LOG2:8 | NUM_QUEUES:8}. // Defaults: TID=6, RING_LOG2=16, NUM_QUEUES=1. VM_ENABLED reflects // the build-config so the config-agnostic libvortex.so can @@ -125,7 +126,8 @@ uint32_t CommandProcessor::mmio_read(uint32_t off) const { const uint32_t supports_draw = 1u << 25; const uint32_t supports_qmd = 1u << 26; const uint32_t fault_report = hooks_.mmu_fault_report ? (1u << 27) : 0; - return supports_qmd | supports_draw | vm_enabled | fault_report + const uint32_t llc_client = hooks_.dma_llc_client ? (1u << 28) : 0; + return llc_client | supports_qmd | supports_draw | vm_enabled | fault_report | (uint32_t(6) << 16) | (uint32_t(16) << 8) | uint32_t(1); } case 0x010: return uint32_t(cycle_counter_ & 0xFFFFFFFF); @@ -547,6 +549,8 @@ void CommandProcessor::tick_engine() { src = cp_translate(src, physical); } const uint64_t total = cur_cmd_.arg2; + const bool no_alloc = hooks_.dma_no_alloc_min != 0 + && total >= hooks_.dma_no_alloc_min; constexpr uint64_t CHUNK = 64 * 1024; std::vector buf( std::size_t(total < CHUNK ? total : CHUNK)); @@ -554,7 +558,11 @@ void CommandProcessor::tick_engine() { uint64_t n = total - done; if (n > CHUNK) n = CHUNK; hooks_.dram_read (src + done, buf.data(), n); - hooks_.dram_write(dst + done, buf.data(), n); + if (hooks_.dma_write) { + hooks_.dma_write(dst + done, buf.data(), n, no_alloc); + } else { + hooks_.dram_write(dst + done, buf.data(), n); + } done += n; } } diff --git a/sim/common/cmd_processor.h b/sim/common/cmd_processor.h index 956d786e03..f249b89ca5 100644 --- a/sim/common/cmd_processor.h +++ b/sim/common/cmd_processor.h @@ -107,6 +107,17 @@ class CommandProcessor { // so the runtime does not read them on a target that would leave // the DCR bus waiting for a response. bool mmu_fault_report = false; + + // Write `bytes` bytes from `src` to `addr` for a CMD_MEM_* transfer. + // `no_alloc` marks a transfer of at least dma_no_alloc_min bytes, + // whose writes should not allocate in the device caches. Unset: + // dram_write. + std::function dma_write; + uint64_t dma_no_alloc_min = 0; + + // Device-memory accesses go through the last-level cache, so host + // transfers need no cache flush. Published in DEV_CAPS. + bool dma_llc_client = false; }; explicit CommandProcessor(const Hooks& hooks); diff --git a/sim/rtlsim/processor.cpp b/sim/rtlsim/processor.cpp index eaef68cc0e..92724c8923 100644 --- a/sim/rtlsim/processor.cpp +++ b/sim/rtlsim/processor.cpp @@ -29,7 +29,9 @@ #endif +#include #include +#include #include #include #include @@ -39,6 +41,7 @@ #include #include +#include #include #include #include @@ -72,6 +75,15 @@ typedef uint64_t Word; #define VL_WDATA_GETW(lwp, i, n, w) \ VL_SEL_IWII(0, n * w, 0, 0, lwp, i * w, w) +// Command-processor DMA port into the last-level cache: one cache word per +// request, and as many reads in flight as one 4 KB DMA burst holds. +#if VX_CFG_L3_ENABLED +#define CP_MEM_DATA_SIZE VX_CFG_L2_SECTOR_SIZE +#else +#define CP_MEM_DATA_SIZE VX_CFG_L1_LINE_SIZE +#endif +#define CP_MEM_RD_SLOTS (4096 / CP_MEM_DATA_SIZE) + using namespace vortex; static uint64_t g_mem_done_seq = 0; @@ -253,11 +265,66 @@ class Processor::Impl { } } + void cp_mem_write(uint64_t addr, const uint8_t* src, uint64_t size, bool noalloc) { + while (size != 0) { + uint64_t offset = addr % CP_MEM_DATA_SIZE; + uint64_t n = std::min(size, CP_MEM_DATA_SIZE - offset); + cp_req_t req; + req.rw = true; + req.noalloc = noalloc; + req.addr = addr / CP_MEM_DATA_SIZE; + req.byteen = 0; + req.data.fill(0); + for (uint64_t i = 0; i < n; ++i) { + req.data[offset + i] = src[i]; + req.byteen |= uint64_t(1) << (offset + i); + } + req.tag = 0; + cp_reqs_.push_back(req); + addr += n; + src += n; + size -= n; + } + while (!cp_reqs_.empty()) { + this->tick(); + } + } + + void cp_mem_read(uint64_t addr, uint8_t* dst, uint64_t size) { + uint32_t inflight = 0; + while (size != 0 || inflight != 0) { + // two queued requests keep one read entering the port every cycle + while (size != 0 && cp_reqs_.size() < 2 && !cp_free_slots_.empty()) { + uint64_t offset = addr % CP_MEM_DATA_SIZE; + uint64_t n = std::min(size, CP_MEM_DATA_SIZE - offset); + uint32_t slot = cp_free_slots_.back(); + cp_free_slots_.pop_back(); + cp_rd_slots_[slot] = {dst, uint32_t(offset), uint32_t(n)}; + cp_req_t req; + req.rw = false; + req.noalloc = false; + req.addr = addr / CP_MEM_DATA_SIZE; + req.byteen = ~uint64_t(0); + req.data.fill(0); + req.tag = slot; + cp_reqs_.push_back(req); + ++inflight; + addr += n; + dst += n; + size -= n; + } + this->tick(); + inflight -= cp_rd_completed_; + cp_rd_completed_ = 0; + } + } + private: void reset() { this->mem_bus_reset(); this->dcr_bus_reset(); + this->cp_mem_reset(); print_bufs_.clear(); @@ -301,11 +368,13 @@ class Processor::Impl { this->eval(); this->mem_bus_eval(0); + this->cp_mem_eval(0); device_->clk = 1; this->eval(); this->mem_bus_eval(1); + this->cp_mem_eval(1); dram_sim_.tick(); @@ -447,6 +516,51 @@ class Processor::Impl { device_->dcr_req_rw = 0; } + void cp_mem_reset() { + cp_reqs_.clear(); + cp_free_slots_.clear(); + for (uint32_t i = 0; i < CP_MEM_RD_SLOTS; ++i) { + cp_free_slots_.push_back(CP_MEM_RD_SLOTS - 1 - i); + } + cp_req_fire_ = false; + cp_rd_completed_ = 0; + device_->cp_mem_req_valid = 0; + device_->cp_mem_rsp_ready = 1; + } + + // Inputs change after the rising edge; the handshakes they take part in + // are sampled after the falling-edge evaluation, ahead of the next edge. + void cp_mem_eval(bool clk) { + if (!clk) { + cp_req_fire_ = device_->cp_mem_req_valid && device_->cp_mem_req_ready; + if (device_->cp_mem_rsp_valid) { + uint32_t slot = device_->cp_mem_rsp_tag; + auto& rd = cp_rd_slots_.at(slot); + auto data = VDataCast::get(device_->cp_mem_rsp_data); + std::memcpy(rd.dst, data + rd.offset, rd.size); + cp_free_slots_.push_back(slot); + ++cp_rd_completed_; + } + return; + } + if (cp_req_fire_) { + cp_reqs_.pop_front(); + cp_req_fire_ = false; + } + if (cp_reqs_.empty()) { + device_->cp_mem_req_valid = 0; + return; + } + auto& req = cp_reqs_.front(); + device_->cp_mem_req_valid = 1; + device_->cp_mem_req_rw = req.rw; + device_->cp_mem_req_noalloc = req.noalloc; + device_->cp_mem_req_addr = req.addr; + device_->cp_mem_req_byteen = req.byteen; + device_->cp_mem_req_tag = req.tag; + memcpy(VDataCast::get(device_->cp_mem_req_data), req.data.data(), CP_MEM_DATA_SIZE); + } + void wait(uint32_t cycles) { for (int i = 0; i < cycles; ++i) { this->tick(); @@ -468,6 +582,27 @@ class Processor::Impl { std::unordered_map print_bufs_; + typedef struct { + std::array data; + uint64_t addr; + uint64_t byteen; + uint32_t tag; + bool rw; + bool noalloc; + } cp_req_t; + + typedef struct { + uint8_t* dst; + uint32_t offset; + uint32_t size; + } cp_rd_t; + + std::deque cp_reqs_; + std::array cp_rd_slots_; + std::vector cp_free_slots_; + uint32_t cp_rd_completed_; + bool cp_req_fire_; + std::list pending_mem_reqs_[VX_CFG_PLATFORM_MEMORY_NUM_BANKS]; std::queue dram_queue_[VX_CFG_PLATFORM_MEMORY_NUM_BANKS]; @@ -515,6 +650,14 @@ int Processor::dcr_read(uint32_t addr, uint32_t tag, uint32_t* value) { return impl_->dcr_read(addr, tag, value); } +void Processor::cp_mem_write(uint64_t addr, const void* src, uint64_t size, bool noalloc) { + impl_->cp_mem_write(addr, static_cast(src), size, noalloc); +} + +void Processor::cp_mem_read(uint64_t addr, void* dst, uint64_t size) { + impl_->cp_mem_read(addr, static_cast(dst), size); +} + void Processor::drain() { impl_->drain(); } \ No newline at end of file diff --git a/sim/rtlsim/processor.h b/sim/rtlsim/processor.h index c6ae7ae464..9fe3c2f68e 100644 --- a/sim/rtlsim/processor.h +++ b/sim/rtlsim/processor.h @@ -44,6 +44,13 @@ class Processor { // write it issued. void drain(); + // Device-memory access through the command processor's port into the + // last-level cache. A write returns once every request entered the port; + // the device orders it ahead of the next launch. A read returns with data. + void cp_mem_write(uint64_t addr, const void* src, uint64_t size, bool noalloc); + + void cp_mem_read(uint64_t addr, void* dst, uint64_t size); + private: class Impl; diff --git a/sim/rtlsim/rtlsim_shim.sv b/sim/rtlsim/rtlsim_shim.sv index bde1d6501f..9e214e1c90 100644 --- a/sim/rtlsim/rtlsim_shim.sv +++ b/sim/rtlsim/rtlsim_shim.sv @@ -50,6 +50,21 @@ module rtlsim_shim import VX_gpu_pkg::*; #( output wire dcr_rsp_valid, output wire [VX_DCR_DATA_WIDTH-1:0] dcr_rsp_data, + // Command-processor DMA into the last-level cache + input wire cp_mem_req_valid, + input wire cp_mem_req_rw, + input wire cp_mem_req_noalloc, + input wire [CP_MEM_ADDR_WIDTH-1:0] cp_mem_req_addr, + input wire [CP_MEM_DATA_SIZE*8-1:0] cp_mem_req_data, + input wire [CP_MEM_DATA_SIZE-1:0] cp_mem_req_byteen, + input wire [CP_MEM_TAG_WIDTH-1:0] cp_mem_req_tag, + output wire cp_mem_req_ready, + + output wire cp_mem_rsp_valid, + output wire [CP_MEM_DATA_SIZE*8-1:0] cp_mem_rsp_data, + output wire [CP_MEM_TAG_WIDTH-1:0] cp_mem_rsp_tag, + input wire cp_mem_rsp_ready, + // ctrl/status input wire start, output wire busy @@ -104,6 +119,19 @@ module rtlsim_shim import VX_gpu_pkg::*; #( .dcr_rsp_valid (dcr_rsp_valid), .dcr_rsp_data (dcr_rsp_data), + .cp_mem_req_valid (cp_mem_req_valid), + .cp_mem_req_rw (cp_mem_req_rw), + .cp_mem_req_noalloc (cp_mem_req_noalloc), + .cp_mem_req_addr (cp_mem_req_addr), + .cp_mem_req_data (cp_mem_req_data), + .cp_mem_req_byteen (cp_mem_req_byteen), + .cp_mem_req_tag (cp_mem_req_tag), + .cp_mem_req_ready (cp_mem_req_ready), + .cp_mem_rsp_valid (cp_mem_rsp_valid), + .cp_mem_rsp_data (cp_mem_rsp_data), + .cp_mem_rsp_tag (cp_mem_rsp_tag), + .cp_mem_rsp_ready (cp_mem_rsp_ready), + .start (start), .busy (vx_busy) ); diff --git a/sw/runtime/common/device.cpp b/sw/runtime/common/device.cpp index c8d25999af..a32f18e9c2 100644 --- a/sw/runtime/common/device.cpp +++ b/sw/runtime/common/device.cpp @@ -363,6 +363,9 @@ vx_result_t Device::cp_init() { // Reading them on a target that does not decode them stalls the DCR // bus waiting for a response that never arrives. mmu_fault_report_ = (dev_caps & (1u << 27)) != 0; + // DMA_LLC_CLIENT (bit 28): the CP DMA is a client of the last-level + // cache, which is then the point of coherence for host transfers. + cp_dma_llc_client_ = (dev_caps & (1u << 28)) != 0; } // Resolve the core count now, while nothing holds cp_mu_. Deferring it to @@ -552,9 +555,10 @@ vx_result_t Device::cp_ring_append_(const void* cl) { // Caller holds cp_mu_. A kernel's results are flushed lazily, by the // next command that needs them: the next launch gets an L1-only flush // (the last-level cache is the point of coherence between kernels), and - // the next transfer to or from device memory a full one, since the CP - // DMA reaches device memory beneath the last-level cache. Appending the - // flush under the same lock keeps any other command from slipping between. + // the next transfer to or from device memory a full one when the CP DMA + // reaches device memory beneath the last-level cache, an L1-only one when + // it is a client of that cache. Appending the flush under the same lock + // keeps any other command from slipping between. const uint8_t* b = static_cast(cl); const bool dma = cp_dma_touches_device_mem(b); const bool launch = (b[0] == CP_OPCODE_LAUNCH) @@ -562,11 +566,12 @@ vx_result_t Device::cp_ring_append_(const void* cl) { || (b[0] == CP_OPCODE_DRAW); if ((dma && (l1_dirty_ || llc_dirty_)) || (launch && l1_dirty_)) { uint8_t flush[CP_CL_BYTES]; - build_cache_flush(flush, cp_num_cores_, /*l1_only=*/!dma); + const bool full = dma && llc_dirty_; + build_cache_flush(flush, cp_num_cores_, /*l1_only=*/!full); auto r = cp_ring_write_(flush); if (r != VX_SUCCESS) return r; l1_dirty_ = false; - llc_dirty_ = llc_dirty_ && !dma; + llc_dirty_ = llc_dirty_ && !full; } assert(!(dma && (l1_dirty_ || llc_dirty_))); auto r = cp_ring_write_(cl); @@ -574,7 +579,7 @@ vx_result_t Device::cp_ring_append_(const void* cl) { if (launch) { // a draw descriptor ends on its own L1-only flush step l1_dirty_ = (b[0] != CP_OPCODE_DRAW); - llc_dirty_ = true; + llc_dirty_ = !cp_dma_llc_client_; } else if (b[0] == CP_OPCODE_CACHE_FLUSH) { l1_dirty_ = false; llc_dirty_ = llc_dirty_ && (b[1] & CP_FLUSH_FLAG_L1_ONLY); diff --git a/sw/runtime/common/vortex2_internal.h b/sw/runtime/common/vortex2_internal.h index 23c688ad63..724449d825 100644 --- a/sw/runtime/common/vortex2_internal.h +++ b/sw/runtime/common/vortex2_internal.h @@ -387,8 +387,8 @@ class Device : public RefCounted { // the non-batch path, or via cp_batch_begin). No doorbell, no poll. // Inserts the lazy kernel-boundary CMD_CACHE_FLUSH (AMD ACQUIRE_MEM // model) under the same lock: L1-only ahead of a launch while - // l1_dirty_, full ahead of a CMD_MEM_* touching device memory while - // either flag is set. + // l1_dirty_, and ahead of a CMD_MEM_* touching device memory while + // either flag is set -- full while llc_dirty_. vx_result_t cp_ring_append_(const void* cl); vx_result_t cp_ring_write_(const void* cl); @@ -460,7 +460,7 @@ class Device : public RefCounted { // stale lines. bool l1_dirty_ = false; // A kernel ran since the last full cache flush, so the last-level cache - // may hold lines the CP DMA (which reaches device memory beneath it) + // may hold lines the CP DMA (when it reaches device memory beneath it) // would miss or later overwrite. Both flags are guarded by cp_mu_. bool llc_dirty_ = false; std::mutex cp_mu_; // serialize ring writes @@ -492,6 +492,9 @@ class Device : public RefCounted { // the OP_DRAW mirror). Discovered at open. bool cp_supports_draw_ = false; bool cp_supports_qmd_ = false; + // The CP DMA reaches device memory through the last-level cache (CP + // DEV_CAPS bit 28), so transfers never need that cache flushed. + bool cp_dma_llc_client_ = false; class CpMemIO; std::unique_ptr vm_io_; std::unique_ptr vm_mgr_; diff --git a/sw/runtime/rtlsim/vortex.cpp b/sw/runtime/rtlsim/vortex.cpp index 8d7f24e96f..0376f03e77 100644 --- a/sw/runtime/rtlsim/vortex.cpp +++ b/sw/runtime/rtlsim/vortex.cpp @@ -23,6 +23,7 @@ // dram hooks dereference it directly (the sim runs in-process). // ============================================================================ +#include #include #include @@ -143,15 +144,46 @@ class vx_device { std::memcpy(dst, hp, bytes); return; } + #if VX_CFG_CP_ON_LLC + if (future_.valid()) future_.wait(); + processor_.cp_mem_read(addr, dst, bytes); + #else ram_.read(static_cast(dst), addr, bytes); + #endif }; h.dram_write = [this](uint64_t addr, const void* src, std::size_t bytes) { if (void* hp = host_region_ptr(addr)) { std::memcpy(hp, src, bytes); return; } + #if VX_CFG_CP_ON_LLC + if (future_.valid()) future_.wait(); + processor_.cp_mem_write(addr, src, bytes, false); + #else ram_.write(static_cast(src), addr, bytes); + #endif }; + #if VX_CFG_CP_ON_LLC + // The CP DMA is a client of the last-level cache: transfers stay coherent + // with it, and those of at least half its size write around it. + h.dma_write = [this](uint64_t addr, const void* src, std::size_t bytes, bool no_alloc) { + if (void* hp = host_region_ptr(addr)) { + std::memcpy(hp, src, bytes); + return; + } + if (future_.valid()) future_.wait(); + processor_.cp_mem_write(addr, src, bytes, no_alloc); + }; + h.dma_llc_client = true; + #if VX_CFG_L3_ENABLED + h.dma_no_alloc_min = VX_CFG_L3_SIZE / 2; + #else + h.dma_no_alloc_min = VX_CFG_L2_SIZE / 2; + #endif + if (const char* v = std::getenv("VORTEX_CP_NO_ALLOC_MIN")) { + h.dma_no_alloc_min = std::strtoull(v, nullptr, 0); + } + #endif h.vortex_dcr_write = [this](uint32_t addr, uint32_t value) { // Wait for any background processor_.run() to finish so dcr_write // does not race the Verilator state. From 1de9126e6ba1dbe40f19b78a3ca4e4a9538df5b7 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 19 Oct 2026 02:47:48 -0700 Subject: [PATCH 049/124] Check a victim's dirty bytes only when the victim way is valid With lazy fetch on, the dirty-byte mask is one RAM indexed by way and line. The reset walk leaves it alone and every install clears its line's mask, so an invalid way's mask is undefined until its first install. In simulation the banks can commit random operations for a few cycles while reset propagates, which may write the RAM. The commit-stage check that a clean victim carries no dirty bytes then fired on an invalid victim, at 2 clusters + L3 with L3 lazy fetch on every app, and at 4 cores + L2 with L2 lazy fetch once the CP joined the LLC. Hardware never uses those bytes: write-backs take bytes only from dirty sectors, and merges only read lines installed since reset. The tags now report whether the victim way holds a valid sector, and the check applies only to valid victims. The flag reaches the commit stage through a simulation-only register; synthesis drops it. The check that a dirty sector carries bytes is unchanged. rtlsim with lazy fetch on, at 2 clusters + L3 and at 4 cores + L2: sgemm, raycast, dogfood, multikernel, flush_coherence and mstress pass, as do the lazy_fetch, mstress and flush_coherence CI cases. The unmodified bank fails multikernel and dogfood at 2 clusters + L3 on the check. Lazy fetch off is cycle-identical. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/cache/VX_cache_bank.sv | 22 ++++++++++++++++++++-- hw/rtl/cache/VX_cache_tags.sv | 4 ++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/hw/rtl/cache/VX_cache_bank.sv b/hw/rtl/cache/VX_cache_bank.sv index 070cdadcdf..4187d85858 100644 --- a/hw/rtl/cache/VX_cache_bank.sv +++ b/hw/rtl/cache/VX_cache_bank.sv @@ -557,7 +557,8 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire [NUM_WAYS-1:0] tag_matches_st0; wire [NUM_WAYS-1:0] line_present_st0; wire [`CS_WAY_SEL_WIDTH-1:0] hit_idx_st0; - wire evict_dirty_st0; + wire evict_dirty_st0, evict_valid_st0; + `UNUSED_VAR (evict_valid_st0) wire [`CS_SECTORS_PER_LINE-1:0] evict_dirty_mask_st0; wire [`CS_TAG_SEL_BITS-1:0] evict_tag_st0; @@ -660,6 +661,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .tag_matches (tag_matches_st0), .line_present (line_present_st0), .evict_dirty (evict_dirty_st0), + .evict_valid (evict_valid_st0), .evict_dirty_mask (evict_dirty_mask_st0), .evict_tag (evict_tag_st0), .partial_match (partial_match_st0) @@ -1419,7 +1421,23 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire has_dirty_bytes = (| evict_byteen_stc); // a refetch evicts nothing: the mask it reads is its resident way's wire check_evict_bytes = do_fill_or_flush_stc && ~refetch_first_stc; - `RUNTIME_ASSERT (~check_evict_bytes || (stC.lk.is_dirty == has_dirty_bytes), ("missmatch dirty bytes: dirty_line=%b, dirty_bytes=%b, addr=0x%0h", stC.lk.is_dirty, has_dirty_bytes, `CS_BANK_TO_FULL_ADDR(addr_stc, BANK_ID))) + `ifndef SYNTHESIS + // An invalid way's mask is undefined: the reset walk leaves the + // mask alone and every install clears its line's mask, so only a + // valid victim's bytes must agree with its dirty flag. + wire evict_valid_stc; + VX_pipe_register #( + .DATAW (1), + .DEPTH (1 + PIPE_EX) + ) reg_evict_valid ( + .clk (clk), + .reset (reset), + .enable (~pipe_stall), + .data_in (evict_valid_st0), + .data_out (evict_valid_stc) + ); + `RUNTIME_ASSERT (~(check_evict_bytes && evict_valid_stc) || (stC.lk.is_dirty == has_dirty_bytes), ("missmatch dirty bytes: dirty_line=%b, dirty_bytes=%b, addr=0x%0h", stC.lk.is_dirty, has_dirty_bytes, `CS_BANK_TO_FULL_ADDR(addr_stc, BANK_ID))) + `endif // per-sector: a dirty sector must carry dirty bytes, or its writeback beat is dropped. wire [SEC-1:0] sec_has_dirty_bytes; for (genvar s = 0; s < SEC; ++s) begin : g_sec_has_dirty_bytes diff --git a/hw/rtl/cache/VX_cache_tags.sv b/hw/rtl/cache/VX_cache_tags.sv index 5ecfc5c0be..62433812f8 100644 --- a/hw/rtl/cache/VX_cache_tags.sv +++ b/hw/rtl/cache/VX_cache_tags.sv @@ -76,6 +76,8 @@ module VX_cache_tags import VX_gpu_pkg::*; #( // writeback); a single bit == evict_dirty when 1 sector/line. output wire [`CS_SECTORS_PER_LINE-1:0] evict_dirty_mask, output wire [`CS_TAG_SEL_BITS-1:0] evict_tag, + // the evict way holds a valid sector + output wire evict_valid, // the requested sector is resident but partial (a load/AMO to it refetches) output wire partial_match ); @@ -92,6 +94,8 @@ module VX_cache_tags import VX_gpu_pkg::*; #( wire [NUM_WAYS-1:0][SEC-1:0] read_partial; wire [NUM_WAYS-1:0] partial_hit; + assign evict_valid = (| read_valid[evict_way]); + if (WRITEBACK) begin : g_evict_tag_wb assign evict_dirty = (| read_dirty[evict_way]); // dirty if any sector dirty assign evict_dirty_mask = read_dirty[evict_way]; From 4e1d2a7ec81452365d50cabada2a9f53281446a0 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 19 Oct 2026 03:05:26 -0700 Subject: [PATCH 050/124] Serve a warp's identical loads with one dcache bank access The LSU coalescer merges lanes only inside each output channel's lane group, so a load whose lanes in different groups read the same word, such as every uniform load (kernel arguments, the shared operand of SGEMM, x[k] in SGEMV), sent one identical request per channel, all to the same bank. Each was served separately, and the bank was the bottleneck. With VX_CFG_DCACHE_READ_MERGE (default on), the dcache serves them with one bank access: - Plain reads (no write, flush, io, local, OM or AMO attribute) from ports of one LSU block, with the same word address and tag in the same cycle, form a group. The lowest port leads; the others stay out of the request crossbar and are accepted in the cycle their leader is. Each port's order into the bank is therefore unchanged: a thread's earlier store to the word is already in that bank's queue ahead of the load, and a later one cannot overtake it. Fences and flush tokens are consumed before the merge and block every input while visible. - The merged ports' mask rides below the tag through the bank, MSHR and replay. - The response network becomes a flat multicast crossbar: each bank lane keeps a sent mask, delivers a merged response to all of its ports, normally in one cycle, and releases it once every port has taken it. - The dcache reqs counter counts a merged read once. Merging is tied to an unregistered request crossbar by a static assert, since the flush unit counts in-flight requests by port handshakes. uniform_ld is a new test. It checks that a warp sees its own lane's store to a shared word, does not see a store issued after the load, and gets its own word when lanes in groups of 1 to 16 share one line. rtlsim sgemm, off -> on: 16 threads, 8-byte dcache word (8 banks) -n128 2,464,685 -> 2,041,896 (-17.2%) 16 threads, 2 dcache banks -n128 1,844,005 -> 1,662,951 (-9.8%) 32 threads -n128 874,395 -> 805,401 (-7.9%) 16 threads -n128 1,780,951 -> 1,665,973 (-6.5%) 4 cores + L2, 16 threads -n128 439,572 -> 411,124 (-6.5%) default config 801,222 -> 799,610 (-0.2%) At 16 threads, dcache reads fall 25% and bank stalls 55%. sgemv 512 at 4 cores + L2: -6.3%. raycast: 0 to -1%. ldst_order, fence, fence_mp, uniform_ld and amo (with A) pass, including at 2 clusters + L3. Comparing lines instead of words, releasing a follower before its leader is accepted, and merging AMOs each fail a test. Co-Authored-By: Claude Opus 5.5 --- VX_config.toml | 4 + hw/rtl/VX_socket.sv | 1 + hw/rtl/cache/VX_cache.sv | 174 +++++++++++++++++++++---- hw/rtl/cache/VX_cache_cluster.sv | 2 + hw/rtl/cache/VX_cache_wrap.sv | 2 + tests/regression/Makefile | 2 +- tests/regression/uniform_ld/Makefile | 16 +++ tests/regression/uniform_ld/common.h | 27 ++++ tests/regression/uniform_ld/kernel.cpp | 60 +++++++++ tests/regression/uniform_ld/main.cpp | 164 +++++++++++++++++++++++ 10 files changed, 423 insertions(+), 29 deletions(-) create mode 100644 tests/regression/uniform_ld/Makefile create mode 100644 tests/regression/uniform_ld/common.h create mode 100644 tests/regression/uniform_ld/kernel.cpp create mode 100644 tests/regression/uniform_ld/main.cpp diff --git a/VX_config.toml b/VX_config.toml index 51115ae941..2b6d21a4fe 100644 --- a/VX_config.toml +++ b/VX_config.toml @@ -255,6 +255,10 @@ VX_CFG_DCACHE_NUM_BANKS = "expr: pow(2, clog2(min($VX_CFG_DCACHE_NUM_REQS, 16))) # With two banks it can only swap a line on parity, and the extra parallel # misses cost more in memory queuing than the spreading saves. VX_CFG_DCACHE_BANK_HASH = "expr: int($VX_CFG_DCACHE_NUM_BANKS >= 4)" +# One bank access serves a warp's same-word reads arriving together on its +# coalescer channels (uniform loads), and the response returns to all of +# them in the same cycle. +VX_CFG_DCACHE_READ_MERGE = 1 VX_CFG_L1_MEM_PORTS = "expr: min($VX_CFG_DCACHE_NUM_BANKS, $VX_CFG_PLATFORM_MEMORY_NUM_BANKS) if ($VX_CFG_ICACHE_ENABLE or $VX_CFG_DCACHE_ENABLE) else min($VX_CFG_DCACHE_NUM_REQS, $VX_CFG_PLATFORM_MEMORY_NUM_BANKS)" diff --git a/hw/rtl/VX_socket.sv b/hw/rtl/VX_socket.sv index 08aa35fc86..cf04039049 100644 --- a/hw/rtl/VX_socket.sv +++ b/hw/rtl/VX_socket.sv @@ -234,6 +234,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .IS_LLC (DCACHE_IS_LLC), .SHARED_LLC (DCACHE_SHARED_LLC), .BANK_HASH (`VX_CFG_DCACHE_BANK_HASH), + .READ_MERGE (`VX_CFG_DCACHE_READ_MERGE ? DCACHE_CHANNELS : 0), .AMO_ENABLE (`VX_CFG_EXT_A_ENABLED) ) dcache ( `ifdef PERF_ENABLE diff --git a/hw/rtl/cache/VX_cache.sv b/hw/rtl/cache/VX_cache.sv index c5e7989c83..446b7355e1 100644 --- a/hw/rtl/cache/VX_cache.sv +++ b/hw/rtl/cache/VX_cache.sv @@ -58,6 +58,8 @@ module VX_cache import VX_gpu_pkg::*; #( parameter BANK_HASH = 0, // hashed bank select (cache_bank_hash) + parameter READ_MERGE = 0, // merge same-word reads within groups of this many adjacent ports (0: off) + parameter AMO_ENABLE = 0 // synthesize atomic-op logic in the banks ) ( `ifdef PERF_ENABLE @@ -87,7 +89,12 @@ module VX_cache import VX_gpu_pkg::*; #( localparam BANK_SEL_BITS = `CLOG2(NUM_BANKS); localparam BANK_SEL_WIDTH = `UP(BANK_SEL_BITS); localparam LINE_ADDR_WIDTH = (`CS_WORD_ADDR_WIDTH - BANK_SEL_BITS - WORD_SEL_BITS); - localparam CORE_REQ_DATAW = LINE_ADDR_WIDTH + 1 + WORD_SEL_WIDTH + WORD_SIZE + WORD_WIDTH + TAG_WIDTH + `UP(MEM_ATTR_WIDTH); + // Read merging carries the merged ports' mask (one bit per port of the + // leader's group) below the request tag through the bank. + localparam MERGE_GROUP = (READ_MERGE != 0) ? `MIN(READ_MERGE, NUM_REQS) : 1; + localparam DO_READ_MERGE = (MERGE_GROUP > 1); + localparam BANK_TAG_WIDTH = TAG_WIDTH + (DO_READ_MERGE ? MERGE_GROUP : 0); + localparam CORE_REQ_DATAW = LINE_ADDR_WIDTH + 1 + WORD_SEL_WIDTH + WORD_SIZE + WORD_WIDTH + BANK_TAG_WIDTH + `UP(MEM_ATTR_WIDTH); localparam CORE_RSP_DATAW = WORD_WIDTH + TAG_WIDTH; localparam BANK_MEM_TAG_WIDTH = UUID_WIDTH + MSHR_ADDR_WIDTH; localparam MEM_REQ_DATAW = (`UP(BANK_SEL_BITS) + `CS_LINE_SECTOR_ADDR_WIDTH + 1 + SECTOR_SIZE + `CS_SECTOR_WIDTH + BANK_MEM_TAG_WIDTH + `UP(MEM_ATTR_WIDTH)); @@ -98,6 +105,10 @@ module VX_cache import VX_gpu_pkg::*; #( localparam REQ_XBAR_BUF = (NUM_REQS > 2) ? 2 : 0; localparam CORE_RSP_BUF_ENABLE = (NUM_BANKS != 1) || (NUM_REQS != 1); localparam MEM_REQ_BUF_ENABLE = (NUM_BANKS != 1); + `STATIC_ASSERT(!DO_READ_MERGE || (`IS_POW2(MERGE_GROUP) && `IS_DIVISBLE(NUM_REQS, MERGE_GROUP)), ("invalid parameter: read merge group")) + // a merged request is one bank access for several port handshakes, which + // the in-flight request count of a registered request crossbar would miss + `STATIC_ASSERT(!DO_READ_MERGE || (`TO_OUT_BUF_REG(REQ_XBAR_BUF) == 0), ("invalid parameter: read merge with a registered request crossbar")) `ifdef PERF_ENABLE wire [NUM_BANKS-1:0] perf_read_miss_per_bank; wire [NUM_BANKS-1:0] perf_write_miss_per_bank; @@ -251,13 +262,13 @@ module VX_cache import VX_gpu_pkg::*; #( wire [NUM_BANKS-1:0][WORD_SEL_WIDTH-1:0] per_bank_core_req_wsel; wire [NUM_BANKS-1:0][WORD_SIZE-1:0] per_bank_core_req_byteen; wire [NUM_BANKS-1:0][`CS_WORD_WIDTH-1:0] per_bank_core_req_data; - wire [NUM_BANKS-1:0][TAG_WIDTH-1:0] per_bank_core_req_tag; + wire [NUM_BANKS-1:0][BANK_TAG_WIDTH-1:0] per_bank_core_req_tag; wire [NUM_BANKS-1:0][REQ_SEL_WIDTH-1:0] per_bank_core_req_idx; wire [NUM_BANKS-1:0][`UP(MEM_ATTR_WIDTH)-1:0] per_bank_core_req_attr; wire [NUM_BANKS-1:0] per_bank_core_req_ready; wire [NUM_BANKS-1:0] per_bank_core_rsp_valid; wire [NUM_BANKS-1:0][`CS_WORD_WIDTH-1:0] per_bank_core_rsp_data; - wire [NUM_BANKS-1:0][TAG_WIDTH-1:0] per_bank_core_rsp_tag; + wire [NUM_BANKS-1:0][BANK_TAG_WIDTH-1:0] per_bank_core_rsp_tag; wire [NUM_BANKS-1:0][REQ_SEL_WIDTH-1:0] per_bank_core_rsp_idx; wire [NUM_BANKS-1:0] per_bank_core_rsp_ready; wire [NUM_BANKS-1:0] per_bank_mem_req_valid; @@ -313,6 +324,57 @@ module VX_cache import VX_gpu_pkg::*; #( assign core_req_bid[i] = '0; end end + // Read merging: plain reads of the same word with the same tag from ports + // of one group in the same cycle are served by one bank access. The lowest + // such port leads; the others stay out of the request crossbar and are + // accepted in the cycle their leader is, so every port keeps its own + // request order into the bank. + wire [NUM_REQS-1:0] core_req_follower; + wire [NUM_REQS-1:0] core_req_xbar_ready; + wire [NUM_REQS-1:0][BANK_TAG_WIDTH-1:0] core_req_bank_tag; + if (DO_READ_MERGE) begin : g_read_merge + // a plain read has no flush, I/O, local, OM or atomic attribute + localparam [`UP(MEM_ATTR_WIDTH)-1:0] SPECIAL_ATTRS = `UP(MEM_ATTR_WIDTH)'((1 << MEM_ATTR_FLUSH_OFFS) + | (1 << MEM_ATTR_IO_OFFS) + | (1 << MEM_ATTR_LOCAL_OFFS) + | (1 << MEM_ATTR_OM_OFFS) + | (1 << MEM_ATTR_AMO_OFFS)); + wire [NUM_REQS-1:0] plain_read; + for (genvar i = 0; i < NUM_REQS; ++i) begin : g_plain_read + assign plain_read[i] = core_req_valid[i] && ~core_req_rw[i] && ((core_req_attr[i] & SPECIAL_ATTRS) == '0); + end + // same_word[i][j]: port i reads what a lower port j of its group reads + wire [NUM_REQS-1:0][NUM_REQS-1:0] same_word; + for (genvar i = 0; i < NUM_REQS; ++i) begin : g_same_i + for (genvar j = 0; j < NUM_REQS; ++j) begin : g_same_j + if (j < i && (j / MERGE_GROUP) == (i / MERGE_GROUP)) begin : g_cmp + assign same_word[i][j] = plain_read[i] && plain_read[j] + && (core_req_addr[i] == core_req_addr[j]) + && (core_req_tag[i] == core_req_tag[j]); + end else begin : g_none + assign same_word[i][j] = 1'b0; + end + end + end + for (genvar i = 0; i < NUM_REQS; ++i) begin : g_merge + localparam GROUP_BASE = (i / MERGE_GROUP) * MERGE_GROUP; + wire [MERGE_GROUP-1:0] merged_mask; + wire [NUM_REQS-1:0] leader_fire; + for (genvar k = 0; k < MERGE_GROUP; ++k) begin : g_merged_mask + assign merged_mask[k] = same_word[GROUP_BASE + k][i]; + end + for (genvar j = 0; j < NUM_REQS; ++j) begin : g_leader_fire + assign leader_fire[j] = same_word[i][j] && ~core_req_follower[j] && core_req_xbar_ready[j]; + end + assign core_req_follower[i] = (| same_word[i]); + assign core_req_bank_tag[i] = {core_req_tag[i], merged_mask}; + assign core_req_ready[i] = core_req_follower[i] ? (| leader_fire) : core_req_xbar_ready[i]; + end + end else begin : g_no_read_merge + assign core_req_follower = '0; + assign core_req_bank_tag = core_req_tag; + assign core_req_ready = core_req_xbar_ready; + end for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_req_data_in assign core_req_data_in[i] = { core_req_line_addr[i], @@ -320,7 +382,7 @@ module VX_cache import VX_gpu_pkg::*; #( core_req_wsel[i], core_req_byteen[i], core_req_data[i], - core_req_tag[i], + core_req_bank_tag[i], core_req_attr[i] }; end @@ -343,10 +405,10 @@ module VX_cache import VX_gpu_pkg::*; #( `else `UNUSED_PIN(collisions), `endif - .valid_in (core_req_valid), + .valid_in (core_req_valid & ~core_req_follower), .data_in (core_req_data_in), .sel_in (core_req_bid), - .ready_in (core_req_ready), + .ready_in (core_req_xbar_ready), .valid_out (per_bank_core_req_valid), .data_out (core_req_data_out), .sel_out (per_bank_core_req_idx), @@ -386,7 +448,7 @@ module VX_cache import VX_gpu_pkg::*; #( .MRSQ_SIZE (MRSQ_SIZE), .MREQ_SIZE (MREQ_SIZE), .LATENCY (LATENCY), - .TAG_WIDTH (TAG_WIDTH), + .TAG_WIDTH (BANK_TAG_WIDTH), .CORE_OUT_BUF (CORE_RSP_BUF_ENABLE ? 3 : 0), .MEM_OUT_BUF (MEM_REQ_BUF_ENABLE ? 3 : 0), .IS_LLC (IS_LLC), @@ -438,7 +500,7 @@ module VX_cache import VX_gpu_pkg::*; #( wire [NUM_REQS-1:0] core_rsp_ready; wire [NUM_BANKS-1:0][CORE_RSP_DATAW-1:0] per_bank_core_rsp_pdata; for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_per_bank_core_rsp_pdata - assign per_bank_core_rsp_pdata[i] = {per_bank_core_rsp_data[i], per_bank_core_rsp_tag[i]}; + assign per_bank_core_rsp_pdata[i] = {per_bank_core_rsp_data[i], per_bank_core_rsp_tag[i][BANK_TAG_WIDTH-1 -: TAG_WIDTH]}; end // A flush token's answer takes bank 0's lane into the response crossbar // when the bank has no response of its own, so it leaves through the same @@ -462,25 +524,80 @@ module VX_cache import VX_gpu_pkg::*; #( assign per_bank_core_rsp_ready[i] = core_rsp_xbar_ready[i]; end end - VX_stream_omega #( - .NUM_INPUTS (NUM_BANKS), - .NUM_OUTPUTS (NUM_REQS), - .DATAW (CORE_RSP_DATAW), - .ARBITER ("R"), - .OUT_BUF (CORE_OUT_BUF) - ) core_rsp_xbar ( - .clk (clk), - .reset (reset), - .valid_in (core_rsp_xbar_valid), - .data_in (core_rsp_xbar_pdata), - .sel_in (core_rsp_xbar_idx), - .ready_in (core_rsp_xbar_ready), - .valid_out (core_rsp_valid), - .data_out (core_rsp_pdata), - `UNUSED_PIN (sel_out), - .ready_out (core_rsp_ready), - `UNUSED_PIN (collisions) - ); + if (DO_READ_MERGE) begin : g_core_rsp_mcast + // A merged read's response goes to its leader's port and to every + // merged port: each output takes it when its arbiter grants it, and the + // bank's response is released once all its targets have taken it. + localparam GROUP_BITS = `CLOG2(MERGE_GROUP); + wire [NUM_BANKS-1:0][NUM_REQS-1:0] lane_pending; + wire [NUM_BANKS-1:0][NUM_REQS-1:0] lane_taken; + wire [NUM_REQS-1:0][NUM_BANKS-1:0] per_output_valid; + wire [NUM_REQS-1:0][NUM_BANKS-1:0] per_output_ready; + for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_lanes + wire [MERGE_GROUP-1:0] merged_mask; + if (i == 0) begin : g_flush_mask + assign merged_mask = per_bank_core_rsp_valid[0] ? per_bank_core_rsp_tag[0][MERGE_GROUP-1:0] : '0; + end else begin : g_bank_mask + assign merged_mask = per_bank_core_rsp_tag[i][MERGE_GROUP-1:0]; + end + wire [REQ_SEL_WIDTH-1:0] group_base = (core_rsp_xbar_idx[i] >> GROUP_BITS) << GROUP_BITS; + wire [NUM_REQS-1:0] targets = (NUM_REQS'(1) << core_rsp_xbar_idx[i]) + | (NUM_REQS'(merged_mask) << group_base); + reg [NUM_REQS-1:0] sent_r; + assign lane_pending[i] = targets & ~sent_r & {NUM_REQS{core_rsp_xbar_valid[i]}}; + for (genvar j = 0; j < NUM_REQS; ++j) begin : g_taken + assign per_output_valid[j][i] = lane_pending[i][j]; + assign lane_taken[i][j] = lane_pending[i][j] && per_output_ready[j][i]; + end + assign core_rsp_xbar_ready[i] = core_rsp_xbar_valid[i] && ((lane_pending[i] & ~lane_taken[i]) == '0); + always @(posedge clk) begin + if (reset) begin + sent_r <= '0; + end else if (core_rsp_xbar_valid[i]) begin + sent_r <= core_rsp_xbar_ready[i] ? '0 : (sent_r | lane_taken[i]); + end + end + end + for (genvar j = 0; j < NUM_REQS; ++j) begin : g_outputs + VX_stream_arb #( + .NUM_INPUTS (NUM_BANKS), + .NUM_OUTPUTS (1), + .DATAW (CORE_RSP_DATAW), + .ARBITER ("R"), + .OUT_BUF (CORE_OUT_BUF) + ) core_rsp_arb ( + .clk (clk), + .reset (reset), + .valid_in (per_output_valid[j]), + .data_in (core_rsp_xbar_pdata), + .ready_in (per_output_ready[j]), + .valid_out (core_rsp_valid[j]), + .data_out (core_rsp_pdata[j]), + `UNUSED_PIN (sel_out), + .ready_out (core_rsp_ready[j]) + ); + end + end else begin : g_core_rsp_xbar + VX_stream_omega #( + .NUM_INPUTS (NUM_BANKS), + .NUM_OUTPUTS (NUM_REQS), + .DATAW (CORE_RSP_DATAW), + .ARBITER ("R"), + .OUT_BUF (CORE_OUT_BUF) + ) core_rsp_xbar ( + .clk (clk), + .reset (reset), + .valid_in (core_rsp_xbar_valid), + .data_in (core_rsp_xbar_pdata), + .sel_in (core_rsp_xbar_idx), + .ready_in (core_rsp_xbar_ready), + .valid_out (core_rsp_valid), + .data_out (core_rsp_pdata), + `UNUSED_PIN (sel_out), + .ready_out (core_rsp_ready), + `UNUSED_PIN (collisions) + ); + end for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_rsp assign core_bus2_if[i].rsp_valid = core_rsp_valid[i]; assign {core_bus2_if[i].rsp_data.data, core_bus2_if[i].rsp_data.tag} = core_rsp_pdata[i]; @@ -661,7 +778,8 @@ module VX_cache import VX_gpu_pkg::*; #( wire [NUM_REQS-1:0] perf_core_writes_per_req; wire [NUM_REQS-1:0] perf_crsp_stall_per_req; wire [MEM_PORTS-1:0] perf_mem_stall_per_port; - `BUFFER(perf_core_reads_per_req, core_req_valid & core_req_ready & ~core_req_rw); + // a merged read counts once, as the bank access that serves it + `BUFFER(perf_core_reads_per_req, core_req_valid & core_req_ready & ~core_req_rw & ~core_req_follower); `BUFFER(perf_core_writes_per_req, core_req_valid & core_req_ready & core_req_rw); for (genvar i = 0; i < NUM_REQS; ++i) begin : g_perf_crsp_stall_per_req assign perf_crsp_stall_per_req[i] = core_bus_if[i].rsp_valid && ~core_bus_if[i].rsp_ready; diff --git a/hw/rtl/cache/VX_cache_cluster.sv b/hw/rtl/cache/VX_cache_cluster.sv index 88dc19d40d..94a9ebdb58 100644 --- a/hw/rtl/cache/VX_cache_cluster.sv +++ b/hw/rtl/cache/VX_cache_cluster.sv @@ -60,6 +60,7 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( parameter SHARED_LLC = 0, // The LLC serves every memory client alone (point of coherence). parameter FLUSH_FORWARD = 1, // A flush also flushes the next level. parameter BANK_HASH = 0, // Hashed bank select (cache_bank_hash). + parameter READ_MERGE = 0, // Merge same-word reads within groups of this many adjacent ports (0: off). parameter AMO_ENABLE = 0 // Enable atomic memory operations. ) ( @@ -183,6 +184,7 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( .SHARED_LLC (SHARED_LLC), .FLUSH_FORWARD (FLUSH_FORWARD), .BANK_HASH (BANK_HASH), + .READ_MERGE (READ_MERGE), .AMO_ENABLE (AMO_ENABLE) ) cache_wrap ( `ifdef PERF_ENABLE diff --git a/hw/rtl/cache/VX_cache_wrap.sv b/hw/rtl/cache/VX_cache_wrap.sv index 2f04c76436..f0d21ecb98 100644 --- a/hw/rtl/cache/VX_cache_wrap.sv +++ b/hw/rtl/cache/VX_cache_wrap.sv @@ -62,6 +62,7 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( parameter SHARED_LLC = 0, // The LLC serves every memory client alone (point of coherence). parameter FLUSH_FORWARD = 1, // A flush also flushes the next level. parameter BANK_HASH = 0, // Hashed bank select (cache_bank_hash). + parameter READ_MERGE = 0, // Merge same-word reads within groups of this many adjacent ports (0: off). parameter AMO_ENABLE = 0 // Enable atomic operations in cache banks. ) ( @@ -164,6 +165,7 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( .SHARED_LLC (SHARED_LLC), .FLUSH_FORWARD (FLUSH_FORWARD), .BANK_HASH (BANK_HASH), + .READ_MERGE (READ_MERGE), .AMO_ENABLE (AMO_ENABLE) ) cache ( .clk (clk), diff --git a/tests/regression/Makefile b/tests/regression/Makefile index 15db9427dd..5b2f655a92 100644 --- a/tests/regression/Makefile +++ b/tests/regression/Makefile @@ -8,7 +8,7 @@ TESTS := \ arith printf diverge diverge_large sort fence fence_mp ldst_order \ vecadd vecadd_v1 sgemm sgemmx sgemm_v1 conv3 relu sgemv \ sgemm2 sgemm2_v1 madmax stencil3d raycast bfs jacobi pathfinder \ - softmax \ + softmax uniform_ld \ sgemm_tcu sgemm_tcu_mx sgemm_tcu_sp_mx sgemm_tcu_wg sgemm_tcu_wg_mx sgemm_tcu_sp sgemm_tcu_wg_sp \ sgemm2_dxa sgemm2_tcu sgemm_tcu_wg_dxa sgemm_tcu_wg_sp_dxa \ sgemm2_dxa_mcast sgemm_tcu_wg_dxa_mcast \ diff --git a/tests/regression/uniform_ld/Makefile b/tests/regression/uniform_ld/Makefile new file mode 100644 index 0000000000..fb9ed28371 --- /dev/null +++ b/tests/regression/uniform_ld/Makefile @@ -0,0 +1,16 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := uniform_ld + +SRC_DIR := $(VORTEX_HOME)/tests/regression/$(PROJECT) + +SRCS := $(SRC_DIR)/main.cpp + +VX_SRCS := $(SRC_DIR)/kernel.cpp + +OPTS ?= -n64 + +KERNEL_LIB := vortex2 + +include ../common.mk diff --git a/tests/regression/uniform_ld/common.h b/tests/regression/uniform_ld/common.h new file mode 100644 index 0000000000..010b9dc91b --- /dev/null +++ b/tests/regression/uniform_ld/common.h @@ -0,0 +1,27 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include + +#define UNIFORM_MARK 0x80000000u + +// Per warp and iteration: one RAW word and one WAR word (four words apart, +// so four iterations share a line), and one 16-word line of read-only words. +#define RAW_STRIDE 4 +#define WAR_STRIDE 4 +#define RO_LINE 16 + +typedef struct { + uint32_t num_iters; // iterations per warp + uint32_t num_threads; // total threads launched + uint32_t warp_size; // threads per warp + uint32_t scatter_words; + uint64_t raw_addr; // in: initial words; out: RAW words overwritten with UNIFORM_MARK + index + uint64_t war_addr; // in: initial words; out: WAR words overwritten with UNIFORM_MARK + index + uint64_t ro_addr; // in: read-only words + uint64_t scatter_addr;// in: read-only words scattered across lines + uint64_t err_addr; // out: per-thread error count + uint64_t sink_addr; // out: per-thread sum of the scattered loads (keeps them alive) +} kernel_arg_t; + +#endif diff --git a/tests/regression/uniform_ld/kernel.cpp b/tests/regression/uniform_ld/kernel.cpp new file mode 100644 index 0000000000..7673a99508 --- /dev/null +++ b/tests/regression/uniform_ld/kernel.cpp @@ -0,0 +1,60 @@ +#include +#include "common.h" + +// Warp-uniform loads around single-lane stores. Per iteration k, lane +// s = k % W (so every lane, and with it every dcache channel, takes a turn): +// +// scattered loads - every lane a different line, so the channels drain +// unevenly before the accesses below +// RAW: lane s stores its word, then the whole warp loads it (one address): +// lane s must see its own store +// WAR: the whole warp loads a word, then lane s stores it: lane s's load +// must not see its own later store +// read-only loads - lanes in groups of 1, 2, 4, 8 or 16 share a word of +// one line: every lane must get its own word +__kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { + auto raw = reinterpret_cast(arg->raw_addr); + auto war = reinterpret_cast(arg->war_addr); + auto ro = reinterpret_cast(arg->ro_addr); + auto scatter = reinterpret_cast(arg->scatter_addr); + auto err = reinterpret_cast(arg->err_addr); + auto sink = reinterpret_cast(arg->sink_addr); + const uint32_t W = arg->warp_size; + const uint32_t tid = blockIdx.x * blockDim.x + threadIdx.x; + const uint32_t lane = tid % W; + const uint32_t warp = tid / W; + uint32_t errors = 0; + uint32_t acc = 0; + for (uint32_t k = 0; k < arg->num_iters; ++k) { + uint32_t s = k % W; + uint32_t slot = warp * arg->num_iters + k; + + acc += scatter[(lane * RO_LINE + slot * 5) % arg->scatter_words]; + + uint32_t ri = slot * RAW_STRIDE; + if (lane == s) { + raw[ri] = UNIFORM_MARK + ri; + } + uint32_t rv = raw[ri]; + if (lane == s && rv != UNIFORM_MARK + ri) { + ++errors; + } + + uint32_t wi = slot * WAR_STRIDE; + uint32_t wv = war[wi]; + if (lane == s) { + war[wi] = UNIFORM_MARK + wi; + } + if (lane == s && wv != wi * 7 + 1) { + ++errors; + } + + uint32_t shift = k % 5; + uint32_t oi = slot * RO_LINE + ((lane >> shift) & (RO_LINE - 1)); + if (ro[oi] != oi * 3 + 2) { + ++errors; + } + } + err[tid] = errors; + sink[tid] = acc; +} diff --git a/tests/regression/uniform_ld/main.cpp b/tests/regression/uniform_ld/main.cpp new file mode 100644 index 0000000000..40d0943689 --- /dev/null +++ b/tests/regression/uniform_ld/main.cpp @@ -0,0 +1,164 @@ +// uniform_ld — warp-uniform loads must keep each thread's order with its own +// stores, and every lane must receive its own word. + +#include +#include "common.h" + +#include +#include +#include +#include +#include +#include + +#define CHECK(expr) do { \ + vx_result_t _r = (expr); \ + if (_r != VX_SUCCESS) { \ + std::fprintf(stderr, "FAIL %s:%d: '%s' returned %s\n", \ + __FILE__, __LINE__, #expr, vx_result_string(_r)); \ + std::exit(1); \ + } \ +} while (0) + +namespace { +const char* kernel_file = "kernel.vxbin"; +uint32_t num_iters = 64; +uint32_t scatter_words = 65536; + +void parse_args(int argc, char** argv) { + int c; + while ((c = getopt(argc, argv, "n:s:k:h")) != -1) { + switch (c) { + case 'n': num_iters = std::atoi(optarg); break; + case 's': scatter_words = std::atoi(optarg); break; + case 'k': kernel_file = optarg; break; + default: + std::cout << "Usage: [-n iterations] [-s scattered words] [-k kernel] [-h]" << std::endl; + std::exit(c == 'h' ? 0 : -1); + } + } +} +} // namespace + +int main(int argc, char** argv) { + parse_args(argc, argv); + if (num_iters == 0) num_iters = 1; + if (scatter_words == 0) scatter_words = 1; + + vx_device_h dev = nullptr; + CHECK(vx_device_open(0, &dev)); + + // Fill every warp of every core. + uint64_t nt = 0, nw = 0, nc = 0; + CHECK(vx_device_query(dev, VX_CAPS_NUM_THREADS, &nt)); + CHECK(vx_device_query(dev, VX_CAPS_NUM_WARPS, &nw)); + CHECK(vx_device_query(dev, VX_CAPS_NUM_CORES, &nc)); + const uint32_t W = static_cast(nt); + const uint32_t num_threads = static_cast(nt * nw * nc); + const uint32_t num_slots = (num_threads / W) * num_iters; + std::cout << "uniform_ld: iters=" << num_iters << " threads=" << num_threads + << " scatter=" << scatter_words << std::endl; + + vx_queue_info_t qi = { sizeof(qi), nullptr, VX_QUEUE_PRIORITY_NORMAL, 0 }; + vx_queue_h q = nullptr; + CHECK(vx_queue_create(dev, &qi, &q)); + + const uint32_t raw_words = num_slots * RAW_STRIDE; + const uint32_t war_words = num_slots * WAR_STRIDE; + const uint32_t ro_words = num_slots * RO_LINE; + vx_buffer_h raw_buf = nullptr, war_buf = nullptr, ro_buf = nullptr; + vx_buffer_h scatter_buf = nullptr, err_buf = nullptr, sink_buf = nullptr; + CHECK(vx_buffer_create(dev, raw_words * sizeof(uint32_t), VX_MEM_READ_WRITE, &raw_buf)); + CHECK(vx_buffer_create(dev, war_words * sizeof(uint32_t), VX_MEM_READ_WRITE, &war_buf)); + CHECK(vx_buffer_create(dev, ro_words * sizeof(uint32_t), VX_MEM_READ, &ro_buf)); + CHECK(vx_buffer_create(dev, scatter_words * sizeof(uint32_t), VX_MEM_READ, &scatter_buf)); + CHECK(vx_buffer_create(dev, num_threads * sizeof(uint32_t), VX_MEM_WRITE, &err_buf)); + CHECK(vx_buffer_create(dev, num_threads * sizeof(uint32_t), VX_MEM_WRITE, &sink_buf)); + + vx_module_h mod = nullptr; + vx_kernel_h kern = nullptr; + CHECK(vx_module_load_file(dev, kernel_file, &mod)); + CHECK(vx_module_get_kernel(mod, "main", &kern)); + + kernel_arg_t kernel_arg{}; + kernel_arg.num_iters = num_iters; + kernel_arg.num_threads = num_threads; + kernel_arg.warp_size = W; + kernel_arg.scatter_words = scatter_words; + CHECK(vx_buffer_address(raw_buf, &kernel_arg.raw_addr)); + CHECK(vx_buffer_address(war_buf, &kernel_arg.war_addr)); + CHECK(vx_buffer_address(ro_buf, &kernel_arg.ro_addr)); + CHECK(vx_buffer_address(scatter_buf, &kernel_arg.scatter_addr)); + CHECK(vx_buffer_address(err_buf, &kernel_arg.err_addr)); + CHECK(vx_buffer_address(sink_buf, &kernel_arg.sink_addr)); + + std::vector h_raw(raw_words), h_war(war_words), h_ro(ro_words), h_scatter(scatter_words); + for (uint32_t i = 0; i < raw_words; ++i) h_raw[i] = i * 7 + 1; + for (uint32_t i = 0; i < war_words; ++i) h_war[i] = i * 7 + 1; + for (uint32_t i = 0; i < ro_words; ++i) h_ro[i] = i * 3 + 2; + for (uint32_t i = 0; i < scatter_words; ++i) h_scatter[i] = i; + CHECK(vx_enqueue_write(q, raw_buf, 0, h_raw.data(), raw_words * sizeof(uint32_t), 0, nullptr, nullptr)); + CHECK(vx_enqueue_write(q, war_buf, 0, h_war.data(), war_words * sizeof(uint32_t), 0, nullptr, nullptr)); + CHECK(vx_enqueue_write(q, ro_buf, 0, h_ro.data(), ro_words * sizeof(uint32_t), 0, nullptr, nullptr)); + CHECK(vx_enqueue_write(q, scatter_buf, 0, h_scatter.data(), scatter_words * sizeof(uint32_t), 0, nullptr, nullptr)); + + uint32_t grid[1], block[1]; + CHECK(vx_device_max_occupancy_grid(dev, 1, &num_threads, grid, block)); + + vx_launch_info_t li{}; + li.struct_size = sizeof(li); + li.kernel = kern; + li.args_host = &kernel_arg; + li.args_size = sizeof(kernel_arg); + li.ndim = 1; + li.grid_dim[0] = grid[0]; + li.block_dim[0]= block[0]; + + std::vector h_err(num_threads); + vx_event_h launch_ev = nullptr, read_ev = nullptr; + CHECK(vx_enqueue_launch(q, &li, 0, nullptr, &launch_ev)); + CHECK(vx_enqueue_read(q, h_err.data(), err_buf, 0, num_threads * sizeof(uint32_t), 1, &launch_ev, nullptr)); + CHECK(vx_enqueue_read(q, h_raw.data(), raw_buf, 0, raw_words * sizeof(uint32_t), 1, &launch_ev, nullptr)); + CHECK(vx_enqueue_read(q, h_war.data(), war_buf, 0, war_words * sizeof(uint32_t), 1, &launch_ev, &read_ev)); + CHECK(vx_event_wait_value(read_ev, 1, VX_TIMEOUT_INFINITE)); + + int errors = 0; + for (uint32_t t = 0; t < num_threads; ++t) { + if (h_err[t] == 0) continue; + if (errors < 16) + std::printf("*** thread %u: %u wrong loads\n", t, h_err[t]); + errors += h_err[t]; + } + for (uint32_t slot = 0; slot < num_slots; ++slot) { + uint32_t ri = slot * RAW_STRIDE, wi = slot * WAR_STRIDE; + if (h_raw[ri] != UNIFORM_MARK + ri) { + if (errors < 16) std::printf("*** raw[%u] expected=0x%x actual=0x%x\n", ri, UNIFORM_MARK + ri, h_raw[ri]); + ++errors; + } + if (h_war[wi] != UNIFORM_MARK + wi) { + if (errors < 16) std::printf("*** war[%u] expected=0x%x actual=0x%x\n", wi, UNIFORM_MARK + wi, h_war[wi]); + ++errors; + } + } + + vx_event_release(read_ev); + vx_event_release(launch_ev); + vx_buffer_release(sink_buf); + vx_buffer_release(err_buf); + vx_buffer_release(scatter_buf); + vx_buffer_release(ro_buf); + vx_buffer_release(war_buf); + vx_buffer_release(raw_buf); + vx_kernel_release(kern); + vx_module_release(mod); + vx_queue_release(q); + vx_device_dump_perf(dev, stdout); + vx_device_release(dev); + + if (errors) { + std::cout << "Found " << errors << " errors!\nFAILED!" << std::endl; + return 1; + } + std::cout << "PASSED!" << std::endl; + return 0; +} From 5e8e5c234dbabe2354434ff427436ded036606de Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 19 Oct 2026 03:18:19 -0700 Subject: [PATCH 051/124] Size the local-memory reservation hold budget by the core's harts A local-memory reservation refuses LRs from other harts until its own SC lands, spending one credit per refused LR, and yields to the next LR once the credits run out. The budget was fixed at 3 bits (7 refusals), sized as if credits were spent per cycle of the local round trip. A warp's lr.w reaches the bank as one request per lane, back to back, so at 16 threads one warp's own burst spent the budget and its 9th lane took the reservation from its first. Most SCs then found their reservation gone, and a contended LR/SC loop stopped making progress: amo's lrsc_lmem case failed at 16 threads with the A extension. The budget now covers two refusals per hart of the core: CLOG2(NUM_WARPS * NUM_THREADS) + 1 bits. rtlsim, amo with A: lrsc_lmem -n4, 16 threads FAILED (1,825,530 cycles) -> PASSED (35,313) full amo, 16 threads 14/15 -> 15/15 full amo, 8 threads and default config: 15/15 lrsc_counter on global memory at 4 cores + L2, 16 threads (256 contending harts) passes with the cache bank's existing budget. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/mem/VX_local_mem.sv | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/hw/rtl/mem/VX_local_mem.sv b/hw/rtl/mem/VX_local_mem.sv index 6d780960e5..5614595503 100644 --- a/hw/rtl/mem/VX_local_mem.sv +++ b/hw/rtl/mem/VX_local_mem.sv @@ -479,11 +479,16 @@ module VX_local_mem import VX_gpu_pkg::*; #( // holder-credit protocol a hot word maps to exactly one station, so // depth only reduces false eviction between distinct reserved words // aliasing in one bank -- and a spurious SC failure is architecturally - // legal. Clamp the depth and shorten the credit budget to the local - // round trip instead of inheriting the L1-miss sizing. + // legal. Clamp the depth. + // A credit is spent per refused LR, not per cycle: a warp's LR reaches + // the bank as one request per lane, back to back, so between a holder's + // LR and its SC every other hart of the core can be refused, and a + // refused hart can retry once more. The budget covers two refusals per + // hart; a smaller one lets one warp's own LR burst displace its holder + // and a contended retry loop stops making progress. VX_amo_unit #( .NUM_RES_ENTRIES (`MIN(`VX_CFG_AMO_RS_SIZE, 4)), - .HOLD_CREDIT_BITS (3), + .HOLD_CREDIT_BITS (`CLOG2(`VX_CFG_NUM_WARPS * `VX_CFG_NUM_THREADS) + 1), .LINE_ADDR_BITS (BANK_ADDR_WIDTH), .DATA_WIDTH (WORD_WIDTH) ) amo_unit ( From 75984cccd24f1440424d7c496678a393c238fc75 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 19 Oct 2026 03:41:10 -0700 Subject: [PATCH 052/124] Track MSHR misses in a line table so a cache can hold more in flight At realistic DRAM latency the dcache MSHR becomes the limit: with 16 entries and 400-cycle memory, sgemm -n128 at 16 threads finds a bank's MSHR full 20% of its cycles. The full MSHR holds only one or two distinct missing lines; the entries are requests merging onto them. A bigger flat MSHR helps, but every entry stores a full line address and takes part in the wide address compares and read muxes, so its area grows with the request count. With VX_CFG_{DCACHE,L2,L3}_MSHR_LINES set, the line address and sector move into a table of MSHR_LINES entries keyed by {line, sector}. Each entry is reference-counted by the request entries pointing at it and frees when the count reaches zero. Request entries (still MSHR_SIZE) keep only a small line index, so the wide compare covers the line table, and per-request matching, the AMO probe, fill, dequeue and the debug checks go through the index. Allocation, finalize, tail-find, the same-cycle late-joiner splice, word hazards, lazy-fetch parking and the payload RAM are unchanged, so replay and AMO order are too. The MSHR admits a request only while two line entries are free, folded into the existing almost-full stall, so the mshr stall counter covers it. Simulation checks that each count equals its live requests and that no line is needed when none is free. The default, 0, keeps the per-request address. sector_merge is a new test: alternate warps read opposite halves of one 128-byte line, so both sectors of a sectored L2 line miss together in one bank, and every word is checked against a host reference. It fails when line matching ignores the sector. rtlsim, MSHR 16 -> 64 requests over 16 lines (dcache, L2, L3), memory latency 400, 16 threads x 16 warps: sgemm -n128 2,733,260 -> 2,315,677 (-15.3%) raycast 539,032 -> 495,457 (-8.1%) Without B2's read merge the same sgemm gains 24.8%. At the stock latency the MSHR does not bind and the gain is under 0.5%, so the default stays off. 64/16 gives a flat 64-entry MSHR's cycles at about half its area (estimated). Off, every config is cycle-identical. mstress-6, amo (with A) at 1 core and at 4 cores + L2, uniform_ld, lazy_fetch-1 and flush_coherence chain-3 with the CP on the LLC pass with the line table on. Co-Authored-By: Claude Opus 5.5 --- VX_config.toml | 6 + ci/testcases/cache.yaml | 22 +++ hw/rtl/VX_cluster.sv | 1 + hw/rtl/VX_socket.sv | 1 + hw/rtl/Vortex.sv | 1 + hw/rtl/cache/VX_cache.sv | 2 + hw/rtl/cache/VX_cache_bank.sv | 8 +- hw/rtl/cache/VX_cache_cluster.sv | 2 + hw/rtl/cache/VX_cache_mshr.sv | 192 ++++++++++++++++++++--- hw/rtl/cache/VX_cache_wrap.sv | 2 + hw/unittest/cache/VX_cache_top.sv | 2 + tests/regression/Makefile | 2 +- tests/regression/sector_merge/Makefile | 16 ++ tests/regression/sector_merge/common.h | 13 ++ tests/regression/sector_merge/kernel.cpp | 23 +++ tests/regression/sector_merge/main.cpp | 180 +++++++++++++++++++++ 16 files changed, 448 insertions(+), 25 deletions(-) create mode 100644 tests/regression/sector_merge/Makefile create mode 100644 tests/regression/sector_merge/common.h create mode 100644 tests/regression/sector_merge/kernel.cpp create mode 100644 tests/regression/sector_merge/main.cpp diff --git a/VX_config.toml b/VX_config.toml index 2b6d21a4fe..ab8a07a5dc 100644 --- a/VX_config.toml +++ b/VX_config.toml @@ -235,6 +235,10 @@ VX_CFG_DCACHE_DIRTYBYTES = "expr: $VX_CFG_DCACHE_LAZY_FETCH" VX_CFG_DCACHE_LAZY_FETCH = 0 VX_CFG_DCACHE_REPL_POLICY = "expr: $__cache_repl_fifo" VX_CFG_DCACHE_MSHR_SIZE = 16 +# MSHR line entries: distinct {line, sector} misses a bank tracks, each naming +# the chain of requests merged on it; MSHR_SIZE then sizes the request pool. +# 0 keeps the address in every request entry (lines = requests). +VX_CFG_DCACHE_MSHR_LINES = 0 # dcache is unsectored: line = sector = mem transaction granule = L1_LINE (which # defaults to MEM_BLOCK but can be configured smaller). Banking comes from the # reduced WORD above (>= sector is clamped), not the line. Deriving from L1_LINE @@ -270,6 +274,7 @@ VX_CFG_L2_DIRTYBYTES = "expr: $VX_CFG_L2_LAZY_FETCH" VX_CFG_L2_LAZY_FETCH = 0 VX_CFG_L2_REPL_POLICY = "expr: $__cache_repl_fifo" VX_CFG_L2_MSHR_SIZE = 16 +VX_CFG_L2_MSHR_LINES = 0 VX_CFG_L2_LATENCY = "expr: 4 + max(0, clog2($VX_CFG_L2_SIZE) - clog2(1048576))" VX_CFG_L2_MRSQ_SIZE = 4 VX_CFG_L2_MREQ_SIZE = 0 @@ -289,6 +294,7 @@ VX_CFG_L3_DIRTYBYTES = "expr: $VX_CFG_L3_LAZY_FETCH" VX_CFG_L3_LAZY_FETCH = 0 VX_CFG_L3_REPL_POLICY = "expr: $__cache_repl_fifo" VX_CFG_L3_MSHR_SIZE = 16 +VX_CFG_L3_MSHR_LINES = 0 VX_CFG_L3_LATENCY = "expr: 4 + max(0, clog2($VX_CFG_L3_SIZE) - clog2(2097152))" VX_CFG_L3_MRSQ_SIZE = 4 VX_CFG_L3_MREQ_SIZE = 0 diff --git a/ci/testcases/cache.yaml b/ci/testcases/cache.yaml index ceb79a2788..361ec300e2 100644 --- a/ci/testcases/cache.yaml +++ b/ci/testcases/cache.yaml @@ -337,3 +337,25 @@ tests: shape: cores: 4 l2cache: true +# Both sectors of one sectored L2 line miss together from different warps and +# cores: each must get its own fill, never chain behind the other sector's. +# Run with the address held per request and with the MSHR line table. +- id: sector_merge-1 + via: blackbox + drivers: + - simx + - rtlsim + app: sector_merge + configs: -DVX_CFG_SOCKET_SIZE=1 + shape: + cores: 4 + l2cache: true +- id: sector_merge-2 + via: blackbox + drivers: + - rtlsim + app: sector_merge + configs: -DVX_CFG_SOCKET_SIZE=1 -DVX_CFG_DCACHE_MSHR_SIZE=32 -DVX_CFG_DCACHE_MSHR_LINES=8 -DVX_CFG_L2_MSHR_SIZE=32 -DVX_CFG_L2_MSHR_LINES=8 + shape: + cores: 4 + l2cache: true diff --git a/hw/rtl/VX_cluster.sv b/hw/rtl/VX_cluster.sv index 7e42f65be8..1b2ff700c5 100644 --- a/hw/rtl/VX_cluster.sv +++ b/hw/rtl/VX_cluster.sv @@ -224,6 +224,7 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; .MEM_PORTS (L2_MEM_PORTS), .CRSQ_SIZE (`VX_CFG_L2_CRSQ_SIZE), .MSHR_SIZE (`VX_CFG_L2_MSHR_SIZE), + .MSHR_LINES (`VX_CFG_L2_MSHR_LINES), .MRSQ_SIZE (`VX_CFG_L2_MRSQ_SIZE), .MREQ_SIZE (`VX_CFG_L2_MREQ_SIZE), .LATENCY (`VX_CFG_L2_LATENCY), diff --git a/hw/rtl/VX_socket.sv b/hw/rtl/VX_socket.sv index cf04039049..69c111f121 100644 --- a/hw/rtl/VX_socket.sv +++ b/hw/rtl/VX_socket.sv @@ -219,6 +219,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .MEM_PORTS (L1_MEM_PORTS), .CRSQ_SIZE (`VX_CFG_DCACHE_CRSQ_SIZE), .MSHR_SIZE (`VX_CFG_DCACHE_MSHR_SIZE), + .MSHR_LINES (`VX_CFG_DCACHE_MSHR_LINES), .MRSQ_SIZE (`VX_CFG_DCACHE_MRSQ_SIZE), .MREQ_SIZE (`VX_CFG_DCACHE_MREQ_SIZE), .LATENCY (`VX_CFG_DCACHE_LATENCY), diff --git a/hw/rtl/Vortex.sv b/hw/rtl/Vortex.sv index 434d9ab4ab..59bebc3865 100644 --- a/hw/rtl/Vortex.sv +++ b/hw/rtl/Vortex.sv @@ -163,6 +163,7 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( .MEM_PORTS (L3_MEM_PORTS), .CRSQ_SIZE (`VX_CFG_L3_CRSQ_SIZE), .MSHR_SIZE (`VX_CFG_L3_MSHR_SIZE), + .MSHR_LINES (`VX_CFG_L3_MSHR_LINES), .MRSQ_SIZE (`VX_CFG_L3_MRSQ_SIZE), .MREQ_SIZE (`VX_CFG_L3_MREQ_SIZE), .LATENCY (`VX_CFG_L3_LATENCY), diff --git a/hw/rtl/cache/VX_cache.sv b/hw/rtl/cache/VX_cache.sv index 446b7355e1..d51f0c3e28 100644 --- a/hw/rtl/cache/VX_cache.sv +++ b/hw/rtl/cache/VX_cache.sv @@ -29,6 +29,7 @@ module VX_cache import VX_gpu_pkg::*; #( parameter CRSQ_SIZE = 0, // Core Response Queue Size parameter MSHR_SIZE = 16, // Miss Reserv Queue Knob + parameter MSHR_LINES = 0, // MSHR distinct line entries (0 = address per request) parameter MRSQ_SIZE = 4, // Memory Response Queue Size parameter MREQ_SIZE = 0, // Memory Request Queue Size (0 = derived minimum) @@ -445,6 +446,7 @@ module VX_cache import VX_gpu_pkg::*; #( .REPL_POLICY (REPL_POLICY), .CRSQ_SIZE (CRSQ_SIZE), .MSHR_SIZE (MSHR_SIZE), + .MSHR_LINES (MSHR_LINES), .MRSQ_SIZE (MRSQ_SIZE), .MREQ_SIZE (MREQ_SIZE), .LATENCY (LATENCY), diff --git a/hw/rtl/cache/VX_cache_bank.sv b/hw/rtl/cache/VX_cache_bank.sv index 4187d85858..ae219feccb 100644 --- a/hw/rtl/cache/VX_cache_bank.sv +++ b/hw/rtl/cache/VX_cache_bank.sv @@ -26,6 +26,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( parameter SECTOR_SIZE = LINE_SIZE,// sector (fill/eviction granule); = LINE_SIZE => 1 sector parameter CRSQ_SIZE = 0, // extra core-response queue slots over minimum parameter MSHR_SIZE = 1, // miss reservation queue size + parameter MSHR_LINES = 0, // MSHR distinct line entries (0 = address per request) parameter MRSQ_SIZE = 1, // memory response queue size (sized at wrapper) parameter MREQ_SIZE = 0, // memory request queue size (0 = derived minimum) parameter WRITE_ENABLE = 1, @@ -173,6 +174,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // Shared signals // ------------------------------------------------------------------------ wire crsp_queue_stall, mshr_alm_full; + wire mshr_req_full, mshr_line_ready; wire mshr_allocate_st0; (* max_fanout = 64 *) wire pipe_stall; // Authoritative "no entry held", straight off the MSHR's valid mask. The @@ -1005,11 +1007,13 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .decr (mshr_dequeue), `UNUSED_PIN (empty), `UNUSED_PIN (alm_empty), - .full (mshr_alm_full), + .full (mshr_req_full), `UNUSED_PIN (alm_full), `UNUSED_PIN (size) ); + assign mshr_alm_full = mshr_req_full || ~mshr_line_ready; + VX_cache_mshr #( .INSTANCE_ID (`SFORMATF(("%s-mshr", INSTANCE_ID))), .BANK_ID (BANK_ID), @@ -1018,6 +1022,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .SECTOR_SIZE (SECTOR_SIZE), .NUM_BANKS (NUM_BANKS), .MSHR_SIZE (MSHR_SIZE), + .MSHR_LINES (MSHR_LINES), .WRITEBACK (WRITEBACK), .AMO_ENABLE (AMO_ENABLE != 0), .AMO_PASSTHRU ((AMO_ENABLE != 0) && (IS_LLC == 0)), @@ -1069,6 +1074,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .allocate_pending_ovl (mshr_pending_ovl_raw), .allocate_previd (mshr_previd), `UNUSED_PIN (allocate_ready), + .allocate_line_ready (mshr_line_ready), .finalize_valid (mshr_finalize_st1 && ~pipe_stall), .finalize_is_release (mshr_release_st1), // Only link an entry into the pending chain if it is KEPT (a miss). A diff --git a/hw/rtl/cache/VX_cache_cluster.sv b/hw/rtl/cache/VX_cache_cluster.sv index 94a9ebdb58..a8d0f99a44 100644 --- a/hw/rtl/cache/VX_cache_cluster.sv +++ b/hw/rtl/cache/VX_cache_cluster.sv @@ -33,6 +33,7 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( parameter CRSQ_SIZE = 0, // Core Response Queue Size parameter MSHR_SIZE = 16, // Miss Reserv Queue Knob + parameter MSHR_LINES = 0, // MSHR distinct line entries (0 = address per request) parameter MRSQ_SIZE = 4, // Memory Response Queue Size parameter MREQ_SIZE = 0, // Memory Request Queue Size (0 = derived minimum) @@ -171,6 +172,7 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( .REPL_POLICY (REPL_POLICY), .CRSQ_SIZE (CRSQ_SIZE), .MSHR_SIZE (MSHR_SIZE), + .MSHR_LINES (MSHR_LINES), .MRSQ_SIZE (MRSQ_SIZE), .MREQ_SIZE (MREQ_SIZE), .LATENCY (LATENCY), diff --git a/hw/rtl/cache/VX_cache_mshr.sv b/hw/rtl/cache/VX_cache_mshr.sv index c8e66e0c10..c314df553c 100644 --- a/hw/rtl/cache/VX_cache_mshr.sv +++ b/hw/rtl/cache/VX_cache_mshr.sv @@ -38,6 +38,15 @@ // The finalize request is used to persit or release the currently allocated MSHR entry // if we had a cache miss or a hit, respectively. // +// With MSHR_LINES != 0 the line address lives in a separate table of +// MSHR_LINES entries, one per distinct {line, sector} held, and each request +// entry stores only an index into it. The address CAM then spans the lines +// rather than the requests, so the request pool can grow (many requests merge +// on few lines) without growing the wide compare. A line entry is reference +// counted by the request entries naming it and frees when the count drains. +// The bank admits a core request only while two line entries are free: the +// request already past admission may claim one before this one reaches S0. +// // Warning: This MSHR implementation is strongly coupled with the bank pipeline // and as such changes to either module requires careful evaluation. // @@ -50,6 +59,7 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( parameter SECTOR_SIZE = LINE_SIZE, // Size of a sector in bytes (coalescing/fill granule); = LINE_SIZE => 1 sector parameter NUM_BANKS = 1, // Number of banks parameter MSHR_SIZE = 4, // Miss Reserv Queue Knob + parameter MSHR_LINES = 0, // distinct line entries (0 = address held per request) parameter DATA_WIDTH = 1, // MSHR parameters parameter WRITEBACK = 0, // Enable cache writeback parameter AMO_ENABLE = 0, // Per-entry AMO tracking (probe outputs) @@ -127,6 +137,8 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( output wire allocate_pending_ovl, output wire [MSHR_ADDR_WIDTH-1:0] allocate_previd, output wire allocate_ready, + // a line entry is free for every request the bank can still admit + output wire allocate_line_ready, // finalize input wire finalize_valid, @@ -144,11 +156,8 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( `UNUSED_PARAM (BANK_ID) `UNUSED_PARAM (BANK_HASH) - reg [`CS_LINE_ADDR_WIDTH-1:0] addr_table [0:MSHR_SIZE-1]; - // Per-entry sector. Coalescing matches on {line, sector} so same-line - // different-sector misses get independent fills (each replay then hits its - // own filled sector). Zero-width-equivalent (1 bit, all 0) when 1 sector/line. - reg [`UP(`CS_SECTOR_SEL_BITS)-1:0] sector_table [0:MSHR_SIZE-1]; + `STATIC_ASSERT((MSHR_LINES == 0) || ((MSHR_LINES >= 2) && (MSHR_LINES <= MSHR_SIZE)), ("invalid parameter: MSHR_LINES")) + // Per-entry word-in-line select, kept outside the payload RAM so the // allocate probe can compare it: the hit-demotion order hazard is // word-granular (a chained entry constrains only requests that touch @@ -182,6 +191,13 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( wire dequeue_park_fire = (LAZY_FETCH != 0) && dequeue_fire && dequeue_park; wire dequeue_free = dequeue_fire && ~dequeue_park_fire; + // Entry holds the allocate request's {line, sector} / the probe's line. + // Coalescing matches on {line, sector} so same-line different-sector misses + // get independent fills (each replay then hits its own filled sector). + wire [MSHR_SIZE-1:0] entry_match; + wire [MSHR_SIZE-1:0] entry_probe; + wire [`CS_LINE_ADDR_WIDTH-1:0] dequeue_addr_n; + wire [MSHR_SIZE-1:0] addr_matches; for (genvar i = 0; i < MSHR_SIZE; ++i) begin : g_addr_matches // Exclude the entry being consumed this cycle: an allocate that links @@ -193,8 +209,7 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( // cycle: a released head never fills, so a request that linked behind // it would wait forever. Exclude it too — the requester re-looks-up // and hits the line the releasing entry just installed. - assign addr_matches[i] = valid_table[i] && (addr_table[i] == allocate_addr) - && (sector_table[i] == allocate_sector) && ~amo_mask[i] + assign addr_matches[i] = valid_table[i] && entry_match[i] && ~amo_mask[i] && ~(dequeue_free && (dequeue_id == MSHR_ADDR_WIDTH'(i))) && ~(finalize_valid && finalize_is_release && (finalize_id == MSHR_ADDR_WIDTH'(i))); end @@ -288,8 +303,6 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( end if (allocate_fire) begin - addr_table[allocate_id] <= allocate_addr; - sector_table[allocate_id] <= allocate_sector; word_table[allocate_id] <= allocate_word_idx; write_table[allocate_id] <= allocate_rw; end @@ -303,14 +316,150 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( next_table <= next_table_n; end + // Per-entry address, for checks and traces only. + wire [`CS_LINE_ADDR_WIDTH-1:0] entry_addr [MSHR_SIZE]; + + if (MSHR_LINES != 0) begin : g_line_table + localparam LINE_ID_W = `LOG2UP(MSHR_LINES); + localparam LCNT_W = `CLOG2(MSHR_SIZE + 1); + + reg [`CS_LINE_ADDR_WIDTH-1:0] line_addr [0:MSHR_LINES-1]; + reg [`UP(`CS_SECTOR_SEL_BITS)-1:0] line_sector [0:MSHR_LINES-1]; + reg [LCNT_W-1:0] line_cnt [0:MSHR_LINES-1]; + reg [LINE_ID_W-1:0] entry_line [0:MSHR_SIZE-1]; + + reg [LINE_ID_W-1:0] line_free_id_r; + reg line_free_r, line_ready_r; + + wire [MSHR_LINES-1:0] line_valid, line_hits, line_probes, line_free_n; + for (genvar l = 0; l < MSHR_LINES; ++l) begin : g_line_match + assign line_valid[l] = (line_cnt[l] != '0); + assign line_hits[l] = line_valid[l] && (line_addr[l] == allocate_addr) && (line_sector[l] == allocate_sector); + assign line_probes[l] = line_valid[l] && (line_addr[l] == probe_addr); + end + + // a {line, sector} occupies at most one line entry + wire line_hit; + wire [LINE_ID_W-1:0] line_hit_id; + VX_priority_encoder #( + .N (MSHR_LINES) + ) line_hit_sel ( + .data_in (line_hits), + .index_out (line_hit_id), + .valid_out (line_hit), + `UNUSED_PIN (onehot_out) + ); + + wire [LINE_ID_W-1:0] alloc_line = line_hit ? line_hit_id : line_free_id_r; + + for (genvar i = 0; i < MSHR_SIZE; ++i) begin : g_entry_match + assign entry_match[i] = line_hit && (entry_line[i] == line_hit_id); + assign entry_probe[i] = line_probes[entry_line[i]]; + assign entry_addr[i] = line_addr[entry_line[i]]; + end + + wire [LINE_ID_W-1:0] dequeue_line = entry_line[dequeue_id]; + wire [LINE_ID_W-1:0] finalize_line = entry_line[finalize_id]; + wire finalize_free = finalize_valid && finalize_is_release; + + for (genvar l = 0; l < MSHR_LINES; ++l) begin : g_line_cnt + wire inc = allocate_fire && (alloc_line == LINE_ID_W'(l)); + wire dec_d = dequeue_free && (dequeue_line == LINE_ID_W'(l)); + wire dec_f = finalize_free && (finalize_line == LINE_ID_W'(l)); + wire [LCNT_W-1:0] cnt_n = line_cnt[l] + LCNT_W'(inc) - LCNT_W'(dec_d) - LCNT_W'(dec_f); + assign line_free_n[l] = (cnt_n == '0); + always @(posedge clk) begin + if (reset) begin + line_cnt[l] <= '0; + end else begin + line_cnt[l] <= cnt_n; + end + end + end + + wire [MSHR_LINES-1:0] line_free_oh; + wire [LINE_ID_W-1:0] line_free_id_n; + wire line_free_any_n; + VX_priority_encoder #( + .N (MSHR_LINES) + ) line_free_sel ( + .data_in (line_free_n), + .onehot_out (line_free_oh), + .index_out (line_free_id_n), + .valid_out (line_free_any_n) + ); + + always @(posedge clk) begin + if (reset) begin + line_free_r <= 1'b0; + line_ready_r <= 1'b0; + end else begin + line_free_r <= line_free_any_n; + line_ready_r <= line_free_any_n && (| (line_free_n & ~line_free_oh)); + end + line_free_id_r <= line_free_id_n; + if (allocate_fire) begin + entry_line[allocate_id] <= alloc_line; + if (~line_hit) begin + line_addr[line_free_id_r] <= allocate_addr; + line_sector[line_free_id_r] <= allocate_sector; + end + end + end + + assign fill_addr = line_addr[entry_line[fill_id]]; + assign fill_sector = line_sector[entry_line[fill_id]]; + assign dequeue_addr_n = line_addr[entry_line[dequeue_id_n]]; + assign allocate_line_ready = line_ready_r; + + `RUNTIME_ASSERT(~(allocate_fire && ~line_hit && ~line_free_r), ("*** %s line table overflow: addr=0x%0h (#%0d)", INSTANCE_ID, + `CS_BANK_TO_FULL_ADDR(allocate_addr, BANK_ID), alc_req_uuid)) + + `ifdef SIMULATION + // Each line count equals the live entries naming the line. + for (genvar l = 0; l < MSHR_LINES; ++l) begin : g_line_check + integer refs; + always @(*) begin + refs = 0; + for (integer i = 0; i < MSHR_SIZE; ++i) begin + refs += 32'(valid_table[i] && (entry_line[i] == LINE_ID_W'(l))); + end + end + `RUNTIME_ASSERT(refs == 32'(line_cnt[l]), ("*** %s line count mismatch: line=%0d, count=%0d, refs=%0d", INSTANCE_ID, + l, line_cnt[l], refs)) + end + `endif + end else begin : g_entry_table + reg [`CS_LINE_ADDR_WIDTH-1:0] addr_table [0:MSHR_SIZE-1]; + reg [`UP(`CS_SECTOR_SEL_BITS)-1:0] sector_table [0:MSHR_SIZE-1]; + + always @(posedge clk) begin + if (allocate_fire) begin + addr_table[allocate_id] <= allocate_addr; + sector_table[allocate_id] <= allocate_sector; + end + end + + for (genvar i = 0; i < MSHR_SIZE; ++i) begin : g_entry_match + assign entry_match[i] = (addr_table[i] == allocate_addr) && (sector_table[i] == allocate_sector); + assign entry_probe[i] = (addr_table[i] == probe_addr); + assign entry_addr[i] = addr_table[i]; + end + + assign fill_addr = addr_table[fill_id]; + assign fill_sector = sector_table[fill_id]; + assign dequeue_addr_n = addr_table[dequeue_id_n]; + assign allocate_line_ready = 1'b1; + end + `RUNTIME_ASSERT(~(allocate_fire && valid_table[allocate_id_r]), ("*** %s inuse allocation: addr=0x%0h, id=%0d (#%0d)", INSTANCE_ID, `CS_BANK_TO_FULL_ADDR(allocate_addr, BANK_ID), allocate_id_r, alc_req_uuid)) `RUNTIME_ASSERT(~(finalize_valid && ~valid_table[finalize_id]), ("*** %s invalid release: addr=0x%0h, id=%0d (#%0d)", INSTANCE_ID, - `CS_BANK_TO_FULL_ADDR(addr_table[finalize_id], BANK_ID), finalize_id, fin_req_uuid)) + `CS_BANK_TO_FULL_ADDR(entry_addr[finalize_id], BANK_ID), finalize_id, fin_req_uuid)) `RUNTIME_ASSERT(~(fill_valid && ~valid_table[fill_id]), ("*** %s invalid fill: addr=0x%0h, id=%0d", INSTANCE_ID, - `CS_BANK_TO_FULL_ADDR(addr_table[fill_id], BANK_ID), fill_id)) + `CS_BANK_TO_FULL_ADDR(entry_addr[fill_id], BANK_ID), fill_id)) // A walk confined to its own live chain never lands on an entry that is // already invalid. next_table and next_index are cleared only at allocate, @@ -328,15 +477,15 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( reg [`CS_LINE_ADDR_WIDTH-1:0] chain_addr_r; always @(posedge clk) begin if (fill_valid) begin - chain_addr_r <= addr_table[fill_id]; + chain_addr_r <= entry_addr[fill_id]; end end - `RUNTIME_ASSERT(~(dequeue_val && ~fill_valid && (addr_table[dequeue_id] != chain_addr_r)), ("*** %s chain entry on foreign line: dequeue_id=%0d, entry=0x%0h, chain=0x%0h", INSTANCE_ID, - dequeue_id, `CS_BANK_TO_FULL_ADDR(addr_table[dequeue_id], BANK_ID), `CS_BANK_TO_FULL_ADDR(chain_addr_r, BANK_ID))) + `RUNTIME_ASSERT(~(dequeue_val && ~fill_valid && (entry_addr[dequeue_id] != chain_addr_r)), ("*** %s chain entry on foreign line: dequeue_id=%0d, entry=0x%0h, chain=0x%0h", INSTANCE_ID, + dequeue_id, `CS_BANK_TO_FULL_ADDR(entry_addr[dequeue_id], BANK_ID), `CS_BANK_TO_FULL_ADDR(chain_addr_r, BANK_ID))) // The slot the dequeue pointer names must not be handed to another line - // while the pointer still rests on it: `dequeue_addr` is combinational on - // `addr_table[dequeue_id_r]`, so the rewrite retargets an in-flight replay. + // while the pointer still rests on it: a combinational `dequeue_addr` on + // the entry at `dequeue_id_r` would retarget an in-flight replay. `RUNTIME_ASSERT(~(allocate_fire && dequeue_val && (allocate_id == dequeue_id)), ("*** %s allocate over live dequeue pointer: id=%0d, addr=0x%0h", INSTANCE_ID, allocate_id, `CS_BANK_TO_FULL_ADDR(allocate_addr, BANK_ID))) `endif @@ -358,9 +507,6 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( .rdata (dequeue_data) ); - assign fill_addr = addr_table[fill_id]; - assign fill_sector = sector_table[fill_id]; - if (LAZY_FETCH != 0) begin : g_refetch reg [MSHR_SIZE-1:0] refetch_table, retry_table; always @(posedge clk) begin @@ -430,7 +576,7 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( wire [MSHR_SIZE-1:0] probe_ld, probe_amo; for (genvar i = 0; i < MSHR_SIZE; ++i) begin : g_probe_matches - wire addr_match = valid_table[i] && persisted_table[i] && (addr_table[i] == probe_addr); + wire addr_match = valid_table[i] && persisted_table[i] && entry_probe[i]; assign probe_ld[i] = addr_match && ~amo_table[i]; assign probe_amo[i] = addr_match && amo_table[i]; end @@ -466,7 +612,7 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( assign probe_pending_amo = 1'b0; assign {probe_any_ld, probe_any_amo, persist_ld, persist_amo} = '0; `UNUSED_PARAM (AMO_PASSTHRU) - `UNUSED_VAR ({allocate_is_amo, probe_addr}) + `UNUSED_VAR ({allocate_is_amo, probe_addr, entry_probe}) end assign allocate_ready = allocate_rdy; @@ -518,7 +664,7 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( reg [`CS_LINE_ADDR_WIDTH-1:0] dequeue_addr_r; reg dequeue_rw_r; always @(posedge clk) begin - dequeue_addr_r <= addr_table[dequeue_id_n]; + dequeue_addr_r <= dequeue_addr_n; dequeue_rw_r <= write_table[dequeue_id_n]; end @@ -556,7 +702,7 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( `TRACE(3, ("%t: %s table: ", $time, INSTANCE_ID)) for (integer i = 0; i < MSHR_SIZE; ++i) begin if (valid_table[i]) begin - `TRACE(3, (" %0d=0x%0h", i, `CS_BANK_TO_FULL_ADDR(addr_table[i], BANK_ID))) + `TRACE(3, (" %0d=0x%0h", i, `CS_BANK_TO_FULL_ADDR(entry_addr[i], BANK_ID))) if (write_table[i]) begin `TRACE(3, ("(w)")) end else begin diff --git a/hw/rtl/cache/VX_cache_wrap.sv b/hw/rtl/cache/VX_cache_wrap.sv index f0d21ecb98..07b7e2b310 100644 --- a/hw/rtl/cache/VX_cache_wrap.sv +++ b/hw/rtl/cache/VX_cache_wrap.sv @@ -31,6 +31,7 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( parameter CRSQ_SIZE = 0, // Core Response Queue Size parameter MSHR_SIZE = 16, // Miss Reserv Queue Knob + parameter MSHR_LINES = 0, // MSHR distinct line entries (0 = address per request) parameter MRSQ_SIZE = 4, // Memory Response Queue Size parameter MREQ_SIZE = 0, // Memory Request Queue Size (0 = derived minimum) @@ -155,6 +156,7 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( .REPL_POLICY (REPL_POLICY), .CRSQ_SIZE (CRSQ_SIZE), .MSHR_SIZE (MSHR_SIZE), + .MSHR_LINES (MSHR_LINES), .MRSQ_SIZE (MRSQ_SIZE), .MREQ_SIZE (MREQ_SIZE), .LATENCY (LATENCY), diff --git a/hw/unittest/cache/VX_cache_top.sv b/hw/unittest/cache/VX_cache_top.sv index 5d797998c9..68e16685af 100644 --- a/hw/unittest/cache/VX_cache_top.sv +++ b/hw/unittest/cache/VX_cache_top.sv @@ -39,6 +39,7 @@ module VX_cache_top import VX_gpu_pkg::*; #( parameter CRSQ_SIZE = 0, // Miss Reserv Queue Knob parameter MSHR_SIZE = 16, + parameter MSHR_LINES = 0, // Memory Response Queue Size parameter MRSQ_SIZE = 8, parameter MREQ_SIZE = 0, @@ -179,6 +180,7 @@ module VX_cache_top import VX_gpu_pkg::*; #( .MEM_PORTS (MEM_PORTS), .CRSQ_SIZE (CRSQ_SIZE), .MSHR_SIZE (MSHR_SIZE), + .MSHR_LINES (MSHR_LINES), .MRSQ_SIZE (MRSQ_SIZE), .MREQ_SIZE (MREQ_SIZE), .LATENCY (LATENCY), diff --git a/tests/regression/Makefile b/tests/regression/Makefile index 5b2f655a92..a11c131c2f 100644 --- a/tests/regression/Makefile +++ b/tests/regression/Makefile @@ -15,7 +15,7 @@ TESTS := \ dxa_copy dxa_copy_mcast dxa_kmajor_check \ async_barrier async_gbarrier gbar_phase bar_slot_phase packld wgather wsync quad_lod \ occupancy multikernel reopen module_reload single_cta \ - flush_coherence lazy_fetch \ + flush_coherence lazy_fetch sector_merge \ vm_test vm_fault vm_stress # --- common exclude list --------------------------------------------- diff --git a/tests/regression/sector_merge/Makefile b/tests/regression/sector_merge/Makefile new file mode 100644 index 0000000000..62863e5aee --- /dev/null +++ b/tests/regression/sector_merge/Makefile @@ -0,0 +1,16 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := sector_merge + +SRC_DIR := $(VORTEX_HOME)/tests/regression/$(PROJECT) + +SRCS := $(SRC_DIR)/main.cpp + +VX_SRCS := $(SRC_DIR)/kernel.cpp + +OPTS ?= -n256 + +KERNEL_LIB := vortex2 + +include ../common.mk diff --git a/tests/regression/sector_merge/common.h b/tests/regression/sector_merge/common.h new file mode 100644 index 0000000000..7b3a54079a --- /dev/null +++ b/tests/regression/sector_merge/common.h @@ -0,0 +1,13 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +typedef struct { + uint32_t num_lines; + uint32_t line_words; + uint32_t group; + uint32_t reserved; + uint64_t src_addr; + uint64_t dst_addr; +} kernel_arg_t; + +#endif diff --git a/tests/regression/sector_merge/kernel.cpp b/tests/regression/sector_merge/kernel.cpp new file mode 100644 index 0000000000..389b7cbb55 --- /dev/null +++ b/tests/regression/sector_merge/kernel.cpp @@ -0,0 +1,23 @@ +#include +#include "common.h" + +// Consecutive thread groups read opposite halves of the same lines, so the +// misses for both sectors of a sectored line are in flight together. +__kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { + const uint32_t* src = (const uint32_t*)arg->src_addr; + uint32_t* dst = (uint32_t*)arg->dst_addr; + uint32_t num_lines = arg->num_lines; + uint32_t line_words = arg->line_words; + uint32_t half_words = line_words / 2; + + uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; + uint32_t half = (gid / arg->group) & 1; + uint32_t word = half * half_words + (gid % arg->group) % half_words; + + uint32_t sum = 0; + #pragma unroll 4 + for (uint32_t k = 0; k < num_lines; ++k) { + sum += src[k * line_words + word] * (k + 1); + } + dst[gid] = sum; +} diff --git a/tests/regression/sector_merge/main.cpp b/tests/regression/sector_merge/main.cpp new file mode 100644 index 0000000000..54e9134872 --- /dev/null +++ b/tests/regression/sector_merge/main.cpp @@ -0,0 +1,180 @@ +#include +#include +#include +#include +#include "common.h" + +#define RT_CHECK(_expr) \ + do { \ + int _ret = _expr; \ + if (0 == _ret) \ + break; \ + printf("Error: '%s' returned %d!\n", #_expr, (int)_ret); \ + cleanup(); \ + exit(-1); \ + } while (false) + +/////////////////////////////////////////////////////////////////////////////// + +const char* kernel_file = "kernel.vxbin"; +uint32_t num_lines = 256; +uint32_t line_bytes = 128; + +vx_device_h device = nullptr; +vx_buffer_h src_buffer = nullptr; +vx_buffer_h dst_buffer = nullptr; +vx_queue_h queue = nullptr; +vx_module_h module_ = nullptr; +vx_kernel_h kernel = nullptr; +kernel_arg_t kernel_arg = {}; + +static void show_usage() { + std::cout << "Vortex Test." << std::endl; + std::cout << "Usage: [-k: kernel] [-n lines] [-l line bytes] [-h: help]" << std::endl; +} + +static void parse_args(int argc, char **argv) { + int c; + while ((c = getopt(argc, argv, "n:l:k:h")) != -1) { + switch (c) { + case 'n': + num_lines = atoi(optarg); + break; + case 'l': + line_bytes = atoi(optarg); + break; + case 'k': + kernel_file = optarg; + break; + case 'h': + show_usage(); + exit(0); + break; + default: + show_usage(); + exit(-1); + } + } +} + +void cleanup() { + if (device) { + if (src_buffer) vx_buffer_release(src_buffer); + if (dst_buffer) vx_buffer_release(dst_buffer); + if (kernel) vx_kernel_release(kernel); + if (module_) vx_module_release(module_); + if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); + vx_device_release(device); + } +} + +int main(int argc, char *argv[]) { + parse_args(argc, argv); + + // open device connection + std::cout << "open device connection" << std::endl; + RT_CHECK(vx_device_open(0, &device)); + + vx_queue_info_t qi = { sizeof(qi), nullptr, VX_QUEUE_PRIORITY_NORMAL, 0 }; + RT_CHECK(vx_queue_create(device, &qi, &queue)); + + uint64_t num_cores, num_warps, num_threads; + RT_CHECK(vx_device_query(device, VX_CAPS_NUM_CORES, &num_cores)); + RT_CHECK(vx_device_query(device, VX_CAPS_NUM_WARPS, &num_warps)); + RT_CHECK(vx_device_query(device, VX_CAPS_NUM_THREADS, &num_threads)); + + uint32_t line_words = line_bytes / sizeof(uint32_t); + uint32_t num_tasks = num_cores * num_warps * num_threads; + uint32_t num_words = num_lines * line_words; + + kernel_arg.num_lines = num_lines; + kernel_arg.line_words = line_words; + kernel_arg.group = num_threads; + + std::cout << "number of tasks: " << num_tasks << ", lines: " << num_lines + << ", line bytes: " << line_bytes << std::endl; + + // allocate device memory + std::cout << "allocate device memory" << std::endl; + RT_CHECK(vx_buffer_create(device, num_words * sizeof(uint32_t), VX_MEM_READ, &src_buffer)); + RT_CHECK(vx_buffer_address(src_buffer, &kernel_arg.src_addr)); + RT_CHECK(vx_buffer_create(device, num_tasks * sizeof(uint32_t), VX_MEM_WRITE, &dst_buffer)); + RT_CHECK(vx_buffer_address(dst_buffer, &kernel_arg.dst_addr)); + + // upload source buffer + std::cout << "upload source buffer" << std::endl; + std::vector h_src(num_words); + for (uint32_t i = 0; i < num_words; ++i) { + h_src[i] = i * 2654435761u + 12345u; + } + RT_CHECK(vx_enqueue_write(queue, src_buffer, 0, h_src.data(), num_words * sizeof(uint32_t), 0, nullptr, nullptr)); + + // load kernel module + std::cout << "load kernel module" << std::endl; + RT_CHECK(vx_module_load_file(device, kernel_file, &module_)); + RT_CHECK(vx_module_get_kernel(module_, "main", &kernel)); + + // start device + std::cout << "start device" << std::endl; + vx_event_h launch_ev = nullptr, read_ev = nullptr; + { + uint32_t n = num_tasks; + uint32_t grid_dim[1], block_dim[1]; + RT_CHECK(vx_device_max_occupancy_grid(device, 1, &n, grid_dim, block_dim)); + vx_launch_info_t li = {}; + li.struct_size = sizeof(li); + li.kernel = kernel; + li.args_host = &kernel_arg; + li.args_size = sizeof(kernel_arg); + li.ndim = 1; + li.grid_dim[0] = grid_dim[0]; + li.block_dim[0] = block_dim[0]; + RT_CHECK(vx_enqueue_launch(queue, &li, 0, nullptr, &launch_ev)); + } + + // download destination buffer + std::cout << "download destination buffer" << std::endl; + std::vector h_dst(num_tasks); + RT_CHECK(vx_enqueue_read(queue, h_dst.data(), dst_buffer, 0, num_tasks * sizeof(uint32_t), 1, &launch_ev, &read_ev)); + + // wait for completion + std::cout << "wait for completion" << std::endl; + RT_CHECK(vx_event_wait_value(read_ev, 1, VX_TIMEOUT_INFINITE)); + vx_event_release(read_ev); + vx_event_release(launch_ev); + + // verify result + std::cout << "verify result" << std::endl; + uint32_t half_words = line_words / 2; + int errors = 0; + for (uint32_t gid = 0; gid < num_tasks; ++gid) { + uint32_t half = (gid / num_threads) & 1; + uint32_t word = half * half_words + (gid % num_threads) % half_words; + uint32_t ref = 0; + for (uint32_t k = 0; k < num_lines; ++k) { + ref += h_src[k * line_words + word] * (k + 1); + } + if (h_dst[gid] != ref) { + if (errors < 100) { + std::cout << "error at result #" << gid << ": actual 0x" << std::hex << h_dst[gid] + << ", expected 0x" << ref << std::dec << std::endl; + } + ++errors; + } + } + + // cleanup + std::cout << "cleanup" << std::endl; + cleanup(); + + if (errors != 0) { + std::cout << "Found " << errors << " errors!" << std::endl; + std::cout << "FAILED!" << std::endl; + return 1; + } + + std::cout << "PASSED!" << std::endl; + + return 0; +} From 227c49e35e1f85d602e137922ce1830812e04e63 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 19 Oct 2026 05:17:40 -0700 Subject: [PATCH 053/124] tests: make mstress check every loaded value mstress's host reference was `ref *= value` starting from 0, so every expected result was 0 and the float compare allowed a few ULPs: the test never checked loaded data. A store with a flipped bit, or a fill that returned the wrong sector, still passed. The stress pattern is unchanged: an index table drives eight dependent indirect gathers per point, with the same strides and addresses. The data is now uint32, each point folds its eight loads as value * 31 + src[i], and the host computes the same fold and compares exactly, so every loaded value reaches the result and no rounding can hide or fake an error. All mstress CI cases (cache mstress-1..6, config2 mstress-4/13/14) pass on rtlsim and simx. A cache bank that flips bit 0 of store data now fails with 512 wrong results; the old test passed it. Co-Authored-By: Claude Opus 5.5 --- tests/regression/mstress/kernel.cpp | 8 ++-- tests/regression/mstress/main.cpp | 66 ++++------------------------- 2 files changed, 13 insertions(+), 61 deletions(-) diff --git a/tests/regression/mstress/kernel.cpp b/tests/regression/mstress/kernel.cpp index 8a66f2047c..2ae7f15ac3 100644 --- a/tests/regression/mstress/kernel.cpp +++ b/tests/regression/mstress/kernel.cpp @@ -4,18 +4,18 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { size_t stride = arg->stride; uint32_t* addr_ptr = (uint32_t*)arg->src0_addr; - float* src_ptr = (float*)arg->src1_addr; - float* dst_ptr = (float*)arg->dst_addr; + uint32_t* src_ptr = (uint32_t*)arg->src1_addr; + uint32_t* dst_ptr = (uint32_t*)arg->dst_addr; size_t gid = blockIdx.x * blockDim.x + threadIdx.x; size_t offset = gid * stride; for (size_t i = 0; i < stride; ++i) { - float value = 0.0f; + uint32_t value = 0; for (uint32_t j = 0; j < NUM_LOADS; ++j) { size_t addr = offset + i + j; uint32_t index = addr_ptr[addr]; - value *= src_ptr[index]; + value = value * 31 + src_ptr[index]; } dst_ptr[offset+i] = value; } diff --git a/tests/regression/mstress/main.cpp b/tests/regression/mstress/main.cpp index 9754c79856..b5a6d63bd7 100644 --- a/tests/regression/mstress/main.cpp +++ b/tests/regression/mstress/main.cpp @@ -4,8 +4,6 @@ #include #include "common.h" #include -#include -#include #include #define RT_CHECK(_expr) \ @@ -20,51 +18,6 @@ /////////////////////////////////////////////////////////////////////////////// -union Float_t { - float f; - int i; - struct { - uint32_t man : 23; - uint32_t exp : 8; - uint32_t sign : 1; - } parts; -}; - -inline float fround(float x, int32_t precision = 8) { - auto power_of_10 = std::pow(10, precision); - return std::round(x * power_of_10) / power_of_10; -} - -inline bool almost_equal_eps(float a, float b, int ulp = 128) { - auto eps = std::numeric_limits::epsilon() * (std::max(fabs(a), fabs(b)) * ulp); - auto d = fabs(a - b); - if (d > eps) { - std::cout << "*** almost_equal_eps: d=" << d << ", eps=" << eps << std::endl; - return false; - } - return true; -} - -inline bool almost_equal_ulp(float a, float b, int32_t ulp = 6) { - Float_t fa{a}, fb{b}; - auto d = std::abs(fa.i - fb.i); - if (d > ulp) { - std::cout << "*** almost_equal_ulp: a=" << a << ", b=" << b << ", ulp=" << d << ", ia=" << std::hex << fa.i << ", ib=" << fb.i << std::endl; - return false; - } - return true; -} - -inline bool almost_equal(float a, float b) { - if (a == b) - return true; - /*if (almost_equal_eps(a, b)) - return true;*/ - return almost_equal_ulp(a, b); -} - -/////////////////////////////////////////////////////////////////////////////// - const char* kernel_file = "kernel.vxbin"; uint32_t count = 0; @@ -116,7 +69,7 @@ void cleanup() { } } -void gen_src_data(std::vector& test_data, +void gen_src_data(std::vector& test_data, std::vector& addr_table, uint32_t num_points, uint32_t num_addrs) { @@ -124,8 +77,7 @@ void gen_src_data(std::vector& test_data, addr_table.resize(num_addrs); for (uint32_t i = 0; i < num_points; ++i) { - float r = static_cast(std::rand()) / RAND_MAX; - test_data[i] = r; + test_data[i] = static_cast(std::rand()); } for (uint32_t i = 0; i < num_addrs; ++i) { @@ -190,8 +142,8 @@ int main(int argc, char *argv[]) { // allocate host buffers std::cout << "allocate host buffers" << std::endl; std::vector h_addr; - std::vector h_src; - std::vector h_dst(num_points); + std::vector h_src; + std::vector h_dst(num_points); gen_src_data(h_src, h_addr, num_points, num_addrs); // upload source buffer0 @@ -239,17 +191,17 @@ int main(int argc, char *argv[]) { std::cout << "verify result" << std::endl; int errors = 0; for (uint32_t i = 0; i < num_points; ++i) { - float ref = 0.0f; + uint32_t ref = 0; for (uint32_t j = 0; j < NUM_LOADS; ++j) { uint32_t addr = i + j; uint32_t index = h_addr[addr]; - float value = h_src[index]; + uint32_t value = h_src[index]; //printf("*** [%d] addr=%d, index=%d, value=%f\n", i, addr, index, value); - ref *= value; + ref = ref * 31 + value; } - float cur = h_dst[i]; - if (!almost_equal(cur, ref)) { + uint32_t cur = h_dst[i]; + if (cur != ref) { std::cout << "error at result #" << std::dec << i << ": actual " << cur << ", expected " << ref << std::endl; ++errors; From f0b75042815cde63cc0f3430ddbc8c12b188a81b Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 19 Oct 2026 05:22:04 -0700 Subject: [PATCH 054/124] simx: give each warp's instruction buffer its staging slot In the RTL a warp's instructions wait in an IBUF_SIZE elastic buffer, pass the uop sequencer, and enter the scoreboard's per-warp staging register (a one-entry pipe buffer) before they issue. The buffer frees a slot when its head moves on to staging, and the uop sequencer takes a macro-op out of the buffer only with its last micro-op. SimX released the head's slot early as the RTL does, but its buffer FIFO held only IBUF_SIZE entries, the staged head included. Fetch then ran one instruction ahead into a buffer with no room for it, and decode blocked. A macro-op also gave up its slot with its first micro-op. The FIFO now holds IBUF_SIZE entries plus the head standing in for the staging register, and a macro-op releases its slot with its last micro-op. SimX vs rtlsim cycles (sgemm -n128 unless noted), tip -> this change: 16 threads x 16 warps, IW4, FW2 579,368 -> 573,905 raycast, 16 threads x 16 warps 376,910 -> 359,059 8 threads x 8 warps, 2 cores + L2 1,162,128 -> 1,133,705 sgemm default 797,701 -> 793,868 Co-Authored-By: Claude Opus 5.5 --- sim/simx/core.cpp | 13 +++++++++---- sim/simx/sequencer.h | 3 +++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index 4fdf2f54ce..57b7a1c555 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -125,7 +125,9 @@ class Core::Impl { // create ibuffers for (uint32_t i = 0; i < ibuffers_.size(); ++i) { snprintf(sname, 100, "%s-ibuffer%d", name.c_str(), i); - ibuffers_.at(i) = TFifo::Create(sname, 1, VX_CFG_IBUF_SIZE); + // IBUF_SIZE entries plus the head, which stands for the warp's staging + // register once it has moved there + ibuffers_.at(i) = TFifo::Create(sname, 1, VX_CFG_IBUF_SIZE + 1); } // create the memory coalescer, or the lane adapter that replaces it. @@ -651,16 +653,19 @@ class Core::Impl { // check scoreboard against sequencer micro-op (handles uop WAW/RAW) auto trace = ibuffer->peek(); + auto seq = sequencers_.at(wid); + auto uop_trace = seq->get(trace); // returns cached uop or generates next + // The head moves into the warp's staging register, freeing its buffer // slot, once it has arrived and the warp's previous instruction has - // issued; the scheduler sees the freed slot the cycle after. + // issued; a macro-op leaves the buffer with its last micro-op. The + // scheduler sees the freed slot the cycle after. if (!head_staged_.at(wid) + && (!trace->instr_ptr->is_macro_op() || seq->last()) && SimPlatform::instance().cycles() > std::max(last_issue_.at(wid), trace->ibuf_time)) { head_staged_.at(wid) = true; this->ibuf_release(wid); } - auto seq = sequencers_.at(wid); - auto uop_trace = seq->get(trace); // returns cached uop or generates next // The uop sequencer starts a macro-op once it reaches the front of the // buffer behind the staged instruction, and offers its first uop the diff --git a/sim/simx/sequencer.h b/sim/simx/sequencer.h index 9f7222f270..31f525ec21 100644 --- a/sim/simx/sequencer.h +++ b/sim/simx/sequencer.h @@ -51,6 +51,9 @@ class Sequencer : public SimObject { // A macro-op's first micro-op is the current one. bool starting() const { return state_.active && state_.uop_index == 0; } + // A macro-op's last micro-op is the current one. + bool last() const { return state_.active && (state_.uop_index + 1) == state_.uop_count; } + // Drop any cached uop / macro-op state for this sequencer. Used by // Core::flush_warp_pipeline at async-trap entry: when the ibuffer is // flushed, the trace pointer this sequencer cached in state_.current_uop From 62dc764f4cfa7e78491bf1c8f54212da0afbe403 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 19 Oct 2026 05:22:05 -0700 Subject: [PATCH 055/124] simx: send a fence to the data cache as a flush token In the RTL the LSU sends a fence to the data cache as a flush token on the lanes of its last packet, behind every request the block issued before it, and holds the block until the token is answered. A cache that sees a token holds its cacheable inputs, drains, walks its banks (writing back dirty lines and invalidating), sends one token down each memory port behind that port's write-backs, and answers the tokens visible at the end of the walk once every port has answered, one per cycle. SimX treated a fence as an ordering barrier in the LSU only. SimX now models the token end to end: - The LSU sends the token and releases the fence on its response. - The cache's port arbiter holds cacheable requests while a token is visible. The banks walk one entry per cycle. The token for port i takes bank i's lane into the bank-to-port arbiter, and a tap on each port's response path consumes the token's answer in the same cycle, so other responses are not delayed. - Bank i drives memory port i mod mem_ports, as in the RTL. SimX grouped contiguous banks per port, which sent tokens down the wrong port and deadlocked a fence on any cache with more banks than ports (at least 2); it also changes timing on such caches (IW=2 dcache, the 2-frontend shape, multi-bank L2/L3 with several ports) to match the RTL. - Flush tokens skip address translation. Fence-free runs at the default shape are cycle-identical. CI adds printf parity cases at the default shape, at IW=2 and on the 2-frontend shape. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/core.yaml | 18 ++ docs/designs/multicache_amo_coherence.md | 3 +- sim/simx/lsu_unit.cpp | 52 +++- sim/simx/lsu_unit.h | 6 + sim/simx/mem/cache.cpp | 288 ++++++++++++++++++++++- sim/simx/mem/mmu.cpp | 5 +- sim/simx/types.h | 5 +- 7 files changed, 365 insertions(+), 12 deletions(-) diff --git a/ci/testcases/core.yaml b/ci/testcases/core.yaml index 6f9187911d..d7ab85e2bd 100644 --- a/ci/testcases/core.yaml +++ b/ci/testcases/core.yaml @@ -184,6 +184,24 @@ tests: app: sgemm args: -n128 configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_FETCH_WIDTH=2 +# Device printf: per-character IO ring stores, each published behind a fence +# that drains, flushes and invalidates the data cache. +- id: parity-printf + check: model_parity + via: blackbox + app: printf +# The same fences on data caches with more banks than memory ports: every +# port's flush token must be answered on that port. +- id: parity-printf-iw2 + check: model_parity + via: blackbox + app: printf + configs: -DVX_CFG_ISSUE_WIDTH=2 +- id: parity-printf-fw2 + check: model_parity + via: blackbox + app: printf + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_FETCH_WIDTH=2 # SGEMM with local-memory tiling (kernel-managed shared memory). - id: parity-sgemmx check: model_parity diff --git a/docs/designs/multicache_amo_coherence.md b/docs/designs/multicache_amo_coherence.md index b0233a0252..d553e9ce57 100644 --- a/docs/designs/multicache_amo_coherence.md +++ b/docs/designs/multicache_amo_coherence.md @@ -260,8 +260,7 @@ A fence ordering only device accesses (`fence ow, ow`, no R or W) sends an until its uncached path is empty, then forwards, so the answer proves the earlier IO writes reached memory. -SimX does not model the tokens yet: there the fence is an ordering barrier in -the LSU only. +SimX sends every fence as a full flush token, walking every level. Programs that share data across cores through plain memory are outside what is validated. Share through atomics, or through the single shared level. diff --git a/sim/simx/lsu_unit.cpp b/sim/simx/lsu_unit.cpp index c81c3458aa..40c2b6db80 100644 --- a/sim/simx/lsu_unit.cpp +++ b/sim/simx/lsu_unit.cpp @@ -236,6 +236,16 @@ void LsuUnit::process_response_step(uint32_t b) { } #endif + if (entry.is_flush) { + DT(3, this->name() << " fence-token-rsp: " << lsu_rsp); + entry.count -= lsu_rsp.mask.count(); + if (entry.count == 0) { + state.pending_reqs.release(lsu_rsp.tag); + } + lsu_rsp_in.pop(); + return; + } + auto trace = entry.trace; auto& output = Outputs.at(b); // Every response fragment retires through its own writeback slot, one @@ -351,6 +361,42 @@ void LsuUnit::ingest_inputs(uint32_t b) { input.pop(); } +// The fence travels to the data cache as a flush token on the lanes of its +// last packet, behind every request the block issued before it. The cache +// answers once it has drained, flushed and invalidated, so the fence holds +// the block until the token's response returns. +void LsuUnit::send_fence_token(uint32_t b) { + auto& state = states_.at(b); + if (state.pending_reqs.full() || core_->lmem_switch(b)->ReqIn.full()) + return; + auto trace = state.fence.trace(); + uint32_t base = ((VX_CFG_NUM_THREADS - 1) / VX_CFG_NUM_LSU_LANES) * VX_CFG_NUM_LSU_LANES; + LsuReq lsu_req(VX_CFG_NUM_LSU_LANES); + lsu_req.op = MemOp::LD; + lsu_req.flags.flush = 1; + std::vector lane_entries(VX_CFG_NUM_LSU_LANES); + for (uint32_t t = base; t < VX_CFG_NUM_THREADS; ++t) { + if (trace->tmask.test(t)) { + lsu_req.mask.set(t - base); + lsu_req.tids.at(t - base) = t; + } + } + if (lsu_req.mask.none()) { + lsu_req.mask.set(0); + lsu_req.tids.at(0) = base; + } + uint32_t count = lsu_req.mask.count(); + pending_req_t entry{nullptr, count, true, std::move(lane_entries), IntrLsuArgs{}, false}; + entry.is_flush = true; + lsu_req.tag = state.pending_reqs.allocate(std::move(entry)); + lsu_req.cid = trace->cid; + lsu_req.wid = trace->wid; + lsu_req.uuid = trace->uuid; + core_->lmem_switch(b)->ReqIn.send(lsu_req); + DT(3, this->name() << " fence-token: " << lsu_req); + state.fence_token = false; +} + // Per-cycle request handler. Dispatches at most one batch (one // memory-side mem_req) per call. The head trace stays at the queue // front until all its batches are issued (req_sent_all-driven). @@ -360,7 +406,10 @@ void LsuUnit::process_request_step(uint32_t b) { // Attempt fence release first so a steady stream of TCU client forwards // (which consume the dispatch slot and return) cannot starve it. const bool fence_locked = state.fence.locked(); - if (fence_locked) { + if (fence_locked && state.fence_token) { + this->send_fence_token(b); + } + if (fence_locked && !state.fence_token) { bool released = state.fence.try_release(Outputs.at(b), state.pending_reqs.empty()); if (released) { DT(3, this->name() << " fence-unlock: " << state.fence.trace()); @@ -438,6 +487,7 @@ void LsuUnit::process_request_step(uint32_t b) { // Engage fence lock; the next call will try to release it once // the pending-reqs table drains. state.fence.engage(trace); + state.fence_token = true; DT(3, this->name() << " fence-lock: " << *trace); state.req_queue.pop(); return; diff --git a/sim/simx/lsu_unit.h b/sim/simx/lsu_unit.h index 02a693998a..8ab00012dc 100644 --- a/sim/simx/lsu_unit.h +++ b/sim/simx/lsu_unit.h @@ -100,6 +100,7 @@ class LsuUnit : public FuncUnit { // Per-cycle handlers. void process_response_step(uint32_t b); void process_request_step(uint32_t b); + void send_fence_token(uint32_t b); // Drain Inputs[b] into req_queue. void ingest_inputs(uint32_t b); @@ -123,6 +124,8 @@ class LsuUnit : public FuncUnit { // TcuRspOut with client_tag restored instead of written to registers. bool is_tcu = false; uint32_t client_tag = 0; + // A fence's flush token: its response only releases the fence. + bool is_flush = false; }; // Per-block LSU state. Each member is a named hardware sub-block. @@ -140,6 +143,8 @@ class LsuUnit : public FuncUnit { uint32_t staged_reads = 0; bool head_staged = false; FenceController fence; + // the engaged fence's flush token is still to be sent + bool fence_token = false; std::vector addr_list; uint32_t remain_addrs = 0; @@ -149,6 +154,7 @@ class LsuUnit : public FuncUnit { this->staged_reads = 0; this->head_staged = false; this->fence.reset(); + this->fence_token = false; this->addr_list.clear(); this->remain_addrs = 0; } diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index efd5979a7e..a61cbc8ebd 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -779,6 +779,8 @@ class CacheBank : public SimObject { flush_sector_idx_ = 0; return; } + // the two walks share the sector cursor + __assert(!token_flushing_, "flush walk started during a flush-token walk"); flushing_ = true; flush_set_idx_ = 0; flush_way_idx_ = 0; @@ -790,6 +792,20 @@ class CacheBank : public SimObject { return !flushing_; } + // A flush token's walk: one entry per cycle, every set (every way of a + // write-back cache), writing back dirty data and invalidating the lines. + void token_flush_begin() { + __assert(!flushing_, "flush-token walk started during a flush walk"); + token_flushing_ = true; + token_flush_idx_ = 0; + flush_sector_idx_ = 0; + this->tick_wake(); + } + + bool token_flush_done() const { + return !token_flushing_; + } + protected: void on_reset() { perf_stats_ = Cache::PerfStats(); @@ -824,6 +840,8 @@ class CacheBank : public SimObject { flush_set_idx_ = 0; flush_way_idx_ = 0; flush_sector_idx_ = 0; + token_flushing_ = false; + token_flush_idx_ = 0; #if VX_CFG_EXT_A_ENABLED amo_unit_.reset(); for (auto &e : amo_passthru_) { @@ -859,6 +877,9 @@ class CacheBank : public SimObject { if (flushing_) { this->processFlush(); } + if (token_flushing_) { + this->processTokenFlush(); + } // calculate memory latency perf_stats_.mem_latency += pending_fill_reqs_; @@ -869,7 +890,7 @@ class CacheBank : public SimObject { // any packet reserved toward core_req_in/mem_rsp_in re-arms on arrival. if (this->core_req_in.size() == 0 && this->mem_rsp_in.size() == 0 && pipe_req_->size() == 0 && lazy_q_->size() == 0 && !mshr_.has_ready_reqs() - && !fwd_active_ && !flushing_ && pending_fill_reqs_ == 0) { + && !fwd_active_ && !flushing_ && !token_flushing_ && pending_fill_reqs_ == 0) { this->tick_sleep(); } } @@ -1973,6 +1994,47 @@ class CacheBank : public SimObject { DT(3, this->name() << " flush-done"); } + void processTokenFlush() { + if (pending_fill_reqs_ != 0 + || pipe_req_->size() != 0 + || !mshr_.empty()) { + return; + } + uint32_t ways = config_.write_back ? params_.lines_per_set : 1; + uint32_t set_id = token_flush_idx_ % params_.sets_per_bank; + uint32_t way_id = token_flush_idx_ / params_.sets_per_bank; + auto &set = sets_.at(set_id); + if (config_.write_back) { + auto &line = set.lines.at(way_id); + while (flush_sector_idx_ < line.sectors.size()) { + auto &sec = line.sectors.at(flush_sector_idx_); + if (sec.valid && sec.dirty) { + if (this->mem_req_out.full()) + return; + MemReq mem_req; + mem_req.addr = params_.mem_addr_sector(bank_id_, set_id, line.tag, flush_sector_idx_); + mem_req.op = MemOp::ST; + mem_req.data = sec.data; + mem_req.byteen = sec.dirty_mask; + this->mem_req_out.send(mem_req, MEM_REQ_DELAY); + DT(3, this->name() << " flush-wb: " << mem_req); + ++perf_stats_.evictions; + } + ++flush_sector_idx_; + } + flush_sector_idx_ = 0; + line.reset(); + } else { + for (auto &line : set.lines) { + line.reset(); + } + } + if (++token_flush_idx_ == params_.sets_per_bank * ways) { + token_flushing_ = false; + DT(3, this->name() << " token-flush-done"); + } + } + Cache::Config config_; params_t params_; uint32_t bank_id_; @@ -2037,6 +2099,8 @@ class CacheBank : public SimObject { uint32_t flush_set_idx_; uint32_t flush_way_idx_; uint32_t flush_sector_idx_; + bool token_flushing_; + uint32_t token_flush_idx_; #if VX_CFG_EXT_A_ENABLED AmoUnit amo_unit_; @@ -2091,6 +2155,39 @@ class CachePortArb : public SimObject { uint64_t collisions() const { return collisions_; } + // Ports whose head is a flush token. While any is visible, cacheable + // requests are held at every port. + uint64_t tokens() const { + uint64_t mask = 0; + for (uint32_t i = 0, n = ReqIn.size(); i < n; ++i) { + if (!ReqIn.at(i).empty() && ReqIn.at(i).peek().flags.flush) { + mask |= 1ull << i; + } + } + return mask; + } + + // No cacheable request is left between the ports and the banks. + bool drained() const { + for (auto &stage : tree_stage_) { + if (stage.size() != 0) { + return false; + } + } + for (auto &out : ReqOut) { + if (out.size() != 0) { + return false; + } + } + return true; + } + + MemReq take_token(uint32_t i) { + auto req = ReqIn.at(i).peek(); + ReqIn.at(i).pop(); + return req; + } + // The cache's line initialization holds the core ports for these cycles. void set_init_cycles(uint32_t cycles) { init_cycles_ = cycles; } @@ -2114,6 +2211,7 @@ class CachePortArb : public SimObject { } } uint32_t n = ReqIn.size(); + const bool hold = (this->tokens() != 0); // requesting ports per bank; non-cacheable requests leave on their own port std::vector want(ReqOut.size(), 0); for (uint32_t i = 0; i < n; ++i) { @@ -2122,13 +2220,18 @@ class CachePortArb : public SimObject { continue; } auto& req = in.peek(); + if (req.flags.flush) { + continue; + } if (req.flags.io) { if (IoOut.at(i).try_send(req, 0)) { in.pop(); } continue; } - want.at(params_.addr_bank_id(req.addr)) |= 1ull << i; + if (!hold) { + want.at(params_.addr_bank_id(req.addr)) |= 1ull << i; + } } for (uint32_t b = 0, nb = ReqOut.size(); b < nb; ++b) { auto& out = tree_stage_.empty() ? ReqOut.at(b) : tree_stage_.at(b); @@ -2160,6 +2263,64 @@ class CachePortArb : public SimObject { friend class SimObject; }; +// Sits on one memory port's cache-side response path, ahead of the response +// network. While the port's flush token is out the cache is quiescent, so the +// port's next response is the token's answer and stops here; every other +// response passes in the same cycle. +class FlushTap : public SimObject { +public: + using Ptr = std::shared_ptr; + + SimChannel RspIn; + SimChannel RspOut; + + FlushTap(const SimContext& ctx, const char* name) + : SimObject(ctx, name) + , RspIn(this) + , RspOut(this) + {} + + // the port's flush token has been sent + void arm() { + armed_ = true; + answered_ = false; + } + + // the token's answer has arrived; reading it disarms the tap + bool take_answer() { + bool answered = answered_; + answered_ = false; + return answered; + } + +protected: + void on_reset() { + armed_ = false; + answered_ = false; + } + + void on_tick() { + if (RspIn.empty()) { + this->tick_sleep(); + return; + } + if (armed_) { + DT(3, this->name() << " flush-fwd-rsp"); + armed_ = false; + answered_ = true; + RspIn.pop(); + } else if (RspOut.try_send(RspIn.peek(), 0)) { + RspIn.pop(); + } + } + +private: + bool armed_ = false; + bool answered_ = false; + + friend class SimObject; +}; + class Cache::Impl { public: Impl(Cache *simobject, const Config &config) @@ -2188,6 +2349,15 @@ class Cache::Impl { nc_mem_arbs_.at(i) = MemArbiter::Create(sname, ArbiterType::Priority, 2, 1); } + // flush-token taps, ticking after the arbiters that feed them and before + // the response network they feed + flush_taps_.resize(config_.mem_ports); + for (uint32_t i = 0; i < config_.mem_ports; ++i) { + snprintf(sname, 100, "%s-flush_tap%d", simobject->name().c_str(), i); + flush_taps_.at(i) = FlushTap::Create(sname); + nc_mem_arbs_.at(i)->RspOut.at(0).bind(&flush_taps_.at(i)->RspIn); + } + // Connect non-cacheable arbiter output port 0 to outgoing memory ports for (uint32_t i = 0; i < config_.mem_ports; ++i) { nc_mem_arbs_.at(i)->ReqOut.at(0).bind(&simobject->mem_req_out.at(i)); @@ -2202,6 +2372,7 @@ class Cache::Impl { uint32_t mem_arb_delay = ((num_banks != config_.mem_ports) && (num_banks > 2 || config_.mem_ports > 2)) ? 1 : 0; uint32_t mem_net_delay = omega_stages(num_banks, config_.mem_ports) - 1; uint32_t mem_net_extra = (mem_net_delay > mem_arb_delay) ? (mem_net_delay - mem_arb_delay) : 0; + __assert(num_banks >= config_.mem_ports, "a cache needs a bank per memory port"); auto bank_mem_arb = MemArbiter::Create(sname, ArbiterType::RoundRobin, num_banks, config_.mem_ports, mem_arb_delay + mem_net_extra, mem_arb_delay + mem_net_extra); @@ -2218,8 +2389,11 @@ class Cache::Impl { } else { bank_mem_arb->ReqOut.at(i).bind(&nc_mem_arbs_.at(i)->ReqIn.at(0)); } - nc_mem_arbs_.at(i)->RspOut.at(0).bind(&bank_mem_arb->RspIn.at(i)); } + for (uint32_t i = 0; i < config_.mem_ports; ++i) { + flush_taps_.at(i)->RspOut.bind(&bank_mem_arb->RspIn.at(i)); + } + bank_arb_ = bank_mem_arb; // Create the core-port arbiter ahead of the banks it feeds snprintf(sname, 100, "%s-port_arb", simobject->name().c_str()); @@ -2255,8 +2429,8 @@ class Cache::Impl { banks_.at(i)->core_rsp_out.bind(&bank_core_xbar_->RspIn.at(i)); // bind memory ports - banks_.at(i)->mem_req_out.bind(&bank_mem_arb->ReqIn.at(i)); - bank_mem_arb->RspOut.at(i).bind(&banks_.at(i)->mem_rsp_in); + banks_.at(i)->mem_req_out.bind(&bank_mem_arb->ReqIn.at(this->bank_lane(i))); + bank_mem_arb->RspOut.at(this->bank_lane(i)).bind(&banks_.at(i)->mem_rsp_in); } } @@ -2292,6 +2466,8 @@ class Cache::Impl { } } + this->flush_step(); + // bank memory requests through the memory-side network's deeper levels for (uint32_t i = 0, n = mem_req_net_.size(); i < n; ++i) { auto &net = mem_req_net_.at(i); @@ -2321,6 +2497,94 @@ class Cache::Impl { } } + // Flush tokens (fences) at any port start one walk: the ports hold their + // cacheable requests, in-flight ones drain, every bank writes back and + // invalidates its lines, a token goes down each memory port behind that + // port's write-backs, and once every port has answered, each token + // visible at the end of the walk is answered through the response path, + // one per cycle. + void flush_step() { + switch (flush_state_) { + case FlushState::Idle: + if (port_arb_->tokens() != 0) { + flush_state_ = FlushState::Drain; + } + break; + case FlushState::Drain: { + if (!port_arb_->drained()) + break; + for (auto &bank : banks_) { + if (bank->core_req_in.size() != 0) + return; + } + for (auto &bank : banks_) { + bank->token_flush_begin(); + } + flush_state_ = FlushState::Walk; + break; + } + case FlushState::Walk: + for (auto &bank : banks_) { + if (!bank->token_flush_done()) + return; + } + rsp_pending_ = port_arb_->tokens(); + fwd_sent_ = 0; + fwd_done_ = 0; + flush_state_ = FlushState::Forward; + break; + case FlushState::Forward: { + bool mem_req_empty = true; + for (auto &lane : bank_arb_->ReqIn) { + mem_req_empty &= (lane.size() == 0); + } + for (auto &net : mem_req_net_) { + mem_req_empty &= (net.size() == 0); + } + for (uint32_t i = 0, n = config_.mem_ports; i < n && mem_req_empty; ++i) { + if (fwd_sent_ & (1ull << i)) + continue; + // the token for port i is a read of the port's first line, sent down + // bank i's lane (bank i maps to port i) behind its write-backs + MemReq token; + token.addr = uint64_t(i) << config_.L; + token.flags.flush = 1; + token.tag = 0; + if (bank_arb_->ReqIn.at(this->bank_lane(i)).try_send(token, 1)) { + DT(3, simobject_->name() << " flush-fwd: " << token); + flush_taps_.at(i)->arm(); + fwd_sent_ |= 1ull << i; + } + } + for (uint32_t i = 0, n = config_.mem_ports; i < n; ++i) { + if (flush_taps_.at(i)->take_answer()) { + fwd_done_ |= 1ull << i; + } + } + uint64_t all = (config_.mem_ports < 64) ? ((1ull << config_.mem_ports) - 1) : ~0ull; + if (fwd_done_ == all) { + flush_state_ = FlushState::Respond; + } + break; + } + case FlushState::Respond: { + if (rsp_pending_ == 0) { + flush_state_ = FlushState::Idle; + break; + } + uint32_t i = __builtin_ctzll(rsp_pending_); + auto &token = port_arb_->ReqIn.at(i).peek(); + MemRsp rsp{token.tag, token.hart_id, token.uuid}; + if (simobject_->core_rsp_out.at(i).try_send(rsp, core_rsp_delay_)) { + DT(3, simobject_->name() << " flush-rsp: " << rsp); + port_arb_->take_token(i); + rsp_pending_ &= ~(1ull << i); + } + break; + } + } + } + PerfStats perf_stats() const { PerfStats perf_stats; if (!config_.bypass) { @@ -2388,7 +2652,21 @@ class Cache::Impl { // the core-response network's levels, then the non-cacheable bypass // layer's response arbiter uint32_t core_rsp_delay_ = 0; + enum class FlushState { Idle, Drain, Walk, Forward, Respond }; + FlushState flush_state_ = FlushState::Idle; + uint64_t rsp_pending_ = 0; + uint64_t fwd_sent_ = 0; + uint64_t fwd_done_ = 0; std::vector> mem_req_net_; + std::vector flush_taps_; + + // Bank i drives memory port i mod mem_ports: its lane into the bank-to-port + // arbiter, which serves each port from a contiguous range of lanes. + uint32_t bank_lane(uint32_t bank_id) const { + uint32_t num_banks = 1 << config_.B; + uint32_t banks_per_port = num_banks / config_.mem_ports; + return (bank_id % config_.mem_ports) * banks_per_port + bank_id / config_.mem_ports; + } uint32_t init_cycles_; bool keep_lines_ = false; }; diff --git a/sim/simx/mem/mmu.cpp b/sim/simx/mem/mmu.cpp index f2c440a3fe..e2d578ba9e 100644 --- a/sim/simx/mem/mmu.cpp +++ b/sim/simx/mem/mmu.cpp @@ -91,11 +91,12 @@ bool Mmu::needs_translation(const MemReq& req) const { // needed. BARE mode (SATP unprogrammed) skips translation, covering // the few instruction fetches between reset and the kernel's csrw // satp. IO-flagged requests skip it too: the IO/OM apertures carry - // device registers or encoded coordinates, not virtual addresses. + // device registers or encoded coordinates, not virtual addresses. So do + // flush tokens, which carry no address. if (!satp_ || satp_->get_mode() == BARE) { return false; } - if (req.flags.io) { + if (req.flags.io || req.flags.flush) { return false; } return true; diff --git a/sim/simx/types.h b/sim/simx/types.h index 85b248ade7..d85c1186db 100644 --- a/sim/simx/types.h +++ b/sim/simx/types.h @@ -444,6 +444,7 @@ struct MemFlags { uint32_t io : 1; // bit 1: uncacheable I/O uint32_t local : 1; // bit 2: LMEM port uint32_t amo_unsigned : 1; // bit 3: MIN/MAX signedness (1=unsigned) + uint32_t flush : 1; // bit 4: flush token (a fence) #ifdef VX_CFG_EXT_DXA_ENABLE // DXA completion sideband — populated only on DXA's LMEM-DMA writes. // notify_bar_id carries the RAW (kernel-encoded) bar handle: @@ -453,8 +454,8 @@ struct MemFlags { // be wide enough to hold (bar_no << 8) | (MAX_CTAS_per_core - 1). // For NUM_BARRIERS = 8 and NUM_WARPS ≤ 16, that's at most 11 bits; // 16 bits leaves headroom for larger configs. - uint32_t dxa_notify_done : 1; // bit 4 - uint32_t dxa_notify_bar_id : 16; // bits 5..20 + uint32_t dxa_notify_done : 1; // bit 5 + uint32_t dxa_notify_bar_id : 16; // bits 6..21 #endif }; }; From 18b6fe5219cbd69796788d4cd5c1df57df496eb8 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 19 Oct 2026 05:22:05 -0700 Subject: [PATCH 056/124] simx: stop fences at the point of coherence and skip the walk for IO fences SimX side of the targeted fences in the RTL (VX_decode, VX_lsu_block, VX_cache_init, VX_cache_flush, VX_cache_bypass): - A memory fence (any R or W in its predecessor or successor set) sends an L1-only token. Caches above the shared LLC walk and forward it with the same kind; the shared LLC answers it without a walk; a replicated LLC still walks. - A fence ordering only device accesses decodes as an IO fence and sends an IO token. No level walks: each quiesces its banks, waits until nothing is held between its inputs and its memory ports on the cacheable and the non-cacheable path, and forwards an IO token on every port. - Tokens of different kinds visible together combine to the weakest kind that serves them all; a quiesce without a walk answers only the tokens that owe none. - The LSU adapters keep the token's io bit instead of deriving it from the address. fence_mp now also runs on simx. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/regression.yaml | 4 + docs/designs/multicache_amo_coherence.md | 2 - sim/simx/decode.cpp | 5 +- sim/simx/lsu_unit.cpp | 7 +- sim/simx/mem/cache.cpp | 118 ++++++++++++++++++++--- sim/simx/mem/lsu_mem_adapter.cpp | 4 +- sim/simx/mem/mem_coalescer.cpp | 2 +- sim/simx/types.h | 9 +- 8 files changed, 130 insertions(+), 21 deletions(-) diff --git a/ci/testcases/regression.yaml b/ci/testcases/regression.yaml index b59f66d3a7..ccaf86aa66 100644 --- a/ci/testcases/regression.yaml +++ b/ci/testcases/regression.yaml @@ -245,11 +245,13 @@ tests: - id: fence_mp-1 via: blackbox drivers: + - simx - rtlsim app: fence_mp - id: fence_mp-2 via: blackbox drivers: + - simx - rtlsim app: fence_mp shape: @@ -258,6 +260,7 @@ tests: - id: fence_mp-3 via: blackbox drivers: + - simx - rtlsim app: fence_mp shape: @@ -267,6 +270,7 @@ tests: - id: fence_mp-4 via: blackbox drivers: + - simx - rtlsim app: fence_mp shape: diff --git a/docs/designs/multicache_amo_coherence.md b/docs/designs/multicache_amo_coherence.md index d553e9ce57..9525593bc1 100644 --- a/docs/designs/multicache_amo_coherence.md +++ b/docs/designs/multicache_amo_coherence.md @@ -260,8 +260,6 @@ A fence ordering only device accesses (`fence ow, ow`, no R or W) sends an until its uncached path is empty, then forwards, so the answer proves the earlier IO writes reached memory. -SimX sends every fence as a full flush token, walking every level. - Programs that share data across cores through plain memory are outside what is validated. Share through atomics, or through the single shared level. diff --git a/sim/simx/decode.cpp b/sim/simx/decode.cpp index 9fb835db56..e0a3d8593e 100644 --- a/sim/simx/decode.cpp +++ b/sim/simx/decode.cpp @@ -631,7 +631,10 @@ Instr::Ptr Decoder::decode(uint32_t code, uint64_t uuid) { case Opcode::FENCE: { instr->set_fu_type(FUType::LSU); instr->set_op_type(LsuType::FENCE); - instr->set_args(IntrLsuArgs{0, 0, 0}); + // neither the predecessor nor the successor set holds a memory read or + // write (pred = code[27:24], succ = code[23:20], IORW) + uint32_t fence_io = (funct3 == 0) && ((code >> 24) & 0x3) == 0 && ((code >> 20) & 0x3) == 0; + instr->set_args(IntrLsuArgs{fence_io, 0, 0}); } break; case Opcode::AMO: { instr->set_fu_type(FUType::LSU); diff --git a/sim/simx/lsu_unit.cpp b/sim/simx/lsu_unit.cpp index 40c2b6db80..e17a131bee 100644 --- a/sim/simx/lsu_unit.cpp +++ b/sim/simx/lsu_unit.cpp @@ -367,13 +367,18 @@ void LsuUnit::ingest_inputs(uint32_t b) { // the block until the token's response returns. void LsuUnit::send_fence_token(uint32_t b) { auto& state = states_.at(b); - if (state.pending_reqs.full() || core_->lmem_switch(b)->ReqIn.full()) + if (state.pending_reqs.full() || core_->lmem_switch(b)->ReqIn.full()) { return; + } auto trace = state.fence.trace(); uint32_t base = ((VX_CFG_NUM_THREADS - 1) / VX_CFG_NUM_LSU_LANES) * VX_CFG_NUM_LSU_LANES; LsuReq lsu_req(VX_CFG_NUM_LSU_LANES); lsu_req.op = MemOp::LD; lsu_req.flags.flush = 1; + // an IO fence's token walks nowhere; a memory fence's stops at the shared LLC + auto fence_args = std::get(trace->instr_ptr->get_args()); + lsu_req.flags.io = lsu_fence_is_io(fence_args); + lsu_req.flags.flush_l1 = !lsu_fence_is_io(fence_args); std::vector lane_entries(VX_CFG_NUM_LSU_LANES); for (uint32_t t = base; t < VX_CFG_NUM_THREADS; ++t) { if (trace->tmask.test(t)) { diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index a61cbc8ebd..b74b4bce1a 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -794,9 +794,10 @@ class CacheBank : public SimObject { // A flush token's walk: one entry per cycle, every set (every way of a // write-back cache), writing back dirty data and invalidating the lines. - void token_flush_begin() { + void token_flush_begin(bool skip) { __assert(!flushing_, "flush-token walk started during a flush walk"); token_flushing_ = true; + token_flush_skip_ = skip; token_flush_idx_ = 0; flush_sector_idx_ = 0; this->tick_wake(); @@ -2000,6 +2001,11 @@ class CacheBank : public SimObject { || !mshr_.empty()) { return; } + if (token_flush_skip_) { + token_flushing_ = false; + DT(3, this->name() << " token-quiesce-done"); + return; + } uint32_t ways = config_.write_back ? params_.lines_per_set : 1; uint32_t set_id = token_flush_idx_ % params_.sets_per_bank; uint32_t way_id = token_flush_idx_ / params_.sets_per_bank; @@ -2009,8 +2015,9 @@ class CacheBank : public SimObject { while (flush_sector_idx_ < line.sectors.size()) { auto &sec = line.sectors.at(flush_sector_idx_); if (sec.valid && sec.dirty) { - if (this->mem_req_out.full()) + if (this->mem_req_out.full()) { return; + } MemReq mem_req; mem_req.addr = params_.mem_addr_sector(bank_id_, set_id, line.tag, flush_sector_idx_); mem_req.op = MemOp::ST; @@ -2100,6 +2107,7 @@ class CacheBank : public SimObject { uint32_t flush_way_idx_; uint32_t flush_sector_idx_; bool token_flushing_; + bool token_flush_skip_ = false; uint32_t token_flush_idx_; #if VX_CFG_EXT_A_ENABLED @@ -2167,6 +2175,22 @@ class CachePortArb : public SimObject { return mask; } + // Tokens owing no walk: IO tokens, and L1-only ones unless they still need + // a walk here. + uint64_t tokens(bool l1_needs_walk) const { + uint64_t mask = 0; + for (uint32_t i = 0, n = ReqIn.size(); i < n; ++i) { + if (ReqIn.at(i).empty()) { + continue; + } + auto &flags = ReqIn.at(i).peek().flags; + if (flags.flush && (flags.io || (flags.flush_l1 && !l1_needs_walk))) { + mask |= 1ull << i; + } + } + return mask; + } + // No cacheable request is left between the ports and the banks. bool drained() const { for (auto &stage : tree_stage_) { @@ -2511,28 +2535,43 @@ class Cache::Impl { } break; case FlushState::Drain: { - if (!port_arb_->drained()) + if (!port_arb_->drained()) { break; + } for (auto &bank : banks_) { - if (bank->core_req_in.size() != 0) + if (bank->core_req_in.size() != 0) { return; + } + } + auto kinds = this->token_kinds(); + bool walk = kinds.walk(config_.shared_llc); + if (!walk && !kinds.any_io) { + rsp_pending_ = port_arb_->tokens(); + flush_state_ = FlushState::Respond; + break; } + // with no walk owed, an IO token still quiesces the banks and goes down + // to memory + walk_skip_ = !walk; for (auto &bank : banks_) { - bank->token_flush_begin(); + bank->token_flush_begin(walk_skip_); } flush_state_ = FlushState::Walk; break; } - case FlushState::Walk: + case FlushState::Walk: { for (auto &bank : banks_) { - if (!bank->token_flush_done()) + if (!bank->token_flush_done()) { return; + } } - rsp_pending_ = port_arb_->tokens(); - fwd_sent_ = 0; - fwd_done_ = 0; + auto kinds = this->token_kinds(); + // a quiesce without a walk serves only the tokens owing none + rsp_pending_ = walk_skip_ ? port_arb_->tokens(!config_.shared_llc) : port_arb_->tokens(); + this->forward_begin(walk_skip_, !walk_skip_ && kinds.any_l1 && !kinds.any_full && !kinds.any_io); flush_state_ = FlushState::Forward; break; + } case FlushState::Forward: { bool mem_req_empty = true; for (auto &lane : bank_arb_->ReqIn) { @@ -2541,14 +2580,28 @@ class Cache::Impl { for (auto &net : mem_req_net_) { mem_req_empty &= (net.size() == 0); } - for (uint32_t i = 0, n = config_.mem_ports; i < n && mem_req_empty; ++i) { - if (fwd_sent_ & (1ull << i)) + // an IO token also waits for the non-cacheable path, which carries the + // IO writes it orders + if (fwd_io_ && mem_req_empty) { + for (uint32_t i = 0, n = config_.num_inputs; i < n; ++i) { + mem_req_empty &= (port_arb_->IoOut.at(i).size() == 0); + } + for (auto &arb : nc_mem_arbs_) { + mem_req_empty &= (arb->ReqIn.at(0).size() == 0) && (arb->ReqIn.at(1).size() == 0); + } + } + fwd_go_ |= mem_req_empty; + for (uint32_t i = 0, n = config_.mem_ports; i < n && fwd_go_; ++i) { + if (fwd_sent_ & (1ull << i)) { continue; + } // the token for port i is a read of the port's first line, sent down // bank i's lane (bank i maps to port i) behind its write-backs MemReq token; token.addr = uint64_t(i) << config_.L; token.flags.flush = 1; + token.flags.io = fwd_io_; + token.flags.flush_l1 = fwd_l1_; token.tag = 0; if (bank_arb_->ReqIn.at(this->bank_lane(i)).try_send(token, 1)) { DT(3, simobject_->name() << " flush-fwd: " << token); @@ -2585,6 +2638,43 @@ class Cache::Impl { } } + struct token_kinds_t { + bool any_full = false; + bool any_l1 = false; + bool any_io = false; + // a walk is owed to a full token, and to an L1-only one above the point of coherence + bool walk(bool shared_llc) const { return any_full || (any_l1 && !shared_llc); } + }; + + // Kinds of the visible tokens. Several kinds together combine to the + // weakest one that serves them all. + token_kinds_t token_kinds() const { + token_kinds_t kinds; + uint64_t tokens = port_arb_->tokens(); + for (uint32_t i = 0, n = config_.num_inputs; i < n; ++i) { + if (!(tokens & (1ull << i))) { + continue; + } + auto &flags = port_arb_->ReqIn.at(i).peek().flags; + if (flags.io) { + kinds.any_io = true; + } else if (flags.flush_l1) { + kinds.any_l1 = true; + } else { + kinds.any_full = true; + } + } + return kinds; + } + + void forward_begin(bool io, bool l1_only) { + fwd_io_ = io; + fwd_l1_ = l1_only; + fwd_go_ = false; + fwd_sent_ = 0; + fwd_done_ = 0; + } + PerfStats perf_stats() const { PerfStats perf_stats; if (!config_.bypass) { @@ -2657,6 +2747,10 @@ class Cache::Impl { uint64_t rsp_pending_ = 0; uint64_t fwd_sent_ = 0; uint64_t fwd_done_ = 0; + bool fwd_go_ = false; // nothing is left ahead of the forwarded tokens + bool fwd_io_ = false; // the forwarded tokens' kind + bool fwd_l1_ = false; + bool walk_skip_ = false; // the banks only quiesce: no walk is owed std::vector> mem_req_net_; std::vector flush_taps_; diff --git a/sim/simx/mem/lsu_mem_adapter.cpp b/sim/simx/mem/lsu_mem_adapter.cpp index a34aacfe6c..60076bb902 100644 --- a/sim/simx/mem/lsu_mem_adapter.cpp +++ b/sim/simx/mem/lsu_mem_adapter.cpp @@ -47,7 +47,7 @@ LsuMemAdapter::LsuMemAdapter( mr.flags = req.flags; mr.amo_cmp = req.amo_cmp.at(0); auto t = get_addr_type(mr.addr); - mr.flags.io = (t == AddrType::IO); + mr.flags.io = mr.flags.flush ? mr.flags.io : (t == AddrType::IO); mr.flags.local = (t == AddrType::Shared); return mr; }); @@ -142,7 +142,7 @@ void LsuMemAdapter::on_tick() { out_req.flags = in_req.flags; { auto t = get_addr_type(out_req.addr); - out_req.flags.io = (t == AddrType::IO); + out_req.flags.io = out_req.flags.flush ? out_req.flags.io : (t == AddrType::IO); out_req.flags.local = (t == AddrType::Shared); } diff --git a/sim/simx/mem/mem_coalescer.cpp b/sim/simx/mem/mem_coalescer.cpp index 51a0b95a12..239acc1b50 100644 --- a/sim/simx/mem/mem_coalescer.cpp +++ b/sim/simx/mem/mem_coalescer.cpp @@ -273,7 +273,7 @@ void MemCoalescer::on_tick() { mr.uuid = in_req.uuid; mr.flags = in_req.flags; auto t = get_addr_type(mr.addr); - mr.flags.io = (t == AddrType::IO); + mr.flags.io = mr.flags.flush ? mr.flags.io : (t == AddrType::IO); mr.flags.local = (t == AddrType::Shared); } diff --git a/sim/simx/types.h b/sim/simx/types.h index d85c1186db..f3f9b4cf84 100644 --- a/sim/simx/types.h +++ b/sim/simx/types.h @@ -386,6 +386,10 @@ struct IntrLsuArgs { int32_t offset; // immediate addr offset (signed); 0 for packLD uops }; +// A fence carries no access width; its width field says it orders only +// device (IO) accesses. +inline bool lsu_fence_is_io(const IntrLsuArgs& args) { return args.width != 0; } + inline std::ostream &operator<<(std::ostream &os, const LsuType& type) { switch (type) { case LsuType::LOAD: os << "LOAD"; break; @@ -445,6 +449,7 @@ struct MemFlags { uint32_t local : 1; // bit 2: LMEM port uint32_t amo_unsigned : 1; // bit 3: MIN/MAX signedness (1=unsigned) uint32_t flush : 1; // bit 4: flush token (a fence) + uint32_t flush_l1 : 1; // bit 5: L1-only flush token: stops at the shared LLC #ifdef VX_CFG_EXT_DXA_ENABLE // DXA completion sideband — populated only on DXA's LMEM-DMA writes. // notify_bar_id carries the RAW (kernel-encoded) bar handle: @@ -454,8 +459,8 @@ struct MemFlags { // be wide enough to hold (bar_no << 8) | (MAX_CTAS_per_core - 1). // For NUM_BARRIERS = 8 and NUM_WARPS ≤ 16, that's at most 11 bits; // 16 bits leaves headroom for larger configs. - uint32_t dxa_notify_done : 1; // bit 5 - uint32_t dxa_notify_bar_id : 16; // bits 6..21 + uint32_t dxa_notify_done : 1; // bit 6 + uint32_t dxa_notify_bar_id : 16; // bits 7..22 #endif }; }; From 2342ffdcd323e764c8f544348104d81c7ecd3af8 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 19 Oct 2026 05:22:05 -0700 Subject: [PATCH 057/124] simx: serve a warp's identical loads with one dcache bank access SimX side of the dcache read merge in the RTL (VX_cache with READ_MERGE, VX_CFG_DCACHE_READ_MERGE, default on): - Port arbiter: plain reads (no write, flush, IO, local or atomic attribute) of the same word with the same tag, from ports of one LSU block in the same cycle, are served by one bank access. The lowest such port leads; the others stay out of arbitration and are accepted in the cycle their leader is granted. Nothing merges while a flush token holds the inputs. - The merged ports' mask rides below the request tag through the bank. - Response network: with merging, the RTL replaces the omega network with a flat crossbar of one buffered round-robin arbiter per core port over the bank lanes. A lane delivers a merged response to every target port, each when that port's arbiter grants it, and is released once all have taken it. Its depth is the output buffer plus the arbiter tree's buffered level above 12 banks, in place of the omega's levels. - The read counter counts a merged read once, since it reaches the bank once. A new parity case runs uniform_ld at 16 threads, where up to four channels of one warp merge. SimX, merge off -> on: sgemm -n128 on the 2-frontend shape (16 dcache ports in groups of 4) 571,848 -> 556,176 with dcache requests 1,069,056 -> 741,224 (rtlsim: 719,541, 528,299 cycles). Co-Authored-By: Claude Opus 5.5 --- ci/testcases/core.yaml | 7 ++ sim/simx/mem/cache.cpp | 159 +++++++++++++++++++++++++++++++++++++++-- sim/simx/mem/cache.h | 1 + sim/simx/socket.cpp | 1 + 4 files changed, 161 insertions(+), 7 deletions(-) diff --git a/ci/testcases/core.yaml b/ci/testcases/core.yaml index d7ab85e2bd..1ae0064d19 100644 --- a/ci/testcases/core.yaml +++ b/ci/testcases/core.yaml @@ -202,6 +202,13 @@ tests: via: blackbox app: printf configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_FETCH_WIDTH=2 +# Same-word reads from several coalescer channels of one warp, served by one +# dcache bank access and returned to every channel. +- id: parity-uniform-ld + check: model_parity + via: blackbox + app: uniform_ld + configs: -DVX_CFG_NUM_THREADS=16 # SGEMM with local-memory tiling (kernel-managed shared memory). - id: parity-sgemmx check: model_parity diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index b74b4bce1a..f813cdec29 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -2139,6 +2139,10 @@ class CacheBank : public SimObject { // while the bank's input buffer has room; non-cacheable requests are split off // for the bypass path. It ticks ahead of the banks, so a grant reaches its bank // in the same cycle unless the crossbar registers its output. +// With read merging, plain reads of the same word with the same tag from ports +// of one group in the same cycle are served by one bank access: the lowest +// such port leads, the others stay out of arbitration and are accepted with +// it. The merged ports' mask rides below the tag through the bank. class CachePortArb : public SimObject { public: std::vector> ReqIn; // core ports @@ -2146,7 +2150,7 @@ class CachePortArb : public SimObject { std::vector> IoOut; // non-cacheable requests, per port CachePortArb(const SimContext& ctx, const char* name, const params_t& params, - uint32_t num_inputs, uint32_t num_banks) + uint32_t num_inputs, uint32_t num_banks, uint32_t merge_group) : SimObject(ctx, name) , ReqIn(num_inputs, this) , ReqOut(num_banks, this) @@ -2157,10 +2161,16 @@ class CachePortArb : public SimObject { , delay_((num_inputs > 2) ? 1 : 0) , rr_(num_banks, num_inputs - 1) , tree_stage_((arb_tree_delay(num_inputs) != 0) ? num_banks : 0, this) + , merge_group_(merge_group) { assert(num_inputs <= 64); + __assert(merge_group_ <= 1 || (ispow2(merge_group_) && (num_inputs % merge_group_) == 0), + "a read-merge group must be a power of two dividing the ports"); } + // Bits of the merged-port mask carried below the request tag. + uint32_t merge_bits() const { return (merge_group_ > 1) ? merge_group_ : 0; } + uint64_t collisions() const { return collisions_; } // Ports whose head is a flush token. While any is visible, cacheable @@ -2236,11 +2246,34 @@ class CachePortArb : public SimObject { } uint32_t n = ReqIn.size(); const bool hold = (this->tokens() != 0); + // merged ports riding each leader's bank access + uint64_t followers = 0; + std::vector merged(n, 0); + if (merge_group_ > 1 && !hold) { + for (uint32_t i = 0; i < n; ++i) { + if (!this->is_plain_read(i)) { + continue; + } + auto& req_i = ReqIn.at(i).peek(); + for (uint32_t j = (i / merge_group_) * merge_group_; j < i; ++j) { + if ((followers & (1ull << j)) || !this->is_plain_read(j)) { + continue; + } + auto& req_j = ReqIn.at(j).peek(); + if ((req_j.addr >> params_.word_select_addr_start) == (req_i.addr >> params_.word_select_addr_start) + && req_j.tag == req_i.tag) { + merged.at(j) |= 1ull << i; + followers |= 1ull << i; + break; + } + } + } + } // requesting ports per bank; non-cacheable requests leave on their own port std::vector want(ReqOut.size(), 0); for (uint32_t i = 0; i < n; ++i) { auto& in = ReqIn.at(i); - if (in.empty()) { + if (in.empty() || (followers & (1ull << i))) { continue; } auto& req = in.peek(); @@ -2268,13 +2301,32 @@ class CachePortArb : public SimObject { } while (!(want[b] & (1ull << i))); collisions_ += __builtin_popcountll(want[b]) - 1; auto req = ReqIn.at(i).peek(); - req.tag = (req.tag << lg2_inputs_) | i; + uint64_t mask = 0; + if (merge_group_ > 1) { + mask = merged.at(i) >> ((i / merge_group_) * merge_group_); + for (uint64_t f = merged.at(i); f != 0; f &= f - 1) { + DT(3, this->name() << " read-merge: port=" << __builtin_ctzll(f) << ", leader=" << i << ", " << req); + ReqIn.at(__builtin_ctzll(f)).pop(); + } + } + uint64_t tag = (((uint64_t(req.tag) << this->merge_bits()) | mask) << lg2_inputs_) | i; + __assert((tag >> 32) == 0, "cache bank tag overflow"); + req.tag = uint32_t(tag); out.send(req, tree_stage_.empty() ? delay_ : 1); ReqIn.at(i).pop(); rr_[b] = i; } } + // A plain read: no write, flush, IO, local or atomic attribute. + bool is_plain_read(uint32_t i) const { + if (ReqIn.at(i).empty()) { + return false; + } + auto& req = ReqIn.at(i).peek(); + return req.op == MemOp::LD && !req.flags.flush && !req.flags.io && !req.flags.local; + } + private: params_t params_; uint32_t lg2_inputs_; @@ -2282,11 +2334,97 @@ class CachePortArb : public SimObject { uint32_t init_cycles_ = 0; std::vector rr_; std::vector> tree_stage_; + uint32_t merge_group_; uint64_t collisions_ = 0; friend class SimObject; }; +// The core-response network under read merging: a flat crossbar of one +// round-robin arbiter per core port over the bank lanes. A merged read's +// response goes to the port that sent it and to every merged port; each port +// takes it when its arbiter grants that lane, and the lane is released once +// all of them have. +class CacheRspMcast : public SimObject { +public: + using Ptr = std::shared_ptr; + + std::vector> RspIn; // banks + std::vector> RspOut; // core ports + + CacheRspMcast(const SimContext& ctx, const char* name, uint32_t num_banks, + uint32_t num_outputs, uint32_t merge_group) + : SimObject(ctx, name) + , RspIn(num_banks, this) + , RspOut(num_outputs, this) + , lg2_outputs_(log2ceil(num_outputs)) + , merge_group_(merge_group) + , sent_(num_banks, 0) + , last_grant_(num_outputs, num_banks - 1) + { + assert(num_outputs <= 64); + assert(merge_group < 64); + } + +protected: + void on_reset() { + std::fill(sent_.begin(), sent_.end(), 0); + std::fill(last_grant_.begin(), last_grant_.end(), RspIn.size() - 1); + } + + void on_tick() { + uint32_t nb = RspIn.size(); + uint32_t no = RspOut.size(); + std::vector pending(nb, 0); + bool idle = true; + for (uint32_t b = 0; b < nb; ++b) { + if (RspIn.at(b).empty()) { + continue; + } + idle = false; + auto tag = RspIn.at(b).peek().tag; + uint32_t port = tag & ((1u << lg2_outputs_) - 1); + uint64_t mask = (tag >> lg2_outputs_) & ((1ull << merge_group_) - 1); + uint64_t targets = (1ull << port) | (mask << ((port / merge_group_) * merge_group_)); + pending.at(b) = targets & ~sent_.at(b); + } + if (idle) { + this->tick_sleep(); + return; + } + for (uint32_t o = 0; o < no; ++o) { + for (uint32_t k = 1; k <= nb; ++k) { + uint32_t b = (last_grant_.at(o) + k) % nb; + if (!(pending.at(b) & (1ull << o))) { + continue; + } + auto rsp = RspIn.at(b).peek(); + rsp.tag >>= (lg2_outputs_ + merge_group_); + if (RspOut.at(o).try_send(rsp, 0)) { + DT(4, this->name() << " rsp" << b << "_" << o << ": " << rsp); + sent_.at(b) |= 1ull << o; + last_grant_.at(o) = b; + } + break; + } + } + for (uint32_t b = 0; b < nb; ++b) { + if (pending.at(b) != 0 && (pending.at(b) & ~sent_.at(b)) == 0) { + RspIn.at(b).pop(); + sent_.at(b) = 0; + } + } + } + +private: + uint32_t lg2_outputs_; + uint32_t merge_group_; + std::vector sent_; // [bank] ports that took the lane's response + std::vector last_grant_; // [port] round-robin pointer + + friend class SimObject; +}; + // Sits on one memory port's cache-side response path, ahead of the response // network. While the port's flush token is out the cache is quiescent, so the // port's next response is the token's answer and stops here; every other @@ -2421,7 +2559,8 @@ class Cache::Impl { // Create the core-port arbiter ahead of the banks it feeds snprintf(sname, 100, "%s-port_arb", simobject->name().c_str()); - port_arb_ = CachePortArb::Create(sname, params_, config_.num_inputs, num_banks); + uint32_t merge_group = std::min(config_.read_merge, config_.num_inputs); + port_arb_ = CachePortArb::Create(sname, params_, config_.num_inputs, num_banks, merge_group); for (uint32_t i = 0; i < config_.num_inputs; ++i) { simobject->core_req_in.at(i).bind(&port_arb_->ReqIn.at(i)); } @@ -2433,7 +2572,12 @@ class Cache::Impl { auto bank_sel = [&](const MemCrossBar::ReqType &req) { return params_.addr_bank_id(req.addr); }; - if (config_.core_out_buf) { + if (port_arb_->merge_bits() != 0) { + // one buffered arbiter per core port over the bank lanes + __assert(config_.core_out_buf, "read merging needs a registered core-response port"); + rsp_mcast_ = CacheRspMcast::Create(sname, num_banks, config_.num_inputs, port_arb_->merge_bits()); + core_rsp_delay_ = 1 + arb_tree_delay(num_banks); + } else if (config_.core_out_buf) { bank_core_xbar_ = MemCrossBar::Create(sname, ArbiterType::RoundRobin, config_.num_inputs, num_banks, bank_sel, 0, 0); core_rsp_delay_ = omega_stages(num_banks, config_.num_inputs); } else { @@ -2450,7 +2594,7 @@ class Cache::Impl { // bind core ports port_arb_->ReqOut.at(i).bind(&banks_.at(i)->core_req_in); - banks_.at(i)->core_rsp_out.bind(&bank_core_xbar_->RspIn.at(i)); + banks_.at(i)->core_rsp_out.bind(rsp_mcast_ ? &rsp_mcast_->RspIn.at(i) : &bank_core_xbar_->RspIn.at(i)); // bind memory ports banks_.at(i)->mem_req_out.bind(&bank_mem_arb->ReqIn.at(this->bank_lane(i))); @@ -2502,7 +2646,7 @@ class Cache::Impl { // schedule core responses for (uint32_t req_id = 0, n = config_.num_inputs; req_id < n; ++req_id) { - auto &bank_rsp = bank_core_xbar_->RspOut.at(req_id); + auto &bank_rsp = rsp_mcast_ ? rsp_mcast_->RspOut.at(req_id) : bank_core_xbar_->RspOut.at(req_id); if (bank_rsp.empty()) continue; auto &core_rsp = bank_rsp.peek(); @@ -2739,6 +2883,7 @@ class Cache::Impl { std::vector nc_mem_arbs_; CachePortArb::Ptr port_arb_; MemCrossBar::Ptr bank_core_xbar_; + CacheRspMcast::Ptr rsp_mcast_; // the core-response network's levels, then the non-cacheable bypass // layer's response arbiter uint32_t core_rsp_delay_ = 0; diff --git a/sim/simx/mem/cache.h b/sim/simx/mem/cache.h index 6371f75d09..7c61c03bdd 100644 --- a/sim/simx/mem/cache.h +++ b/sim/simx/mem/cache.h @@ -48,6 +48,7 @@ class Cache : public SimObject { bool shared_llc = false; // the LLC serves every memory client alone bool lazy_fetch = false; // write miss allocates without a fill (write-back LLC only) bool core_out_buf = false; // registered core-response port + uint8_t read_merge = 0; // merge same-word reads within groups of this many adjacent ports (0: off) }; struct PerfStats { diff --git a/sim/simx/socket.cpp b/sim/simx/socket.cpp index 7c8a76942f..e1dbe03d09 100644 --- a/sim/simx/socket.cpp +++ b/sim/simx/socket.cpp @@ -101,6 +101,7 @@ class Socket::Impl { DCACHE_SHARED_LLC, // shared_llc VX_CFG_DCACHE_LAZY_FETCH != 0, // lazy_fetch true, // core-response register + (VX_CFG_DCACHE_READ_MERGE != 0) ? DCACHE_CHANNELS : 0, // read_merge }); #ifdef VX_CFG_EXT_TEX_ENABLE From c1516d3e1d9d00e8d238e4f4a84c4ad42e0087a4 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 19 Oct 2026 05:22:05 -0700 Subject: [PATCH 058/124] simx: route a single cluster's global barrier through its arbiters With one cluster, SimX counted a global-barrier arrival as it left the core and released the cores one cycle after the last one. In the RTL the arrival still crosses the socket's gbar arbiter and the cluster's (each with an output buffer when it has several inputs, the cluster's taking one arrival per cycle) to the device unit, whose registered release then crosses the cluster's and the socket's response forks. Only the device arbiter is a wire with one cluster. The single-cluster path now takes the same arbiter lags and serialization as the multi-cluster path, without the device arbiter's buffer. Co-Authored-By: Claude Opus 5.5 --- sim/simx/cluster.cpp | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/sim/simx/cluster.cpp b/sim/simx/cluster.cpp index 18fa87ad96..481fbe601a 100644 --- a/sim/simx/cluster.cpp +++ b/sim/simx/cluster.cpp @@ -349,16 +349,24 @@ class Cluster::Impl { DT(4, "*** Global barrier arrive: cluster #" << simobject_->id() << ", core #" << core_id << " at barrier #" << bar_id << ", arrived=" << gbar.mask.count()); - if constexpr (VX_CFG_NUM_CLUSTERS > 1) { - // Up to the device unit: the socket arbiter's output buffer, the cluster - // arbiter (one arrival per cycle) and its output buffer, then the device - // arbiter's output buffer. + // Up to the device unit: the socket arbiter's output buffer, then the + // cluster arbiter (one arrival per cycle) and its output buffer. A + // single-core device has no global-barrier path. + uint32_t lag = 0; + if constexpr (VX_CFG_NUM_CLUSTERS * VX_CFG_NUM_CORES > 1) { uint64_t now = SimPlatform::instance().cycles(); uint64_t out = std::max(now + kSocketArbLag, gbar_fwd_next_); gbar_fwd_next_ = out + 1; - simobject_->gbar_fwd_out.send({bar_id, count, core_id}, out + kClusterArbLag + 1 - now); + lag = out + kClusterArbLag - now; + } + if constexpr (VX_CFG_NUM_CLUSTERS > 1) { + // the device arbiter's output buffer + simobject_->gbar_fwd_out.send({bar_id, count, core_id}, lag + 1); } else if (gbar.mask.count() == (size_t)count) { - this->global_barrier_resume(bar_id, 1); + // With one cluster the device arbiter is a wire. The unit registers its + // release, which then crosses the cluster's and the socket's response + // forks. + this->global_barrier_resume(bar_id, lag + 1 + kClusterArbLag + kSocketArbLag); } } From 46080c037ec27def01d5f0263047c9a27bda1c9d Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 19 Oct 2026 05:22:06 -0700 Subject: [PATCH 059/124] simx: track MSHR misses in a line table when MSHR_LINES is set SimX side of the MSHR line table in the RTL (VX_cache_mshr with MSHR_LINES, VX_CFG_{DCACHE,L2,L3}_MSHR_LINES, default 0 = off): - The cache config carries mshr_lines for the dcache, L2 and L3. - The MSHR keeps a table of the distinct {line, sector} misses its request entries hold, each reference-counted by the entries chained on it and freed when the count drains. With a cap, overflowing it is a modelling error. - The bank admits a core request only while a line entry is free for it after every request ahead of it has claimed its own: the lines in the MSHR plus each distinct line a request still inside the pipe may allocate, since the RTL allocates at the pipe's first stage. This is the RTL's two-free-entries rule (the request admitted the cycle before plus this one) in SimX's later-allocating pipe. The stall counts as an MSHR stall, as the RTL folds it into its almost-full stall. - The pipe mirror of in-flight lines keeps the sector, which the line table keys on. With the cap at 0 every config is cycle-identical. sector_merge-2 (MSHR 32 requests over 8 lines on the dcache and L2) now also runs on simx. sgemm -n128, 16 threads x 16 warps, memory latency 400: dcache MSHR 16 (no table): SimX 2,628,803 rtlsim 2,733,260 dcache MSHR 64 over 16 lines: SimX 2,296,987 rtlsim 2,315,677 (0.8%) Co-Authored-By: Claude Opus 5.5 --- ci/testcases/cache.yaml | 1 + sim/simx/cluster.cpp | 2 + sim/simx/mem/cache.cpp | 111 +++++++++++++++++++++++++++++++++++----- sim/simx/mem/cache.h | 1 + sim/simx/processor.cpp | 2 + sim/simx/socket.cpp | 1 + 6 files changed, 106 insertions(+), 12 deletions(-) diff --git a/ci/testcases/cache.yaml b/ci/testcases/cache.yaml index 361ec300e2..92148b00ed 100644 --- a/ci/testcases/cache.yaml +++ b/ci/testcases/cache.yaml @@ -353,6 +353,7 @@ tests: - id: sector_merge-2 via: blackbox drivers: + - simx - rtlsim app: sector_merge configs: -DVX_CFG_SOCKET_SIZE=1 -DVX_CFG_DCACHE_MSHR_SIZE=32 -DVX_CFG_DCACHE_MSHR_LINES=8 -DVX_CFG_L2_MSHR_SIZE=32 -DVX_CFG_L2_MSHR_LINES=8 diff --git a/sim/simx/cluster.cpp b/sim/simx/cluster.cpp index 481fbe601a..adf1f221fc 100644 --- a/sim/simx/cluster.cpp +++ b/sim/simx/cluster.cpp @@ -95,6 +95,8 @@ class Cluster::Impl { L2_SHARED_LLC, // shared_llc VX_CFG_L2_LAZY_FETCH != 0, // lazy_fetch true, // core-response register (behind the bypass layer) + 0, // read_merge + VX_CFG_L2_MSHR_LINES, // mshr_lines }); diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index f813cdec29..cb54186e6b 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -506,10 +506,21 @@ struct mshr_entry_t { } }; +// With a line cap, the {line, sector} misses live in a table of that many +// entries, each reference-counted by the request entries chained on it. class MSHR { public: - MSHR(uint32_t size) - : entries_(size), ready_reqs_(0), size_(0) {} + MSHR(uint32_t size, uint32_t line_cap) + : entries_(size), ready_reqs_(0), size_(0), line_cap_(line_cap) {} + + // Distinct {line, sector} misses held. + uint32_t lines() const { + return lines_.size(); + } + + bool holds_line(uint32_t set_id, uint64_t addr_tag, uint32_t sector_id) const { + return this->find_line(set_id, addr_tag, sector_id) != lines_.end(); + } uint32_t capacity() const { return entries_.size(); @@ -604,6 +615,7 @@ class MSHR { entry.refetch = refetch; entry.refetch_retry = false; ++size_; + this->line_ref(set_id, addr_tag, sector_id); return i; } } @@ -706,6 +718,7 @@ class MSHR { picked->bank_req.type = bank_req_t::None; --ready_reqs_; --size_; + this->line_unref(picked->set_id, picked->addr_tag, picked->sector_id); } void reset() { @@ -714,12 +727,51 @@ class MSHR { } ready_reqs_ = 0; size_ = 0; + lines_.clear(); } private: + struct line_ref_t { + uint32_t set_id; + uint64_t addr_tag; + uint32_t sector_id; + uint32_t count; + }; + + std::vector::const_iterator find_line(uint32_t set_id, uint64_t addr_tag, uint32_t sector_id) const { + return std::find_if(lines_.begin(), lines_.end(), [&](const line_ref_t& l) { + return l.set_id == set_id && l.addr_tag == addr_tag && l.sector_id == sector_id; + }); + } + + void line_ref(uint32_t set_id, uint64_t addr_tag, uint32_t sector_id) { + for (auto &l : lines_) { + if (l.set_id == set_id && l.addr_tag == addr_tag && l.sector_id == sector_id) { + ++l.count; + return; + } + } + __assert(line_cap_ == 0 || lines_.size() < line_cap_, "MSHR line table overflow"); + lines_.push_back({set_id, addr_tag, sector_id, 1}); + } + + void line_unref(uint32_t set_id, uint64_t addr_tag, uint32_t sector_id) { + for (auto it = lines_.begin(); it != lines_.end(); ++it) { + if (it->set_id == set_id && it->addr_tag == addr_tag && it->sector_id == sector_id) { + if (--it->count == 0) { + lines_.erase(it); + } + return; + } + } + __assert(false, "MSHR line count underflow"); + } + std::vector entries_; uint32_t ready_reqs_; uint32_t size_; + uint32_t line_cap_; + std::vector lines_; }; class CacheBank : public SimObject { @@ -750,7 +802,7 @@ class CacheBank : public SimObject { const Cache::Config &config, const params_t ¶ms, uint32_t bank_id) - : SimObject(ctx, name), core_req_in(this), core_rsp_out(this), mem_req_out(this), mem_rsp_in(this, config.mshr_size + AMO_PASSTHRU_CAP), config_(config), params_(params), bank_id_(bank_id), sets_(params.sets_per_bank, set_t(params.lines_per_set, params.sectors_per_line)), mshr_(config.mshr_size), pipe_req_(TFifo::Create("", config.latency, std::max(2u, config.latency))), lazy_q_(TFifo::Create("", 1, lazyq_size(config))), rand_ctr_(0), adm_seq_ctr_(0) + : SimObject(ctx, name), core_req_in(this), core_rsp_out(this), mem_req_out(this), mem_rsp_in(this, config.mshr_size + AMO_PASSTHRU_CAP), config_(config), params_(params), bank_id_(bank_id), sets_(params.sets_per_bank, set_t(params.lines_per_set, params.sectors_per_line)), mshr_(config.mshr_size, config.mshr_lines), pipe_req_(TFifo::Create("", config.latency, std::max(2u, config.latency))), lazy_q_(TFifo::Create("", 1, lazyq_size(config))), rand_ctr_(0), adm_seq_ctr_(0) #if VX_CFG_EXT_A_ENABLED , amo_unit_(__MAX(2u, (uint32_t)VX_CFG_AMO_RS_SIZE)) #endif @@ -1112,6 +1164,13 @@ class CacheBank : public SimObject { ++perf_stats_.mshr_stalls; return; } + // With a line table, a line entry must stay free for this request once + // every request ahead of it has claimed its own. + if (needs_mshr && config_.mshr_lines != 0 + && this->mshrLinesClaimed() >= config_.mshr_lines) { + ++perf_stats_.mshr_stalls; + return; + } // Any admitted request may become a write-miss root: the lazy queue // keeps room for every request already in the pipe. if (config_.lazy_fetch && lazy_q_->size() >= lazyq_size(config_) - config_.latency) { @@ -1209,8 +1268,9 @@ class CacheBank : public SimObject { // stays out of this counter. if (!is_amo_passthru) { ++pending_mshr_size_; - pipe_core_lines_.emplace_back(params_.addr_set_id(core_req.addr), - params_.addr_tag(core_req.addr)); + pipe_core_lines_.push_back({params_.addr_set_id(core_req.addr), + params_.addr_tag(core_req.addr), + params_.addr_sector_id(core_req.addr)}); } #if VX_CFG_EXT_A_ENABLED if (is_amo_passthru) ++pending_amo_probes_; @@ -2050,29 +2110,56 @@ class CacheBank : public SimObject { MSHR mshr_; uint32_t pending_mshr_size_; uint32_t pending_amo_probes_; // AmoProbe requests in the pipe, each holding a table reservation - // (set, tag) of Core requests inside the pipe: their MSHR entries exist - // only from pipe exit, so ordering checks against in-flight lines (the - // AMO admission defer) must consult this mirror as well. - std::vector> pipe_core_lines_; + // {set, tag, sector} of Core requests inside the pipe: their MSHR entries + // exist only from pipe exit, so ordering checks against in-flight lines (the + // AMO admission defer) and the line-table admission must consult this + // mirror as well. + struct pipe_line_t { + uint32_t set_id; + uint64_t addr_tag; + uint32_t sector_id; + }; + std::vector pipe_core_lines_; bool pipeLineBusy(uint32_t set_id, uint64_t addr_tag) const { - for (const auto &kv : pipe_core_lines_) { - if (kv.first == set_id && kv.second == addr_tag) { + for (const auto &l : pipe_core_lines_) { + if (l.set_id == set_id && l.addr_tag == addr_tag) { return true; } } return false; } + // The pipe is in order, so the oldest match is the request leaving it. void pipeLineErase(uint32_t set_id, uint64_t addr_tag) { for (auto it = pipe_core_lines_.begin(); it != pipe_core_lines_.end(); ++it) { - if (it->first == set_id && it->second == addr_tag) { + if (it->set_id == set_id && it->addr_tag == addr_tag) { pipe_core_lines_.erase(it); return; } } assert(false && "in-pipe line mirror underflow"); } + + // MSHR line entries held or claimed: the lines in the MSHR, plus a line for + // each distinct one a request inside the pipe may still allocate. The RTL + // allocates at the pipe's first stage, so these already hold their entries. + uint32_t mshrLinesClaimed() const { + uint32_t n = mshr_.lines(); + for (uint32_t i = 0, k = pipe_core_lines_.size(); i < k; ++i) { + auto &l = pipe_core_lines_.at(i); + if (mshr_.holds_line(l.set_id, l.addr_tag, l.sector_id)) { + continue; + } + bool seen = false; + for (uint32_t j = 0; j < i && !seen; ++j) { + auto &o = pipe_core_lines_.at(j); + seen = (o.set_id == l.set_id && o.addr_tag == l.addr_tag && o.sector_id == l.sector_id); + } + n += !seen; + } + return n; + } TFifo::Ptr pipe_req_; TFifo::Ptr lazy_q_; // write-miss roots awaiting their data-less fill diff --git a/sim/simx/mem/cache.h b/sim/simx/mem/cache.h index 7c61c03bdd..1bd4e925f3 100644 --- a/sim/simx/mem/cache.h +++ b/sim/simx/mem/cache.h @@ -49,6 +49,7 @@ class Cache : public SimObject { bool lazy_fetch = false; // write miss allocates without a fill (write-back LLC only) bool core_out_buf = false; // registered core-response port uint8_t read_merge = 0; // merge same-word reads within groups of this many adjacent ports (0: off) + uint16_t mshr_lines = 0; // MSHR distinct {line, sector} entries (0: one per request) }; struct PerfStats { diff --git a/sim/simx/processor.cpp b/sim/simx/processor.cpp index 268773e2d8..aa318ab00a 100644 --- a/sim/simx/processor.cpp +++ b/sim/simx/processor.cpp @@ -117,6 +117,8 @@ ProcessorImpl::ProcessorImpl() L3_SHARED_LLC, // shared_llc VX_CFG_L3_LAZY_FETCH != 0, // lazy_fetch true, // core-response register (behind the bypass layer) + 0, // read_merge + VX_CFG_L3_MSHR_LINES, // mshr_lines } ); diff --git a/sim/simx/socket.cpp b/sim/simx/socket.cpp index e1dbe03d09..b44fb21845 100644 --- a/sim/simx/socket.cpp +++ b/sim/simx/socket.cpp @@ -102,6 +102,7 @@ class Socket::Impl { VX_CFG_DCACHE_LAZY_FETCH != 0, // lazy_fetch true, // core-response register (VX_CFG_DCACHE_READ_MERGE != 0) ? DCACHE_CHANNELS : 0, // read_merge + VX_CFG_DCACHE_MSHR_LINES, // mshr_lines }); #ifdef VX_CFG_EXT_TEX_ENABLE From a2b16ad26af4dc90c5d229d88b177e347f9d4c36 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 19 Oct 2026 16:06:10 -0700 Subject: [PATCH 060/124] simx: let each slice of a cache arbiter tree grant on its own Over 12 requesters the RTL's arbiters (VX_stream_arb with MAX_FANOUT 8) become a tree: each slice of up to 8 inputs has its own round-robin arbiter and a 2-entry output buffer, and a join arbiter picks one slice buffer per cycle. Two slices can therefore each accept a request for the same output in one cycle. SimX modeled the tree as one arbiter plus one buffer per output, so it accepted only one request per output per cycle and over-counted bank conflicts. This affected the cache's request crossbar (one arbiter per bank over the core ports) and, under read merging, the response crossbar (one arbiter per core port over the bank lanes). Both now have a buffer and a round-robin pointer per slice, and a join arbiter per output. The response tree's buffered level is now the slice buffers rather than an added delay. sgemm on the 2-frontend shape (16 dcache ports, 16 banks), SimX vs rtlsim: -n32 11,029 -> 10,549 (rtlsim 9,885) -n64 72,572 -> 66,039 (rtlsim 65,949) -n128 556,176 -> 553,250 (rtlsim 528,299) Configurations with 12 or fewer ports per cache are unchanged. Co-Authored-By: Claude Opus 5.5 --- sim/simx/mem/cache.cpp | 173 +++++++++++++++++++++++++++++------------ 1 file changed, 124 insertions(+), 49 deletions(-) diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index cb54186e6b..cdfe079bdc 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -31,13 +31,20 @@ using namespace vortex; namespace { +constexpr uint32_t kMaxFanout = 8; + // An arbiter over more than 1.5x the fan-out limit is split into a buffered // tree, one cycle deeper. uint32_t arb_tree_delay(uint32_t requesters) { - constexpr uint32_t kMaxFanout = 8; return (requesters > kMaxFanout + kMaxFanout / 2) ? 1 : 0; } +// Slices of the tree's first level: each arbitrates up to the fan-out limit +// into its own buffer, and a join arbiter picks among the slice buffers. +uint32_t arb_tree_slices(uint32_t requesters) { + return (arb_tree_delay(requesters) != 0) ? (requesters + kMaxFanout - 1) / kMaxFanout : 1; +} + // Registered stages of a radix-2 omega network between a and b ports; up to // two ports on each side it is a single crossbar. uint32_t omega_stages(uint32_t a, uint32_t b) { @@ -2246,8 +2253,10 @@ class CachePortArb : public SimObject { , lg2_inputs_(log2ceil(num_inputs)) // the request crossbar registers its output when it has more than two inputs , delay_((num_inputs > 2) ? 1 : 0) - , rr_(num_banks, num_inputs - 1) - , tree_stage_((arb_tree_delay(num_inputs) != 0) ? num_banks : 0, this) + , slices_(arb_tree_slices(num_inputs)) + , rr_(num_banks * slices_, num_inputs - 1) + , join_rr_(num_banks, slices_ - 1) + , tree_stage_((slices_ > 1) ? num_banks * slices_ : 0, this) , merge_group_(merge_group) { assert(num_inputs <= 64); @@ -2315,6 +2324,7 @@ class CachePortArb : public SimObject { protected: void on_reset() { std::fill(rr_.begin(), rr_.end(), ReqIn.size() - 1); + std::fill(join_rr_.begin(), join_rr_.end(), slices_ - 1); collisions_ = 0; } @@ -2323,12 +2333,20 @@ class CachePortArb : public SimObject { --init_cycles_; return; } - // the arbiter tree's buffered level ahead of each bank - for (uint32_t b = 0, nb = tree_stage_.size(); b < nb; ++b) { - auto& stage = tree_stage_.at(b); - if (!stage.empty() && !ReqOut.at(b).full()) { - ReqOut.at(b).send(stage.peek(), delay_); - stage.pop(); + // the arbiter tree's join level: one slice buffer per bank per cycle + for (uint32_t b = 0, nb = (slices_ > 1) ? ReqOut.size() : 0; b < nb; ++b) { + if (ReqOut.at(b).full()) { + continue; + } + for (uint32_t k = 1; k <= slices_; ++k) { + uint32_t sl = (join_rr_[b] + k) % slices_; + auto& stage = tree_stage_.at(b * slices_ + sl); + if (!stage.empty()) { + ReqOut.at(b).send(stage.peek(), delay_); + stage.pop(); + join_rr_[b] = sl; + break; + } } } uint32_t n = ReqIn.size(); @@ -2377,32 +2395,55 @@ class CachePortArb : public SimObject { want.at(params_.addr_bank_id(req.addr)) |= 1ull << i; } } + // each bank's arbiter, or each slice of its tree, grants one port per cycle for (uint32_t b = 0, nb = ReqOut.size(); b < nb; ++b) { - auto& out = tree_stage_.empty() ? ReqOut.at(b) : tree_stage_.at(b); - if (want[b] == 0 || out.full()) { + if (want[b] == 0) { continue; } - uint32_t i = rr_[b]; - do { - i = (i + 1) % n; - } while (!(want[b] & (1ull << i))); - collisions_ += __builtin_popcountll(want[b]) - 1; - auto req = ReqIn.at(i).peek(); - uint64_t mask = 0; - if (merge_group_ > 1) { - mask = merged.at(i) >> ((i / merge_group_) * merge_group_); - for (uint64_t f = merged.at(i); f != 0; f &= f - 1) { - DT(3, this->name() << " read-merge: port=" << __builtin_ctzll(f) << ", leader=" << i << ", " << req); - ReqIn.at(__builtin_ctzll(f)).pop(); + bool granted = false; + for (uint32_t sl = 0; sl < slices_; ++sl) { + uint64_t slice_want = want[b]; + if (slices_ > 1) { + uint32_t lo = sl * kMaxFanout; + uint32_t hi = std::min(lo + kMaxFanout, n); + uint64_t range = ((hi < 64) ? (1ull << hi) : 0) - (1ull << lo); + slice_want &= range; + } + auto& out = (slices_ > 1) ? tree_stage_.at(b * slices_ + sl) : ReqOut.at(b); + if (slice_want == 0 || out.full()) { + continue; } + auto& rr = rr_[b * slices_ + sl]; + uint32_t i = rr; + do { + i = (i + 1) % n; + } while (!(slice_want & (1ull << i))); + this->grant(i, out, merged); + rr = i; + granted = true; + } + if (granted) { + collisions_ += __builtin_popcountll(want[b]) - 1; + } + } + } + + // Send port i's request to its bank, with the ports merged into it. + void grant(uint32_t i, SimChannel& out, const std::vector& merged) { + auto req = ReqIn.at(i).peek(); + uint64_t mask = 0; + if (merge_group_ > 1) { + mask = merged.at(i) >> ((i / merge_group_) * merge_group_); + for (uint64_t f = merged.at(i); f != 0; f &= f - 1) { + DT(3, this->name() << " read-merge: port=" << __builtin_ctzll(f) << ", leader=" << i << ", " << req); + ReqIn.at(__builtin_ctzll(f)).pop(); } - uint64_t tag = (((uint64_t(req.tag) << this->merge_bits()) | mask) << lg2_inputs_) | i; - __assert((tag >> 32) == 0, "cache bank tag overflow"); - req.tag = uint32_t(tag); - out.send(req, tree_stage_.empty() ? delay_ : 1); - ReqIn.at(i).pop(); - rr_[b] = i; } + uint64_t tag = (((uint64_t(req.tag) << this->merge_bits()) | mask) << lg2_inputs_) | i; + __assert((tag >> 32) == 0, "cache bank tag overflow"); + req.tag = uint32_t(tag); + out.send(req, tree_stage_.empty() ? delay_ : 1); + ReqIn.at(i).pop(); } // A plain read: no write, flush, IO, local or atomic attribute. @@ -2419,8 +2460,10 @@ class CachePortArb : public SimObject { uint32_t lg2_inputs_; uint32_t delay_; uint32_t init_cycles_ = 0; - std::vector rr_; - std::vector> tree_stage_; + uint32_t slices_; + std::vector rr_; // [bank][slice] + std::vector join_rr_; // [bank] + std::vector> tree_stage_; // [bank][slice] uint32_t merge_group_; uint64_t collisions_ = 0; @@ -2431,7 +2474,8 @@ class CachePortArb : public SimObject { // round-robin arbiter per core port over the bank lanes. A merged read's // response goes to the port that sent it and to every merged port; each port // takes it when its arbiter grants that lane, and the lane is released once -// all of them have. +// all of them have. Over many banks a port's arbiter is a tree: each slice of +// lanes grants into its own buffer, and a join picks among the slices. class CacheRspMcast : public SimObject { public: using Ptr = std::shared_ptr; @@ -2446,8 +2490,11 @@ class CacheRspMcast : public SimObject { , RspOut(num_outputs, this) , lg2_outputs_(log2ceil(num_outputs)) , merge_group_(merge_group) + , slices_(arb_tree_slices(num_banks)) , sent_(num_banks, 0) - , last_grant_(num_outputs, num_banks - 1) + , last_grant_(num_outputs * slices_, num_banks - 1) + , join_rr_(num_outputs, slices_ - 1) + , tree_stage_((slices_ > 1) ? num_outputs * slices_ : 0, this) { assert(num_outputs <= 64); assert(merge_group < 64); @@ -2457,13 +2504,32 @@ class CacheRspMcast : public SimObject { void on_reset() { std::fill(sent_.begin(), sent_.end(), 0); std::fill(last_grant_.begin(), last_grant_.end(), RspIn.size() - 1); + std::fill(join_rr_.begin(), join_rr_.end(), slices_ - 1); } void on_tick() { uint32_t nb = RspIn.size(); uint32_t no = RspOut.size(); - std::vector pending(nb, 0); bool idle = true; + // the tree's join level: one slice buffer per port per cycle + for (uint32_t o = 0; o < no && slices_ > 1; ++o) { + for (uint32_t k = 1; k <= slices_; ++k) { + uint32_t sl = (join_rr_.at(o) + k) % slices_; + auto& stage = tree_stage_.at(o * slices_ + sl); + if (stage.empty()) { + continue; + } + if (RspOut.at(o).try_send(stage.peek(), 0)) { + stage.pop(); + join_rr_.at(o) = sl; + } + break; + } + } + for (auto& stage : tree_stage_) { + idle &= (stage.size() == 0); + } + std::vector pending(nb, 0); for (uint32_t b = 0; b < nb; ++b) { if (RspIn.at(b).empty()) { continue; @@ -2480,19 +2546,25 @@ class CacheRspMcast : public SimObject { return; } for (uint32_t o = 0; o < no; ++o) { - for (uint32_t k = 1; k <= nb; ++k) { - uint32_t b = (last_grant_.at(o) + k) % nb; - if (!(pending.at(b) & (1ull << o))) { - continue; - } - auto rsp = RspIn.at(b).peek(); - rsp.tag >>= (lg2_outputs_ + merge_group_); - if (RspOut.at(o).try_send(rsp, 0)) { - DT(4, this->name() << " rsp" << b << "_" << o << ": " << rsp); - sent_.at(b) |= 1ull << o; - last_grant_.at(o) = b; + for (uint32_t sl = 0; sl < slices_; ++sl) { + uint32_t lo = (slices_ > 1) ? sl * kMaxFanout : 0; + uint32_t hi = (slices_ > 1) ? std::min(lo + kMaxFanout, nb) : nb; + auto& out = (slices_ > 1) ? tree_stage_.at(o * slices_ + sl) : RspOut.at(o); + auto& rr = last_grant_.at(o * slices_ + sl); + for (uint32_t k = 1; k <= nb; ++k) { + uint32_t b = (rr + k) % nb; + if (b < lo || b >= hi || !(pending.at(b) & (1ull << o))) { + continue; + } + auto rsp = RspIn.at(b).peek(); + rsp.tag >>= (lg2_outputs_ + merge_group_); + if (out.try_send(rsp, (slices_ > 1) ? 1 : 0)) { + DT(4, this->name() << " rsp" << b << "_" << o << ": " << rsp); + sent_.at(b) |= 1ull << o; + rr = b; + } + break; } - break; } } for (uint32_t b = 0; b < nb; ++b) { @@ -2506,8 +2578,11 @@ class CacheRspMcast : public SimObject { private: uint32_t lg2_outputs_; uint32_t merge_group_; - std::vector sent_; // [bank] ports that took the lane's response - std::vector last_grant_; // [port] round-robin pointer + uint32_t slices_; + std::vector sent_; // [bank] ports that took the lane's response + std::vector last_grant_; // [port][slice] round-robin pointer + std::vector join_rr_; // [port] + std::vector> tree_stage_; // [port][slice] friend class SimObject; }; @@ -2663,7 +2738,7 @@ class Cache::Impl { // one buffered arbiter per core port over the bank lanes __assert(config_.core_out_buf, "read merging needs a registered core-response port"); rsp_mcast_ = CacheRspMcast::Create(sname, num_banks, config_.num_inputs, port_arb_->merge_bits()); - core_rsp_delay_ = 1 + arb_tree_delay(num_banks); + core_rsp_delay_ = 1; } else if (config_.core_out_buf) { bank_core_xbar_ = MemCrossBar::Create(sname, ArbiterType::RoundRobin, config_.num_inputs, num_banks, bank_sel, 0, 0); core_rsp_delay_ = omega_stages(num_banks, config_.num_inputs); From a9bd77bc12c3a562ffedeb4ac8e843e6e86f0e25 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 19 Oct 2026 16:06:10 -0700 Subject: [PATCH 061/124] simx: register a shared cache's input arbiter as the RTL does When the cores of a socket share an L1 cache (SOCKET_SIZE larger than the cache count), VX_cache_cluster puts a VX_mem_bus_arb in front of each cache port, with a registered request buffer (REQ_OUT_BUF 3) and, when the cache registers its core-response port, a registered response buffer (RSP_OUT_BUF = CORE_OUT_BUF). SimX's arbiter for two cores onto one cache had no delay in either direction, so instruction fetch and data accesses on a shared icache and dcache were two cycles short. The arbiter now takes one cycle on the request and, with a registered core-response port, one on the response, whenever cores share a cache. async_gbarrier at 2 cores in one socket with an L2: SimX 69,852 -> 78,230, rtlsim 78,237. A new parity case covers this shape. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/regression.yaml | 10 ++++++++++ sim/simx/mem/cache_cluster.cpp | 10 ++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/ci/testcases/regression.yaml b/ci/testcases/regression.yaml index ccaf86aa66..9238aa4c88 100644 --- a/ci/testcases/regression.yaml +++ b/ci/testcases/regression.yaml @@ -156,6 +156,16 @@ tests: app: async_gbarrier shape: cores: 2 +# Two cores sharing one socket's instruction and data caches: their requests +# and responses cross each cache's input arbiter. +- id: model_parity-async-gbarrier-ss2 + check: model_parity + via: blackbox + app: async_gbarrier + configs: -DVX_CFG_SOCKET_SIZE=2 + shape: + cores: 2 + l2cache: true - id: bar-slot-phase-1 via: blackbox diff --git a/sim/simx/mem/cache_cluster.cpp b/sim/simx/mem/cache_cluster.cpp index ee88f524d9..5b8bddc2b8 100644 --- a/sim/simx/mem/cache_cluster.cpp +++ b/sim/simx/mem/cache_cluster.cpp @@ -35,11 +35,17 @@ CacheCluster::CacheCluster(const SimContext& ctx, char sname[100]; - // Arbitrate incoming core interfaces + // Arbitrate incoming core interfaces. Arbitrating cores onto fewer caches + // registers the request, and the response once more when the cache + // registers its core-response port. + const bool shared = (num_inputs != num_units); + const uint32_t arb_req_delay = shared ? 1 : 0; + const uint32_t arb_rsp_delay = (shared && cache_config.core_out_buf) ? 1 : 0; std::vector input_arbs(cache_config.num_inputs); for (uint32_t i = 0; i < cache_config.num_inputs; ++i) { snprintf(sname, 100, "%s-input-arb%d", name, i); - input_arbs.at(i) = MemArbiter::Create(sname, ArbiterType::RoundRobin, num_inputs, num_units); + input_arbs.at(i) = MemArbiter::Create(sname, ArbiterType::RoundRobin, num_inputs, num_units, + arb_req_delay, arb_rsp_delay); for (uint32_t j = 0; j < num_inputs; ++j) { this->core_req_in.at(j).at(i).bind(&input_arbs.at(i)->ReqIn.at(j)); input_arbs.at(i)->RspOut.at(j).bind(&this->core_rsp_out.at(j).at(i)); From bbad636e32e8f7a7ab727105f8174d8f46d25c81 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 19 Oct 2026 16:06:11 -0700 Subject: [PATCH 062/124] simx: let the command processor's DMA use the shared last-level cache SimX side of VX_CFG_CP_LLC_CLIENT (VX_cp_llc_port, the LLC client slot and write-around in the RTL). The default is off; off, every configuration is unchanged. With VX_CFG_CP_ON_LLC: - The shared LLC gets one more client slot beside the cores: the cluster L2 when it is the LLC of one cluster, the L3 otherwise. It does not change the bank count. - The CP port sends host transfers into that slot, one LLC word per request. IO-aperture addresses carry the io attribute and bypass the cache. Reads keep up to one 4 KB burst in flight. - Writes of transfers of at least half the LLC carry a no-allocate attribute. A write-back bank sends a no-allocate write miss with no fill pending straight to memory and updates a hit in place. - A launch that follows DMA writes first sends an L1-only flush token on the slot behind them and starts once the LLC answers it. - The SimX runtime asks the simulator whether its CP is an LLC client and then routes CP transfers through the port, so they are timed and counted, and publishes the DMA_LLC_CLIENT capability, so the runtime issues only L1-only flushes around transfers. It does not read the hardware build configuration. The model resets its state at each launch. The LLC keeps the lines that transfers left in it into the launch, and the launch first waits for the transfers' fills, which in the RTL would simply keep going. dogfood, flush_coherence and fence_mp pass with the option on at 4 cores + L2 and 2 clusters x 2 cores + L2 + L3. A parity case runs dogfood at 4 cores + L2. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/regression.yaml | 11 ++ sim/simx/cluster.cpp | 27 ++++- sim/simx/cluster.h | 7 ++ sim/simx/constants.h | 15 +++ sim/simx/cp_llc_port.h | 191 +++++++++++++++++++++++++++++++++++ sim/simx/mem/cache.cpp | 24 ++++- sim/simx/mem/cache.h | 4 + sim/simx/processor.cpp | 88 +++++++++++++++- sim/simx/processor.h | 13 +++ sim/simx/processor_impl.h | 18 ++++ sim/simx/types.h | 5 +- sw/runtime/simx/vortex.cpp | 34 +++++++ 12 files changed, 432 insertions(+), 5 deletions(-) create mode 100644 sim/simx/cp_llc_port.h diff --git a/ci/testcases/regression.yaml b/ci/testcases/regression.yaml index 9238aa4c88..a6042df1ef 100644 --- a/ci/testcases/regression.yaml +++ b/ci/testcases/regression.yaml @@ -355,6 +355,17 @@ tests: app: flush_coherence args: -n2048 -r16 -c -x configs: -DVX_CFG_EXT_DXA_ENABLE +# The command processor's DMA as a client of the shared L2: host transfers +# are timed through the L2, and each launch waits for the DMA's fence. +- id: model_parity-dogfood-cp-llc + check: model_parity + via: blackbox + app: dogfood + args: -n16 -xbar -xgbar + configs: -DVX_CFG_CP_LLC_CLIENT=1 + shape: + cores: 4 + l2cache: true # The shared L2 keeps its lines across the L1-only launch flushes. - id: model_parity-flush-chain check: model_parity diff --git a/sim/simx/cluster.cpp b/sim/simx/cluster.cpp index adf1f221fc..a2c83774b9 100644 --- a/sim/simx/cluster.cpp +++ b/sim/simx/cluster.cpp @@ -82,7 +82,7 @@ class Cluster::Impl { log2ceil(VX_CFG_L2_NUM_WAYS), // A log2ceil(VX_CFG_L2_NUM_BANKS), // B VX_CFG_XLEN, // address bits - VX_CFG_L2_NUM_REQS, // request size + L2_CACHE_REQS, // request size VX_CFG_L2_MEM_PORTS, // memory ports VX_CFG_L2_WRITEBACK, // write-back false, // write response @@ -97,6 +97,7 @@ class Cluster::Impl { true, // core-response register (behind the bypass layer) 0, // read_merge VX_CFG_L2_MSHR_LINES, // mshr_lines + CP_ON_L2 != 0, // write_around }); @@ -580,6 +581,18 @@ class Cluster::Impl { l2cache_->flush_begin(l1_only); } + void l2_keep_lines() { + l2cache_->keep_lines(); + } + + SimChannel& cp_mem_req_in() { + return l2cache_->core_req_in.at(L2_CP_IDX); + } + + SimChannel& cp_mem_rsp_out() { + return l2cache_->core_rsp_out.at(L2_CP_IDX); + } + bool l2_flush_done() const { return l2cache_->flush_done(); } @@ -743,6 +756,18 @@ void Cluster::l2_flush_begin(bool l1_only) { impl_->l2_flush_begin(l1_only); } +void Cluster::l2_keep_lines() { + impl_->l2_keep_lines(); +} + +SimChannel& Cluster::cp_mem_req_in() { + return impl_->cp_mem_req_in(); +} + +SimChannel& Cluster::cp_mem_rsp_out() { + return impl_->cp_mem_rsp_out(); +} + bool Cluster::l2_flush_done() const { return impl_->l2_flush_done(); } diff --git a/sim/simx/cluster.h b/sim/simx/cluster.h index 1bcc518427..e47453af1d 100644 --- a/sim/simx/cluster.h +++ b/sim/simx/cluster.h @@ -143,6 +143,13 @@ class Cluster : public SimObject { bool rtcache_flush_done() const; #endif void l2_flush_begin(bool l1_only); + // The L2, when it is the shared last-level cache, keeps its lines into the + // next launch. + void l2_keep_lines(); + // The command processor DMA's client slot on this cluster's L2, valid only + // under CP_ON_L2 (the L2 is the LLC). + SimChannel& cp_mem_req_in(); + SimChannel& cp_mem_rsp_out(); bool l2_flush_done() const; #ifdef VX_CFG_EXT_RASTER_ENABLE diff --git a/sim/simx/constants.h b/sim/simx/constants.h index 64a61c14d2..fabb7e3925 100644 --- a/sim/simx/constants.h +++ b/sim/simx/constants.h @@ -102,6 +102,21 @@ inline constexpr uint32_t VX_CFG_L2_NUM_REQS = VX_CFG_L2_DEMAND_REQS; inline constexpr uint32_t VX_CFG_L3_NUM_REQS = VX_CFG_NUM_CLUSTERS * VX_CFG_L2_MEM_PORTS; #endif +// With VX_CFG_CP_ON_LLC the command processor's DMA is one more client of the +// shared last-level cache: the cluster L2 when it is the LLC of one cluster, +// the L3 otherwise. The slot does not size the banks. +inline constexpr uint32_t CP_ON_L2 = ((VX_CFG_CP_ON_LLC != 0) + && (VX_CFG_NUM_CLUSTERS == 1) + && (VX_CFG_L2_ENABLED != 0) + && (VX_CFG_L3_ENABLED == 0)) ? 1 : 0; +inline constexpr uint32_t CP_ON_L3 = ((VX_CFG_CP_ON_LLC != 0) && (VX_CFG_L3_ENABLED != 0)) ? 1 : 0; +inline constexpr uint32_t L2_CP_IDX = VX_CFG_L2_NUM_REQS; +inline constexpr uint32_t L3_CP_IDX = VX_CFG_L3_NUM_REQS; +inline constexpr uint32_t L2_CACHE_REQS = VX_CFG_L2_NUM_REQS + CP_ON_L2; +inline constexpr uint32_t L3_CACHE_REQS = VX_CFG_L3_NUM_REQS + CP_ON_L3; +// one last-level cache word per DMA request +inline constexpr uint32_t CP_MEM_DATA_SIZE = (CP_ON_L2 != 0) ? VX_CFG_L1_LINE_SIZE : VX_CFG_L2_SECTOR_SIZE; + inline constexpr uint32_t PER_ISSUE_WARPS = VX_CFG_NUM_WARPS / VX_CFG_ISSUE_WIDTH; inline constexpr uint32_t ISSUE_WIS_BITS = log2ceil(PER_ISSUE_WARPS); diff --git a/sim/simx/cp_llc_port.h b/sim/simx/cp_llc_port.h new file mode 100644 index 0000000000..a298216046 --- /dev/null +++ b/sim/simx/cp_llc_port.h @@ -0,0 +1,191 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include "types.h" +#include "constants.h" +#include "mem_block_pool.h" +#include "debug.h" + +namespace vortex { + +// The command processor's DMA port into the last-level cache's client slot. +// Requests in the IO aperture carry the IO attribute and bypass the cache; a +// write flagged no-allocate writes around it on a miss. A launch that follows +// DMA writes first sends an L1-only flush token behind them, which the +// last-level cache answers once its inputs have drained. +class CpLlcPort : public SimObject { +public: + using Ptr = std::shared_ptr; + + SimChannel MemReqOut; + SimChannel MemRspIn; + + // one 4 KB DMA burst of reads in flight + static constexpr uint32_t kReadSlots = 4096 / CP_MEM_DATA_SIZE; + + CpLlcPort(const SimContext& ctx, const char* name) + : SimObject(ctx, name) + , MemReqOut(this) + , MemRspIn(this) + , slots_(kReadSlots) + {} + + // Queue one LLC word of a transfer; `src` is null for a read into `dst`. + void push(uint64_t addr, const uint8_t* src, uint8_t* dst, uint32_t offset, + uint32_t size, bool noalloc) { + MemReq req; + req.addr = addr; + req.op = src ? MemOp::ST : MemOp::LD; + req.flags.io = (get_addr_type(addr) == AddrType::IO); + req.flags.noalloc = src && noalloc; + // the word's bytes sit at its offset in the memory block + uint32_t boff = uint32_t(addr % VX_CFG_MEM_BLOCK_SIZE) + offset; + if (src) { + req.data = make_mem_block(); + std::memset(req.data->data(), 0, req.data->size()); + std::memcpy(req.data->data() + boff, src, size); + req.byteen = ((size < 64) ? ((1ull << size) - 1) : ~0ull) << boff; + } else { + uint32_t woff = uint32_t(addr % VX_CFG_MEM_BLOCK_SIZE); + req.byteen = ((CP_MEM_DATA_SIZE < 64) ? ((1ull << CP_MEM_DATA_SIZE) - 1) : ~0ull) << woff; + } + queue_.push_back({req, dst, boff, size}); + this->tick_wake(); + } + + // Every queued request has entered the port, and every read has returned. + bool idle() const { + return queue_.empty() && reads_ == 0 && !fencing_; + } + + // DMA writes entered the port since the last fence. + bool writes_pending() const { + return wr_pending_; + } + + // Send the launch fence behind the pending writes. + void fence() { + fencing_ = true; + fence_sent_ = false; + this->tick_wake(); + } + +protected: + void on_reset() { + queue_.clear(); + free_slots_.clear(); + for (uint32_t i = 0; i < kReadSlots; ++i) { + free_slots_.push_back(kReadSlots - 1 - i); + } + reads_ = 0; + wr_pending_ = false; + fencing_ = false; + fence_sent_ = false; + } + + void on_tick() { + if (!MemRspIn.empty()) { + auto& rsp = MemRspIn.peek(); + DT(3, this->name() << " rsp: " << rsp); + if (rsp.tag == kFenceTag) { + fencing_ = false; + wr_pending_ = false; + } else { + uint32_t slot = rsp.tag >> 1; + auto& rd = slots_.at(slot); + std::memcpy(rd.dst, rsp.data->data() + rd.offset, rd.size); + free_slots_.push_back(slot); + --reads_; + } + MemRspIn.pop(); + } + if (fencing_) { + if (!fence_sent_) { + MemReq token; + token.flags.flush = 1; + token.flags.flush_l1 = 1; + token.tag = kFenceTag; + if (MemReqOut.try_send(token, 1)) { + DT(3, this->name() << " fence: " << token); + fence_sent_ = true; + } + } + return; + } + if (queue_.empty()) { + if (reads_ == 0 && MemRspIn.empty()) { + this->tick_sleep(); + } + return; + } + auto& entry = queue_.front(); + bool is_read = !entry.req.is_write(); + if (is_read && free_slots_.empty()) { + return; + } + MemReq req = entry.req; + uint32_t slot = 0; + if (is_read) { + slot = free_slots_.back(); + req.tag = slot << 1; + } else { + req.tag = 0; + } + if (!MemReqOut.try_send(req, 1)) { + return; + } + DT(3, this->name() << " req: " << req); + if (is_read) { + free_slots_.pop_back(); + slots_.at(slot) = {entry.dst, entry.offset, entry.size}; + ++reads_; + } else { + wr_pending_ = true; + } + queue_.pop_front(); + } + +private: + static constexpr uint32_t kFenceTag = 1; + + struct entry_t { + MemReq req; + uint8_t* dst; + uint32_t offset; + uint32_t size; + }; + + struct read_slot_t { + uint8_t* dst = nullptr; + uint32_t offset = 0; + uint32_t size = 0; + }; + + std::deque queue_; + std::vector slots_; + std::vector free_slots_; + uint32_t reads_ = 0; + bool wr_pending_ = false; + bool fencing_ = false; + bool fence_sent_ = false; + + friend class SimObject; +}; + +} diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index cdfe079bdc..14fee5673c 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -1941,7 +1941,17 @@ class CacheBank : public SimObject { } } else { // Cache miss. - if (bank_req.write && !config_.write_back) { + if (config_.write_around && bank_req.write && bank_req.flags.noalloc + && !mshr_.lookup(set_id, addr_tag, sector_id)) { + // A no-allocate write miss with no fill pending on its line goes to + // memory as is: nothing is allocated or fetched. + if (!this->emitWritethrough(set_id, addr_tag, sector_id, + bank_req.addr, bank_req.hart_id, bank_req.uuid, + bank_req.data, bank_req.byteen, + need_core_rsp(bank_req), bank_req.req_tag)) { + return; + } + } else if (bank_req.write && !config_.write_back) { // Write-through write miss: forward store to memory and respond // immediately. No fill / no MSHR allocation (this store doesn't // wait for a line). If a fill for this line is already in flight @@ -3002,6 +3012,14 @@ class Cache::Impl { } } + void keep_lines() { + if (config_.bypass || !config_.shared_llc) return; + keep_lines_ = true; + for (auto &bank : banks_) { + bank->flush_begin(true); + } + } + bool flush_done() const { if (config_.bypass) return true; for (const auto &bank : banks_) { @@ -3100,3 +3118,7 @@ void Cache::flush_begin(bool l1_only) { bool Cache::flush_done() const { return impl_->flush_done(); } + +void Cache::keep_lines() { + impl_->keep_lines(); +} diff --git a/sim/simx/mem/cache.h b/sim/simx/mem/cache.h index 1bd4e925f3..ae8b783dd9 100644 --- a/sim/simx/mem/cache.h +++ b/sim/simx/mem/cache.h @@ -50,6 +50,7 @@ class Cache : public SimObject { bool core_out_buf = false; // registered core-response port uint8_t read_merge = 0; // merge same-word reads within groups of this many adjacent ports (0: off) uint16_t mshr_lines = 0; // MSHR distinct {line, sector} entries (0: one per request) + bool write_around = false; // a no-allocate write miss goes to memory (write-back only) }; struct PerfStats { @@ -93,6 +94,9 @@ class Cache : public SimObject { void flush_begin(bool l1_only = false); bool flush_done() const; + // A shared last-level cache keeps its lines into the next launch. + void keep_lines(); + protected: void on_reset(); void on_tick(); diff --git a/sim/simx/processor.cpp b/sim/simx/processor.cpp index aa318ab00a..d0a8a8c6e8 100644 --- a/sim/simx/processor.cpp +++ b/sim/simx/processor.cpp @@ -104,7 +104,7 @@ ProcessorImpl::ProcessorImpl() log2ceil(VX_CFG_L3_NUM_WAYS), // A log2ceil(VX_CFG_L3_NUM_BANKS), // B VX_CFG_XLEN, // address bits - VX_CFG_L3_NUM_REQS, // request size + L3_CACHE_REQS, // request size VX_CFG_L3_MEM_PORTS, // memory ports VX_CFG_L3_WRITEBACK, // write-back false, // write response @@ -119,6 +119,7 @@ ProcessorImpl::ProcessorImpl() true, // core-response register (behind the bypass layer) 0, // read_merge VX_CFG_L3_MSHR_LINES, // mshr_lines + CP_ON_L3 != 0, // write_around } ); @@ -171,6 +172,18 @@ ProcessorImpl::ProcessorImpl() } #endif + // The CP DMA's client slot on the shared last-level cache. + if constexpr (CP_ON_L2 != 0 || CP_ON_L3 != 0) { + cp_llc_port_ = CpLlcPort::Create("cp-llc-port"); + if constexpr (CP_ON_L2 != 0) { + cp_llc_port_->MemReqOut.bind(&clusters_.at(0)->cp_mem_req_in()); + clusters_.at(0)->cp_mem_rsp_out().bind(&cp_llc_port_->MemRspIn); + } else { + cp_llc_port_->MemReqOut.bind(&l3cache_->core_req_in.at(L3_CP_IDX)); + l3cache_->core_rsp_out.at(L3_CP_IDX).bind(&cp_llc_port_->MemRspIn); + } + } + // connect L3 memory interfaces for (uint32_t i = 0; i < VX_CFG_L3_MEM_PORTS; ++i) { l3cache_->mem_req_out.at(i).bind(&memsim_->mem_req_in.at(i)); @@ -287,6 +300,7 @@ void ProcessorImpl::flush_caches(bool l1_only) { } int ProcessorImpl::run() { + this->cp_mem_fence(); this->reset(); kmu_->start(); this->forward_delegated_launch(); @@ -326,6 +340,61 @@ int ProcessorImpl::run() { return exitcode; } +void ProcessorImpl::cp_mem_write(uint64_t addr, const void* src, uint64_t size, bool noalloc) { + this->cp_mem_transfer(addr, static_cast(src), nullptr, size, noalloc); +} + +void ProcessorImpl::cp_mem_read(uint64_t addr, void* dst, uint64_t size) { + this->cp_mem_transfer(addr, nullptr, static_cast(dst), size, false); +} + +void ProcessorImpl::cp_mem_transfer(uint64_t addr, const uint8_t* src, uint8_t* dst, uint64_t size, bool noalloc) { + __assert(cp_llc_port_, "no CP port on the last-level cache"); + if (!platform_reset_) { + this->reset(); + } + while (size != 0) { + uint32_t offset = uint32_t(addr % CP_MEM_DATA_SIZE); + uint32_t n = uint32_t(std::min(size, CP_MEM_DATA_SIZE - offset)); + cp_llc_port_->push(addr - offset, src, dst, offset, n, noalloc); + addr += n; + size -= n; + if (src) { + src += n; + } + if (dst) { + dst += n; + } + } + while (!cp_llc_port_->idle()) { + SimPlatform::instance().tick(); + } + this->llc_keep_lines(); +} + +void ProcessorImpl::cp_mem_fence() { + if (!cp_llc_port_) { + return; + } + if (cp_llc_port_->writes_pending()) { + cp_llc_port_->fence(); + } + // The launch resets the model, so a transfer's fills land first: memory + // holds a read in flight without a channel packet. + while (!cp_llc_port_->idle() || !SimPlatform::instance().idle() || perf_mem_pending_reads_ != 0) { + SimPlatform::instance().tick(); + } + this->llc_keep_lines(); +} + +void ProcessorImpl::llc_keep_lines() { + if constexpr (CP_ON_L2 != 0) { + clusters_.at(0)->l2_keep_lines(); + } else { + l3cache_->keep_lines(); + } +} + bool ProcessorImpl::mmu_fault_pending() const { #ifdef VX_CFG_VM_ENABLE return dev_ptw_->fault_info().valid; @@ -346,6 +415,7 @@ void ProcessorImpl::forward_delegated_launch() { void ProcessorImpl::reset() { SimPlatform::instance().reset(); + platform_reset_ = true; perf_mem_reads_ = 0; perf_mem_writes_ = 0; perf_mem_latency_ = 0; @@ -493,6 +563,22 @@ void Processor::reset() { impl_->reset(); } +void Processor::cp_mem_write(uint64_t addr, const void* src, uint64_t size, bool noalloc) { + impl_->cp_mem_write(addr, src, size, noalloc); +} + +void Processor::cp_mem_read(uint64_t addr, void* dst, uint64_t size) { + impl_->cp_mem_read(addr, dst, size); +} + +bool Processor::cp_llc_client() const { + return (CP_ON_L2 != 0) || (CP_ON_L3 != 0); +} + +uint64_t Processor::cp_llc_size() const { + return (CP_ON_L3 != 0) ? VX_CFG_L3_SIZE : VX_CFG_L2_SIZE; +} + void Processor::start_kmu() { impl_->kmu().start(); } diff --git a/sim/simx/processor.h b/sim/simx/processor.h index 66081ebbd2..13bd99aa40 100644 --- a/sim/simx/processor.h +++ b/sim/simx/processor.h @@ -59,6 +59,19 @@ class Processor { int dcr_read(uint32_t addr, uint32_t tag, uint32_t* value); + // Device-memory access through the command processor's port into the + // last-level cache, when the configuration has one (VX_CFG_CP_ON_LLC). A + // write returns once every request entered the port; the next launch is + // ordered behind it. A read returns with its data. + void cp_mem_write(uint64_t addr, const void* src, uint64_t size, bool noalloc); + + void cp_mem_read(uint64_t addr, void* dst, uint64_t size); + + // The command processor's DMA is a client of the last-level cache, of + // this size in bytes. + bool cp_llc_client() const; + uint64_t cp_llc_size() const; + private: ProcessorImpl* impl_; }; diff --git a/sim/simx/processor_impl.h b/sim/simx/processor_impl.h index 50ecc05106..052d06f8f4 100644 --- a/sim/simx/processor_impl.h +++ b/sim/simx/processor_impl.h @@ -19,6 +19,7 @@ #include "cluster.h" #include "kmu.h" #include "gbar_unit.h" +#include "cp_llc_port.h" namespace vortex { @@ -75,12 +76,27 @@ class ProcessorImpl { // each level's evictions reach the next before that level itself flushes. void flush_caches(bool l1_only); + // Device-memory access through the command processor's port into the + // last-level cache (CP_ON_L2 / CP_ON_L3). A write returns once every + // request entered the port, a read once its data returned. + void cp_mem_write(uint64_t addr, const void* src, uint64_t size, bool noalloc); + void cp_mem_read(uint64_t addr, void* dst, uint64_t size); + private: // A grid-less launch is a delegated draw launch: the KMU walks no CTAs and // the frame kick is forwarded to every cluster's raster engine instead. void forward_delegated_launch(); + // Queue a transfer on the CP port and tick until it has entered, or for a + // read returned; the last-level cache keeps what it took into the next launch. + void cp_mem_transfer(uint64_t addr, const uint8_t* src, uint8_t* dst, uint64_t size, bool noalloc); + + // A launch that follows DMA writes waits for the port's fence. + void cp_mem_fence(); + + void llc_keep_lines(); + // True once the device walker has latched a page fault. bool mmu_fault_pending() const; @@ -97,6 +113,8 @@ class ProcessorImpl { #endif RAM* ram_ = nullptr; // functional backing store (set by attach_ram) Cache::Ptr l3cache_; + CpLlcPort::Ptr cp_llc_port_; // CP DMA client of the last-level cache + bool platform_reset_ = false; uint64_t perf_mem_reads_; uint64_t perf_mem_writes_; uint64_t perf_mem_latency_; diff --git a/sim/simx/types.h b/sim/simx/types.h index f3f9b4cf84..595891d401 100644 --- a/sim/simx/types.h +++ b/sim/simx/types.h @@ -450,6 +450,7 @@ struct MemFlags { uint32_t amo_unsigned : 1; // bit 3: MIN/MAX signedness (1=unsigned) uint32_t flush : 1; // bit 4: flush token (a fence) uint32_t flush_l1 : 1; // bit 5: L1-only flush token: stops at the shared LLC + uint32_t noalloc : 1; // bit 6: a write that misses a write-around cache goes to memory #ifdef VX_CFG_EXT_DXA_ENABLE // DXA completion sideband — populated only on DXA's LMEM-DMA writes. // notify_bar_id carries the RAW (kernel-encoded) bar handle: @@ -459,8 +460,8 @@ struct MemFlags { // be wide enough to hold (bar_no << 8) | (MAX_CTAS_per_core - 1). // For NUM_BARRIERS = 8 and NUM_WARPS ≤ 16, that's at most 11 bits; // 16 bits leaves headroom for larger configs. - uint32_t dxa_notify_done : 1; // bit 6 - uint32_t dxa_notify_bar_id : 16; // bits 7..22 + uint32_t dxa_notify_done : 1; // bit 7 + uint32_t dxa_notify_bar_id : 16; // bits 8..23 #endif }; }; diff --git a/sw/runtime/simx/vortex.cpp b/sw/runtime/simx/vortex.cpp index d72ba8d60d..b2120aaf0b 100644 --- a/sw/runtime/simx/vortex.cpp +++ b/sw/runtime/simx/vortex.cpp @@ -33,6 +33,7 @@ #include #include +#include #include #include #include @@ -150,6 +151,13 @@ class vx_device { std::memcpy(dst, hp, bytes); return; } + if (processor_.cp_llc_client()) { + if (future_.valid()) { + future_.wait(); + } + processor_.cp_mem_read(addr, dst, bytes); + return; + } ram_.read(static_cast(dst), addr, bytes); }; h.dram_write = [this](uint64_t addr, const void* src, std::size_t bytes) { @@ -157,8 +165,34 @@ class vx_device { std::memcpy(hp, src, bytes); return; } + if (processor_.cp_llc_client()) { + if (future_.valid()) { + future_.wait(); + } + processor_.cp_mem_write(addr, src, bytes, false); + return; + } ram_.write(static_cast(src), addr, bytes); }; + if (processor_.cp_llc_client()) { + // The CP DMA is a client of the last-level cache: transfers stay + // coherent with it, and those of at least half its size write around it. + h.dma_write = [this](uint64_t addr, const void* src, std::size_t bytes, bool no_alloc) { + if (void* hp = host_region_ptr(addr)) { + std::memcpy(hp, src, bytes); + return; + } + if (future_.valid()) { + future_.wait(); + } + processor_.cp_mem_write(addr, src, bytes, no_alloc); + }; + h.dma_llc_client = true; + h.dma_no_alloc_min = processor_.cp_llc_size() / 2; + if (const char* v = std::getenv("VORTEX_CP_NO_ALLOC_MIN")) { + h.dma_no_alloc_min = std::strtoull(v, nullptr, 0); + } + } h.vortex_dcr_write = [this](uint32_t addr, uint32_t value) { // Wait for any background processor_.run() to finish before writing DCR. if (future_.valid()) { From fd151efaa9fb735c9897828178f91d37fcdc5e3d Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 19 Oct 2026 18:29:25 -0700 Subject: [PATCH 063/124] Build VX_csr_fpu only when the F extension is enabled VX_csr_fpu included VX_fpu_define.vh unconditionally. The DUT source lists preprocess every file under hw/rtl/core, and the tcu DUT is built with VX_CFG_EXT_F_DISABLE and no FPU include path, so both synthesis gates failed on tcu before synthesis. Both instantiation sites already sit under VX_CFG_EXT_F_ENABLE; the module now does too, like the FPU include in VX_csr_data and VX_sfu_unit. fpga_gate tcu: PASS (300 MHz, LUT unchanged). asic_gate tcu: PASS (839 MHz, area unchanged). Co-Authored-By: Claude Opus 5.5 --- hw/rtl/core/VX_csr_fpu.sv | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hw/rtl/core/VX_csr_fpu.sv b/hw/rtl/core/VX_csr_fpu.sv index 7e3889c40f..d19d308279 100644 --- a/hw/rtl/core/VX_csr_fpu.sv +++ b/hw/rtl/core/VX_csr_fpu.sv @@ -11,6 +11,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +`ifdef VX_CFG_EXT_F_ENABLE + `include "VX_define.vh" `include "VX_fpu_define.vh" @@ -101,3 +103,5 @@ module VX_csr_fpu import VX_gpu_pkg::*, VX_fpu_pkg::*; #( end endmodule + +`endif From cc7bb17ccf8305a3220e9ad31903424dda529943 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 19 Oct 2026 18:30:54 -0700 Subject: [PATCH 064/124] rtlsim: ask the processor whether the CP DMA is an LLC client The rtlsim runtime included VX_config.h to decide whether the command processor's DMA goes through the last-level cache and to size the write-around threshold, which ci/check_config_boundary.sh forbids: the hardware build configuration is private to RTL and the simulators. The rtlsim Processor now answers both: cp_llc_client() (VX_CFG_CP_ON_LLC) and cp_llc_size() (the L3 size with an L3, else the L2 size). The runtime routes its DMA hooks and sets DMA_LLC_CLIENT and the no-alloc threshold from them, as the SimX runtime does. rtlsim dogfood -n16 at 4 cores + L2 with VX_CFG_CP_LLC_CLIENT=1 is cycle-identical (353,584) and passes all 22 tests. All ci/check_*.sh pass. Co-Authored-By: Claude Opus 5.5 --- sim/rtlsim/processor.cpp | 8 +++++ sim/rtlsim/processor.h | 5 +++ sw/runtime/rtlsim/vortex.cpp | 61 ++++++++++++++++++------------------ 3 files changed, 44 insertions(+), 30 deletions(-) diff --git a/sim/rtlsim/processor.cpp b/sim/rtlsim/processor.cpp index 92724c8923..327a131bb3 100644 --- a/sim/rtlsim/processor.cpp +++ b/sim/rtlsim/processor.cpp @@ -658,6 +658,14 @@ void Processor::cp_mem_read(uint64_t addr, void* dst, uint64_t size) { impl_->cp_mem_read(addr, static_cast(dst), size); } +bool Processor::cp_llc_client() const { + return VX_CFG_CP_ON_LLC != 0; +} + +uint64_t Processor::cp_llc_size() const { + return VX_CFG_L3_ENABLED ? VX_CFG_L3_SIZE : VX_CFG_L2_SIZE; +} + void Processor::drain() { impl_->drain(); } \ No newline at end of file diff --git a/sim/rtlsim/processor.h b/sim/rtlsim/processor.h index 9fe3c2f68e..05e0b4605c 100644 --- a/sim/rtlsim/processor.h +++ b/sim/rtlsim/processor.h @@ -51,6 +51,11 @@ class Processor { void cp_mem_read(uint64_t addr, void* dst, uint64_t size); + // The command processor's DMA is a client of the last-level cache, of + // this size in bytes. + bool cp_llc_client() const; + uint64_t cp_llc_size() const; + private: class Impl; diff --git a/sw/runtime/rtlsim/vortex.cpp b/sw/runtime/rtlsim/vortex.cpp index 0376f03e77..7f32b98eb9 100644 --- a/sw/runtime/rtlsim/vortex.cpp +++ b/sw/runtime/rtlsim/vortex.cpp @@ -23,7 +23,6 @@ // dram hooks dereference it directly (the sim runs in-process). // ============================================================================ -#include #include #include @@ -144,46 +143,48 @@ class vx_device { std::memcpy(dst, hp, bytes); return; } - #if VX_CFG_CP_ON_LLC - if (future_.valid()) future_.wait(); - processor_.cp_mem_read(addr, dst, bytes); - #else + if (processor_.cp_llc_client()) { + if (future_.valid()) { + future_.wait(); + } + processor_.cp_mem_read(addr, dst, bytes); + return; + } ram_.read(static_cast(dst), addr, bytes); - #endif }; h.dram_write = [this](uint64_t addr, const void* src, std::size_t bytes) { if (void* hp = host_region_ptr(addr)) { std::memcpy(hp, src, bytes); return; } - #if VX_CFG_CP_ON_LLC - if (future_.valid()) future_.wait(); - processor_.cp_mem_write(addr, src, bytes, false); - #else - ram_.write(static_cast(src), addr, bytes); - #endif - }; - #if VX_CFG_CP_ON_LLC - // The CP DMA is a client of the last-level cache: transfers stay coherent - // with it, and those of at least half its size write around it. - h.dma_write = [this](uint64_t addr, const void* src, std::size_t bytes, bool no_alloc) { - if (void* hp = host_region_ptr(addr)) { - std::memcpy(hp, src, bytes); + if (processor_.cp_llc_client()) { + if (future_.valid()) { + future_.wait(); + } + processor_.cp_mem_write(addr, src, bytes, false); return; } - if (future_.valid()) future_.wait(); - processor_.cp_mem_write(addr, src, bytes, no_alloc); + ram_.write(static_cast(src), addr, bytes); }; - h.dma_llc_client = true; - #if VX_CFG_L3_ENABLED - h.dma_no_alloc_min = VX_CFG_L3_SIZE / 2; - #else - h.dma_no_alloc_min = VX_CFG_L2_SIZE / 2; - #endif - if (const char* v = std::getenv("VORTEX_CP_NO_ALLOC_MIN")) { - h.dma_no_alloc_min = std::strtoull(v, nullptr, 0); + if (processor_.cp_llc_client()) { + // The CP DMA is a client of the last-level cache: transfers stay + // coherent with it, and those of at least half its size write around it. + h.dma_write = [this](uint64_t addr, const void* src, std::size_t bytes, bool no_alloc) { + if (void* hp = host_region_ptr(addr)) { + std::memcpy(hp, src, bytes); + return; + } + if (future_.valid()) { + future_.wait(); + } + processor_.cp_mem_write(addr, src, bytes, no_alloc); + }; + h.dma_llc_client = true; + h.dma_no_alloc_min = processor_.cp_llc_size() / 2; + if (const char* v = std::getenv("VORTEX_CP_NO_ALLOC_MIN")) { + h.dma_no_alloc_min = std::strtoull(v, nullptr, 0); + } } - #endif h.vortex_dcr_write = [this](uint32_t addr, uint32_t value) { // Wait for any background processor_.run() to finish so dcr_write // does not race the Verilator state. From 54754389c2fd51fd6ccf166c9725f10557c0315c Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 19 Oct 2026 18:30:54 -0700 Subject: [PATCH 065/124] simx: drop the extra register on the LSU's path to commit In the RTL a load response crosses three registers between the LSU block and the commit arbiter: the block's response buffer (rsp_buf), its result arbiter (rsp_arb, OUT_BUF 3) and the unit's lane gather (OUT_BUF 3). SimX counted the first two in the LSU's output channel and then added the two-cycle register that the commit stage applies to every unit, which for the ALU stands for its result buffer and its PE switch. Loads therefore reached commit one cycle late: uncontended, the response-to-commit distance was 4 cycles in SimX and 3 in rtlsim, at the default and the 8-thread shapes. The commit stage now adds only the lane gather for the LSU. SimX vs rtlsim: parity-sgemm-mc 1,220,795 -> 1,152,088 (rtlsim 1,102,494; 10.73% -> 4.50%) parity-sgemm-fw2 553,250 -> 543,888 (rtlsim 528,299; 4.72% -> 2.95%) parity-sgemm 794,010 -> 793,955 (rtlsim 799,610; 0.71%) parity-raycast 1,497,605 -> 1,494,492 (rtlsim 1,498,691; 0.28%) parity-vecadd 367,367 -> 365,304 (rtlsim 368,717; 0.93%) parity-sgemv 893,874 -> 901,575 (rtlsim 883,260; 2.07%) Co-Authored-By: Claude Opus 5.5 --- sim/simx/core.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index 57b7a1c555..8b5682742e 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -877,8 +877,11 @@ class Core::Impl { auto trace = fu_out.peek(); uint32_t iw = trace->wid % VX_CFG_ISSUE_WIDTH; auto& arb_in = *commit_queues_.at(iw).at(fu); - // A unit registers its result once more before the commit arbiter. - if (arb_in.try_send(trace, 2)) { + // The unit's output registers ahead of the commit arbiter that its + // output channel does not already carry: the LSU's response buffer and + // result arbiter are in its channel, leaving its lane gather. + uint32_t commit_delay = (fu == (uint32_t)FUType::LSU) ? 1 : 2; + if (arb_in.try_send(trace, commit_delay)) { // Release the warp as soon as its stalling instruction's result leaves // the functional unit — the branch target / fence / warp-control is // resolved at that point. The release lands in stalled_warps so the From 14f944c668c6a8f43bbd9a544e27e46ec50d3d1c Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 19 Oct 2026 18:57:26 -0700 Subject: [PATCH 066/124] dxa: stream GMEM responses straight to smem_wr again VX_dxa_gmem_req buffered every GMEM response in a MAX_OUTSTANDING-deep line FIFO with a bypass, so a busy smem_wr would not hold the socket memory response path it shares with the L1 fills. The FIFO costs eight cache lines of storage plus a line-wide bypass mux, and it only engages when smem_wr's own line registers (fill buffer, one captured line, the deferred last line) are full. rtlsim, buffered -> direct (all PASS): dxa_copy -d1 (NT4) 6827 -> 6827 dxa_copy -d3, 2 cores 17649 -> 17649 dxa_copy_mcast 3381 -> 3381 sgemm2_dxa 40390 -> 40390 sgemm2_dxa, 2 cores 20731 -> 20731 sgemm2_dxa_mcast 37578 -> 37578 sgemm_tcu_wg_dxa_mcast 128x128x64 NT16 78882 -> 78882 sgemm_tcu_wg_dxa 128x128x64 (IW4 NT32) 32312 -> 32823 (+1.6%) dxa_kmajor_check PASS -> PASS Only the tiled-scatter WGMMA producer ever fills smem_wr's line registers, and there the buffer saves 1.6%. It does not prevent a deadlock: smem_wr only waits on LMEM writes, and they never depend on an L1 fill. Synthesis, dxa DUT: fpga_gate: LUT 11157 -> 10294 (baseline 10102), LUTRAM 938 -> 638, Fmax 291 -> 301 MHz asic_gate: cell area 14919 -> 12449 um^2 (baseline 12449) The responses go back to smem_wr directly. SimX now models this. smem_wr takes one line per cycle, and a GMEM response goes ahead of a pending OOB line. A line is taken only into a free line register. A response that smem_wr cannot take stays on the memory response path, so the responses queued behind it wait too. Before, SimX accepted every response at once. On the wgmma-dxa parity config the SimX/rtlsim gap drops from 5.2% (30638 vs 32312) to 4.0% (31519 vs 32823). Co-Authored-By: Claude Opus 5.5 --- hw/rtl/dxa/VX_dxa_gmem_req.sv | 50 ++------------------ sim/simx/dxa/dxa_core.cpp | 88 +++++++++++++++++++++++++---------- 2 files changed, 68 insertions(+), 70 deletions(-) diff --git a/hw/rtl/dxa/VX_dxa_gmem_req.sv b/hw/rtl/dxa/VX_dxa_gmem_req.sv index 929b02c3e7..43e467ab46 100644 --- a/hw/rtl/dxa/VX_dxa_gmem_req.sv +++ b/hw/rtl/dxa/VX_dxa_gmem_req.sv @@ -15,9 +15,9 @@ // // Manages a slot-table-indexed pool of MAX_OUTSTANDING in-flight tags. On // each accept, allocates a free slot and writes per-tag metadata. GMEM -// responses (out-of-order from the L1) queue in a response buffer with one -// entry per slot and drain to smem_wr via the `sw_*` channel. OOB entries -// skip the bus and are presented to smem_wr via an oob_pending[] bitvector. +// responses (out-of-order from the L1) are streamed directly to smem_wr +// via the `sw_*` channel — no rsp_buf BRAM. OOB entries skip the bus and +// are presented to smem_wr via an oob_pending[] bitvector. // // Tag reuse is per-slot via a busy[] bitvector and outstanding counter; // release order is independent of issue order (OOO drain in smem_wr). @@ -242,48 +242,6 @@ module VX_dxa_gmem_req import VX_gpu_pkg::*, VX_dxa_pkg::*; #( end end - VX_mem_bus_if #( - .DATA_SIZE (GMEM_BYTES), - .TAG_WIDTH (GMEM_TAG_WIDTH) - ) mem_bus_raw (); - - assign mem_bus_raw.req_valid = mem_bus_w.req_valid; - assign mem_bus_raw.req_data = mem_bus_w.req_data; - assign mem_bus_w.req_ready = mem_bus_raw.req_ready; - - // smem_wr takes a scattered line one element per cycle. The memory - // response path is shared with the L1 fills, so a response must never - // wait on it: one entry per slot means the buffer is never full. A - // response passes straight through when the buffer is empty and smem_wr - // is ready. - wire rsp_buf_empty, rsp_buf_full; - wire [GMEM_DATAW + GMEM_TAG_WIDTH-1:0] rsp_buf_data; - wire rsp_bypass = rsp_buf_empty && mem_bus_w.rsp_ready; - - VX_fifo_queue #( - .DATAW (GMEM_DATAW + GMEM_TAG_WIDTH), - .DEPTH (MAX_OUTSTANDING) - ) rsp_buf ( - .clk (clk), - .reset (reset), - .push (mem_bus_raw.rsp_valid && ~rsp_bypass), - .pop (~rsp_buf_empty && mem_bus_w.rsp_ready), - .data_in (mem_bus_raw.rsp_data), - .data_out (rsp_buf_data), - .empty (rsp_buf_empty), - .full (rsp_buf_full), - `UNUSED_PIN (alm_empty), - `UNUSED_PIN (alm_full), - `UNUSED_PIN (size) - ); - - `RUNTIME_ASSERT(~(mem_bus_raw.rsp_valid && ~rsp_bypass && rsp_buf_full), - ("%t: GMEM response buffer overflow", $time)) - - assign mem_bus_raw.rsp_ready = 1'b1; - assign mem_bus_w.rsp_valid = mem_bus_raw.rsp_valid || ~rsp_buf_empty; - assign mem_bus_w.rsp_data = rsp_buf_empty ? mem_bus_raw.rsp_data : rsp_buf_data; - VX_mem_bus_slice #( .DATA_SIZE (GMEM_BYTES), .TAG_WIDTH (GMEM_TAG_WIDTH), @@ -292,7 +250,7 @@ module VX_dxa_gmem_req import VX_gpu_pkg::*, VX_dxa_pkg::*; #( ) gmem_out_slice ( .clk (clk), .reset (reset), - .bus_in_if (mem_bus_raw), + .bus_in_if (mem_bus_w), .bus_out_if (gmem_bus_if) ); diff --git a/sim/simx/dxa/dxa_core.cpp b/sim/simx/dxa/dxa_core.cpp index 3ad602dd27..7a227118a7 100644 --- a/sim/simx/dxa/dxa_core.cpp +++ b/sim/simx/dxa/dxa_core.cpp @@ -97,6 +97,8 @@ class DxaCore::Impl { struct InflightSlot { bool allocated = false; bool rsp_arrived = false; + bool oob_pending = false; // OOB line waiting for its turn into smem_wr + bool deferred = false; // last line parked until the others drain uint64_t ready_cycle = 0; // first cycle the line can be written to SMEM uint64_t free_cycle = 0; // first cycle the slot can carry a new read LineWork work; // captured at allocation @@ -126,7 +128,7 @@ class DxaCore::Impl { std::vector work_list; uint32_t ag_idx = 0; // next entry to issue (gmem_req) - // gmem_req → rsp_buf → smem_wr inflight bookkeeping. + // gmem_req → smem_wr inflight bookkeeping. std::array inflight; std::deque issued_order; // outstanding tags in issue order uint32_t drain_slot = UINT32_MAX; // slot currently draining @@ -217,27 +219,15 @@ class DxaCore::Impl { void tick() { ++cycle_; - // Drain in reverse-pipeline order (smem_wr → rsp_buf → gmem_req → - // addr_gen → setup → req-arb) so each stage's tick reads what its - // upstream produced earlier this cycle, mimicking forward stage flow. + // Drain in reverse-pipeline order (smem_wr → gmem_req → addr_gen → + // setup → req-arb) so each stage's tick reads what its upstream produced + // earlier this cycle, mimicking forward stage flow. - // 1) Drain GMEM responses → mark inflight slots arrived. - // gmem_arb_ already routes responses back to RspOut[worker_id] (its - // input index) and restores the user's tag (the per-worker slot id). + // 1) Hand one line per worker to smem_wr. gmem_arb_ already routes + // responses back to RspOut[worker_id] (its input index) and restores the + // user's tag (the per-worker slot id). for (uint32_t worker_id = 0; worker_id < workers_.size(); ++worker_id) { - auto& ch = gmem_arb_->RspOut.at(worker_id); - // One response per port per cycle (each worker has its own RspOut port). - if (!ch.empty()) { - auto& rsp = ch.peek(); - uint32_t slot_id = uint32_t(rsp.tag) & (VX_CFG_DXA_MAX_INFLIGHT - 1); - auto& w = workers_[worker_id]; - if (slot_id < w.inflight.size() && w.inflight[slot_id].allocated) { - w.inflight[slot_id].rsp_arrived = true; - w.inflight[slot_id].ready_cycle = cycle_ + kSmemWrLatency; - w.inflight[slot_id].rsp_data = rsp.data; - } - ch.pop(); - } + capture_line(workers_[worker_id], gmem_arb_->RspOut.at(worker_id)); } // 2) Tick workers (smem_wr drain → gmem_req issue). @@ -564,6 +554,52 @@ class DxaCore::Impl { } } + // ── smem_wr capture: one line per cycle into its line registers ─────── + // A GMEM response has priority over a pending OOB line. smem_wr holds one + // line being written, one captured behind it, and the transfer's last line + // while others are outstanding. A response it cannot take stays on the + // memory response path and holds the responses queued behind it. + void capture_line(Worker& w, SimChannel& ch) { + bool pend_busy = false; + bool defer_busy = false; + uint32_t oob_slot = UINT32_MAX; + for (uint32_t i = 0; i < w.inflight.size(); ++i) { + const auto& s = w.inflight[i]; + if (s.rsp_arrived && i != w.drain_slot) { + (s.deferred ? defer_busy : pend_busy) = true; + } + if (s.oob_pending && oob_slot == UINT32_MAX) { + oob_slot = i; + } + } + + uint32_t slot = oob_slot; + if (!ch.empty()) { + slot = uint32_t(ch.peek().tag) & (VX_CFG_DXA_MAX_INFLIGHT - 1); + if (!w.inflight[slot].allocated) { + ch.pop(); + return; + } + } + if (slot == UINT32_MAX) { + return; + } + + auto& s = w.inflight[slot]; + bool defer = s.work.last && (w.issued_order.size() > 1); + if (defer ? defer_busy : pend_busy) { + return; + } + if (!ch.empty()) { + s.rsp_data = ch.peek().data; + ch.pop(); + } + s.oob_pending = false; + s.rsp_arrived = true; + s.deferred = defer; + s.ready_cycle = cycle_ + kSmemWrLatency; + } + // ── gmem_req: issue MemReqs with available inflight slots ──────────── void tick_worker_gmem_req(Worker& w) { // One AGU step per cycle: advance ag_idx by at most one, issuing at most @@ -592,12 +628,13 @@ class DxaCore::Impl { return; } - // OOB lines skip the GMEM request entirely; we synthesize an immediate - // arrival (rsp data left null — smem_wr will use cfill). + // OOB lines skip the GMEM request entirely; smem_wr takes them in a cycle + // with no GMEM response (rsp data left null — smem_wr will use cfill). if (lw.oob) { w.inflight[slot].allocated = true; - w.inflight[slot].rsp_arrived = true; - w.inflight[slot].ready_cycle = cycle_ + kSmemWrLatency; + w.inflight[slot].rsp_arrived = false; + w.inflight[slot].oob_pending = true; + w.inflight[slot].deferred = false; w.inflight[slot].rsp_data.reset(); w.inflight[slot].work = lw; w.issued_order.push_back(slot); @@ -619,6 +656,8 @@ class DxaCore::Impl { w.inflight[slot].allocated = true; w.inflight[slot].rsp_arrived = false; + w.inflight[slot].oob_pending = false; + w.inflight[slot].deferred = false; w.inflight[slot].rsp_data.reset(); w.inflight[slot].work = lw; w.issued_order.push_back(slot); @@ -816,6 +855,7 @@ class DxaCore::Impl { // Slot done — release. s.allocated = false; s.rsp_arrived = false; + s.deferred = false; s.free_cycle = cycle_ + 1; s.rsp_data.reset(); auto it = std::find(w.issued_order.begin(), w.issued_order.end(), slot); From 9e9500226b3c86e41da5e7bf8f2045e0b7e8d6ae Mon Sep 17 00:00:00 2001 From: tinebp Date: Tue, 20 Oct 2026 02:50:30 -0700 Subject: [PATCH 067/124] Include VX_define.vh before VX_csr_fpu's F-extension guard fd151efaa put the VX_CFG_EXT_F_ENABLE guard above the include of VX_define.vh, which is where the macro gets defined. Verilator compiles all files as one unit and saw the macro from earlier files, so rtlsim and xrtsim never noticed. Vivado project mode reads the file on its own, dropped the module, and the U55C xclbin build failed at elaboration (module 'VX_csr_fpu' not found). Include the config first, as VX_csr_data does. With this, the default U55C xclbin builds, and sgemm, raycast, dogfood and printf pass on the board. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/core/VX_csr_fpu.sv | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hw/rtl/core/VX_csr_fpu.sv b/hw/rtl/core/VX_csr_fpu.sv index d19d308279..1aefbf8642 100644 --- a/hw/rtl/core/VX_csr_fpu.sv +++ b/hw/rtl/core/VX_csr_fpu.sv @@ -11,9 +11,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +`include "VX_define.vh" + `ifdef VX_CFG_EXT_F_ENABLE -`include "VX_define.vh" `include "VX_fpu_define.vh" // Per-warp FP CSRs (frm + fflags) for one residue class of warps modulo WID_STRIDE, From 360cd99e2ba9e2e97d973d8c3199d961ee419a2f Mon Sep 17 00:00:00 2001 From: tinebp Date: Tue, 20 Oct 2026 07:12:20 -0700 Subject: [PATCH 068/124] hw/unittest: test the CP DMA port's launch fence VX_cp_llc_port holds a launch that follows DMA writes until an L1-only flush token sent behind them is answered by the last-level cache. No system test sees this: removing the fence passed flush_coherence and dogfood on rtlsim and on xrtsim at 4 cores + L2 with VX_CFG_CP_LLC_CLIENT=1, also with DBG_FLUSH_STALL. The runtime submits every DMA (vx_enqueue_write, the launch args, the QMD) as its own ring command and polls for its retire, and command batches carry only DCR writes and launches. So a host round trip always separates the last DMA write from the next launch, and the write has long drained by then. This unit test plays the cache on the port's slot. Requests enter an in-order input queue that the test can stall, and a request takes effect only when it leaves that queue. In every cycle that start_out is high, every DMA write the port accepted must already have taken effect. The test covers: - a launch with no write passes straight through; - reads do not arm the fence; - a launch behind a write held at the cache input waits for that write and for the token's answer, and the port takes no DMA request in the meantime; - the token is a read with the flush attribute and slot tag bit 0, and its answer does not reach the DMA; - the fence clears after the launch; - bursts of writes into a cache that accepts and drains intermittently; - the IO and no-allocate attributes. Known-bad, both caught by the start_out check: fence removed (start_out = start_in) FAIL at cycle 31 start released when the token is issued FAIL at cycle 96 tip PASSED (XLEN 32 and 64, NDEBUG and DEBUG builds) The hw unittest CI lane only builds the tests, so a new lane runs this one when VX_cp_llc_port.sv or the test changes. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/unittest.yaml | 7 + hw/unittest/Makefile | 3 + hw/unittest/cp_llc_port/Makefile | 27 ++ hw/unittest/cp_llc_port/VX_cp_llc_port_top.sv | 102 +++++ hw/unittest/cp_llc_port/main.cpp | 393 ++++++++++++++++++ 5 files changed, 532 insertions(+) create mode 100644 hw/unittest/cp_llc_port/Makefile create mode 100644 hw/unittest/cp_llc_port/VX_cp_llc_port_top.sv create mode 100644 hw/unittest/cp_llc_port/main.cpp diff --git a/ci/testcases/unittest.yaml b/ci/testcases/unittest.yaml index 89c8ba08e8..12619a17cd 100644 --- a/ci/testcases/unittest.yaml +++ b/ci/testcases/unittest.yaml @@ -31,6 +31,13 @@ tests: via: script run: "make -C hw/unittest run-tcu-dsp" touches: [hw/rtl/tcu, hw/unittest/tcu_fedp] + # The command processor's DMA port into the last-level cache: a launch behind + # DMA writes must wait for them to reach the cache. No runtime path launches + # right behind a DMA write, so no system test observes this ordering. + - id: hw-cp-llc-port + via: script + run: "make -C hw/unittest/cp_llc_port run" + touches: [hw/rtl/mem/VX_cp_llc_port.sv, hw/unittest/cp_llc_port] # Texel blend (VX_tex_lerp, both forms the sampler uses) against the C model # the SW sampler and SimX share, and the exact Vulkan f/256 blend, over every # input. diff --git a/hw/unittest/Makefile b/hw/unittest/Makefile index 0387f2a267..fb703fade9 100644 --- a/hw/unittest/Makefile +++ b/hw/unittest/Makefile @@ -21,6 +21,7 @@ all: $(MAKE) -C afu_reset_seq $(MAKE) -C cp_axi_path $(MAKE) -C cp_dma + $(MAKE) -C cp_llc_port $(MAKE) -C cp_core $(MAKE) -C fma_unit $(MAKE) -C fdiv_unit @@ -52,6 +53,7 @@ run: $(MAKE) -C afu_reset_seq run $(MAKE) -C cp_axi_path run $(MAKE) -C cp_dma run + $(MAKE) -C cp_llc_port run $(MAKE) -C cp_core run $(MAKE) -C fma_unit run $(MAKE) -C fdiv_unit run @@ -147,6 +149,7 @@ clean: $(MAKE) -C cp_axil_regfile clean $(MAKE) -C cp_axi_path clean $(MAKE) -C cp_dma clean + $(MAKE) -C cp_llc_port clean $(MAKE) -C cp_core clean $(MAKE) -C fma_unit clean $(MAKE) -C fdiv_unit clean diff --git a/hw/unittest/cp_llc_port/Makefile b/hw/unittest/cp_llc_port/Makefile new file mode 100644 index 0000000000..f37bc1700d --- /dev/null +++ b/hw/unittest/cp_llc_port/Makefile @@ -0,0 +1,27 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := cp_llc_port + +RTL_DIR := $(VORTEX_HOME)/hw/rtl +DPI_DIR := $(VORTEX_HOME)/hw/dpi + +SRC_DIR := $(VORTEX_HOME)/hw/unittest/$(PROJECT) + +CXXFLAGS := -I$(SRC_DIR) -I$(VORTEX_HOME)/hw/unittest/common -I$(SW_COMMON_DIR) +CXXFLAGS += -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw + +SRCS := $(SRC_DIR)/main.cpp + +DBG_TRACE_FLAGS := + +RTL_PKGS := $(RTL_DIR)/VX_gpu_pkg.sv $(RTL_DIR)/VX_trace_pkg.sv + +RTL_INCLUDE := -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw -I$(RTL_DIR) -I$(DPI_DIR) \ + -I$(RTL_DIR)/libs -I$(VORTEX_HOME)/hw/unittest/$(PROJECT) +RTL_INCLUDE += -I$(RTL_DIR)/interfaces -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm -I$(RTL_DIR)/fpu \ + -I$(RTL_DIR)/core -I$(RTL_DIR)/cp + +TOP := VX_cp_llc_port_top + +include ../common.mk diff --git a/hw/unittest/cp_llc_port/VX_cp_llc_port_top.sv b/hw/unittest/cp_llc_port/VX_cp_llc_port_top.sv new file mode 100644 index 0000000000..ad339b8342 --- /dev/null +++ b/hw/unittest/cp_llc_port/VX_cp_llc_port_top.sv @@ -0,0 +1,102 @@ +// Copyright © 2019-2023 +// Licensed under the Apache License, Version 2.0. + +`include "VX_define.vh" + +// Test wrapper around VX_cp_llc_port: the last-level cache side of the +// port's VX_mem_bus_if is flattened so the C++ harness can play the cache. +module VX_cp_llc_port_top import VX_gpu_pkg::*; #( + parameter DATA_SIZE = 64, + parameter TAG_WIDTH = 4, + parameter SLOT_TAG_WIDTH = UUID_WIDTH + 1 + TAG_WIDTH, + parameter ADDR_WIDTH = `VX_CFG_MEM_ADDR_WIDTH - `CLOG2(DATA_SIZE) +) ( + input wire clk, + input wire reset, + + input wire req_valid, + input wire req_rw, + input wire req_noalloc, + input wire [ADDR_WIDTH-1:0] req_addr, + input wire [DATA_SIZE*8-1:0] req_data, + input wire [DATA_SIZE-1:0] req_byteen, + input wire [TAG_WIDTH-1:0] req_tag, + output wire req_ready, + + output wire rsp_valid, + output wire [DATA_SIZE*8-1:0] rsp_data, + output wire [TAG_WIDTH-1:0] rsp_tag, + input wire rsp_ready, + + input wire start_in, + output wire start_out, + output wire busy, + + output wire mem_req_valid, + output wire mem_req_rw, + output wire [ADDR_WIDTH-1:0] mem_req_addr, + output wire [DATA_SIZE*8-1:0] mem_req_data, + output wire [DATA_SIZE-1:0] mem_req_byteen, + output wire mem_req_flush, + output wire mem_req_io, + output wire mem_req_noalloc, + output wire [SLOT_TAG_WIDTH-1:0] mem_req_tag, + input wire mem_req_ready, + + input wire mem_rsp_valid, + input wire [DATA_SIZE*8-1:0] mem_rsp_data, + input wire [SLOT_TAG_WIDTH-1:0] mem_rsp_tag, + output wire mem_rsp_ready +); + VX_mem_bus_if #( + .DATA_SIZE (DATA_SIZE), + .TAG_WIDTH (SLOT_TAG_WIDTH), + .ADDR_WIDTH (ADDR_WIDTH) + ) mem_bus_if (); + + VX_cp_llc_port #( + .DATA_SIZE (DATA_SIZE), + .SLOT_TAG_WIDTH (SLOT_TAG_WIDTH), + .TAG_WIDTH (TAG_WIDTH), + .ADDR_WIDTH (ADDR_WIDTH) + ) dut ( + .clk (clk), + .reset (reset), + .req_valid (req_valid), + .req_rw (req_rw), + .req_noalloc (req_noalloc), + .req_addr (req_addr), + .req_data (req_data), + .req_byteen (req_byteen), + .req_tag (req_tag), + .req_ready (req_ready), + .rsp_valid (rsp_valid), + .rsp_data (rsp_data), + .rsp_tag (rsp_tag), + .rsp_ready (rsp_ready), + .start_in (start_in), + .start_out (start_out), + .busy (busy), + .mem_bus_if (mem_bus_if) + ); + + assign mem_req_valid = mem_bus_if.req_valid; + assign mem_req_rw = mem_bus_if.req_data.rw; + assign mem_req_addr = mem_bus_if.req_data.addr; + assign mem_req_data = mem_bus_if.req_data.data; + assign mem_req_byteen = mem_bus_if.req_data.byteen; + mem_bus_attr_t mem_req_attr; + assign mem_req_attr = mem_bus_attr_t'(mem_bus_if.req_data.attr); + assign mem_req_flush = mem_req_attr.is_flush; + assign mem_req_io = mem_req_attr.is_addr_io; + assign mem_req_noalloc = mem_req_attr.is_noalloc; + `UNUSED_VAR (mem_req_attr) + assign mem_req_tag = mem_bus_if.req_data.tag; + assign mem_bus_if.req_ready = mem_req_ready; + + assign mem_bus_if.rsp_valid = mem_rsp_valid; + assign mem_bus_if.rsp_data.data = mem_rsp_data; + assign mem_bus_if.rsp_data.tag = mem_rsp_tag; + assign mem_rsp_ready = mem_bus_if.rsp_ready; + +endmodule diff --git a/hw/unittest/cp_llc_port/main.cpp b/hw/unittest/cp_llc_port/main.cpp new file mode 100644 index 0000000000..5975b84561 --- /dev/null +++ b/hw/unittest/cp_llc_port/main.cpp @@ -0,0 +1,393 @@ +// Copyright © 2019-2023 +// Licensed under the Apache License, Version 2.0. + +// Verilator unit test for VX_cp_llc_port, the command processor's DMA port +// into the last-level cache. +// +// The harness plays the cache: requests enter an in-order input queue that the +// test can stall, and a request takes effect only when it leaves that queue, so +// an accepted DMA write is not yet visible while it sits in the port's buffer +// or in the queue. Every cycle the launch leaves the port (start_out), all DMA +// writes the port accepted must already have taken effect. +// +// Coverage: +// 1. A launch with no DMA write behind it passes straight through. +// 2. DMA reads return data and tag, and do not arm the launch fence. +// 3. A launch behind a DMA write held at the cache input waits for that +// write, then for the fence token's answer; the token is a read with the +// flush attribute and slot tag bit 0 set, and its answer is not forwarded +// to the DMA. The port takes no DMA request while the fence is pending. +// 4. The fence clears: a following launch with no new write passes through. +// 5. Several writes into a cache that accepts and drains in bursts. +// 6. Request attributes: IO aperture, and no-allocate on writes only. + +#include "vl_simulator.h" +#include "VVX_cp_llc_port_top.h" +#include +#include +#include +#include +#include + +#ifndef TRACE_START_TIME +#define TRACE_START_TIME 0ull +#endif +#ifndef TRACE_STOP_TIME +#define TRACE_STOP_TIME -1ull +#endif + +static uint64_t timestamp = 0; +static bool trace_en = false; +double sc_time_stamp() { return timestamp; } +bool sim_trace_enabled() { return trace_en; } +void sim_trace_enable(bool e) { trace_en = e; } + +#define EXPECT(cond, msg) do { \ + if (!(cond)) { \ + std::fprintf(stderr, "FAIL %s:%d (cycle %llu): %s\n", __FILE__, __LINE__, \ + (unsigned long long)cycles, msg); \ + std::exit(1); \ + } \ +} while (0) + +using Top = VVX_cp_llc_port_top; + +static constexpr uint32_t LINE_SIZE = 64; +static constexpr uint32_t TAG_BITS = 4; +static constexpr uint32_t DATA_LINE = 0x100000; // above the IO aperture +static constexpr uint32_t IO_LINE = VX_MEM_IO_BASE_ADDR / LINE_SIZE; + +static_assert(VX_MEM_IO_BASE_ADDR % LINE_SIZE == 0, "IO base must be line aligned"); +static_assert(DATA_LINE >= VX_MEM_IO_END_ADDR / LINE_SIZE, "data line must be outside the IO aperture"); + +struct CacheReq { + bool rw; + bool flush; + uint32_t addr; + uint32_t data; + uint64_t tag; +}; + +struct CacheRsp { + uint32_t data; + uint64_t tag; +}; + +static uint64_t cycles = 0; + +// The last-level cache seen through the port's slot. +struct Cache { + bool accept = true; // input ready + uint32_t period = 0; // with duty: ready duty of every period cycles + uint32_t duty = 0; + uint32_t service = 1; // cycles per request + std::deque input; + std::deque rsps; + std::unordered_map mem; + uint32_t busy_cycles = 0; + uint64_t writes_applied = 0; + + // Retire at most one request from the head of the input queue. + void step() { + if (input.empty()) { + return; + } + if (++busy_cycles < service) { + return; + } + busy_cycles = 0; + CacheReq r = input.front(); + input.pop_front(); + if (r.flush) { + rsps.push_back({0, r.tag}); + } else if (r.rw) { + mem[r.addr] = r.data; + ++writes_applied; + } else { + auto it = mem.find(r.addr); + rsps.push_back({it != mem.end() ? it->second : 0, r.tag}); + } + } +}; + +struct Bench { + vl_simulator sim; + uint64_t tick = 0; + Cache cache; + + uint64_t dma_writes = 0; // DMA writes the port accepted + uint64_t fence_tokens = 0; + uint64_t starts = 0; + std::deque dma_rsps; + + Bench() { + idle_inputs(); + tick = sim.reset(tick); + } + + void idle_inputs() { + sim->req_valid = 0; + sim->req_rw = 0; + sim->req_noalloc = 0; + sim->req_addr = 0; + sim->req_byteen = 0; + sim->req_tag = 0; + sim->rsp_ready = 1; + sim->start_in = 0; + sim->mem_req_ready = 0; + sim->mem_rsp_valid = 0; + sim->mem_rsp_tag = 0; + for (uint32_t i = 0; i < LINE_SIZE / 4; ++i) { + sim->req_data[i] = 0; + sim->mem_rsp_data[i] = 0; + } + } + + // One clock: present the cache's outputs, sample every handshake on the + // settled inputs, then advance the edge and the cache model. + void cycle() { + sim->mem_req_ready = cache.accept && (cache.period == 0 || (cycles % cache.period) < cache.duty); + sim->mem_rsp_valid = !cache.rsps.empty(); + if (!cache.rsps.empty()) { + sim->mem_rsp_tag = cache.rsps.front().tag; + sim->mem_rsp_data[0] = cache.rsps.front().data; + } + sim->eval(); + + if (sim->start_out) { + ++starts; + EXPECT(cache.writes_applied == dma_writes, + "launch left the port before its DMA writes reached the cache"); + } + if (sim->req_valid && sim->req_ready && sim->req_rw) { + ++dma_writes; + } + if (sim->mem_req_valid && sim->mem_req_ready) { + CacheReq r; + r.rw = sim->mem_req_rw; + r.flush = sim->mem_req_flush; + r.addr = sim->mem_req_addr; + r.data = sim->mem_req_data[0]; + r.tag = sim->mem_req_tag; + if (r.flush) { + ++fence_tokens; + EXPECT(!r.rw, "fence token must be a read"); + EXPECT((r.tag & 1) == 1, "fence token must carry slot tag bit 0"); + } else { + EXPECT((r.tag & 1) == 0, "DMA request must clear slot tag bit 0"); + } + cache.input.push_back(r); + } + if (sim->mem_rsp_valid && sim->mem_rsp_ready) { + CacheRsp rsp = cache.rsps.front(); + cache.rsps.pop_front(); + if ((rsp.tag & 1) == 0) { + dma_rsps.push_back({rsp.data, (rsp.tag >> 1) & ((1u << TAG_BITS) - 1)}); + } + } + if (sim->rsp_valid && sim->rsp_ready) { + EXPECT(!dma_rsps.empty(), "the port forwarded a response the DMA never asked for"); + EXPECT(sim->rsp_tag == dma_rsps.front().tag, "DMA response tag mismatch"); + EXPECT(sim->rsp_data[0] == dma_rsps.front().data, "DMA response data mismatch"); + dma_rsps.pop_front(); + } + + tick = sim.step(tick, 2); + ++cycles; + cache.step(); + } + + // Hold one DMA request until the port takes it. + void dma_req(bool rw, uint32_t addr, uint32_t data, uint32_t tag, bool noalloc = false) { + sim->req_valid = 1; + sim->req_rw = rw; + sim->req_noalloc = noalloc; + sim->req_addr = addr; + sim->req_data[0] = data; + sim->req_byteen = ~0ull; + sim->req_tag = tag; + for (int i = 0;; ++i) { + EXPECT(i < 1000, "DMA request never accepted"); + sim->eval(); + bool fire = sim->req_ready; + cycle(); + if (fire) { + break; + } + } + sim->req_valid = 0; + } + + // Pulse start_in for one cycle; returns whether start_out fired with it. + bool launch() { + sim->start_in = 1; + uint64_t before = starts; + cycle(); + sim->start_in = 0; + return starts != before; + } + + void run(int n) { + for (int i = 0; i < n; ++i) { + cycle(); + } + } + + // Wait for the launch to leave the port; returns the cycles waited. + int wait_start(int limit) { + uint64_t before = starts; + for (int i = 0; i < limit; ++i) { + cycle(); + if (starts != before) { + return i + 1; + } + } + EXPECT(false, "launch never left the port"); + return -1; + } +}; + +static void test_launch_without_writes(Bench& b) { + b.cache.accept = true; + b.run(4); + EXPECT(!b.sim->busy, "port busy at idle"); + EXPECT(b.launch(), "launch with no DMA write must pass through in the same cycle"); + b.run(8); + EXPECT(b.fence_tokens == 0, "no fence token expected without a DMA write"); + EXPECT(!b.sim->busy, "port busy after a pass-through launch"); +} + +static void test_reads(Bench& b) { + b.cache.accept = true; + b.cache.mem[DATA_LINE + 7] = 0xcafe0007; + b.cache.mem[DATA_LINE + 9] = 0xcafe0009; + b.dma_req(false, DATA_LINE + 7, 0, 3); + b.dma_req(false, DATA_LINE + 9, 0, 5); + b.run(10); + EXPECT(b.dma_rsps.empty(), "DMA read responses missing"); + uint64_t tokens = b.fence_tokens; + EXPECT(b.launch(), "DMA reads must not hold the launch"); + b.run(4); + EXPECT(b.fence_tokens == tokens, "DMA reads must not arm the fence"); +} + +static void test_fence_holds_launch(Bench& b) { + constexpr int STALL = 64; + b.cache.accept = false; + b.dma_req(true, DATA_LINE + 1, 0x11111111, 1); + EXPECT(b.cache.writes_applied < b.dma_writes, "write should still be held at the cache input"); + + uint64_t starts = b.starts; + EXPECT(!b.launch(), "launch passed the port with a DMA write still in front of the cache"); + for (int i = 0; i < STALL; ++i) { + b.sim->eval(); + EXPECT(b.sim->busy, "port must be busy while the fence is pending"); + EXPECT(!b.sim->req_ready, "port must take no DMA request while the fence is pending"); + b.cycle(); + } + EXPECT(b.starts == starts, "launch left the port while the cache input was stalled"); + + // Release the cache slowly: the write, then the token, each take 6 cycles. + uint64_t tokens = b.fence_tokens; + b.cache.accept = true; + b.cache.service = 6; + int waited = b.wait_start(200); + EXPECT(waited >= 2 * 6, "launch left before the cache answered the fence token"); + EXPECT(b.fence_tokens == tokens + 1, "exactly one fence token expected"); + EXPECT(b.dma_rsps.empty(), "fence answer must not reach the DMA"); + b.run(2); + EXPECT(!b.sim->busy, "port still busy after the fence answered"); + b.cache.service = 1; +} + +static void test_fence_clears(Bench& b) { + uint64_t tokens = b.fence_tokens; + EXPECT(b.launch(), "a launch with no new DMA write must pass through after a fence"); + b.run(8); + EXPECT(b.fence_tokens == tokens, "fence re-armed without a new DMA write"); +} + +static void test_write_bursts(Bench& b) { + for (int round = 0; round < 8; ++round) { + b.cache.service = 1 + (round % 4); + b.cache.accept = true; + b.cache.period = 5 + round; + b.cache.duty = 1 + (round % 3); + for (int i = 0; i < 6; ++i) { + b.dma_req(true, DATA_LINE + 32 * round + i, 0x20000000 + 16 * round + i, i & 0xf); + } + b.cache.period = 0; + b.cache.accept = (round % 2) == 0; + uint64_t starts = b.starts; + bool passed = b.launch(); + EXPECT(!passed || b.cache.writes_applied == b.dma_writes, "launch passed undrained writes"); + if (!passed) { + b.run(round * 5); + b.cache.accept = true; + for (int i = 0; b.starts == starts; ++i) { + EXPECT(i < 500, "launch never left the port"); + b.cycle(); + } + } + b.run(4); + EXPECT(!b.sim->busy, "port busy after a burst launch"); + } + // Every written line reads back through the port. + b.cache.accept = true; + b.cache.service = 1; + b.dma_req(false, DATA_LINE + 32 * 7 + 5, 0, 2); + b.run(10); + EXPECT(b.cache.mem[DATA_LINE + 32 * 7 + 5] == 0x20000000 + 16 * 7 + 5, "burst write lost"); +} + +static void test_attributes(Bench& b) { + b.cache.accept = false; + struct Case { bool rw; uint32_t addr; bool noalloc; bool io; bool exp_noalloc; }; + const Case cases[] = { + {true, IO_LINE, false, true, false}, + {false, IO_LINE, true, true, false}, + {true, DATA_LINE + 3, true, false, true }, + {false, DATA_LINE + 3, true, false, false}, + }; + for (const auto& c : cases) { + b.sim->req_valid = 1; + b.sim->req_rw = c.rw; + b.sim->req_noalloc = c.noalloc; + b.sim->req_addr = c.addr; + b.sim->req_tag = 1; + b.sim->eval(); + EXPECT(b.sim->req_ready, "port buffer should be empty"); + b.cycle(); + b.sim->req_valid = 0; + b.sim->eval(); + EXPECT(b.sim->mem_req_valid, "request missing at the cache input"); + EXPECT(b.sim->mem_req_io == c.io, "IO attribute mismatch"); + EXPECT(b.sim->mem_req_noalloc == c.exp_noalloc, "no-allocate attribute mismatch"); + EXPECT(!b.sim->mem_req_flush, "DMA request carries the flush attribute"); + b.cache.accept = true; + b.cycle(); + b.cache.accept = false; + } + b.cache.accept = true; + b.run(10); + // Leave the fence armed by these writes discharged. + b.launch(); + b.wait_start(100); + b.run(4); +} + +int main(int argc, char** argv) { + Verilated::commandArgs(argc, argv); + Bench b; + + test_launch_without_writes(b); + test_reads(b); + test_fence_holds_launch(b); + test_fence_clears(b); + test_write_bursts(b); + test_attributes(b); + + EXPECT(b.cache.input.empty() && b.cache.rsps.empty(), "cache traffic left over"); + std::printf("PASSED\n"); + return 0; +} From 64f4268018454cd4b3ab5323f8f314a10c0f779b Mon Sep 17 00:00:00 2001 From: tinebp Date: Tue, 20 Oct 2026 07:12:20 -0700 Subject: [PATCH 069/124] docs: select the FPGA target with blackbox.sh --target blackbox.sh sets TARGET to empty before it parses its options and passes TARGET to make only from --target. The FPGA docs exported TARGET=hw instead, so the run fell back to the xrt driver's default xrtsim target and failed with "No such file 'vortex_afu.xclbin'". This changes the xrt runs in docs/fpga_setup.md, hw/syn/xilinx/README and the apptainer README, and the FireSim run in docs/simulation.md, to use --target. It also drops the PLATFORM variable from those runs, since nothing on the run path reads it. The blackbox.sh help no longer says --target is for aved only. It lists the targets for xrt, firesim and aved. With --target=hw, sgemm, raycast, dogfood and printf ran on the U55C. Co-Authored-By: Claude Opus 5.5 --- ci/blackbox.sh | 2 +- docs/fpga_setup.md | 12 +++++++----- docs/simulation.md | 3 +-- hw/syn/xilinx/README | 6 +++--- miscs/apptainer/README.md | 2 +- 5 files changed, 13 insertions(+), 12 deletions(-) diff --git a/ci/blackbox.sh b/ci/blackbox.sh index 21215167a0..c541d04c14 100755 --- a/ci/blackbox.sh +++ b/ci/blackbox.sh @@ -27,7 +27,7 @@ show_help() show_usage echo " where" echo "--driver: gpu, simx, rtlsim, opae, xrt, aved" - echo "--target: aved driver only -- avedsim (default), sim, hw" + echo "--target: xrt -- xrtsim (default), hw_emu, hw; firesim -- hw, hw_emu; aved -- avedsim (default), sim, hw" echo "--app: any subfolder test under regression, graphics, mpi, opencl, or hip" echo "--perf: 0=disable, 1=core, 3=icache, 4=dcache, 5=l2cache, 6=l3cache, 7=mem," echo " 11=tcu, 12=raster, 13=tex, 14=om, 15=rtu, 16=dxa" diff --git a/docs/fpga_setup.md b/docs/fpga_setup.md index 0419cfd18d..7701c11383 100644 --- a/docs/fpga_setup.md +++ b/docs/fpga_setup.md @@ -148,20 +148,22 @@ sudo bash -c 'source /opt/xilinx/xrt/setup.sh && \ The [blackbox.sh](./simulation.md) script within the build directory can be used to run a test with Vortex’s xrt driver using the following command: -`FPGA_BIN_DIR= TARGET=hw|hw_emu PLATFORM= ./ci/blackbox.sh --driver=xrt --app=` +`FPGA_BIN_DIR= ./ci/blackbox.sh --driver=xrt --target=hw|hw_emu --app=` + +The target is a `blackbox.sh` option, not an environment variable: `blackbox.sh` resets `TARGET`, so an exported `TARGET=hw` is ignored and the run falls back to the `xrtsim` simulator. For example: -```FPGA_BIN_DIR= hw/syn/xilinx/xrt/build_4c_u280_xilinx_u280_gen3x16_xdma_1_202211_1_hw/bin TARGET=hw PLATFORM=xilinx_u280_gen3x16_xdma_1_202211_1 ./ci/blackbox.sh --driver=xrt --app=demo``` +```FPGA_BIN_DIR= hw/syn/xilinx/xrt/build_4c_u280_xilinx_u280_gen3x16_xdma_1_202211_1_hw/bin ./ci/blackbox.sh --driver=xrt --target=hw --app=demo``` Run sgemm on a U55C `hw` bitstream (host memory enabled per the section above), `-n` sets the matrix size: ```bash -FPGA_BIN_DIR=$(realpath hw/syn/xilinx/xrt//bin) TARGET=hw \ - ./ci/blackbox.sh --driver=xrt --app=sgemm --args="-n128" +FPGA_BIN_DIR=$(realpath hw/syn/xilinx/xrt//bin) \ + ./ci/blackbox.sh --driver=xrt --target=hw --app=sgemm --args="-n128" ``` -The same command with `TARGET=hw_emu` against a `hw_emu` build runs it in emulation +The same command with `--target=hw_emu` against a `hw_emu` build runs it in emulation (use a small `-n`, e.g. `-n16`, since emulation is slow). Synthesis for Intel (Altera) Boards diff --git a/docs/simulation.md b/docs/simulation.md index 8fbc55f94d..7803c072b5 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -29,9 +29,8 @@ For scale, `sgemm -n32` takes ~1228 ms on rtlsim and ~8 ms on an Alveo U55C, and `sgemm -n1024` — over a billion instructions — completes in ~85 s. ```bash -export TARGET=hw export FPGA_BIN_DIR= -./ci/blackbox.sh --driver=firesim --app=sgemm --args="-n1024" +./ci/blackbox.sh --driver=firesim --target=hw --app=sgemm --args="-n1024" ``` The architecture, build flow and design invariants are documented in diff --git a/hw/syn/xilinx/README b/hw/syn/xilinx/README index 7f0c43d38c..0ae222a173 100644 --- a/hw/syn/xilinx/README +++ b/hw/syn/xilinx/README @@ -70,9 +70,9 @@ TARGET=hw PLATFORM=xilinx_u55c_gen3x16_xdma_3_202210_1 VPP_FLAGS="--from_step vp TARGET=hw PLATFORM=xilinx_u55c_gen3x16_xdma_3_202210_1 VPP_FLAGS="--from_step vpl.impl.route_design" make > build.log 2>&1 & # running test -FPGA_BIN_DIR= TARGET=hw_emu ./ci/blackbox.sh --driver=xrt --app=demo -FPGA_BIN_DIR= TARGET=hw ./ci/blackbox.sh --driver=xrt --app=sgemm --args="-n1024" -FPGA_BIN_DIR= XRT_DEVICE_INDEX=1 TARGET=hw ./ci/blackbox.sh --driver=xrt --app=sgemm --args="-n1024" +FPGA_BIN_DIR= ./ci/blackbox.sh --driver=xrt --target=hw_emu --app=demo +FPGA_BIN_DIR= ./ci/blackbox.sh --driver=xrt --target=hw --app=sgemm --args="-n1024" +FPGA_BIN_DIR= XRT_DEVICE_INDEX=1 ./ci/blackbox.sh --driver=xrt --target=hw --app=sgemm --args="-n1024" # build report logs /bin/vortex_afu.xclbin.info diff --git a/miscs/apptainer/README.md b/miscs/apptainer/README.md index 06e6a4e3f2..541db951f7 100644 --- a/miscs/apptainer/README.md +++ b/miscs/apptainer/README.md @@ -105,7 +105,7 @@ Apptainer> pwd Apptainer> make -C runtime/ clean -Apptainer> FPGA_BIN_DIR=/home/vortex/build/hw/syn/xilinx/xrt/test1_xilinx_u50_gen3x16_xdma_5_202210_1_hw/bin TARGET=hw PLATFORM=xilinx_u50_gen3x16_xdma_5_202210_1 ./ci/blackbox.sh --driver=xrt --app=demo +Apptainer> FPGA_BIN_DIR=/home/vortex/build/hw/syn/xilinx/xrt/test1_xilinx_u50_gen3x16_xdma_5_202210_1_hw/bin ./ci/blackbox.sh --driver=xrt --target=hw --app=demo Verify following line being printed: info: device name=xilinx_u50_gen3x16_xdma_base_5, memory_capacity=0x200000000 bytes, memory_banks=32. From c961402531c10f46ca68ee5b635903eec9276437 Mon Sep 17 00:00:00 2001 From: tinebp Date: Tue, 20 Oct 2026 09:29:46 -0700 Subject: [PATCH 070/124] Round-robin the DXA and TCU on the LMEM DMA port The LMEM DMA port is shared by two clients: DXA writes into SMEM and the TCU's WGMMA operand reads out of it. The port gave the DXA fixed priority. A tiled-scatter DXA transfer writes one bank row almost every cycle for about 540 cycles, and during that time the TCU's B-tile reads got no grant. In sgemm_tcu_wg_dxa, a WGMMA batch that had not finished when such a transfer started stalled for about 510 cycles. The stall held up the warpgroup's barrier, so the producer's next DXA request arrived late and the DXA engine sat idle. Whether a batch lost this race came down to a few cycles of timing. The race started to go the wrong way once rtlsim began completing DRAM requests in per-bank arrival order. With that testbench change reverted, the same RTL runs in 31,037 cycles instead of 32,206. Grant the port round-robin so both clients keep moving. This adds one bit of arbiter state and no storage. A TCU read now waits at most one DXA write. Configurations with a single DMA client are unchanged, because the arbiter there is a pass-through. SimX serves its DMA clients round-robin in the same way. It also lets the DXA capture a GMEM line in the same cycle that its smem_wr pend register loads the fill buffer. VX_dxa_smem_wr's pend is free again that cycle, so rtlsim takes a line every two cycles. SimX had captured before draining, which held pend busy one extra cycle and admitted a line only every three cycles. rtlsim cycles, before -> after: wgmma-dxa (IW4 NT32, 128x128x64) 32,823 -> 29,487 (-10.2%) wgmma-sp-dxa (IW4 NT32, 128x128x128) 54,973 -> 48,573 (-11.6%) wgmma-dxa-mcast NT4 358,969 -> 358,897 wgmma-dxa-mcast NT16 78,882 -> 78,840 copy NT4 / NT16 / mcast, wgmma-fp16-ss, wgmma-fedp2k-rs, wgmma-sparse unchanged SimX vs rtlsim: wgmma-dxa 2.29% (was 5.24%), wgmma-sp-dxa 2.52% (was 4.97%), wgmma-dxa-mcast 0.60%. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/core/VX_mem_unit.sv | 3 ++- sim/simx/core.cpp | 4 ++-- sim/simx/dxa/dxa_core.cpp | 17 +++++++++-------- sim/simx/mem/local_mem.cpp | 12 ++++++++---- sim/simx/mem/local_mem.h | 2 +- 5 files changed, 22 insertions(+), 16 deletions(-) diff --git a/hw/rtl/core/VX_mem_unit.sv b/hw/rtl/core/VX_mem_unit.sv index a766858777..17b5d7c665 100644 --- a/hw/rtl/core/VX_mem_unit.sv +++ b/hw/rtl/core/VX_mem_unit.sv @@ -166,6 +166,7 @@ module VX_mem_unit import VX_gpu_pkg::*; #( `ASSIGN_VX_MEM_BUS_IF_EX (dma_arb_in_if[LMEM_DMA_TCU_IDX], tcu_lmem_if, LMEM_DMA_IN_TAG_W, TCU_LMEM_TAG_W, UUID_WIDTH); `endif + // Round-robin, so a DXA write stream cannot starve the TCU's operand reads. VX_mem_bus_arb #( .NUM_INPUTS (LMEM_DMA_INPUTS), .NUM_OUTPUTS (1), @@ -173,7 +174,7 @@ module VX_mem_unit import VX_gpu_pkg::*; #( .TAG_WIDTH (LMEM_DMA_IN_TAG_W), .ATTR_WIDTH (LMEM_DMA_ATTR_W), .ADDR_WIDTH (LMEM_DMA_ADDR_WIDTH), - .ARBITER ("P") + .ARBITER ("R") ) lmem_dma_arb ( .clk (clk), .reset (reset), diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index 8b5682742e..2455e026b1 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -146,7 +146,7 @@ class Core::Impl { // create local memory. // The DXA writes and the TCU tile-buffer reads share the row-wide DMA - // port, DXA first. + // port, round-robin. snprintf(sname, 100, "%s-lmem", name.c_str()); local_mem_ = LocalMem::Create(sname, LocalMem::Config{ (1 << VX_CFG_LMEM_LOG_SIZE), @@ -274,7 +274,7 @@ class Core::Impl { #ifdef VX_CFG_TCU_WGMMA_ENABLE // Bind the TCU tile-buffer subsystem (TcuTbuf) to the LMEM DMA port, - // after the DXA. + // as the client after the DXA. { auto& tbuf = tcu_unit_->tbuf(); uint32_t base = VX_CFG_EXT_DXA_ENABLED * LocalMem::DMA_PORTS; diff --git a/sim/simx/dxa/dxa_core.cpp b/sim/simx/dxa/dxa_core.cpp index 7a227118a7..55ef8786d3 100644 --- a/sim/simx/dxa/dxa_core.cpp +++ b/sim/simx/dxa/dxa_core.cpp @@ -223,17 +223,18 @@ class DxaCore::Impl { // setup → req-arb) so each stage's tick reads what its upstream produced // earlier this cycle, mimicking forward stage flow. - // 1) Hand one line per worker to smem_wr. gmem_arb_ already routes - // responses back to RspOut[worker_id] (its input index) and restores the - // user's tag (the per-worker slot id). + // 1) smem_wr drain, then 2) hand it one line per worker, then 3) issue + // the next GMEM read. A line leaving pend for the fill buffer frees pend + // for a capture in the same cycle. gmem_arb_ already routes responses + // back to RspOut[worker_id] (its input index) and restores the user's tag + // (the per-worker slot id). for (uint32_t worker_id = 0; worker_id < workers_.size(); ++worker_id) { - capture_line(workers_[worker_id], gmem_arb_->RspOut.at(worker_id)); - } - - // 2) Tick workers (smem_wr drain → gmem_req issue). - for (auto& w : workers_) { + auto& w = workers_[worker_id]; if (w.state == WState::RUNNING) { tick_worker_smem_wr(w); + } + capture_line(w, gmem_arb_->RspOut.at(worker_id)); + if (w.state == WState::RUNNING) { tick_worker_gmem_req(w); } } diff --git a/sim/simx/mem/local_mem.cpp b/sim/simx/mem/local_mem.cpp index 9e3dc16884..925bcc3b92 100644 --- a/sim/simx/mem/local_mem.cpp +++ b/sim/simx/mem/local_mem.cpp @@ -54,6 +54,7 @@ class LocalMem::Impl { #endif MemCrossBar::Ptr mem_xbar_; mutable PerfStats perf_stats_; + uint32_t dma_last_grant_ = 0; uint64_t to_local_addr(uint64_t addr) { return bit_getw(addr, 0, addr_bits_-1); @@ -110,6 +111,7 @@ class LocalMem::Impl { void reset() { perf_stats_ = PerfStats(); + dma_last_grant_ = config_.dma_clients ? (config_.dma_clients - 1) : 0; #if VX_CFG_EXT_A_ENABLED amo_unit_.reset(); std::fill(amo_busy_.begin(), amo_busy_.end(), 0); @@ -119,13 +121,14 @@ class LocalMem::Impl { } // Serve one DMA client's row access this cycle and return the banks it - // occupies. Clients are served at fixed priority, and DMA wins every bank - // it touches over the LSU: a read touches all banks, a write those its - // byte enables cover. + // occupies. Clients are served round-robin, starting after the last one + // served, and DMA wins every bank it touches over the LSU: a read touches + // all banks, a write those its byte enables cover. uint64_t dma_tick() { uint32_t num_banks = (1 << config_.B); uint32_t lg2_line_size = log2ceil(config_.line_size); - for (uint32_t c = 0; c < config_.dma_clients; ++c) { + for (uint32_t i = 0; i < config_.dma_clients; ++i) { + uint32_t c = (dma_last_grant_ + 1 + i) % config_.dma_clients; bool pending = false; bool rsp_ready = true; for (uint32_t p = 0; p < DMA_PORTS; ++p) { @@ -178,6 +181,7 @@ class LocalMem::Impl { DT(4, simobject_->name() << "-dma" << c << " req : " << req); in.pop(); } + dma_last_grant_ = c; return banks; } return 0; diff --git a/sim/simx/mem/local_mem.h b/sim/simx/mem/local_mem.h index 2a879974bd..6772c8b04f 100644 --- a/sim/simx/mem/local_mem.h +++ b/sim/simx/mem/local_mem.h @@ -31,7 +31,7 @@ class LocalMem : public SimObject { uint32_t num_reqs; uint32_t B; // log2 number of banks bool write_reponse; - uint32_t dma_clients; // row-wide DMA masters, in priority order + uint32_t dma_clients; // row-wide DMA masters, served round-robin }; struct PerfStats { From 4c0f266ead4a923026cda861122160242ff938a1 Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 21 Oct 2026 07:47:54 -0700 Subject: [PATCH 071/124] Size each slice of a stream arbiter tree by its own requests VX_stream_arb turns an arbiter with more than 1.5x MAX_FANOUT requests into a tree. On the input side, slices of MAX_FANOUT rows feed a join arbiter. On the output side, a fork arbiter feeds the slices. Every slice was wired as if it were full: its select was declared CLOG2(MAX_FANOUT) bits wide, and it got all NUM_OUTPUTS lanes (NUM_INPUTS on the output side). A last slice with fewer rows has a narrower select, so the slice's packed sel_out did not line up with the wire it drove. Verilator rejects that width mismatch. With more than one output, the selects of different outputs also landed at the wrong offsets. A last slice with fewer requests than the other side's width was elaborated as the opposite kind of arbiter, and it delivered requests to the wrong outputs. This happens whenever the requests are not a whole number of slices. One example is VX_CFG_CP_LLC_CLIENT: the command processor's DMA slot sits beside 2^k L1 ports on the shared L2. At 4 cores x 16 L1 ports that gives 65 inputs per bank arbiter, the last slice has one input, and the rtlsim build fails lint (WIDTHTRUNC at VX_stream_arb.sv:85). Each slice is now sized by the rows it actually has. Its select is that wide and is zero-extended into the join's data or the fork's select table. A last slice with fewer requests than the other side keeps only the lanes it can serve, and the join (or fork) gets that many lanes in its last row. Nothing is buffered on a lane that cannot drain. The flat output-side arbiter also stops indexing ready_out past NUM_OUTPUTS when NUM_OUTPUTS is not a multiple of NUM_INPUTS. The missing outputs of the last row never request. Full slices produce the same logic as before. For every parameter set in a sweep where the old module is Verilator-lint clean, Yosys proves the new module sequentially equivalent to it (equiv_make/equiv_simple/equiv_induct after sv2v). The sweep has 748 shapes: NUM_INPUTS 1-40 x NUM_OUTPUTS 1-8 at MAX_FANOUT 4 and 8; 48-130 requests on one side with 1, 2 or 4 on the other, in both directions; and OUT_BUF 0-3 and ARBITER R/P/C/M on the slice shapes. The old module lints clean on 584 of them, and all 584 are proven. The new module also lints clean on the other 164, which the old one could not build. A mutant with a corrupted slice select fails the same check. The synthesis-gate DUTs elaborate only full slices (16x1 in tensor at both gates, and in the 16-thread core at the ASIC gate), so their netlists do not change. hw/unittest/stream_arb drives random valids and output stalls through 21 shapes. They include last slices of one lane, last slices narrower than the other side, last slices that are not a whole number of rows, two-level joins, and both arbiter directions. The test checks that every packet leaves exactly once, on an output its input maps to, with its row as select. Against the old module the test does not build (14 lint warnings). Built anyway with -Wno-fatal, it misroutes packets. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/unittest.yaml | 8 + hw/rtl/libs/VX_stream_arb.sv | 79 ++++++--- hw/unittest/Makefile | 3 + hw/unittest/stream_arb/Makefile | 23 +++ hw/unittest/stream_arb/VX_stream_arb_tb.sv | 179 ++++++++++++++++++++ hw/unittest/stream_arb/VX_stream_arb_top.sv | 63 +++++++ hw/unittest/stream_arb/main.cpp | 62 +++++++ 7 files changed, 391 insertions(+), 26 deletions(-) create mode 100644 hw/unittest/stream_arb/Makefile create mode 100644 hw/unittest/stream_arb/VX_stream_arb_tb.sv create mode 100644 hw/unittest/stream_arb/VX_stream_arb_top.sv create mode 100644 hw/unittest/stream_arb/main.cpp diff --git a/ci/testcases/unittest.yaml b/ci/testcases/unittest.yaml index 12619a17cd..2f3f33d952 100644 --- a/ci/testcases/unittest.yaml +++ b/ci/testcases/unittest.yaml @@ -38,6 +38,14 @@ tests: via: script run: "make -C hw/unittest/cp_llc_port run" touches: [hw/rtl/mem/VX_cp_llc_port.sv, hw/unittest/cp_llc_port] + # Stream arbiters over more than 1.5x MAX_FANOUT requests become trees of + # slices; a last slice narrower than the arbiter's other side must keep every + # request on its own output and select. The system shapes reach only a few of + # these slice shapes. + - id: hw-stream-arb + via: script + run: "make -C hw/unittest/stream_arb run" + touches: [hw/rtl/libs/VX_stream_arb.sv, hw/unittest/stream_arb] # Texel blend (VX_tex_lerp, both forms the sampler uses) against the C model # the SW sampler and SimX share, and the exact Vulkan f/256 blend, over every # input. diff --git a/hw/rtl/libs/VX_stream_arb.sv b/hw/rtl/libs/VX_stream_arb.sv index 9ef71a6cfd..64e4eba814 100644 --- a/hw/rtl/libs/VX_stream_arb.sv +++ b/hw/rtl/libs/VX_stream_arb.sv @@ -50,24 +50,29 @@ module VX_stream_arb #( localparam LOG_NUM_REQS2 = `CLOG2(MAX_FANOUT); localparam LOG_NUM_REQS3 = `CLOG2(NUM_SLICES); localparam DATAW2 = DATAW + LOG_NUM_REQS2; + localparam SLICE_STRIDE = MAX_FANOUT * NUM_OUTPUTS; + // a last slice with fewer inputs than outputs feeds only the outputs of its inputs + localparam LAST_OUTPUTS = `MIN(NUM_INPUTS - (NUM_SLICES - 1) * SLICE_STRIDE, NUM_OUTPUTS); + localparam JOIN_INPUTS = (NUM_SLICES - 1) * NUM_OUTPUTS + LAST_OUTPUTS; - wire [NUM_SLICES-1:0][NUM_OUTPUTS-1:0] valid_tmp; - wire [NUM_SLICES-1:0][NUM_OUTPUTS-1:0][DATAW2-1:0] data_tmp; - wire [NUM_SLICES-1:0][NUM_OUTPUTS-1:0] ready_tmp; + wire [JOIN_INPUTS-1:0] valid_tmp; + wire [JOIN_INPUTS-1:0][DATAW2-1:0] data_tmp; + wire [JOIN_INPUTS-1:0] ready_tmp; for (genvar s = 0; s < NUM_SLICES; ++s) begin : g_slice_arbs - localparam SLICE_STRIDE= MAX_FANOUT * NUM_OUTPUTS; localparam SLICE_BEGIN = s * SLICE_STRIDE; localparam SLICE_END = `MIN(SLICE_BEGIN + SLICE_STRIDE, NUM_INPUTS); localparam SLICE_SIZE = SLICE_END - SLICE_BEGIN; + localparam SLICE_OUTPUTS = `MIN(SLICE_SIZE, NUM_OUTPUTS); + localparam SLICE_SEL_W = `UP(`CLOG2(`CDIV(SLICE_SIZE, SLICE_OUTPUTS))); - wire [NUM_OUTPUTS-1:0][DATAW-1:0] data_tmp_u; - wire [NUM_OUTPUTS-1:0][LOG_NUM_REQS2-1:0] sel_tmp_u; + wire [SLICE_OUTPUTS-1:0][DATAW-1:0] data_tmp_u; + wire [SLICE_OUTPUTS-1:0][SLICE_SEL_W-1:0] sel_tmp_u; VX_stream_arb #( .NUM_INPUTS (SLICE_SIZE), - .NUM_OUTPUTS (NUM_OUTPUTS), + .NUM_OUTPUTS (SLICE_OUTPUTS), .DATAW (DATAW), .ARBITER (ARBITER), .STICKY (STICKY), @@ -79,14 +84,14 @@ module VX_stream_arb #( .valid_in (valid_in[SLICE_END-1: SLICE_BEGIN]), .data_in (data_in[SLICE_END-1: SLICE_BEGIN]), .ready_in (ready_in[SLICE_END-1: SLICE_BEGIN]), - .valid_out (valid_tmp[s]), + .valid_out (valid_tmp[s * NUM_OUTPUTS +: SLICE_OUTPUTS]), .data_out (data_tmp_u), - .ready_out (ready_tmp[s]), + .ready_out (ready_tmp[s * NUM_OUTPUTS +: SLICE_OUTPUTS]), .sel_out (sel_tmp_u) ); - for (genvar o = 0; o < NUM_OUTPUTS; ++o) begin : g_data_tmp - assign data_tmp[s][o] = {data_tmp_u[o], sel_tmp_u[o]}; + for (genvar o = 0; o < SLICE_OUTPUTS; ++o) begin : g_data_tmp + assign data_tmp[s * NUM_OUTPUTS + o] = {data_tmp_u[o], LOG_NUM_REQS2'(sel_tmp_u[o])}; end end @@ -94,7 +99,7 @@ module VX_stream_arb #( wire [NUM_OUTPUTS-1:0][LOG_NUM_REQS3-1:0] sel_out_u; VX_stream_arb #( - .NUM_INPUTS (NUM_SLICES * NUM_OUTPUTS), + .NUM_INPUTS (JOIN_INPUTS), .NUM_OUTPUTS (NUM_OUTPUTS), .DATAW (DATAW2), .ARBITER (ARBITER), @@ -210,15 +215,19 @@ module VX_stream_arb #( localparam NUM_SLICES = `CDIV(NUM_REQS, MAX_FANOUT); localparam LOG_NUM_REQS2 = `CLOG2(MAX_FANOUT); localparam LOG_NUM_REQS3 = `CLOG2(NUM_SLICES); - - wire [NUM_SLICES-1:0][NUM_INPUTS-1:0] valid_tmp; - wire [NUM_SLICES-1:0][NUM_INPUTS-1:0][DATAW-1:0] data_tmp; - wire [NUM_SLICES-1:0][NUM_INPUTS-1:0] ready_tmp; + localparam SLICE_STRIDE = MAX_FANOUT * NUM_INPUTS; + // a last slice with fewer outputs than inputs takes only the inputs of its outputs + localparam LAST_INPUTS = `MIN(NUM_OUTPUTS - (NUM_SLICES - 1) * SLICE_STRIDE, NUM_INPUTS); + localparam FORK_OUTPUTS = (NUM_SLICES - 1) * NUM_INPUTS + LAST_INPUTS; + + wire [FORK_OUTPUTS-1:0] valid_tmp; + wire [FORK_OUTPUTS-1:0][DATAW-1:0] data_tmp; + wire [FORK_OUTPUTS-1:0] ready_tmp; wire [NUM_INPUTS-1:0][LOG_NUM_REQS3-1:0] sel_tmp; VX_stream_arb #( .NUM_INPUTS (NUM_INPUTS), - .NUM_OUTPUTS (NUM_SLICES * NUM_INPUTS), + .NUM_OUTPUTS (FORK_OUTPUTS), .DATAW (DATAW), .ARBITER (ARBITER), .STICKY (STICKY), @@ -240,15 +249,16 @@ module VX_stream_arb #( for (genvar s = 0; s < NUM_SLICES; ++s) begin : g_slice_arbs - localparam SLICE_STRIDE= MAX_FANOUT * NUM_INPUTS; localparam SLICE_BEGIN = s * SLICE_STRIDE; localparam SLICE_END = `MIN(SLICE_BEGIN + SLICE_STRIDE, NUM_OUTPUTS); localparam SLICE_SIZE = SLICE_END - SLICE_BEGIN; + localparam SLICE_INPUTS= `MIN(SLICE_SIZE, NUM_INPUTS); + localparam SLICE_SEL_W = `UP(`CLOG2(`CDIV(SLICE_SIZE, SLICE_INPUTS))); - wire [NUM_INPUTS-1:0][LOG_NUM_REQS2-1:0] sel_out_u; + wire [SLICE_INPUTS-1:0][SLICE_SEL_W-1:0] sel_out_u; VX_stream_arb #( - .NUM_INPUTS (NUM_INPUTS), + .NUM_INPUTS (SLICE_INPUTS), .NUM_OUTPUTS (SLICE_SIZE), .DATAW (DATAW), .ARBITER (ARBITER), @@ -258,14 +268,23 @@ module VX_stream_arb #( ) fanout_slice_arb ( .clk (clk), .reset (reset), - .valid_in (valid_tmp[s]), - .ready_in (ready_tmp[s]), - .data_in (data_tmp[s]), + .valid_in (valid_tmp[s * NUM_INPUTS +: SLICE_INPUTS]), + .ready_in (ready_tmp[s * NUM_INPUTS +: SLICE_INPUTS]), + .data_in (data_tmp[s * NUM_INPUTS +: SLICE_INPUTS]), .data_out (data_out[SLICE_END-1: SLICE_BEGIN]), .valid_out (valid_out[SLICE_END-1: SLICE_BEGIN]), .ready_out (ready_out[SLICE_END-1: SLICE_BEGIN]), - .sel_out (sel_out_w[s]) + .sel_out (sel_out_u) ); + + // inputs past a narrow last slice are never forked to it + for (genvar i = 0; i < NUM_INPUTS; ++i) begin : g_sel_out_w + if (i < SLICE_INPUTS) begin : g_sel + assign sel_out_w[s][i] = LOG_NUM_REQS2'(sel_out_u[i]); + end else begin : g_pad + assign sel_out_w[s][i] = '0; + end + end end for (genvar i = 0; i < NUM_INPUTS; ++i) begin : g_sel_out @@ -284,7 +303,11 @@ module VX_stream_arb #( wire [NUM_INPUTS-1:0] requests; for (genvar i = 0; i < NUM_INPUTS; ++i) begin : g_i localparam o = r * NUM_INPUTS + i; - assign requests[i] = ready_out[o]; + if (o < NUM_OUTPUTS) begin : g_req_ready + assign requests[i] = ready_out[o]; + end else begin : g_req_pad + assign requests[i] = 1'b0; + end end assign arb_requests[r] = (| requests); end @@ -318,7 +341,11 @@ module VX_stream_arb #( wire [NUM_REQS-1:0] ready_out_s; for (genvar r = 0; r < NUM_REQS; ++r) begin : g_r localparam o = r * NUM_INPUTS + i; - assign ready_out_s[r] = ready_out_w[o]; + if (o < NUM_OUTPUTS) begin : g_ready + assign ready_out_s[r] = ready_out_w[o]; + end else begin : g_padding + assign ready_out_s[r] = 1'b0; + end end assign ready_in[i] = (NUM_INPUTS == 1) ? arb_valid : (| (ready_out_s & arb_onehot)); end diff --git a/hw/unittest/Makefile b/hw/unittest/Makefile index fb703fade9..87c8e63f85 100644 --- a/hw/unittest/Makefile +++ b/hw/unittest/Makefile @@ -22,6 +22,7 @@ all: $(MAKE) -C cp_axi_path $(MAKE) -C cp_dma $(MAKE) -C cp_llc_port + $(MAKE) -C stream_arb $(MAKE) -C cp_core $(MAKE) -C fma_unit $(MAKE) -C fdiv_unit @@ -54,6 +55,7 @@ run: $(MAKE) -C cp_axi_path run $(MAKE) -C cp_dma run $(MAKE) -C cp_llc_port run + $(MAKE) -C stream_arb run $(MAKE) -C cp_core run $(MAKE) -C fma_unit run $(MAKE) -C fdiv_unit run @@ -150,6 +152,7 @@ clean: $(MAKE) -C cp_axi_path clean $(MAKE) -C cp_dma clean $(MAKE) -C cp_llc_port clean + $(MAKE) -C stream_arb clean $(MAKE) -C cp_core clean $(MAKE) -C fma_unit clean $(MAKE) -C fdiv_unit clean diff --git a/hw/unittest/stream_arb/Makefile b/hw/unittest/stream_arb/Makefile new file mode 100644 index 0000000000..ee2f11ccfd --- /dev/null +++ b/hw/unittest/stream_arb/Makefile @@ -0,0 +1,23 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := stream_arb + +RTL_DIR := $(VORTEX_HOME)/hw/rtl +DPI_DIR := $(VORTEX_HOME)/hw/dpi + +SRC_DIR := $(VORTEX_HOME)/hw/unittest/$(PROJECT) + +CXXFLAGS := -I$(SRC_DIR) -I$(VORTEX_HOME)/hw/unittest/common -I$(SW_COMMON_DIR) + +SRCS := $(SRC_DIR)/main.cpp + +DBG_TRACE_FLAGS := + +RTL_PKGS := + +RTL_INCLUDE := -I$(RTL_DIR) -I$(DPI_DIR) -I$(RTL_DIR)/libs -I$(SRC_DIR) + +TOP := VX_stream_arb_top + +include ../common.mk diff --git a/hw/unittest/stream_arb/VX_stream_arb_tb.sv b/hw/unittest/stream_arb/VX_stream_arb_tb.sv new file mode 100644 index 0000000000..d94197bf9e --- /dev/null +++ b/hw/unittest/stream_arb/VX_stream_arb_tb.sv @@ -0,0 +1,179 @@ +// Copyright © 2019-2023 +// Licensed under the Apache License, Version 2.0. + +`include "VX_platform.vh" + +// Self-checking harness around one VX_stream_arb: every input sends numbered +// packets under random valid and output backpressure, and every packet must +// leave once, on an output its input maps to, with the select of its row. +module VX_stream_arb_tb #( + parameter NUM_INPUTS = 1, + parameter NUM_OUTPUTS = 1, + parameter MAX_FANOUT = 8, + parameter SEED = 1 +) ( + input wire clk, + input wire reset, + input wire drain, + output wire idle, + output wire [31:0] delivered, + output wire [31:0] errors +); + localparam SRC_W = 8; + localparam SEQ_W = 16; + localparam DATAW = SRC_W + SEQ_W; + localparam NUM_REQS = (NUM_INPUTS > NUM_OUTPUTS) ? `CDIV(NUM_INPUTS, NUM_OUTPUTS) : `CDIV(NUM_OUTPUTS, NUM_INPUTS); + localparam SEL_COUNT = `MIN(NUM_INPUTS, NUM_OUTPUTS); + localparam SEL_W = `UP(`CLOG2(NUM_REQS)); + + `STATIC_ASSERT (NUM_INPUTS < (1 << SRC_W), ("invalid parameter: NUM_INPUTS=%0d", NUM_INPUTS)) + + function automatic logic [31:0] hash32(input logic [31:0] x); + logic [31:0] h; + h = x ^ (x >> 16); + h = h * 32'h7feb352d; + h = h ^ (h >> 15); + h = h * 32'h846ca68b; + return h ^ (h >> 16); + endfunction + + reg [31:0] cycle; + + reg [NUM_INPUTS-1:0] valid_in; + reg [NUM_INPUTS-1:0][SEQ_W-1:0] seq_in; + wire [NUM_INPUTS-1:0][DATAW-1:0] data_in; + wire [NUM_INPUTS-1:0] ready_in; + + wire [NUM_OUTPUTS-1:0] valid_out; + wire [NUM_OUTPUTS-1:0][DATAW-1:0] data_out; + wire [NUM_OUTPUTS-1:0] ready_out; + wire [SEL_COUNT-1:0][SEL_W-1:0] sel_out; + + for (genvar i = 0; i < NUM_INPUTS; ++i) begin : g_data_in + assign data_in[i] = {SRC_W'(i), seq_in[i]}; + end + + // outputs stall at random, with a longer stall now and then so the + // arbiter buffers fill up + for (genvar o = 0; o < NUM_OUTPUTS; ++o) begin : g_ready_out + wire [1:0] rnd = 2'(hash32(cycle * 32'd977 + 32'd7919 * 32'(o + 1024) + 32'(SEED))); + assign ready_out[o] = (rnd != 0) && (cycle[9:6] != 4'(SEED)); + end + + VX_stream_arb #( + .NUM_INPUTS (NUM_INPUTS), + .NUM_OUTPUTS (NUM_OUTPUTS), + .DATAW (DATAW), + .ARBITER ("R"), + .MAX_FANOUT (MAX_FANOUT), + .OUT_BUF (3) + ) dut ( + .clk (clk), + .reset (reset), + .valid_in (valid_in), + .data_in (data_in), + .ready_in (ready_in), + .valid_out (valid_out), + .data_out (data_out), + .ready_out (ready_out), + .sel_out (sel_out) + ); + + wire [NUM_OUTPUTS-1:0][SRC_W-1:0] src_out; + wire [NUM_OUTPUTS-1:0][SEQ_W-1:0] seq_out; + + for (genvar o = 0; o < NUM_OUTPUTS; ++o) begin : g_data_out + assign {src_out[o], seq_out[o]} = data_out[o]; + end + + reg [NUM_INPUTS-1:0][31:0] sent_cnt, recv_cnt; + reg [NUM_INPUTS-1:0][31:0] sent_sum, recv_sum; + reg [31:0] errors_r; + + logic [NUM_INPUTS-1:0][31:0] recv_inc, recv_sum_inc; + logic [31:0] errors_inc; + + always_comb begin + recv_inc = '0; + recv_sum_inc = '0; + errors_inc = '0; + for (integer o = 0; o < NUM_OUTPUTS; ++o) begin + if (valid_out[o] && ready_out[o]) begin + if (32'(src_out[o]) >= NUM_INPUTS) begin + errors_inc = errors_inc + 1; + end else begin + recv_inc[src_out[o]] = recv_inc[src_out[o]] + 1; + recv_sum_inc[src_out[o]] = recv_sum_inc[src_out[o]] + 32'(seq_out[o]); + if (NUM_INPUTS >= NUM_OUTPUTS) begin + // an input reaches one output, in order, with its row as select + if ((32'(src_out[o]) % NUM_OUTPUTS) != o + || seq_out[o] != recv_cnt[src_out[o]][SEQ_W-1:0] + || 32'(sel_out[o % SEL_COUNT]) != (32'(src_out[o]) / NUM_OUTPUTS)) begin + errors_inc = errors_inc + 1; + end + end else begin + // an input reaches any output of its column + if ((o % NUM_INPUTS) != 32'(src_out[o])) begin + errors_inc = errors_inc + 1; + end + end + end + end + end + end + + always @(posedge clk) begin + if (reset) begin + cycle <= '0; + valid_in <= '0; + seq_in <= '0; + sent_cnt <= '0; + sent_sum <= '0; + recv_cnt <= '0; + recv_sum <= '0; + errors_r <= '0; + end else begin + cycle <= cycle + 1; + for (integer i = 0; i < NUM_INPUTS; ++i) begin + logic [2:0] rnd; + rnd = 3'(hash32(cycle * 32'd977 + 32'd7919 * 32'(i) + 32'(SEED))); + if (valid_in[i] && ready_in[i]) begin + sent_cnt[i] <= sent_cnt[i] + 1; + sent_sum[i] <= sent_sum[i] + 32'(seq_in[i]); + seq_in[i] <= seq_in[i] + 1; + valid_in[i] <= ~drain && (rnd != 0); + end else if (~valid_in[i]) begin + valid_in[i] <= ~drain && (rnd != 0); + end + end + for (integer i = 0; i < NUM_INPUTS; ++i) begin + recv_cnt[i] <= recv_cnt[i] + recv_inc[i]; + recv_sum[i] <= recv_sum[i] + recv_sum_inc[i]; + end + errors_r <= errors_r + errors_inc; + if (errors_inc != 0) begin + $display("cycle %0d: stream_arb %0dx%0d (fanout %0d): misrouted packet", cycle, NUM_INPUTS, NUM_OUTPUTS, MAX_FANOUT); + end + end + end + + logic [31:0] delivered_w; + logic idle_w; + + always_comb begin + delivered_w = '0; + idle_w = (valid_in == '0); + for (integer i = 0; i < NUM_INPUTS; ++i) begin + delivered_w = delivered_w + recv_cnt[i]; + // every input must have been served, and nothing lost + idle_w = idle_w && (recv_cnt[i] != 0) + && (recv_cnt[i] == sent_cnt[i]) + && (recv_sum[i] == sent_sum[i]); + end + end + + assign idle = idle_w; + assign delivered = delivered_w; + assign errors = errors_r; + +endmodule diff --git a/hw/unittest/stream_arb/VX_stream_arb_top.sv b/hw/unittest/stream_arb/VX_stream_arb_top.sv new file mode 100644 index 0000000000..7698928767 --- /dev/null +++ b/hw/unittest/stream_arb/VX_stream_arb_top.sv @@ -0,0 +1,63 @@ +// Copyright © 2019-2023 +// Licensed under the Apache License, Version 2.0. + +`include "VX_platform.vh" + +// VX_stream_arb shapes under test, each in its own self-checking harness. +// Over 1.5x MAX_FANOUT requests an arbiter becomes a tree of slices; the +// shapes cover full slices, a last slice of one input or output, one smaller +// than the other side, one not a whole number of rows, and a join that is a +// tree itself. +module VX_stream_arb_top #( + parameter NUM_CFGS = 21 +) ( + input wire clk, + input wire reset, + input wire drain, + output wire idle, + output wire [NUM_CFGS-1:0] busy, + output wire [31:0] delivered, + output wire [31:0] errors +); + // input select | output select | flat + localparam int CFG_NI [NUM_CFGS] = '{64, 17, 65, 13, 19, 34, 36, 40, 26, 27, 129, 200, 1, 4, 2, 1, 4, 5, 8, 3, 1}; + localparam int CFG_NO [NUM_CFGS] = '{ 1, 1, 1, 1, 1, 4, 4, 4, 2, 2, 1, 1, 17, 34, 27, 64, 32, 2, 8, 5, 1}; + localparam int CFG_MF [NUM_CFGS] = '{ 8, 8, 8, 8, 8, 4, 4, 4, 4, 4, 8, 4, 8, 4, 4, 8, 8, 8, 8, 8, 8}; + + wire [NUM_CFGS-1:0] idle_w; + wire [NUM_CFGS-1:0][31:0] delivered_w; + wire [NUM_CFGS-1:0][31:0] errors_w; + + for (genvar c = 0; c < NUM_CFGS; ++c) begin : g_cfgs + VX_stream_arb_tb #( + .NUM_INPUTS (CFG_NI[c]), + .NUM_OUTPUTS (CFG_NO[c]), + .MAX_FANOUT (CFG_MF[c]), + .SEED (c + 1) + ) tb ( + .clk (clk), + .reset (reset), + .drain (drain), + .idle (idle_w[c]), + .delivered (delivered_w[c]), + .errors (errors_w[c]) + ); + end + + logic [31:0] delivered_s, errors_s; + + always_comb begin + delivered_s = '0; + errors_s = '0; + for (integer c = 0; c < NUM_CFGS; ++c) begin + delivered_s = delivered_s + delivered_w[c]; + errors_s = errors_s + errors_w[c]; + end + end + + assign idle = (& idle_w); + assign busy = ~idle_w; + assign delivered = delivered_s; + assign errors = errors_s; + +endmodule diff --git a/hw/unittest/stream_arb/main.cpp b/hw/unittest/stream_arb/main.cpp new file mode 100644 index 0000000000..740795a1ca --- /dev/null +++ b/hw/unittest/stream_arb/main.cpp @@ -0,0 +1,62 @@ +// Copyright © 2019-2023 +// Licensed under the Apache License, Version 2.0. + +// Verilator unit test for VX_stream_arb over the shapes in VX_stream_arb_top. +// Each shape's harness checks its own traffic: random input valids and output +// stalls for a while, then the inputs stop and every packet must have left +// exactly once, on the right output, with the right select. + +#include "vl_simulator.h" +#include "VVX_stream_arb_top.h" +#include +#include + +#ifndef TRACE_START_TIME +#define TRACE_START_TIME 0ull +#endif +#ifndef TRACE_STOP_TIME +#define TRACE_STOP_TIME -1ull +#endif + +static uint64_t timestamp = 0; +static bool trace_en = false; + +double sc_time_stamp() { return timestamp; } +bool sim_trace_enabled() { return trace_en; } +void sim_trace_enable(bool e) { trace_en = e; } + +int main(int argc, char** argv) { + Verilated::commandArgs(argc, argv); + vl_simulator sim; + uint64_t tick = 0; + + constexpr uint32_t kTrafficCycles = 20000; + constexpr uint32_t kDrainCycles = 20000; + + sim->drain = 0; + sim->reset = 1; + tick = sim.step(tick, 8); + sim->reset = 0; + + tick = sim.step(tick, 2 * kTrafficCycles); + sim->drain = 1; + uint32_t cycles = 0; + while (!sim->idle && sim->errors == 0 && cycles < kDrainCycles) { + tick = sim.step(tick, 2); + ++cycles; + } + timestamp = tick; + + std::printf("delivered=%u, errors=%u, idle=%d\n", sim->delivered, sim->errors, sim->idle); + for (uint32_t c = 0; c < 8 * sizeof(sim->busy); ++c) { + if ((sim->busy >> c) & 1) { + std::printf("shape %u did not drain\n", c); + } + } + if (sim->errors != 0 || !sim->idle) { + std::printf("FAILED!\n"); + return 1; + } + std::printf("PASSED!\n"); + return 0; +} From f321887e6f48c72c7635eeed7d7dfd11df044730 Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 21 Oct 2026 07:47:54 -0700 Subject: [PATCH 072/124] simx: let a cache's port arbiter take more than 64 ports The cache's core-port arbiter kept its requesting ports, read-merge followers and flush tokens in 64-bit masks. With VX_CFG_CP_LLC_CLIENT, a shared L2 with 64 L1 ports plus the command processor's DMA slot has 65 ports. On x86 the shift for port 64 wraps around to port 0, so port 0 was granted while its input channel was empty, and SimX aborted with "channel is empty" (simobject.h peek). Release builds compile out the assert(num_inputs <= 64) that should have caught this. Requests are now collected per bank and per first-level slice of the arbiter tree; one slice is at most 12 ports wide. The flush tokens are kept per port. Read-merge masks are taken relative to the merge group. The tree now follows the RTL's recursion. When a level has more than 1.5x the fan-out limit of slice buffers, those buffers are sliced again behind another buffered level before the join. Each slice, including a partial last one, round-robins over its own inputs into a 2-entry buffer. Configurations of up to 96 ports keep the single level they had, and their cycle counts are unchanged. The read-merge response network keeps its 64-port, one-level limit, now as an always-on check. SimX cycles, c96140253 -> this change (sgemm -n32 / raycast): default 115,067 / 1,494,492 unchanged 4 cores + L2 30,086 / 478,511 unchanged 4c, 16 L1 ports (64-port L2) 29,996 / 478,186 unchanged 8c IW4, CP_LLC_CLIENT (65) abort -> PASS rtx5080 4c x 32t, CP (65) abort -> PASS (sgemm -n32 10,009) CI adds model_parity-sgemm-cp-llc-17 (2 cores x 8 L1 ports + DMA = 17 L2 ports, a one-port last slice) and a SimX sgemm case at 65 ports. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/regression.yaml | 22 ++++ sim/simx/mem/cache.cpp | 238 +++++++++++++++++++++-------------- 2 files changed, 169 insertions(+), 91 deletions(-) diff --git a/ci/testcases/regression.yaml b/ci/testcases/regression.yaml index a6042df1ef..e6475f33c4 100644 --- a/ci/testcases/regression.yaml +++ b/ci/testcases/regression.yaml @@ -366,6 +366,28 @@ tests: shape: cores: 4 l2cache: true +# Eight L1 ports per core and the DMA's slot give the shared L2 2^k + 1 +# ports, so each bank's port arbiter is a tree whose last slice has one port: +# 17 ports on two cores, and 65 on eight, past a 64-bit port mask. +- id: model_parity-sgemm-cp-llc-17 + check: model_parity + via: blackbox + app: sgemm + args: -n32 + configs: -DVX_CFG_CP_LLC_CLIENT=1 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=8 + shape: + cores: 2 + l2cache: true +- id: sgemm-cp-llc-65 + via: blackbox + drivers: + - simx + app: sgemm + args: -n32 + configs: -DVX_CFG_CP_LLC_CLIENT=1 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=8 + shape: + cores: 8 + l2cache: true # The shared L2 keeps its lines across the L1-only launch flushes. - id: model_parity-flush-chain check: model_parity diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index 14fee5673c..67a8e88131 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -34,15 +34,23 @@ namespace { constexpr uint32_t kMaxFanout = 8; // An arbiter over more than 1.5x the fan-out limit is split into a buffered -// tree, one cycle deeper. -uint32_t arb_tree_delay(uint32_t requesters) { - return (requesters > kMaxFanout + kMaxFanout / 2) ? 1 : 0; +// tree: slices of up to the fan-out limit each grant into their own buffer, +// and those buffers are arbitrated the same way, one level per cycle, until +// few enough remain for a single join arbiter. Returns each buffered level's +// slice count, none for a single arbiter. +std::vector arb_tree_levels(uint32_t requesters) { + std::vector levels; + for (uint32_t n = requesters; n > kMaxFanout + kMaxFanout / 2;) { + n = (n + kMaxFanout - 1) / kMaxFanout; + levels.push_back(n); + } + return levels; } -// Slices of the tree's first level: each arbitrates up to the fan-out limit -// into its own buffer, and a join arbiter picks among the slice buffers. +// Slices of the tree's first level, 1 for a single arbiter. uint32_t arb_tree_slices(uint32_t requesters) { - return (arb_tree_delay(requesters) != 0) ? (requesters + kMaxFanout - 1) / kMaxFanout : 1; + auto levels = arb_tree_levels(requesters); + return levels.empty() ? 1 : levels.front(); } // Registered stages of a radix-2 omega network between a and b ports; up to @@ -2263,15 +2271,19 @@ class CachePortArb : public SimObject { , lg2_inputs_(log2ceil(num_inputs)) // the request crossbar registers its output when it has more than two inputs , delay_((num_inputs > 2) ? 1 : 0) - , slices_(arb_tree_slices(num_inputs)) - , rr_(num_banks * slices_, num_inputs - 1) - , join_rr_(num_banks, slices_ - 1) - , tree_stage_((slices_ > 1) ? num_banks * slices_ : 0, this) + , levels_(arb_tree_levels(num_inputs)) , merge_group_(merge_group) { - assert(num_inputs <= 64); - __assert(merge_group_ <= 1 || (ispow2(merge_group_) && (num_inputs % merge_group_) == 0), - "a read-merge group must be a power of two dividing the ports"); + __assert(merge_group_ <= 1 || (merge_group_ <= 64 && ispow2(merge_group_) && (num_inputs % merge_group_) == 0), + "a read-merge group must be a power of two of at most 64 dividing the ports"); + tree_stage_.reserve(levels_.size()); + for (auto slices : levels_) { + tree_stage_.emplace_back(num_banks * slices, this); + } + rr_.resize(levels_.size() + 1); + for (uint32_t s = 0; s <= levels_.size(); ++s) { + rr_.at(s).resize(num_banks * this->arb_slices(s)); + } } // Bits of the merged-port mask carried below the request tag. @@ -2279,39 +2291,51 @@ class CachePortArb : public SimObject { uint64_t collisions() const { return collisions_; } - // Ports whose head is a flush token. While any is visible, cacheable - // requests are held at every port. - uint64_t tokens() const { - uint64_t mask = 0; + bool is_token(uint32_t i) const { + return !ReqIn.at(i).empty() && ReqIn.at(i).peek().flags.flush; + } + + // While any port's head is a flush token, cacheable requests are held at + // every port. + bool any_token() const { for (uint32_t i = 0, n = ReqIn.size(); i < n; ++i) { - if (!ReqIn.at(i).empty() && ReqIn.at(i).peek().flags.flush) { - mask |= 1ull << i; + if (this->is_token(i)) { + return true; } } + return false; + } + + // Ports whose head is a flush token. + std::vector tokens() const { + std::vector mask(ReqIn.size(), false); + for (uint32_t i = 0, n = ReqIn.size(); i < n; ++i) { + mask.at(i) = this->is_token(i); + } return mask; } // Tokens owing no walk: IO tokens, and L1-only ones unless they still need // a walk here. - uint64_t tokens(bool l1_needs_walk) const { - uint64_t mask = 0; + std::vector tokens(bool l1_needs_walk) const { + std::vector mask(ReqIn.size(), false); for (uint32_t i = 0, n = ReqIn.size(); i < n; ++i) { - if (ReqIn.at(i).empty()) { + if (!this->is_token(i)) { continue; } auto &flags = ReqIn.at(i).peek().flags; - if (flags.flush && (flags.io || (flags.flush_l1 && !l1_needs_walk))) { - mask |= 1ull << i; - } + mask.at(i) = flags.io || (flags.flush_l1 && !l1_needs_walk); } return mask; } // No cacheable request is left between the ports and the banks. bool drained() const { - for (auto &stage : tree_stage_) { - if (stage.size() != 0) { - return false; + for (auto &level : tree_stage_) { + for (auto &stage : level) { + if (stage.size() != 0) { + return false; + } } } for (auto &out : ReqOut) { @@ -2333,36 +2357,65 @@ class CachePortArb : public SimObject { protected: void on_reset() { - std::fill(rr_.begin(), rr_.end(), ReqIn.size() - 1); - std::fill(join_rr_.begin(), join_rr_.end(), slices_ - 1); + for (auto &rr : rr_) { + std::fill(rr.begin(), rr.end(), 0); + } collisions_ = 0; } + // Slices of the arbiter level s: level 0 grants the ports, each level above + // the buffers of the level below, and the last level is the join arbiter. + uint32_t arb_slices(uint32_t s) const { + return (s < levels_.size()) ? levels_.at(s) : 1; + } + + // Inputs of slice sl of the arbiter level s. + uint32_t arb_slice_size(uint32_t s, uint32_t sl) const { + uint32_t inputs = (s == 0) ? ReqIn.size() : levels_.at(s - 1); + if (s == levels_.size()) { + return inputs; + } + return std::min(kMaxFanout, inputs - sl * kMaxFanout); + } + void on_tick() { if (init_cycles_ != 0) { --init_cycles_; return; } - // the arbiter tree's join level: one slice buffer per bank per cycle - for (uint32_t b = 0, nb = (slices_ > 1) ? ReqOut.size() : 0; b < nb; ++b) { - if (ReqOut.at(b).full()) { - continue; - } - for (uint32_t k = 1; k <= slices_; ++k) { - uint32_t sl = (join_rr_[b] + k) % slices_; - auto& stage = tree_stage_.at(b * slices_ + sl); - if (!stage.empty()) { - ReqOut.at(b).send(stage.peek(), delay_); - stage.pop(); - join_rr_[b] = sl; - break; + uint32_t nb = ReqOut.size(); + uint32_t nl = levels_.size(); + // the tree's buffered levels, join first: each slice passes one buffer of + // the level below per cycle + for (uint32_t s = nl; s > 0; --s) { + uint32_t slices = this->arb_slices(s); + uint32_t inputs = levels_.at(s - 1); + for (uint32_t b = 0; b < nb; ++b) { + for (uint32_t sl = 0; sl < slices; ++sl) { + auto& out = (s < nl) ? tree_stage_.at(s).at(b * slices + sl) : ReqOut.at(b); + if (out.full()) { + continue; + } + uint32_t lo = (s < nl) ? sl * kMaxFanout : 0; + uint32_t size = this->arb_slice_size(s, sl); + auto& rr = rr_.at(s).at(b * slices + sl); + for (uint32_t k = 0; k < size; ++k) { + uint32_t j = (rr + k) % size; + auto& in = tree_stage_.at(s - 1).at(b * inputs + lo + j); + if (!in.empty()) { + out.send(in.peek(), (s < nl) ? 1 : delay_); + in.pop(); + rr = (j + 1) % size; + break; + } + } } } } uint32_t n = ReqIn.size(); - const bool hold = (this->tokens() != 0); - // merged ports riding each leader's bank access - uint64_t followers = 0; + const bool hold = this->any_token(); + // merged ports riding each leader's bank access, by offset in its group + std::vector followers(n, false); std::vector merged(n, 0); if (merge_group_ > 1 && !hold) { for (uint32_t i = 0; i < n; ++i) { @@ -2370,25 +2423,30 @@ class CachePortArb : public SimObject { continue; } auto& req_i = ReqIn.at(i).peek(); - for (uint32_t j = (i / merge_group_) * merge_group_; j < i; ++j) { - if ((followers & (1ull << j)) || !this->is_plain_read(j)) { + uint32_t base = (i / merge_group_) * merge_group_; + for (uint32_t j = base; j < i; ++j) { + if (followers.at(j) || !this->is_plain_read(j)) { continue; } auto& req_j = ReqIn.at(j).peek(); if ((req_j.addr >> params_.word_select_addr_start) == (req_i.addr >> params_.word_select_addr_start) && req_j.tag == req_i.tag) { - merged.at(j) |= 1ull << i; - followers |= 1ull << i; + merged.at(j) |= 1ull << (i - base); + followers.at(i) = true; break; } } } } - // requesting ports per bank; non-cacheable requests leave on their own port - std::vector want(ReqOut.size(), 0); + // requesting ports per bank and first-level slice; non-cacheable requests + // leave on their own port + uint32_t slices = this->arb_slices(0); + uint32_t slice_ports = (nl != 0) ? kMaxFanout : n; + std::vector want(nb * slices, 0); + std::vector wants(nb, 0); for (uint32_t i = 0; i < n; ++i) { auto& in = ReqIn.at(i); - if (in.empty() || (followers & (1ull << i))) { + if (in.empty() || followers.at(i)) { continue; } auto& req = in.peek(); @@ -2402,38 +2460,35 @@ class CachePortArb : public SimObject { continue; } if (!hold) { - want.at(params_.addr_bank_id(req.addr)) |= 1ull << i; + uint32_t b = params_.addr_bank_id(req.addr); + want.at(b * slices + i / slice_ports) |= 1u << (i % slice_ports); + ++wants.at(b); } } // each bank's arbiter, or each slice of its tree, grants one port per cycle - for (uint32_t b = 0, nb = ReqOut.size(); b < nb; ++b) { - if (want[b] == 0) { + for (uint32_t b = 0; b < nb; ++b) { + if (wants.at(b) == 0) { continue; } bool granted = false; - for (uint32_t sl = 0; sl < slices_; ++sl) { - uint64_t slice_want = want[b]; - if (slices_ > 1) { - uint32_t lo = sl * kMaxFanout; - uint32_t hi = std::min(lo + kMaxFanout, n); - uint64_t range = ((hi < 64) ? (1ull << hi) : 0) - (1ull << lo); - slice_want &= range; - } - auto& out = (slices_ > 1) ? tree_stage_.at(b * slices_ + sl) : ReqOut.at(b); + for (uint32_t sl = 0; sl < slices; ++sl) { + uint32_t slice_want = want.at(b * slices + sl); + auto& out = (nl != 0) ? tree_stage_.at(0).at(b * slices + sl) : ReqOut.at(b); if (slice_want == 0 || out.full()) { continue; } - auto& rr = rr_[b * slices_ + sl]; - uint32_t i = rr; - do { - i = (i + 1) % n; - } while (!(slice_want & (1ull << i))); - this->grant(i, out, merged); - rr = i; + uint32_t size = this->arb_slice_size(0, sl); + auto& rr = rr_.at(0).at(b * slices + sl); + uint32_t j = rr; + while (!(slice_want & (1u << j))) { + j = (j + 1) % size; + } + this->grant(sl * slice_ports + j, out, merged); + rr = (j + 1) % size; granted = true; } if (granted) { - collisions_ += __builtin_popcountll(want[b]) - 1; + collisions_ += wants.at(b) - 1; } } } @@ -2443,16 +2498,18 @@ class CachePortArb : public SimObject { auto req = ReqIn.at(i).peek(); uint64_t mask = 0; if (merge_group_ > 1) { - mask = merged.at(i) >> ((i / merge_group_) * merge_group_); - for (uint64_t f = merged.at(i); f != 0; f &= f - 1) { - DT(3, this->name() << " read-merge: port=" << __builtin_ctzll(f) << ", leader=" << i << ", " << req); - ReqIn.at(__builtin_ctzll(f)).pop(); + uint32_t base = (i / merge_group_) * merge_group_; + mask = merged.at(i); + for (uint64_t f = mask; f != 0; f &= f - 1) { + uint32_t port = base + __builtin_ctzll(f); + DT(3, this->name() << " read-merge: port=" << port << ", leader=" << i << ", " << req); + ReqIn.at(port).pop(); } } uint64_t tag = (((uint64_t(req.tag) << this->merge_bits()) | mask) << lg2_inputs_) | i; __assert((tag >> 32) == 0, "cache bank tag overflow"); req.tag = uint32_t(tag); - out.send(req, tree_stage_.empty() ? delay_ : 1); + out.send(req, levels_.empty() ? delay_ : 1); ReqIn.at(i).pop(); } @@ -2470,10 +2527,9 @@ class CachePortArb : public SimObject { uint32_t lg2_inputs_; uint32_t delay_; uint32_t init_cycles_ = 0; - uint32_t slices_; - std::vector rr_; // [bank][slice] - std::vector join_rr_; // [bank] - std::vector> tree_stage_; // [bank][slice] + std::vector levels_; // [level] slices per bank + std::vector>> tree_stage_; // [level][bank][slice] + std::vector> rr_; // [arbiter level][bank][slice] next to grant uint32_t merge_group_; uint64_t collisions_ = 0; @@ -2506,8 +2562,8 @@ class CacheRspMcast : public SimObject { , join_rr_(num_outputs, slices_ - 1) , tree_stage_((slices_ > 1) ? num_outputs * slices_ : 0, this) { - assert(num_outputs <= 64); - assert(merge_group < 64); + __assert(num_outputs <= 64 && merge_group < 64, "the read-merge response network takes at most 64 ports"); + __assert(arb_tree_levels(num_banks).size() <= 1, "the read-merge response network takes one tree level"); } protected: @@ -2846,7 +2902,7 @@ class Cache::Impl { void flush_step() { switch (flush_state_) { case FlushState::Idle: - if (port_arb_->tokens() != 0) { + if (port_arb_->any_token()) { flush_state_ = FlushState::Drain; } break; @@ -2937,17 +2993,18 @@ class Cache::Impl { break; } case FlushState::Respond: { - if (rsp_pending_ == 0) { + auto next = std::find(rsp_pending_.begin(), rsp_pending_.end(), true); + if (next == rsp_pending_.end()) { flush_state_ = FlushState::Idle; break; } - uint32_t i = __builtin_ctzll(rsp_pending_); + uint32_t i = std::distance(rsp_pending_.begin(), next); auto &token = port_arb_->ReqIn.at(i).peek(); MemRsp rsp{token.tag, token.hart_id, token.uuid}; if (simobject_->core_rsp_out.at(i).try_send(rsp, core_rsp_delay_)) { DT(3, simobject_->name() << " flush-rsp: " << rsp); port_arb_->take_token(i); - rsp_pending_ &= ~(1ull << i); + rsp_pending_.at(i) = false; } break; } @@ -2966,9 +3023,8 @@ class Cache::Impl { // weakest one that serves them all. token_kinds_t token_kinds() const { token_kinds_t kinds; - uint64_t tokens = port_arb_->tokens(); for (uint32_t i = 0, n = config_.num_inputs; i < n; ++i) { - if (!(tokens & (1ull << i))) { + if (!port_arb_->is_token(i)) { continue; } auto &flags = port_arb_->ReqIn.at(i).peek().flags; @@ -3069,7 +3125,7 @@ class Cache::Impl { uint32_t core_rsp_delay_ = 0; enum class FlushState { Idle, Drain, Walk, Forward, Respond }; FlushState flush_state_ = FlushState::Idle; - uint64_t rsp_pending_ = 0; + std::vector rsp_pending_; uint64_t fwd_sent_ = 0; uint64_t fwd_done_ = 0; bool fwd_go_ = false; // nothing is left ahead of the forwarded tokens From 9681b608b7c18c42a6d88708bc00cd23218ce8c6 Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 21 Oct 2026 08:10:06 -0700 Subject: [PATCH 073/124] Keep each uncached request on its own memory port when a cache has fewer core ports A cache with more memory ports than core ports does not build. One example is VX_CFG_DCACHE_NUM_BANKS=16 at 4 threads: 2 dcache requests and 16 memory ports. Two separate faults cause it. First, VX_cache_bypass sends uncached core requests to the memory ports through a VX_mem_bus_arb. With fewer inputs than outputs, that is a VX_stream_arb that picks a ready output, so its valid depends on ready_out. With REQ_OUT_BUF(0) this valid feeds mem_bus_out_arb, which grants on valid and returns ready. That is a real combinational loop: valid -> grant -> ready -> grant -> valid (Verilator UNOPTFLAT). Beside a cache, each uncached core request i now keeps memory port i, as SimX has always modelled the bypass (req_id % mem_ports). The memory ports past the last core port carry cached traffic only. Without a cache, the memory side is a pass-through, there is no loop, and the bypass still spreads requests over the ports as before. Second, VX_mem_bus_arb and VX_lsu_mem_arb sized the arbiter's select from ARB_SEL_BITS(I, O), which is 0 when I < O. VX_stream_arb's output-side select is CLOG2(CDIV(O, I)) bits per input. Once O > 2I the widths differ (WIDTHEXPAND), and VX_lsu_mem_arb also declared it per output instead of per input. Both now take the arbiter's own select shape, as VX_tex_bus_arb already does. Both faults are already present at the merge-base with origin/master (7fe193695). Configurations that build today are unchanged. When there are at least as many core ports as memory ports, or no cache, the bypass elaborates exactly as before. The select width equals UP(ARB_SEL_BITS) whenever the old code was lint-clean. Yosys proves the new VX_mem_bus_arb and VX_cache_bypass sequentially equivalent to the old ones for every parameter set where the old ones are lint-clean: 388 shapes (VX_mem_bus_arb 1-12 x 1-12 inputs and outputs at buffer depths 0 and 2; VX_cache_bypass with 1-8 core ports, 1-16 memory ports, with and without a cache, at 4- and 16-byte lines). The old modules lint clean on 234 of them. A mutant fails the same check. None of the synthesis-gate DUTs elaborates a bypass or bus arbiter with fewer inputs than outputs. With DCACHE_NUM_BANKS=16 and 16 memory banks at 4 threads, the rtlsim build is now lint clean. sgemm and raycast pass on rtlsim and SimX: sgemm -n32 115,253 rtlsim / 115,177 SimX (-0.07%), raycast 1,501,399 / 1,494,417 (-0.47%). CI adds model_parity-sgemm-dcache-16-banks. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/regression.yaml | 8 ++++++++ hw/rtl/cache/VX_cache_bypass.sv | 13 ++++++++++--- hw/rtl/mem/VX_lsu_mem_arb.sv | 4 +++- hw/rtl/mem/VX_mem_bus_arb.sv | 3 ++- 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/ci/testcases/regression.yaml b/ci/testcases/regression.yaml index e6475f33c4..d2d6c695d6 100644 --- a/ci/testcases/regression.yaml +++ b/ci/testcases/regression.yaml @@ -388,6 +388,14 @@ tests: shape: cores: 8 l2cache: true +# More dcache banks and memory ports than core requests: the uncached path keeps +# each core request on its own memory port. +- id: model_parity-sgemm-dcache-16-banks + check: model_parity + via: blackbox + app: sgemm + args: -n32 + configs: -DVX_CFG_DCACHE_NUM_BANKS=16 -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=16 # The shared L2 keeps its lines across the L1-only launch flushes. - id: model_parity-flush-chain check: model_parity diff --git a/hw/rtl/cache/VX_cache_bypass.sv b/hw/rtl/cache/VX_cache_bypass.sv index 950f434392..f3e6d36ac4 100644 --- a/hw/rtl/cache/VX_cache_bypass.sv +++ b/hw/rtl/cache/VX_cache_bypass.sv @@ -53,6 +53,10 @@ module VX_cache_bypass import VX_gpu_pkg::*; #( localparam MEM_TAG_NC1_WIDTH = UUID_WIDTH + MEM_TAG_ID_WIDTH; localparam MEM_TAG_NC2_WIDTH = MEM_TAG_NC1_WIDTH + WSEL_BITS; localparam MEM_TAG_OUT_WIDTH = CACHE_ENABLE ? `MAX(MEM_TAG_IN_WIDTH, MEM_TAG_NC2_WIDTH) : MEM_TAG_NC2_WIDTH; + // Beside a cache, the memory-side arbiter grants on valid, so spreading + // uncached requests over more memory ports by their ready would close a + // valid-ready loop: with fewer core ports, core port i keeps memory port i. + localparam NC_PORTS = CACHE_ENABLE ? `MIN(NUM_REQS, MEM_PORTS) : MEM_PORTS; `STATIC_ASSERT(0 == (`VX_MEM_IO_BASE_ADDR % `VX_CFG_MEM_BLOCK_SIZE), ("invalid parameter")) VX_mem_bus_if #( .DATA_SIZE (WORD_SIZE), @@ -108,10 +112,10 @@ module VX_cache_bypass import VX_gpu_pkg::*; #( VX_mem_bus_if #( .DATA_SIZE (WORD_SIZE), .TAG_WIDTH (MEM_TAG_NC1_WIDTH) - ) core_bus_nc_arb_if[MEM_PORTS](); + ) core_bus_nc_arb_if[NC_PORTS](); VX_mem_bus_arb #( .NUM_INPUTS (NUM_REQS), - .NUM_OUTPUTS(MEM_PORTS), + .NUM_OUTPUTS(NC_PORTS), .DATA_SIZE (WORD_SIZE), .TAG_WIDTH (CORE_TAG_WIDTH), .TAG_SEL_IDX(TAG_SEL_IDX), @@ -128,7 +132,7 @@ module VX_cache_bypass import VX_gpu_pkg::*; #( .DATA_SIZE (LINE_SIZE), .TAG_WIDTH (MEM_TAG_NC2_WIDTH) ) mem_bus_out_nc_if[MEM_PORTS](); - for (genvar i = 0; i < MEM_PORTS; ++i) begin : g_mem_bus_out_nc + for (genvar i = 0; i < NC_PORTS; ++i) begin : g_mem_bus_out_nc wire core_req_nc_arb_rw; wire [WORD_SIZE-1:0] core_req_nc_arb_byteen; wire [CORE_ADDR_WIDTH-1:0] core_req_nc_arb_addr; @@ -203,6 +207,9 @@ module VX_cache_bypass import VX_gpu_pkg::*; #( }; assign mem_bus_out_nc_if[i].rsp_ready = core_bus_nc_arb_if[i].rsp_ready; end + for (genvar i = NC_PORTS; i < MEM_PORTS; ++i) begin : g_mem_bus_out_nc_idle + `INIT_VX_MEM_BUS_IF (mem_bus_out_nc_if[i]) + end VX_mem_bus_if #( .DATA_SIZE (LINE_SIZE), .TAG_WIDTH (MEM_TAG_OUT_WIDTH) diff --git a/hw/rtl/mem/VX_lsu_mem_arb.sv b/hw/rtl/mem/VX_lsu_mem_arb.sv index 29aa903045..90df8a233a 100644 --- a/hw/rtl/mem/VX_lsu_mem_arb.sv +++ b/hw/rtl/mem/VX_lsu_mem_arb.sv @@ -37,6 +37,8 @@ module VX_lsu_mem_arb import VX_gpu_pkg::*; #( ); localparam DATA_WIDTH = (8 * DATA_SIZE); localparam LOG_NUM_REQS = `ARB_SEL_BITS(NUM_INPUTS, NUM_OUTPUTS); + localparam SEL_COUNT = `MIN(NUM_INPUTS, NUM_OUTPUTS); + localparam SEL_W = `UP(`CLOG2(`CDIV(`MAX(NUM_INPUTS, NUM_OUTPUTS), SEL_COUNT))); localparam REQ_DATAW = 1 + NUM_LANES * (1 + ADDR_WIDTH + DATA_WIDTH + DATA_SIZE + `UP(USER_WIDTH)) + TAG_WIDTH; localparam RSP_DATAW = NUM_LANES * (1 + DATA_WIDTH) + TAG_WIDTH; @@ -45,7 +47,7 @@ module VX_lsu_mem_arb import VX_gpu_pkg::*; #( wire [NUM_OUTPUTS-1:0] req_valid_out; wire [NUM_OUTPUTS-1:0][REQ_DATAW-1:0] req_data_out; wire [NUM_OUTPUTS-1:0] req_ready_out; - wire [NUM_OUTPUTS-1:0][`UP(LOG_NUM_REQS)-1:0] req_sel_out; + wire [SEL_COUNT-1:0][SEL_W-1:0] req_sel_out; wire [NUM_INPUTS-1:0] req_valid_in; wire [NUM_INPUTS-1:0][REQ_DATAW-1:0] req_data_in; diff --git a/hw/rtl/mem/VX_mem_bus_arb.sv b/hw/rtl/mem/VX_mem_bus_arb.sv index 3b9fc8198f..70f16613d7 100644 --- a/hw/rtl/mem/VX_mem_bus_arb.sv +++ b/hw/rtl/mem/VX_mem_bus_arb.sv @@ -37,6 +37,7 @@ module VX_mem_bus_arb import VX_gpu_pkg::*; #( localparam REQ_DATAW = 1 + ADDR_WIDTH + DATA_WIDTH + DATA_SIZE + ATTR_WIDTH + TAG_WIDTH; localparam RSP_DATAW = DATA_WIDTH + TAG_WIDTH; localparam SEL_COUNT = `MIN(NUM_INPUTS, NUM_OUTPUTS); + localparam SEL_W = `UP(`CLOG2(`CDIV(`MAX(NUM_INPUTS, NUM_OUTPUTS), SEL_COUNT))); wire [NUM_INPUTS-1:0] req_valid_in; wire [NUM_INPUTS-1:0][REQ_DATAW-1:0] req_data_in; @@ -44,7 +45,7 @@ module VX_mem_bus_arb import VX_gpu_pkg::*; #( wire [NUM_OUTPUTS-1:0] req_valid_out; wire [NUM_OUTPUTS-1:0][REQ_DATAW-1:0] req_data_out; - wire [SEL_COUNT-1:0][`UP(LOG_NUM_REQS)-1:0] req_sel_out; + wire [SEL_COUNT-1:0][SEL_W-1:0] req_sel_out; wire [NUM_OUTPUTS-1:0] req_ready_out; for (genvar i = 0; i < NUM_INPUTS; ++i) begin : g_req_data_in From 18cccfdd5ce2dc6a17c9968688d564cf4327c6db Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 21 Oct 2026 15:37:58 -0700 Subject: [PATCH 074/124] lsu: select the stack-interleave high bits instead of adding the correction The AGU's last step, addr_hi = lin_hi - off_borrow + addr_carry, was a carry chain waiting on the two group-field corrections, which settle last. Their difference is -1, 0 or +1, so lin_hi - 1 and lin_hi + 1 are now formed alongside them and the corrections only select. The result is exactly the same for every input. This path (lane dispatch -> AGU -> memory scheduler request queue) limits the core DUT. On pblocks: core 276 -> 290.3 MHz, all 49 perf_gate rtlsim cases cycle-identical. rtl-port's core DUT met 285.8 MHz before this change. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/core/VX_lsu_agu.sv | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/hw/rtl/core/VX_lsu_agu.sv b/hw/rtl/core/VX_lsu_agu.sv index c9dde01271..960e9c1b6e 100644 --- a/hw/rtl/core/VX_lsu_agu.sv +++ b/hw/rtl/core/VX_lsu_agu.sv @@ -155,9 +155,12 @@ module VX_lsu_agu import VX_gpu_pkg::*; ( assign addr_carry = sum[CORR_BITS]; end - // The window base cancels on the high bits; only the disagreement - // between the two group-field corrections survives. - wire [HI_BITS-1:0] addr_hi = lin_hi - HI_BITS'(off_borrow) + HI_BITS'(addr_carry); + // Only the two group-field corrections survive on the high bits. They + // settle last and differ by at most one, so they select a precomputed neighbour. + wire [HI_BITS-1:0] lin_hi_dec = lin_hi - HI_BITS'(1); + wire [HI_BITS-1:0] lin_hi_inc = lin_hi + HI_BITS'(1); + wire [HI_BITS-1:0] addr_hi = (off_borrow == addr_carry) ? lin_hi + : (off_borrow ? lin_hi_dec : lin_hi_inc); assign addr = in_stack ? {addr_hi, addr_lo} : lin_addr; end else begin : g_stack_linear From 5a3e84facf639ab11575db8885545d365f3c8a98 Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 21 Oct 2026 15:37:59 -0700 Subject: [PATCH 075/124] core: decide the warp-control unit's ready from the opcode VX_wctl_unit's execute_if.ready depended on whether a tmc/pred exits, which reduces the operand-selected thread mask to zero. That put a deep cone from the SFU dispatch buffer's output register back to its own enable; on pblocks it dropped the core DUT from 288.5 to 250 MHz. Ready now uses opcode-only terms. Every tmc/pred at the end of a packet counts as a possible posted request, so it waits for room for one; the request itself is still sent only for a real exit, and the request state machine is unchanged. With one SFU block the shared unit always accepts, so nothing changes there. With several, a non-exiting tmc/pred waits while the 2-entry request buffer is full. A direct request is answered in the cycle it is sent and never reaches the wait state, so the wait-state term, which carries the shared unit's same-cycle response, is dropped for it. On pblocks the core DUT recovered to 276 MHz (290.3 MHz together with the LSU AGU carry-select), with all 49 perf_gate rtlsim cases cycle-identical. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/core/VX_wctl_unit.sv | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/hw/rtl/core/VX_wctl_unit.sv b/hw/rtl/core/VX_wctl_unit.sv index 7a2a0117de..60b4cef82a 100644 --- a/hw/rtl/core/VX_wctl_unit.sv +++ b/hw/rtl/core/VX_wctl_unit.sv @@ -186,6 +186,11 @@ module VX_wctl_unit import VX_gpu_pkg::*; #( wire is_shared = execute_if.data.header.eop && (is_bar || is_wspawn || is_exit); wire sh_posted = is_shared && ~bar_rsp_valid; + // Ready is decided from the opcode: whether a tmc/pred exits depends on its + // operand mask, so every tmc/pred waits for room for a posted request. + wire may_share = execute_if.data.header.eop && (is_bar || is_wspawn || is_tmc || is_pred); + wire may_post = may_share && ~bar_rsp_valid; + // an exit is retired by the shared unit, which deactivates the warp itself assign tmc_valid = wctl_valid && (is_tmc || is_pred) && !is_shared; @@ -235,10 +240,12 @@ module VX_wctl_unit import VX_gpu_pkg::*; #( end wire sh_req_fire = sh_req_valid && sh_req_ready; - wire sh_ok = ~is_shared || (sh_state == SH_DONE) - || ((sh_state == SH_WAIT) && warp_shared_if.rsp_valid) - || (sh_req_fire && (sh_posted || warp_shared_if.rsp_valid)); - wire exec_ok = rsp_buf_ready && !wsync_drain && sh_ok && (is_shared || !bar_drain); + wire sh_can_req = (sh_state == SH_IDLE) && !bar_drain && sh_req_ready; + // a direct request is answered in the cycle it is sent, so it never waits + wire sh_ok = ~may_share || (sh_state == SH_DONE) + || (!SH_DIRECT && (sh_state == SH_WAIT) && warp_shared_if.rsp_valid) + || (sh_can_req && (may_post || SH_DIRECT)); + wire exec_ok = rsp_buf_ready && !wsync_drain && sh_ok && (may_share || !bar_drain); assign execute_if.ready = exec_ok; wire execute_fire = execute_if.valid && execute_if.ready; From f0667ebd8999aedb845f3512874ffc52b75a2d75 Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 21 Oct 2026 15:38:00 -0700 Subject: [PATCH 076/124] simx: hold off a local-memory read after any write and release the coalescer input early Two SimX timing fixes on the LSU's local-memory path, each matching the RTL: - A local-memory read of the word written the previous cycle waits one cycle whichever port wrote it: DMA, an LSU store or an atomic's write-back. The RTL stalls on all three; SimX modeled the atomic only. - The coalescer releases its input as it builds the last batch of a request, as the RTL raises its input ready in that state, not when the batch is accepted downstream. On pblocks, model_parity mean error went from 1.38% to 1.22%, max 4.48%. Co-Authored-By: Claude Opus 5.5 --- sim/simx/mem/local_mem.cpp | 73 ++++++++++++++++++++-------------- sim/simx/mem/mem_coalescer.cpp | 15 ++++--- sim/simx/mem/mem_coalescer.h | 1 - 3 files changed, 50 insertions(+), 39 deletions(-) diff --git a/sim/simx/mem/local_mem.cpp b/sim/simx/mem/local_mem.cpp index 925bcc3b92..211f46c5d7 100644 --- a/sim/simx/mem/local_mem.cpp +++ b/sim/simx/mem/local_mem.cpp @@ -40,18 +40,19 @@ class LocalMem::Impl { // else meanwhile. static constexpr uint32_t AMO_BANK_STALL_CYCLES = 2; std::vector amo_busy_; - // A read issued the cycle after a write to the same word returns the word's - // pre-update value, so the bank holds such a read off for one cycle. The - // write-back is the third cycle of the occupancy, so this shadow covers the - // fourth -- the cycle the next contending atomic arrives, which is why - // back-to-back atomics on one word cost four cycles and not three. - std::vector amo_rdw_addr_; - std::vector amo_rdw_valid_; - // Whether the atomic now occupying the bank ends in a write at all. A - // load-reserved never writes and a failed store-conditional must not, so - // neither leaves a shadow behind. + // The word the atomic now occupying the bank writes back, and whether it + // writes at all: a load-reserved never writes and a failed + // store-conditional must not, so neither leaves a shadow behind. + std::vector amo_wb_addr_; std::vector amo_rdw_pending_; #endif + // A read of a word written the previous cycle (by DMA, a store or an atomic's + // write-back) would see the old value, so the bank holds it off one cycle. + // rdw_* records this cycle's writes; rdw_cur_* holds last cycle's. + std::vector rdw_addr_; + std::vector rdw_valid_; + std::vector rdw_cur_addr_; + std::vector rdw_cur_valid_; MemCrossBar::Ptr mem_xbar_; mutable PerfStats perf_stats_; uint32_t dma_last_grant_ = 0; @@ -60,13 +61,11 @@ class LocalMem::Impl { return bit_getw(addr, 0, addr_bits_-1); } -#if VX_CFG_EXT_A_ENABLED // The word a bank is addressed by. Two byte addresses falling in one word // are the same access as far as the bank is concerned. uint64_t bank_word(uint64_t addr) { return to_local_addr(addr) >> log2ceil(config_.line_size); } -#endif public: Impl(LocalMem* simobject, const Config& config) @@ -77,10 +76,13 @@ class LocalMem::Impl { #if VX_CFG_EXT_A_ENABLED , amo_unit_(VX_CFG_AMO_RS_SIZE < 2 ? 2u : (uint32_t)VX_CFG_AMO_RS_SIZE) , amo_busy_(1 << config.B, 0) - , amo_rdw_addr_(1 << config.B, 0) - , amo_rdw_valid_(1 << config.B, false) + , amo_wb_addr_(1 << config.B, 0) , amo_rdw_pending_(1 << config.B, false) #endif + , rdw_addr_(1 << config.B, 0) + , rdw_valid_(1 << config.B, false) + , rdw_cur_addr_(1 << config.B, 0) + , rdw_cur_valid_(1 << config.B, false) { char sname[100]; snprintf(sname, 100, "%s-xbar", simobject->name().c_str()); @@ -115,9 +117,10 @@ class LocalMem::Impl { #if VX_CFG_EXT_A_ENABLED amo_unit_.reset(); std::fill(amo_busy_.begin(), amo_busy_.end(), 0); - std::fill(amo_rdw_valid_.begin(), amo_rdw_valid_.end(), false); std::fill(amo_rdw_pending_.begin(), amo_rdw_pending_.end(), false); #endif + std::fill(rdw_valid_.begin(), rdw_valid_.end(), false); + std::fill(rdw_cur_valid_.begin(), rdw_cur_valid_.end(), false); } // Serve one DMA client's row access this cycle and return the banks it @@ -160,7 +163,10 @@ class LocalMem::Impl { if (req.byteen & (1ull << b)) { uint8_t value = (*req.data)[b]; ram_.write(&value, line_addr + b, 1); - banks |= 1ull << (((line_addr + b) >> lg2_line_size) & (num_banks - 1)); + uint32_t bank = ((line_addr + b) >> lg2_line_size) & (num_banks - 1); + banks |= 1ull << bank; + rdw_addr_[bank] = this->bank_word(line_addr + b); + rdw_valid_[bank] = true; } } #if VX_CFG_EXT_A_ENABLED @@ -188,6 +194,10 @@ class LocalMem::Impl { } void tick() { + rdw_cur_addr_.swap(rdw_addr_); + rdw_cur_valid_.swap(rdw_valid_); + std::fill(rdw_valid_.begin(), rdw_valid_.end(), false); + uint64_t dma_banks = this->dma_tick(); // process bank requets from xbar @@ -196,16 +206,13 @@ class LocalMem::Impl { #if VX_CFG_EXT_A_ENABLED // Cycles an in-flight atomic owns; the bank serves nothing else. if (amo_busy_[i] != 0) { - if (0 == --amo_busy_[i]) { - amo_rdw_valid_[i] = amo_rdw_pending_[i]; // write-back just issued + if (0 == --amo_busy_[i] && amo_rdw_pending_[i]) { + // write-back just issued + rdw_addr_[i] = amo_wb_addr_[i]; + rdw_valid_[i] = true; } continue; } - // The shadow is one cycle wide: consume it here whether or not - // anything is waiting on the bank. - const bool rdw_shadow = amo_rdw_valid_[i]; - const uint64_t rdw_addr = amo_rdw_addr_[i]; - amo_rdw_valid_[i] = false; #endif if ((dma_banks >> i) & 1) { continue; @@ -216,18 +223,22 @@ class LocalMem::Impl { auto& bank_req = xbar_req_out.peek(); + // A read of the word written last cycle waits a cycle. An atomic + // reads the bank for its old value, so it counts as a reader here + // even though the request carries a write flag; that is what makes + // contending atomics on one word cost four cycles. #if VX_CFG_EXT_A_ENABLED - // A read of the word the write-back just stored waits a cycle. An - // atomic reads the bank for its old value, so it counts as a reader - // here even though the request carries a write flag; that is what - // makes contending atomics on one word cost four cycles. const bool reads_bank = !bank_req.is_write() || memop_is_atomic(bank_req.op); - if (rdw_shadow +#else + const bool reads_bank = !bank_req.is_write(); +#endif + if (rdw_cur_valid_[i] && reads_bank - && this->bank_word(bank_req.addr) == rdw_addr) { + && this->bank_word(bank_req.addr) == rdw_cur_addr_[i]) { continue; } +#if VX_CFG_EXT_A_ENABLED // Shared-memory atomics: read-modify-write in place and ALWAYS // return the old word. An AMO has a destination register, so the // LSU reserved a response slot; a missing response hangs the warp @@ -285,7 +296,7 @@ class LocalMem::Impl { perf_stats_.reads += !do_store; perf_stats_.writes += do_store; DT(4, simobject_->name() << "-bank" << i << " amo : " << bank_req); - amo_rdw_addr_[i] = this->bank_word(bank_req.addr); + amo_wb_addr_[i] = this->bank_word(bank_req.addr); amo_rdw_pending_[i] = do_store; xbar_req_out.pop(); amo_busy_[i] = AMO_BANK_STALL_CYCLES; // capture, then write back @@ -302,6 +313,8 @@ class LocalMem::Impl { ram_.write(&value, line_addr + b, 1); } } + rdw_addr_[i] = this->bank_word(bank_req.addr); + rdw_valid_[i] = true; #if VX_CFG_EXT_A_ENABLED // A plain store breaks a reservation on the word it writes, the // same as an atomic's commit does; otherwise a store between a diff --git a/sim/simx/mem/mem_coalescer.cpp b/sim/simx/mem/mem_coalescer.cpp index 239acc1b50..bf25a8f05e 100644 --- a/sim/simx/mem/mem_coalescer.cpp +++ b/sim/simx/mem/mem_coalescer.cpp @@ -256,7 +256,6 @@ void MemCoalescer::on_tick() { // build per-channel memory requests out_round_.valid = true; out_round_.lanes = out_mask; - out_round_.cur_mask = cur_mask; out_round_.reqs.assign(output_size_, MemReq{}); for (uint32_t o = 0; o < output_size_; ++o) { if (!out_mask.test(o)) { @@ -285,7 +284,13 @@ void MemCoalescer::on_tick() { // The built round is presented to the downstream on the next tick, not // this one: the coalescer alternates a build tick with a drain tick, so a // batch issues at most every other cycle. Draining is handled at the top - // of on_tick when out_round_ is valid. + // of on_tick when out_round_ is valid. The input is released as its last + // batch is built, without waiting for that batch to be accepted. + sent_mask_ |= cur_mask; + if (sent_mask_ == in_req.mask) { + ReqIn.pop(); + sent_mask_.reset(); + } } // Issue the pending round's per-channel requests; commit the round once @@ -304,12 +309,6 @@ void MemCoalescer::flush_out_round() { } out_round_.valid = false; - auto& in_req = ReqIn.peek(); - sent_mask_ |= out_round_.cur_mask; - if (sent_mask_ == in_req.mask) { - ReqIn.pop(); - sent_mask_.reset(); - } } const MemCoalescer::PerfStats& MemCoalescer::perf_stats() const { diff --git a/sim/simx/mem/mem_coalescer.h b/sim/simx/mem/mem_coalescer.h index e133c4be1a..3cc6d72c8d 100644 --- a/sim/simx/mem/mem_coalescer.h +++ b/sim/simx/mem/mem_coalescer.h @@ -67,7 +67,6 @@ class MemCoalescer : public SimObject { bool valid = false; BitVector<> lanes; // output lanes still to send std::vector reqs; // per output lane - BitVector<> cur_mask; // input lanes covered by this round }; uint32_t input_size_; From b3cad81fb8b0ac00f59f1ac6cf2e81b7fe59650c Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 21 Oct 2026 15:43:21 -0700 Subject: [PATCH 077/124] simx: launch each cluster on the nearest ready core The RTL launch fan-out (VX_kmu_bus_arb) round-robins compute messages over its ready outputs, starting from its pointer, and puts a two-entry skid in front of each output. SimX sent every cluster to the core the pointer named and stalled while that core was busy. It also let each core hold four CTAs it had not yet admitted. The two differences only matter when cores finish their CTAs at different times. The raycaster's grid is four CTAs wide, so the fixed rotation gave each of four cores one column of the image. The two middle columns hold the teapot and carry most of the work: SimX put 114,408 of 259,276 instructions on core 1 and took 478,511 cycles. The RTL lets the cores that finish early take the next CTAs and took 420,640 cycles, so SimX was 13.8% high. An rtlsim trace of a two-core SGEMM shows the same steering: at the start of the second wave, core 1 accepts three CTAs before core 0 accepts any. The four-deep lanes came in with the admission credits. Strict rotation was then added to undo the skew they caused on a two-core SGEMM. Two credits, which match the RTL's skid depth, together with the RTL's work-conserving choice, fix both. A single-core launch is unchanged, since it has only one lane to choose. The lanes stay flat, one per core. That matches the RTL wherever only one level of the launch tree fans out, which covers every parity shape. Where two levels fan out, as in several clusters of several cores, the upper level's shared skids are not modelled. The design doc now says so, and its description of SimX's steering and credits is corrected. raycast cycles, SimX before -> after vs rtlsim: 4c + L2 478,511 -> 419,820 vs 420,640 (+13.8% -> -0.2%) 2c IW4 8-bank 829,292 -> 767,309 vs 769,586 (+7.8% -> -0.3%) 2c IW4 8-bank CP_LLC=1 841,841 -> 748,610 vs 731,508 (the remaining +2.3% is the cycle counter, fixed separately) default (1 core) 1,494,492, unchanged On two cores the per-core instruction split now matches the RTL exactly: 122,571 / 136,673 with CP_LLC=1 and 128,064 / 131,180 without. sgemm -n32 is unchanged at 115,067 (1 core) and 30,086 (4c + L2). CI adds parity-raycast-mc (raycast, four cores, L2), so CTA placement under uneven work is checked against the RTL. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/core.yaml | 9 +++++++ docs/designs/cta_dispatch_architecture.md | 32 +++++++++++++---------- sim/simx/kmu/kmu.cpp | 25 ++++++++++-------- sim/simx/kmu/kmu.h | 9 +++---- 4 files changed, 45 insertions(+), 30 deletions(-) diff --git a/ci/testcases/core.yaml b/ci/testcases/core.yaml index 1ae0064d19..1ccfd9dcd7 100644 --- a/ci/testcases/core.yaml +++ b/ci/testcases/core.yaml @@ -233,3 +233,12 @@ tests: via: blackbox app: raycast configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 +# The raycaster on four cores sharing an L2: its image columns carry very +# different work, so where the launch places each CTA sets the critical path. +- id: parity-raycast-mc + check: model_parity + via: blackbox + app: raycast + shape: + cores: 4 + l2cache: true diff --git a/docs/designs/cta_dispatch_architecture.md b/docs/designs/cta_dispatch_architecture.md index 7b8af01a77..7906fd124a 100644 --- a/docs/designs/cta_dispatch_architecture.md +++ b/docs/designs/cta_dispatch_architecture.md @@ -462,10 +462,10 @@ Two preconditions make this sound, both already enforced: SimX has no bus, so its `Kmu` carries the routing itself: the destination is fixed **before** a message is sent — a locked cluster's remaining members go -only to the core that took the first, and a new message goes to the core the -round-robin already points at, stalling on a busy core rather than yielding -its turn to a neighbour -([`kmu.cpp:122-153`](../../sim/simx/kmu/kmu.cpp#L122)). +only to the core that took the first, and a new message goes to the nearest +ready core from the round-robin pointer, as the RTL fan-out picks the nearest +ready output +([`kmu.cpp:122-156`](../../sim/simx/kmu/kmu.cpp#L122)). **Why this is load-bearing rather than an optimisation.** The obvious consequence of a split cluster is that LMEM peers resolve to another CTA's @@ -733,16 +733,20 @@ is cycle-for-cycle identical to it on `vecadd` / `sgemm` / `sgemm_tcu_wg` / The KMU→core transport is modelled as per-core **launch lanes**: each lane is a registered boundary stage feeding the dispatcher's endpoint, and the KMU -additionally paces every core with explicit credits — `LAUNCH_CREDITS = 4` -per core ([`kmu.h:59`](../../sim/simx/kmu/kmu.h#L59)) — spent when a CTA is +additionally paces every core with explicit credits — `LAUNCH_CREDITS = 2` +per core ([`kmu.h:58`](../../sim/simx/kmu/kmu.h#L58)) — spent when a CTA is sent and returned by the dispatcher one message per admission -([`cta_dispatcher.cpp:142`](../../sim/simx/cta_dispatcher.cpp#L142)). The -bound is the transport's occupancy made explicit — the boundary stage's -endpoint (2) plus the dispatcher endpoint (1) plus one credit in return -flight — so a busy core never accumulates work beyond what the RTL's skids -would hold, and pacing rests on slot accounting rather than on peeking at -the consumer's state. RTL needs no counter: its registered output skids -apply the same bound as wires. +([`cta_dispatcher.cpp:175`](../../sim/simx/cta_dispatcher.cpp#L175)). The +bound is the two-entry output skid in front of each core made explicit, so a +busy core never holds more unadmitted CTAs than the RTL's skid would, and +pacing rests on slot accounting rather than on peeking at the consumer's +state. The depth matters for placement, not only for pacing: every CTA +committed to a busy core is one a core that frees a slot sooner cannot take. +RTL needs no counter: its registered output skids apply the same bound as +wires. The lanes are flat, one per core, so the model is exact while a single +level of the tree fans out; where two levels fan out (several clusters of +several cores, or several sockets of several cores) the upper level's shared +skids are not modelled. The differences are deliberate, not drift: @@ -754,7 +758,7 @@ The differences are deliberate, not drift: | Bus payload | `cluster_size` (scalar), `aligned_lmem_size` | `cluster_dim[3]`, raw `lmem_size` | | `cta_id` on the bus | absent | present (model-only, unused as identity) | | Stride rounding | in the KMU | in the dispatcher | -| Launch pacing | elastic-skid back-pressure | explicit credits (`LAUNCH_CREDITS = 4`) | +| Launch pacing | elastic-skid back-pressure | explicit credits (`LAUNCH_CREDITS = 2`) | | Fragment path | through `kmu_bus_if` → `VX_cta_dispatch` | injected at the scheduler by the FWD | The last row is the significant one: in RTL a fragment wave is a launch that diff --git a/sim/simx/kmu/kmu.cpp b/sim/simx/kmu/kmu.cpp index ab20f74be8..58734cce0e 100644 --- a/sim/simx/kmu/kmu.cpp +++ b/sim/simx/kmu/kmu.cpp @@ -120,15 +120,12 @@ void Kmu::on_tick() { return; } // Steering: a locked cluster's remaining members stay with the owning core; - // otherwise the next cluster goes to the core the round-robin already points - // at. One CTA per cycle — the launch bus carries a single beat. A lane is - // ready when its transport has room and the core still owes fewer than - // LAUNCH_CREDITS admissions, so a busy core never accumulates work. - // - // The destination is fixed before the message is sent, so a busy core stalls - // the bus instead of yielding its turn to a neighbour. Scanning for any ready - // lane here would let a core that drains faster absorb its neighbour's share - // and leave the placement lopsided. + // otherwise the next cluster goes to the nearest ready lane from the + // round-robin pointer, so a core that drains its work sooner takes the next + // cluster instead of the launch waiting on a busy neighbour. One CTA per + // cycle — the launch bus carries a single beat. A lane is ready when its + // transport has room and the core holds fewer than LAUNCH_CREDITS CTAs it + // has not yet admitted. auto lane_ready = [&](uint32_t c) { return !bus_out.at(c).full() && inflight_[c] < LAUNCH_CREDITS; }; @@ -137,8 +134,14 @@ void Kmu::on_tick() { if (lane_ready(cluster_core_)) { target = cluster_core_; } - } else if (lane_ready(rr_ptr_)) { - target = rr_ptr_; + } else { + for (uint32_t k = 0; k < num_cores; ++k) { + uint32_t c = (rr_ptr_ + k) % num_cores; + if (lane_ready(c)) { + target = c; + break; + } + } } if (target == num_cores) return; // no ready lane this cycle diff --git a/sim/simx/kmu/kmu.h b/sim/simx/kmu/kmu.h index 537703601c..687bd40ec9 100644 --- a/sim/simx/kmu/kmu.h +++ b/sim/simx/kmu/kmu.h @@ -52,11 +52,10 @@ class Kmu : public SimObject { // rests on explicit slot accounting, not on transport back-pressure. std::vector> credit_in; - // In-flight bound per core: the boundary stage's endpoint (2) plus the - // dispatcher endpoint (1) plus one admission credit in return flight, so - // the lane's full() view stays the binding gate whenever the transport - // applies back-pressure. - static constexpr uint32_t LAUNCH_CREDITS = 4; + // Each core's launch buffer holds two CTAs it has not yet admitted. A + // deeper buffer would commit work to a busy core early and keep it from a + // core that frees a slot sooner. + static constexpr uint32_t LAUNCH_CREDITS = 2; Kmu(const SimContext& ctx, const char* name, uint32_t num_cores); From 4470f277f1828129e575d1bd1575cfcadad4e76f Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 21 Oct 2026 15:43:21 -0700 Subject: [PATCH 078/124] simx: count a core's cycles only while it has work In the RTL, a core's cycle counter advances only while its scheduler is busy. VX_scheduler increments cycles_r under `if (busy)`. busy is the registered OR of three terms: a warp is active (active_warps != 0), a CTA is firing (cta_fire), or an instruction is in flight (the warp scheduler's per-warp issued-minus-committed counter is not empty). That is ORed with the CTA dispatcher's busy, which covers dispatching and accepting a launch. The counter, sched_cycles, is the one VX_csr_mpm returns to the host for VX_CSR_MCYCLE, and the one VX_csr_data returns to a kernel that reads mcycle. SimX advanced its counter on every tick. It therefore also charged a core for the cycles before its first CTA arrived, the cycles after its last instruction committed, and every cycle the host spent moving data while the simulator ticked. The last case is the one that showed. With VX_CFG_CP_LLC_CLIENT, the command processor's uploads and readbacks go through the shared L2, and SimX ticks the whole machine until they drain. On raycast at eight cores, IW4 and 8 memory banks, that added about 19,000 cycles to every core: SimX reported 300,432 against the RTL's 282,021. The RTL's per-core counts show that its counter is gated. They range from 26,749 on the most idle core to 282,021 on the busiest. SimX now reports 26,747 to 281,015, within 0.4% on every core. Count while Scheduler::running() or an instruction is in flight. Those are the RTL's terms: a warp active, a CTA firing, the dispatcher holding a CTA, and an instruction scheduled but not yet committed. SimX's dispatcher also counts the one or two cycles a CTA waits at the core's launch endpoint before it is admitted, which the RTL holds in the fan-out's skid instead. A core that stays busy from launch to completion moves only by those edge cycles: raycast -74 and sgemm -n32 -39 at one core. raycast cycles, SimX before -> after vs rtlsim (on top of the launch fix): 8c IW4 8-bank CP_LLC=1 300,432 -> 281,015 vs 282,021 (+6.5% -> -0.4%) 2c IW4 8-bank CP_LLC=1 748,610 -> 729,206 vs 731,508 (+2.3% -> -0.3%) 4c + L2 419,820 -> 419,477 vs 420,640 (-0.3%) model_parity-dogfood-cp-llc moves from +3.39% to +0.23%. Co-Authored-By: Claude Opus 5.5 --- sim/simx/core.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index 2455e026b1..bd84b24ea5 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -354,8 +354,8 @@ class Core::Impl { std::fill(ibuf_inflight_.begin(), ibuf_inflight_.end(), 0); - // perf_stats_ deliberately survives: these are the MPM counters, which are - // free-running from device reset. A kernel launch re-enters reset() here but + // perf_stats_ deliberately survives: these are the MPM counters, which + // accumulate from device reset. A kernel launch re-enters reset() here but // does not reset the hardware, so clearing them would make each dump report // only the last launch -- an app that dumps between launches would then see // per-launch counts where the device reports a running total. @@ -370,7 +370,13 @@ class Core::Impl { this->fetch(); this->schedule(); - ++perf_stats_.cycles; + // The cycle counter advances only while the core has work -- a CTA + // launching, a warp active, or an instruction in flight -- so cycles it + // spends idle before its first CTA, after its last warp, or while the host + // moves data are not charged to it. + if (scheduler_->running() || !pending_instrs_.empty()) { + ++perf_stats_.cycles; + } DPN(2, std::flush); } From 7039eea2666195fca071dfa1afd0781f4b014622 Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 21 Oct 2026 15:43:22 -0700 Subject: [PATCH 079/124] ci: size the fp16-mc parity case like its perf_gate twin At its default 64^3, model_parity-fp16-mc runs 64 one-warp CTAs on two cores with eight slots each, which is four waves in about 18,000 cycles. A core admits its next CTA into its round-robin tail slot, so one CTA that runs long blocks the core's next wave, and the work-conserving launch then gives that core's share to the other core. The split, and with it the total, moves with the timing of a single CTA. With the launch fix, SimX's per-CTA times match the RTL's to within 1% on average in each of the first two waves (5,074 / 5,077 against 5,103 / 5,050, and 4,160 / 4,155 against 4,191 / 4,255 cycles). But one second-wave CTA on core 0 ran 4,494 cycles in SimX against about 4,000 for the rest. That held core 0's third wave back by 550 cycles, and core 1 ended with 34 CTAs to core 0's 30, where the RTL split them 32 / 32. The case then read +8.15% (19,315 vs 17,859). Before the launch fix, strict rotation forced a 32 / 32 split whatever the timing, so the sensitivity was hidden. At 128^3, the size perf_gate-fp16-mc already runs, the same configuration gives 123,402 against the RTL's 120,867 (+2.1%; +1.3% before the launch fix). The variation of a single CTA no longer decides the result. The tolerance stays at 5%. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/tensor.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/testcases/tensor.yaml b/ci/testcases/tensor.yaml index 246774db01..52b0799215 100644 --- a/ci/testcases/tensor.yaml +++ b/ci/testcases/tensor.yaml @@ -193,6 +193,9 @@ tests: check: model_parity via: blackbox app: sgemm_tcu + # Sized like its perf_gate twin: at the default 64^3 the grid is four waves of + # CTAs, so one CTA running long moves the per-core split and the cycle count. + args: -m 128 -n 128 -k 128 configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 shape: {cores: 2, l2cache: true} From 8d9335a34b4b1f5dd16f356c196feea8ee3c449d Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 21 Oct 2026 16:45:49 -0700 Subject: [PATCH 080/124] CHANGELOG: memory-level parallelism, phase 1 Record the features, design changes and bug fixes on this branch that are not yet on master: per-partition fetch, the hashed bank select, the dcache read merge, write-allocate without fetch, the MSHR line table, the command processor's DMA through the shared LLC, the flush and fence redesign, the LMEM DMA arbitration, the SimX timing model, and the bugs whose faults were also present on master. Optimizations under 10% and fixes to code that is new on this branch are left out. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b384f6a82b..ec2d93d728 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,31 @@ follows the version pins recorded in [VERSION](VERSION) (`VORTEX_VERSION`, - **Hardware profiles.** `profiles/.json` names a machine (`tinygpu`, `rtx1080`, `v100`, `a100`, `h100`, `rtx5080`) that every flow builds through `configure --profile=` or `blackbox.sh --profile=`; explicit `CONFIGS` win over the profile. See [docs/simulation.md](docs/simulation.md#hardware-profiles). +- **Per-partition fetch (`VX_CFG_FETCH_WIDTH`, default 1).** The front end splits into FETCH_WIDTH slices, each with its own warp + scheduler, fetch, decode and icache port, serving the warps with `wid % FETCH_WIDTH == f`; CTA dispatch, barriers and warp control + stay core-wide. The core takes one of three structures from the fetch width, issue width and per-unit block counts; with one block + of every unit per issue slice and FETCH_WIDTH equal to ISSUE_WIDTH, each slice is a self-contained compute slice (the NVIDIA + processing-block organization). *Why:* sgemm at 16 warps x 16 threads, ISSUE_WIDTH 4: -31.6% cycles at FETCH_WIDTH 2. +- **Hashed cache bank select (`VX_CFG_{DCACHE,L2}_BANK_HASH`).** The bank index is the raw bank bits XORed with a hash of the line + address above them (IPOLY(5) for 16 banks, IPOLY(37) for 32, an XOR fold otherwise), so power-of-two strides spread over the banks. + On by default for the L2 and for a dcache with 4 or more banks. *Why:* rtlsim sgemv 512x512 at 16 threads -32.8%, fp16 TCU SGEMM + on 2 cores + L2 at 16 threads -13.8%, WGMMA with DXA -13.5%. +- **Dcache read merge (`VX_CFG_DCACHE_READ_MERGE`, on by default).** Plain reads of the same word from the ports of one LSU block in + the same cycle are served by one bank access, and the response is multicast to every port; each port's order into the bank is + unchanged. *Why:* uniform loads (kernel arguments, SGEMM's shared operand) no longer serialize on one bank: rtlsim sgemm -n128 at + 16 threads -6.5%, -17.2% with an 8-byte dcache word. +- **Write-allocate without fetch (`VX_CFG_{DCACHE,L2,L3}_LAZY_FETCH`, off by default).** A store miss in a write-back cache installs + the sector with its dirty-byte mask and a partial bit instead of fetching the line; a later load or AMO merges memory data under the + unwritten bytes, and evictions write back only dirty bytes. *Why:* streaming writes no longer read every line first: 1-core vecadd + DRAM reads -33%. +- **MSHR line table (`VX_CFG_{DCACHE,L2,L3}_MSHR_LINES`, off by default).** Misses are tracked in a reference-counted table of + distinct {line, sector} entries, and request entries keep only an index into it, so the MSHR holds more requests without widening + its address compares. *Why:* at 400-cycle memory latency, 64 requests over 16 lines: sgemm -n128 -15.3%, raycast -8.1%. +- **Command-processor DMA through the shared LLC (`VX_CFG_CP_LLC_CLIENT`, off by default).** When one LLC serves every core, the + CP's transfers enter it on their own client slot, so host transfers need only an L1 flush, not a walk of the whole LLC; transfers + of at least half the LLC write around it. A launch behind DMA writes waits for an L1-only flush token sent on the slot. The runtime + sees the new `DMA_LLC_CLIENT` CP capability. *Why:* rtlsim at 2 clusters + L2 + L3: dogfood -30%, multikernel -42.7%; dogfood + -64% with an 8 MB L3. ### Changed @@ -23,6 +48,48 @@ follows the version pins recorded in [VERSION](VERSION) (`VORTEX_VERSION`, (JEDEC HBM2/HBM3/GDDR6 organization and nanosecond timing, 32 B bursts, permuted open-page bank interleave, per-channel responses), configured from data-sheet values: `VX_CFG_PLATFORM_MEMORY_{TYPE,DATA_RATE,LATENCY,QUEUE_SIZE}`. The default machine keeps its idle read latency (14/26 cycles open-row/conflict, was 13/27) and one-block-per-cycle peak. +- **Cache flush architecture.** The CP starts every core's flush with one broadcast `CACHE_FLUSH` DCR write, then reads each core's + completion, so the flushes overlap; a flush read without a preceding write no longer starts one. A cache walks its banks, sends one + flush token down every memory port and answers once all ports have; every independently buffered write path (each dcache lane, each + OM input of the ocache) carries its own token. Read-only caches (icache, tcache, rtcache, rcache) no longer forward flushes. + `DBG_FLUSH_STALL` adds random input backpressure to the flush path for stress tests. +- **L1-only flush between launches.** When a single LLC serves every memory client it is the point of coherence, so a launch + boundary cleans only the L1s: `CMD_CACHE_FLUSH` gains an L1-only flag, carried as a flush-token kind that a shared LLC answers + without a walk. Replicated LLCs and LLCs with peer clients (TEX, RTU, RASTER, OM, DXA) still walk. The runtime posts flushes + lazily and flushes fully before any device-memory transfer, and device busy stays high until every core write, uncached IO stores + included, is visible in memory. *Why:* per-launch flush cost at 4 cores + L2 drops from 2,750 to 180 cycles. +- **Targeted fences.** A memory fence sends an L1-only flush token that the shared LLC answers without a walk. A fence that orders + only device (I/O) accesses decodes as its own LSU op, and its token makes each level drain to its memory ports without walking. + *Why:* printf, which fences after each device write, runs 13.6x faster on rtlsim at 2 cores + L2. +- **LMEM DMA arbitration.** DXA writes into shared memory and the TCU's WGMMA operand reads now share the local-memory DMA port + round-robin. DXA had fixed priority and starved the TCU for about 510 cycles per tiled-scatter transfer. *Why:* rtlsim WGMMA with + DXA -10.2%, sparse WGMMA with DXA -11.6%. +- **SimX timing model follows the RTL's structure.** Buffer capacities and registered ready at every pipeline boundary, per-slice + cache arbiter trees and network depths, registered cache core-response ports, flush and fence tokens end to end, and CTA launch onto + the nearest ready core with two launch credits per core. SimX's per-core cycle counter (`mcycle`) now advances only while the core + has work, as the RTL's does. rtlsim completes DRAM requests in per-bank arrival order, as the AXI platform does. + +### Fixed + +- **Global barriers across clusters.** Each cluster ran its own global-barrier unit that saw only its own cores, while + `vx_num_cores()`, and so `vortex::gbarrier`, counts the whole device. On SimX every global barrier hung with more than one + cluster; on the RTL, 2 clusters x 1 core timed out and 2 clusters x 2 cores released each cluster on its own arrivals, with no + ordering between clusters. One device-level unit now counts every core. +- **Writes left dirty by a flush.** A cache forwarded its flush token on memory port 0 only, so writes queued on another port could + reach the next level after that level's walk and stay dirty there. +- **Dirty sectors dropped on a sector refill.** With dirty-byte tracking and several sectors per line, installing a sector into a + resident line cleared the whole line's byte mask, so a dirty sibling sector wrote back with no bytes enabled and its data was lost + (mstress at 4 cores with a write-back, sectored L2). +- **Local-memory LR/SC livelock at 16 threads.** One warp's per-lane LR burst used up a reservation's fixed hold budget of 7 refusals, + so contended LR/SC loops on shared memory stopped making progress. The budget now scales with the core's harts. +- **`VX_stream_arb` trees with a partial last slice.** When the request count was not a whole number of slices, the last slice's + select was sized and placed as if it were full: the build failed lint, and with several outputs requests went to the wrong output. +- **`-DSCOPE` builds.** The scope buses of fetch, issue, LSU, TCU, TEX, RASTER and OM were declared only under their `DBG_SCOPE_*` + taps, left unbound, or did not parse, so debug-scope builds failed to elaborate. +- **mstress checked no data.** Its host reference was always 0, so the test passed with corrupted loads; it now checks every loaded + value exactly. +- **FPGA run instructions.** The docs exported `TARGET=hw`, which `blackbox.sh` ignores, so runs fell back to xrtsim; they now pass + `--target=hw`. ## [3.0] — 2026-10-07 From 60e7a2bbca0df5be6320029462e8ece86348348a Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 21 Oct 2026 16:45:49 -0700 Subject: [PATCH 081/124] tests: render raycast at 64x64 by default The 16x12 default gives each core only a handful of CTAs, so multi-core runs measure launch and tail effects more than the raycaster itself. A 64x64 render target gives every configuration enough work to reach a steady state. OPTS still overrides it. Co-Authored-By: Claude Opus 5.5 --- tests/regression/raycast/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/regression/raycast/Makefile b/tests/regression/raycast/Makefile index d736258fb2..b1dedb71a2 100644 --- a/tests/regression/raycast/Makefile +++ b/tests/regression/raycast/Makefile @@ -11,8 +11,8 @@ VX_SRCS := $(SRC_DIR)/kernel.cpp CXXFLAGS += -DASSETS_PATHS='"$(SRC_DIR)"' -# Default workload is small for fast CI sweeps; override OPTS for full-resolution runs. -OPTS ?= -w 16 -h 12 +# Default workload is a 64x64 render target; override OPTS for other resolutions. +OPTS ?= -w 64 -h 64 KERNEL_LIB := vortex2 From 4faf32cb4c0ba94eb108d677a628af73e793a0db Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 21 Oct 2026 18:07:24 -0700 Subject: [PATCH 082/124] cache: size the flush-token attribute and hash input to their fields Two Verilator warnings, fatal in CI, broke every rtlsim, opaesim and xrtsim build that reached them: - The memory-side bank hash read its line address through a full-width wire and used only its low 32 bits. At XLEN 64 the upper bits were unused (UNUSEDSIGNAL) whenever the hash was on: a dcache with 4 or more banks, or the L2/L3. The hash now takes the 32-bit slice directly, as the request side already does; the hash itself only reads the low bits, so the result is unchanged. - The flush token's attribute shifted 32-bit operands into a field that is wider with the A extension (47 bits with Zacas at XLEN 32, 79 at 64), a WIDTHEXPAND. Both operands are now cast to the field width. Lint with -Wall at the failing CI configs (amo with A+Zacas at XLEN 32 and 64, L1_LINE_SIZE=8 at 64, 2 clusters x 2 cores + L2 + L3 at 64) and at the default 32/64: the unfixed file reproduces each CI warning, the fixed one is clean. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/cache/VX_cache.sv | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hw/rtl/cache/VX_cache.sv b/hw/rtl/cache/VX_cache.sv index d51f0c3e28..926f807452 100644 --- a/hw/rtl/cache/VX_cache.sv +++ b/hw/rtl/cache/VX_cache.sv @@ -610,8 +610,8 @@ module VX_cache import VX_gpu_pkg::*; #( // the raw bank bits of the memory address, formed while the bank id is a constant wire [`UP(BANK_SEL_BITS)-1:0] raw_bid; if (NUM_BANKS > 1 && BANK_HASH != 0) begin : g_hash - wire [`CS_LINE_ADDR_WIDTH-1:0] line_addr = per_bank_mem_req_addr[i][`CS_LINE_SECTOR_ADDR_WIDTH-1 -: `CS_LINE_ADDR_WIDTH]; - assign raw_bid = BANK_SEL_BITS'(i) ^ BANK_SEL_BITS'(cache_bank_hash(32'(line_addr), BANK_SEL_BITS)); + assign raw_bid = BANK_SEL_BITS'(i) ^ BANK_SEL_BITS'(cache_bank_hash( + 32'(per_bank_mem_req_addr[i][`CS_LINE_SECTOR_ADDR_WIDTH-1 -: `CS_LINE_ADDR_WIDTH]), BANK_SEL_BITS)); end else begin : g_raw assign raw_bid = `UP(BANK_SEL_BITS)'(i); end @@ -656,7 +656,7 @@ module VX_cache import VX_gpu_pkg::*; #( SECTOR_SIZE'(mem_flush_l1_only), `CS_SECTOR_WIDTH'(0), BANK_MEM_TAG_WIDTH'(0), - `UP(MEM_ATTR_WIDTH)'((1 << MEM_ATTR_FLUSH_OFFS) | (32'(mem_flush_io) << MEM_ATTR_IO_OFFS)) + (`UP(MEM_ATTR_WIDTH)'(1) << MEM_ATTR_FLUSH_OFFS) | (`UP(MEM_ATTR_WIDTH)'(mem_flush_io) << MEM_ATTR_IO_OFFS) }; assign mem_req_xbar_valid[i] = per_bank_mem_req_valid[i] || mem_flush_valid[i]; assign mem_req_xbar_pdata[i] = per_bank_mem_req_valid[i] ? per_bank_mem_req_pdata[i] : mem_flush_pdata; From 3b241d7b208cfa8c70d9b013853bb1e3d9fd7d44 Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 21 Oct 2026 18:07:24 -0700 Subject: [PATCH 083/124] simx: let the CP LLC port build in the functional simulator cp_llc_port.h included simobject.h directly, while types.h picks simobject_functional.h under SIMX_FUNCTIONAL. A functional build saw both SimContext classes and failed (debug:simx_functional-1/2 in CI). The port now gets the simobject header through types.h like every other model. simx_functional-1 (sgemm -n128) and -2 (SIMX_MT=4, 4 cores + L2) pass. Co-Authored-By: Claude Opus 5.5 --- sim/simx/cp_llc_port.h | 1 - 1 file changed, 1 deletion(-) diff --git a/sim/simx/cp_llc_port.h b/sim/simx/cp_llc_port.h index a298216046..ab90a85876 100644 --- a/sim/simx/cp_llc_port.h +++ b/sim/simx/cp_llc_port.h @@ -13,7 +13,6 @@ #pragma once -#include #include #include #include From 8e2aa619c6e33a5cd2155e3e447afa6069153457 Mon Sep 17 00:00:00 2001 From: tinebp Date: Thu, 22 Oct 2026 02:15:49 -0700 Subject: [PATCH 084/124] core: give each LSU block its own memory path Each LSU block's memory scheduler and its per-block share of VX_mem_unit (the local-memory switch, the local-memory adapter, the coalescer and the D-cache adapter) were instantiated in VX_core, outside the slice they serve, so every load left the slice and came back. They now live in the block, in a new VX_lsu_mem_path, with their parameters and generate conditions unchanged. VX_mem_unit keeps what the blocks share: the local-memory banks and the D-cache flush injectors. The injectors stay in the core so that every slice stays identical (a private D-cache has an injector on lane 0 only); the flush token is told apart from the lane's responses by a tag bit, and the cache's response reaches the block as wires. No register is added or removed. rtlsim sgemm_tcu_wg_sp_dxa -m 128 -n 128 -k 128 (IW 4, FW 4, 4 LSU and TCU blocks, 32 threads): 144,956 cycles, identical to the parent, PASSED. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/VX_gpu_pkg.sv | 2 +- hw/rtl/core/VX_backend_slice.sv | 32 ++- hw/rtl/core/VX_compute_slice.sv | 32 ++- hw/rtl/core/VX_core.sv | 172 ++++++++------ hw/rtl/core/VX_execute.sv | 36 ++- hw/rtl/core/VX_execute_slice.sv | 42 +++- hw/rtl/core/VX_lsu_block.sv | 47 +++- hw/rtl/core/VX_lsu_mem_path.sv | 287 ++++++++++++++++++++++ hw/rtl/core/VX_lsu_unit.sv | 54 ++++- hw/rtl/core/VX_mem_unit.sv | 304 +++++------------------- hw/unittest/mem_unit/VX_mem_unit_top.sv | 101 ++++---- hw/unittest/tensor/VX_tensor_top.sv | 126 +++++----- 12 files changed, 786 insertions(+), 449 deletions(-) create mode 100644 hw/rtl/core/VX_lsu_mem_path.sv diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index 7acc4b6daf..782f1361c9 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -1721,7 +1721,7 @@ package VX_gpu_pkg; localparam DCACHE_CORE_TAG_WIDTH = (UUID_WIDTH + DCACHE_TAG_ID_BITS); // Core request tag bits on dcache_bus_if port of VX_core (+1 for flush-arb sel bit) - // Pre-MMU width (what VX_mem_unit produces, including the flush sel bit). + // Pre-MMU width (an LSU block's D-cache port, including the flush sel bit). // When VM_ENABLE is on, the dMMU widens this by DCACHE_TLB_SOURCE_BITS // for its internal bypass/TLB/PTW arbiter, so the cache cluster sees // the wider DCACHE_TAG_WIDTH below. diff --git a/hw/rtl/core/VX_backend_slice.sv b/hw/rtl/core/VX_backend_slice.sv index 93732a84cd..acc5767a67 100644 --- a/hw/rtl/core/VX_backend_slice.sv +++ b/hw/rtl/core/VX_backend_slice.sv @@ -56,7 +56,21 @@ import VX_tcu_pkg::*; `endif // LSU - VX_lsu_sched_if.master lsu_client_if, +`ifdef PERF_ENABLE + output wire [`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_rd_req_fire, + output wire [`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_wr_req_fire, + output wire [`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_rsp_fire, + output wire [PERF_CTR_BITS-1:0] lsu_coalescer_misses, +`endif +`ifdef TCU_META_ENABLE + VX_lsu_sched_if.slave lsu_tcu_mem_if, +`endif + output wire lsu_sched_empty, + output wire lsu_store_empty, +`ifdef VX_CFG_LMEM_ENABLE + VX_mem_bus_if.master lmem_bus_if [`VX_CFG_NUM_LSU_LANES], +`endif + VX_mem_bus_if.master dcache_bus_if [DCACHE_CHANNELS], // SFU `ifdef VX_CFG_EXT_DXA_ENABLE @@ -141,7 +155,21 @@ import VX_tcu_pkg::*; .dispatch_if (dispatch_if), .commit_if (commit_if), .branch_ctl_if (branch_ctl_if), - .lsu_client_if (lsu_client_if), + `ifdef PERF_ENABLE + .lsu_perf_rd_req_fire (lsu_perf_rd_req_fire), + .lsu_perf_wr_req_fire (lsu_perf_wr_req_fire), + .lsu_perf_rsp_fire (lsu_perf_rsp_fire), + .lsu_coalescer_misses (lsu_coalescer_misses), + `endif + `ifdef TCU_META_ENABLE + .lsu_tcu_mem_if (lsu_tcu_mem_if), + `endif + .lsu_sched_empty (lsu_sched_empty), + .lsu_store_empty (lsu_store_empty), + `ifdef VX_CFG_LMEM_ENABLE + .lmem_bus_if (lmem_bus_if), + `endif + .dcache_bus_if (dcache_bus_if), `ifdef VX_CFG_EXT_DXA_ENABLE .dxa_req_bus_if (dxa_req_bus_if), `endif diff --git a/hw/rtl/core/VX_compute_slice.sv b/hw/rtl/core/VX_compute_slice.sv index 0111b36d12..63be670437 100644 --- a/hw/rtl/core/VX_compute_slice.sv +++ b/hw/rtl/core/VX_compute_slice.sv @@ -59,7 +59,21 @@ import VX_tcu_pkg::*; `endif // LSU - VX_lsu_sched_if.master lsu_client_if, +`ifdef PERF_ENABLE + output wire [`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_rd_req_fire, + output wire [`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_wr_req_fire, + output wire [`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_rsp_fire, + output wire [PERF_CTR_BITS-1:0] lsu_coalescer_misses, +`endif +`ifdef TCU_META_ENABLE + VX_lsu_sched_if.slave lsu_tcu_mem_if, +`endif + output wire lsu_sched_empty, + output wire lsu_store_empty, +`ifdef VX_CFG_LMEM_ENABLE + VX_mem_bus_if.master lmem_bus_if [`VX_CFG_NUM_LSU_LANES], +`endif + VX_mem_bus_if.master dcache_bus_if [DCACHE_CHANNELS], // SFU `ifdef VX_CFG_EXT_DXA_ENABLE @@ -251,7 +265,21 @@ import VX_tcu_pkg::*; `ifdef VX_CFG_EXT_RTU_ENABLE .sched_unlock_if (sched_unlock_if), `endif - .lsu_client_if (lsu_client_if), + `ifdef PERF_ENABLE + .lsu_perf_rd_req_fire (lsu_perf_rd_req_fire), + .lsu_perf_wr_req_fire (lsu_perf_wr_req_fire), + .lsu_perf_rsp_fire (lsu_perf_rsp_fire), + .lsu_coalescer_misses (lsu_coalescer_misses), + `endif + `ifdef TCU_META_ENABLE + .lsu_tcu_mem_if (lsu_tcu_mem_if), + `endif + .lsu_sched_empty (lsu_sched_empty), + .lsu_store_empty (lsu_store_empty), + `ifdef VX_CFG_LMEM_ENABLE + .lmem_bus_if (lmem_bus_if), + `endif + .dcache_bus_if (dcache_bus_if), `ifdef VX_CFG_EXT_DXA_ENABLE .dxa_req_bus_if (dxa_req_bus_if), `endif diff --git a/hw/rtl/core/VX_core.sv b/hw/rtl/core/VX_core.sv index 4b1b9c6e42..aee719c076 100644 --- a/hw/rtl/core/VX_core.sv +++ b/hw/rtl/core/VX_core.sv @@ -80,18 +80,38 @@ import VX_tcu_pkg::*; VX_txbar_bus_if dxa_txbar_bus_if(); `endif - VX_lsu_mem_if #( - .NUM_LANES (`VX_CFG_NUM_LSU_LANES), + // the LSU blocks' memory paths + wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_sched_empty; + wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_store_empty; +`ifdef VX_CFG_LMEM_ENABLE + VX_mem_bus_if #( .DATA_SIZE (LSU_WORD_SIZE), .TAG_WIDTH (LSU_TAG_WIDTH) - ) lsu_mem_if[`VX_CFG_NUM_LSU_BLOCKS](); - - // per-LSU-block memory scheduler clients from execute - VX_lsu_sched_if lsu_client_if [`VX_CFG_NUM_LSU_BLOCKS](); - wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_sched_empty; + ) lsu_lmem_bus_if[LSU_NUM_REQS](); +`endif + VX_mem_bus_if #( + .DATA_SIZE (DCACHE_WORD_SIZE), + .TAG_WIDTH (DCACHE_TAG_WIDTH) + ) lsu_dcache_bus_if[DCACHE_NUM_REQS](); +`ifdef PERF_ENABLE + wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_rd_req_fire; + wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_wr_req_fire; + wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_rsp_fire; + wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][PERF_CTR_BITS-1:0] lsu_coalescer_misses; +`endif `ifdef TCU_META_ENABLE - VX_lsu_sched_if tcu_mem_if(); + // The TCU metadata AGU is the second client of LSU block 0's scheduler; the + // other blocks' second client is idle. + VX_lsu_sched_if lsu_tcu_mem_if [`VX_CFG_NUM_LSU_BLOCKS](); + for (genvar b = 1; b < `VX_CFG_NUM_LSU_BLOCKS; ++b) begin : g_lsu_tcu_tieoff + assign lsu_tcu_mem_if[b].req_valid = 1'b0; + assign lsu_tcu_mem_if[b].req_data = '0; + assign lsu_tcu_mem_if[b].rsp_ready = 1'b1; + `UNUSED_VAR (lsu_tcu_mem_if[b].req_ready) + `UNUSED_VAR (lsu_tcu_mem_if[b].rsp_valid) + `UNUSED_VAR (lsu_tcu_mem_if[b].rsp_data) + end `endif // frontend-slice icache requests; port 0 also carries the injected flush @@ -136,7 +156,7 @@ import VX_tcu_pkg::*; wire dcr_flush_l1_only; // the in-core global-store path is empty - wire mem_unit_empty; + wire mem_unit_empty = (& lsu_store_empty); // Stores are ack-less and lag warp exit, so the dcache flush waits until every // store of this core has reached L1. @@ -353,7 +373,7 @@ import VX_tcu_pkg::*; .meta_wr_wid (tcu_meta_wr_wid), .meta_wr_idx (tcu_meta_wr_idx), .meta_wr_data (tcu_meta_wr_data), - .tcu_mem_if (tcu_mem_if), + .tcu_mem_if (lsu_tcu_mem_if[0]), `endif .clk (clk), .reset (reset) @@ -425,7 +445,21 @@ import VX_tcu_pkg::*; .satp_wr_valid (satp_wr_valid[i]), .satp_wr_data (satp_wr_data[i]), `endif - .lsu_client_if (lsu_client_if[i]), + `ifdef PERF_ENABLE + .lsu_perf_rd_req_fire (lsu_perf_rd_req_fire[i]), + .lsu_perf_wr_req_fire (lsu_perf_wr_req_fire[i]), + .lsu_perf_rsp_fire (lsu_perf_rsp_fire[i]), + .lsu_coalescer_misses (lsu_coalescer_misses[i]), + `endif + `ifdef TCU_META_ENABLE + .lsu_tcu_mem_if (lsu_tcu_mem_if[i]), + `endif + .lsu_sched_empty (lsu_sched_empty[i]), + .lsu_store_empty (lsu_store_empty[i]), + `ifdef VX_CFG_LMEM_ENABLE + .lmem_bus_if (lsu_lmem_bus_if[i * `VX_CFG_NUM_LSU_LANES +: `VX_CFG_NUM_LSU_LANES]), + `endif + .dcache_bus_if (lsu_dcache_bus_if[i * DCACHE_CHANNELS +: DCACHE_CHANNELS]), `ifdef VX_CFG_EXT_DXA_ENABLE .dxa_req_bus_if (block_dxa_req_bus_if[i]), `endif @@ -647,7 +681,21 @@ import VX_tcu_pkg::*; `ifdef VX_CFG_EXT_RTU_ENABLE .sched_unlock_if (sched_unlock_if[i]), `endif - .lsu_client_if (lsu_client_if[i]), + `ifdef PERF_ENABLE + .lsu_perf_rd_req_fire (lsu_perf_rd_req_fire[i]), + .lsu_perf_wr_req_fire (lsu_perf_wr_req_fire[i]), + .lsu_perf_rsp_fire (lsu_perf_rsp_fire[i]), + .lsu_coalescer_misses (lsu_coalescer_misses[i]), + `endif + `ifdef TCU_META_ENABLE + .lsu_tcu_mem_if (lsu_tcu_mem_if[i]), + `endif + .lsu_sched_empty (lsu_sched_empty[i]), + .lsu_store_empty (lsu_store_empty[i]), + `ifdef VX_CFG_LMEM_ENABLE + .lmem_bus_if (lsu_lmem_bus_if[i * `VX_CFG_NUM_LSU_LANES +: `VX_CFG_NUM_LSU_LANES]), + `endif + .dcache_bus_if (lsu_dcache_bus_if[i * DCACHE_CHANNELS +: DCACHE_CHANNELS]), `ifdef VX_CFG_EXT_DXA_ENABLE .dxa_req_bus_if (block_dxa_req_bus_if[i]), `endif @@ -727,9 +775,21 @@ import VX_tcu_pkg::*; .reset (reset), .core_id (core_id), - // execute exposes LSU client interfaces; lsu_scheduler sits between - // execute and lsu_mem_if (which connects to mem_unit). - .lsu_client_if (lsu_client_if), + `ifdef PERF_ENABLE + .lsu_perf_rd_req_fire (lsu_perf_rd_req_fire), + .lsu_perf_wr_req_fire (lsu_perf_wr_req_fire), + .lsu_perf_rsp_fire (lsu_perf_rsp_fire), + .lsu_coalescer_misses (lsu_coalescer_misses), + `endif + `ifdef TCU_META_ENABLE + .lsu_tcu_mem_if (lsu_tcu_mem_if), + `endif + .lsu_sched_empty (lsu_sched_empty), + .lsu_store_empty (lsu_store_empty), + `ifdef VX_CFG_LMEM_ENABLE + .lmem_bus_if (lsu_lmem_bus_if), + `endif + .dcache_bus_if (lsu_dcache_bus_if), .dispatch_if (dispatch_if), .commit_if (commit_if), @@ -791,60 +851,6 @@ import VX_tcu_pkg::*; end end - // One memory scheduler per LSU block. With the TCU metadata AGU, every block - // has a second client: block 0 serves the AGU and the others tie it off, which - // keeps the blocks identical at the cost of a few arbiter muxes. -`ifdef TCU_META_ENABLE - localparam LSU_SCHED_NUM_CLIENTS = 2; -`else - localparam LSU_SCHED_NUM_CLIENTS = 1; -`endif - for (genvar block_idx = 0; block_idx < `VX_CFG_NUM_LSU_BLOCKS; ++block_idx) begin : g_lsu_scheduler - VX_lsu_sched_if sched_client_if [LSU_SCHED_NUM_CLIENTS](); - - // client 0: the block's LSU - assign sched_client_if[0].req_valid = lsu_client_if[block_idx].req_valid; - assign sched_client_if[0].req_data = lsu_client_if[block_idx].req_data; - assign lsu_client_if[block_idx].req_ready = sched_client_if[0].req_ready; - assign lsu_client_if[block_idx].rsp_valid = sched_client_if[0].rsp_valid; - assign lsu_client_if[block_idx].rsp_data = sched_client_if[0].rsp_data; - assign sched_client_if[0].rsp_ready = lsu_client_if[block_idx].rsp_ready; - - `ifdef TCU_META_ENABLE - // client 1: the TCU metadata AGU on block 0 - if (block_idx == 0) begin : g_tcu_client - assign sched_client_if[1].req_valid = tcu_mem_if.req_valid; - assign sched_client_if[1].req_data = tcu_mem_if.req_data; - assign tcu_mem_if.req_ready = sched_client_if[1].req_ready; - assign tcu_mem_if.rsp_valid = sched_client_if[1].rsp_valid; - assign tcu_mem_if.rsp_data = sched_client_if[1].rsp_data; - assign sched_client_if[1].rsp_ready = tcu_mem_if.rsp_ready; - end else begin : g_tcu_client_tieoff - assign sched_client_if[1].req_valid = 1'b0; - assign sched_client_if[1].req_data = '0; - assign sched_client_if[1].rsp_ready = 1'b1; - `UNUSED_VAR (sched_client_if[1].req_ready) - `UNUSED_VAR (sched_client_if[1].rsp_valid) - `UNUSED_VAR (sched_client_if[1].rsp_data) - end - `endif - - VX_lsu_scheduler #( - .INSTANCE_ID (`SFORMATF(("%s-lsusched%0d", INSTANCE_ID, block_idx))), - .NUM_CLIENTS (LSU_SCHED_NUM_CLIENTS), - .NUM_LANES (`VX_CFG_NUM_LSU_LANES), - .CORE_QUEUE_SIZE (`VX_CFG_LSU_QUEUE_IN_SIZE), - .MEM_QUEUE_SIZE (LSU_QUEUE_OUT_SIZE), - .PENDING_SIZE (`VX_CFG_LSU_PENDING_SIZE) - ) lsu_scheduler ( - .clk (clk), - .reset (reset), - .client_if (sched_client_if), - .empty (lsu_sched_empty[block_idx]), - .lsu_mem_if (lsu_mem_if[block_idx]) - ); - end - VX_mem_unit #( .INSTANCE_ID (INSTANCE_ID) ) mem_unit ( @@ -852,7 +858,6 @@ import VX_tcu_pkg::*; .reset (reset), `ifdef PERF_ENABLE .lmem_perf (lmem_perf), - .coalescer_perf (coalescer_perf), `endif `ifdef VX_CFG_TCU_WGMMA_ENABLE .tcu_lmem_if (tcu_lmem_if), @@ -861,8 +866,10 @@ import VX_tcu_pkg::*; .dxa_lmem_bus_if (dxa_lmem_bus_if), .dxa_txbar_bus_if(dxa_txbar_bus_if), `endif - .empty (mem_unit_empty), - .lsu_mem_if (lsu_mem_if), + `ifdef VX_CFG_LMEM_ENABLE + .lmem_bus_if (lsu_lmem_bus_if), + `endif + .lsu_dcache_bus_if (lsu_dcache_bus_if), .dcr_flush_if (dcr_flush_dcache_if), .flush_l1_only (dcr_flush_l1_only), .dcache_bus_if (dcache_bus_if) @@ -926,13 +933,20 @@ import VX_tcu_pkg::*; wire [LSU_NUM_REQS-1:0] perf_dcache_wr_req_fire, perf_dcache_wr_req_fire_r; wire [LSU_NUM_REQS-1:0] perf_dcache_rsp_fire; - for (genvar i = 0; i < `VX_CFG_NUM_LSU_BLOCKS; ++i) begin : g_perf_dcache - for (genvar j = 0; j < `VX_CFG_NUM_LSU_LANES; ++j) begin : g_j - assign perf_dcache_rd_req_fire[i * `VX_CFG_NUM_LSU_LANES + j] = lsu_mem_if[i].req_valid && lsu_mem_if[i].req_data.mask[j] && lsu_mem_if[i].req_ready && ~lsu_mem_if[i].req_data.rw; - assign perf_dcache_wr_req_fire[i * `VX_CFG_NUM_LSU_LANES + j] = lsu_mem_if[i].req_valid && lsu_mem_if[i].req_data.mask[j] && lsu_mem_if[i].req_ready && lsu_mem_if[i].req_data.rw; - assign perf_dcache_rsp_fire[i * `VX_CFG_NUM_LSU_LANES + j] = lsu_mem_if[i].rsp_valid && lsu_mem_if[i].rsp_data.mask[j] && lsu_mem_if[i].rsp_ready; - end - end + assign perf_dcache_rd_req_fire = lsu_perf_rd_req_fire; + assign perf_dcache_wr_req_fire = lsu_perf_wr_req_fire; + assign perf_dcache_rsp_fire = lsu_perf_rsp_fire; + + wire [PERF_CTR_BITS-1:0] coalescer_misses; + VX_reduce_tree #( + .IN_W (PERF_CTR_BITS), + .N (`VX_CFG_NUM_LSU_BLOCKS), + .OP ("+") + ) coalescer_reduce ( + .data_in (lsu_coalescer_misses), + .data_out (coalescer_misses) + ); + `BUFFER(coalescer_perf.misses, coalescer_misses); `BUFFER(perf_dcache_rd_req_fire_r, perf_dcache_rd_req_fire); `BUFFER(perf_dcache_wr_req_fire_r, perf_dcache_wr_req_fire); diff --git a/hw/rtl/core/VX_execute.sv b/hw/rtl/core/VX_execute.sv index 94e33c191a..8258555eaf 100644 --- a/hw/rtl/core/VX_execute.sv +++ b/hw/rtl/core/VX_execute.sv @@ -27,10 +27,22 @@ import VX_tcu_pkg::*; input wire [CORE_ID_WIDTH-1:0] core_id, - // Per-block LSU client connections to VX_lsu_scheduler (at VX_core). - // The dcache port (lsu_mem_if) lives at VX_core; execute exposes - // only the LSU client interfaces. - VX_lsu_sched_if.master lsu_client_if [`VX_CFG_NUM_LSU_BLOCKS], + // LSU memory paths +`ifdef PERF_ENABLE + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_rd_req_fire, + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_wr_req_fire, + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_rsp_fire, + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][PERF_CTR_BITS-1:0] lsu_coalescer_misses, +`endif +`ifdef TCU_META_ENABLE + VX_lsu_sched_if.slave lsu_tcu_mem_if [`VX_CFG_NUM_LSU_BLOCKS], +`endif + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_sched_empty, + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_store_empty, +`ifdef VX_CFG_LMEM_ENABLE + VX_mem_bus_if.master lmem_bus_if [LSU_NUM_REQS], +`endif + VX_mem_bus_if.master dcache_bus_if [DCACHE_NUM_REQS], // dispatch interface VX_dispatch_if.slave dispatch_if [NUM_EX_UNITS * `VX_CFG_ISSUE_WIDTH], @@ -115,7 +127,21 @@ import VX_tcu_pkg::*; .core_id (core_id), .dispatch_if (dispatch_if[EX_LSU * `VX_CFG_ISSUE_WIDTH +: `VX_CFG_ISSUE_WIDTH]), .commit_if (commit_if[EX_LSU * `VX_CFG_ISSUE_WIDTH +: `VX_CFG_ISSUE_WIDTH]), - .per_block_client_if (lsu_client_if) + `ifdef PERF_ENABLE + .perf_rd_req_fire (lsu_perf_rd_req_fire), + .perf_wr_req_fire (lsu_perf_wr_req_fire), + .perf_rsp_fire (lsu_perf_rsp_fire), + .coalescer_misses (lsu_coalescer_misses), + `endif + `ifdef TCU_META_ENABLE + .tcu_mem_if (lsu_tcu_mem_if), + `endif + .sched_empty (lsu_sched_empty), + .store_empty (lsu_store_empty), + `ifdef VX_CFG_LMEM_ENABLE + .lmem_bus_if (lmem_bus_if), + `endif + .dcache_bus_if (dcache_bus_if) ); `ifdef VX_CFG_EXT_F_ENABLE diff --git a/hw/rtl/core/VX_execute_slice.sv b/hw/rtl/core/VX_execute_slice.sv index 541c84efe7..be0a1d001d 100644 --- a/hw/rtl/core/VX_execute_slice.sv +++ b/hw/rtl/core/VX_execute_slice.sv @@ -45,7 +45,21 @@ import VX_tcu_pkg::*; VX_branch_ctl_if.master branch_ctl_if, // LSU - VX_lsu_sched_if.master lsu_client_if, +`ifdef PERF_ENABLE + output wire [`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_rd_req_fire, + output wire [`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_wr_req_fire, + output wire [`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_rsp_fire, + output wire [PERF_CTR_BITS-1:0] lsu_coalescer_misses, +`endif +`ifdef TCU_META_ENABLE + VX_lsu_sched_if.slave lsu_tcu_mem_if, +`endif + output wire lsu_sched_empty, + output wire lsu_store_empty, +`ifdef VX_CFG_LMEM_ENABLE + VX_mem_bus_if.master lmem_bus_if [`VX_CFG_NUM_LSU_LANES], +`endif + VX_mem_bus_if.master dcache_bus_if [DCACHE_CHANNELS], // SFU `ifdef VX_CFG_EXT_DXA_ENABLE @@ -172,12 +186,26 @@ import VX_tcu_pkg::*; .INSTANCE_ID (`SFORMATF(("%s-lsu%s", INSTANCE_ID, SLICE_STR))) ) lsu_block ( `SCOPE_IO_BIND (0) - .clk (clk), - .reset (reset), - .core_id (core_id), - .execute_if (lsu_execute_if[0]), - .result_if (lsu_result_if[0]), - .client_if (lsu_client_if) + .clk (clk), + .reset (reset), + .core_id (core_id), + .execute_if (lsu_execute_if[0]), + .result_if (lsu_result_if[0]), + `ifdef PERF_ENABLE + .perf_rd_req_fire (lsu_perf_rd_req_fire), + .perf_wr_req_fire (lsu_perf_wr_req_fire), + .perf_rsp_fire (lsu_perf_rsp_fire), + .coalescer_misses (lsu_coalescer_misses), + `endif + `ifdef TCU_META_ENABLE + .tcu_mem_if (lsu_tcu_mem_if), + `endif + .sched_empty (lsu_sched_empty), + .store_empty (lsu_store_empty), + `ifdef VX_CFG_LMEM_ENABLE + .lmem_bus_if (lmem_bus_if), + `endif + .dcache_bus_if (dcache_bus_if) ); VX_lane_gather #( diff --git a/hw/rtl/core/VX_lsu_block.sv b/hw/rtl/core/VX_lsu_block.sv index eebfad60a4..83c8ed4190 100644 --- a/hw/rtl/core/VX_lsu_block.sv +++ b/hw/rtl/core/VX_lsu_block.sv @@ -29,9 +29,24 @@ module VX_lsu_block import VX_gpu_pkg::*; #( // Outputs VX_result_if.master result_if, - // The slice exposes the memory-client side as a VX_lsu_sched_if master; - // the parent wires it into VX_lsu_scheduler. - VX_lsu_sched_if.master client_if +`ifdef PERF_ENABLE + output wire [`VX_CFG_NUM_LSU_LANES-1:0] perf_rd_req_fire, + output wire [`VX_CFG_NUM_LSU_LANES-1:0] perf_wr_req_fire, + output wire [`VX_CFG_NUM_LSU_LANES-1:0] perf_rsp_fire, + output wire [PERF_CTR_BITS-1:0] coalescer_misses, +`endif + +`ifdef TCU_META_ENABLE + VX_lsu_sched_if.slave tcu_mem_if, +`endif + + output wire sched_empty, + output wire store_empty, + +`ifdef VX_CFG_LMEM_ENABLE + VX_mem_bus_if.master lmem_bus_if [`VX_CFG_NUM_LSU_LANES], +`endif + VX_mem_bus_if.master dcache_bus_if [DCACHE_CHANNELS] ); localparam NUM_LANES = `VX_CFG_NUM_LSU_LANES; localparam PID_BITS = `CLOG2(`VX_CFG_NUM_THREADS / NUM_LANES); @@ -51,7 +66,6 @@ module VX_lsu_block import VX_gpu_pkg::*; #( `ifndef VX_CFG_EXT_A_ENABLE `UNUSED_VAR (core_id) `endif - `UNUSED_SPARAM (INSTANCE_ID) // tag_width = header + op_type + align + pkt_addr + fence localparam TAG_WIDTH = $bits(lsu_header_t) + INST_LSU_BITS + (NUM_LANES * REQ_ASHIFT) + LSUQ_SIZEW + 1; @@ -385,6 +399,31 @@ module VX_lsu_block import VX_gpu_pkg::*; #( req_is_fence }; + VX_lsu_sched_if client_if(); + + VX_lsu_mem_path #( + .INSTANCE_ID (INSTANCE_ID) + ) mem_path ( + .clk (clk), + .reset (reset), + `ifdef PERF_ENABLE + .perf_rd_req_fire (perf_rd_req_fire), + .perf_wr_req_fire (perf_wr_req_fire), + .perf_rsp_fire (perf_rsp_fire), + .coalescer_misses (coalescer_misses), + `endif + .client_if (client_if), + `ifdef TCU_META_ENABLE + .tcu_mem_if (tcu_mem_if), + `endif + .sched_empty (sched_empty), + .store_empty (store_empty), + `ifdef VX_CFG_LMEM_ENABLE + .lmem_bus_if (lmem_bus_if), + `endif + .dcache_bus_if (dcache_bus_if) + ); + // Pack per-lane mem_req signals into the client_if request payload; // unpack the response payload into per-lane mem_rsp signals. assign client_if.req_valid = mem_req_valid; diff --git a/hw/rtl/core/VX_lsu_mem_path.sv b/hw/rtl/core/VX_lsu_mem_path.sv new file mode 100644 index 0000000000..7fd861ca86 --- /dev/null +++ b/hw/rtl/core/VX_lsu_mem_path.sv @@ -0,0 +1,287 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// The memory path of one LSU block: its memory scheduler, the global/local +// switch, the local-memory adapter, the coalescer and the D-cache adapter. +// The D-cache port carries one extra tag LSB: requests leave with it clear, and +// a response with it set is the core's flush token, not this block's. +module VX_lsu_mem_path import VX_gpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "" +) ( + input wire clk, + input wire reset, + +`ifdef PERF_ENABLE + output wire [`VX_CFG_NUM_LSU_LANES-1:0] perf_rd_req_fire, + output wire [`VX_CFG_NUM_LSU_LANES-1:0] perf_wr_req_fire, + output wire [`VX_CFG_NUM_LSU_LANES-1:0] perf_rsp_fire, + output wire [PERF_CTR_BITS-1:0] coalescer_misses, +`endif + + VX_lsu_sched_if.slave client_if, +`ifdef TCU_META_ENABLE + VX_lsu_sched_if.slave tcu_mem_if, +`endif + + // the scheduler holds no request; the global-store path holds no store + output wire sched_empty, + output wire store_empty, + +`ifdef VX_CFG_LMEM_ENABLE + VX_mem_bus_if.master lmem_bus_if [`VX_CFG_NUM_LSU_LANES], +`endif + VX_mem_bus_if.master dcache_bus_if [DCACHE_CHANNELS] +); + `UNUSED_SPARAM (INSTANCE_ID) + + VX_lsu_mem_if #( + .NUM_LANES (`VX_CFG_NUM_LSU_LANES), + .DATA_SIZE (LSU_WORD_SIZE), + .TAG_WIDTH (LSU_TAG_WIDTH) + ) lsu_mem_if(); + + // scheduler: client 0 is the block's lanes, client 1 the TCU metadata AGU + +`ifdef TCU_META_ENABLE + localparam NUM_CLIENTS = 2; +`else + localparam NUM_CLIENTS = 1; +`endif + + VX_lsu_sched_if sched_client_if [NUM_CLIENTS](); + + assign sched_client_if[0].req_valid = client_if.req_valid; + assign sched_client_if[0].req_data = client_if.req_data; + assign client_if.req_ready = sched_client_if[0].req_ready; + assign client_if.rsp_valid = sched_client_if[0].rsp_valid; + assign client_if.rsp_data = sched_client_if[0].rsp_data; + assign sched_client_if[0].rsp_ready = client_if.rsp_ready; + +`ifdef TCU_META_ENABLE + assign sched_client_if[1].req_valid = tcu_mem_if.req_valid; + assign sched_client_if[1].req_data = tcu_mem_if.req_data; + assign tcu_mem_if.req_ready = sched_client_if[1].req_ready; + assign tcu_mem_if.rsp_valid = sched_client_if[1].rsp_valid; + assign tcu_mem_if.rsp_data = sched_client_if[1].rsp_data; + assign sched_client_if[1].rsp_ready = tcu_mem_if.rsp_ready; +`endif + + VX_lsu_scheduler #( + .INSTANCE_ID (`SFORMATF(("%s-lsusched", INSTANCE_ID))), + .NUM_CLIENTS (NUM_CLIENTS), + .NUM_LANES (`VX_CFG_NUM_LSU_LANES), + .CORE_QUEUE_SIZE (`VX_CFG_LSU_QUEUE_IN_SIZE), + .MEM_QUEUE_SIZE (LSU_QUEUE_OUT_SIZE), + .PENDING_SIZE (`VX_CFG_LSU_PENDING_SIZE) + ) lsu_scheduler ( + .clk (clk), + .reset (reset), + .client_if (sched_client_if), + .empty (sched_empty), + .lsu_mem_if (lsu_mem_if) + ); + +`ifdef PERF_ENABLE + for (genvar j = 0; j < `VX_CFG_NUM_LSU_LANES; ++j) begin : g_perf + assign perf_rd_req_fire[j] = lsu_mem_if.req_valid && lsu_mem_if.req_data.mask[j] && lsu_mem_if.req_ready && ~lsu_mem_if.req_data.rw; + assign perf_wr_req_fire[j] = lsu_mem_if.req_valid && lsu_mem_if.req_data.mask[j] && lsu_mem_if.req_ready && lsu_mem_if.req_data.rw; + assign perf_rsp_fire[j] = lsu_mem_if.rsp_valid && lsu_mem_if.rsp_data.mask[j] && lsu_mem_if.rsp_ready; + end +`endif + + // Two terms keep a store visible until it reaches the D-cache port: the + // coalescer (or the adapter input without one), and the adapter's request + // buffer. + wire path_empty; + wire adapter_empty; + assign store_empty = path_empty && adapter_empty; + + VX_lsu_mem_if #( + .NUM_LANES (`VX_CFG_NUM_LSU_LANES), + .DATA_SIZE (LSU_WORD_SIZE), + .TAG_WIDTH (LSU_TAG_WIDTH) + ) lsu_dcache_if(); + +`ifdef VX_CFG_LMEM_ENABLE + + VX_lsu_mem_if #( + .NUM_LANES (`VX_CFG_NUM_LSU_LANES), + .DATA_SIZE (LSU_WORD_SIZE), + .TAG_WIDTH (LSU_TAG_WIDTH) + ) lsu_lmem_if(); + + VX_lmem_switch #( + .GLOBAL_OUT_BUF(1), + .LOCAL_OUT_BUF(1), + .RSP_OUT_BUF (1), + .ARBITER ("P") + ) lmem_switch ( + .clk (clk), + .reset (reset), + .lsu_in_if (lsu_mem_if), + .global_out_if(lsu_dcache_if), + .local_out_if (lsu_lmem_if) + ); + + // A dedicated local-memory adapter per block: sharing one adapter's + // unpack/pack buffers between blocks would deadlock. + VX_lsu_adapter #( + .NUM_LANES (`VX_CFG_NUM_LSU_LANES), + .DATA_SIZE (LSU_WORD_SIZE), + .TAG_WIDTH (LSU_TAG_WIDTH), + .TAG_SEL_BITS (LSU_TAG_WIDTH - UUID_WIDTH), + .ARBITER ("P"), + .REQ_OUT_BUF (3), + .RSP_OUT_BUF (0) + ) lmem_adapter ( + .clk (clk), + .reset (reset), + .lsu_mem_if (lsu_lmem_if), + .mem_bus_if (lmem_bus_if) + ); + +`else + + `ASSIGN_VX_MEM_BUS_IF (lsu_dcache_if, lsu_mem_if); + +`endif + + VX_lsu_mem_if #( + .NUM_LANES (DCACHE_CHANNELS), + .DATA_SIZE (DCACHE_WORD_SIZE), + .TAG_WIDTH (DCACHE_CORE_TAG_WIDTH) + ) dcache_coalesced_if(); + + if ((`VX_CFG_NUM_LSU_LANES > 1) && (DCACHE_WORD_SIZE > LSU_WORD_SIZE)) begin : g_coalescing + + VX_mem_coalescer #( + .INSTANCE_ID (`SFORMATF(("%s-coalescer", INSTANCE_ID))), + .NUM_REQS (`VX_CFG_NUM_LSU_LANES), + .DATA_IN_SIZE (LSU_WORD_SIZE), + .DATA_OUT_SIZE (DCACHE_WORD_SIZE), + .ADDR_WIDTH (LSU_ADDR_WIDTH), + .USER_WIDTH (MEM_ATTR_WIDTH), + .TAG_WIDTH (LSU_TAG_WIDTH), + .UUID_WIDTH (UUID_WIDTH), + .QUEUE_SIZE (LSU_QUEUE_OUT_SIZE), + .PERF_CTR_BITS (PERF_CTR_BITS) + ) mem_coalescer ( + .clk (clk), + .reset (reset), + + `ifdef PERF_ENABLE + .misses (coalescer_misses), + `else + `UNUSED_PIN (misses), + `endif + + .empty (path_empty), + + // Input request + .in_req_valid (lsu_dcache_if.req_valid), + .in_req_mask (lsu_dcache_if.req_data.mask), + .in_req_rw (lsu_dcache_if.req_data.rw), + .in_req_byteen (lsu_dcache_if.req_data.byteen), + .in_req_addr (lsu_dcache_if.req_data.addr), + .in_req_user (lsu_dcache_if.req_data.user), + .in_req_no_merge(lsu_dcache_if.req_data.user[0][MEM_ATTR_AMO_OFFS]), + .in_req_data (lsu_dcache_if.req_data.data), + .in_req_tag (lsu_dcache_if.req_data.tag), + .in_req_ready (lsu_dcache_if.req_ready), + + // Input response + .in_rsp_valid (lsu_dcache_if.rsp_valid), + .in_rsp_mask (lsu_dcache_if.rsp_data.mask), + .in_rsp_data (lsu_dcache_if.rsp_data.data), + .in_rsp_tag (lsu_dcache_if.rsp_data.tag), + .in_rsp_ready (lsu_dcache_if.rsp_ready), + + // Output request + .out_req_valid (dcache_coalesced_if.req_valid), + .out_req_mask (dcache_coalesced_if.req_data.mask), + .out_req_rw (dcache_coalesced_if.req_data.rw), + .out_req_byteen (dcache_coalesced_if.req_data.byteen), + .out_req_addr (dcache_coalesced_if.req_data.addr), + .out_req_user (dcache_coalesced_if.req_data.user), + .out_req_data (dcache_coalesced_if.req_data.data), + .out_req_tag (dcache_coalesced_if.req_data.tag), + .out_req_ready (dcache_coalesced_if.req_ready), + + // Output response + .out_rsp_valid (dcache_coalesced_if.rsp_valid), + .out_rsp_mask (dcache_coalesced_if.rsp_data.mask), + .out_rsp_data (dcache_coalesced_if.rsp_data.data), + .out_rsp_tag (dcache_coalesced_if.rsp_data.tag), + .out_rsp_ready (dcache_coalesced_if.rsp_ready) + ); + + end else begin : g_no_coalescing + + // The passthrough narrows the tag from LSU_TAG_WIDTH to + // DCACHE_CORE_TAG_WIDTH: the dcache tag-id space must cover the + // mem-scheduler slot id or responses alias. + `STATIC_ASSERT (DCACHE_CORE_TAG_WIDTH >= LSU_TAG_WIDTH, ("dcache tag-id space cannot hold the LSU outstanding slot id")) + + `ASSIGN_VX_MEM_BUS_IF (dcache_coalesced_if, lsu_dcache_if); + `ifdef PERF_ENABLE + assign coalescer_misses = '0; + `endif + // Without a coalescer, a store still upstream of the adapter is one + // presented at its input. + assign path_empty = ~dcache_coalesced_if.req_valid; + + end + + VX_mem_bus_if #( + .DATA_SIZE (DCACHE_WORD_SIZE), + .TAG_WIDTH (DCACHE_CORE_TAG_WIDTH) + ) dcache_adapter_if[DCACHE_CHANNELS](); + + VX_lsu_adapter #( + .NUM_LANES (DCACHE_CHANNELS), + .DATA_SIZE (DCACHE_WORD_SIZE), + .TAG_WIDTH (DCACHE_CORE_TAG_WIDTH), + .TAG_SEL_BITS (DCACHE_CORE_TAG_WIDTH - UUID_WIDTH), + .ARBITER ("P"), + .REQ_OUT_BUF (3), + .RSP_OUT_BUF (0) + ) dcache_adapter ( + .clk (clk), + .reset (reset), + .lsu_mem_if (dcache_coalesced_if), + .mem_bus_if (dcache_adapter_if) + ); + + wire [DCACHE_CHANNELS-1:0] adapter_req_valid; + for (genvar j = 0; j < DCACHE_CHANNELS; ++j) begin : g_dcache_bus_if + assign adapter_req_valid[j] = dcache_adapter_if[j].req_valid; + + assign dcache_bus_if[j].req_valid = dcache_adapter_if[j].req_valid; + assign dcache_bus_if[j].req_data.rw = dcache_adapter_if[j].req_data.rw; + assign dcache_bus_if[j].req_data.addr = dcache_adapter_if[j].req_data.addr; + assign dcache_bus_if[j].req_data.data = dcache_adapter_if[j].req_data.data; + assign dcache_bus_if[j].req_data.byteen = dcache_adapter_if[j].req_data.byteen; + assign dcache_bus_if[j].req_data.attr = dcache_adapter_if[j].req_data.attr; + assign dcache_bus_if[j].req_data.tag = {dcache_adapter_if[j].req_data.tag, 1'b0}; + assign dcache_adapter_if[j].req_ready = dcache_bus_if[j].req_ready; + + assign dcache_adapter_if[j].rsp_valid = dcache_bus_if[j].rsp_valid && ~dcache_bus_if[j].rsp_data.tag[0]; + assign dcache_adapter_if[j].rsp_data.data = dcache_bus_if[j].rsp_data.data; + assign dcache_adapter_if[j].rsp_data.tag = dcache_bus_if[j].rsp_data.tag[DCACHE_TAG_WIDTH-1:1]; + assign dcache_bus_if[j].rsp_ready = dcache_adapter_if[j].rsp_ready; + end + assign adapter_empty = ~(| adapter_req_valid); + +endmodule diff --git a/hw/rtl/core/VX_lsu_unit.sv b/hw/rtl/core/VX_lsu_unit.sv index 2623bc8162..e383dfe5c8 100644 --- a/hw/rtl/core/VX_lsu_unit.sv +++ b/hw/rtl/core/VX_lsu_unit.sv @@ -29,8 +29,24 @@ module VX_lsu_unit import VX_gpu_pkg::*; #( // Outputs VX_commit_if.master commit_if [`VX_CFG_ISSUE_WIDTH], - // Per-block client connection to VX_lsu_scheduler (at VX_core). - VX_lsu_sched_if.master per_block_client_if [`VX_CFG_NUM_LSU_BLOCKS] +`ifdef PERF_ENABLE + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0] perf_rd_req_fire, + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0] perf_wr_req_fire, + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0] perf_rsp_fire, + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][PERF_CTR_BITS-1:0] coalescer_misses, +`endif + +`ifdef TCU_META_ENABLE + VX_lsu_sched_if.slave tcu_mem_if [`VX_CFG_NUM_LSU_BLOCKS], +`endif + + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] sched_empty, + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] store_empty, + +`ifdef VX_CFG_LMEM_ENABLE + VX_mem_bus_if.master lmem_bus_if [LSU_NUM_REQS], +`endif + VX_mem_bus_if.master dcache_bus_if [DCACHE_NUM_REQS] ); localparam BLOCK_SIZE = `VX_CFG_NUM_LSU_BLOCKS; localparam NUM_LANES = `VX_CFG_NUM_LSU_LANES; @@ -58,6 +74,24 @@ module VX_lsu_unit import VX_gpu_pkg::*; #( ) per_block_result_if[BLOCK_SIZE](); for (genvar block_idx = 0; block_idx < BLOCK_SIZE; ++block_idx) begin : g_blocks + `ifdef VX_CFG_LMEM_ENABLE + VX_mem_bus_if #( + .DATA_SIZE (LSU_WORD_SIZE), + .TAG_WIDTH (LSU_TAG_WIDTH) + ) block_lmem_bus_if[NUM_LANES](); + for (genvar j = 0; j < NUM_LANES; ++j) begin : g_lmem_bus_if + `ASSIGN_VX_MEM_BUS_IF (lmem_bus_if[block_idx * NUM_LANES + j], block_lmem_bus_if[j]); + end + `endif + + VX_mem_bus_if #( + .DATA_SIZE (DCACHE_WORD_SIZE), + .TAG_WIDTH (DCACHE_TAG_WIDTH) + ) block_dcache_bus_if[DCACHE_CHANNELS](); + for (genvar j = 0; j < DCACHE_CHANNELS; ++j) begin : g_dcache_bus_if + `ASSIGN_VX_MEM_BUS_IF (dcache_bus_if[block_idx * DCACHE_CHANNELS + j], block_dcache_bus_if[j]); + end + VX_lsu_block #( .INSTANCE_ID (`SFORMATF(("%s%0d", INSTANCE_ID, block_idx))) ) lsu_block( @@ -67,7 +101,21 @@ module VX_lsu_unit import VX_gpu_pkg::*; #( .core_id (core_id), .execute_if (per_block_execute_if[block_idx]), .result_if (per_block_result_if[block_idx]), - .client_if (per_block_client_if[block_idx]) + `ifdef PERF_ENABLE + .perf_rd_req_fire (perf_rd_req_fire[block_idx]), + .perf_wr_req_fire (perf_wr_req_fire[block_idx]), + .perf_rsp_fire (perf_rsp_fire[block_idx]), + .coalescer_misses (coalescer_misses[block_idx]), + `endif + `ifdef TCU_META_ENABLE + .tcu_mem_if (tcu_mem_if[block_idx]), + `endif + .sched_empty (sched_empty[block_idx]), + .store_empty (store_empty[block_idx]), + `ifdef VX_CFG_LMEM_ENABLE + .lmem_bus_if (block_lmem_bus_if), + `endif + .dcache_bus_if (block_dcache_bus_if) ); end diff --git a/hw/rtl/core/VX_mem_unit.sv b/hw/rtl/core/VX_mem_unit.sv index 17b5d7c665..a72e33c8c0 100644 --- a/hw/rtl/core/VX_mem_unit.sv +++ b/hw/rtl/core/VX_mem_unit.sv @@ -21,7 +21,6 @@ module VX_mem_unit import VX_gpu_pkg::*; #( `ifdef PERF_ENABLE output lmem_perf_t lmem_perf, - output coalescer_perf_t coalescer_perf, `endif `ifdef VX_CFG_EXT_DXA_ENABLE @@ -34,28 +33,19 @@ module VX_mem_unit import VX_gpu_pkg::*; #( VX_mem_bus_if.slave tcu_lmem_if, `endif - output wire empty, + // the LSU blocks' memory paths +`ifdef VX_CFG_LMEM_ENABLE + VX_mem_bus_if.slave lmem_bus_if [LSU_NUM_REQS], +`endif + VX_mem_bus_if.slave lsu_dcache_bus_if [DCACHE_NUM_REQS], - VX_lsu_mem_if.slave lsu_mem_if [`VX_CFG_NUM_LSU_BLOCKS], VX_dcr_flush_if.slave dcr_flush_if, input wire flush_l1_only, VX_mem_bus_if.master dcache_bus_if [DCACHE_NUM_REQS] ); - // Per-block empty: high when the global-store path holds no in-flight - // write. Folded into core `busy` so the end-of-kernel cache flush is - // ordered strictly behind the last store. Two terms: the coalescer/passthru - // producer, and the dcache adapter's request buffer (which carries state - // when REQ_OUT_BUF > 0 and would otherwise hide a still-settling store). - wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] per_block_path_empty; - wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] per_block_adapter_empty; - wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] per_block_empty; - assign per_block_empty = per_block_path_empty & per_block_adapter_empty; - VX_lsu_mem_if #( - .NUM_LANES (`VX_CFG_NUM_LSU_LANES), - .DATA_SIZE (LSU_WORD_SIZE), - .TAG_WIDTH (LSU_TAG_WIDTH) - ) lsu_dcache_if[`VX_CFG_NUM_LSU_BLOCKS](); - +`ifndef VX_CFG_LMEM_ENABLE + `UNUSED_SPARAM (INSTANCE_ID) +`endif `ifdef VX_CFG_TCU_WGMMA_ENABLE `STATIC_ASSERT(`VX_CFG_LMEM_ENABLED, ("TCU_WGMMA_ENABLE requires LMEM_ENABLE")) // The WGMMA shared-memory descriptor carries the LMEM byte offset in a 16-bit @@ -71,63 +61,6 @@ module VX_mem_unit import VX_gpu_pkg::*; #( localparam LMEM_ADDR_WIDTH = `VX_CFG_LMEM_LOG_SIZE - `CLOG2(LSU_WORD_SIZE); - VX_lsu_mem_if #( - .NUM_LANES (`VX_CFG_NUM_LSU_LANES), - .DATA_SIZE (LSU_WORD_SIZE), - .TAG_WIDTH (LSU_TAG_WIDTH) - ) lsu_lmem_if[`VX_CFG_NUM_LSU_BLOCKS](); - - for (genvar i = 0; i < `VX_CFG_NUM_LSU_BLOCKS; ++i) begin : g_lmem_switches - VX_lmem_switch #( - .GLOBAL_OUT_BUF(1), - .LOCAL_OUT_BUF(1), - .RSP_OUT_BUF (1), - .ARBITER ("P") - ) lmem_switch ( - .clk (clk), - .reset (reset), - .lsu_in_if (lsu_mem_if[i]), - .global_out_if(lsu_dcache_if[i]), - .local_out_if (lsu_lmem_if[i]) - ); - end - - // Per-block local memory adapters to avoid deadlock when NUM_LSU_BLOCKS > 1. - // Each block gets dedicated ports into local memory, eliminating the circular - // dependency that arises from sharing a single adapter's unpack/pack buffers. - - VX_mem_bus_if #( - .DATA_SIZE (LSU_WORD_SIZE), - .TAG_WIDTH (LSU_TAG_WIDTH) - ) lmem_adapt_if[LSU_NUM_REQS](); - - for (genvar i = 0; i < `VX_CFG_NUM_LSU_BLOCKS; ++i) begin : g_lmem_adapters - - VX_mem_bus_if #( - .DATA_SIZE (LSU_WORD_SIZE), - .TAG_WIDTH (LSU_TAG_WIDTH) - ) lmem_block_if[`VX_CFG_NUM_LSU_LANES](); - - VX_lsu_adapter #( - .NUM_LANES (`VX_CFG_NUM_LSU_LANES), - .DATA_SIZE (LSU_WORD_SIZE), - .TAG_WIDTH (LSU_TAG_WIDTH), - .TAG_SEL_BITS (LSU_TAG_WIDTH - UUID_WIDTH), - .ARBITER ("P"), - .REQ_OUT_BUF (3), - .RSP_OUT_BUF (0) - ) lmem_adapter ( - .clk (clk), - .reset (reset), - .lsu_mem_if (lsu_lmem_if[i]), - .mem_bus_if (lmem_block_if) - ); - - for (genvar j = 0; j < `VX_CFG_NUM_LSU_LANES; ++j) begin : g_lmem_adapt_if - `ASSIGN_VX_MEM_BUS_IF (lmem_adapt_if[i * `VX_CFG_NUM_LSU_LANES + j], lmem_block_if[j]); - end - end - // DMA arbiter: merge DXA writes and/or TCU reads onto single DMA port. VX_mem_bus_if #( @@ -247,127 +180,17 @@ module VX_mem_unit import VX_gpu_pkg::*; #( .lmem_perf (lmem_perf), `endif .dma_bus_if (lmem_dma_if), - .lsu_bus_if (lmem_adapt_if) + .lsu_bus_if (lmem_bus_if) ); `else - for (genvar i = 0; i < `VX_CFG_NUM_LSU_BLOCKS; ++i) begin : g_lsu_dcache_if - `ASSIGN_VX_MEM_BUS_IF (lsu_dcache_if[i], lsu_mem_if[i]); - end - `ifdef PERF_ENABLE assign lmem_perf = '0; `endif `endif - VX_lsu_mem_if #( - .NUM_LANES (DCACHE_CHANNELS), - .DATA_SIZE (DCACHE_WORD_SIZE), - .TAG_WIDTH (DCACHE_CORE_TAG_WIDTH) - ) dcache_coalesced_if[`VX_CFG_NUM_LSU_BLOCKS](); - -`ifdef PERF_ENABLE - wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][PERF_CTR_BITS-1:0] per_block_coalescer_misses; - wire [PERF_CTR_BITS-1:0] coalescer_misses; - VX_reduce_tree #( - .IN_W (PERF_CTR_BITS), - .N (`VX_CFG_NUM_LSU_BLOCKS), - .OP ("+") - ) coalescer_reduce ( - .data_in (per_block_coalescer_misses), - .data_out (coalescer_misses) - ); - `BUFFER(coalescer_perf.misses, coalescer_misses); -`endif - - if ((`VX_CFG_NUM_LSU_LANES > 1) && (DCACHE_WORD_SIZE > LSU_WORD_SIZE)) begin : g_coalescing - - for (genvar i = 0; i < `VX_CFG_NUM_LSU_BLOCKS; ++i) begin : g_coalescers - - VX_mem_coalescer #( - .INSTANCE_ID (`SFORMATF(("%s-coalescer%0d", INSTANCE_ID, i))), - .NUM_REQS (`VX_CFG_NUM_LSU_LANES), - .DATA_IN_SIZE (LSU_WORD_SIZE), - .DATA_OUT_SIZE (DCACHE_WORD_SIZE), - .ADDR_WIDTH (LSU_ADDR_WIDTH), - .USER_WIDTH (MEM_ATTR_WIDTH), - .TAG_WIDTH (LSU_TAG_WIDTH), - .UUID_WIDTH (UUID_WIDTH), - .QUEUE_SIZE (LSU_QUEUE_OUT_SIZE), - .PERF_CTR_BITS (PERF_CTR_BITS) - ) mem_coalescer ( - .clk (clk), - .reset (reset), - - `ifdef PERF_ENABLE - .misses (per_block_coalescer_misses[i]), - `else - `UNUSED_PIN (misses), - `endif - - .empty (per_block_path_empty[i]), - - // Input request - .in_req_valid (lsu_dcache_if[i].req_valid), - .in_req_mask (lsu_dcache_if[i].req_data.mask), - .in_req_rw (lsu_dcache_if[i].req_data.rw), - .in_req_byteen (lsu_dcache_if[i].req_data.byteen), - .in_req_addr (lsu_dcache_if[i].req_data.addr), - .in_req_user (lsu_dcache_if[i].req_data.user), - .in_req_no_merge(lsu_dcache_if[i].req_data.user[0][MEM_ATTR_AMO_OFFS]), - .in_req_data (lsu_dcache_if[i].req_data.data), - .in_req_tag (lsu_dcache_if[i].req_data.tag), - .in_req_ready (lsu_dcache_if[i].req_ready), - - // Input response - .in_rsp_valid (lsu_dcache_if[i].rsp_valid), - .in_rsp_mask (lsu_dcache_if[i].rsp_data.mask), - .in_rsp_data (lsu_dcache_if[i].rsp_data.data), - .in_rsp_tag (lsu_dcache_if[i].rsp_data.tag), - .in_rsp_ready (lsu_dcache_if[i].rsp_ready), - - // Output request - .out_req_valid (dcache_coalesced_if[i].req_valid), - .out_req_mask (dcache_coalesced_if[i].req_data.mask), - .out_req_rw (dcache_coalesced_if[i].req_data.rw), - .out_req_byteen (dcache_coalesced_if[i].req_data.byteen), - .out_req_addr (dcache_coalesced_if[i].req_data.addr), - .out_req_user (dcache_coalesced_if[i].req_data.user), - .out_req_data (dcache_coalesced_if[i].req_data.data), - .out_req_tag (dcache_coalesced_if[i].req_data.tag), - .out_req_ready (dcache_coalesced_if[i].req_ready), - - // Output response - .out_rsp_valid (dcache_coalesced_if[i].rsp_valid), - .out_rsp_mask (dcache_coalesced_if[i].rsp_data.mask), - .out_rsp_data (dcache_coalesced_if[i].rsp_data.data), - .out_rsp_tag (dcache_coalesced_if[i].rsp_data.tag), - .out_rsp_ready (dcache_coalesced_if[i].rsp_ready) - ); - end - - end else begin : g_no_coalescing - - // The passthrough narrows the tag from LSU_TAG_WIDTH to - // DCACHE_CORE_TAG_WIDTH: the dcache tag-id space must cover the - // mem-scheduler slot id or responses alias. - `STATIC_ASSERT (DCACHE_CORE_TAG_WIDTH >= LSU_TAG_WIDTH, ("dcache tag-id space cannot hold the LSU outstanding slot id")) - - for (genvar i = 0; i < `VX_CFG_NUM_LSU_BLOCKS; ++i) begin : g_dcache_coalesced_if - `ASSIGN_VX_MEM_BUS_IF (dcache_coalesced_if[i], lsu_dcache_if[i]); - `ifdef PERF_ENABLE - assign per_block_coalescer_misses[i] = '0; - `endif - // No coalescer: a store still in flight upstream of the adapter is - // one being presented to the adapter input. The adapter's own buffer - // is accounted for separately by per_block_adapter_empty. - assign per_block_path_empty[i] = ~dcache_coalesced_if[i].req_valid; - end - - end - // A private dcache takes the lanes straight from the adapter, whose drain // store_drained already waits for, so one token suffices. Otherwise (a // shared dcache, or none) the lanes are buffered independently before the @@ -384,61 +207,62 @@ module VX_mem_unit import VX_gpu_pkg::*; #( end assign dcr_flush_if.done = (& lane_flush_done); - for (genvar i = 0; i < `VX_CFG_NUM_LSU_BLOCKS; ++i) begin : g_dcache_adapters - - VX_mem_bus_if #( - .DATA_SIZE (DCACHE_WORD_SIZE), - .TAG_WIDTH (DCACHE_CORE_TAG_WIDTH) - ) dcache_bus_tmp_if[DCACHE_CHANNELS](); - - VX_lsu_adapter #( - .NUM_LANES (DCACHE_CHANNELS), - .DATA_SIZE (DCACHE_WORD_SIZE), - .TAG_WIDTH (DCACHE_CORE_TAG_WIDTH), - .TAG_SEL_BITS (DCACHE_CORE_TAG_WIDTH - UUID_WIDTH), - .ARBITER ("P"), - .REQ_OUT_BUF (3), // register dcache-request master boundary on all channels (passthru ports 1+) - .RSP_OUT_BUF (0) - ) dcache_adapter ( - .clk (clk), - .reset (reset), - .lsu_mem_if (dcache_coalesced_if[i]), - .mem_bus_if (dcache_bus_tmp_if) - ); - - // A store buffered in the adapter's REQ_OUT_BUF has left dcache_coalesced_if - // but not yet reached the flush-injection point; keep the block non-empty - // until every channel's request has drained past the adapter output. - wire [DCACHE_CHANNELS-1:0] adapter_req_valid; - for (genvar j = 0; j < DCACHE_CHANNELS; ++j) begin : g_adapter_req_valid - assign adapter_req_valid[j] = dcache_bus_tmp_if[j].req_valid; - end - assign per_block_adapter_empty[i] = ~(| adapter_req_valid); - - for (genvar j = 0; j < DCACHE_CHANNELS; ++j) begin : g_dcache_bus_if - localparam LANE = i * DCACHE_CHANNELS + j; - if (LANE < FLUSH_LANES) begin : g_flush_lane - VX_dcr_flush #( - .WORD_SIZE (DCACHE_WORD_SIZE), - .TAG_WIDTH (DCACHE_CORE_TAG_WIDTH), - .REQ_OUT_BUF (3) // register core dcache-request boundary; rsp already registered by L1 CORE_OUT_BUF - ) dcr_flush ( - .clk (clk), - .reset (reset), - .dcr_flush_if (lane_flush_if[LANE]), - .l1_only (flush_l1_only), - .core_bus_if (dcache_bus_tmp_if[j]), - .cache_bus_if (dcache_bus_if[LANE]) - ); - end else begin : g_passthru_lane - // tag is zero-extended from DCACHE_CORE_TAG_WIDTH to - // DCACHE_TAG_WIDTH on request, and MSB-stripped on response - `ASSIGN_VX_MEM_BUS_IF_EX (dcache_bus_if[LANE], dcache_bus_tmp_if[j], DCACHE_TAG_WIDTH_BASE, DCACHE_CORE_TAG_WIDTH, 0); - end + // A block's D-cache request carries a clear tag LSB, the flush arbiter's + // select bit; the block takes the cache response directly and drops the + // flush token's. + for (genvar i = 0; i < DCACHE_NUM_REQS; ++i) begin : g_dcache_bus_if + if (i < FLUSH_LANES) begin : g_flush_lane + VX_mem_bus_if #( + .DATA_SIZE (DCACHE_WORD_SIZE), + .TAG_WIDTH (DCACHE_CORE_TAG_WIDTH) + ) flush_core_if(); + + assign flush_core_if.req_valid = lsu_dcache_bus_if[i].req_valid; + assign flush_core_if.req_data.rw = lsu_dcache_bus_if[i].req_data.rw; + assign flush_core_if.req_data.addr = lsu_dcache_bus_if[i].req_data.addr; + assign flush_core_if.req_data.data = lsu_dcache_bus_if[i].req_data.data; + assign flush_core_if.req_data.byteen = lsu_dcache_bus_if[i].req_data.byteen; + assign flush_core_if.req_data.attr = lsu_dcache_bus_if[i].req_data.attr; + assign flush_core_if.req_data.tag = lsu_dcache_bus_if[i].req_data.tag[DCACHE_TAG_WIDTH-1:1]; + assign lsu_dcache_bus_if[i].req_ready = flush_core_if.req_ready; + `UNUSED_VAR (lsu_dcache_bus_if[i].req_data.tag[0]) + + assign lsu_dcache_bus_if[i].rsp_valid = dcache_bus_if[i].rsp_valid; + assign lsu_dcache_bus_if[i].rsp_data = dcache_bus_if[i].rsp_data; + assign flush_core_if.rsp_ready = lsu_dcache_bus_if[i].rsp_ready; + `UNUSED_VAR (flush_core_if.rsp_valid) + `UNUSED_VAR (flush_core_if.rsp_data) + + VX_dcr_flush #( + .WORD_SIZE (DCACHE_WORD_SIZE), + .TAG_WIDTH (DCACHE_CORE_TAG_WIDTH), + .REQ_OUT_BUF (3) // register core dcache-request boundary; rsp already registered by L1 CORE_OUT_BUF + ) dcr_flush ( + .clk (clk), + .reset (reset), + .dcr_flush_if (lane_flush_if[i]), + .l1_only (flush_l1_only), + .core_bus_if (flush_core_if), + .cache_bus_if (dcache_bus_if[i]) + ); + end else begin : g_passthru_lane + // the cache sees the select bit as a clear tag MSB + assign dcache_bus_if[i].req_valid = lsu_dcache_bus_if[i].req_valid; + assign dcache_bus_if[i].req_data.rw = lsu_dcache_bus_if[i].req_data.rw; + assign dcache_bus_if[i].req_data.addr = lsu_dcache_bus_if[i].req_data.addr; + assign dcache_bus_if[i].req_data.data = lsu_dcache_bus_if[i].req_data.data; + assign dcache_bus_if[i].req_data.byteen = lsu_dcache_bus_if[i].req_data.byteen; + assign dcache_bus_if[i].req_data.attr = lsu_dcache_bus_if[i].req_data.attr; + assign dcache_bus_if[i].req_data.tag = {1'b0, lsu_dcache_bus_if[i].req_data.tag[DCACHE_TAG_WIDTH-1:1]}; + assign lsu_dcache_bus_if[i].req_ready = dcache_bus_if[i].req_ready; + `UNUSED_VAR (lsu_dcache_bus_if[i].req_data.tag[0]) + + assign lsu_dcache_bus_if[i].rsp_valid = dcache_bus_if[i].rsp_valid; + assign lsu_dcache_bus_if[i].rsp_data.data = dcache_bus_if[i].rsp_data.data; + assign lsu_dcache_bus_if[i].rsp_data.tag = {dcache_bus_if[i].rsp_data.tag[DCACHE_TAG_WIDTH-2:0], 1'b0}; + assign dcache_bus_if[i].rsp_ready = lsu_dcache_bus_if[i].rsp_ready; + `UNUSED_VAR (dcache_bus_if[i].rsp_data.tag[DCACHE_TAG_WIDTH-1]) end - end - assign empty = (& per_block_empty); - endmodule diff --git a/hw/unittest/mem_unit/VX_mem_unit_top.sv b/hw/unittest/mem_unit/VX_mem_unit_top.sv index ad2c75034e..ed556ebf4c 100644 --- a/hw/unittest/mem_unit/VX_mem_unit_top.sv +++ b/hw/unittest/mem_unit/VX_mem_unit_top.sv @@ -14,29 +14,19 @@ `include "VX_define.vh" module VX_mem_unit_top import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - parameter LSU_WORD_WIDTH = LSU_WORD_SIZE * 8 + parameter `STRING INSTANCE_ID = "" ) ( // Clock input wire clk, input wire reset, - // LSU memory request + // LSU block memory clients input wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_req_valid, - input wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_req_rw, - input wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0] lsu_req_mask, - input wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0][LSU_WORD_SIZE-1:0] lsu_req_byteen, - input wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0][LSU_ADDR_WIDTH-1:0] lsu_req_addr, - input wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0][MEM_ATTR_WIDTH-1:0] lsu_req_user, - input wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0][LSU_WORD_WIDTH-1:0] lsu_req_data, - input wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][LSU_TAG_WIDTH-1:0] lsu_req_tag, + input lsu_req_data_t [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_req_data, output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_req_ready, - // LSU memory response output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_rsp_valid, - output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0] lsu_rsp_mask, - output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0][LSU_WORD_WIDTH-1:0] lsu_rsp_data, - output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][LSU_TAG_WIDTH-1:0] lsu_rsp_tag, + output lsu_rsp_data_t [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_rsp_data, input wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_rsp_ready, // Memory request @@ -55,32 +45,60 @@ module VX_mem_unit_top import VX_gpu_pkg::*; #( input wire [DCACHE_NUM_REQS-1:0][DCACHE_TAG_WIDTH-1:0] mem_rsp_tag, output wire [DCACHE_NUM_REQS-1:0] mem_rsp_ready ); - VX_lsu_mem_if #( - .NUM_LANES (`VX_CFG_NUM_LSU_LANES), +`ifdef VX_CFG_LMEM_ENABLE + VX_mem_bus_if #( .DATA_SIZE (LSU_WORD_SIZE), .TAG_WIDTH (LSU_TAG_WIDTH) - ) lsu_mem_if[`VX_CFG_NUM_LSU_BLOCKS](); - - // LSU memory request - for (genvar i = 0; i < `VX_CFG_NUM_LSU_BLOCKS; ++i) begin : g_lsu_mem_req - assign lsu_mem_if[i].req_valid = lsu_req_valid[i]; - assign lsu_mem_if[i].req_data.rw = lsu_req_rw[i]; - assign lsu_mem_if[i].req_data.mask = lsu_req_mask[i]; - assign lsu_mem_if[i].req_data.byteen = lsu_req_byteen[i]; - assign lsu_mem_if[i].req_data.addr = lsu_req_addr[i]; - assign lsu_mem_if[i].req_data.user = lsu_req_user[i]; - assign lsu_mem_if[i].req_data.data = lsu_req_data[i]; - assign lsu_mem_if[i].req_data.tag = lsu_req_tag[i]; - assign lsu_req_ready[i] = lsu_mem_if[i].req_ready; - end + ) lsu_lmem_bus_if[LSU_NUM_REQS](); +`endif + + VX_mem_bus_if #( + .DATA_SIZE (DCACHE_WORD_SIZE), + .TAG_WIDTH (DCACHE_TAG_WIDTH) + ) lsu_dcache_bus_if[DCACHE_NUM_REQS](); + + for (genvar i = 0; i < `VX_CFG_NUM_LSU_BLOCKS; ++i) begin : g_lsu_mem_paths + VX_lsu_sched_if client_if(); + + assign client_if.req_valid = lsu_req_valid[i]; + assign client_if.req_data = lsu_req_data[i]; + assign lsu_req_ready[i] = client_if.req_ready; + assign lsu_rsp_valid[i] = client_if.rsp_valid; + assign lsu_rsp_data[i] = client_if.rsp_data; + assign client_if.rsp_ready = lsu_rsp_ready[i]; + + `ifdef TCU_META_ENABLE + VX_lsu_sched_if tcu_mem_if(); + assign tcu_mem_if.req_valid = 1'b0; + assign tcu_mem_if.req_data = '0; + assign tcu_mem_if.rsp_ready = 1'b1; + `UNUSED_VAR (tcu_mem_if.req_ready) + `UNUSED_VAR (tcu_mem_if.rsp_valid) + `UNUSED_VAR (tcu_mem_if.rsp_data) + `endif - // LSU memory response - for (genvar i = 0; i < `VX_CFG_NUM_LSU_BLOCKS; ++i) begin : g_lsu_rsp - assign lsu_rsp_valid[i] = lsu_mem_if[i].rsp_valid; - assign lsu_rsp_mask[i] = lsu_mem_if[i].rsp_data.mask; - assign lsu_rsp_data[i] = lsu_mem_if[i].rsp_data.data; - assign lsu_rsp_tag[i] = lsu_mem_if[i].rsp_data.tag; - assign lsu_mem_if[i].rsp_ready = lsu_rsp_ready[i]; + VX_lsu_mem_path #( + .INSTANCE_ID (`SFORMATF(("%s-lsu%0d", INSTANCE_ID, i))) + ) mem_path ( + .clk (clk), + .reset (reset), + `ifdef PERF_ENABLE + `UNUSED_PIN (perf_rd_req_fire), + `UNUSED_PIN (perf_wr_req_fire), + `UNUSED_PIN (perf_rsp_fire), + `UNUSED_PIN (coalescer_misses), + `endif + .client_if (client_if), + `ifdef TCU_META_ENABLE + .tcu_mem_if (tcu_mem_if), + `endif + `UNUSED_PIN (sched_empty), + `UNUSED_PIN (store_empty), + `ifdef VX_CFG_LMEM_ENABLE + .lmem_bus_if (lsu_lmem_bus_if[i * `VX_CFG_NUM_LSU_LANES +: `VX_CFG_NUM_LSU_LANES]), + `endif + .dcache_bus_if (lsu_dcache_bus_if[i * DCACHE_CHANNELS +: DCACHE_CHANNELS]) + ); end VX_mem_bus_if #( @@ -116,9 +134,6 @@ module VX_mem_unit_top import VX_gpu_pkg::*; #( assign dcr_flush_if.req = 1'b0; `UNUSED_VAR (dcr_flush_if.done) - wire mem_unit_empty; - `UNUSED_VAR (mem_unit_empty) - VX_mem_unit #( .INSTANCE_ID (INSTANCE_ID) ) mem_unit ( @@ -127,8 +142,10 @@ module VX_mem_unit_top import VX_gpu_pkg::*; #( `ifdef PERF_ENABLE .lmem_perf (lmem_perf), `endif - .empty (mem_unit_empty), - .lsu_mem_if (lsu_mem_if), + `ifdef VX_CFG_LMEM_ENABLE + .lmem_bus_if (lsu_lmem_bus_if), + `endif + .lsu_dcache_bus_if (lsu_dcache_bus_if), .dcr_flush_if (dcr_flush_if), .flush_l1_only (1'b0), .dcache_bus_if (mem_bus_if) diff --git a/hw/unittest/tensor/VX_tensor_top.sv b/hw/unittest/tensor/VX_tensor_top.sv index e56e5c14b6..05037a0908 100644 --- a/hw/unittest/tensor/VX_tensor_top.sv +++ b/hw/unittest/tensor/VX_tensor_top.sv @@ -21,10 +21,11 @@ // per core (VX_CFG_SOCKET_SIZE): // VX_tcu_unit : tensor-core unit (dispatch/commit from top ports) // VX_tcu_shared : WGMMA tile buffers + metadata AGU -// VX_lsu_scheduler : per-LSU-block, 2 clients — the ordinary LSU stream -// (top ports) and the TCU AGU metadata-load port -// VX_mem_unit : LMEM + coalescer + LMEM-DMA arb (TCU WGMMA + DXA -// staging converge here) + dcache adapters +// VX_lsu_mem_path : per-LSU-block scheduler (2 clients — the ordinary LSU +// stream from top ports and the TCU AGU metadata-load +// port), lmem switch, coalescer and adapters +// VX_mem_unit : LMEM + LMEM-DMA arb (TCU WGMMA + DXA staging +// converge here) + dcache flush injectors // per socket: // VX_dxa_core : descriptor-driven tile-transfer engine (DCR-fed) // VX_mem_bus_switch : DXA staging writes back to the owning core's LMEM @@ -36,7 +37,7 @@ // // Memory hierarchy: TCU WGMMA prefetches and DXA staging writes meet at the // real LMEM DMA arbiter inside VX_mem_unit; TCU_LD metadata loads merge with -// the ordinary LSU stream at block-0's VX_lsu_scheduler; global traffic +// the ordinary LSU stream at block-0's scheduler; global traffic // (dcache refills, DXA tile reads) contends at the socket arbiter and the // shared L2, whose memory side exits to top-level ports. // @@ -74,7 +75,7 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( // ----------------------------------------------------------------------- // Ordinary LSU client streams, flattened [core][lsu-block] - // (client 0 of each VX_lsu_scheduler; the TCU AGU is client 1 on block 0) + // (client 0 of each LSU block's scheduler; the TCU AGU is client 1 on block 0) // ----------------------------------------------------------------------- input wire [LSU_STREAMS-1:0] lsu_req_valid, input lsu_req_data_t [LSU_STREAMS-1:0] lsu_req_data, @@ -121,12 +122,6 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( ); localparam TCU_LMEM_BANK_ADDR_W = `VX_CFG_LMEM_LOG_SIZE - `CLOG2(LSU_WORD_SIZE) - `CLOG2(`VX_CFG_LMEM_NUM_BANKS); -`ifdef TCU_META_ENABLE - localparam LSU_SCHED_NUM_CLIENTS = 2; -`else - localparam LSU_SCHED_NUM_CLIENTS = 1; -`endif - // ------------------------------------------------------------------------ // Per-core DXA staging returns + collected dcache core ports // ------------------------------------------------------------------------ @@ -149,7 +144,7 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( wire [NC-1:0] lsu_sched_empty; // ======================================================================== - // Per-core tensor slice: TCU + LSU schedulers + mem_unit + // Per-core tensor slice: TCU + LSU memory paths + mem_unit // ======================================================================== for (genvar c = 0; c < NC; ++c) begin : g_cores @@ -173,7 +168,15 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( ) tcu_lmem_if(); `ifdef TCU_META_ENABLE - VX_lsu_sched_if tcu_mem_if(); + VX_lsu_sched_if lsu_tcu_mem_if [NUM_LSU_BLKS](); + for (genvar block_idx = 1; block_idx < NUM_LSU_BLKS; ++block_idx) begin : g_tcu_client_tieoff + assign lsu_tcu_mem_if[block_idx].req_valid = 1'b0; + assign lsu_tcu_mem_if[block_idx].req_data = '0; + assign lsu_tcu_mem_if[block_idx].rsp_ready = 1'b1; + `UNUSED_VAR (lsu_tcu_mem_if[block_idx].req_ready) + `UNUSED_VAR (lsu_tcu_mem_if[block_idx].rsp_valid) + `UNUSED_VAR (lsu_tcu_mem_if[block_idx].rsp_data) + end `endif `ifdef VX_CFG_TCU_WGMMA_ENABLE @@ -249,70 +252,65 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( .meta_wr_wid (tcu_meta_wr_wid), .meta_wr_idx (tcu_meta_wr_idx), .meta_wr_data (tcu_meta_wr_data), - .tcu_mem_if (tcu_mem_if), + .tcu_mem_if (lsu_tcu_mem_if[0]), `endif .clk (clk), .reset (reset) ); - // --- LSU schedulers: client 0 from top ports, client 1 = TCU AGU --- - VX_lsu_mem_if #( - .NUM_LANES (`VX_CFG_NUM_LSU_LANES), - .DATA_SIZE (LSU_WORD_SIZE), - .TAG_WIDTH (LSU_TAG_WIDTH) - ) lsu_mem_if[NUM_LSU_BLKS](); - + // --- LSU memory paths: client 0 from top ports, client 1 = TCU AGU --- wire [NUM_LSU_BLKS-1:0] block_sched_empty; + wire [NUM_LSU_BLKS-1:0] block_store_empty; - for (genvar block_idx = 0; block_idx < NUM_LSU_BLKS; ++block_idx) begin : g_lsu_scheduler - localparam s = c * NUM_LSU_BLKS + block_idx; + VX_mem_bus_if #( + .DATA_SIZE (LSU_WORD_SIZE), + .TAG_WIDTH (LSU_TAG_WIDTH) + ) lsu_lmem_bus_if[LSU_NUM_REQS](); - VX_lsu_sched_if sched_client_if [LSU_SCHED_NUM_CLIENTS](); + VX_mem_bus_if #( + .DATA_SIZE (DCACHE_WORD_SIZE), + .TAG_WIDTH (DCACHE_TAG_WIDTH) + ) lsu_dcache_bus_if[DCACHE_NUM_REQS](); - assign sched_client_if[0].req_valid = lsu_req_valid[s]; - assign sched_client_if[0].req_data = lsu_req_data[s]; - assign lsu_req_ready[s] = sched_client_if[0].req_ready; - assign lsu_rsp_valid[s] = sched_client_if[0].rsp_valid; - assign lsu_rsp_data[s] = sched_client_if[0].rsp_data; - assign sched_client_if[0].rsp_ready = lsu_rsp_ready[s]; - `ifdef TCU_META_ENABLE - if (block_idx == 0) begin : g_tcu_client - assign sched_client_if[1].req_valid = tcu_mem_if.req_valid; - assign sched_client_if[1].req_data = tcu_mem_if.req_data; - assign tcu_mem_if.req_ready = sched_client_if[1].req_ready; - assign tcu_mem_if.rsp_valid = sched_client_if[1].rsp_valid; - assign tcu_mem_if.rsp_data = sched_client_if[1].rsp_data; - assign sched_client_if[1].rsp_ready = tcu_mem_if.rsp_ready; - end else begin : g_tcu_client_tieoff - assign sched_client_if[1].req_valid = 1'b0; - assign sched_client_if[1].req_data = '0; - assign sched_client_if[1].rsp_ready = 1'b1; - `UNUSED_VAR (sched_client_if[1].req_ready) - `UNUSED_VAR (sched_client_if[1].rsp_valid) - `UNUSED_VAR (sched_client_if[1].rsp_data) - end - `endif + for (genvar block_idx = 0; block_idx < NUM_LSU_BLKS; ++block_idx) begin : g_lsu_mem_path + localparam s = c * NUM_LSU_BLKS + block_idx; - VX_lsu_scheduler #( - .INSTANCE_ID (`SFORMATF(("%s-lsusched%0d-%0d", INSTANCE_ID, c, block_idx))), - .NUM_CLIENTS (LSU_SCHED_NUM_CLIENTS), - .NUM_LANES (`VX_CFG_NUM_LSU_LANES), - .CORE_QUEUE_SIZE(`VX_CFG_LSU_QUEUE_IN_SIZE), - .MEM_QUEUE_SIZE (LSU_QUEUE_OUT_SIZE), - .PENDING_SIZE (`VX_CFG_LSU_PENDING_SIZE) - ) lsu_scheduler ( - .clk (clk), - .reset (reset), - .client_if (sched_client_if), - .empty (block_sched_empty[block_idx]), - .lsu_mem_if (lsu_mem_if[block_idx]) + VX_lsu_sched_if client_if(); + + assign client_if.req_valid = lsu_req_valid[s]; + assign client_if.req_data = lsu_req_data[s]; + assign lsu_req_ready[s] = client_if.req_ready; + assign lsu_rsp_valid[s] = client_if.rsp_valid; + assign lsu_rsp_data[s] = client_if.rsp_data; + assign client_if.rsp_ready = lsu_rsp_ready[s]; + + VX_lsu_mem_path #( + .INSTANCE_ID (`SFORMATF(("%s-lsu%0d-%0d", INSTANCE_ID, c, block_idx))) + ) mem_path ( + .clk (clk), + .reset (reset), + `ifdef PERF_ENABLE + `UNUSED_PIN (perf_rd_req_fire), + `UNUSED_PIN (perf_wr_req_fire), + `UNUSED_PIN (perf_rsp_fire), + `UNUSED_PIN (coalescer_misses), + `endif + .client_if (client_if), + `ifdef TCU_META_ENABLE + .tcu_mem_if (lsu_tcu_mem_if[block_idx]), + `endif + .sched_empty (block_sched_empty[block_idx]), + .store_empty (block_store_empty[block_idx]), + .lmem_bus_if (lsu_lmem_bus_if[block_idx * `VX_CFG_NUM_LSU_LANES +: `VX_CFG_NUM_LSU_LANES]), + .dcache_bus_if (lsu_dcache_bus_if[block_idx * DCACHE_CHANNELS +: DCACHE_CHANNELS]) ); end assign lsu_sched_empty[c] = & block_sched_empty; + assign mem_unit_empty[c] = & block_store_empty; - // --- mem_unit: LMEM + DMA arb + dcache adapters --- + // --- mem_unit: LMEM + DMA arb + dcache flush injectors --- VX_txbar_bus_if dxa_txbar_bus_if(); assign txbar_valid[c] = dxa_txbar_bus_if.valid; assign txbar_data[c] = dxa_txbar_bus_if.data; @@ -339,8 +337,8 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( .dxa_lmem_bus_if (per_core_dxa_lmem_bus_if[c]), .dxa_txbar_bus_if(dxa_txbar_bus_if), `endif - .empty (mem_unit_empty[c]), - .lsu_mem_if (lsu_mem_if), + .lmem_bus_if (lsu_lmem_bus_if), + .lsu_dcache_bus_if (lsu_dcache_bus_if), .dcr_flush_if (dcr_flush_dcache_if), .flush_l1_only (1'b0), .dcache_bus_if (dcache_bus_if) From 304e0db175998cbff826db1d7c9a7e359e48c7f8 Mon Sep 17 00:00:00 2001 From: tinebp Date: Thu, 22 Oct 2026 02:15:49 -0700 Subject: [PATCH 085/124] core: export the LSU blocks' status bits instead of combining them in the core With the memory path inside each LSU block, the core would build busy and lsu_drained from other modules' outputs in the same cycle. Each block now exports sched_empty, store_empty and busy. Every slice ANDs the sched_empty vector, with mmu_empty, next to its warp control; the socket ORs the blocks' busy vector into its registered busy, beside the core's own terms; store_drained stays in the core, feeding its flush injectors. Each AND and OR is the same function as before, so no status signal moves in any cycle. rtlsim sgemm_tcu_wg_sp_dxa -m 128 -n 128 -k 128: cycle-identical to the parent (144,956), PASSED. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/VX_socket.sv | 4 +- hw/rtl/core/VX_backend_slice.sv | 2 + hw/rtl/core/VX_compute_slice.sv | 8 +++- hw/rtl/core/VX_core.sv | 53 +++++++++++++------------ hw/rtl/core/VX_execute.sv | 2 + hw/rtl/core/VX_execute_slice.sv | 2 + hw/rtl/core/VX_frontend_xbar.sv | 6 ++- hw/rtl/core/VX_lsu_block.sv | 2 + hw/rtl/core/VX_lsu_mem_path.sv | 2 + hw/rtl/core/VX_lsu_unit.sv | 2 + hw/rtl/core/VX_mem_unit.sv | 11 ++++- hw/rtl/core/VX_wctl_unit.sv | 3 +- hw/rtl/interfaces/VX_warp_ctl_if.sv | 20 ++++++---- hw/unittest/core/VX_core_top.sv | 7 +++- hw/unittest/mem_unit/VX_mem_unit_top.sv | 4 +- hw/unittest/tensor/VX_tensor_top.sv | 4 +- 16 files changed, 88 insertions(+), 44 deletions(-) diff --git a/hw/rtl/VX_socket.sv b/hw/rtl/VX_socket.sv index 69c111f121..3df30eaef2 100644 --- a/hw/rtl/VX_socket.sv +++ b/hw/rtl/VX_socket.sv @@ -842,6 +842,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; /////////////////////////////////////////////////////////////////////////// wire [`VX_CFG_SOCKET_SIZE-1:0] per_core_busy; + wire [`VX_CFG_SOCKET_SIZE-1:0][`VX_CFG_NUM_LSU_BLOCKS-1:0] per_core_lsu_busy; `ifdef EXT_GFX_ANY_ENABLE // Core flush requests fan out to the socket-local gfx caches (tcache, // rtcache) and up to the cluster (rcache, ocache); each leg's done is @@ -936,6 +937,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .gbar_bus_if (per_core_gbar_bus_if[core_id]), + .lsu_busy (per_core_lsu_busy[core_id]), .busy (per_core_busy[core_id]) ); end @@ -960,7 +962,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; wire mem_req_empty = icache_req_empty && dcache_req_empty && tcache_req_empty && rtcache_req_empty && mem_arb_empty; wire busy_r; - `BUFFER_EX(busy_r, dcr_bus_if.req_valid | (|per_core_busy) | ~mem_req_empty, 1'b1, 1, (`VX_CFG_SOCKET_SIZE > 1)); + `BUFFER_EX(busy_r, dcr_bus_if.req_valid | (|per_core_busy) | (|per_core_lsu_busy) | ~mem_req_empty, 1'b1, 1, (`VX_CFG_SOCKET_SIZE > 1)); assign busy = busy_r | dcr_bus_if.req_valid | kmu_bus_if[0].valid | (|per_core_kmu_valid); endmodule diff --git a/hw/rtl/core/VX_backend_slice.sv b/hw/rtl/core/VX_backend_slice.sv index acc5767a67..24e380cae0 100644 --- a/hw/rtl/core/VX_backend_slice.sv +++ b/hw/rtl/core/VX_backend_slice.sv @@ -67,6 +67,7 @@ import VX_tcu_pkg::*; `endif output wire lsu_sched_empty, output wire lsu_store_empty, + output wire lsu_busy, `ifdef VX_CFG_LMEM_ENABLE VX_mem_bus_if.master lmem_bus_if [`VX_CFG_NUM_LSU_LANES], `endif @@ -166,6 +167,7 @@ import VX_tcu_pkg::*; `endif .lsu_sched_empty (lsu_sched_empty), .lsu_store_empty (lsu_store_empty), + .lsu_busy (lsu_busy), `ifdef VX_CFG_LMEM_ENABLE .lmem_bus_if (lmem_bus_if), `endif diff --git a/hw/rtl/core/VX_compute_slice.sv b/hw/rtl/core/VX_compute_slice.sv index 63be670437..ddf14588e9 100644 --- a/hw/rtl/core/VX_compute_slice.sv +++ b/hw/rtl/core/VX_compute_slice.sv @@ -49,7 +49,8 @@ import VX_tcu_pkg::*; input wire [PERF_CTR_BITS-1:0] cycles, input wire [PERF_CTR_BITS-1:0] instret, input wire [`VX_CFG_NUM_WARPS-1:0] active_warps, - input wire lsu_sched_drained, + input wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] all_lsu_sched_empty, + input wire mmu_empty, output wire [NCTA_WIDTH-1:0] csr_rd_cta_id, input cta_ctx_t csr_rd_ctx, `ifdef VX_CFG_VM_ENABLE @@ -70,6 +71,7 @@ import VX_tcu_pkg::*; `endif output wire lsu_sched_empty, output wire lsu_store_empty, + output wire lsu_busy, `ifdef VX_CFG_LMEM_ENABLE VX_mem_bus_if.master lmem_bus_if [`VX_CFG_NUM_LSU_LANES], `endif @@ -201,7 +203,8 @@ import VX_tcu_pkg::*; assign warp_ctl_if.dvstack_ptr = dvstack_ptr[0]; assign warp_ctl_if.mscratch = dvstack_mscratch[0]; assign warp_ctl_if.warp_pending_alm_empty = pending_alm_empty; - assign warp_ctl_if.lsu_sched_drained = lsu_sched_drained; + assign warp_ctl_if.lsu_sched_empty = all_lsu_sched_empty; + assign warp_ctl_if.mmu_empty = mmu_empty; assign csr_rd_wid[0] = sched_csr_if.csr_rd_wid; assign csr_rd_cta_id = sched_csr_if.csr_rd_cta_id; @@ -276,6 +279,7 @@ import VX_tcu_pkg::*; `endif .lsu_sched_empty (lsu_sched_empty), .lsu_store_empty (lsu_store_empty), + .lsu_busy (lsu_busy), `ifdef VX_CFG_LMEM_ENABLE .lmem_bus_if (lmem_bus_if), `endif diff --git a/hw/rtl/core/VX_core.sv b/hw/rtl/core/VX_core.sv index aee719c076..7ef6230dfa 100644 --- a/hw/rtl/core/VX_core.sv +++ b/hw/rtl/core/VX_core.sv @@ -71,7 +71,8 @@ import VX_tcu_pkg::*; // Global barrier VX_gbar_bus_if.master gbar_bus_if, - // Status + // Status: the core's own busy, and each LSU block's memory path busy + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_busy, output wire busy ); VX_warp_shared_if warp_shared_if[`VX_CFG_NUM_SFU_BLOCKS](); @@ -157,6 +158,19 @@ import VX_tcu_pkg::*; // the in-core global-store path is empty wire mem_unit_empty = (& lsu_store_empty); + wire mem_unit_busy; + +`ifdef VX_CFG_VM_ENABLE +`ifdef PERF_ENABLE + // the MMU perf counters are kept at the socket + assign pipeline_perf.mmu = '0; +`endif + // Translation drains at the socket MMU; the socket returns its empty status. + wire mmu_empty = mmu_drained; +`else + wire mmu_empty = 1'b1; +`endif + // Stores are ack-less and lag warp exit, so the dcache flush waits until every // store of this core has reached L1. @@ -189,7 +203,6 @@ import VX_tcu_pkg::*; wire sched_busy; wire [PERF_CTR_BITS-1:0] sched_cycles; wire [PERF_CTR_BITS-1:0] sched_instret; - wire lsu_drained; VX_csr_mpm #( .INSTANCE_ID (`SFORMATF(("%s-mpm", INSTANCE_ID))) @@ -437,7 +450,8 @@ import VX_tcu_pkg::*; .cycles (sched_cycles), .instret (sched_instret), .active_warps (sched_active_warps), - .lsu_sched_drained (lsu_drained), + .all_lsu_sched_empty (lsu_sched_empty), + .mmu_empty (mmu_empty), .csr_rd_cta_id (csr_rd_cta_id[i]), .csr_rd_ctx (csr_rd_ctx[i]), `ifdef VX_CFG_VM_ENABLE @@ -456,6 +470,7 @@ import VX_tcu_pkg::*; `endif .lsu_sched_empty (lsu_sched_empty[i]), .lsu_store_empty (lsu_store_empty[i]), + .lsu_busy (lsu_busy[i]), `ifdef VX_CFG_LMEM_ENABLE .lmem_bus_if (lsu_lmem_bus_if[i * `VX_CFG_NUM_LSU_LANES +: `VX_CFG_NUM_LSU_LANES]), `endif @@ -551,7 +566,8 @@ import VX_tcu_pkg::*; .cycles (sched_cycles), .instret (sched_instret), .active_warps (sched_active_warps), - .lsu_sched_drained (lsu_drained), + .lsu_sched_empty (lsu_sched_empty), + .mmu_empty (mmu_empty), .csr_rd_ctx (csr_rd_ctx), `ifdef VX_CFG_VM_ENABLE .satp (sched_satp), @@ -692,6 +708,7 @@ import VX_tcu_pkg::*; `endif .lsu_sched_empty (lsu_sched_empty[i]), .lsu_store_empty (lsu_store_empty[i]), + .lsu_busy (lsu_busy[i]), `ifdef VX_CFG_LMEM_ENABLE .lmem_bus_if (lsu_lmem_bus_if[i * `VX_CFG_NUM_LSU_LANES +: `VX_CFG_NUM_LSU_LANES]), `endif @@ -786,6 +803,7 @@ import VX_tcu_pkg::*; `endif .lsu_sched_empty (lsu_sched_empty), .lsu_store_empty (lsu_store_empty), + .lsu_busy (lsu_busy), `ifdef VX_CFG_LMEM_ENABLE .lmem_bus_if (lsu_lmem_bus_if), `endif @@ -872,35 +890,18 @@ import VX_tcu_pkg::*; .lsu_dcache_bus_if (lsu_dcache_bus_if), .dcr_flush_if (dcr_flush_dcache_if), .flush_l1_only (dcr_flush_l1_only), - .dcache_bus_if (dcache_bus_if) + .dcache_bus_if (dcache_bus_if), + .busy (mem_unit_busy) ); -`ifdef VX_CFG_VM_ENABLE -`ifdef PERF_ENABLE - // the MMU perf counters are kept at the socket - assign pipeline_perf.mmu = '0; -`endif - // Translation drains at the socket MMU; the socket returns its empty status. - wire mmu_empty = mmu_drained; -`else - wire mmu_empty = 1'b1; -`endif - // Fragment work drains at the producer (VX_raster_core.busy), not here. // DCR reads (cache flush, MPM readback) order on their own DCR response, so // busy tracks only kernel execution: warps resident or stores not yet at L1. // ~mmu_empty also keeps the core busy while a translation or parked store is // still in flight, so kernel completion cannot race un-drained accesses. - // A request waiting at the dcache port sits past the flush injector's - // buffer, which mem_unit_empty does not cover. - wire [DCACHE_NUM_REQS-1:0] dcache_req_valid; - for (genvar i = 0; i < DCACHE_NUM_REQS; ++i) begin : g_dcache_req_valid - assign dcache_req_valid[i] = dcache_bus_if[i].req_valid; - end - assign busy = sched_busy || ~(&lsu_sched_empty) || ~mem_unit_empty || (| dcache_req_valid) || ~mmu_empty; - - // BAR (vx_barrier / vx_barrier_arrive) drains LSU + MMU before suspending or registering arrival. - assign lsu_drained = (&lsu_sched_empty) && mmu_empty; + // A store still in an LSU block's memory path shows on lsu_busy, which the + // parent ORs in; one past a flush injector's buffer shows on mem_unit_busy. + assign busy = sched_busy || mem_unit_busy || ~mmu_empty; `ifdef PERF_ENABLE diff --git a/hw/rtl/core/VX_execute.sv b/hw/rtl/core/VX_execute.sv index 8258555eaf..f4fd71f0c3 100644 --- a/hw/rtl/core/VX_execute.sv +++ b/hw/rtl/core/VX_execute.sv @@ -39,6 +39,7 @@ import VX_tcu_pkg::*; `endif output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_sched_empty, output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_store_empty, + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_busy, `ifdef VX_CFG_LMEM_ENABLE VX_mem_bus_if.master lmem_bus_if [LSU_NUM_REQS], `endif @@ -138,6 +139,7 @@ import VX_tcu_pkg::*; `endif .sched_empty (lsu_sched_empty), .store_empty (lsu_store_empty), + .busy (lsu_busy), `ifdef VX_CFG_LMEM_ENABLE .lmem_bus_if (lmem_bus_if), `endif diff --git a/hw/rtl/core/VX_execute_slice.sv b/hw/rtl/core/VX_execute_slice.sv index be0a1d001d..9e9bfb34eb 100644 --- a/hw/rtl/core/VX_execute_slice.sv +++ b/hw/rtl/core/VX_execute_slice.sv @@ -56,6 +56,7 @@ import VX_tcu_pkg::*; `endif output wire lsu_sched_empty, output wire lsu_store_empty, + output wire lsu_busy, `ifdef VX_CFG_LMEM_ENABLE VX_mem_bus_if.master lmem_bus_if [`VX_CFG_NUM_LSU_LANES], `endif @@ -202,6 +203,7 @@ import VX_tcu_pkg::*; `endif .sched_empty (lsu_sched_empty), .store_empty (lsu_store_empty), + .busy (lsu_busy), `ifdef VX_CFG_LMEM_ENABLE .lmem_bus_if (lmem_bus_if), `endif diff --git a/hw/rtl/core/VX_frontend_xbar.sv b/hw/rtl/core/VX_frontend_xbar.sv index 8f4d7cf3c7..252d4df822 100644 --- a/hw/rtl/core/VX_frontend_xbar.sv +++ b/hw/rtl/core/VX_frontend_xbar.sv @@ -43,7 +43,8 @@ module VX_frontend_xbar import VX_gpu_pkg::*; ( input wire [PERF_CTR_BITS-1:0] cycles, input wire [PERF_CTR_BITS-1:0] instret, input wire [`VX_CFG_NUM_WARPS-1:0] active_warps, - input wire lsu_sched_drained, + input wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_sched_empty, + input wire mmu_empty, input cta_ctx_t [`VX_CFG_NUM_SFU_BLOCKS-1:0] csr_rd_ctx, `ifdef VX_CFG_VM_ENABLE input wire [`VX_CFG_NUM_SFU_BLOCKS-1:0][`VX_CFG_XLEN-1:0] satp, @@ -196,7 +197,8 @@ module VX_frontend_xbar import VX_gpu_pkg::*; ( for (genvar k = 0; k < SFU_BLOCK_WARPS; ++k) begin : g_pending assign warp_ctl_if[b].warp_pending_alm_empty[k] = pending_alm_empty[k * BS + b]; end - assign warp_ctl_if[b].lsu_sched_drained = lsu_sched_drained; + assign warp_ctl_if[b].lsu_sched_empty = lsu_sched_empty; + assign warp_ctl_if[b].mmu_empty = mmu_empty; assign sched_csr_if[b].cycles = cycles; assign sched_csr_if[b].instret = instret; diff --git a/hw/rtl/core/VX_lsu_block.sv b/hw/rtl/core/VX_lsu_block.sv index 83c8ed4190..c43935bc21 100644 --- a/hw/rtl/core/VX_lsu_block.sv +++ b/hw/rtl/core/VX_lsu_block.sv @@ -42,6 +42,7 @@ module VX_lsu_block import VX_gpu_pkg::*; #( output wire sched_empty, output wire store_empty, + output wire busy, `ifdef VX_CFG_LMEM_ENABLE VX_mem_bus_if.master lmem_bus_if [`VX_CFG_NUM_LSU_LANES], @@ -418,6 +419,7 @@ module VX_lsu_block import VX_gpu_pkg::*; #( `endif .sched_empty (sched_empty), .store_empty (store_empty), + .busy (busy), `ifdef VX_CFG_LMEM_ENABLE .lmem_bus_if (lmem_bus_if), `endif diff --git a/hw/rtl/core/VX_lsu_mem_path.sv b/hw/rtl/core/VX_lsu_mem_path.sv index 7fd861ca86..3b3a255e0c 100644 --- a/hw/rtl/core/VX_lsu_mem_path.sv +++ b/hw/rtl/core/VX_lsu_mem_path.sv @@ -38,6 +38,7 @@ module VX_lsu_mem_path import VX_gpu_pkg::*; #( // the scheduler holds no request; the global-store path holds no store output wire sched_empty, output wire store_empty, + output wire busy, `ifdef VX_CFG_LMEM_ENABLE VX_mem_bus_if.master lmem_bus_if [`VX_CFG_NUM_LSU_LANES], @@ -107,6 +108,7 @@ module VX_lsu_mem_path import VX_gpu_pkg::*; #( wire path_empty; wire adapter_empty; assign store_empty = path_empty && adapter_empty; + assign busy = ~sched_empty || ~store_empty; VX_lsu_mem_if #( .NUM_LANES (`VX_CFG_NUM_LSU_LANES), diff --git a/hw/rtl/core/VX_lsu_unit.sv b/hw/rtl/core/VX_lsu_unit.sv index e383dfe5c8..ccd14130a3 100644 --- a/hw/rtl/core/VX_lsu_unit.sv +++ b/hw/rtl/core/VX_lsu_unit.sv @@ -42,6 +42,7 @@ module VX_lsu_unit import VX_gpu_pkg::*; #( output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] sched_empty, output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] store_empty, + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] busy, `ifdef VX_CFG_LMEM_ENABLE VX_mem_bus_if.master lmem_bus_if [LSU_NUM_REQS], @@ -112,6 +113,7 @@ module VX_lsu_unit import VX_gpu_pkg::*; #( `endif .sched_empty (sched_empty[block_idx]), .store_empty (store_empty[block_idx]), + .busy (busy[block_idx]), `ifdef VX_CFG_LMEM_ENABLE .lmem_bus_if (block_lmem_bus_if), `endif diff --git a/hw/rtl/core/VX_mem_unit.sv b/hw/rtl/core/VX_mem_unit.sv index a72e33c8c0..11a8228b36 100644 --- a/hw/rtl/core/VX_mem_unit.sv +++ b/hw/rtl/core/VX_mem_unit.sv @@ -41,7 +41,10 @@ module VX_mem_unit import VX_gpu_pkg::*; #( VX_dcr_flush_if.slave dcr_flush_if, input wire flush_l1_only, - VX_mem_bus_if.master dcache_bus_if [DCACHE_NUM_REQS] + VX_mem_bus_if.master dcache_bus_if [DCACHE_NUM_REQS], + + // a flush lane holds a request at the D-cache port + output wire busy ); `ifndef VX_CFG_LMEM_ENABLE `UNUSED_SPARAM (INSTANCE_ID) @@ -207,6 +210,12 @@ module VX_mem_unit import VX_gpu_pkg::*; #( end assign dcr_flush_if.done = (& lane_flush_done); + wire [FLUSH_LANES-1:0] flush_req_valid; + for (genvar k = 0; k < FLUSH_LANES; ++k) begin : g_flush_req_valid + assign flush_req_valid[k] = dcache_bus_if[k].req_valid; + end + assign busy = (| flush_req_valid); + // A block's D-cache request carries a clear tag LSB, the flush arbiter's // select bit; the block takes the cache response directly and drops the // flush token's. diff --git a/hw/rtl/core/VX_wctl_unit.sv b/hw/rtl/core/VX_wctl_unit.sv index 60b4cef82a..49dbabb063 100644 --- a/hw/rtl/core/VX_wctl_unit.sv +++ b/hw/rtl/core/VX_wctl_unit.sv @@ -175,7 +175,8 @@ module VX_wctl_unit import VX_gpu_pkg::*; #( && !warp_ctl_if.warp_pending_alm_empty[wid_to_sfu_local(execute_if.data.header.wid)]; // BAR (vx_barrier or vx_barrier_arrive) drains LSU before continuing. - wire bar_drain = execute_if.valid && is_bar && !warp_ctl_if.lsu_sched_drained; + wire lsu_sched_drained = (& warp_ctl_if.lsu_sched_empty) && warp_ctl_if.mmu_empty; + wire bar_drain = execute_if.valid && is_bar && !lsu_sched_drained; // Barriers, wspawn and warp retirement are core-wide requests to the shared // unit. Only a barrier arrive writes a result, the barrier phase, so it alone diff --git a/hw/rtl/interfaces/VX_warp_ctl_if.sv b/hw/rtl/interfaces/VX_warp_ctl_if.sv index 0e49757335..26d648a659 100644 --- a/hw/rtl/interfaces/VX_warp_ctl_if.sv +++ b/hw/rtl/interfaces/VX_warp_ctl_if.sv @@ -27,12 +27,14 @@ interface VX_warp_ctl_if import VX_gpu_pkg::*; (); // Pipeline drain status of the scheduler's warps, by SFU-block-local warp id wire [SFU_BLOCK_WARPS-1:0] warp_pending_alm_empty; - // LSU memory-scheduler drained signal (AND-reduced across all LSU blocks). - // Used by VX_wctl_unit to gate BAR (both vx_barrier and vx_barrier_arrive) - // so the barrier acts as a SMEM/GMEM fence — pending LSU memory ops finish - // before the warp suspends or the barrier arrival is registered. This is - // the CUDA __syncthreads / OpenCL barrier(CLK_LOCAL_MEM_FENCE) contract. - wire lsu_sched_drained; + // Every LSU block's memory scheduler is empty, and the translation path + // drained. VX_wctl_unit ANDs them to gate BAR (both vx_barrier and + // vx_barrier_arrive) so the barrier acts as a SMEM/GMEM fence — pending + // LSU memory ops finish before the warp suspends or the barrier arrival is + // registered. This is the CUDA __syncthreads / OpenCL + // barrier(CLK_LOCAL_MEM_FENCE) contract. + wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_sched_empty; + wire mmu_empty; // IPDOM stack address and mscratch of the warp at dvstack_wid wire [NW_WIDTH-1:0] dvstack_wid; @@ -49,7 +51,8 @@ interface VX_warp_ctl_if import VX_gpu_pkg::*; (); output split, output sjoin, input warp_pending_alm_empty, - input lsu_sched_drained, + input lsu_sched_empty, + input mmu_empty, output dvstack_wid, input dvstack_ptr, input mscratch @@ -65,7 +68,8 @@ interface VX_warp_ctl_if import VX_gpu_pkg::*; (); input split, input sjoin, output warp_pending_alm_empty, - output lsu_sched_drained, + output lsu_sched_empty, + output mmu_empty, input dvstack_wid, output dvstack_ptr, output mscratch diff --git a/hw/unittest/core/VX_core_top.sv b/hw/unittest/core/VX_core_top.sv index d47bfad744..d5aef6f3d4 100644 --- a/hw/unittest/core/VX_core_top.sv +++ b/hw/unittest/core/VX_core_top.sv @@ -209,6 +209,10 @@ module VX_core_top import VX_gpu_pkg::*; `UNUSED_VAR (scope_bus_out_w) `endif + wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] core_lsu_busy; + wire core_busy; + assign busy = core_busy || (| core_lsu_busy); + VX_core #( .INSTANCE_ID (`SFORMATF(("core"))) ) core ( @@ -245,7 +249,8 @@ module VX_core_top import VX_gpu_pkg::*; `endif .kmu_bus_if (kmu_bus_if), - .busy (busy) + .lsu_busy (core_lsu_busy), + .busy (core_busy) ); endmodule diff --git a/hw/unittest/mem_unit/VX_mem_unit_top.sv b/hw/unittest/mem_unit/VX_mem_unit_top.sv index ed556ebf4c..1f39567731 100644 --- a/hw/unittest/mem_unit/VX_mem_unit_top.sv +++ b/hw/unittest/mem_unit/VX_mem_unit_top.sv @@ -94,6 +94,7 @@ module VX_mem_unit_top import VX_gpu_pkg::*; #( `endif `UNUSED_PIN (sched_empty), `UNUSED_PIN (store_empty), + `UNUSED_PIN (busy), `ifdef VX_CFG_LMEM_ENABLE .lmem_bus_if (lsu_lmem_bus_if[i * `VX_CFG_NUM_LSU_LANES +: `VX_CFG_NUM_LSU_LANES]), `endif @@ -148,7 +149,8 @@ module VX_mem_unit_top import VX_gpu_pkg::*; #( .lsu_dcache_bus_if (lsu_dcache_bus_if), .dcr_flush_if (dcr_flush_if), .flush_l1_only (1'b0), - .dcache_bus_if (mem_bus_if) + .dcache_bus_if (mem_bus_if), + `UNUSED_PIN (busy) ); endmodule diff --git a/hw/unittest/tensor/VX_tensor_top.sv b/hw/unittest/tensor/VX_tensor_top.sv index 05037a0908..232f6475cb 100644 --- a/hw/unittest/tensor/VX_tensor_top.sv +++ b/hw/unittest/tensor/VX_tensor_top.sv @@ -302,6 +302,7 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( `endif .sched_empty (block_sched_empty[block_idx]), .store_empty (block_store_empty[block_idx]), + `UNUSED_PIN (busy), .lmem_bus_if (lsu_lmem_bus_if[block_idx * `VX_CFG_NUM_LSU_LANES +: `VX_CFG_NUM_LSU_LANES]), .dcache_bus_if (lsu_dcache_bus_if[block_idx * DCACHE_CHANNELS +: DCACHE_CHANNELS]) ); @@ -341,7 +342,8 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( .lsu_dcache_bus_if (lsu_dcache_bus_if), .dcr_flush_if (dcr_flush_dcache_if), .flush_l1_only (1'b0), - .dcache_bus_if (dcache_bus_if) + .dcache_bus_if (dcache_bus_if), + `UNUSED_PIN (busy) ); for (genvar i = 0; i < DCACHE_NUM_REQS; ++i) begin : g_dcache_passthru From 8410590f44445da0645bee427d7774f35f382b11 Mon Sep 17 00:00:00 2001 From: tinebp Date: Thu, 22 Oct 2026 02:15:50 -0700 Subject: [PATCH 086/124] core: keep each SFU block's CTA context RAM in the block VX_scheduler held one CTA context RAM per SFU block, each read only by that block's CSR unit, so every read address and its data crossed the core. Each RAM now sits in its VX_sfu_block, written by the same broadcast from CTA dispatch, with the same size and registered read. SATP stays in the scheduler: it is one core-wide live CSR whose writes are ordered across SFU blocks. rtlsim sgemm_tcu_wg_sp_dxa -m 128 -n 128 -k 128: cycle-identical to the parent (144,956), PASSED. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/core/VX_backend_slice.sv | 8 +++++--- hw/rtl/core/VX_compute_slice.sv | 18 ++++-------------- hw/rtl/core/VX_core.sv | 19 +++++++------------ hw/rtl/core/VX_csr_data.sv | 24 +++++++++++++----------- hw/rtl/core/VX_csr_unit.sv | 2 ++ hw/rtl/core/VX_execute.sv | 6 ++++-- hw/rtl/core/VX_execute_slice.sv | 6 ++++-- hw/rtl/core/VX_frontend_xbar.sv | 12 +----------- hw/rtl/core/VX_scheduler.sv | 26 ++------------------------ hw/rtl/core/VX_sfu_block.sv | 27 ++++++++++++++++++++++++++- hw/rtl/core/VX_sfu_unit.sv | 7 +++++-- hw/rtl/interfaces/VX_sched_csr_if.sv | 14 +++++--------- 12 files changed, 78 insertions(+), 91 deletions(-) diff --git a/hw/rtl/core/VX_backend_slice.sv b/hw/rtl/core/VX_backend_slice.sv index 24e380cae0..385db201a5 100644 --- a/hw/rtl/core/VX_backend_slice.sv +++ b/hw/rtl/core/VX_backend_slice.sv @@ -106,8 +106,9 @@ import VX_tcu_pkg::*; input wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] tcu_meta_wr_data, `endif - // core-wide warp control - VX_warp_shared_if.master warp_shared_if + // core-wide warp control and CTA contexts + VX_warp_shared_if.master warp_shared_if, + input cta_bcast_t cta_bcast ); `STATIC_ASSERT (BACKEND_SLICES, ("invalid parameter")) @@ -204,7 +205,8 @@ import VX_tcu_pkg::*; `endif .warp_shared_if (warp_shared_if), .warp_ctl_if (warp_ctl_if), - .sched_csr_if (sched_csr_if) + .sched_csr_if (sched_csr_if), + .cta_bcast (cta_bcast) ); // commit diff --git a/hw/rtl/core/VX_compute_slice.sv b/hw/rtl/core/VX_compute_slice.sv index ddf14588e9..f241dd9d96 100644 --- a/hw/rtl/core/VX_compute_slice.sv +++ b/hw/rtl/core/VX_compute_slice.sv @@ -51,8 +51,7 @@ import VX_tcu_pkg::*; input wire [`VX_CFG_NUM_WARPS-1:0] active_warps, input wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] all_lsu_sched_empty, input wire mmu_empty, - output wire [NCTA_WIDTH-1:0] csr_rd_cta_id, - input cta_ctx_t csr_rd_ctx, + input cta_bcast_t cta_bcast, `ifdef VX_CFG_VM_ENABLE input wire [`VX_CFG_XLEN-1:0] satp, output wire satp_wr_valid, @@ -207,7 +206,6 @@ import VX_tcu_pkg::*; assign warp_ctl_if.mmu_empty = mmu_empty; assign csr_rd_wid[0] = sched_csr_if.csr_rd_wid; - assign csr_rd_cta_id = sched_csr_if.csr_rd_cta_id; assign csr_wr[0].valid = sched_csr_if.csr_wr_valid; assign csr_wr[0].wid = sched_csr_if.csr_wr_wid; assign csr_wr[0].data = sched_csr_if.csr_wr_data; @@ -224,16 +222,7 @@ import VX_tcu_pkg::*; assign sched_csr_if.csr_mepc = csr_rd[0].mepc; assign sched_csr_if.csr_mcause = csr_rd[0].mcause; assign sched_csr_if.csr_mtval = csr_rd[0].mtval; - assign sched_csr_if.cta_csrs.cta_id = sched_csr_if.csr_rd_cta_id; - assign sched_csr_if.cta_csrs.cta_rank = csr_rd[0].cta_rank; - assign sched_csr_if.cta_csrs.cta_size = csr_rd_ctx.cta_size; - assign sched_csr_if.cta_csrs.block_idx = csr_rd_ctx.block_idx; - assign sched_csr_if.cta_csrs.block_dim = csr_rd_ctx.block_dim; - assign sched_csr_if.cta_csrs.grid_dim = csr_rd_ctx.grid_dim; - assign sched_csr_if.cta_csrs.entry = csr_rd_ctx.entry; - assign sched_csr_if.cta_csrs.param = csr_rd_ctx.param; - assign sched_csr_if.cta_csrs.lmem_addr = csr_rd_ctx.lmem_addr; - assign sched_csr_if.cta_csrs.cluster_size = csr_rd_ctx.cluster_size; + assign sched_csr_if.cta_rank = csr_rd[0].cta_rank; assign sched_csr_if.cta_lane = csr_rd[0].cta_lane; `ifdef VX_CFG_VM_ENABLE assign sched_csr_if.csr_satp = satp; @@ -313,7 +302,8 @@ import VX_tcu_pkg::*; .tcu_meta_wr_idx (tcu_meta_wr_idx), .tcu_meta_wr_data (tcu_meta_wr_data), `endif - .warp_shared_if (warp_shared_if) + .warp_shared_if (warp_shared_if), + .cta_bcast (cta_bcast) ); endmodule diff --git a/hw/rtl/core/VX_core.sv b/hw/rtl/core/VX_core.sv index 7ef6230dfa..eedcb21820 100644 --- a/hw/rtl/core/VX_core.sv +++ b/hw/rtl/core/VX_core.sv @@ -266,8 +266,7 @@ import VX_tcu_pkg::*; wire [`VX_CFG_NUM_SFU_BLOCKS-1:0][`VX_CFG_XLEN-1:0] satp_wr_data; `endif - wire [`VX_CFG_NUM_SFU_BLOCKS-1:0][NCTA_WIDTH-1:0] csr_rd_cta_id; - cta_ctx_t [`VX_CFG_NUM_SFU_BLOCKS-1:0] csr_rd_ctx; + cta_bcast_t cta_bcast; VX_scheduler #( .INSTANCE_ID (`SFORMATF(("%s-scheduler", INSTANCE_ID))) @@ -284,8 +283,7 @@ import VX_tcu_pkg::*; .txbar_bus_if (dxa_txbar_bus_if), `endif .committed_warps(committed_warps), - .csr_rd_cta_id (csr_rd_cta_id), - .csr_rd_ctx (csr_rd_ctx), + .cta_bcast (cta_bcast), .kmu_bus_if (kmu_bus_if), .gbar_bus_if (gbar_bus_if), .cycles (sched_cycles), @@ -452,8 +450,7 @@ import VX_tcu_pkg::*; .active_warps (sched_active_warps), .all_lsu_sched_empty (lsu_sched_empty), .mmu_empty (mmu_empty), - .csr_rd_cta_id (csr_rd_cta_id[i]), - .csr_rd_ctx (csr_rd_ctx[i]), + .cta_bcast (cta_bcast), `ifdef VX_CFG_VM_ENABLE .satp (sched_satp[i]), .satp_wr_valid (satp_wr_valid[i]), @@ -520,9 +517,6 @@ import VX_tcu_pkg::*; VX_branch_ctl_if branch_ctl_if[`VX_CFG_NUM_ALU_BLOCKS](); VX_warp_ctl_if warp_ctl_if[`VX_CFG_NUM_SFU_BLOCKS](); VX_sched_csr_if sched_csr_if[`VX_CFG_NUM_SFU_BLOCKS](); - for (genvar b = 0; b < `VX_CFG_NUM_SFU_BLOCKS; ++b) begin : g_csr_rd_cta_id - assign csr_rd_cta_id[b] = sched_csr_if[b].csr_rd_cta_id; - end `ifdef VX_CFG_EXT_RTU_ENABLE VX_sched_unlock_if sched_unlock_if[`VX_CFG_NUM_SFU_BLOCKS](); `endif @@ -568,7 +562,6 @@ import VX_tcu_pkg::*; .active_warps (sched_active_warps), .lsu_sched_empty (lsu_sched_empty), .mmu_empty (mmu_empty), - .csr_rd_ctx (csr_rd_ctx), `ifdef VX_CFG_VM_ENABLE .satp (sched_satp), .satp_wr_valid (satp_wr_valid), @@ -742,7 +735,8 @@ import VX_tcu_pkg::*; .tcu_meta_wr_idx (tcu_meta_wr_idx), .tcu_meta_wr_data (tcu_meta_wr_data), `endif - .warp_shared_if (warp_shared_if[i]) + .warp_shared_if (warp_shared_if[i]), + .cta_bcast (cta_bcast) ); // wid = {wis, isw}: slice i owns every ISSUE_WIDTH-th warp starting at i @@ -848,7 +842,8 @@ import VX_tcu_pkg::*; .warp_shared_if (warp_shared_if), .warp_ctl_if (warp_ctl_if), - .branch_ctl_if (branch_ctl_if) + .branch_ctl_if (branch_ctl_if), + .cta_bcast (cta_bcast) ); VX_commit #( diff --git a/hw/rtl/core/VX_csr_data.sv b/hw/rtl/core/VX_csr_data.sv index e7a8799dc6..6a2300fe1a 100644 --- a/hw/rtl/core/VX_csr_data.sv +++ b/hw/rtl/core/VX_csr_data.sv @@ -39,6 +39,9 @@ import VX_fpu_pkg::*; VX_sched_csr_if.slave sched_csr_if, + // context of the request's CTA, read by req_cta_id + input cta_ctx_t cta_ctx, + // The request is held for at least one cycle before it fires; that cycle // decodes it, and the fire cycle only merges the values that can still move. input wire req_fire, @@ -54,7 +57,7 @@ import VX_fpu_pkg::*; `UNUSED_SPARAM (INSTANCE_ID) `UNUSED_VAR (reset) `UNUSED_VAR (req_uuid) - wire [`VX_CFG_MEM_ADDR_WIDTH-1:0] __cta_param = sched_csr_if.cta_csrs.param; + wire [`VX_CFG_MEM_ADDR_WIDTH-1:0] __cta_param = cta_ctx.param; `UNUSED_VAR (__cta_param) // Values that can change between the decode cycle and the fire cycle are @@ -104,7 +107,6 @@ import VX_fpu_pkg::*; // Scheduler CSRs read interface assign sched_csr_if.csr_rd_wid = req_wid; - assign sched_csr_if.csr_rd_cta_id = req_cta_id; reg [`VX_CFG_XLEN-1:0] stable_w; reg [NUM_LIVE-1:0] live_sel_w; @@ -126,7 +128,7 @@ import VX_fpu_pkg::*; `endif `VX_CSR_MSCRATCH : begin stable_w = `VX_CFG_XLEN'(sched_csr_if.mscratch); wr_sel_w[WR_MSCRATCH] = 1; end - `VX_CSR_CTA_ID : stable_w = `VX_CFG_XLEN'(sched_csr_if.cta_csrs.cta_id); + `VX_CSR_CTA_ID : stable_w = `VX_CFG_XLEN'(req_cta_id); `VX_CSR_CTA_RANK : live_sel_w[LIVE_CTA_RANK] = 1; `VX_CSR_CTA_SIZE : live_sel_w[LIVE_CTA_SIZE] = 1; `VX_CSR_CTA_BLOCK_ID_X : live_sel_w[LIVE_BLOCK_ID + 0] = 1; @@ -239,16 +241,16 @@ import VX_fpu_pkg::*; `endif // The CTA context RAMs are addressed by the held request and return in the // fire cycle. - assign live_src[LIVE_CTA_RANK] = `VX_CFG_XLEN'(sched_csr_if.cta_csrs.cta_rank); - assign live_src[LIVE_CTA_SIZE] = `VX_CFG_XLEN'(sched_csr_if.cta_csrs.cta_size); + assign live_src[LIVE_CTA_RANK] = `VX_CFG_XLEN'(sched_csr_if.cta_rank); + assign live_src[LIVE_CTA_SIZE] = `VX_CFG_XLEN'(cta_ctx.cta_size); for (genvar i = 0; i < 3; ++i) begin : g_cta_dims - assign live_src[LIVE_BLOCK_ID + i] = `VX_CFG_XLEN'(sched_csr_if.cta_csrs.block_idx[i]); - assign live_src[LIVE_BLOCK_DIM + i] = `VX_CFG_XLEN'(sched_csr_if.cta_csrs.block_dim[i]); - assign live_src[LIVE_GRID_DIM + i] = `VX_CFG_XLEN'(sched_csr_if.cta_csrs.grid_dim[i]); + assign live_src[LIVE_BLOCK_ID + i] = `VX_CFG_XLEN'(cta_ctx.block_idx[i]); + assign live_src[LIVE_BLOCK_DIM + i] = `VX_CFG_XLEN'(cta_ctx.block_dim[i]); + assign live_src[LIVE_GRID_DIM + i] = `VX_CFG_XLEN'(cta_ctx.grid_dim[i]); end - assign live_src[LIVE_LMEM_ADDR] = `VX_CFG_XLEN'(sched_csr_if.cta_csrs.lmem_addr); - assign live_src[LIVE_CLUSTER] = `VX_CFG_XLEN'(sched_csr_if.cta_csrs.cluster_size); - assign live_src[LIVE_ENTRY] = `VX_CFG_XLEN'(to_fullPC(sched_csr_if.cta_csrs.entry)); + assign live_src[LIVE_LMEM_ADDR] = `VX_CFG_XLEN'(cta_ctx.lmem_addr); + assign live_src[LIVE_CLUSTER] = `VX_CFG_XLEN'(cta_ctx.cluster_size); + assign live_src[LIVE_ENTRY] = `VX_CFG_XLEN'(to_fullPC(cta_ctx.entry)); `ifdef VX_CFG_VM_ENABLE assign live_src[LIVE_SATP] = sched_csr_if.csr_satp; `else diff --git a/hw/rtl/core/VX_csr_unit.sv b/hw/rtl/core/VX_csr_unit.sv index d4f501574b..0a4e859da9 100644 --- a/hw/rtl/core/VX_csr_unit.sv +++ b/hw/rtl/core/VX_csr_unit.sv @@ -34,6 +34,7 @@ module VX_csr_unit import VX_gpu_pkg::*; #( `endif VX_sched_csr_if.slave sched_csr_if, + input cta_ctx_t cta_ctx, VX_execute_if.slave execute_if, VX_result_if.master result_if ); @@ -87,6 +88,7 @@ module VX_csr_unit import VX_gpu_pkg::*; #( .core_id (core_id), .sched_csr_if (sched_csr_if), + .cta_ctx (cta_ctx), `ifdef VX_CFG_EXT_F_ENABLE .fcsr_rd_data (fcsr_rd_data), diff --git a/hw/rtl/core/VX_execute.sv b/hw/rtl/core/VX_execute.sv index f4fd71f0c3..84ebe921fb 100644 --- a/hw/rtl/core/VX_execute.sv +++ b/hw/rtl/core/VX_execute.sv @@ -97,7 +97,8 @@ import VX_tcu_pkg::*; VX_sched_csr_if.slave sched_csr_if [`VX_CFG_NUM_SFU_BLOCKS], VX_branch_ctl_if.master branch_ctl_if [`VX_CFG_NUM_ALU_BLOCKS], VX_warp_shared_if.master warp_shared_if [`VX_CFG_NUM_SFU_BLOCKS], - VX_warp_ctl_if.master warp_ctl_if [`VX_CFG_NUM_SFU_BLOCKS] + VX_warp_ctl_if.master warp_ctl_if [`VX_CFG_NUM_SFU_BLOCKS], + input cta_bcast_t cta_bcast ); `ifdef VX_CFG_EXT_F_ENABLE @@ -213,7 +214,8 @@ import VX_tcu_pkg::*; `endif .sched_csr_if (sched_csr_if), .warp_shared_if (warp_shared_if), - .warp_ctl_if (warp_ctl_if) + .warp_ctl_if (warp_ctl_if), + .cta_bcast (cta_bcast) ); endmodule diff --git a/hw/rtl/core/VX_execute_slice.sv b/hw/rtl/core/VX_execute_slice.sv index 9e9bfb34eb..7436dc3573 100644 --- a/hw/rtl/core/VX_execute_slice.sv +++ b/hw/rtl/core/VX_execute_slice.sv @@ -99,7 +99,8 @@ import VX_tcu_pkg::*; // SFU warp control and CSRs VX_warp_shared_if.master warp_shared_if, VX_warp_ctl_if.master warp_ctl_if, - VX_sched_csr_if.slave sched_csr_if + VX_sched_csr_if.slave sched_csr_if, + input cta_bcast_t cta_bcast ); localparam ALU_PARTIAL_BW = (`VX_CFG_NUM_ALU_LANES != `VX_CFG_SIMD_WIDTH); `ifdef VX_CFG_EXT_F_ENABLE @@ -279,7 +280,8 @@ import VX_tcu_pkg::*; `endif .sched_csr_if (sched_csr_if), .warp_shared_if (warp_shared_if), - .warp_ctl_if (warp_ctl_if) + .warp_ctl_if (warp_ctl_if), + .cta_bcast (cta_bcast) ); VX_lane_gather #( diff --git a/hw/rtl/core/VX_frontend_xbar.sv b/hw/rtl/core/VX_frontend_xbar.sv index 252d4df822..0426b3255c 100644 --- a/hw/rtl/core/VX_frontend_xbar.sv +++ b/hw/rtl/core/VX_frontend_xbar.sv @@ -45,7 +45,6 @@ module VX_frontend_xbar import VX_gpu_pkg::*; ( input wire [`VX_CFG_NUM_WARPS-1:0] active_warps, input wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_sched_empty, input wire mmu_empty, - input cta_ctx_t [`VX_CFG_NUM_SFU_BLOCKS-1:0] csr_rd_ctx, `ifdef VX_CFG_VM_ENABLE input wire [`VX_CFG_NUM_SFU_BLOCKS-1:0][`VX_CFG_XLEN-1:0] satp, output wire [`VX_CFG_NUM_SFU_BLOCKS-1:0] satp_wr_valid, @@ -210,16 +209,7 @@ module VX_frontend_xbar import VX_gpu_pkg::*; ( assign sched_csr_if[b].csr_mepc = csr_rd.mepc; assign sched_csr_if[b].csr_mcause = csr_rd.mcause; assign sched_csr_if[b].csr_mtval = csr_rd.mtval; - assign sched_csr_if[b].cta_csrs.cta_id = sched_csr_if[b].csr_rd_cta_id; - assign sched_csr_if[b].cta_csrs.cta_rank = csr_rd.cta_rank; - assign sched_csr_if[b].cta_csrs.cta_size = csr_rd_ctx[b].cta_size; - assign sched_csr_if[b].cta_csrs.block_idx = csr_rd_ctx[b].block_idx; - assign sched_csr_if[b].cta_csrs.block_dim = csr_rd_ctx[b].block_dim; - assign sched_csr_if[b].cta_csrs.grid_dim = csr_rd_ctx[b].grid_dim; - assign sched_csr_if[b].cta_csrs.entry = csr_rd_ctx[b].entry; - assign sched_csr_if[b].cta_csrs.param = csr_rd_ctx[b].param; - assign sched_csr_if[b].cta_csrs.lmem_addr = csr_rd_ctx[b].lmem_addr; - assign sched_csr_if[b].cta_csrs.cluster_size = csr_rd_ctx[b].cluster_size; + assign sched_csr_if[b].cta_rank = csr_rd.cta_rank; assign sched_csr_if[b].cta_lane = csr_rd.cta_lane; `ifdef VX_CFG_VM_ENABLE assign sched_csr_if[b].csr_satp = satp[b]; diff --git a/hw/rtl/core/VX_scheduler.sv b/hw/rtl/core/VX_scheduler.sv index 246274c7ca..efcc981018 100644 --- a/hw/rtl/core/VX_scheduler.sv +++ b/hw/rtl/core/VX_scheduler.sv @@ -35,9 +35,8 @@ module VX_scheduler import VX_gpu_pkg::*; #( `endif input wire [`VX_CFG_NUM_WARPS-1:0] committed_warps, - // CTA context read by each SFU block's CSR unit, valid a cycle after the address - input wire [`VX_CFG_NUM_SFU_BLOCKS-1:0][NCTA_WIDTH-1:0] csr_rd_cta_id, - output cta_ctx_t [`VX_CFG_NUM_SFU_BLOCKS-1:0] csr_rd_ctx, + // CTA context writes, to every SFU block's copy + output cta_bcast_t cta_bcast, VX_kmu_bus_if.slave kmu_bus_if, @@ -77,7 +76,6 @@ module VX_scheduler import VX_gpu_pkg::*; #( wire [FW-1:0][`VX_CFG_NUM_THREADS-1:0] cta_tmask; wire [`VX_CFG_MEM_ADDR_WIDTH-1:0] cta_param; wire [FW-1:0] cta_init; - cta_bcast_t cta_bcast; cta_lane_bcast_t [FW-1:0] cta_lane_bcast; wire cta_dispatcher_busy; wire cta_warp_done; @@ -104,26 +102,6 @@ module VX_scheduler import VX_gpu_pkg::*; #( .busy (cta_dispatcher_busy) ); - // CTA contexts, one read port per SFU block - for (genvar b = 0; b < `VX_CFG_NUM_SFU_BLOCKS; ++b) begin : g_cta_ctx - VX_dp_ram #( - .DATAW ($bits(cta_ctx_t)), - .SIZE (NUM_CTA_MAX), - .RDW_MODE ("R"), - .OUT_REG (1) - ) cta_ctx_ram ( - .clk (clk), - .reset (reset), - .read (1'b1), - .write (cta_bcast.ctx_write), - .wren (1'b1), - .waddr (cta_bcast.ctx_waddr), - .wdata (cta_bcast.ctx_wdata), - .raddr (csr_rd_cta_id[b]), - .rdata (csr_rd_ctx[b]) - ); - end - // core-wide warp-control resources wire bar_unlock_valid; wire [`VX_CFG_NUM_WARPS-1:0] bar_unlock_mask; diff --git a/hw/rtl/core/VX_sfu_block.sv b/hw/rtl/core/VX_sfu_block.sv index 15ce7ad91d..92d4fed066 100644 --- a/hw/rtl/core/VX_sfu_block.sv +++ b/hw/rtl/core/VX_sfu_block.sv @@ -52,7 +52,10 @@ module VX_sfu_block import VX_gpu_pkg::*; #( VX_sched_csr_if.slave sched_csr_if, VX_warp_shared_if.master warp_shared_if, - VX_warp_ctl_if.master warp_ctl_if + VX_warp_ctl_if.master warp_ctl_if, + + // CTA dispatch's context writes + input cta_bcast_t cta_bcast ); localparam NUM_LANES = `VX_CFG_NUM_SFU_LANES; `ifndef VX_CFG_EXT_RTU_ENABLE @@ -147,6 +150,27 @@ module VX_sfu_block import VX_gpu_pkg::*; #( .result_if (pe_result_if[PE_IDX_WCTL]) ); + // The block's copy of the CTA contexts, read at the CSR request's CTA and + // valid a cycle later. + cta_ctx_t cta_ctx; + + VX_dp_ram #( + .DATAW ($bits(cta_ctx_t)), + .SIZE (NUM_CTA_MAX), + .RDW_MODE ("R"), + .OUT_REG (1) + ) cta_ctx_ram ( + .clk (clk), + .reset (reset), + .read (1'b1), + .write (cta_bcast.ctx_write), + .wren (1'b1), + .waddr (cta_bcast.ctx_waddr), + .wdata (cta_bcast.ctx_wdata), + .raddr (pe_execute_if[PE_IDX_CSRS].data.header.cta_id), + .rdata (cta_ctx) + ); + VX_csr_unit #( .INSTANCE_ID (`SFORMATF(("%s-csr", INSTANCE_ID))), .NUM_LANES (NUM_LANES) @@ -162,6 +186,7 @@ module VX_sfu_block import VX_gpu_pkg::*; #( .fcsr_wr_data (fcsr_wr_data), `endif .sched_csr_if (sched_csr_if), + .cta_ctx (cta_ctx), .result_if (pe_result_if[PE_IDX_CSRS]) ); diff --git a/hw/rtl/core/VX_sfu_unit.sv b/hw/rtl/core/VX_sfu_unit.sv index a7f14fe32d..59ba4c8dc3 100644 --- a/hw/rtl/core/VX_sfu_unit.sv +++ b/hw/rtl/core/VX_sfu_unit.sv @@ -50,7 +50,9 @@ module VX_sfu_unit import VX_gpu_pkg::*; #( // Outputs VX_commit_if.master commit_if [`VX_CFG_ISSUE_WIDTH], VX_warp_shared_if.master warp_shared_if [`VX_CFG_NUM_SFU_BLOCKS], - VX_warp_ctl_if.master warp_ctl_if [`VX_CFG_NUM_SFU_BLOCKS] + VX_warp_ctl_if.master warp_ctl_if [`VX_CFG_NUM_SFU_BLOCKS], + + input cta_bcast_t cta_bcast ); localparam BLOCK_SIZE = `VX_CFG_NUM_SFU_BLOCKS; localparam NUM_LANES = `VX_CFG_NUM_SFU_LANES; @@ -110,7 +112,8 @@ module VX_sfu_unit import VX_gpu_pkg::*; #( `endif .sched_csr_if (sched_csr_if[block_idx]), .warp_shared_if (warp_shared_if[block_idx]), - .warp_ctl_if (warp_ctl_if[block_idx]) + .warp_ctl_if (warp_ctl_if[block_idx]), + .cta_bcast (cta_bcast) ); end diff --git a/hw/rtl/interfaces/VX_sched_csr_if.sv b/hw/rtl/interfaces/VX_sched_csr_if.sv index d773df084d..cc88a4d0e2 100644 --- a/hw/rtl/interfaces/VX_sched_csr_if.sv +++ b/hw/rtl/interfaces/VX_sched_csr_if.sv @@ -20,13 +20,11 @@ interface VX_sched_csr_if import VX_gpu_pkg::*; (); wire [`VX_CFG_NUM_WARPS-1:0] active_warps; wire [`VX_CFG_NUM_THREADS-1:0] thread_mask; // selected by csr_rd_wid - // Read port: slave sends wid + cta_id, master returns selected mscratch - // and cta_csrs. csr_rd_cta_id must carry the executing instruction's - // cta_id so the scheduler can index its per-CTA table directly. + // Read port: slave sends wid, master returns the selected warp's mscratch + // and CTA rank. logic [NW_WIDTH-1:0] csr_rd_wid; - logic [NCTA_WIDTH-1:0] csr_rd_cta_id; logic [`VX_CFG_MEM_ADDR_WIDTH-1:0] mscratch; - cta_csrs_t cta_csrs; + logic [NW_WIDTH-1:0] cta_rank; // Per-lane launch record; the CSR unit reads the view (thread index or fragment // stamp) the accessed CSR needs. cta_lane_t [`VX_CFG_NUM_THREADS-1:0] cta_lane; @@ -61,7 +59,7 @@ interface VX_sched_csr_if import VX_gpu_pkg::*; (); output active_warps, output thread_mask, output mscratch, - output cta_csrs, + output cta_rank, output cta_lane, output csr_mstatus, output csr_mtvec, @@ -74,7 +72,6 @@ interface VX_sched_csr_if import VX_gpu_pkg::*; (); input satp_wr_data, `endif input csr_rd_wid, - input csr_rd_cta_id, input csr_wr_valid, input csr_wr_wid, input csr_wr_data, @@ -88,7 +85,7 @@ interface VX_sched_csr_if import VX_gpu_pkg::*; (); input active_warps, input thread_mask, input mscratch, - input cta_csrs, + input cta_rank, input cta_lane, input csr_mstatus, input csr_mtvec, @@ -101,7 +98,6 @@ interface VX_sched_csr_if import VX_gpu_pkg::*; (); output satp_wr_data, `endif output csr_rd_wid, - output csr_rd_cta_id, output csr_wr_valid, output csr_wr_wid, output csr_wr_data, From 47844854f4cd405b0098e652d0ef942a5e62399d Mon Sep 17 00:00:00 2001 From: tinebp Date: Thu, 22 Oct 2026 02:47:51 -0700 Subject: [PATCH 087/124] tcu: give each TCU block its own metadata port into its slice's LSU scheduler When the LSU has one block per TCU block, each TCU block now issues its TCU_LD metadata loads through the scheduler of the LSU block that serves the same warps, instead of all blocks sharing LSU block 0's scheduler. This keeps each slice's memory traffic inside the slice. With a different block count the AGU keeps one shared port on LSU block 0. The AGU keeps its per-block holding registers and shared commit arbiter. Metadata write-after-read safety still comes from TCU_LD being dispatched in order behind the warp's MMA micro-ops in the same TCU block. SimX models the same ports (TCU_AGU_PORTS, same derivation as the RTL). sgemm_tcu_wg_sp_dxa 128x128x128 (IW4, FW4, 4 LSU/4 TCU blocks, 32 threads): rtlsim 144,956 cycles, PASSED, unchanged from the shared port. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/core/VX_core.sv | 13 +- hw/rtl/tcu/VX_tcu_agu.sv | 309 ++++++++++++++++++++---------------- hw/rtl/tcu/VX_tcu_shared.sv | 32 +++- sim/simx/core.cpp | 8 +- sim/simx/lsu_unit.cpp | 26 +-- sim/simx/lsu_unit.h | 18 ++- sim/simx/tcu/tcu_unit.cpp | 116 ++++++++------ sim/simx/tcu/tcu_unit.h | 17 +- 8 files changed, 313 insertions(+), 226 deletions(-) diff --git a/hw/rtl/core/VX_core.sv b/hw/rtl/core/VX_core.sv index eedcb21820..a5b31c5e58 100644 --- a/hw/rtl/core/VX_core.sv +++ b/hw/rtl/core/VX_core.sv @@ -102,17 +102,8 @@ import VX_tcu_pkg::*; `endif `ifdef TCU_META_ENABLE - // The TCU metadata AGU is the second client of LSU block 0's scheduler; the - // other blocks' second client is idle. + // The TCU metadata AGU is the second client of the LSU schedulers. VX_lsu_sched_if lsu_tcu_mem_if [`VX_CFG_NUM_LSU_BLOCKS](); - for (genvar b = 1; b < `VX_CFG_NUM_LSU_BLOCKS; ++b) begin : g_lsu_tcu_tieoff - assign lsu_tcu_mem_if[b].req_valid = 1'b0; - assign lsu_tcu_mem_if[b].req_data = '0; - assign lsu_tcu_mem_if[b].rsp_ready = 1'b1; - `UNUSED_VAR (lsu_tcu_mem_if[b].req_ready) - `UNUSED_VAR (lsu_tcu_mem_if[b].rsp_valid) - `UNUSED_VAR (lsu_tcu_mem_if[b].rsp_data) - end `endif // frontend-slice icache requests; port 0 also carries the injected flush @@ -384,7 +375,7 @@ import VX_tcu_pkg::*; .meta_wr_wid (tcu_meta_wr_wid), .meta_wr_idx (tcu_meta_wr_idx), .meta_wr_data (tcu_meta_wr_data), - .tcu_mem_if (lsu_tcu_mem_if[0]), + .tcu_mem_if (lsu_tcu_mem_if), `endif .clk (clk), .reset (reset) diff --git a/hw/rtl/tcu/VX_tcu_agu.sv b/hw/rtl/tcu/VX_tcu_agu.sv index 6690cf17b1..70a77b8126 100644 --- a/hw/rtl/tcu/VX_tcu_agu.sv +++ b/hw/rtl/tcu/VX_tcu_agu.sv @@ -16,12 +16,16 @@ `ifdef TCU_META_ENABLE // Warp-level AGU for TCU_LD instructions. Each TCU block can hold one -// transaction, while the LSU request and metadata write ports are shared. +// transaction. With one memory port the blocks share it; with one port per +// block (NUM_PORTS == BLOCK_SIZE) each block loads through its own LSU +// scheduler, so the loads of a warpgroup's warps proceed in parallel. The +// metadata write port is shared. module VX_tcu_agu import VX_gpu_pkg::*, VX_tcu_pkg::*; #( parameter `STRING INSTANCE_ID = "", parameter BLOCK_SIZE = `VX_CFG_NUM_TCU_BLOCKS, - parameter NUM_LANES = `VX_CFG_NUM_TCU_LANES + parameter NUM_LANES = `VX_CFG_NUM_TCU_LANES, + parameter NUM_PORTS = 1 ) ( input wire clk, input wire reset, @@ -33,8 +37,8 @@ module VX_tcu_agu import VX_gpu_pkg::*, VX_tcu_pkg::*; #( input tcu_execute_t per_block_ld_data [BLOCK_SIZE], output wire [BLOCK_SIZE-1:0] per_block_ld_ready, - // Memory client connection (to VX_lsu_scheduler at VX_core). - VX_lsu_sched_if.master client_if, + // Memory client connections (to the LSU schedulers). + VX_lsu_sched_if.master client_if [NUM_PORTS], // Meta SRAM write port (broadcast — same data lands in every // block's VX_tcu_meta so subsequent wmma_sp reads see it @@ -51,6 +55,7 @@ module VX_tcu_agu import VX_gpu_pkg::*, VX_tcu_pkg::*; #( input wire [BLOCK_SIZE-1:0] result_ready ); `UNUSED_SPARAM (INSTANCE_ID) + `STATIC_ASSERT ((NUM_PORTS == 1) || (NUM_PORTS == BLOCK_SIZE), ("invalid parameter: NUM_PORTS=%0d", NUM_PORTS)) localparam BLOCK_IDX_BITS = (BLOCK_SIZE > 1) ? $clog2(BLOCK_SIZE) : 1; logic [BLOCK_SIZE-1:0] busy_r; @@ -64,22 +69,6 @@ module VX_tcu_agu import VX_gpu_pkg::*, VX_tcu_pkg::*; #( logic [NUM_LANES-1:0] rsp_lane_done_r [BLOCK_SIZE]; wire [BLOCK_SIZE-1:0] req_pending = busy_r & ~issued_r; - wire [BLOCK_IDX_BITS-1:0] req_block; - wire [BLOCK_SIZE-1:0] req_grant; - wire req_valid; - - VX_generic_arbiter #( - .NUM_REQS (BLOCK_SIZE), - .TYPE ("R") - ) req_arb ( - .clk (clk), - .reset (reset), - .requests (req_pending), - .grant_index (req_block), - .grant_onehot (req_grant), - .grant_valid (req_valid), - .grant_ready (client_if.req_ready) - ); wire [BLOCK_SIZE-1:0] commit_pending = complete_r; wire [BLOCK_IDX_BITS-1:0] commit_block; @@ -99,15 +88,148 @@ module VX_tcu_agu import VX_gpu_pkg::*, VX_tcu_pkg::*; #( .grant_valid (commit_valid), .grant_ready (commit_ready) ); - - wire [BLOCK_IDX_BITS-1:0] rsp_block = - client_if.rsp_data.tag[BLOCK_IDX_BITS-1:0]; - wire rsp_fire = client_if.rsp_valid && client_if.rsp_ready; - wire req_fire = req_valid && client_if.req_ready; wire commit_fire = commit_valid && commit_ready; - `UNUSED_VAR (req_grant) `UNUSED_VAR (commit_grant) + // ----------------------------------------------------------------------- + // Memory request: address from rs1 (warp-broadcast). NUM_LANES mask + // all-ones — we fetch the metadata tile in one warp-level request. + // Lane T loads the 32-bit word at base + 4*T; the host packs h_meta so + // that word holds the metadata destined for SRAM cell + // (bank = T%PWD, col = T/PWD). One formula serves WMMA-SP and WGMMA-SP. + // ----------------------------------------------------------------------- + localparam PWD = TCU_META_PER_WARP_DEPTH; + `UNUSED_PARAM (PWD) + + // LSU bus carries LSU_WORD_SIZE-byte chunks per lane (= XLEN/8 bytes). + // The metadata layout is contiguous 32-bit words, so when XLEN > 32 + // each LSU response carries XLEN/32 logical 32-bit words and we need + // to pick the right one. LSU_W32_RATIO = bytes/4, so XLEN=64 → 2. + localparam LSU_W32_RATIO = LSU_WORD_SIZE / 4; + localparam LG_W32_RATIO = (LSU_W32_RATIO > 1) ? $clog2(LSU_W32_RATIO) : 0; + + // Per-lane sub-word selector: lane i reads 32-bit word i, so its half of + // the LSU word is fixed by i. + logic [NUM_LANES-1:0][`UP(LG_W32_RATIO)-1:0] half_sel_w; + always_comb begin + half_sel_w = '0; + for (int i = 0; i < NUM_LANES; i++) begin + if (LG_W32_RATIO > 0) begin + half_sel_w[i] = `UP(LG_W32_RATIO)'(i); + end + end + end + if (LG_W32_RATIO == 0) begin : g_half_sel_unused + `UNUSED_VAR (half_sel_w) + end + +`ifdef VX_CFG_LMEM_ENABLE + // Equality on the word-address bits above the window: the base is asserted + // aligned to the capacity, so the prefix compare is exact and needs no end + // address -- which is not representable when the window ends at the top of + // the address space. + localparam LMEM_WORD_LSB = `VX_CFG_LMEM_LOG_SIZE - `CLOG2(LSU_WORD_SIZE); + localparam [LSU_ADDR_WIDTH-1:0] LMEM_WORD_BASE = + LSU_ADDR_WIDTH'(`VX_CFG_XLEN'(`VX_MEM_LMEM_BASE_ADDR) >> `CLOG2(LSU_WORD_SIZE)); + `STATIC_ASSERT (LSU_ADDR_WIDTH > LMEM_WORD_LSB, ("LMEM window wider than the word address space")) +`endif + + function automatic lsu_req_data_t make_req(input logic [LSU_ADDR_WIDTH-1:0] base_word_addr, + input logic [BLOCK_IDX_BITS-1:0] block); + lsu_req_data_t r; + logic base_is_local; + `ifdef VX_CFG_LMEM_ENABLE + base_is_local = (base_word_addr[LSU_ADDR_WIDTH-1:LMEM_WORD_LSB] + == LMEM_WORD_BASE[LSU_ADDR_WIDTH-1:LMEM_WORD_LSB]); + `else + base_is_local = 1'b0; + `endif + r = '0; + r.rw = 1'b0; // load + r.mask = {NUM_LANES{1'b1}}; + for (int i = 0; i < NUM_LANES; i++) begin + mem_bus_attr_t a; + a = '0; + a.is_addr_local = base_is_local; + // base_addr is pre-advanced per TCU_LD slot by the kernel, so lane + // i reads 32-bit word i relative to it, in LSU-word units. + r.addr[i] = base_word_addr + (LSU_ADDR_WIDTH'(i) >> LG_W32_RATIO); + r.byteen[i] = {LSU_WORD_SIZE{1'b1}}; + r.data[i] = '0; + r.attr[i] = a; + end + r.tag = '0; + r.tag[BLOCK_IDX_BITS-1:0] = block; + return r; + endfunction + + `STATIC_ASSERT (LSU_CLIENT_TAG_WIDTH >= BLOCK_IDX_BITS, ("LSU client tag cannot encode TCU block")) + + // Per-block request and response events, from either port structure. + wire [BLOCK_SIZE-1:0] req_fire_b; + wire [BLOCK_SIZE-1:0] rsp_fire_b; + wire [BLOCK_SIZE-1:0] rsp_eop_b; + wire [NUM_LANES-1:0] rsp_mask_b [BLOCK_SIZE]; + wire [NUM_LANES-1:0][(LSU_WORD_SIZE*8)-1:0] rsp_data_b [BLOCK_SIZE]; + wire [`VX_CFG_XLEN-1:0] req_addr_chk [NUM_PORTS]; + wire [NUM_PORTS-1:0] req_fire_chk; + + if (NUM_PORTS == 1) begin : g_shared_port + wire [BLOCK_IDX_BITS-1:0] req_block; + wire [BLOCK_SIZE-1:0] req_grant; + wire req_valid; + + VX_generic_arbiter #( + .NUM_REQS (BLOCK_SIZE), + .TYPE ("R") + ) req_arb ( + .clk (clk), + .reset (reset), + .requests (req_pending), + .grant_index (req_block), + .grant_onehot (req_grant), + .grant_valid (req_valid), + .grant_ready (client_if[0].req_ready) + ); + `UNUSED_VAR (req_grant) + + wire [BLOCK_IDX_BITS-1:0] rsp_block = client_if[0].rsp_data.tag[BLOCK_IDX_BITS-1:0]; + wire req_fire = req_valid && client_if[0].req_ready; + wire rsp_fire = client_if[0].rsp_valid && client_if[0].rsp_ready; + + for (genvar bi = 0; bi < BLOCK_SIZE; ++bi) begin : g_ev + assign req_fire_b[bi] = req_fire && (req_block == BLOCK_IDX_BITS'(bi)); + assign rsp_fire_b[bi] = rsp_fire && (rsp_block == BLOCK_IDX_BITS'(bi)); + assign rsp_eop_b[bi] = client_if[0].rsp_data.eop; + assign rsp_mask_b[bi] = client_if[0].rsp_data.mask; + assign rsp_data_b[bi] = client_if[0].rsp_data.data; + end + + assign client_if[0].req_valid = req_valid; + assign client_if[0].req_data = make_req(LSU_ADDR_WIDTH'(addr_r[req_block] >> `CLOG2(LSU_WORD_SIZE)), req_block); + assign client_if[0].rsp_ready = busy_r[rsp_block] && issued_r[rsp_block] + && !complete_r[rsp_block]; + assign req_addr_chk[0] = addr_r[req_block]; + assign req_fire_chk[0] = req_fire; + `UNUSED_VAR (client_if[0].rsp_data.sop) + end else begin : g_port_per_block + for (genvar bi = 0; bi < BLOCK_SIZE; ++bi) begin : g_port + assign client_if[bi].req_valid = req_pending[bi]; + assign client_if[bi].req_data = make_req(LSU_ADDR_WIDTH'(addr_r[bi] >> `CLOG2(LSU_WORD_SIZE)), BLOCK_IDX_BITS'(0)); + assign client_if[bi].rsp_ready = busy_r[bi] && issued_r[bi] && !complete_r[bi]; + + assign req_fire_b[bi] = req_pending[bi] && client_if[bi].req_ready; + assign rsp_fire_b[bi] = client_if[bi].rsp_valid && client_if[bi].rsp_ready; + assign rsp_eop_b[bi] = client_if[bi].rsp_data.eop; + assign rsp_mask_b[bi] = client_if[bi].rsp_data.mask; + assign rsp_data_b[bi] = client_if[bi].rsp_data.data; + assign req_addr_chk[bi] = addr_r[bi]; + assign req_fire_chk[bi] = req_fire_b[bi]; + `UNUSED_VAR (client_if[bi].rsp_data.sop) + `UNUSED_VAR (client_if[bi].rsp_data.tag) + end + end + always @(posedge clk) begin if (reset) begin busy_r <= '0; @@ -132,21 +254,20 @@ module VX_tcu_agu import VX_gpu_pkg::*, VX_tcu_pkg::*; #( rsp_data_r[bi] <= '0; rsp_lane_done_r[bi] <= '0; end - end - if (req_fire) begin - issued_r[req_block] <= 1'b1; - end - if (rsp_fire) begin - for (int t = 0; t < NUM_LANES; ++t) begin - if (client_if.rsp_data.mask[t]) begin - rsp_data_r[rsp_block][t] <= client_if.rsp_data.data[t]; - rsp_lane_done_r[rsp_block][t] <= 1'b1; - end + if (req_fire_b[bi]) begin + issued_r[bi] <= 1'b1; end - if (client_if.rsp_data.eop - && ((rsp_lane_done_r[rsp_block] | client_if.rsp_data.mask) - == {NUM_LANES{1'b1}})) begin - complete_r[rsp_block] <= 1'b1; + if (rsp_fire_b[bi]) begin + for (int t = 0; t < NUM_LANES; ++t) begin + if (rsp_mask_b[bi][t]) begin + rsp_data_r[bi][t] <= rsp_data_b[bi][t]; + rsp_lane_done_r[bi][t] <= 1'b1; + end + end + if (rsp_eop_b[bi] + && ((rsp_lane_done_r[bi] | rsp_mask_b[bi]) == {NUM_LANES{1'b1}})) begin + complete_r[bi] <= 1'b1; + end end end if (commit_fire) begin @@ -158,116 +279,24 @@ module VX_tcu_agu import VX_gpu_pkg::*, VX_tcu_pkg::*; #( end end - `UNUSED_VAR (client_if.rsp_data.sop) - for (genvar bi = 0; bi < BLOCK_SIZE; ++bi) begin : g_ld_ready assign per_block_ld_ready[bi] = ~busy_r[bi]; end - // ----------------------------------------------------------------------- - // Memory request: address from rs1 (warp-broadcast). NUM_LANES mask - // all-ones — we fetch the metadata tile in one warp-level request. - // - // Per-lane offset matches the host's pack_metadata_wg layout - // (sgemm_tcu_wg_sp/main.cpp): the host writes bank `b` and word `c` - // at index `b * stride + c`. The AGU writes wr_data[T] into - // VX_tcu_meta at (bank=T%PWD, col_in_group=T/PWD), so lane T must - // load `meta_base[(T % PWD) * stride + (T / PWD)]`. - // ----------------------------------------------------------------------- - wire [`VX_CFG_XLEN-1:0] base_addr = addr_r[req_block]; - `UNUSED_VAR (base_addr[`CLOG2(LSU_WORD_SIZE)-1:0]) - - wire [LSU_ADDR_WIDTH-1:0] base_word_addr = - LSU_ADDR_WIDTH'(base_addr[`VX_CFG_XLEN-1:`CLOG2(LSU_WORD_SIZE)]); - - localparam PWD = TCU_META_PER_WARP_DEPTH; - // Per-thread metadata layout: lane T loads h_meta[slot*NT + T]. The - // host packs h_meta so the 32-bit word at offset (slot*NT + T) holds - // the metadata destined for SRAM cell (bank = T%PWD, col = T/PWD). - // One formula serves both WMMA-SP and WGMMA-SP. - - // LSU bus carries LSU_WORD_SIZE-byte chunks per lane (= XLEN/8 bytes). - // The metadata layout is contiguous 32-bit words, so when XLEN > 32 - // each LSU response carries XLEN/32 logical 32-bit words and we need - // to pick the right one. LSU_W32_RATIO = bytes/4, so XLEN=64 → 2. - localparam LSU_W32_RATIO = LSU_WORD_SIZE / 4; - localparam LG_W32_RATIO = (LSU_W32_RATIO > 1) ? $clog2(LSU_W32_RATIO) : 0; - - // is_addr_local route bit — LMEM range check. The metadata buffer is - // per-warp inside shared memory so LMEM detection is uniform across - // lanes; compute once from the base address. -`ifdef VX_CFG_LMEM_ENABLE - // Equality on the word-address bits above the window: the base is asserted - // aligned to the capacity, so the prefix compare is exact and needs no end - // address -- which is not representable when the window ends at the top of - // the address space. - localparam LMEM_WORD_LSB = `VX_CFG_LMEM_LOG_SIZE - `CLOG2(LSU_WORD_SIZE); - localparam [LSU_ADDR_WIDTH-1:0] LMEM_WORD_BASE = - LSU_ADDR_WIDTH'(`VX_CFG_XLEN'(`VX_MEM_LMEM_BASE_ADDR) >> `CLOG2(LSU_WORD_SIZE)); - `STATIC_ASSERT (LSU_ADDR_WIDTH > LMEM_WORD_LSB, ("LMEM window wider than the word address space")) - wire base_is_local = (base_word_addr[LSU_ADDR_WIDTH-1:LMEM_WORD_LSB] - == LMEM_WORD_BASE[LSU_ADDR_WIDTH-1:LMEM_WORD_LSB]); -`else - wire base_is_local = 1'b0; -`endif - - // Per-lane sub-word selector: which 32-bit slice of the LSU response - // carries the desired metadata word. For XLEN=32 (LSU_W32_RATIO=1) - // every lane reads exactly one 32-bit word, so half_sel is always 0. - // For XLEN=64 (LSU_W32_RATIO=2) we round the 32-bit word offset down - // to an 8-byte LSU address and use the LSB to select low/high half. - logic [NUM_LANES-1:0][`UP(LG_W32_RATIO)-1:0] half_sel_w; - // For XLEN=32, LG_W32_RATIO=0 and half_sel_w is never read downstream; - // tag it unused so Verilator's UNUSEDSIGNAL doesn't fail the build. - if (LG_W32_RATIO == 0) begin : g_half_sel_unused - `UNUSED_VAR (half_sel_w) - end - - lsu_req_data_t req_w; - always_comb begin - req_w = '0; - req_w.rw = 1'b0; // load - req_w.mask = {NUM_LANES{1'b1}}; - half_sel_w = '0; - for (int i = 0; i < NUM_LANES; i++) begin - automatic mem_bus_attr_t a; - automatic logic [LSU_ADDR_WIDTH-1:0] word_off_32; - a = '0; - a.is_addr_local = base_is_local; - // base_addr is pre-advanced per TCU_LD slot by the kernel - // (slot s passes base + s*NT*sizeof(float)), so lane i reads word i - // relative to base_word_addr. - word_off_32 = LSU_ADDR_WIDTH'(i); - // word_off_32 is in 32-bit-word units. The LSU bus addresses - // LSU_WORD_SIZE-byte chunks (= XLEN/32 × 32-bit words), so we - // shift down to bus stride and keep the sub-word half for the - // response extraction below. - req_w.addr[i] = base_word_addr + (word_off_32 >> LG_W32_RATIO); - if (LG_W32_RATIO > 0) begin - half_sel_w[i] = `UP(LG_W32_RATIO)'(word_off_32[`UP(LG_W32_RATIO)-1:0]); - end - req_w.byteen[i] = {LSU_WORD_SIZE{1'b1}}; - req_w.data[i] = '0; - req_w.attr[i] = a; - end - req_w.tag = '0; - req_w.tag[BLOCK_IDX_BITS-1:0] = req_block; - end - `UNUSED_PARAM (PWD) - `STATIC_ASSERT (LSU_CLIENT_TAG_WIDTH >= BLOCK_IDX_BITS, ("LSU client tag cannot encode TCU block")) `ifdef VX_CFG_LSU_STACK_INTERLEAVE_ENABLE // Metadata is read as a linear warp-wide tile, which an interleaved // per-thread stack cannot provide. if (`VX_CFG_NUM_THREADS > 1) begin : g_meta_stack_check - wire [`VX_CFG_XLEN-1:0] meta_end_addr = base_addr + `VX_CFG_XLEN'(NUM_LANES * 4 - 1); - `RUNTIME_ASSERT(~req_fire || (meta_end_addr < STACK_WINDOW_BOTTOM) || (base_addr >= STACK_WINDOW_TOP), - ("%t: *** %s TCU metadata inside the thread stack window: addr=0x%0h", $time, INSTANCE_ID, base_addr)) + for (genvar pi = 0; pi < NUM_PORTS; ++pi) begin : g_port + wire [`VX_CFG_XLEN-1:0] meta_end_addr = req_addr_chk[pi] + `VX_CFG_XLEN'(NUM_LANES * 4 - 1); + `RUNTIME_ASSERT(~req_fire_chk[pi] || (meta_end_addr < STACK_WINDOW_BOTTOM) || (req_addr_chk[pi] >= STACK_WINDOW_TOP), + ("%t: *** %s TCU metadata inside the thread stack window: addr=0x%0h", $time, INSTANCE_ID, req_addr_chk[pi])) + end end +`else + `UNUSED_VAR (req_addr_chk) + `UNUSED_VAR (req_fire_chk) `endif - assign client_if.req_valid = req_valid; - assign client_if.req_data = req_w; - assign client_if.rsp_ready = busy_r[rsp_block] && issued_r[rsp_block] - && !complete_r[rsp_block]; // ----------------------------------------------------------------------- // VX_tcu_meta write (broadcast to all blocks' meta instances). diff --git a/hw/rtl/tcu/VX_tcu_shared.sv b/hw/rtl/tcu/VX_tcu_shared.sv index 77ff231a72..4fee552c69 100644 --- a/hw/rtl/tcu/VX_tcu_shared.sv +++ b/hw/rtl/tcu/VX_tcu_shared.sv @@ -45,8 +45,8 @@ module VX_tcu_shared import VX_gpu_pkg::*, VX_tcu_pkg::*; #( output wire [NW_WIDTH-1:0] meta_wr_wid, output wire [4:0] meta_wr_idx, output wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] meta_wr_data, - // TCU_LD memory client of the LSU scheduler - VX_lsu_sched_if.master tcu_mem_if, + // TCU_LD memory clients, one per LSU block's scheduler + VX_lsu_sched_if.master tcu_mem_if [`VX_CFG_NUM_LSU_BLOCKS], `endif input wire clk, @@ -94,17 +94,41 @@ module VX_tcu_shared import VX_gpu_pkg::*, VX_tcu_pkg::*; #( // counts must match for the per-lane completion tracking to be correct. `STATIC_ASSERT (`VX_CFG_NUM_TCU_LANES == `VX_CFG_NUM_LSU_LANES, ("VX_tcu_agu requires NUM_TCU_LANES == NUM_LSU_LANES")); + // With one LSU block per TCU block, each block loads its metadata + // through the scheduler of the LSU block that serves the same warps; + // otherwise every block shares LSU block 0's scheduler. + localparam AGU_PORTS = (`VX_CFG_NUM_LSU_BLOCKS == BLOCK_SIZE) ? BLOCK_SIZE : 1; + for (genvar b = AGU_PORTS; b < `VX_CFG_NUM_LSU_BLOCKS; ++b) begin : g_mem_tieoff + assign tcu_mem_if[b].req_valid = 1'b0; + assign tcu_mem_if[b].req_data = '0; + assign tcu_mem_if[b].rsp_ready = 1'b1; + `UNUSED_VAR (tcu_mem_if[b].req_ready) + `UNUSED_VAR (tcu_mem_if[b].rsp_valid) + `UNUSED_VAR (tcu_mem_if[b].rsp_data) + end + + VX_lsu_sched_if agu_mem_if [AGU_PORTS](); + for (genvar b = 0; b < AGU_PORTS; ++b) begin : g_mem_port + assign tcu_mem_if[b].req_valid = agu_mem_if[b].req_valid; + assign tcu_mem_if[b].req_data = agu_mem_if[b].req_data; + assign agu_mem_if[b].req_ready = tcu_mem_if[b].req_ready; + assign agu_mem_if[b].rsp_valid = tcu_mem_if[b].rsp_valid; + assign agu_mem_if[b].rsp_data = tcu_mem_if[b].rsp_data; + assign tcu_mem_if[b].rsp_ready = agu_mem_if[b].rsp_ready; + end + VX_tcu_agu #( .INSTANCE_ID (`SFORMATF(("%s-agu", INSTANCE_ID))), .BLOCK_SIZE (BLOCK_SIZE), - .NUM_LANES (`VX_CFG_NUM_TCU_LANES) + .NUM_LANES (`VX_CFG_NUM_TCU_LANES), + .NUM_PORTS (AGU_PORTS) ) agu ( .clk (clk), .reset (reset), .per_block_ld_valid (agu_ld_valid), .per_block_ld_data (agu_ld_data), .per_block_ld_ready (agu_ld_ready), - .client_if (tcu_mem_if), + .client_if (agu_mem_if), .meta_wr_en (meta_wr_en), .meta_wr_wid (meta_wr_wid), .meta_wr_idx (meta_wr_idx), diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index bd84b24ea5..bfc91db066 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -285,11 +285,13 @@ class Core::Impl { } #endif #ifdef TCU_META_ENABLE - // Bind the TCU metadata AGU to the LSU block-0 client port. + // Bind the TCU metadata AGU ports to the LSU client ports. { auto lsu = std::static_pointer_cast(func_units_.at((int)FUType::LSU)); - tcu_unit_->agu_req_out.bind(&lsu->TcuReqIn); - lsu->TcuRspOut.bind(&tcu_unit_->agu_rsp_in); + for (uint32_t p = 0; p < TCU_AGU_PORTS; ++p) { + tcu_unit_->agu_req_out.at(p).bind(&lsu->TcuReqIn.at(p)); + lsu->TcuRspOut.at(p).bind(&tcu_unit_->agu_rsp_in.at(p)); + } } #endif #endif diff --git a/sim/simx/lsu_unit.cpp b/sim/simx/lsu_unit.cpp index e17a131bee..eb0b032917 100644 --- a/sim/simx/lsu_unit.cpp +++ b/sim/simx/lsu_unit.cpp @@ -78,8 +78,8 @@ LsuUnit::LsuUnit(const SimContext& ctx, const char* name, Core* core) // instead of throttling the memory response stream. : FuncUnit(ctx, name, core, 6) #ifdef TCU_META_ENABLE - , TcuReqIn(this) - , TcuRspOut(this) + , TcuReqIn(TCU_AGU_PORTS, this) + , TcuRspOut(TCU_AGU_PORTS, this) #endif , pending_loads_(0) {} @@ -94,8 +94,10 @@ bool LsuUnit::drained() const { #ifdef TCU_META_ENABLE // TcuRspOut forwards to the TCU's endpoint, so query via size() (empty() // asserts on forwarding channels). - if (!TcuReqIn.empty() || TcuRspOut.size() != 0) { - return false; + for (uint32_t p = 0; p < TCU_AGU_PORTS; ++p) { + if (!TcuReqIn.at(p).empty() || TcuRspOut.at(p).size() != 0) { + return false; + } } #endif for (uint32_t b = 0; b < VX_CFG_NUM_LSU_BLOCKS; ++b) { @@ -221,11 +223,11 @@ void LsuUnit::process_response_step(uint32_t b) { // TCU metadata response: forward the fragment with the client's tag // restored; no register writeback, no trace retirement. if (entry.is_tcu) { - if (TcuRspOut.full()) + if (TcuRspOut.at(b).full()) return; // stall LsuRsp fwd = lsu_rsp; fwd.tag = entry.client_tag; - TcuRspOut.send(fwd); + TcuRspOut.at(b).send(fwd); DT(3, this->name() << " tcu-meta-rsp: " << fwd); entry.count -= lsu_rsp.mask.count(); if (entry.count == 0) { @@ -422,15 +424,15 @@ void LsuUnit::process_request_step(uint32_t b) { } #ifdef TCU_META_ENABLE - // TCU metadata client (block 0 only): forward one pending request per + // TCU metadata client (blocks below TCU_AGU_PORTS): forward one pending request per // cycle into lmem_switch, ahead of the block's own dispatch. The wire tag // is re-allocated from pending_reqs so it stays unique on the switch; the // client's own tag is restored on the response. An independent client, so // not gated by the fence lock — its entries still hold fence release via // pending_reqs.empty(). - if (b == 0 && !TcuReqIn.empty()) { - if (!state.pending_reqs.full() && !core_->lmem_switch(0)->ReqIn.full()) { - LsuReq lsu_req = TcuReqIn.peek(); + if (b < TCU_AGU_PORTS && !TcuReqIn.at(b).empty()) { + if (!state.pending_reqs.full() && !core_->lmem_switch(b)->ReqIn.full()) { + LsuReq lsu_req = TcuReqIn.at(b).peek(); uint32_t count = lsu_req.mask.count(); std::vector lane_entries(VX_CFG_NUM_LSU_LANES); for (uint32_t i = 0; i < VX_CFG_NUM_LSU_LANES; ++i) { @@ -452,9 +454,9 @@ void LsuUnit::process_request_step(uint32_t b) { entry.client_tag = lsu_req.tag; uint32_t tag = state.pending_reqs.allocate(std::move(entry)); lsu_req.tag = tag; - core_->lmem_switch(0)->ReqIn.send(lsu_req); + core_->lmem_switch(b)->ReqIn.send(lsu_req); DT(3, this->name() << " tcu-meta-req: " << lsu_req); - TcuReqIn.pop(); + TcuReqIn.at(b).pop(); return; // one request per cycle into the switch } } diff --git a/sim/simx/lsu_unit.h b/sim/simx/lsu_unit.h index 8ab00012dc..6030fc0a39 100644 --- a/sim/simx/lsu_unit.h +++ b/sim/simx/lsu_unit.h @@ -27,6 +27,13 @@ #define TCU_META_ENABLE #endif +// TCU_LD memory ports: one per TCU block, through the LSU block that serves +// the same warps, when the LSU has one block per TCU block; otherwise all +// TCU blocks share LSU block 0. Same derivation as the RTL's AGU_PORTS. +#ifndef TCU_AGU_PORTS +#define TCU_AGU_PORTS (uint32_t((VX_CFG_NUM_LSU_BLOCKS == VX_CFG_NUM_TCU_BLOCKS) ? VX_CFG_NUM_TCU_BLOCKS : 1)) +#endif + namespace vortex { // Micro-op generator for packed-load macro instructions (PACKLB.F / PACKLH.F). @@ -79,11 +86,12 @@ class LsuUnit : public FuncUnit { ~LsuUnit(); #ifdef TCU_META_ENABLE - // TCU metadata client port on block 0: TCU_LD load requests arbitrate - // with the LSU's own traffic into lmem_switch; response fragments return - // here with the client's own tag restored. - SimChannel TcuReqIn; - SimChannel TcuRspOut; + // TCU metadata client ports (TCU_AGU_PORTS, starting at block 0): TCU_LD + // load requests arbitrate with the block's own traffic into its + // lmem_switch; response fragments return here with the client's own tag + // restored. + std::vector> TcuReqIn; + std::vector> TcuRspOut; #endif // Returns true when all LSU blocks have no in-flight requests. diff --git a/sim/simx/tcu/tcu_unit.cpp b/sim/simx/tcu/tcu_unit.cpp index 5e2bd89173..0e88015cb3 100644 --- a/sim/simx/tcu/tcu_unit.cpp +++ b/sim/simx/tcu/tcu_unit.cpp @@ -452,58 +452,82 @@ class TcuUnit::Impl { } } - void agu_step() { - if (!simobject_->agu_req_out.full()) { - for (uint32_t offset = 0; offset < VX_CFG_NUM_TCU_BLOCKS; ++offset) { - uint32_t block = (agu_issue_rr_ + offset) % VX_CFG_NUM_TCU_BLOCKS; - auto& agu = agu_.at(block); - if (!agu.busy || agu.next_beat >= kAguBeats) { - continue; - } - LsuReq req(VX_CFG_NUM_LSU_LANES); - req.op = MemOp::LD; - req.tag = block * kAguBeats + agu.next_beat; - req.wid = agu.wid; - req.cid = agu.trace->cid; - req.uuid = agu.trace->uuid; - for (uint32_t i = 0; i < VX_CFG_NUM_LSU_LANES; ++i) { - uint32_t idx = agu.next_beat * VX_CFG_NUM_LSU_LANES + i; - if (idx >= VX_CFG_NUM_THREADS) { - break; - } - req.mask.set(i); - req.addrs.at(i) = agu.addrs.at(idx); - req.tids.at(i) = idx % VX_CFG_NUM_THREADS; - } - simobject_->agu_req_out.send(req); - ++agu.next_beat; - agu_issue_rr_ = (block + 1) % VX_CFG_NUM_TCU_BLOCKS; + LsuReq agu_make_req(uint32_t block, agu_state_t& agu) { + LsuReq req(VX_CFG_NUM_LSU_LANES); + req.op = MemOp::LD; + req.tag = block * kAguBeats + agu.next_beat; + req.wid = agu.wid; + req.cid = agu.trace->cid; + req.uuid = agu.trace->uuid; + for (uint32_t i = 0; i < VX_CFG_NUM_LSU_LANES; ++i) { + uint32_t idx = agu.next_beat * VX_CFG_NUM_LSU_LANES + i; + if (idx >= VX_CFG_NUM_THREADS) { break; } + req.mask.set(i); + req.addrs.at(i) = agu.addrs.at(idx); + req.tids.at(i) = idx % VX_CFG_NUM_THREADS; } + ++agu.next_beat; + return req; + } - if (!simobject_->agu_rsp_in.empty()) { - auto& rsp = simobject_->agu_rsp_in.peek(); - uint32_t block = (uint32_t)rsp.tag / kAguBeats; - uint32_t beat = (uint32_t)rsp.tag % kAguBeats; - auto& agu = agu_.at(block); - for (uint32_t lane = 0; lane < rsp.mask.size(); ++lane) { - if (!rsp.mask.test(lane)) { + void agu_receive(const LsuRsp& rsp) { + uint32_t block = (uint32_t)rsp.tag / kAguBeats; + uint32_t beat = (uint32_t)rsp.tag % kAguBeats; + auto& agu = agu_.at(block); + for (uint32_t lane = 0; lane < rsp.mask.size(); ++lane) { + if (!rsp.mask.test(lane)) { + continue; + } + uint32_t idx = beat * VX_CFG_NUM_LSU_LANES + lane; + assert(idx < VX_CFG_NUM_THREADS && rsp.data.at(lane)); + uint32_t off = agu.addrs.at(idx) & (VX_CFG_MEM_BLOCK_SIZE - 1); + uint32_t word = 0; + std::memcpy(&word, rsp.data.at(lane)->data() + off, 4); + agu.words.at(idx) = word; + ++agu.rsp_count; + } + if (agu.busy && !agu.complete && agu.rsp_count == VX_CFG_NUM_THREADS) { + this->agu_deposit(agu); + agu.complete = true; + } + } + + void agu_step() { + if (TCU_AGU_PORTS == 1) { + // every block shares one port: round-robin one request per cycle + if (!simobject_->agu_req_out.at(0).full()) { + for (uint32_t offset = 0; offset < VX_CFG_NUM_TCU_BLOCKS; ++offset) { + uint32_t block = (agu_issue_rr_ + offset) % VX_CFG_NUM_TCU_BLOCKS; + auto& agu = agu_.at(block); + if (!agu.busy || agu.next_beat >= kAguBeats) { + continue; + } + simobject_->agu_req_out.at(0).send(this->agu_make_req(block, agu)); + agu_issue_rr_ = (block + 1) % VX_CFG_NUM_TCU_BLOCKS; + break; + } + } + } else { + // each block issues on its own port + for (uint32_t block = 0; block < VX_CFG_NUM_TCU_BLOCKS; ++block) { + auto& agu = agu_.at(block); + if (!agu.busy || agu.next_beat >= kAguBeats + || simobject_->agu_req_out.at(block).full()) { continue; } - uint32_t idx = beat * VX_CFG_NUM_LSU_LANES + lane; - assert(idx < VX_CFG_NUM_THREADS && rsp.data.at(lane)); - uint32_t off = agu.addrs.at(idx) & (VX_CFG_MEM_BLOCK_SIZE - 1); - uint32_t word = 0; - std::memcpy(&word, rsp.data.at(lane)->data() + off, 4); - agu.words.at(idx) = word; - ++agu.rsp_count; + simobject_->agu_req_out.at(block).send(this->agu_make_req(block, agu)); } - simobject_->agu_rsp_in.pop(); - if (agu.busy && !agu.complete && agu.rsp_count == VX_CFG_NUM_THREADS) { - this->agu_deposit(agu); - agu.complete = true; + } + + for (uint32_t p = 0; p < TCU_AGU_PORTS; ++p) { + auto& rsp_in = simobject_->agu_rsp_in.at(p); + if (rsp_in.empty()) { + continue; } + this->agu_receive(rsp_in.peek()); + rsp_in.pop(); } } @@ -1834,8 +1858,8 @@ Instr::Ptr TcuUopGen::get(const Instr& macro_instr, uint32_t uop_index) { TcuUnit::TcuUnit(const SimContext &ctx, const char* name, Core* core) : FuncUnit(ctx, name, core, 1u << log2ceil(VX_CFG_TCU_LATENCY + 1)) // landing-queue depth bounds results awaiting the consumer #ifdef TCU_META_ENABLE - , agu_req_out(this) - , agu_rsp_in(this) + , agu_req_out(TCU_AGU_PORTS, this) + , agu_rsp_in(TCU_AGU_PORTS, this) #endif , impl_(new Impl(this, core)) { diff --git a/sim/simx/tcu/tcu_unit.h b/sim/simx/tcu/tcu_unit.h index 93328060f4..681c448b5c 100644 --- a/sim/simx/tcu/tcu_unit.h +++ b/sim/simx/tcu/tcu_unit.h @@ -28,6 +28,13 @@ #define TCU_META_ENABLE #endif +// TCU_LD memory ports: one per TCU block, through the LSU block that serves +// the same warps, when the LSU has one block per TCU block; otherwise all +// TCU blocks share LSU block 0. Same derivation as the RTL's AGU_PORTS. +#ifndef TCU_AGU_PORTS +#define TCU_AGU_PORTS (uint32_t((VX_CFG_NUM_LSU_BLOCKS == VX_CFG_NUM_TCU_BLOCKS) ? VX_CFG_NUM_TCU_BLOCKS : 1)) +#endif + namespace vortex { class Core; @@ -77,11 +84,11 @@ class TcuUnit : public FuncUnit { virtual ~TcuUnit(); #ifdef TCU_META_ENABLE - // Metadata AGU port: TCU_LD issues load requests through the LSU block-0 - // client port; response fragments return here and accumulate into the - // metadata SRAM. - SimChannel agu_req_out; - SimChannel agu_rsp_in; + // Metadata AGU ports (TCU_AGU_PORTS): TCU_LD issues load requests through + // the LSU client ports; response fragments return here and accumulate into + // the metadata SRAM. + std::vector> agu_req_out; + std::vector> agu_rsp_in; #endif void wmma(uint32_t wid, From fc0a35bb423603e919343fc1b1a67e58aeaaa315 Mon Sep 17 00:00:00 2001 From: tinebp Date: Thu, 22 Oct 2026 04:43:07 -0700 Subject: [PATCH 088/124] tcu: move the TCU_LD AGU into each TCU block and drop its gather buffer The metadata AGU was one core-level unit: a TCU_LD left its slice for the AGU and its metadata came back over a core-wide write bus. Each TCU block now holds its own AGU, wired to the scheduler of the LSU block serving the same warps (inside the execute slice in the sliced core modes) and to the block's own metadata RAM. The AGU no longer buffers a warp's response. Each response fragment writes its lanes straight into the metadata RAM through a new per-lane write mask, and the scheduler's end-of-packet marks completion, so the per-block gather buffer (NUM_LANES x XLEN bits), the lane-done mask, the request and commit arbiters, the cross-block commit mux and the core-wide write bus are gone. A block keeps only busy/issued/done flags, the address, slot and header. Writing early is safe: TCU_LD enters its block in order behind the older MMA micro-ops, which read the metadata RAM when they issue. When the LSU has fewer blocks than the TCU, the blocks share LSU block 0's client through a round-robin arbiter in VX_tcu_unit, as before. SimX now writes each returned lane into the metadata on arrival, matching the RTL. sgemm_tcu_wg_sp_dxa 128x128x128, rtlsim (IW4, FW4, 32 threads), PASSED: stage-sliced 144,956 cycles, compute slices 144,830, both identical to the previous commit; SimX 145,323, identical. Lint clean in 13 configs (XLEN 32/64; stage-sliced, backend and compute slices; shared LSU port; sparse and MX). Co-Authored-By: Claude Opus 5.5 --- hw/rtl/core/VX_backend_slice.sv | 30 --- hw/rtl/core/VX_compute_slice.sv | 30 --- hw/rtl/core/VX_core.sv | 73 ------ hw/rtl/core/VX_execute.sv | 31 +-- hw/rtl/core/VX_execute_slice.sv | 31 +-- hw/rtl/tcu/VX_tcu_agu.sv | 380 +++++++++----------------------- hw/rtl/tcu/VX_tcu_block.sv | 48 ++-- hw/rtl/tcu/VX_tcu_core.sv | 6 +- hw/rtl/tcu/VX_tcu_meta.sv | 29 ++- hw/rtl/tcu/VX_tcu_shared.sv | 70 +----- hw/rtl/tcu/VX_tcu_unit.sv | 98 +++++--- sim/simx/lsu_unit.h | 2 +- sim/simx/tcu/tcu_unit.cpp | 51 ++--- sim/simx/tcu/tcu_unit.h | 2 +- 14 files changed, 272 insertions(+), 609 deletions(-) diff --git a/hw/rtl/core/VX_backend_slice.sv b/hw/rtl/core/VX_backend_slice.sv index 385db201a5..2d7059d12a 100644 --- a/hw/rtl/core/VX_backend_slice.sv +++ b/hw/rtl/core/VX_backend_slice.sv @@ -61,9 +61,6 @@ import VX_tcu_pkg::*; output wire [`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_wr_req_fire, output wire [`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_rsp_fire, output wire [PERF_CTR_BITS-1:0] lsu_coalescer_misses, -`endif -`ifdef TCU_META_ENABLE - VX_lsu_sched_if.slave lsu_tcu_mem_if, `endif output wire lsu_sched_empty, output wire lsu_store_empty, @@ -93,18 +90,6 @@ import VX_tcu_pkg::*; input wire [TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs2_data, input wire tcu_tbuf_ready, `endif -`ifdef TCU_META_ENABLE - output wire tcu_agu_ld_valid, - output tcu_execute_t tcu_agu_ld_data, - input wire tcu_agu_ld_ready, - input wire tcu_agu_result_valid, - input tcu_result_t tcu_agu_result_data, - output wire tcu_agu_result_ready, - input wire tcu_meta_wr_en, - input wire [NW_WIDTH-1:0] tcu_meta_wr_wid, - input wire [4:0] tcu_meta_wr_idx, - input wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] tcu_meta_wr_data, -`endif // core-wide warp control and CTA contexts VX_warp_shared_if.master warp_shared_if, @@ -162,9 +147,6 @@ import VX_tcu_pkg::*; .lsu_perf_wr_req_fire (lsu_perf_wr_req_fire), .lsu_perf_rsp_fire (lsu_perf_rsp_fire), .lsu_coalescer_misses (lsu_coalescer_misses), - `endif - `ifdef TCU_META_ENABLE - .lsu_tcu_mem_if (lsu_tcu_mem_if), `endif .lsu_sched_empty (lsu_sched_empty), .lsu_store_empty (lsu_store_empty), @@ -190,18 +172,6 @@ import VX_tcu_pkg::*; .tcu_tbuf_rs1_data (tcu_tbuf_rs1_data), .tcu_tbuf_rs2_data (tcu_tbuf_rs2_data), .tcu_tbuf_ready (tcu_tbuf_ready), - `endif - `ifdef TCU_META_ENABLE - .tcu_agu_ld_valid (tcu_agu_ld_valid), - .tcu_agu_ld_data (tcu_agu_ld_data), - .tcu_agu_ld_ready (tcu_agu_ld_ready), - .tcu_agu_result_valid (tcu_agu_result_valid), - .tcu_agu_result_data (tcu_agu_result_data), - .tcu_agu_result_ready (tcu_agu_result_ready), - .tcu_meta_wr_en (tcu_meta_wr_en), - .tcu_meta_wr_wid (tcu_meta_wr_wid), - .tcu_meta_wr_idx (tcu_meta_wr_idx), - .tcu_meta_wr_data (tcu_meta_wr_data), `endif .warp_shared_if (warp_shared_if), .warp_ctl_if (warp_ctl_if), diff --git a/hw/rtl/core/VX_compute_slice.sv b/hw/rtl/core/VX_compute_slice.sv index f241dd9d96..9eb31133da 100644 --- a/hw/rtl/core/VX_compute_slice.sv +++ b/hw/rtl/core/VX_compute_slice.sv @@ -64,9 +64,6 @@ import VX_tcu_pkg::*; output wire [`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_wr_req_fire, output wire [`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_rsp_fire, output wire [PERF_CTR_BITS-1:0] lsu_coalescer_misses, -`endif -`ifdef TCU_META_ENABLE - VX_lsu_sched_if.slave lsu_tcu_mem_if, `endif output wire lsu_sched_empty, output wire lsu_store_empty, @@ -96,18 +93,6 @@ import VX_tcu_pkg::*; input wire [TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs2_data, input wire tcu_tbuf_ready, `endif -`ifdef TCU_META_ENABLE - output wire tcu_agu_ld_valid, - output tcu_execute_t tcu_agu_ld_data, - input wire tcu_agu_ld_ready, - input wire tcu_agu_result_valid, - input tcu_result_t tcu_agu_result_data, - output wire tcu_agu_result_ready, - input wire tcu_meta_wr_en, - input wire [NW_WIDTH-1:0] tcu_meta_wr_wid, - input wire [4:0] tcu_meta_wr_idx, - input wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] tcu_meta_wr_data, -`endif // core-wide warp control VX_warp_shared_if.master warp_shared_if @@ -262,9 +247,6 @@ import VX_tcu_pkg::*; .lsu_perf_wr_req_fire (lsu_perf_wr_req_fire), .lsu_perf_rsp_fire (lsu_perf_rsp_fire), .lsu_coalescer_misses (lsu_coalescer_misses), - `endif - `ifdef TCU_META_ENABLE - .lsu_tcu_mem_if (lsu_tcu_mem_if), `endif .lsu_sched_empty (lsu_sched_empty), .lsu_store_empty (lsu_store_empty), @@ -289,18 +271,6 @@ import VX_tcu_pkg::*; .tcu_tbuf_rs1_data (tcu_tbuf_rs1_data), .tcu_tbuf_rs2_data (tcu_tbuf_rs2_data), .tcu_tbuf_ready (tcu_tbuf_ready), - `endif - `ifdef TCU_META_ENABLE - .tcu_agu_ld_valid (tcu_agu_ld_valid), - .tcu_agu_ld_data (tcu_agu_ld_data), - .tcu_agu_ld_ready (tcu_agu_ld_ready), - .tcu_agu_result_valid (tcu_agu_result_valid), - .tcu_agu_result_data (tcu_agu_result_data), - .tcu_agu_result_ready (tcu_agu_result_ready), - .tcu_meta_wr_en (tcu_meta_wr_en), - .tcu_meta_wr_wid (tcu_meta_wr_wid), - .tcu_meta_wr_idx (tcu_meta_wr_idx), - .tcu_meta_wr_data (tcu_meta_wr_data), `endif .warp_shared_if (warp_shared_if), .cta_bcast (cta_bcast) diff --git a/hw/rtl/core/VX_core.sv b/hw/rtl/core/VX_core.sv index a5b31c5e58..c018683547 100644 --- a/hw/rtl/core/VX_core.sv +++ b/hw/rtl/core/VX_core.sv @@ -101,11 +101,6 @@ import VX_tcu_pkg::*; wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][PERF_CTR_BITS-1:0] lsu_coalescer_misses; `endif -`ifdef TCU_META_ENABLE - // The TCU metadata AGU is the second client of the LSU schedulers. - VX_lsu_sched_if lsu_tcu_mem_if [`VX_CFG_NUM_LSU_BLOCKS](); -`endif - // frontend-slice icache requests; port 0 also carries the injected flush VX_mem_bus_if #( .DATA_SIZE (ICACHE_WORD_SIZE), @@ -317,16 +312,6 @@ import VX_tcu_pkg::*; wire [`VX_CFG_NUM_TCU_BLOCKS-1:0][TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs2_data; wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_tbuf_ready; `endif -`ifdef TCU_META_ENABLE - wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_agu_ld_valid, tcu_agu_ld_ready; - tcu_execute_t tcu_agu_ld_data [`VX_CFG_NUM_TCU_BLOCKS]; - wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_agu_result_valid, tcu_agu_result_ready; - tcu_result_t tcu_agu_result_data [`VX_CFG_NUM_TCU_BLOCKS]; - wire tcu_meta_wr_en; - wire [NW_WIDTH-1:0] tcu_meta_wr_wid; - wire [4:0] tcu_meta_wr_idx; - wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] tcu_meta_wr_data; -`endif VX_sfu_shared #( .INSTANCE_ID (`SFORMATF(("%s-sfu-shared", INSTANCE_ID))) @@ -363,19 +348,6 @@ import VX_tcu_pkg::*; .tbuf_rs2_data (tcu_tbuf_rs2_data), .tbuf_ready_eff (tcu_tbuf_ready), .tcu_lmem_if (tcu_lmem_if), - `endif - `ifdef TCU_META_ENABLE - .agu_ld_valid (tcu_agu_ld_valid), - .agu_ld_data (tcu_agu_ld_data), - .agu_ld_ready (tcu_agu_ld_ready), - .agu_result_valid (tcu_agu_result_valid), - .agu_result_data (tcu_agu_result_data), - .agu_result_ready (tcu_agu_result_ready), - .meta_wr_en (tcu_meta_wr_en), - .meta_wr_wid (tcu_meta_wr_wid), - .meta_wr_idx (tcu_meta_wr_idx), - .meta_wr_data (tcu_meta_wr_data), - .tcu_mem_if (lsu_tcu_mem_if), `endif .clk (clk), .reset (reset) @@ -452,9 +424,6 @@ import VX_tcu_pkg::*; .lsu_perf_wr_req_fire (lsu_perf_wr_req_fire[i]), .lsu_perf_rsp_fire (lsu_perf_rsp_fire[i]), .lsu_coalescer_misses (lsu_coalescer_misses[i]), - `endif - `ifdef TCU_META_ENABLE - .lsu_tcu_mem_if (lsu_tcu_mem_if[i]), `endif .lsu_sched_empty (lsu_sched_empty[i]), .lsu_store_empty (lsu_store_empty[i]), @@ -479,18 +448,6 @@ import VX_tcu_pkg::*; .tcu_tbuf_rs1_data (tcu_tbuf_rs1_data[i]), .tcu_tbuf_rs2_data (tcu_tbuf_rs2_data[i]), .tcu_tbuf_ready (tcu_tbuf_ready[i]), - `endif - `ifdef TCU_META_ENABLE - .tcu_agu_ld_valid (tcu_agu_ld_valid[i]), - .tcu_agu_ld_data (tcu_agu_ld_data[i]), - .tcu_agu_ld_ready (tcu_agu_ld_ready[i]), - .tcu_agu_result_valid (tcu_agu_result_valid[i]), - .tcu_agu_result_data (tcu_agu_result_data[i]), - .tcu_agu_result_ready (tcu_agu_result_ready[i]), - .tcu_meta_wr_en (tcu_meta_wr_en && (NW_WIDTH'(tcu_meta_wr_wid % `VX_CFG_NUM_TCU_BLOCKS) == NW_WIDTH'(i))), - .tcu_meta_wr_wid (tcu_meta_wr_wid), - .tcu_meta_wr_idx (tcu_meta_wr_idx), - .tcu_meta_wr_data (tcu_meta_wr_data), `endif .warp_shared_if (warp_shared_if[i]) ); @@ -686,9 +643,6 @@ import VX_tcu_pkg::*; .lsu_perf_wr_req_fire (lsu_perf_wr_req_fire[i]), .lsu_perf_rsp_fire (lsu_perf_rsp_fire[i]), .lsu_coalescer_misses (lsu_coalescer_misses[i]), - `endif - `ifdef TCU_META_ENABLE - .lsu_tcu_mem_if (lsu_tcu_mem_if[i]), `endif .lsu_sched_empty (lsu_sched_empty[i]), .lsu_store_empty (lsu_store_empty[i]), @@ -713,18 +667,6 @@ import VX_tcu_pkg::*; .tcu_tbuf_rs1_data (tcu_tbuf_rs1_data[i]), .tcu_tbuf_rs2_data (tcu_tbuf_rs2_data[i]), .tcu_tbuf_ready (tcu_tbuf_ready[i]), - `endif - `ifdef TCU_META_ENABLE - .tcu_agu_ld_valid (tcu_agu_ld_valid[i]), - .tcu_agu_ld_data (tcu_agu_ld_data[i]), - .tcu_agu_ld_ready (tcu_agu_ld_ready[i]), - .tcu_agu_result_valid (tcu_agu_result_valid[i]), - .tcu_agu_result_data (tcu_agu_result_data[i]), - .tcu_agu_result_ready (tcu_agu_result_ready[i]), - .tcu_meta_wr_en (tcu_meta_wr_en && (NW_WIDTH'(tcu_meta_wr_wid % `VX_CFG_NUM_TCU_BLOCKS) == NW_WIDTH'(i))), - .tcu_meta_wr_wid (tcu_meta_wr_wid), - .tcu_meta_wr_idx (tcu_meta_wr_idx), - .tcu_meta_wr_data (tcu_meta_wr_data), `endif .warp_shared_if (warp_shared_if[i]), .cta_bcast (cta_bcast) @@ -782,9 +724,6 @@ import VX_tcu_pkg::*; .lsu_perf_wr_req_fire (lsu_perf_wr_req_fire), .lsu_perf_rsp_fire (lsu_perf_rsp_fire), .lsu_coalescer_misses (lsu_coalescer_misses), - `endif - `ifdef TCU_META_ENABLE - .lsu_tcu_mem_if (lsu_tcu_mem_if), `endif .lsu_sched_empty (lsu_sched_empty), .lsu_store_empty (lsu_store_empty), @@ -807,18 +746,6 @@ import VX_tcu_pkg::*; .tcu_tbuf_rs2_data (tcu_tbuf_rs2_data), .tcu_tbuf_ready (tcu_tbuf_ready), `endif - `ifdef TCU_META_ENABLE - .tcu_agu_ld_valid (tcu_agu_ld_valid), - .tcu_agu_ld_data (tcu_agu_ld_data), - .tcu_agu_ld_ready (tcu_agu_ld_ready), - .tcu_agu_result_valid (tcu_agu_result_valid), - .tcu_agu_result_data (tcu_agu_result_data), - .tcu_agu_result_ready (tcu_agu_result_ready), - .tcu_meta_wr_en (tcu_meta_wr_en), - .tcu_meta_wr_wid (tcu_meta_wr_wid), - .tcu_meta_wr_idx (tcu_meta_wr_idx), - .tcu_meta_wr_data (tcu_meta_wr_data), - `endif `ifdef VX_CFG_EXT_DXA_ENABLE .dxa_req_bus_if (block_dxa_req_bus_if), `endif diff --git a/hw/rtl/core/VX_execute.sv b/hw/rtl/core/VX_execute.sv index 84ebe921fb..330e01a254 100644 --- a/hw/rtl/core/VX_execute.sv +++ b/hw/rtl/core/VX_execute.sv @@ -33,9 +33,6 @@ import VX_tcu_pkg::*; output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_wr_req_fire, output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_rsp_fire, output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][PERF_CTR_BITS-1:0] lsu_coalescer_misses, -`endif -`ifdef TCU_META_ENABLE - VX_lsu_sched_if.slave lsu_tcu_mem_if [`VX_CFG_NUM_LSU_BLOCKS], `endif output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_sched_empty, output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_store_empty, @@ -80,18 +77,6 @@ import VX_tcu_pkg::*; input wire [`VX_CFG_NUM_TCU_BLOCKS-1:0][TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs2_data, input wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_tbuf_ready, `endif -`ifdef TCU_META_ENABLE - output wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_agu_ld_valid, - output tcu_execute_t tcu_agu_ld_data [`VX_CFG_NUM_TCU_BLOCKS], - input wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_agu_ld_ready, - input wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_agu_result_valid, - input tcu_result_t tcu_agu_result_data [`VX_CFG_NUM_TCU_BLOCKS], - output wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_agu_result_ready, - input wire tcu_meta_wr_en, - input wire [NW_WIDTH-1:0] tcu_meta_wr_wid, - input wire [4:0] tcu_meta_wr_idx, - input wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] tcu_meta_wr_data, -`endif // scheduler interfaces VX_sched_csr_if.slave sched_csr_if [`VX_CFG_NUM_SFU_BLOCKS], @@ -120,6 +105,11 @@ import VX_tcu_pkg::*; `endif `SCOPE_IO_SWITCH (SCOPE_COUNT); +`ifdef TCU_META_ENABLE + // TCU_LD memory clients of the LSU blocks' schedulers + VX_lsu_sched_if lsu_tcu_mem_if [`VX_CFG_NUM_LSU_BLOCKS](); +`endif + VX_lsu_unit #( .INSTANCE_ID (`SFORMATF(("%s-lsu", INSTANCE_ID))) ) lsu_unit ( @@ -175,16 +165,7 @@ import VX_tcu_pkg::*; .tbuf_ready (tcu_tbuf_ready), `endif `ifdef TCU_META_ENABLE - .agu_ld_valid (tcu_agu_ld_valid), - .agu_ld_data (tcu_agu_ld_data), - .agu_ld_ready (tcu_agu_ld_ready), - .agu_result_valid (tcu_agu_result_valid), - .agu_result_data (tcu_agu_result_data), - .agu_result_ready (tcu_agu_result_ready), - .meta_wr_en (tcu_meta_wr_en), - .meta_wr_wid (tcu_meta_wr_wid), - .meta_wr_idx (tcu_meta_wr_idx), - .meta_wr_data (tcu_meta_wr_data), + .tcu_mem_if (lsu_tcu_mem_if), `endif .dispatch_if (dispatch_if[EX_TCU * `VX_CFG_ISSUE_WIDTH +: `VX_CFG_ISSUE_WIDTH]), .commit_if (commit_if[EX_TCU * `VX_CFG_ISSUE_WIDTH +: `VX_CFG_ISSUE_WIDTH]) diff --git a/hw/rtl/core/VX_execute_slice.sv b/hw/rtl/core/VX_execute_slice.sv index 7436dc3573..5542697bc2 100644 --- a/hw/rtl/core/VX_execute_slice.sv +++ b/hw/rtl/core/VX_execute_slice.sv @@ -50,9 +50,6 @@ import VX_tcu_pkg::*; output wire [`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_wr_req_fire, output wire [`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_rsp_fire, output wire [PERF_CTR_BITS-1:0] lsu_coalescer_misses, -`endif -`ifdef TCU_META_ENABLE - VX_lsu_sched_if.slave lsu_tcu_mem_if, `endif output wire lsu_sched_empty, output wire lsu_store_empty, @@ -83,18 +80,6 @@ import VX_tcu_pkg::*; input wire [TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs2_data, input wire tcu_tbuf_ready, `endif -`ifdef TCU_META_ENABLE - output wire tcu_agu_ld_valid, - output tcu_execute_t tcu_agu_ld_data, - input wire tcu_agu_ld_ready, - input wire tcu_agu_result_valid, - input tcu_result_t tcu_agu_result_data, - output wire tcu_agu_result_ready, - input wire tcu_meta_wr_en, - input wire [NW_WIDTH-1:0] tcu_meta_wr_wid, - input wire [4:0] tcu_meta_wr_idx, - input wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] tcu_meta_wr_data, -`endif // SFU warp control and CSRs VX_warp_shared_if.master warp_shared_if, @@ -184,6 +169,11 @@ import VX_tcu_pkg::*; .execute_if (lsu_execute_if) ); +`ifdef TCU_META_ENABLE + // the TCU block's metadata loads go through this slice's LSU scheduler + VX_lsu_sched_if lsu_tcu_mem_if(); +`endif + VX_lsu_block #( .INSTANCE_ID (`SFORMATF(("%s-lsu%s", INSTANCE_ID, SLICE_STR))) ) lsu_block ( @@ -402,16 +392,7 @@ import VX_tcu_pkg::*; .tbuf_ready (tcu_tbuf_ready), `endif `ifdef TCU_META_ENABLE - .agu_ld_valid (tcu_agu_ld_valid), - .agu_ld_data (tcu_agu_ld_data), - .agu_ld_ready (tcu_agu_ld_ready), - .agu_result_valid (tcu_agu_result_valid), - .agu_result_data (tcu_agu_result_data), - .agu_result_ready (tcu_agu_result_ready), - .meta_wr_en (tcu_meta_wr_en), - .meta_wr_wid (tcu_meta_wr_wid), - .meta_wr_idx (tcu_meta_wr_idx), - .meta_wr_data (tcu_meta_wr_data), + .tcu_mem_if (lsu_tcu_mem_if), `endif .execute_if (tcu_execute_if[0]), .result_if (tcu_result_if[0]) diff --git a/hw/rtl/tcu/VX_tcu_agu.sv b/hw/rtl/tcu/VX_tcu_agu.sv index 70a77b8126..315917365f 100644 --- a/hw/rtl/tcu/VX_tcu_agu.sv +++ b/hw/rtl/tcu/VX_tcu_agu.sv @@ -15,113 +15,89 @@ `ifdef TCU_META_ENABLE -// Warp-level AGU for TCU_LD instructions. Each TCU block can hold one -// transaction. With one memory port the blocks share it; with one port per -// block (NUM_PORTS == BLOCK_SIZE) each block loads through its own LSU -// scheduler, so the loads of a warpgroup's warps proceed in parallel. The -// metadata write port is shared. +// A TCU block's TCU_LD unit: one warp-wide load in flight through an LSU +// scheduler client port. Response fragments are written straight into the +// block's metadata RAM, lane by lane, so nothing is buffered here. The load +// cannot overwrite metadata an older MMA still needs: TCU_LD enters the block +// in order behind that MMA, which reads the RAM when it issues. module VX_tcu_agu import VX_gpu_pkg::*, VX_tcu_pkg::*; #( parameter `STRING INSTANCE_ID = "", - parameter BLOCK_SIZE = `VX_CFG_NUM_TCU_BLOCKS, - parameter NUM_LANES = `VX_CFG_NUM_TCU_LANES, - parameter NUM_PORTS = 1 + parameter NUM_LANES = `VX_CFG_NUM_TCU_LANES ) ( - input wire clk, - input wire reset, + input wire clk, + input wire reset, - // Per-block dispatch observation. The unit feeds INST_TCU_LD - // instructions here (gated by op_type at the wrapper); other - // op_types are masked so the AGU never sees them on .valid. - input wire [BLOCK_SIZE-1:0] per_block_ld_valid, - input tcu_execute_t per_block_ld_data [BLOCK_SIZE], - output wire [BLOCK_SIZE-1:0] per_block_ld_ready, + input wire ld_valid, + input tcu_execute_t ld_data, + output wire ld_ready, - // Memory client connections (to the LSU schedulers). - VX_lsu_sched_if.master client_if [NUM_PORTS], + VX_lsu_sched_if.master client_if, - // Meta SRAM write port (broadcast — same data lands in every - // block's VX_tcu_meta so subsequent wmma_sp reads see it - // regardless of which block the warp resides on). - output wire meta_wr_en, - output wire [NW_WIDTH-1:0] meta_wr_wid, - output wire [4:0] meta_wr_idx, - output wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] meta_wr_data, + output wire meta_wr_en, + output wire [NW_WIDTH-1:0] meta_wr_wid, + output wire [4:0] meta_wr_idx, + output wire [TCU_BLOCK_CAP-1:0] meta_wr_mask, + output wire [TCU_BLOCK_CAP-1:0][31:0] meta_wr_data, - // Result_if hand for the originating block. The wrapper muxes this - // with tcu_core's normal result_if path. - output wire [BLOCK_SIZE-1:0] result_valid, - output tcu_result_t result_data [BLOCK_SIZE], - input wire [BLOCK_SIZE-1:0] result_ready + // releases the TCU_LD's metadata scoreboard bit + output wire result_valid, + output tcu_result_t result_data, + input wire result_ready ); `UNUSED_SPARAM (INSTANCE_ID) - `STATIC_ASSERT ((NUM_PORTS == 1) || (NUM_PORTS == BLOCK_SIZE), ("invalid parameter: NUM_PORTS=%0d", NUM_PORTS)) - localparam BLOCK_IDX_BITS = (BLOCK_SIZE > 1) ? $clog2(BLOCK_SIZE) : 1; + `STATIC_ASSERT (NUM_LANES == `VX_CFG_NUM_LSU_LANES, ("TCU_LD requires NUM_TCU_LANES == NUM_LSU_LANES")) - logic [BLOCK_SIZE-1:0] busy_r; - logic [BLOCK_SIZE-1:0] issued_r; - logic [BLOCK_SIZE-1:0] complete_r; - tcu_header_t header_r [BLOCK_SIZE]; - logic [4:0] slot_r [BLOCK_SIZE]; - logic [`VX_CFG_XLEN-1:0] addr_r [BLOCK_SIZE]; - logic [NUM_LANES-1:0][(LSU_WORD_SIZE*8)-1:0] - rsp_data_r [BLOCK_SIZE]; - logic [NUM_LANES-1:0] rsp_lane_done_r [BLOCK_SIZE]; - - wire [BLOCK_SIZE-1:0] req_pending = busy_r & ~issued_r; - - wire [BLOCK_SIZE-1:0] commit_pending = complete_r; - wire [BLOCK_IDX_BITS-1:0] commit_block; - wire [BLOCK_SIZE-1:0] commit_grant; - wire commit_valid; - wire commit_ready = result_ready[commit_block]; - - VX_generic_arbiter #( - .NUM_REQS (BLOCK_SIZE), - .TYPE ("R") - ) commit_arb ( - .clk (clk), - .reset (reset), - .requests (commit_pending), - .grant_index (commit_block), - .grant_onehot (commit_grant), - .grant_valid (commit_valid), - .grant_ready (commit_ready) - ); - wire commit_fire = commit_valid && commit_ready; - `UNUSED_VAR (commit_grant) - - // ----------------------------------------------------------------------- - // Memory request: address from rs1 (warp-broadcast). NUM_LANES mask - // all-ones — we fetch the metadata tile in one warp-level request. - // Lane T loads the 32-bit word at base + 4*T; the host packs h_meta so - // that word holds the metadata destined for SRAM cell - // (bank = T%PWD, col = T/PWD). One formula serves WMMA-SP and WGMMA-SP. - // ----------------------------------------------------------------------- - localparam PWD = TCU_META_PER_WARP_DEPTH; - `UNUSED_PARAM (PWD) - - // LSU bus carries LSU_WORD_SIZE-byte chunks per lane (= XLEN/8 bytes). - // The metadata layout is contiguous 32-bit words, so when XLEN > 32 - // each LSU response carries XLEN/32 logical 32-bit words and we need - // to pick the right one. LSU_W32_RATIO = bytes/4, so XLEN=64 → 2. + // An LSU word holds XLEN/32 metadata words; lane i reads word i. localparam LSU_W32_RATIO = LSU_WORD_SIZE / 4; localparam LG_W32_RATIO = (LSU_W32_RATIO > 1) ? $clog2(LSU_W32_RATIO) : 0; - // Per-lane sub-word selector: lane i reads 32-bit word i, so its half of - // the LSU word is fixed by i. - logic [NUM_LANES-1:0][`UP(LG_W32_RATIO)-1:0] half_sel_w; - always_comb begin - half_sel_w = '0; - for (int i = 0; i < NUM_LANES; i++) begin - if (LG_W32_RATIO > 0) begin - half_sel_w[i] = `UP(LG_W32_RATIO)'(i); + reg busy_r, issued_r, done_r; + tcu_header_t header_r; + reg [4:0] slot_r; + reg [`VX_CFG_XLEN-1:0] addr_r; + + wire ld_fire = ld_valid && ld_ready; + wire req_fire = client_if.req_valid && client_if.req_ready; + wire rsp_fire = client_if.rsp_valid && client_if.rsp_ready; + wire result_fire = result_valid && result_ready; + + always @(posedge clk) begin + if (reset) begin + busy_r <= 1'b0; + issued_r <= 1'b0; + done_r <= 1'b0; + end else begin + if (ld_fire) begin + busy_r <= 1'b1; + end + if (req_fire) begin + issued_r <= 1'b1; + end + if (rsp_fire && client_if.rsp_data.eop) begin + done_r <= 1'b1; + end + if (result_fire) begin + busy_r <= 1'b0; + issued_r <= 1'b0; + done_r <= 1'b0; end end + if (ld_fire) begin + header_r <= ld_data.header; + slot_r <= ld_data.op_args.tcu.fmt_d; + addr_r <= ld_data.rs1_data[0]; + end end - if (LG_W32_RATIO == 0) begin : g_half_sel_unused - `UNUSED_VAR (half_sel_w) - end + + assign ld_ready = ~busy_r; + `UNUSED_VAR (ld_data) + + // ----------------------------------------------------------------------- + // Request: one warp-wide load of NUM_LANES consecutive 32-bit words. + // ----------------------------------------------------------------------- + wire [LSU_ADDR_WIDTH-1:0] base_word_addr = LSU_ADDR_WIDTH'(addr_r >> `CLOG2(LSU_WORD_SIZE)); + `UNUSED_VAR (addr_r[`CLOG2(LSU_WORD_SIZE)-1:0]) `ifdef VX_CFG_LMEM_ENABLE // Equality on the word-address bits above the window: the base is asserted @@ -132,211 +108,75 @@ module VX_tcu_agu import VX_gpu_pkg::*, VX_tcu_pkg::*; #( localparam [LSU_ADDR_WIDTH-1:0] LMEM_WORD_BASE = LSU_ADDR_WIDTH'(`VX_CFG_XLEN'(`VX_MEM_LMEM_BASE_ADDR) >> `CLOG2(LSU_WORD_SIZE)); `STATIC_ASSERT (LSU_ADDR_WIDTH > LMEM_WORD_LSB, ("LMEM window wider than the word address space")) + wire base_is_local = (base_word_addr[LSU_ADDR_WIDTH-1:LMEM_WORD_LSB] + == LMEM_WORD_BASE[LSU_ADDR_WIDTH-1:LMEM_WORD_LSB]); +`else + wire base_is_local = 1'b0; `endif - function automatic lsu_req_data_t make_req(input logic [LSU_ADDR_WIDTH-1:0] base_word_addr, - input logic [BLOCK_IDX_BITS-1:0] block); - lsu_req_data_t r; - logic base_is_local; - `ifdef VX_CFG_LMEM_ENABLE - base_is_local = (base_word_addr[LSU_ADDR_WIDTH-1:LMEM_WORD_LSB] - == LMEM_WORD_BASE[LSU_ADDR_WIDTH-1:LMEM_WORD_LSB]); - `else - base_is_local = 1'b0; - `endif - r = '0; - r.rw = 1'b0; // load - r.mask = {NUM_LANES{1'b1}}; - for (int i = 0; i < NUM_LANES; i++) begin - mem_bus_attr_t a; - a = '0; - a.is_addr_local = base_is_local; - // base_addr is pre-advanced per TCU_LD slot by the kernel, so lane - // i reads 32-bit word i relative to it, in LSU-word units. - r.addr[i] = base_word_addr + (LSU_ADDR_WIDTH'(i) >> LG_W32_RATIO); - r.byteen[i] = {LSU_WORD_SIZE{1'b1}}; - r.data[i] = '0; - r.attr[i] = a; - end - r.tag = '0; - r.tag[BLOCK_IDX_BITS-1:0] = block; - return r; - endfunction - - `STATIC_ASSERT (LSU_CLIENT_TAG_WIDTH >= BLOCK_IDX_BITS, ("LSU client tag cannot encode TCU block")) - - // Per-block request and response events, from either port structure. - wire [BLOCK_SIZE-1:0] req_fire_b; - wire [BLOCK_SIZE-1:0] rsp_fire_b; - wire [BLOCK_SIZE-1:0] rsp_eop_b; - wire [NUM_LANES-1:0] rsp_mask_b [BLOCK_SIZE]; - wire [NUM_LANES-1:0][(LSU_WORD_SIZE*8)-1:0] rsp_data_b [BLOCK_SIZE]; - wire [`VX_CFG_XLEN-1:0] req_addr_chk [NUM_PORTS]; - wire [NUM_PORTS-1:0] req_fire_chk; - - if (NUM_PORTS == 1) begin : g_shared_port - wire [BLOCK_IDX_BITS-1:0] req_block; - wire [BLOCK_SIZE-1:0] req_grant; - wire req_valid; - - VX_generic_arbiter #( - .NUM_REQS (BLOCK_SIZE), - .TYPE ("R") - ) req_arb ( - .clk (clk), - .reset (reset), - .requests (req_pending), - .grant_index (req_block), - .grant_onehot (req_grant), - .grant_valid (req_valid), - .grant_ready (client_if[0].req_ready) - ); - `UNUSED_VAR (req_grant) - - wire [BLOCK_IDX_BITS-1:0] rsp_block = client_if[0].rsp_data.tag[BLOCK_IDX_BITS-1:0]; - wire req_fire = req_valid && client_if[0].req_ready; - wire rsp_fire = client_if[0].rsp_valid && client_if[0].rsp_ready; - - for (genvar bi = 0; bi < BLOCK_SIZE; ++bi) begin : g_ev - assign req_fire_b[bi] = req_fire && (req_block == BLOCK_IDX_BITS'(bi)); - assign rsp_fire_b[bi] = rsp_fire && (rsp_block == BLOCK_IDX_BITS'(bi)); - assign rsp_eop_b[bi] = client_if[0].rsp_data.eop; - assign rsp_mask_b[bi] = client_if[0].rsp_data.mask; - assign rsp_data_b[bi] = client_if[0].rsp_data.data; - end - - assign client_if[0].req_valid = req_valid; - assign client_if[0].req_data = make_req(LSU_ADDR_WIDTH'(addr_r[req_block] >> `CLOG2(LSU_WORD_SIZE)), req_block); - assign client_if[0].rsp_ready = busy_r[rsp_block] && issued_r[rsp_block] - && !complete_r[rsp_block]; - assign req_addr_chk[0] = addr_r[req_block]; - assign req_fire_chk[0] = req_fire; - `UNUSED_VAR (client_if[0].rsp_data.sop) - end else begin : g_port_per_block - for (genvar bi = 0; bi < BLOCK_SIZE; ++bi) begin : g_port - assign client_if[bi].req_valid = req_pending[bi]; - assign client_if[bi].req_data = make_req(LSU_ADDR_WIDTH'(addr_r[bi] >> `CLOG2(LSU_WORD_SIZE)), BLOCK_IDX_BITS'(0)); - assign client_if[bi].rsp_ready = busy_r[bi] && issued_r[bi] && !complete_r[bi]; - - assign req_fire_b[bi] = req_pending[bi] && client_if[bi].req_ready; - assign rsp_fire_b[bi] = client_if[bi].rsp_valid && client_if[bi].rsp_ready; - assign rsp_eop_b[bi] = client_if[bi].rsp_data.eop; - assign rsp_mask_b[bi] = client_if[bi].rsp_data.mask; - assign rsp_data_b[bi] = client_if[bi].rsp_data.data; - assign req_addr_chk[bi] = addr_r[bi]; - assign req_fire_chk[bi] = req_fire_b[bi]; - `UNUSED_VAR (client_if[bi].rsp_data.sop) - `UNUSED_VAR (client_if[bi].rsp_data.tag) - end + mem_bus_attr_t req_attr; + always_comb begin + req_attr = '0; + req_attr.is_addr_local = base_is_local; end - always @(posedge clk) begin - if (reset) begin - busy_r <= '0; - issued_r <= '0; - complete_r <= '0; - for (int bi = 0; bi < BLOCK_SIZE; ++bi) begin - header_r[bi] <= '0; - slot_r[bi] <= '0; - addr_r[bi] <= '0; - rsp_data_r[bi] <= '0; - rsp_lane_done_r[bi] <= '0; - end - end else begin - for (int bi = 0; bi < BLOCK_SIZE; ++bi) begin - if (per_block_ld_valid[bi] && per_block_ld_ready[bi]) begin - busy_r[bi] <= 1'b1; - issued_r[bi] <= 1'b0; - complete_r[bi] <= 1'b0; - header_r[bi] <= per_block_ld_data[bi].header; - slot_r[bi] <= per_block_ld_data[bi].op_args.tcu.fmt_d; - addr_r[bi] <= per_block_ld_data[bi].rs1_data[0]; - rsp_data_r[bi] <= '0; - rsp_lane_done_r[bi] <= '0; - end - if (req_fire_b[bi]) begin - issued_r[bi] <= 1'b1; - end - if (rsp_fire_b[bi]) begin - for (int t = 0; t < NUM_LANES; ++t) begin - if (rsp_mask_b[bi][t]) begin - rsp_data_r[bi][t] <= rsp_data_b[bi][t]; - rsp_lane_done_r[bi][t] <= 1'b1; - end - end - if (rsp_eop_b[bi] - && ((rsp_lane_done_r[bi] | rsp_mask_b[bi]) == {NUM_LANES{1'b1}})) begin - complete_r[bi] <= 1'b1; - end - end - end - if (commit_fire) begin - busy_r[commit_block] <= 1'b0; - issued_r[commit_block] <= 1'b0; - complete_r[commit_block] <= 1'b0; - rsp_lane_done_r[commit_block] <= '0; - end + lsu_req_data_t req_data; + always_comb begin + req_data = '0; + req_data.rw = 1'b0; + req_data.mask = {NUM_LANES{1'b1}}; + for (int i = 0; i < NUM_LANES; ++i) begin + req_data.addr[i] = base_word_addr + (LSU_ADDR_WIDTH'(i) >> LG_W32_RATIO); + req_data.byteen[i] = {LSU_WORD_SIZE{1'b1}}; + req_data.attr[i] = req_attr; end end - for (genvar bi = 0; bi < BLOCK_SIZE; ++bi) begin : g_ld_ready - assign per_block_ld_ready[bi] = ~busy_r[bi]; - end + assign client_if.req_valid = busy_r && ~issued_r; + assign client_if.req_data = req_data; + assign client_if.rsp_ready = 1'b1; + `UNUSED_VAR (client_if.rsp_data.sop) + `UNUSED_VAR (client_if.rsp_data.tag) + + `RUNTIME_ASSERT(~client_if.rsp_valid || (issued_r && ~done_r), + ("%t: *** %s unexpected TCU_LD response", $time, INSTANCE_ID)) `ifdef VX_CFG_LSU_STACK_INTERLEAVE_ENABLE // Metadata is read as a linear warp-wide tile, which an interleaved // per-thread stack cannot provide. if (`VX_CFG_NUM_THREADS > 1) begin : g_meta_stack_check - for (genvar pi = 0; pi < NUM_PORTS; ++pi) begin : g_port - wire [`VX_CFG_XLEN-1:0] meta_end_addr = req_addr_chk[pi] + `VX_CFG_XLEN'(NUM_LANES * 4 - 1); - `RUNTIME_ASSERT(~req_fire_chk[pi] || (meta_end_addr < STACK_WINDOW_BOTTOM) || (req_addr_chk[pi] >= STACK_WINDOW_TOP), - ("%t: *** %s TCU metadata inside the thread stack window: addr=0x%0h", $time, INSTANCE_ID, req_addr_chk[pi])) - end + wire [`VX_CFG_XLEN-1:0] meta_end_addr = addr_r + `VX_CFG_XLEN'(NUM_LANES * 4 - 1); + `RUNTIME_ASSERT(~req_fire || (meta_end_addr < STACK_WINDOW_BOTTOM) || (addr_r >= STACK_WINDOW_TOP), + ("%t: *** %s TCU metadata inside the thread stack window: addr=0x%0h", $time, INSTANCE_ID, addr_r)) end -`else - `UNUSED_VAR (req_addr_chk) - `UNUSED_VAR (req_fire_chk) `endif // ----------------------------------------------------------------------- - // VX_tcu_meta write (broadcast to all blocks' meta instances). + // Metadata RAM write: each response fragment writes its own lanes. // ----------------------------------------------------------------------- - assign meta_wr_en = commit_valid; - assign meta_wr_wid = header_r[commit_block].wid; - assign meta_wr_idx = slot_r[commit_block]; - // Repack response data into TCU_BLOCK_CAP × XLEN. The downstream - // meta SRAM consumes only the low 32 bits, so for XLEN=64 we mux the - // half of the LSU response that holds this lane's logical 32-bit word. - for (genvar i = 0; i < TCU_BLOCK_CAP; ++i) begin : g_meta_data - if (i < NUM_LANES) begin : g_in - if (LG_W32_RATIO == 0) begin : g_w32_passthru - assign meta_wr_data[i] = `VX_CFG_XLEN'(rsp_data_r[commit_block][i]); - end else begin : g_w32_pick - wire [LSU_W32_RATIO-1:0][31:0] rsp_w32_v = rsp_data_r[commit_block][i]; - assign meta_wr_data[i] = `VX_CFG_XLEN'(rsp_w32_v[half_sel_w[i]]); - end + assign meta_wr_en = rsp_fire; + assign meta_wr_wid = header_r.wid; + assign meta_wr_idx = slot_r; + + for (genvar i = 0; i < TCU_BLOCK_CAP; ++i) begin : g_meta_wr + if (i < NUM_LANES) begin : g_lane + wire [LSU_W32_RATIO-1:0][31:0] rsp_w32 = client_if.rsp_data.data[i]; + assign meta_wr_mask[i] = client_if.rsp_data.mask[i]; + assign meta_wr_data[i] = rsp_w32[i % LSU_W32_RATIO]; + `UNUSED_VAR (rsp_w32) end else begin : g_pad + assign meta_wr_mask[i] = 1'b0; assign meta_wr_data[i] = '0; end end // ----------------------------------------------------------------------- - // Result drive (commits to scoreboard via writeback, releasing - // wr_xregs[0]). Only the owner block's result_if fires. - // tcu_result_t = { header, data[NUM_LANES][XLEN] }. - // The header is propagated from the latched execute_t header — it - // already carries wr_xregs and wb=0 from the decoder. + // Result: no register writeback; its header releases the scoreboard bit. // ----------------------------------------------------------------------- - tcu_result_t commit_data_w; + assign result_valid = done_r; always_comb begin - commit_data_w = '0; - commit_data_w.header = header_r[commit_block]; - commit_data_w.data = '0; - end - - for (genvar bi = 0; bi < BLOCK_SIZE; ++bi) begin : g_result - assign result_valid[bi] = commit_valid && (commit_block == BLOCK_IDX_BITS'(bi)); - assign result_data[bi] = commit_data_w; + result_data = '0; + result_data.header = header_r; end endmodule diff --git a/hw/rtl/tcu/VX_tcu_block.sv b/hw/rtl/tcu/VX_tcu_block.sv index ee6e19a637..1e493f9d83 100644 --- a/hw/rtl/tcu/VX_tcu_block.sv +++ b/hw/rtl/tcu/VX_tcu_block.sv @@ -13,8 +13,9 @@ `include "VX_define.vh" -// One TCU block: the MMA datapath, with TCU_LD handed to the core-wide AGU and -// WGMMA operands supplied by the core-wide tile buffers (VX_tcu_shared). +// One TCU block: the MMA datapath and the TCU_LD unit that fills its metadata +// RAM, with WGMMA operands supplied by the core-wide tile buffers +// (VX_tcu_shared). module VX_tcu_block import VX_gpu_pkg::*, VX_tcu_pkg::*; #( parameter `STRING INSTANCE_ID = "", parameter `STRING BLOCK_STR = "" @@ -35,17 +36,8 @@ module VX_tcu_block import VX_gpu_pkg::*, VX_tcu_pkg::*; #( `endif `ifdef TCU_META_ENABLE - // TCU_LD to and from the AGU - output wire agu_ld_valid, - output tcu_execute_t agu_ld_data, - input wire agu_ld_ready, - input wire agu_result_valid, - input tcu_result_t agu_result_data, - output wire agu_result_ready, - input wire meta_wr_en, - input wire [NW_WIDTH-1:0] meta_wr_wid, - input wire [4:0] meta_wr_idx, - input wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] meta_wr_data, + // TCU_LD memory client of an LSU scheduler + VX_lsu_sched_if.master tcu_mem_if, `endif VX_execute_if.slave execute_if, @@ -63,11 +55,38 @@ module VX_tcu_block import VX_gpu_pkg::*, VX_tcu_pkg::*; #( ) core_result_if(); `ifdef TCU_META_ENABLE + wire agu_ld_valid, agu_ld_ready; + wire agu_result_valid, agu_result_ready; + tcu_result_t agu_result_data; + wire meta_wr_en; + wire [NW_WIDTH-1:0] meta_wr_wid; + wire [4:0] meta_wr_idx; + wire [TCU_BLOCK_CAP-1:0] meta_wr_mask; + wire [TCU_BLOCK_CAP-1:0][31:0] meta_wr_data; + + VX_tcu_agu #( + .INSTANCE_ID (`SFORMATF(("%s-agu%s", INSTANCE_ID, BLOCK_STR))) + ) agu ( + .clk (clk), + .reset (reset), + .ld_valid (agu_ld_valid), + .ld_data (execute_if.data), + .ld_ready (agu_ld_ready), + .client_if (tcu_mem_if), + .meta_wr_en (meta_wr_en), + .meta_wr_wid (meta_wr_wid), + .meta_wr_idx (meta_wr_idx), + .meta_wr_mask (meta_wr_mask), + .meta_wr_data (meta_wr_data), + .result_valid (agu_result_valid), + .result_data (agu_result_data), + .result_ready (agu_result_ready) + ); + // TCU_LD goes to the AGU, every MMA op to the core wire is_tcu_ld = (execute_if.data.op_type == INST_TCU_LD); assign agu_ld_valid = execute_if.valid && is_tcu_ld; - assign agu_ld_data = execute_if.data; assign core_execute_if.valid = execute_if.valid && !is_tcu_ld; assign core_execute_if.data = execute_if.data; assign execute_if.ready = is_tcu_ld ? agu_ld_ready : core_execute_if.ready; @@ -112,6 +131,7 @@ module VX_tcu_block import VX_gpu_pkg::*, VX_tcu_pkg::*; #( .ext_meta_wr_en (meta_wr_en), .ext_meta_wr_wid (meta_wr_wid), .ext_meta_wr_idx (meta_wr_idx), + .ext_meta_wr_mask (meta_wr_mask), .ext_meta_wr_data (meta_wr_data), `endif .execute_if (core_execute_if), diff --git a/hw/rtl/tcu/VX_tcu_core.sv b/hw/rtl/tcu/VX_tcu_core.sv index e4fc89786f..b70a8e3a50 100644 --- a/hw/rtl/tcu/VX_tcu_core.sv +++ b/hw/rtl/tcu/VX_tcu_core.sv @@ -27,12 +27,13 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( input wire tbuf_ready, `endif - // External metadata write port from the shared VX_tcu_agu. + // Metadata write port from the block's VX_tcu_agu. `ifdef TCU_META_ENABLE input wire ext_meta_wr_en, input wire [NW_WIDTH-1:0] ext_meta_wr_wid, input wire [4:0] ext_meta_wr_idx, - input wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] ext_meta_wr_data, + input wire [TCU_BLOCK_CAP-1:0] ext_meta_wr_mask, + input wire [TCU_BLOCK_CAP-1:0][31:0] ext_meta_wr_data, `endif // Inputs @@ -318,6 +319,7 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( .wr_en (ext_meta_wr_en), .wr_wid (ext_meta_wr_wid), .wr_idx (ext_meta_wr_idx), + .wr_mask(ext_meta_wr_mask), .wr_data(ext_meta_wr_data), .rd_wid (execute_if.data.header.wid) `ifdef VX_CFG_TCU_SPARSE_ENABLE diff --git a/hw/rtl/tcu/VX_tcu_meta.sv b/hw/rtl/tcu/VX_tcu_meta.sv index 8a31ca6bad..68bf326390 100644 --- a/hw/rtl/tcu/VX_tcu_meta.sv +++ b/hw/rtl/tcu/VX_tcu_meta.sv @@ -19,7 +19,7 @@ // the MX scale-factor stores (A/B) behind a single warp-indexed block RAM. // // All regions share one warp-indexed address space (SIZE = NUM_WARPS) and the -// one broadcast write port from VX_tcu_agu. wr_idx[4] selects the namespace +// one write port from the block's VX_tcu_agu. wr_idx[4] selects the namespace // (0 = sparse, 1 = MX); within MX wr_idx[0] selects the axis (0 = A, 1 = B); // within sparse wr_idx[3:0] is the column-group index. This matches the // TCU_LD destination-register encoding emitted by vx_tensor.h (x0/x1 = SP, @@ -39,11 +39,12 @@ module VX_tcu_meta import VX_gpu_pkg::*, VX_tcu_pkg::*; #( input wire clk, input wire reset, - // Unified broadcast write port (from VX_tcu_agu). + // Write port from the block's VX_tcu_agu; wr_mask selects the lanes written. input wire wr_en, input wire [NW_WIDTH-1:0] wr_wid, input wire [4:0] wr_idx, - input wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] wr_data, + input wire [TCU_BLOCK_CAP-1:0] wr_mask, + input wire [TCU_BLOCK_CAP-1:0][31:0] wr_data, // Unified read port (warp id from the FEDP path). input wire [NW_WIDTH-1:0] rd_wid @@ -162,15 +163,11 @@ module VX_tcu_meta import VX_gpu_pkg::*, VX_tcu_pkg::*; #( for (genvar b = 0; b < PER_WARP_DEPTH; ++b) begin : g_meta_banks for (genvar c = 0; c < NUM_COLS; ++c) begin : g_col localparam int COL = b * NUM_COLS + c; - if (STORES_PER_COL > 1) begin : g_wr - assign merged_wdata[COL * 32 +: 32] = 32'(wr_data[b % BANKS_PER_STORE]); - end else begin : g_wr - // Thread offset for column c within its group: (c % CPL) * PER_WARP_DEPTH + b - localparam int COL_IN_GROUP = c % COLS_PER_LOAD; - localparam int THREAD_OFF = COL_IN_GROUP * PER_WARP_DEPTH + b; - assign merged_wdata[COL * 32 +: 32] = 32'(wr_data[THREAD_OFF]); - end - assign merged_wren[COL] = sp_wr && col_wren[c] && meta_wr_bank_en[b]; + // Thread offset for column c within its group: (c % CPL) * PER_WARP_DEPTH + b + localparam int THREAD_OFF = (STORES_PER_COL > 1) ? (b % BANKS_PER_STORE) + : ((c % COLS_PER_LOAD) * PER_WARP_DEPTH + b); + assign merged_wdata[COL * 32 +: 32] = wr_data[THREAD_OFF]; + assign merged_wren[COL] = sp_wr && col_wren[c] && meta_wr_bank_en[b] && wr_mask[THREAD_OFF]; end end @@ -196,10 +193,10 @@ module VX_tcu_meta import VX_gpu_pkg::*, VX_tcu_pkg::*; #( wire mx_wr_b = wr_en && wr_idx[4] && wr_idx[0]; for (genvar i = 0; i < TCU_BLOCK_CAP; ++i) begin : g_mx_pack - assign merged_wdata[(MXA_COL_BASE + i) * 32 +: 32] = 32'(wr_data[i]); - assign merged_wdata[(MXB_COL_BASE + i) * 32 +: 32] = 32'(wr_data[i]); - assign merged_wren[MXA_COL_BASE + i] = mx_wr_a; - assign merged_wren[MXB_COL_BASE + i] = mx_wr_b; + assign merged_wdata[(MXA_COL_BASE + i) * 32 +: 32] = wr_data[i]; + assign merged_wdata[(MXB_COL_BASE + i) * 32 +: 32] = wr_data[i]; + assign merged_wren[MXA_COL_BASE + i] = mx_wr_a && wr_mask[i]; + assign merged_wren[MXB_COL_BASE + i] = mx_wr_b && wr_mask[i]; end assign meta_a = merged_rdata[(MXA_COL_BASE * 32) +: (TCU_BLOCK_CAP * 32)]; assign meta_b = merged_rdata[(MXB_COL_BASE * 32) +: (TCU_BLOCK_CAP * 32)]; diff --git a/hw/rtl/tcu/VX_tcu_shared.sv b/hw/rtl/tcu/VX_tcu_shared.sv index 4fee552c69..0b232fa3f4 100644 --- a/hw/rtl/tcu/VX_tcu_shared.sv +++ b/hw/rtl/tcu/VX_tcu_shared.sv @@ -14,7 +14,7 @@ `include "VX_define.vh" // TCU state shared by every TCU block: the WGMMA tile buffers (a warpgroup's -// B operand is common to its blocks) and the warp-level metadata AGU. +// B operand is common to its blocks). module VX_tcu_shared import VX_gpu_pkg::*, VX_tcu_pkg::*; #( parameter `STRING INSTANCE_ID = "", parameter BLOCK_SIZE = `VX_CFG_NUM_TCU_BLOCKS @@ -34,31 +34,14 @@ module VX_tcu_shared import VX_gpu_pkg::*, VX_tcu_pkg::*; #( VX_mem_bus_if.master tcu_lmem_if, `endif -`ifdef TCU_META_ENABLE - input wire [BLOCK_SIZE-1:0] agu_ld_valid, - input tcu_execute_t agu_ld_data [BLOCK_SIZE], - output wire [BLOCK_SIZE-1:0] agu_ld_ready, - output wire [BLOCK_SIZE-1:0] agu_result_valid, - output tcu_result_t agu_result_data [BLOCK_SIZE], - input wire [BLOCK_SIZE-1:0] agu_result_ready, - output wire meta_wr_en, - output wire [NW_WIDTH-1:0] meta_wr_wid, - output wire [4:0] meta_wr_idx, - output wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] meta_wr_data, - // TCU_LD memory clients, one per LSU block's scheduler - VX_lsu_sched_if.master tcu_mem_if [`VX_CFG_NUM_LSU_BLOCKS], -`endif - input wire clk, input wire reset ); `UNUSED_SPARAM (INSTANCE_ID) `ifndef VX_CFG_TCU_WGMMA_ENABLE -`ifndef TCU_META_ENABLE `UNUSED_PARAM (BLOCK_SIZE) `UNUSED_VAR ({clk, reset}) `endif -`endif `ifdef VX_CFG_TCU_WGMMA_ENABLE VX_tcu_wgmma #( @@ -88,55 +71,4 @@ module VX_tcu_shared import VX_gpu_pkg::*, VX_tcu_pkg::*; #( `endif `endif -`ifdef TCU_META_ENABLE - // The AGU streams NUM_TCU_LANES operands over the LSU memory client, whose - // request/response mask and data are sized by NUM_LSU_LANES; the two lane - // counts must match for the per-lane completion tracking to be correct. - `STATIC_ASSERT (`VX_CFG_NUM_TCU_LANES == `VX_CFG_NUM_LSU_LANES, ("VX_tcu_agu requires NUM_TCU_LANES == NUM_LSU_LANES")); - - // With one LSU block per TCU block, each block loads its metadata - // through the scheduler of the LSU block that serves the same warps; - // otherwise every block shares LSU block 0's scheduler. - localparam AGU_PORTS = (`VX_CFG_NUM_LSU_BLOCKS == BLOCK_SIZE) ? BLOCK_SIZE : 1; - for (genvar b = AGU_PORTS; b < `VX_CFG_NUM_LSU_BLOCKS; ++b) begin : g_mem_tieoff - assign tcu_mem_if[b].req_valid = 1'b0; - assign tcu_mem_if[b].req_data = '0; - assign tcu_mem_if[b].rsp_ready = 1'b1; - `UNUSED_VAR (tcu_mem_if[b].req_ready) - `UNUSED_VAR (tcu_mem_if[b].rsp_valid) - `UNUSED_VAR (tcu_mem_if[b].rsp_data) - end - - VX_lsu_sched_if agu_mem_if [AGU_PORTS](); - for (genvar b = 0; b < AGU_PORTS; ++b) begin : g_mem_port - assign tcu_mem_if[b].req_valid = agu_mem_if[b].req_valid; - assign tcu_mem_if[b].req_data = agu_mem_if[b].req_data; - assign agu_mem_if[b].req_ready = tcu_mem_if[b].req_ready; - assign agu_mem_if[b].rsp_valid = tcu_mem_if[b].rsp_valid; - assign agu_mem_if[b].rsp_data = tcu_mem_if[b].rsp_data; - assign tcu_mem_if[b].rsp_ready = agu_mem_if[b].rsp_ready; - end - - VX_tcu_agu #( - .INSTANCE_ID (`SFORMATF(("%s-agu", INSTANCE_ID))), - .BLOCK_SIZE (BLOCK_SIZE), - .NUM_LANES (`VX_CFG_NUM_TCU_LANES), - .NUM_PORTS (AGU_PORTS) - ) agu ( - .clk (clk), - .reset (reset), - .per_block_ld_valid (agu_ld_valid), - .per_block_ld_data (agu_ld_data), - .per_block_ld_ready (agu_ld_ready), - .client_if (agu_mem_if), - .meta_wr_en (meta_wr_en), - .meta_wr_wid (meta_wr_wid), - .meta_wr_idx (meta_wr_idx), - .meta_wr_data (meta_wr_data), - .result_valid (agu_result_valid), - .result_data (agu_result_data), - .result_ready (agu_result_ready) - ); -`endif - endmodule diff --git a/hw/rtl/tcu/VX_tcu_unit.sv b/hw/rtl/tcu/VX_tcu_unit.sv index 91dd5fd292..7d1a5250c2 100644 --- a/hw/rtl/tcu/VX_tcu_unit.sv +++ b/hw/rtl/tcu/VX_tcu_unit.sv @@ -32,17 +32,8 @@ module VX_tcu_unit import VX_gpu_pkg::*, VX_tcu_pkg::*; #( `endif `ifdef TCU_META_ENABLE - // per-block ports of the core-wide metadata AGU (VX_tcu_shared) - output wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] agu_ld_valid, - output tcu_execute_t agu_ld_data [`VX_CFG_NUM_TCU_BLOCKS], - input wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] agu_ld_ready, - input wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] agu_result_valid, - input tcu_result_t agu_result_data [`VX_CFG_NUM_TCU_BLOCKS], - output wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] agu_result_ready, - input wire meta_wr_en, - input wire [NW_WIDTH-1:0] meta_wr_wid, - input wire [4:0] meta_wr_idx, - input wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] meta_wr_data, + // TCU_LD memory clients of the LSU blocks' schedulers + VX_lsu_sched_if.master tcu_mem_if [`VX_CFG_NUM_LSU_BLOCKS], `endif // Inputs @@ -78,11 +69,77 @@ module VX_tcu_unit import VX_gpu_pkg::*, VX_tcu_pkg::*; #( .data_t (tcu_result_t) ) per_block_result_if[BLOCK_SIZE](); +`ifdef TCU_META_ENABLE + VX_lsu_sched_if block_mem_if [BLOCK_SIZE](); + + if (`VX_CFG_NUM_LSU_BLOCKS == BLOCK_SIZE) begin : g_mem_per_block + // each TCU block loads through the LSU block serving the same warps + for (genvar b = 0; b < BLOCK_SIZE; ++b) begin : g_port + assign tcu_mem_if[b].req_valid = block_mem_if[b].req_valid; + assign tcu_mem_if[b].req_data = block_mem_if[b].req_data; + assign block_mem_if[b].req_ready = tcu_mem_if[b].req_ready; + assign block_mem_if[b].rsp_valid = tcu_mem_if[b].rsp_valid; + assign block_mem_if[b].rsp_data = tcu_mem_if[b].rsp_data; + assign tcu_mem_if[b].rsp_ready = block_mem_if[b].rsp_ready; + end + end else begin : g_mem_shared + // the TCU blocks share LSU block 0's client, tagged by block + localparam BLOCK_IDX_BITS = `CLOG2(BLOCK_SIZE); + `STATIC_ASSERT (LSU_CLIENT_TAG_WIDTH >= BLOCK_IDX_BITS, ("LSU client tag cannot encode TCU block")) + + wire [BLOCK_SIZE-1:0] req_valids; + wire [`UP(BLOCK_IDX_BITS)-1:0] req_idx; + wire [BLOCK_SIZE-1:0] req_grant; + wire req_valid; + lsu_req_data_t req_datas [BLOCK_SIZE]; + for (genvar b = 0; b < BLOCK_SIZE; ++b) begin : g_req + assign req_valids[b] = block_mem_if[b].req_valid; + assign req_datas[b] = block_mem_if[b].req_data; + assign block_mem_if[b].req_ready = tcu_mem_if[0].req_ready && req_grant[b]; + end + + VX_generic_arbiter #( + .NUM_REQS (BLOCK_SIZE), + .TYPE ("R") + ) req_arb ( + .clk (clk), + .reset (reset), + .requests (req_valids), + .grant_index (req_idx), + .grant_onehot (req_grant), + .grant_valid (req_valid), + .grant_ready (tcu_mem_if[0].req_ready) + ); + + lsu_req_data_t req_data; + always_comb begin + req_data = req_datas[req_idx]; + req_data.tag[`UP(BLOCK_IDX_BITS)-1:0] = req_idx; + end + assign tcu_mem_if[0].req_valid = req_valid; + assign tcu_mem_if[0].req_data = req_data; + + wire [`UP(BLOCK_IDX_BITS)-1:0] rsp_idx = tcu_mem_if[0].rsp_data.tag[`UP(BLOCK_IDX_BITS)-1:0]; + wire [BLOCK_SIZE-1:0] rsp_readys; + for (genvar b = 0; b < BLOCK_SIZE; ++b) begin : g_rsp + assign block_mem_if[b].rsp_valid = tcu_mem_if[0].rsp_valid && (rsp_idx == `UP(BLOCK_IDX_BITS)'(b)); + assign block_mem_if[b].rsp_data = tcu_mem_if[0].rsp_data; + assign rsp_readys[b] = block_mem_if[b].rsp_ready; + end + assign tcu_mem_if[0].rsp_ready = rsp_readys[rsp_idx]; + + for (genvar b = 1; b < `VX_CFG_NUM_LSU_BLOCKS; ++b) begin : g_tieoff + assign tcu_mem_if[b].req_valid = 1'b0; + assign tcu_mem_if[b].req_data = '0; + assign tcu_mem_if[b].rsp_ready = 1'b1; + `UNUSED_VAR (tcu_mem_if[b].req_ready) + `UNUSED_VAR (tcu_mem_if[b].rsp_valid) + `UNUSED_VAR (tcu_mem_if[b].rsp_data) + end + end +`endif + for (genvar block_idx = 0; block_idx < BLOCK_SIZE; ++block_idx) begin : g_blocks - `ifdef TCU_META_ENABLE - wire block_meta_wr_en = meta_wr_en - && (NW_WIDTH'(meta_wr_wid % `VX_CFG_NUM_TCU_BLOCKS) == NW_WIDTH'(block_idx)); - `endif VX_tcu_block #( .INSTANCE_ID (INSTANCE_ID), @@ -100,16 +157,7 @@ module VX_tcu_unit import VX_gpu_pkg::*, VX_tcu_pkg::*; #( .tbuf_ready (tbuf_ready[block_idx]), `endif `ifdef TCU_META_ENABLE - .agu_ld_valid (agu_ld_valid[block_idx]), - .agu_ld_data (agu_ld_data[block_idx]), - .agu_ld_ready (agu_ld_ready[block_idx]), - .agu_result_valid (agu_result_valid[block_idx]), - .agu_result_data (agu_result_data[block_idx]), - .agu_result_ready (agu_result_ready[block_idx]), - .meta_wr_en (block_meta_wr_en), - .meta_wr_wid (meta_wr_wid), - .meta_wr_idx (meta_wr_idx), - .meta_wr_data (meta_wr_data), + .tcu_mem_if (block_mem_if[block_idx]), `endif .execute_if (per_block_execute_if[block_idx]), .result_if (per_block_result_if[block_idx]) diff --git a/sim/simx/lsu_unit.h b/sim/simx/lsu_unit.h index 6030fc0a39..cb81581e53 100644 --- a/sim/simx/lsu_unit.h +++ b/sim/simx/lsu_unit.h @@ -29,7 +29,7 @@ // TCU_LD memory ports: one per TCU block, through the LSU block that serves // the same warps, when the LSU has one block per TCU block; otherwise all -// TCU blocks share LSU block 0. Same derivation as the RTL's AGU_PORTS. +// TCU blocks share LSU block 0, as in the RTL (VX_tcu_unit). #ifndef TCU_AGU_PORTS #define TCU_AGU_PORTS (uint32_t((VX_CFG_NUM_LSU_BLOCKS == VX_CFG_NUM_TCU_BLOCKS) ? VX_CFG_NUM_TCU_BLOCKS : 1)) #endif diff --git a/sim/simx/tcu/tcu_unit.cpp b/sim/simx/tcu/tcu_unit.cpp index 0e88015cb3..f5e33c6984 100644 --- a/sim/simx/tcu/tcu_unit.cpp +++ b/sim/simx/tcu/tcu_unit.cpp @@ -412,8 +412,8 @@ class TcuUnit::Impl { } #ifdef TCU_META_ENABLE - // Each TCU block tracks one metadata transaction. Requests share the LSU - // client port, while tags allow their memory latency to overlap. + // Each TCU block's AGU tracks one metadata load and writes each returned + // lane straight into the block's metadata SRAM. static constexpr uint32_t kAguBeats = (VX_CFG_NUM_THREADS + VX_CFG_NUM_LSU_LANES - 1) / VX_CFG_NUM_LSU_LANES; @@ -426,7 +426,6 @@ class TcuUnit::Impl { uint32_t next_beat = 0; uint32_t rsp_count = 0; std::array addrs{}; - std::array words{}; }; std::array agu_; uint32_t agu_issue_rr_ = 0; @@ -485,11 +484,10 @@ class TcuUnit::Impl { uint32_t off = agu.addrs.at(idx) & (VX_CFG_MEM_BLOCK_SIZE - 1); uint32_t word = 0; std::memcpy(&word, rsp.data.at(lane)->data() + off, 4); - agu.words.at(idx) = word; + this->agu_write(agu, idx, word); ++agu.rsp_count; } if (agu.busy && !agu.complete && agu.rsp_count == VX_CFG_NUM_THREADS) { - this->agu_deposit(agu); agu.complete = true; } } @@ -531,15 +529,13 @@ class TcuUnit::Impl { } } - // Deposit the fetched words into the selected metadata SRAM. - void agu_deposit(const agu_state_t& agu) { + // Write lane T's fetched word into the selected metadata SRAM. + void agu_write(const agu_state_t& agu, uint32_t T, uint32_t word) { uint32_t wid = agu.wid; #ifdef VX_CFG_TCU_MX_ENABLE if (agu.selector & 0x10) { auto& dst = (agu.selector & 1) ? mx_meta_b_.at(wid) : mx_meta_a_.at(wid); - for (uint32_t lane = 0; lane < VX_CFG_NUM_THREADS; ++lane) { - dst.at(lane) = agu.words.at(lane); - } + dst.at(T) = word; return; } #endif @@ -554,26 +550,25 @@ class TcuUnit::Impl { constexpr uint32_t BPS = cfg::banks_per_store; constexpr uint32_t SPC = cfg::stores_per_col; constexpr uint32_t CPL = cfg::meta_cols_per_load; - for (uint32_t T = 0; T < VX_CFG_NUM_THREADS; ++T) { - uint32_t store_in_load = T / BPS; - uint32_t thread_in_store = T % BPS; - uint32_t flat_store = slot_idx * CPL + store_in_load; - uint32_t col = flat_store / SPC; - uint32_t store_in_col = flat_store % SPC; - uint32_t bank = store_in_col * BPS + thread_in_store; - if (bank >= PWD || col >= kMaxMetaCols) { - continue; - } - uint32_t word = agu.words.at(T); - sparse_meta_.at(wid).at(bank * kMaxMetaCols + col) = word; - // Trace: META_SRAM write (CSV: wid,bank,col,addr,value). - if (const char* p = std::getenv("VORTEX_TCU_TRACE")) { - if (p[0] == '1') { - fprintf(stderr, "META_TRC,%u,%u,%u,0x%lx,0x%08x\n", - wid, bank, col, (unsigned long)agu.addrs.at(T), word); - } + uint32_t store_in_load = T / BPS; + uint32_t thread_in_store = T % BPS; + uint32_t flat_store = slot_idx * CPL + store_in_load; + uint32_t col = flat_store / SPC; + uint32_t store_in_col = flat_store % SPC; + uint32_t bank = store_in_col * BPS + thread_in_store; + if (bank >= PWD || col >= kMaxMetaCols) { + return; + } + sparse_meta_.at(wid).at(bank * kMaxMetaCols + col) = word; + // Trace: META_SRAM write (CSV: wid,bank,col,addr,value). + if (const char* p = std::getenv("VORTEX_TCU_TRACE")) { + if (p[0] == '1') { + fprintf(stderr, "META_TRC,%u,%u,%u,0x%lx,0x%08x\n", + wid, bank, col, (unsigned long)agu.addrs.at(T), word); } } + #else + __unused(T, word); #endif } #endif // TCU_META_ENABLE diff --git a/sim/simx/tcu/tcu_unit.h b/sim/simx/tcu/tcu_unit.h index 681c448b5c..6a4e0e7215 100644 --- a/sim/simx/tcu/tcu_unit.h +++ b/sim/simx/tcu/tcu_unit.h @@ -30,7 +30,7 @@ // TCU_LD memory ports: one per TCU block, through the LSU block that serves // the same warps, when the LSU has one block per TCU block; otherwise all -// TCU blocks share LSU block 0. Same derivation as the RTL's AGU_PORTS. +// TCU blocks share LSU block 0, as in the RTL (VX_tcu_unit). #ifndef TCU_AGU_PORTS #define TCU_AGU_PORTS (uint32_t((VX_CFG_NUM_LSU_BLOCKS == VX_CFG_NUM_TCU_BLOCKS) ? VX_CFG_NUM_TCU_BLOCKS : 1)) #endif From 365895a619e1faff4bc43779251718c22b710b2e Mon Sep 17 00:00:00 2001 From: tinebp Date: Thu, 22 Oct 2026 18:09:12 -0700 Subject: [PATCH 089/124] core: drop the live-warps CSR bits above XLEN explicitly The active-warps CSR is XLEN wide, so with more warps than XLEN bits it holds warps 0..XLEN-1 only, as SimX already reports. The implicit truncation left the upper bits of sched_csr_if.active_warps unread, and lint rejected every build with NUM_WARPS > XLEN (e.g. 48 or 64 warps at XLEN=32), so those shapes could not be simulated at all. Lint clean at 32, 48 and 64 warps, XLEN 32 and 64. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/core/VX_csr_data.sv | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/hw/rtl/core/VX_csr_data.sv b/hw/rtl/core/VX_csr_data.sv index 6a2300fe1a..0214f50e99 100644 --- a/hw/rtl/core/VX_csr_data.sv +++ b/hw/rtl/core/VX_csr_data.sv @@ -229,7 +229,12 @@ import VX_fpu_pkg::*; assign live_src[LIVE_INSTRET_H] = 32'(sched_csr_if.instret[PERF_CTR_BITS-1:32]); `endif assign live_src[LIVE_THREADS] = `VX_CFG_XLEN'(sched_csr_if.thread_mask); - assign live_src[LIVE_WARPS] = `VX_CFG_XLEN'(sched_csr_if.active_warps); + // the CSR is XLEN wide: it holds warps 0..XLEN-1 + wire [`VX_CFG_NUM_WARPS-1:0] active_warps = sched_csr_if.active_warps; + assign live_src[LIVE_WARPS] = `VX_CFG_XLEN'(active_warps); + if (`VX_CFG_NUM_WARPS > `VX_CFG_XLEN) begin : g_active_warps_hi + `UNUSED_VAR (active_warps[`VX_CFG_NUM_WARPS-1:`VX_CFG_XLEN]) + end `ifdef VX_CFG_EXT_F_ENABLE assign live_src[LIVE_FFLAGS] = `VX_CFG_XLEN'(req_fcsr[`FP_FLAGS_BITS-1:0]); assign live_src[LIVE_FRM] = `VX_CFG_XLEN'(req_fcsr[INST_FRM_BITS+`FP_FLAGS_BITS-1:`FP_FLAGS_BITS]); From e311957d9c9d48c64dbda4e7d565fa88f555db81 Mon Sep 17 00:00:00 2001 From: tinebp Date: Thu, 22 Oct 2026 18:09:14 -0700 Subject: [PATCH 090/124] core: support a warp count per issue slice that is not a power of two Three places indexed warps by bit-slicing, which is only right when the count is a power of two. Real GPU shapes are not: an RTX 5080 SM holds 48 warps, 12 per scheduler partition. - Operand collectors: a warp's collector was wis[1:0] and its register slot the upper wis bits. With 12 warps per slice there are 3 collectors, so warps 3, 7 and 11 selected a collector that does not exist, their instructions never left the scoreboard, and the slice deadlocked (scoreboard timeout on sgemm at 4 cores x 48 warps). The collector is now wis % NUM_OPCS and the slot wis / NUM_OPCS, and the per-collector slot count rounds up, as in SimX. - IPDOM stack: entries were addressed {ptr, wid}, which runs past the end of the DEPTH * NUM_WARPS store (up to entry 491 of 372 at 12 warps). The address is now ptr * NUM_WARPS + wid. - mhartid stepped by the next power of two of NUM_WARPS per core, while the stack window holds NUM_WARPS * NUM_THREADS stacks per core, so the last core's stacks fell below the window. It is now dense, (core * NUM_WARPS + wid) * NUM_THREADS + tid, as SimX computes it. For power-of-two counts the constant %, / and * reduce to the same bit selects, so existing configurations are unchanged. Lint clean at 6, 9, 12, 24, 48, 64 and the default warp counts (9 warps failed lint in the IPDOM stack before). rtlsim validation at the 48-warp shape is still running. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/VX_gpu_pkg.sv | 2 +- hw/rtl/core/VX_csr_unit.sv | 5 +++-- hw/rtl/core/VX_ipdom_stack.sv | 6 ++++-- hw/rtl/core/VX_opc_unit.sv | 15 +++++++++++---- hw/rtl/core/VX_operands.sv | 4 ++-- 5 files changed, 21 insertions(+), 11 deletions(-) diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index 782f1361c9..55e2d64049 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -602,7 +602,7 @@ package VX_gpu_pkg; localparam DISPATCH_QSIZE = `VX_CFG_DISPATCH_QUEUE_SIZE; - localparam PER_OPC_WARPS = PER_ISSUE_WARPS / `VX_CFG_NUM_OPCS; + localparam PER_OPC_WARPS = `CDIV(PER_ISSUE_WARPS, `VX_CFG_NUM_OPCS); localparam PER_OPC_NW_BITS = `CLOG2(PER_OPC_WARPS); localparam PER_OPC_NW_W = `UP(PER_OPC_NW_BITS); diff --git a/hw/rtl/core/VX_csr_unit.sv b/hw/rtl/core/VX_csr_unit.sv index 0a4e859da9..0a5358ca90 100644 --- a/hw/rtl/core/VX_csr_unit.sv +++ b/hw/rtl/core/VX_csr_unit.sv @@ -114,8 +114,9 @@ module VX_csr_unit import VX_gpu_pkg::*; #( // Thread ids are the lane index on top of a per-request base. wire [`VX_CFG_XLEN-1:0] wtid_base = (PID_BITS != 0) ? `VX_CFG_XLEN'(execute_if.data.header.pid * NUM_LANES) : '0; - wire [`VX_CFG_XLEN-1:0] gtid_base = (`VX_CFG_XLEN'(core_id) << (NW_BITS + NT_BITS)) - + (`VX_CFG_XLEN'(execute_if.data.header.wid) << NT_BITS) + // mhartid is dense, (core * NUM_WARPS + wid) * NUM_THREADS + tid, for any NUM_WARPS + wire [`VX_CFG_XLEN-1:0] gtid_base = ((`VX_CFG_XLEN'(core_id) * `VX_CFG_XLEN'(`VX_CFG_NUM_WARPS) + + `VX_CFG_XLEN'(execute_if.data.header.wid)) << NT_BITS) + wtid_base; wire is_wtid_w = (csr_addr == `VX_CSR_THREAD_ID); diff --git a/hw/rtl/core/VX_ipdom_stack.sv b/hw/rtl/core/VX_ipdom_stack.sv index a5b8a8b3f7..270bfe0f25 100644 --- a/hw/rtl/core/VX_ipdom_stack.sv +++ b/hw/rtl/core/VX_ipdom_stack.sv @@ -78,8 +78,10 @@ module VX_ipdom_stack import VX_gpu_pkg::*; #( wire [BRAW_ADDRW-1:0] raddr, waddr; if (DEPTH > 1 && NUM_WARPS > 1) begin : g_DW - assign waddr = push ? {wr_ptr_w[wid], wid} : {rd_ptr, wid}; - assign raddr = {rd_ptr, wid}; + // entry = ptr * NUM_WARPS + wid keeps the store dense when NUM_WARPS is not a power of two + wire [ADDRW-1:0] waddr_ptr = push ? wr_ptr_w[wid] : rd_ptr; + assign waddr = BRAW_ADDRW'(waddr_ptr * NUM_WARPS + wid); + assign raddr = BRAW_ADDRW'(rd_ptr * NUM_WARPS + wid); end else if (DEPTH > 1) begin : g_D `UNUSED_VAR (wid) assign waddr = push ? wr_ptr_w : rd_ptr; diff --git a/hw/rtl/core/VX_opc_unit.sv b/hw/rtl/core/VX_opc_unit.sv index 203501046a..75b3b4c708 100644 --- a/hw/rtl/core/VX_opc_unit.sv +++ b/hw/rtl/core/VX_opc_unit.sv @@ -236,14 +236,21 @@ module VX_opc_unit import VX_gpu_pkg::*; #( end end + // warps are dealt round-robin across the collectors: slot = wis / NUM_OPCS + wire [PER_OPC_NW_W-1:0] wr_slot = PER_OPC_NW_W'(writeback_if.data.wis / `VX_CFG_NUM_OPCS); + wire [PER_OPC_NW_W-1:0] rd_slot = PER_OPC_NW_W'(pipe_mdata_st1[META_DATAW-UUID_WIDTH-1 -: ISSUE_WIS_W] / `VX_CFG_NUM_OPCS); + if (PER_OPC_NW_BITS == 0) begin : g_slot_unused + `UNUSED_VAR ({wr_slot, rd_slot}) + end + wire [BANK_ADDR_WIDTH-1:0] gpr_wr_addr; if (SIMD_COUNT != 1) begin : g_gpr_wr_addr_sid wire [PER_OPC_NW_BITS + REG_REM_BITS-1:0] tmp; - `CONCAT(tmp, writeback_if.data.wis[ISSUE_WIS_W-1 -: PER_OPC_NW_W], + `CONCAT(tmp, wr_slot, writeback_if.data.rd[NUM_REGS_BITS-1 -: REG_REM_BITS], PER_OPC_NW_BITS, REG_REM_BITS) assign gpr_wr_addr = {writeback_if.data.sid, tmp}; end else begin : g_gpr_wr_addr - `CONCAT(gpr_wr_addr, writeback_if.data.wis[ISSUE_WIS_W-1 -: PER_OPC_NW_W], + `CONCAT(gpr_wr_addr, wr_slot, writeback_if.data.rd[NUM_REGS_BITS-1 -: REG_REM_BITS], PER_OPC_NW_BITS, REG_REM_BITS) end @@ -271,13 +278,13 @@ module VX_opc_unit import VX_gpu_pkg::*; #( wire [BANK_ADDR_WIDTH-1:0] gpr_rd_addr; if (SIMD_COUNT != 1) begin : g_gpr_rd_addr_sid wire [PER_OPC_NW_BITS + REG_REM_BITS-1:0] tmp; - `CONCAT(tmp, pipe_mdata_st1[META_DATAW-UUID_WIDTH-1 -: PER_OPC_NW_W], + `CONCAT(tmp, rd_slot, gpr_rd_reg_st1[b], PER_OPC_NW_BITS, REG_REM_BITS) // pipe_mdata packs {uuid, wis, cta_id, simd_pid, ...}; skip uuid+wis+cta_id // to extract simd_pid, which must match writeback_if.data.sid on the write path. assign gpr_rd_addr = {pipe_mdata_st1[META_DATAW-UUID_WIDTH-ISSUE_WIS_W-NCTA_WIDTH-1 -: SIMD_IDX_W], tmp}; end else begin : g_gpr_rd_addr - `CONCAT(gpr_rd_addr, pipe_mdata_st1[META_DATAW-UUID_WIDTH-1 -: PER_OPC_NW_W], + `CONCAT(gpr_rd_addr, rd_slot, gpr_rd_reg_st1[b], PER_OPC_NW_BITS, REG_REM_BITS) end diff --git a/hw/rtl/core/VX_operands.sv b/hw/rtl/core/VX_operands.sv index 2c1f11b9bc..9a2d417616 100644 --- a/hw/rtl/core/VX_operands.sv +++ b/hw/rtl/core/VX_operands.sv @@ -42,8 +42,8 @@ module VX_operands import VX_gpu_pkg::*; #( wire [NUM_OPCS_W-1:0] sb_opc, wb_opc; if (`VX_CFG_NUM_OPCS != 1) begin : g_wis_opc - assign sb_opc = scoreboard_if.data.wis[NUM_OPCS_W-1:0]; - assign wb_opc = writeback_if.data.wis[NUM_OPCS_W-1:0]; + assign sb_opc = NUM_OPCS_W'(scoreboard_if.data.wis % `VX_CFG_NUM_OPCS); + assign wb_opc = NUM_OPCS_W'(writeback_if.data.wis % `VX_CFG_NUM_OPCS); end else begin : g_wis_opc assign sb_opc = 0; assign wb_opc = 0; From 0c603f49b5d0f884fb0c04e3d9a364664649068e Mon Sep 17 00:00:00 2001 From: tinebp Date: Thu, 22 Oct 2026 18:09:21 -0700 Subject: [PATCH 091/124] hw/unittest: wire the tcu and tensor wrappers to the per-block TCU memory port fc0a35bb4 moved the TCU_LD AGU into each TCU block and gave VX_tcu_unit a tcu_mem_if port per LSU block, but the two synthesis wrappers still wired the removed core-level AGU and metadata write bus. Both DUTs failed to elaborate in fpga_gate ("unconnected interface port tcu_mem_if"). The wrappers now pass VX_tcu_unit the LSU-block client array and keep VX_tcu_shared for the WGMMA tile buffers only. Both wrappers elaborate under Verilator with the gate configurations; the previous tcu wrapper still fails there (missing tcu_mem_if, 20 removed pins). The fpga_gate and asic_gate tcu/tensor builds have not been rerun. Co-Authored-By: Claude Opus 5.5 --- hw/unittest/tcu_unit/VX_tcu_unit_top.sv | 50 +++++-------------------- hw/unittest/tensor/VX_tensor_top.sv | 50 +++---------------------- 2 files changed, 14 insertions(+), 86 deletions(-) diff --git a/hw/unittest/tcu_unit/VX_tcu_unit_top.sv b/hw/unittest/tcu_unit/VX_tcu_unit_top.sv index 637ba9bd9a..7542fe1600 100644 --- a/hw/unittest/tcu_unit/VX_tcu_unit_top.sv +++ b/hw/unittest/tcu_unit/VX_tcu_unit_top.sv @@ -106,13 +106,13 @@ module VX_tcu_unit_top import VX_gpu_pkg::*, VX_tcu_pkg::*; ( `endif `ifdef TCU_META_ENABLE - VX_lsu_sched_if tcu_mem_if(); - assign tcu_mem_req_valid = tcu_mem_if.req_valid; - assign tcu_mem_req_data = tcu_mem_if.req_data; - assign tcu_mem_if.req_ready = tcu_mem_req_ready; - assign tcu_mem_if.rsp_valid = tcu_mem_rsp_valid; - assign tcu_mem_if.rsp_data = tcu_mem_rsp_data; - assign tcu_mem_rsp_ready = tcu_mem_if.rsp_ready; + VX_lsu_sched_if tcu_mem_if [`VX_CFG_NUM_LSU_BLOCKS](); + assign tcu_mem_req_valid = tcu_mem_if[0].req_valid; + assign tcu_mem_req_data = tcu_mem_if[0].req_data; + assign tcu_mem_if[0].req_ready = tcu_mem_req_ready; + assign tcu_mem_if[0].rsp_valid = tcu_mem_rsp_valid; + assign tcu_mem_if[0].rsp_data = tcu_mem_rsp_data; + assign tcu_mem_rsp_ready = tcu_mem_if[0].rsp_ready; `endif VX_dispatch_if dispatch_if[`VX_CFG_ISSUE_WIDTH](); @@ -171,16 +171,6 @@ module VX_tcu_unit_top import VX_gpu_pkg::*, VX_tcu_pkg::*; ( wire [`VX_CFG_NUM_TCU_BLOCKS-1:0][TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs2_data; wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_tbuf_ready; `endif -`ifdef TCU_META_ENABLE - wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_agu_ld_valid, tcu_agu_ld_ready; - tcu_execute_t tcu_agu_ld_data [`VX_CFG_NUM_TCU_BLOCKS]; - wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_agu_result_valid, tcu_agu_result_ready; - tcu_result_t tcu_agu_result_data [`VX_CFG_NUM_TCU_BLOCKS]; - wire tcu_meta_wr_en; - wire [NW_WIDTH-1:0] tcu_meta_wr_wid; - wire [4:0] tcu_meta_wr_idx; - wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] tcu_meta_wr_data; -`endif VX_tcu_unit #( .INSTANCE_ID ("tcu_unit_top") @@ -197,22 +187,13 @@ module VX_tcu_unit_top import VX_gpu_pkg::*, VX_tcu_pkg::*; ( .tbuf_ready (tcu_tbuf_ready), `endif `ifdef TCU_META_ENABLE - .agu_ld_valid (tcu_agu_ld_valid), - .agu_ld_data (tcu_agu_ld_data), - .agu_ld_ready (tcu_agu_ld_ready), - .agu_result_valid (tcu_agu_result_valid), - .agu_result_data (tcu_agu_result_data), - .agu_result_ready (tcu_agu_result_ready), - .meta_wr_en (tcu_meta_wr_en), - .meta_wr_wid (tcu_meta_wr_wid), - .meta_wr_idx (tcu_meta_wr_idx), - .meta_wr_data (tcu_meta_wr_data), + .tcu_mem_if (tcu_mem_if), `endif .dispatch_if (dispatch_if), .commit_if (commit_if) ); - // the core-wide TCU state the blocks share: WGMMA tile buffers, metadata AGU + // the core-wide WGMMA tile buffers the TCU blocks share VX_tcu_shared #( .INSTANCE_ID ("tcu_unit_top"), .BLOCK_SIZE (`VX_CFG_NUM_TCU_BLOCKS) @@ -225,19 +206,6 @@ module VX_tcu_unit_top import VX_gpu_pkg::*, VX_tcu_pkg::*; ( .tbuf_rs2_data (tcu_tbuf_rs2_data), .tbuf_ready_eff (tcu_tbuf_ready), .tcu_lmem_if (tcu_lmem_if), - `endif - `ifdef TCU_META_ENABLE - .agu_ld_valid (tcu_agu_ld_valid), - .agu_ld_data (tcu_agu_ld_data), - .agu_ld_ready (tcu_agu_ld_ready), - .agu_result_valid (tcu_agu_result_valid), - .agu_result_data (tcu_agu_result_data), - .agu_result_ready (tcu_agu_result_ready), - .meta_wr_en (tcu_meta_wr_en), - .meta_wr_wid (tcu_meta_wr_wid), - .meta_wr_idx (tcu_meta_wr_idx), - .meta_wr_data (tcu_meta_wr_data), - .tcu_mem_if (tcu_mem_if), `endif .clk (clk), .reset (reset) diff --git a/hw/unittest/tensor/VX_tensor_top.sv b/hw/unittest/tensor/VX_tensor_top.sv index 232f6475cb..b3059bcdec 100644 --- a/hw/unittest/tensor/VX_tensor_top.sv +++ b/hw/unittest/tensor/VX_tensor_top.sv @@ -20,7 +20,7 @@ // // per core (VX_CFG_SOCKET_SIZE): // VX_tcu_unit : tensor-core unit (dispatch/commit from top ports) -// VX_tcu_shared : WGMMA tile buffers + metadata AGU +// VX_tcu_shared : WGMMA tile buffers // VX_lsu_mem_path : per-LSU-block scheduler (2 clients — the ordinary LSU // stream from top ports and the TCU AGU metadata-load // port), lmem switch, coalescer and adapters @@ -75,7 +75,7 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( // ----------------------------------------------------------------------- // Ordinary LSU client streams, flattened [core][lsu-block] - // (client 0 of each LSU block's scheduler; the TCU AGU is client 1 on block 0) + // (client 0 of each LSU block's scheduler; the TCU block AGUs are client 1) // ----------------------------------------------------------------------- input wire [LSU_STREAMS-1:0] lsu_req_valid, input lsu_req_data_t [LSU_STREAMS-1:0] lsu_req_data, @@ -169,14 +169,6 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( `ifdef TCU_META_ENABLE VX_lsu_sched_if lsu_tcu_mem_if [NUM_LSU_BLKS](); - for (genvar block_idx = 1; block_idx < NUM_LSU_BLKS; ++block_idx) begin : g_tcu_client_tieoff - assign lsu_tcu_mem_if[block_idx].req_valid = 1'b0; - assign lsu_tcu_mem_if[block_idx].req_data = '0; - assign lsu_tcu_mem_if[block_idx].rsp_ready = 1'b1; - `UNUSED_VAR (lsu_tcu_mem_if[block_idx].req_ready) - `UNUSED_VAR (lsu_tcu_mem_if[block_idx].rsp_valid) - `UNUSED_VAR (lsu_tcu_mem_if[block_idx].rsp_data) - end `endif `ifdef VX_CFG_TCU_WGMMA_ENABLE @@ -186,16 +178,6 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( wire [`VX_CFG_NUM_TCU_BLOCKS-1:0][TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs2_data; wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_tbuf_ready; `endif - `ifdef TCU_META_ENABLE - wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_agu_ld_valid, tcu_agu_ld_ready; - tcu_execute_t tcu_agu_ld_data [`VX_CFG_NUM_TCU_BLOCKS]; - wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_agu_result_valid, tcu_agu_result_ready; - tcu_result_t tcu_agu_result_data [`VX_CFG_NUM_TCU_BLOCKS]; - wire tcu_meta_wr_en; - wire [NW_WIDTH-1:0] tcu_meta_wr_wid; - wire [4:0] tcu_meta_wr_idx; - wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] tcu_meta_wr_data; - `endif VX_tcu_unit #( .INSTANCE_ID (`SFORMATF(("%s-tcu%0d", INSTANCE_ID, c))) @@ -212,22 +194,13 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( .tbuf_ready (tcu_tbuf_ready), `endif `ifdef TCU_META_ENABLE - .agu_ld_valid (tcu_agu_ld_valid), - .agu_ld_data (tcu_agu_ld_data), - .agu_ld_ready (tcu_agu_ld_ready), - .agu_result_valid (tcu_agu_result_valid), - .agu_result_data (tcu_agu_result_data), - .agu_result_ready (tcu_agu_result_ready), - .meta_wr_en (tcu_meta_wr_en), - .meta_wr_wid (tcu_meta_wr_wid), - .meta_wr_idx (tcu_meta_wr_idx), - .meta_wr_data (tcu_meta_wr_data), + .tcu_mem_if (lsu_tcu_mem_if), `endif .dispatch_if (dispatch_if), .commit_if (commit_if) ); - // the core-wide TCU state the blocks share: WGMMA tile buffers, metadata AGU + // the core-wide WGMMA tile buffers the TCU blocks share VX_tcu_shared #( .INSTANCE_ID (`SFORMATF(("%s-tcu%0d", INSTANCE_ID, c))), .BLOCK_SIZE (`VX_CFG_NUM_TCU_BLOCKS) @@ -240,25 +213,12 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( .tbuf_rs2_data (tcu_tbuf_rs2_data), .tbuf_ready_eff (tcu_tbuf_ready), .tcu_lmem_if (tcu_lmem_if), - `endif - `ifdef TCU_META_ENABLE - .agu_ld_valid (tcu_agu_ld_valid), - .agu_ld_data (tcu_agu_ld_data), - .agu_ld_ready (tcu_agu_ld_ready), - .agu_result_valid (tcu_agu_result_valid), - .agu_result_data (tcu_agu_result_data), - .agu_result_ready (tcu_agu_result_ready), - .meta_wr_en (tcu_meta_wr_en), - .meta_wr_wid (tcu_meta_wr_wid), - .meta_wr_idx (tcu_meta_wr_idx), - .meta_wr_data (tcu_meta_wr_data), - .tcu_mem_if (lsu_tcu_mem_if[0]), `endif .clk (clk), .reset (reset) ); - // --- LSU memory paths: client 0 from top ports, client 1 = TCU AGU --- + // --- LSU memory paths: client 0 from top ports, client 1 = the TCU block AGUs --- wire [NUM_LSU_BLKS-1:0] block_sched_empty; wire [NUM_LSU_BLKS-1:0] block_store_empty; From 45e6a23561677d121ad4445336bbee8ed4930014 Mon Sep 17 00:00:00 2001 From: tinebp Date: Fri, 23 Oct 2026 12:29:57 -0700 Subject: [PATCH 092/124] afu: let the memory ports keep many transactions in flight The XRT AFU held memory-level parallelism to one transaction per direction at the platform AXI boundary, which caps a port at one line per DRAM round trip whatever the caches have in flight: - Multi-port shells (u280, u200): VX_afu_axi_limit kept AXI port 0 to one read and one write in flight. Lines interleave across ports, so every stream was paced by port 0. It only reproduced the old 2:1 arbiter's policy; it is removed and port 0 behaves like the other ports. - Single-port shells (u55c/u50 merged HBM, u250, versal): the arbiter that merges Vortex and the command processor ran single-outstanding, so the whole GPU had one DRAM read and one write in flight. It now runs MULTI_OUT: the source index rides in the top ID bit and responses are routed by it. Vortex keeps the remaining ID bits, and a static assert checks that the CP's ID fits. VX_mm_axi_arb could also switch its AR/AW grant while an offer was still waiting for READY, changing ADDR/ID/LEN under a held VALID (AXI A3.2.1). An offer the slave has not taken now holds the grant; every other cycle arbitrates as before. The memory-port drain counters widen to 16 bits, since one port can now carry more than 1024 transactions. xrtsim cycles, base -> fix (all PASSED): single port, 1 core, latency 100: vecadd -n65536 2346805 -> 1883709 (1.25x), sgemm -n64 1367849 -> 1074041 (1.27x) 4 ports, 1 core, latency 100: 1.06x / 1.08x single port, 4 cores + L2: sgemv 45914 -> 35316 (1.30x) default timing (short round trip): 1.00-1.13x The gain grows with DRAM latency, so it is larger on HBM hardware. SimX and rtlsim do not model the AFU; their cycles are unchanged. Validated: 23/23 xrt ci cases, unit tests cp_llc_port, cp_axi_path, cp_core and afu_reset_seq, AFU lint at the u55c/u200/u280/u250 shapes with no new warnings. Vivado synthesis not run. Co-Authored-By: Claude Opus 5.5 --- docs/designs/fpga_afu_shell.md | 16 ++-- hw/rtl/afu/common/VX_afu_axi_limit.sv | 99 ----------------------- hw/rtl/afu/common/VX_afu_wrap.sv | 108 ++++++++++---------------- hw/rtl/libs/VX_mm_axi_arb.sv | 70 +++++++++++++++-- 4 files changed, 115 insertions(+), 178 deletions(-) delete mode 100644 hw/rtl/afu/common/VX_afu_axi_limit.sv diff --git a/docs/designs/fpga_afu_shell.md b/docs/designs/fpga_afu_shell.md index f866d40b7a..413a108129 100644 --- a/docs/designs/fpga_afu_shell.md +++ b/docs/designs/fpga_afu_shell.md @@ -27,7 +27,6 @@ DMA-command engine have been removed. | [`VX_afu_ctrl.sv`](../../hw/rtl/afu/xrt/VX_afu_ctrl.sv) | Slimmed AXI-Lite slave (~322 LOC): `ap_ctrl` stub at 0x00 + a SCOPE serial register pair + SCOPE watchdog. | | [`VX_afu_axil_demux.sv`](../../hw/rtl/afu/common/VX_afu_axil_demux.sv) | AXI-Lite demux splitting the control space on `addr[12]`, one outstanding transaction per direction. | | [`VX_afu_axi_drain.sv`](../../hw/rtl/afu/common/VX_afu_axi_drain.sv) | Outstanding-transaction tracker per AXI master; reports when a port owes the interconnect nothing. | -| [`VX_afu_axi_limit.sv`](../../hw/rtl/afu/common/VX_afu_axi_limit.sv) | Holds an AXI master to one read and one write in flight; applied to port 0 of the multi-port shell. | | [`VX_afu_reset_seq.sv`](../../hw/rtl/afu/common/VX_afu_reset_seq.sv) | Quiesce-before-reset sequencer for the soft reset; refuses rather than resetting a master that will not drain. | - **Control.** Host AXI-Lite `addr[12]` splits the slave: `addr[12]=0` → @@ -42,12 +41,15 @@ DMA-command engine have been removed. Covered by `hw/unittest/afu_axil_demux`. - **Memory.** Two structures, selected by the number of memory ports. *One port* (the merged platforms): `Vortex_axi` and the CP's `axi_dev` - share the port through a 2:1 `VX_mm_axi_arb`, and CP bursts reach the - platform as bursts. *Several ports*: Vortex's memory ports and `axi_dev` - (bridged by `VX_membus_from_axi`) are the inputs of one bank adapter, - `VX_mem_to_axi`, so the CP is merged **upstream** of bank selection and - reaches every bank. In both, port 0 carries one read and one write in - flight (`VX_afu_axi_limit` in the second case). CP `axi_host` drives a + share the port through a 2:1 `VX_mm_axi_arb` in its ID-routed + multi-outstanding mode (`MULTI_OUT=1`: the top ID bit names the source, + Vortex gets the remaining ID bits), and CP bursts reach the platform as + bursts. *Several ports*: Vortex's memory ports and `axi_dev` (bridged by + `VX_membus_from_axi`) are the inputs of one bank adapter, `VX_mem_to_axi`, + so the CP is merged **upstream** of bank selection and reaches every bank. + In both, every port carries as many reads and writes in flight as its + masters issue; one outstanding transaction per direction would cap a port + at one line per DRAM round trip. CP `axi_host` drives a **dedicated `m_axi_host` AXI master** for the command ring + host DMA. `PLATFORM_MEMORY_OFFSET` is applied per bank, at the port. - **Interrupt.** The AFU `interrupt` pin is driven from `cp_interrupt` diff --git a/hw/rtl/afu/common/VX_afu_axi_limit.sv b/hw/rtl/afu/common/VX_afu_axi_limit.sv deleted file mode 100644 index 37bc3bf736..0000000000 --- a/hw/rtl/afu/common/VX_afu_axi_limit.sv +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright © 2019-2023 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -`include "VX_platform.vh" - -// ============================================================================ -// VX_afu_axi_limit — holds an AXI master to one read and one write in flight. -// -// A new AR is withheld until the previous read's last beat, and a new AW until -// the previous write's B. The limits only ever tighten on a handshake, so an -// offer already made to the slave is never withdrawn. -// -// AW and W of one write may fire in either order. W is let through whenever -// its AW has already gone, and AW whenever its W has, whatever the limit says -// by then: withholding half of an accepted write would hang it. -// ============================================================================ - -`TRACING_OFF -module VX_afu_axi_limit ( - input wire clk, - input wire reset, - - input wire in_awvalid, - output wire in_awready, - input wire in_wvalid, - output wire in_wready, - input wire in_wlast, - input wire in_arvalid, - output wire in_arready, - - output wire out_awvalid, - input wire out_awready, - output wire out_wvalid, - input wire out_wready, - output wire out_arvalid, - input wire out_arready, - - input wire b_fire, - input wire r_fire_last -); - reg rd_busy; - reg wr_busy; - reg w_owed; // AW sent, its W still to come - reg aw_owed; // W sent, its AW still to come - - wire aw_allow = aw_owed || ~wr_busy; - wire w_allow = w_owed || (~wr_busy && ~aw_owed); - wire ar_allow = ~rd_busy; - - assign out_awvalid = in_awvalid && aw_allow; - assign in_awready = out_awready && aw_allow; - assign out_wvalid = in_wvalid && w_allow; - assign in_wready = out_wready && w_allow; - assign out_arvalid = in_arvalid && ar_allow; - assign in_arready = out_arready && ar_allow; - - wire aw_fire = out_awvalid && out_awready; - wire w_fire_last = out_wvalid && out_wready && in_wlast; - wire ar_fire = out_arvalid && out_arready; - - always @(posedge clk) begin - if (reset) begin - rd_busy <= 1'b0; - wr_busy <= 1'b0; - w_owed <= 1'b0; - aw_owed <= 1'b0; - end else begin - if (ar_fire) begin - rd_busy <= 1'b1; - end else if (r_fire_last) begin - rd_busy <= 1'b0; - end - if (aw_fire) begin - wr_busy <= 1'b1; - end else if (b_fire) begin - wr_busy <= 1'b0; - end - if (aw_fire && ~w_fire_last) begin - aw_owed <= 1'b0; - w_owed <= ~aw_owed; - end else if (w_fire_last && ~aw_fire) begin - w_owed <= 1'b0; - aw_owed <= ~w_owed; - end - end - end - -endmodule -`TRACING_ON diff --git a/hw/rtl/afu/common/VX_afu_wrap.sv b/hw/rtl/afu/common/VX_afu_wrap.sv index bea895dfe0..f19ed26bb8 100644 --- a/hw/rtl/afu/common/VX_afu_wrap.sv +++ b/hw/rtl/afu/common/VX_afu_wrap.sv @@ -479,11 +479,17 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( if (C_M_AXI_MEM_NUM_BANKS == 1) begin : g_single_port + // The arbiter carries the source index in the top ID bit, so each + // master gets the rest of the platform ID. + localparam ARB_SRC_BITS = 1; + localparam SRC_ID_WIDTH = C_M_AXI_MEM_ID_WIDTH - ARB_SRC_BITS; + `STATIC_ASSERT(`VX_CP_AXI_TID_WIDTH <= SRC_ID_WIDTH, ("CP device-master ID does not fit the platform ID")) + // ---- Vortex's AXI port, ahead of the arbiter ---- wire vx_awvalid_a [C_M_AXI_MEM_NUM_BANKS]; wire vx_awready_a [C_M_AXI_MEM_NUM_BANKS]; wire [M_AXI_MEM_ADDR_WIDTH-1:0] vx_awaddr_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_ID_WIDTH-1:0] vx_awid_a [C_M_AXI_MEM_NUM_BANKS]; + wire [SRC_ID_WIDTH-1:0] vx_awid_a [C_M_AXI_MEM_NUM_BANKS]; wire [7:0] vx_awlen_a [C_M_AXI_MEM_NUM_BANKS]; wire vx_wvalid_a [C_M_AXI_MEM_NUM_BANKS]; @@ -494,26 +500,26 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( wire vx_bvalid_a [C_M_AXI_MEM_NUM_BANKS]; wire vx_bready_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_ID_WIDTH-1:0] vx_bid_a [C_M_AXI_MEM_NUM_BANKS]; + wire [SRC_ID_WIDTH-1:0] vx_bid_a [C_M_AXI_MEM_NUM_BANKS]; wire [1:0] vx_bresp_a [C_M_AXI_MEM_NUM_BANKS]; wire vx_arvalid_a [C_M_AXI_MEM_NUM_BANKS]; wire vx_arready_a [C_M_AXI_MEM_NUM_BANKS]; wire [M_AXI_MEM_ADDR_WIDTH-1:0] vx_araddr_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_ID_WIDTH-1:0] vx_arid_a [C_M_AXI_MEM_NUM_BANKS]; + wire [SRC_ID_WIDTH-1:0] vx_arid_a [C_M_AXI_MEM_NUM_BANKS]; wire [7:0] vx_arlen_a [C_M_AXI_MEM_NUM_BANKS]; wire vx_rvalid_a [C_M_AXI_MEM_NUM_BANKS]; wire vx_rready_a [C_M_AXI_MEM_NUM_BANKS]; wire [C_M_AXI_MEM_DATA_WIDTH-1:0] vx_rdata_a [C_M_AXI_MEM_NUM_BANKS]; wire vx_rlast_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_ID_WIDTH-1:0] vx_rid_a [C_M_AXI_MEM_NUM_BANKS]; + wire [SRC_ID_WIDTH-1:0] vx_rid_a [C_M_AXI_MEM_NUM_BANKS]; wire [1:0] vx_rresp_a [C_M_AXI_MEM_NUM_BANKS]; Vortex_axi #( .AXI_DATA_WIDTH (C_M_AXI_MEM_DATA_WIDTH), .AXI_ADDR_WIDTH (M_AXI_MEM_ADDR_WIDTH), - .AXI_TID_WIDTH (C_M_AXI_MEM_ID_WIDTH), + .AXI_TID_WIDTH (SRC_ID_WIDTH), .AXI_NUM_BANKS (C_M_AXI_MEM_NUM_BANKS) ) vortex_axi ( `SCOPE_IO_BIND (1) @@ -577,12 +583,12 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( ); // ---- 2:1 arbiter merges Vortex + CP axi_dev ---- - // Pad CP's narrower ID into the platform ID width so the arbiter sees - // identical signal widths from both sources. - wire [C_M_AXI_MEM_ID_WIDTH-1:0] cp_awid_padded = - {{(C_M_AXI_MEM_ID_WIDTH - `VX_CP_AXI_TID_WIDTH){1'b0}}, cp_axi_dev.awid}; - wire [C_M_AXI_MEM_ID_WIDTH-1:0] cp_arid_padded = - {{(C_M_AXI_MEM_ID_WIDTH - `VX_CP_AXI_TID_WIDTH){1'b0}}, cp_axi_dev.arid}; + // Zero-extend both sources' IDs to the platform width; the arbiter + // overwrites the top bit with the source index. + wire [C_M_AXI_MEM_ID_WIDTH-1:0] vx_awid_padded = C_M_AXI_MEM_ID_WIDTH'(vx_awid_a[0]); + wire [C_M_AXI_MEM_ID_WIDTH-1:0] vx_arid_padded = C_M_AXI_MEM_ID_WIDTH'(vx_arid_a[0]); + wire [C_M_AXI_MEM_ID_WIDTH-1:0] cp_awid_padded = C_M_AXI_MEM_ID_WIDTH'(cp_axi_dev.awid); + wire [C_M_AXI_MEM_ID_WIDTH-1:0] cp_arid_padded = C_M_AXI_MEM_ID_WIDTH'(cp_axi_dev.arid); // The CP's device addresses come from the same host-side allocator as the // pointers handed to the cores -- Device::global_mem_, based at @@ -599,7 +605,9 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( // Packed 2-master AXI arbiter: index 0 = Vortex bank-0 (priority via // ARBITER="P"), index 1 = CP device master. Input channels are packed // {cp, vx}; the arbiter's slave-side outputs land in local packed wires - // and are split back to the two masters below. + // and are split back to the two masters below. Responses are routed by + // the source bit of their ID, so both masters keep as many transactions + // in flight as they issue. localparam BANK0_STRB_W = C_M_AXI_MEM_DATA_WIDTH/8; wire [1:0] b0_awready; @@ -620,6 +628,7 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( .DATA_WIDTH (C_M_AXI_MEM_DATA_WIDTH), .ID_WIDTH (C_M_AXI_MEM_ID_WIDTH), .ARBITER ("P"), // index 0 (Vortex bank-0) > index 1 (CP) + .MULTI_OUT (1), .STRB_WIDTH (BANK0_STRB_W) ) bank0_arb ( .clk (clk), @@ -628,7 +637,7 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( .s_awvalid ({cp_axi_dev.awvalid, vx_awvalid_a[0]}), .s_awready (b0_awready), .s_awaddr ({cp_awaddr_dev, vx_awaddr_a[0]}), - .s_awid ({cp_awid_padded, vx_awid_a[0]}), + .s_awid ({cp_awid_padded, vx_awid_padded}), .s_awlen ({cp_axi_dev.awlen, vx_awlen_a[0]}), .s_wvalid ({cp_axi_dev.wvalid, vx_wvalid_a[0]}), @@ -645,7 +654,7 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( .s_arvalid ({cp_axi_dev.arvalid, vx_arvalid_a[0]}), .s_arready (b0_arready), .s_araddr ({cp_araddr_dev, vx_araddr_a[0]}), - .s_arid ({cp_arid_padded, vx_arid_a[0]}), + .s_arid ({cp_arid_padded, vx_arid_padded}), .s_arlen ({cp_axi_dev.arlen, vx_arlen_a[0]}), .s_rvalid (b0_rvalid), @@ -675,6 +684,8 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( // index 0 = Vortex bank-0, index 1 = CP device master. // Declared before their first assignment below (the implicit-net footgun: // a pre-declaration use makes Vivado mint 1-bit nets and keep both). + wire [C_M_AXI_MEM_ID_WIDTH-1:0] vx_bid_full; + wire [C_M_AXI_MEM_ID_WIDTH-1:0] vx_rid_full; wire [C_M_AXI_MEM_ID_WIDTH-1:0] cp_axi_dev_bid_full; wire [C_M_AXI_MEM_ID_WIDTH-1:0] cp_axi_dev_rid_full; assign vx_awready_a[0] = b0_awready[0]; @@ -683,7 +694,7 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( assign cp_axi_dev.wready = b0_wready[1]; assign vx_bvalid_a[0] = b0_bvalid[0]; assign cp_axi_dev.bvalid = b0_bvalid[1]; - assign vx_bid_a[0] = b0_bid[0]; + assign vx_bid_full = b0_bid[0]; assign cp_axi_dev_bid_full = b0_bid[1]; assign vx_bresp_a[0] = b0_bresp[0]; assign cp_axi_dev.bresp = b0_bresp[1]; @@ -695,14 +706,18 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( assign cp_axi_dev.rdata = b0_rdata[1]; assign vx_rlast_a[0] = b0_rlast[0]; assign cp_axi_dev.rlast = b0_rlast[1]; - assign vx_rid_a[0] = b0_rid[0]; + assign vx_rid_full = b0_rid[0]; assign cp_axi_dev_rid_full = b0_rid[1]; assign vx_rresp_a[0] = b0_rresp[0]; assign cp_axi_dev.rresp = b0_rresp[1]; - // Truncate the arbiter's wider ID back to CP's narrower native ID width. + // Truncate the arbiter's platform-width IDs back to each master's own. + assign vx_bid_a[0] = vx_bid_full[SRC_ID_WIDTH-1:0]; + assign vx_rid_a[0] = vx_rid_full[SRC_ID_WIDTH-1:0]; assign cp_axi_dev.bid = cp_axi_dev_bid_full[`VX_CP_AXI_TID_WIDTH-1:0]; assign cp_axi_dev.rid = cp_axi_dev_rid_full[`VX_CP_AXI_TID_WIDTH-1:0]; + `UNUSED_VAR (vx_bid_full) + `UNUSED_VAR (vx_rid_full) `UNUSED_VAR (cp_axi_dev_bid_full) `UNUSED_VAR (cp_axi_dev_rid_full) @@ -939,13 +954,6 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( `UNUSED_VAR (cp_axi_dev.arburst) - wire ad_awvalid_a [C_M_AXI_MEM_NUM_BANKS]; - wire ad_awready_a [C_M_AXI_MEM_NUM_BANKS]; - wire ad_arvalid_a [C_M_AXI_MEM_NUM_BANKS]; - wire ad_arready_a [C_M_AXI_MEM_NUM_BANKS]; - wire ad_wvalid_a [C_M_AXI_MEM_NUM_BANKS]; - wire ad_wready_a [C_M_AXI_MEM_NUM_BANKS]; - VX_mem_to_axi #( .DATA_WIDTH (C_M_AXI_MEM_DATA_WIDTH), .ADDR_WIDTH_IN (VX_MEM_ADDR_A_WIDTH), @@ -974,8 +982,8 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( .mem_rsp_tag (fab_rsp_tag), .mem_rsp_ready (fab_rsp_ready), - .m_axi_awvalid (ad_awvalid_a), - .m_axi_awready (ad_awready_a), + .m_axi_awvalid (pre_awvalid_a), + .m_axi_awready (pre_awready_a), .m_axi_awaddr (m_axi_mem_awaddr_u), .m_axi_awid (m_axi_mem_awid_a), .m_axi_awlen (m_axi_mem_awlen_a), @@ -987,8 +995,8 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( `UNUSED_PIN (m_axi_awqos), `UNUSED_PIN (m_axi_awregion), - .m_axi_wvalid (ad_wvalid_a), - .m_axi_wready (ad_wready_a), + .m_axi_wvalid (m_axi_mem_wvalid_a), + .m_axi_wready (m_axi_mem_wready_a), .m_axi_wdata (m_axi_mem_wdata_a), .m_axi_wstrb (m_axi_mem_wstrb_a), .m_axi_wlast (m_axi_mem_wlast_a), @@ -998,8 +1006,8 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( .m_axi_bid (m_axi_mem_bid_a), .m_axi_bresp (m_axi_mem_bresp_a), - .m_axi_arvalid (ad_arvalid_a), - .m_axi_arready (ad_arready_a), + .m_axi_arvalid (pre_arvalid_a), + .m_axi_arready (pre_arready_a), .m_axi_araddr (m_axi_mem_araddr_u), .m_axi_arid (m_axi_mem_arid_a), .m_axi_arlen (m_axi_mem_arlen_a), @@ -1019,40 +1027,6 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( .m_axi_rresp (m_axi_mem_rresp_a) ); - // Bank 0 carries one read and one write at a time, the other banks - // as many as the adapter tracks. Reads in flight cost cycles on a - // saturated channel and save them across several, so moving this - // limit changes cycle counts in both directions. - VX_afu_axi_limit bank0_limit ( - .clk (clk), - .reset (subsys_reset), - .in_awvalid (ad_awvalid_a[0]), - .in_awready (ad_awready_a[0]), - .in_wvalid (ad_wvalid_a[0]), - .in_wready (ad_wready_a[0]), - .in_wlast (m_axi_mem_wlast_a[0]), - .in_arvalid (ad_arvalid_a[0]), - .in_arready (ad_arready_a[0]), - .out_awvalid (pre_awvalid_a[0]), - .out_awready (pre_awready_a[0]), - .out_wvalid (m_axi_mem_wvalid_a[0]), - .out_wready (m_axi_mem_wready_a[0]), - .out_arvalid (pre_arvalid_a[0]), - .out_arready (pre_arready_a[0]), - .b_fire (m_axi_mem_bvalid_a[0] && m_axi_mem_bready_a[0]), - .r_fire_last (m_axi_mem_rvalid_a[0] && m_axi_mem_rready_a[0] - && m_axi_mem_rlast_a[0]) - ); - - for (genvar i = 1; i < C_M_AXI_MEM_NUM_BANKS; ++i) begin : g_bank_direct - assign pre_awvalid_a[i] = ad_awvalid_a[i]; - assign ad_awready_a[i] = pre_awready_a[i]; - assign m_axi_mem_wvalid_a[i] = ad_wvalid_a[i]; - assign ad_wready_a[i] = m_axi_mem_wready_a[i]; - assign pre_arvalid_a[i] = ad_arvalid_a[i]; - assign ad_arready_a[i] = pre_arready_a[i]; - end - end // ======================================================================== @@ -1087,7 +1061,11 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( .out_ready (m_axi_mem_arready_a[i]) ); - VX_afu_axi_drain mem_drain ( + // A memory port carries every miss the last-level caches hold in + // flight, so its counters are sized well past any MSHR total. + VX_afu_axi_drain #( + .COUNT_WIDTH (16) + ) mem_drain ( .clk (clk), .reset (reset), .aw_fire (m_axi_mem_awvalid_a[i] && m_axi_mem_awready_a[i]), diff --git a/hw/rtl/libs/VX_mm_axi_arb.sv b/hw/rtl/libs/VX_mm_axi_arb.sv index 0c9536d093..7bfddde3c2 100644 --- a/hw/rtl/libs/VX_mm_axi_arb.sv +++ b/hw/rtl/libs/VX_mm_axi_arb.sv @@ -175,6 +175,20 @@ module VX_mm_axi_arb #( assign aw_pick = '0; assign aw_any = s_awvalid[0]; end else begin : g_aw_arb + // Once offered, the AW must stay unchanged until the slave takes it + // (AXI stability): only the offering master may compete meanwhile. + reg aw_held; + reg [SEL_WIDTH-1:0] aw_held_idx; + wire [NUM_INPUTS-1:0] aw_reqs = aw_held ? (s_awvalid & (NUM_INPUTS'(1) << aw_held_idx)) + : s_awvalid; + always @(posedge clk) begin + if (reset) begin + aw_held <= 1'b0; + end else begin + aw_held <= c_awvalid && ~c_awready; + end + aw_held_idx <= aw_pick; + end VX_generic_arbiter #( .NUM_REQS (NUM_INPUTS), .TYPE (ARBITER), @@ -182,7 +196,7 @@ module VX_mm_axi_arb #( ) aw_arb ( .clk (clk), .reset (reset), - .requests (s_awvalid), + .requests (aw_reqs), .grant_index (aw_pick), `UNUSED_PIN (grant_onehot), .grant_valid (aw_any), @@ -245,6 +259,20 @@ module VX_mm_axi_arb #( assign ar_pick = '0; assign ar_any = s_arvalid[0]; end else begin : g_ar_arb + // Once offered, the AR must stay unchanged until the slave takes it + // (AXI stability): only the offering master may compete meanwhile. + reg ar_held; + reg [SEL_WIDTH-1:0] ar_held_idx; + wire [NUM_INPUTS-1:0] ar_reqs = ar_held ? (s_arvalid & (NUM_INPUTS'(1) << ar_held_idx)) + : s_arvalid; + always @(posedge clk) begin + if (reset) begin + ar_held <= 1'b0; + end else begin + ar_held <= c_arvalid && ~c_arready; + end + ar_held_idx <= ar_pick; + end VX_generic_arbiter #( .NUM_REQS (NUM_INPUTS), .TYPE (ARBITER), @@ -252,7 +280,7 @@ module VX_mm_axi_arb #( ) ar_arb ( .clk (clk), .reset (reset), - .requests (s_arvalid), + .requests (ar_reqs), .grant_index (ar_pick), `UNUSED_PIN (grant_onehot), .grant_valid (ar_any), @@ -298,8 +326,8 @@ module VX_mm_axi_arb #( // The top SEL_WIDTH bits of the AXI ID carry the source index; masters // must leave them free. Reads are fully concurrent; write bursts serialize // on the shared W channel (AW gated until the prior burst's WLAST) while B - // responses stay concurrent. Sticky arbitration holds the granted addr/id - // stable until the handshake completes (AXI stability requirement). + // responses stay concurrent. An offer the slave has not yet taken holds + // the grant, so its addr/id stay stable until the handshake. localparam LOW_ID = ID_WIDTH - SEL_WIDTH; // source's own low ID bits `STATIC_ASSERT((ID_WIDTH > SEL_WIDTH), ("VX_mm_axi_arb MULTI_OUT: ID_WIDTH must exceed LOG2UP(NUM_INPUTS) to carry the source index")) @@ -312,12 +340,26 @@ module VX_mm_axi_arb #( assign ar_any = s_arvalid[0]; end else begin : g_ar_arb wire ar_fire = c_arvalid && c_arready; + // Once offered, the AR must stay unchanged until the slave takes it + // (AXI stability): only the offering master may compete meanwhile. + reg ar_held; + reg [SEL_WIDTH-1:0] ar_held_idx; + wire [NUM_INPUTS-1:0] ar_reqs = ar_held ? (s_arvalid & (NUM_INPUTS'(1) << ar_held_idx)) + : s_arvalid; + always @(posedge clk) begin + if (reset) begin + ar_held <= 1'b0; + end else begin + ar_held <= c_arvalid && ~c_arready; + end + ar_held_idx <= ar_pick; + end VX_generic_arbiter #( .NUM_REQS (NUM_INPUTS), .TYPE (ARBITER), - .STICKY (1) // hold grant stable until AR fires (AXI stability) + .STICKY (1) ) ar_arb ( - .clk (clk), .reset (reset), .requests (s_arvalid), + .clk (clk), .reset (reset), .requests (ar_reqs), .grant_index (ar_pick), `UNUSED_PIN (grant_onehot), .grant_valid (ar_any), .grant_ready (ar_fire) ); @@ -350,12 +392,26 @@ module VX_mm_axi_arb #( assign aw_pick = '0; assign aw_any = s_awvalid[0]; end else begin : g_aw_arb + // Once offered, the AW must stay unchanged until the slave takes it + // (AXI stability): only the offering master may compete meanwhile. + reg aw_held; + reg [SEL_WIDTH-1:0] aw_held_idx; + wire [NUM_INPUTS-1:0] aw_reqs = aw_held ? (s_awvalid & (NUM_INPUTS'(1) << aw_held_idx)) + : s_awvalid; + always @(posedge clk) begin + if (reset) begin + aw_held <= 1'b0; + end else begin + aw_held <= c_awvalid && ~c_awready; + end + aw_held_idx <= aw_pick; + end VX_generic_arbiter #( .NUM_REQS (NUM_INPUTS), .TYPE (ARBITER), .STICKY (1) ) aw_arb ( - .clk (clk), .reset (reset), .requests (s_awvalid), + .clk (clk), .reset (reset), .requests (aw_reqs), .grant_index (aw_pick), `UNUSED_PIN (grant_onehot), .grant_valid (aw_any), .grant_ready (aw_fire) ); From f93df327f70df5e35d353a1d1a40ff23282f1b75 Mon Sep 17 00:00:00 2001 From: tinebp Date: Fri, 23 Oct 2026 18:17:31 -0700 Subject: [PATCH 093/124] cache: route a crossbar input's ready from its own output, drain slices before a token VX_stream_xbar ORed every output's ready into each input's ready. Flat arbiters only grant valid requesters, so that equalled the selected output's ready. But a bank arbiter with a fan-in wider than 1.5 x MAX_FANOUT splits into registered slices, and a one-input slice passes its buffer's ready through unqualified. With 17 requesters on the L2 (16 L1 ports plus the command processor's DMA port, CP_LLC_CLIENT=1), another bank's slice acknowledged a CP write bound for a full bank, and the write was dropped: - sgemm read stale B (all 16384 outputs wrong, about 86% of expected) at the a100 shape, reduced down to one core with 4 warps x 16 threads, issue width 4, a 32-bank L2 and HBM timing; - at the default L2 geometry the kernel arguments read as 0xbaadf00d and VX_lsu_block's address assertion fired. Each input now takes its selected output's ready. The flush sequencer's in-flight counter, meant to hold a flush token until earlier requests reach the banks, was sized and wired so it never counted. The request crossbar's slices let requests from different inputs pass each other, so VX_cache_init now counts requests entering the crossbar (a merged read once, as its leader) and drains them before it walks or answers a token. Without slices (12 or fewer requesters, every default configuration) the counter is not built. Known-bad: without the change the reduced repro fails (16384 errors) and the default-geometry config asserts at cycle 12897. With it: both pass; sgemm at the a100 and rtx5080 4-core shapes with CP_LLC_CLIENT=1 pass; hw/unittest cp_llc_port and the CP LLC parity cases (dogfood-cp-llc, sgemm-cp-llc-17, sgemm-cp-llc-65) pass; default sgemm and raycast rtlsim cycles are identical (799610, 21106013); lint clean at XLEN 32 and 64. SimX already behaved correctly. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/cache/VX_cache.sv | 28 +++++++++++++++++++----- hw/rtl/cache/VX_cache_init.sv | 41 +++++++++++++++++------------------ hw/rtl/libs/VX_stream_xbar.sv | 5 ++++- 3 files changed, 47 insertions(+), 27 deletions(-) diff --git a/hw/rtl/cache/VX_cache.sv b/hw/rtl/cache/VX_cache.sv index 926f807452..fb54750a87 100644 --- a/hw/rtl/cache/VX_cache.sv +++ b/hw/rtl/cache/VX_cache.sv @@ -104,12 +104,26 @@ module VX_cache import VX_gpu_pkg::*; #( localparam MEM_PORTS_SEL_WIDTH = `UP(MEM_PORTS_SEL_BITS); localparam MEM_ARB_SEL_BITS = `CLOG2(`CDIV(NUM_BANKS, MEM_PORTS)); localparam REQ_XBAR_BUF = (NUM_REQS > 2) ? 2 : 0; + // A bank's request arbiter splits a fan-in wider than 1.5 x MAX_FANOUT + // into registered two-entry slices, where requests from different inputs + // wait apart: entries per bank over all slice levels (0 without slices). + function automatic integer req_xbar_slice_slots(integer n); + integer m, slots; + m = n; + slots = 0; + while (m > (`MAX_FANOUT + `MAX_FANOUT / 2)) begin + m = `CDIV(m, `MAX_FANOUT); + slots = slots + 2 * m; + end + return slots; + endfunction + localparam REQ_XBAR_SLICE_SLOTS = req_xbar_slice_slots(NUM_REQS); + // requests the crossbar can hold when slices let them pass each other; + // the flush sequencer drains them before it walks or answers a token + localparam REQ_XBAR_SIZE = (REQ_XBAR_SLICE_SLOTS != 0) ? (NUM_BANKS * (REQ_XBAR_SLICE_SLOTS + `TO_OUT_BUF_SIZE(REQ_XBAR_BUF))) : 0; localparam CORE_RSP_BUF_ENABLE = (NUM_BANKS != 1) || (NUM_REQS != 1); localparam MEM_REQ_BUF_ENABLE = (NUM_BANKS != 1); `STATIC_ASSERT(!DO_READ_MERGE || (`IS_POW2(MERGE_GROUP) && `IS_DIVISBLE(NUM_REQS, MERGE_GROUP)), ("invalid parameter: read merge group")) - // a merged request is one bank access for several port handshakes, which - // the in-flight request count of a registered request crossbar would miss - `STATIC_ASSERT(!DO_READ_MERGE || (`TO_OUT_BUF_REG(REQ_XBAR_BUF) == 0), ("invalid parameter: read merge with a registered request crossbar")) `ifdef PERF_ENABLE wire [NUM_BANKS-1:0] perf_read_miss_per_bank; wire [NUM_BANKS-1:0] perf_write_miss_per_bank; @@ -124,6 +138,7 @@ module VX_cache import VX_gpu_pkg::*; #( wire flush_skip; wire [`UP(UUID_WIDTH)-1:0] flush_uuid; wire [NUM_BANKS-1:0] per_bank_flush_end; + wire [NUM_REQS-1:0] core_req_xbar_fire; wire [NUM_BANKS-1:0] per_bank_core_req_fire; wire mem_req_empty; wire [MEM_PORTS-1:0] mem_flush_valid; @@ -143,13 +158,14 @@ module VX_cache import VX_gpu_pkg::*; #( .FLUSH_FORWARD (FLUSH_FORWARD), .SHARED_LLC (SHARED_LLC), .TAG_WIDTH (TAG_WIDTH), - .BANK_SEL_LATENCY (`TO_OUT_BUF_REG(REQ_XBAR_BUF)), // request xbar latency + .XBAR_SIZE (REQ_XBAR_SIZE), .REQ_SEL_WIDTH (REQ_SEL_WIDTH) ) cache_init ( .clk (clk), .reset (reset), .core_bus_in_if (core_bus_if), .core_bus_out_if (core_bus2_if), + .xbar_req_fire (core_req_xbar_fire), .bank_req_fire (per_bank_core_req_fire), .flush_begin (per_bank_flush_begin), .flush_skip (flush_skip), @@ -387,7 +403,9 @@ module VX_cache import VX_gpu_pkg::*; #( core_req_attr[i] }; end - assign per_bank_core_req_fire = per_bank_core_req_valid & per_bank_mem_req_ready; + // a merged read enters the crossbar once, as its leader + assign core_req_xbar_fire = core_req_valid & ~core_req_follower & core_req_xbar_ready; + assign per_bank_core_req_fire = per_bank_core_req_valid & per_bank_core_req_ready; `ifdef PERF_ENABLE wire [PERF_CTR_BITS-1:0] perf_collisions; `endif diff --git a/hw/rtl/cache/VX_cache_init.sv b/hw/rtl/cache/VX_cache_init.sv index 75c0aae526..28439232d6 100644 --- a/hw/rtl/cache/VX_cache_init.sv +++ b/hw/rtl/cache/VX_cache_init.sv @@ -23,6 +23,9 @@ // the cache's own response path, and the inputs reopen. // A token that arrives during the walk is visible only after all of its // input's earlier requests were accepted, so the walk covered them. +// A request crossbar that can hold requests apart from their input order +// (XBAR_SIZE != 0) drains into the banks first, while the inputs are held, +// so no earlier request of a token's input is passed by a later one. // An L1-only token (any byteen bit set: a next level with wider words sees // the bit at the token's line offset) is forwarded with its kind. A // last-level cache that alone serves every memory client is the point of @@ -43,13 +46,14 @@ module VX_cache_init import VX_gpu_pkg::*; #( parameter FLUSH_FORWARD = 1, // Forward the flush to the next level parameter SHARED_LLC = 0, // The LLC serving every memory client alone: answers L1-only tokens without a walk parameter TAG_WIDTH = UUID_WIDTH + 1, // core request tag size - parameter BANK_SEL_LATENCY = 1, // Bank select latency + parameter XBAR_SIZE = 0, // Requests the request crossbar can hold apart from input order parameter REQ_SEL_WIDTH = `UP(`CLOG2(NUM_REQS)) ) ( input wire clk, input wire reset, VX_mem_bus_if.slave core_bus_in_if [NUM_REQS], VX_mem_bus_if.master core_bus_out_if [NUM_REQS], + input wire [NUM_REQS-1:0] xbar_req_fire, input wire [NUM_BANKS-1:0] bank_req_fire, output wire [NUM_BANKS-1:0] flush_begin, output wire flush_skip, // the banks only quiesce: no walk is owed @@ -79,35 +83,30 @@ module VX_cache_init import VX_gpu_pkg::*; #( reg [2:0] state, state_n; - // track in-flight core requests + // track requests inside the request crossbar wire no_inflight_reqs; - if (BANK_SEL_LATENCY != 0) begin : g_bank_sel_latency + if (XBAR_SIZE != 0) begin : g_xbar_inflight - localparam NUM_REQS_W = `CLOG2(NUM_REQS+1); - localparam NUM_BANKS_W = `CLOG2(NUM_BANKS+1); + localparam REQS_W = `CLOG2(NUM_REQS+1); + localparam INCRW = `CLOG2(`MIN(NUM_REQS, XBAR_SIZE)+1); + localparam BANKS_W = `CLOG2(NUM_BANKS+1); - wire [NUM_REQS-1:0] core_bus_out_fire; - for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_bus_out_fire - assign core_bus_out_fire[i] = core_bus_out_if[i].req_valid && core_bus_out_if[i].req_ready; - end - - wire [NUM_REQS_W-1:0] core_bus_out_cnt; - wire [NUM_BANKS_W-1:0] bank_req_cnt; + wire [REQS_W-1:0] xbar_req_cnt; + wire [BANKS_W-1:0] bank_req_cnt; - `POP_COUNT(core_bus_out_cnt, core_bus_out_fire); + `POP_COUNT(xbar_req_cnt, xbar_req_fire); `POP_COUNT(bank_req_cnt, bank_req_fire); - `UNUSED_VAR (core_bus_out_cnt) VX_pending_size #( - .SIZE (BANK_SEL_LATENCY * NUM_BANKS), - .INCRW (NUM_BANKS_W), - .DECRW (NUM_BANKS_W) + .SIZE (XBAR_SIZE), + .INCRW (INCRW), + .DECRW (BANKS_W) ) pending_size ( .clk (clk), .reset (reset), - .incr (NUM_BANKS_W'(core_bus_out_cnt)), + .incr (INCRW'(xbar_req_cnt)), .decr (bank_req_cnt), .empty (no_inflight_reqs), `UNUSED_PIN (alm_empty), @@ -116,9 +115,9 @@ module VX_cache_init import VX_gpu_pkg::*; #( `UNUSED_PIN (size) ); - end else begin : g_no_bank_sel_latency + end else begin : g_no_xbar_inflight assign no_inflight_reqs = 0; - `UNUSED_VAR (bank_req_fire) + `UNUSED_VAR ({xbar_req_fire, bank_req_fire}) end wire [NUM_REQS-1:0] flush_req_mask; @@ -248,7 +247,7 @@ module VX_cache_init import VX_gpu_pkg::*; #( // STATE_IDLE: default: begin if (flush_req_enable) begin - if (BANK_SEL_LATENCY != 0) begin + if (XBAR_SIZE != 0) begin state_n = STATE_WAIT1; end else if (~flush_walk && ~any_io) begin rsp_pending_n = flush_req_mask; diff --git a/hw/rtl/libs/VX_stream_xbar.sv b/hw/rtl/libs/VX_stream_xbar.sv index e7b18362c2..c8002a0146 100644 --- a/hw/rtl/libs/VX_stream_xbar.sv +++ b/hw/rtl/libs/VX_stream_xbar.sv @@ -63,8 +63,11 @@ module VX_stream_xbar #( .data_out (per_output_ready_in_w) ); + // An output arbiter may assert ready for an input that does not + // target it (a single-input fan-in slice passes its buffer's + // ready through), so take the ready of the selected output only. for (genvar i = 0; i < NUM_INPUTS; ++i) begin : g_ready_in - assign ready_in[i] = | per_output_ready_in_w[i]; + assign ready_in[i] = per_output_ready_in_w[i][sel_in[i]]; end for (genvar i = 0; i < NUM_INPUTS; ++i) begin : g_sel_in_demux From c5ed76a73201861f38fc339f50748005ad81c730 Mon Sep 17 00:00:00 2001 From: tinebp Date: Fri, 23 Oct 2026 18:17:32 -0700 Subject: [PATCH 094/124] profiles: per-partition fetch and the CP as an LLC client on the GPU shapes All five GPU profiles now fetch per SM processing block (VX_CFG_FETCH_WIDTH=4: four partitions, each with its own instruction buffer and scheduler) and route the command processor's DMA through the last-level cache (VX_CFG_CP_LLC_CLIENT=1), as on the real parts. SimX at 4 cores: FETCH_WIDTH=4 gives sgemm -29.7% (rtx5080 -n192) and -40.7% (a100 -n128); CP_LLC_CLIENT a further -6.5% / -5.8%. Both pass in rtlsim at the a100 and rtx5080 4-core shapes. The rtx5080 DRAM latency is recalibrated from 339 to 675 cycles with the chase app (cold L2, CP_LLC_CLIENT=0): 802 cycles per dependent DRAM access. L2_LAZY_FETCH and the MSHR line table stay off: lazy fetch does not yet promote fully written sectors, so it costs the CP-through-LLC gain. Co-Authored-By: Claude Opus 5.5 --- profiles/a100.json | 10 +++++++--- profiles/h100.json | 10 +++++++--- profiles/rtx1080.json | 10 +++++++--- profiles/rtx5080.json | 12 ++++++++---- profiles/v100.json | 10 +++++++--- 5 files changed, 36 insertions(+), 16 deletions(-) diff --git a/profiles/a100.json b/profiles/a100.json index 7a6003fe0b..a9f778afe1 100644 --- a/profiles/a100.json +++ b/profiles/a100.json @@ -5,7 +5,8 @@ "VX_CFG_NUM_WARPS": 64, "VX_CFG_NUM_THREADS": 32, "VX_CFG_ISSUE_WIDTH": 4, - "VX_CFG_DCACHE_SIZE": 131072 + "VX_CFG_DCACHE_SIZE": 131072, + "VX_CFG_FETCH_WIDTH": 4 }, "chip": { "VX_CFG_NUM_CLUSTERS": 1, @@ -13,7 +14,8 @@ "VX_CFG_L2_ENABLE": true, "VX_CFG_L2_SIZE": 33554432, "VX_CFG_L2_NUM_WAYS": 16, - "VX_CFG_L2_NUM_BANKS": 32 + "VX_CFG_L2_NUM_BANKS": 32, + "VX_CFG_CP_LLC_CLIENT": 1 }, "platform": { "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 32, @@ -30,6 +32,8 @@ "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "40 MB L2 in 80 slices (whitepaper), rounded down to 32 MB and 32 banks; 16 ways as in GPGPU-Sim's SM80_A100 dl2", "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_DATA_RATE": "5 HBM2 stacks x 8 128-bit channels = 40 channels at 2430 MT/s, modelled as 32 channels at 3038 MT/s for a power-of-two interleave: 32 x 16 B x 3038 MT/s = 1555 GB/s, the card's bandwidth; the array timing is in nanoseconds, so latency is unchanged", "PLATFORM_MEMORY_LATENCY, PLATFORM_MEMORY_QUEUE_SIZE": "Fixed DRAM latency and per-channel request queue of GPGPU-Sim's SM80_A100 configuration (accel-sim gpu-simulator/configs/tested-cfgs/SM80_A100/gpgpusim.config)", - "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1410 MHz boost clock, 1555 GB/s (whitepaper, Table 3)" + "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1410 MHz boost clock, 1555 GB/s (whitepaper, Table 3)", + "FETCH_WIDTH": "4 SM processing blocks, each with its own warp scheduler, dispatch unit and L0 instruction cache (NVIDIA A100 Tensor Core GPU Architecture whitepaper)", + "CP_LLC_CLIENT": "Host copies reach device memory through the L2 like every other client, so uploaded data is L2-resident at launch" } } diff --git a/profiles/h100.json b/profiles/h100.json index d95d2fabda..5100b09169 100644 --- a/profiles/h100.json +++ b/profiles/h100.json @@ -5,7 +5,8 @@ "VX_CFG_NUM_WARPS": 64, "VX_CFG_NUM_THREADS": 32, "VX_CFG_ISSUE_WIDTH": 4, - "VX_CFG_DCACHE_SIZE": 262144 + "VX_CFG_DCACHE_SIZE": 262144, + "VX_CFG_FETCH_WIDTH": 4 }, "chip": { "VX_CFG_NUM_CLUSTERS": 1, @@ -13,7 +14,8 @@ "VX_CFG_L2_ENABLE": true, "VX_CFG_L2_SIZE": 33554432, "VX_CFG_L2_NUM_WAYS": 16, - "VX_CFG_L2_NUM_BANKS": 32 + "VX_CFG_L2_NUM_BANKS": 32, + "VX_CFG_CP_LLC_CLIENT": 1 }, "platform": { "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 32, @@ -27,6 +29,8 @@ "DCACHE_SIZE": "256 KB combined L1/shared per SM (whitepaper)", "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "50 MB L2 (whitepaper), rounded down to the power of two the cache accepts; 16 ways and 32 banks as in the a100 profile", "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_DATA_RATE": "5 HBM3 stacks x 16 64-bit channels at 5236 MT/s = 40 128-bit channel pairs, modelled as 32 at 6545 MT/s for a power-of-two interleave: 32 x 16 B x 6545 MT/s = 3.35 TB/s, the card's bandwidth; latency not calibrated", - "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1980 MHz boost clock, 3.35 TB/s (H100 SXM5 datasheet)" + "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1980 MHz boost clock, 3.35 TB/s (H100 SXM5 datasheet)", + "FETCH_WIDTH": "4 SM processing blocks, each with its own warp scheduler, dispatch unit and L0 instruction cache (NVIDIA H100 Tensor Core GPU Architecture whitepaper)", + "CP_LLC_CLIENT": "Host copies reach device memory through the L2 like every other client, so uploaded data is L2-resident at launch" } } diff --git a/profiles/rtx1080.json b/profiles/rtx1080.json index 73908bb3d2..3c95172e2c 100644 --- a/profiles/rtx1080.json +++ b/profiles/rtx1080.json @@ -5,7 +5,8 @@ "VX_CFG_NUM_WARPS": 64, "VX_CFG_NUM_THREADS": 32, "VX_CFG_ISSUE_WIDTH": 4, - "VX_CFG_DCACHE_SIZE": 32768 + "VX_CFG_DCACHE_SIZE": 32768, + "VX_CFG_FETCH_WIDTH": 4 }, "chip": { "VX_CFG_NUM_CLUSTERS": 1, @@ -13,7 +14,8 @@ "VX_CFG_L2_ENABLE": true, "VX_CFG_L2_SIZE": 2097152, "VX_CFG_L2_NUM_WAYS": 16, - "VX_CFG_L2_NUM_BANKS": 16 + "VX_CFG_L2_NUM_BANKS": 16, + "VX_CFG_CP_LLC_CLIENT": 1 }, "platform": { "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 16, @@ -27,6 +29,8 @@ "DCACHE_SIZE": "48 KB unified L1/texture per SM (whitepaper), rounded down to the power of two the cache accepts", "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "2 MB L2 (whitepaper); 16 ways, 16 banks for the 16 memory channels", "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_DATA_RATE": "256-bit GDDR5X at 10 Gbps modelled as 16 GDDR6 x16 channels at 10000 MT/s: 16 x 2 B x 10 GT/s = 320 GB/s, the card's bandwidth; latency not calibrated", - "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1607 MHz base clock, 320 GB/s (whitepaper)" + "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1607 MHz base clock, 320 GB/s (whitepaper)", + "FETCH_WIDTH": "4 SM processing blocks, each with its own instruction buffer and warp scheduler (NVIDIA GeForce GTX 1080 (GP104) whitepaper)", + "CP_LLC_CLIENT": "Host copies reach device memory through the L2 like every other client, so uploaded data is L2-resident at launch" } } diff --git a/profiles/rtx5080.json b/profiles/rtx5080.json index d90b047d99..f703f3e82b 100644 --- a/profiles/rtx5080.json +++ b/profiles/rtx5080.json @@ -6,7 +6,8 @@ "VX_CFG_NUM_THREADS": 32, "VX_CFG_ISSUE_WIDTH": 4, "VX_CFG_DCACHE_SIZE": 131072, - "VX_CFG_DCACHE_MSHR_SIZE": 24 + "VX_CFG_DCACHE_MSHR_SIZE": 24, + "VX_CFG_FETCH_WIDTH": 4 }, "chip": { "VX_CFG_NUM_CLUSTERS": 1, @@ -15,7 +16,8 @@ "VX_CFG_L2_SIZE": 67108864, "VX_CFG_L2_NUM_WAYS": 16, "VX_CFG_L2_NUM_BANKS": 32, - "VX_CFG_L2_MSHR_SIZE": 192 + "VX_CFG_L2_MSHR_SIZE": 192, + "VX_CFG_CP_LLC_CLIENT": 1 }, "platform": { "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 32, @@ -23,7 +25,7 @@ "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 960000, "VX_CFG_PLATFORM_MEMORY_TYPE": "GDDR6", "VX_CFG_PLATFORM_MEMORY_DATA_RATE": 15000, - "VX_CFG_PLATFORM_MEMORY_LATENCY": 339 + "VX_CFG_PLATFORM_MEMORY_LATENCY": 675 }, "reference": { "clock_mhz": 2295, @@ -58,6 +60,8 @@ "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "2295 MHz base clock, 960 GB/s (whitepaper App. B)", "reference": "Nsight Compute on an RTX 5080 at its 2295 MHz base clock with a cold L2, 2026-10-05", "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_DATA_RATE": "30 Gbps GDDR7 x8 channels modelled as GDDR6 x16 channels at 15000 MT/s: the same channel count, 32 B burst time and 960 GB/s (32 x 2 B x 15 GT/s)", - "PLATFORM_MEMORY_LATENCY": "339 core cycles of PHY/NoC latency: the 407 calibrated in SimX against the 802-cycle pointer-chase DRAM latency in 'reference', less the 68 cycles JEDEC GDDR6 timing adds to the device's row-conflict latency (derived, not re-run; recalibrate with a pointer chase when the memory model changes)" + "PLATFORM_MEMORY_LATENCY": "675 core cycles of PHY/NoC latency, calibrated in SimX against the 802-cycle pointer-chase DRAM latency in 'reference' with the same chase (random cyclic permutation, 128 B stride, one thread, cold L2 with CP_LLC_CLIENT=0; cycle difference between 128 KiB and 256 KiB footprints over the 1024 extra loads: 802.1). The L2 link latency is not modelled, so an L2 hit is about 79 cycles against the card's 354. Recalibrate when the memory model changes", + "FETCH_WIDTH": "4 SM processing blocks, each with its own warp scheduler, dispatch unit and L0 instruction cache (NVIDIA RTX Blackwell GPU Architecture whitepaper)", + "CP_LLC_CLIENT": "Host copies reach device memory through the L2 like every other client, so uploaded data is L2-resident at launch" } } diff --git a/profiles/v100.json b/profiles/v100.json index 451551d910..4bccc586bd 100644 --- a/profiles/v100.json +++ b/profiles/v100.json @@ -5,7 +5,8 @@ "VX_CFG_NUM_WARPS": 64, "VX_CFG_NUM_THREADS": 32, "VX_CFG_ISSUE_WIDTH": 4, - "VX_CFG_DCACHE_SIZE": 131072 + "VX_CFG_DCACHE_SIZE": 131072, + "VX_CFG_FETCH_WIDTH": 4 }, "chip": { "VX_CFG_NUM_CLUSTERS": 1, @@ -13,7 +14,8 @@ "VX_CFG_L2_ENABLE": true, "VX_CFG_L2_SIZE": 4194304, "VX_CFG_L2_NUM_WAYS": 16, - "VX_CFG_L2_NUM_BANKS": 32 + "VX_CFG_L2_NUM_BANKS": 32, + "VX_CFG_CP_LLC_CLIENT": 1 }, "platform": { "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 32, @@ -27,6 +29,8 @@ "DCACHE_SIZE": "128 KB combined L1/shared per SM (whitepaper)", "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "6 MB L2 (whitepaper), rounded down to the power of two the cache accepts; 16 ways, 32 banks for the 32 memory channels", "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_DATA_RATE": "4 HBM2 stacks x 8 128-bit channels = 32 channels at 1752 MT/s: 32 x 16 B x 1752 MT/s = 897 GB/s (whitepaper)", - "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1530 MHz boost clock, 900 GB/s (whitepaper, Table 1)" + "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1530 MHz boost clock, 900 GB/s (whitepaper, Table 1)", + "FETCH_WIDTH": "4 SM processing blocks, each with its own warp scheduler, dispatch unit and L0 instruction cache (NVIDIA Tesla V100 GPU Architecture whitepaper)", + "CP_LLC_CLIENT": "Host copies reach device memory through the L2 like every other client, so uploaded data is L2-resident at launch" } } From 1a3055f04d583a75106f985e68f448034c3cd2f5 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 24 Oct 2026 03:05:07 -0700 Subject: [PATCH 095/124] Support core, socket and cluster counts that are not powers of two Real GPUs are not built in powers of two (RTX 5080 84 SMs, A100 108, H100 132), and SimX already ran those shapes, but Vortex.sv asserted power-of-two NUM_CLUSTERS, NUM_CORES and SOCKET_SIZE, so rtlsim and the FPGA could not build them. The requirement is now only that NUM_CORES is a multiple of SOCKET_SIZE, and the sites that sliced core, socket or cluster IDs by bits use constant arithmetic that reduces to the same bits for powers of two: - KMU fragment routing picks the next hop with (dest / DEST_DIV) % N. - The DXA local-memory core select uses % NUM_CORES, then % SOCKET_SIZE. - The AMO hart id dropped the cluster bits, so cores in different clusters aliased in the LR/SC reservation table; it now carries the global core id. - The command processor exposes the exact SOCKET_SIZE in a new read-only capability word (0x00C), so the runtime reports the right core count at socket sizes that are not powers of two. Older CPs answer that offset with garbage, so a word with its reserved bits set is ignored. - SimX rejects RTU configurations whose socket size is not a multiple of the RTU core count, matching what the RTL can build. Known-bad: vecadd at 3 cores + L2 hit the power-of-two assert; sgemm2_dxa at 4 cores with sockets of 2 hung (core 1 scheduler timeout) under an intermediate version of this change. Both pass now. Validation: sgemm, vecadd, async_gbarrier, dogfood and printf at 3 cores, 6 cores (issue width 4) and 3 clusters x 2 cores (L2 + L3) pass on rtlsim and SimX with matching instruction counts; sgemm at 3 cores + L2 is 1951807 cycles on rtlsim and 1972907 on SimX (+1.08%). Power-of-two configurations are unchanged: default sgemm 5943507, raycast 21106013, 4-core + L2 sgemm 1477825 cycles. Lint clean in 22 shapes at XLEN 32 and 64. New CI cases cover 3 and 6 cores, and DXA at 4 cores in sockets of 2 and 6 cores in sockets of 3. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/cache.yaml | 3 +- ci/testcases/config1.yaml | 39 ++++++++++++++++++++++ ci/testcases/dxa.yaml | 39 ++++++++++++++++++++++ docs/assets/img/cta_dispatch_hierarchy.svg | 6 ++-- docs/designs/command_processor.md | 6 ++++ docs/designs/cta_dispatch_architecture.md | 10 +++--- hw/rtl/VX_cluster.sv | 4 +-- hw/rtl/VX_gpu_pkg.sv | 16 +++++---- hw/rtl/VX_socket.sv | 10 +++--- hw/rtl/Vortex.sv | 7 ++-- hw/rtl/core/VX_kmu_bus_arb.sv | 11 +++--- hw/rtl/cp/VX_cp_axil_regfile.sv | 12 +++++-- hw/rtl/dxa/VX_dxa_unit.sv | 6 ++-- hw/rtl/interfaces/VX_kmu_bus_if.sv | 2 +- hw/unittest/cp_axil_regfile/main.cpp | 14 +++++--- sim/common/cmd_processor.cpp | 10 ++++-- sim/common/cmd_processor.h | 1 + sim/simx/constants.h | 2 ++ sim/simx/rtu/rtu_types.h | 2 ++ sw/runtime/common/caps.h | 33 +++++++++++------- sw/runtime/common/device.cpp | 6 ++-- sw/runtime/common/vortex2_internal.h | 5 +-- 22 files changed, 181 insertions(+), 63 deletions(-) diff --git a/ci/testcases/cache.yaml b/ci/testcases/cache.yaml index 92148b00ed..9b52dab9ee 100644 --- a/ci/testcases/cache.yaml +++ b/ci/testcases/cache.yaml @@ -264,7 +264,7 @@ tests: args: -n1 # Non-power-of-two cache sharing: 3 cores over 2 shared icaches exercises the -# SimX arbiter's ceiling-division input grouping. A floor-grouped arbiter +# arbiter's ceiling-division input grouping. A floor-grouped arbiter # (inputs/outputs) never polls the trailing core's fetch port and the run # hangs at boot; VX_stream_arb sizes its groups by ceiling division, so the # SimX model must match. @@ -272,6 +272,7 @@ tests: via: blackbox drivers: - simx + - rtlsim app: demo configs: -DVX_CFG_SOCKET_SIZE=3 -DVX_CFG_NUM_ICACHES=2 shape: diff --git a/ci/testcases/config1.yaml b/ci/testcases/config1.yaml index cb2d49aa71..f8ef17bbcb 100644 --- a/ci/testcases/config1.yaml +++ b/ci/testcases/config1.yaml @@ -174,6 +174,45 @@ tests: shape: cores: 4 args: -n1 +# Core and cluster counts that are not powers of two: the shared L2 serves +# three sockets, the global barrier counts three cores, and three clusters +# share the L3. +- id: async_gbarrier-npot-1 + via: blackbox + drivers: + - rtlsim + app: async_gbarrier + shape: + cores: 3 + l2cache: true +- id: async_gbarrier-npot-2 + via: blackbox + drivers: + - simx + app: async_gbarrier + shape: + cores: 3 + l2cache: true +- id: vecadd-npot-1 + via: blackbox + drivers: + - rtlsim + app: vecadd + shape: + cores: 2 + clusters: 3 + l2cache: true + l3cache: true +- id: vecadd-npot-2 + via: blackbox + drivers: + - simx + app: vecadd + shape: + cores: 2 + clusters: 3 + l2cache: true + l3cache: true - id: diverge-17 via: blackbox drivers: diff --git a/ci/testcases/dxa.yaml b/ci/testcases/dxa.yaml index 913c71cb57..dad905a839 100644 --- a/ci/testcases/dxa.yaml +++ b/ci/testcases/dxa.yaml @@ -353,6 +353,45 @@ tests: configs: -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_DXA_UNITS=4 -DVX_CFG_SOCKET_SIZE=2 shape: cores: 4 +# Shared L2 behind multi-core sockets: the socket routes each DXA lmem write +# to the core named by its cluster-local index modulo the socket size, at a +# power-of-two shape and at sockets of three. +- id: sgemm2_dxa-l2-1 + via: blackbox + drivers: + - simx + app: sgemm2_dxa + configs: -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_DXA_UNITS=4 -DVX_CFG_SOCKET_SIZE=2 + shape: + cores: 4 + l2cache: true +- id: sgemm2_dxa-l2-2 + via: blackbox + drivers: + - rtlsim + app: sgemm2_dxa + configs: -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_DXA_UNITS=4 -DVX_CFG_SOCKET_SIZE=2 + shape: + cores: 4 + l2cache: true +- id: sgemm2_dxa-npot-1 + via: blackbox + drivers: + - simx + app: sgemm2_dxa + configs: -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_DXA_UNITS=4 -DVX_CFG_SOCKET_SIZE=3 + shape: + cores: 6 + l2cache: true +- id: sgemm2_dxa-npot-2 + via: blackbox + drivers: + - rtlsim + app: sgemm2_dxa + configs: -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_DXA_UNITS=4 -DVX_CFG_SOCKET_SIZE=3 + shape: + cores: 6 + l2cache: true - id: sgemm2_dxa-9 via: blackbox drivers: diff --git a/docs/assets/img/cta_dispatch_hierarchy.svg b/docs/assets/img/cta_dispatch_hierarchy.svg index 64634eab83..ca0ce66ba3 100644 --- a/docs/assets/img/cta_dispatch_hierarchy.svg +++ b/docs/assets/img/cta_dispatch_hierarchy.svg @@ -56,7 +56,7 @@ kmu_arb — Vortex.sv:186 1 -> NUM_CLUSTERS - DEST_LSB_DEVICE + DEST_DIV_DEVICE @@ -64,7 +64,7 @@ kmu_arb — VX_cluster.sv:105 2 -> NUM_SOCKETS, ARBITER "R" - DEST_LSB_CLUSTER + DEST_DIV_CLUSTER the only level with 2 inputs @@ -83,7 +83,7 @@ kmu_arb — VX_socket.sv:69 1 -> VX_CFG_SOCKET_SIZE - DEST_LSB_SOCKET + DEST_DIV_SOCKET socket 1..N diff --git a/docs/designs/command_processor.md b/docs/designs/command_processor.md index 356ddb000b..9e0dd9c5a2 100644 --- a/docs/designs/command_processor.md +++ b/docs/designs/command_processor.md @@ -349,6 +349,7 @@ subtracts it first. | `0x000` | `CP_CTRL` | RW | bit0 = enable_global, bit1 = reset_all | | `0x004` | `CP_STATUS` | RO | bit0 = busy, bit1 = error | | `0x008` | `CP_DEV_CAPS` | RO | `[7:0]` NUM_QUEUES, `[15:8]` RING_SIZE_LOG2_MAX, `[23:16]` AXI_TID_WIDTH — plus feature bits 24-26 on the Emulation CP (§6) | +| `0x00C` | `GPU_DEV_CAPS_EXT` | RO | device-config caps extension (§6) | | `0x010` | `CP_CYCLE_LO` | RO | free-running cycle counter, low 32 | | `0x014` | `CP_CYCLE_HI` | RO | high 32 | | `0x018` / `0x01C` | `GPU_DEV_CAPS_LO/HI` | RO | device-config caps (§6) | @@ -434,6 +435,11 @@ The exact counts in `[63:42]` let a machine that is not a power of two falls back to the log2 fields only when they read zero, as on devices built before they existed. +**`GPU_DEV_CAPS_EXT`** (`0x00C`) carries the exact counts that no longer fit +in `GPU_DEV_CAPS`: `[7:0]` socket_size (`VX_CAPS_SOCKET_SIZE`), the rest zero. +It reads zero on devices built before it existed, so the decoder falls back to +the log2 socket_size field `[16:14]`. + `VX_CAPS_NUM_CORES` is derived, not a field: `num_clusters × cluster_size × socket_size`, from the exact counts. `CACHE_LINE_SIZE`, `GLOBAL_MEM_SIZE`, `CLOCK_RATE`, `PEAK_MEM_BW` and the VM ids are not diff --git a/docs/designs/cta_dispatch_architecture.md b/docs/designs/cta_dispatch_architecture.md index 7906fd124a..48c6aa0a85 100644 --- a/docs/designs/cta_dispatch_architecture.md +++ b/docs/designs/cta_dispatch_architecture.md @@ -41,12 +41,12 @@ between them, so the slave-port `ready` would cross both stages combinationally. Where a level needs both, it composes two instances around a registered launch link: -| Level | File | Shape | `dest` slice | +| Level | File | Shape | `dest` digit | |---|---|---|---| -| Device | [`Vortex.sv:185`](../../hw/rtl/Vortex.sv#L185) | 1 → `VX_CFG_NUM_CLUSTERS` | `KMU_DEST_LSB_DEVICE` | +| Device | [`Vortex.sv:185`](../../hw/rtl/Vortex.sv#L185) | 1 → `VX_CFG_NUM_CLUSTERS` | `KMU_DEST_DIV_DEVICE` | | Cluster merge | [`VX_cluster.sv:121`](../../hw/rtl/VX_cluster.sv#L121) | 2 → 1 | — | -| Cluster distribute | [`VX_cluster.sv:133`](../../hw/rtl/VX_cluster.sv#L133) | 1 → `NUM_SOCKETS` | `KMU_DEST_LSB_CLUSTER` | -| Socket | [`VX_socket.sv:72`](../../hw/rtl/VX_socket.sv#L72) | 1 → `VX_CFG_SOCKET_SIZE` | `KMU_DEST_LSB_SOCKET` | +| Cluster distribute | [`VX_cluster.sv:133`](../../hw/rtl/VX_cluster.sv#L133) | 1 → `NUM_SOCKETS` | `KMU_DEST_DIV_CLUSTER` | +| Socket | [`VX_socket.sv:72`](../../hw/rtl/VX_socket.sv#L72) | 1 → `VX_CFG_SOCKET_SIZE` | `KMU_DEST_DIV_SOCKET` | `NUM_SOCKETS = UP(VX_CFG_NUM_CORES / VX_CFG_SOCKET_SIZE)` ([`VX_gpu_pkg.sv:149`](../../hw/rtl/VX_gpu_pkg.sv#L149)). The cluster merge is @@ -228,7 +228,7 @@ Two things are worth noticing about what is *not* here: placement hint, so the fan-out drops its first member on any ready core and the message lock keeps the rest with it. This is what spreads a grid across the machine. -- **FRAGMENT** → routed by this level's slice of `dest`. Fragment work is +- **FRAGMENT** → routed by this level's digit of `dest`. Fragment work is bin→core affine, and that affinity is what keeps same-pixel blend order correct; load-balancing it would break the blend order outright. diff --git a/hw/rtl/VX_cluster.sv b/hw/rtl/VX_cluster.sv index 1b2ff700c5..5dfcb2f4b9 100644 --- a/hw/rtl/VX_cluster.sv +++ b/hw/rtl/VX_cluster.sv @@ -145,7 +145,7 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; VX_kmu_bus_arb #( .NUM_INPUTS (1), .NUM_OUTPUTS (NUM_SOCKETS), - .DEST_LSB (KMU_DEST_LSB_CLUSTER), + .DEST_DIV (KMU_DEST_DIV_CLUSTER), .OUT_BUF ((NUM_SOCKETS > 1) ? 3 : 0) ) kmu_dist ( .clk (clk), @@ -157,7 +157,7 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; VX_kmu_bus_arb #( .NUM_INPUTS (1), .NUM_OUTPUTS (NUM_SOCKETS), - .DEST_LSB (KMU_DEST_LSB_CLUSTER), + .DEST_DIV (KMU_DEST_DIV_CLUSTER), .OUT_BUF ((NUM_SOCKETS > 1) ? 3 : 0) ) kmu_arb ( .clk (clk), diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index 55e2d64049..904bad773c 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -155,8 +155,9 @@ package VX_gpu_pkg; // table keys on hart_id = make_hart_id(cid, wid, tid). // MINU/MAXU collapse into MIN/MAX + amo_unsigned. - // make_hart_id(cid, wid, tid) — packed concatenation, low bits = tid. - localparam HART_ID_BITS = NC_BITS + NW_BITS + NT_BITS; + // make_hart_id(cid, wid, tid) — packed concatenation, low bits = tid; cid is + // device-wide, so harts of different clusters stay distinct at a shared LLC. + localparam HART_ID_BITS = CORE_ID_BITS + NW_BITS + NT_BITS; localparam HART_ID_WIDTH = `UP(HART_ID_BITS); // ─────────────────────────────────────────────────────────────────────── @@ -944,16 +945,17 @@ package VX_gpu_pkg; // Placement hint: the global index of the core a fragment launch must land // on. Fragment work is bin->core affine (that is what makes same-pixel blend // order safe), so a fragment message cannot be load-balanced like a compute - // CTA. Each fan-out level consumes its own slice of this index, LSB first: - // core-in-socket, then socket-in-cluster, then cluster (see KMU_DEST_LSB_*). + // CTA. Each fan-out level picks its output as (dest / DIV) % NUM_OUTPUTS, + // where DIV is the number of cores below each of its outputs: core-in-socket, + // then socket-in-cluster, then cluster (see KMU_DEST_DIV_*). // Device-wide, so the hint can name any core: VX_CFG_NUM_CORES is per-cluster. // `UP` so a single-core build still has a 1-bit (unused) hint rather than a // zero-width vector. localparam KMU_DEST_W = `UP(`CLOG2(`VX_CFG_NUM_CLUSTERS * `VX_CFG_NUM_CORES)); - localparam KMU_DEST_LSB_SOCKET = 0; - localparam KMU_DEST_LSB_CLUSTER = `CLOG2(`VX_CFG_SOCKET_SIZE); - localparam KMU_DEST_LSB_DEVICE = KMU_DEST_LSB_CLUSTER + `CLOG2(NUM_SOCKETS); + localparam KMU_DEST_DIV_SOCKET = 1; + localparam KMU_DEST_DIV_CLUSTER = `VX_CFG_SOCKET_SIZE; + localparam KMU_DEST_DIV_DEVICE = KMU_DEST_DIV_CLUSTER * NUM_SOCKETS; typedef struct packed { logic [NCTA_WIDTH-1:0] cta_id; diff --git a/hw/rtl/VX_socket.sv b/hw/rtl/VX_socket.sv index 3df30eaef2..3734c2771d 100644 --- a/hw/rtl/VX_socket.sv +++ b/hw/rtl/VX_socket.sv @@ -78,7 +78,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; VX_kmu_bus_arb #( .NUM_INPUTS (1), .NUM_OUTPUTS (`VX_CFG_SOCKET_SIZE), - .DEST_LSB (KMU_DEST_LSB_SOCKET), + .DEST_DIV (KMU_DEST_DIV_SOCKET), .OUT_BUF ((`VX_CFG_SOCKET_SIZE > 1) ? 3 : 0) ) kmu_arb ( .clk (clk), @@ -727,9 +727,9 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; `UNUSED_PIN (busy) ); - // Route DXA lmem requests to per-core buses using core_local_id from tag. - // Tag value layout: {core_id[NC_BITS-1:0], engine_value[0]} - // core_local_id = core_id[CORE_LOCAL_BITS-1:0] + // Route DXA lmem requests to per-core buses by the core named in the tag. + // Tag value layout: {core_id[NC_BITS-1:0], engine_value[0]}, core_id being + // the cluster-local index, so the core within this socket is core_id % SOCKET_SIZE. localparam DXA_LMEM_CORE_LOCAL_BITS = `CLOG2(`VX_CFG_SOCKET_SIZE); VX_mem_bus_if #( .DATA_SIZE (DXA_LMEM_WORD_SIZE), @@ -740,7 +740,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; wire [`UP(DXA_LMEM_CORE_LOCAL_BITS)-1:0] dxa_lmem_core_sel; if (`VX_CFG_SOCKET_SIZE > 1) begin : g_dxa_lmem_sel - assign dxa_lmem_core_sel = dxa_lmem_bus_if[0].req_data.tag.value[1 +: DXA_LMEM_CORE_LOCAL_BITS]; + assign dxa_lmem_core_sel = DXA_LMEM_CORE_LOCAL_BITS'(dxa_lmem_bus_if[0].req_data.tag.value[1 +: NC_BITS] % `VX_CFG_SOCKET_SIZE); end else begin : g_dxa_lmem_sel assign dxa_lmem_core_sel = '0; end diff --git a/hw/rtl/Vortex.sv b/hw/rtl/Vortex.sv index 59bebc3865..320693faa3 100644 --- a/hw/rtl/Vortex.sv +++ b/hw/rtl/Vortex.sv @@ -64,10 +64,7 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( input wire start, output wire busy ); - // Check clustering configuration - `STATIC_ASSERT(`IS_POW2(`VX_CFG_NUM_CLUSTERS), ("NUM_CLUSTERS must be a power of 2")); - `STATIC_ASSERT(`IS_POW2(`VX_CFG_NUM_CORES), ("NUM_CORES must be a power of 2")); - `STATIC_ASSERT(`IS_POW2(`VX_CFG_SOCKET_SIZE), ("SOCKET_SIZE must be a power of 2")); + `STATIC_ASSERT((`VX_CFG_NUM_CORES % `VX_CFG_SOCKET_SIZE) == 0, ("NUM_CORES must be a multiple of SOCKET_SIZE")); // A write-back LLC must be the only instance: replicated ones never make // their dirty lines visible to each other. @@ -279,7 +276,7 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( VX_kmu_bus_arb #( .NUM_INPUTS (1), .NUM_OUTPUTS (`VX_CFG_NUM_CLUSTERS), - .DEST_LSB (KMU_DEST_LSB_DEVICE), + .DEST_DIV (KMU_DEST_DIV_DEVICE), .OUT_BUF ((`VX_CFG_NUM_CLUSTERS > 1) ? 3 : 0) // register per-cluster kmu fan-out (SLR-crossing skid) ) kmu_arb ( .clk (clk), diff --git a/hw/rtl/core/VX_kmu_bus_arb.sv b/hw/rtl/core/VX_kmu_bus_arb.sv index 8ce6d4dcbb..55a87eb547 100644 --- a/hw/rtl/core/VX_kmu_bus_arb.sv +++ b/hw/rtl/core/VX_kmu_bus_arb.sv @@ -27,16 +27,16 @@ // lock would never release and would starve every other master forever. // // Fan-out picks the output by kind: -// FRAGMENT — routed by `dest`, this level's slice (DEST_LSB). Fragment work is -// bin->core affine; load-balancing it would break same-pixel blend -// order. +// FRAGMENT — routed by `dest`, this level's digit (dest / DEST_DIV) % NUM_OUTPUTS. +// Fragment work is bin->core affine; load-balancing it would break +// same-pixel blend order. // COMPUTE — round-robin over the ready outputs, which is what the plain // stream arbiter did before and what keeps CTAs spread across cores. module VX_kmu_bus_arb import VX_gpu_pkg::*; #( parameter NUM_INPUTS = 1, parameter NUM_OUTPUTS = 1, - parameter DEST_LSB = 0, // bit offset of this level's slice of `dest` + parameter DEST_DIV = 1, // cores below each output: this level's digit of `dest` parameter OUT_BUF = 0, parameter `STRING ARBITER = "R" ) ( @@ -167,7 +167,7 @@ module VX_kmu_bus_arb import VX_gpu_pkg::*; #( reg [SEL_O-1:0] lock_sel; reg [SEL_O-1:0] rr_ptr; - wire [SEL_O-1:0] dest_sel = SEL_O'(m_data[PW_DEST +: KMU_DEST_W] >> DEST_LSB); + wire [SEL_O-1:0] dest_sel = SEL_O'((32'(m_data[PW_DEST +: KMU_DEST_W]) / DEST_DIV) % NUM_OUTPUTS); // Round-robin over the ready outputs, starting at rr_ptr. Descending // scan so the nearest ready output (smallest step from rr_ptr) is the @@ -213,6 +213,7 @@ module VX_kmu_bus_arb import VX_gpu_pkg::*; #( end else begin : g_fanout_1 + `UNUSED_PARAM (DEST_DIV) assign out_valid[0] = m_valid; assign out_data[0] = m_data; assign m_ready = out_ready[0]; diff --git a/hw/rtl/cp/VX_cp_axil_regfile.sv b/hw/rtl/cp/VX_cp_axil_regfile.sv index 531e5e064d..96a0ca2fbc 100644 --- a/hw/rtl/cp/VX_cp_axil_regfile.sv +++ b/hw/rtl/cp/VX_cp_axil_regfile.sv @@ -16,6 +16,7 @@ // 0x004 CP_STATUS RO bit0=busy, bit1=error // 0x008 CP_DEV_CAPS RO [7:0]NUM_QUEUES | [15:8]RING_SIZE_LOG2_MAX // [23:16]AXI_TID_WIDTH | [28]DMA_LLC_CLIENT +// 0x00C GPU_DEV_CAPS_EXT RO device-config caps extension: [7:0]SOCKET_SIZE // 0x010 CP_CYCLE_LO RO free-running cycle counter low 32 bits // 0x014 CP_CYCLE_HI RO high 32 bits // 0x018 GPU_DEV_CAPS_LO RO device-config caps, low 32 bits @@ -130,7 +131,8 @@ module VX_cp_axil_regfile localparam int GPU_BANK_ADDR_W = `VX_CFG_PLATFORM_MEMORY_ADDR_WIDTH - `CLOG2(`VX_CFG_PLATFORM_MEMORY_NUM_BANKS); - `STATIC_ASSERT(`VX_CFG_NUM_WARPS < 128 && `VX_CFG_NUM_CLUSTERS < 128 && GPU_CLUSTER_SIZE < 256, + `STATIC_ASSERT(`VX_CFG_NUM_WARPS < 128 && `VX_CFG_NUM_CLUSTERS < 128 && GPU_CLUSTER_SIZE < 256 + && `VX_CFG_SOCKET_SIZE < 256, ("GPU_DEV_CAPS exact-count fields overflow")) wire [63:0] gpu_dev_caps = { @@ -149,6 +151,11 @@ module VX_cp_axil_regfile 8'(`VX_ISA_IMPL_ID) }; + wire [31:0] gpu_dev_caps_ext = { + 24'd0, + 8'(`VX_CFG_SOCKET_SIZE) + }; + wire [63:0] gpu_isa_caps = { 32'(`VX_CFG_MISA_EXT), 2'(`CLOG2(`VX_CFG_XLEN)-4), @@ -212,6 +219,7 @@ module VX_cp_axil_regfile 8'(AXI_TID_W), 8'(RING_SIZE_LOG2_MAX), 8'(NUM_QUEUES)}; + 6'h0C: return gpu_dev_caps_ext; 6'h10: return r_cycle_count[31:0]; 6'h14: return r_cycle_count[63:32]; 6'h18: return gpu_dev_caps[31:0]; @@ -226,7 +234,7 @@ module VX_cp_axil_regfile // reports VM_ENABLED=0 in CP_DEV_CAPS and the runtime never writes // them. Reading zero is the honest answer; implement them for real if // an MMU is added. - 6'h0C, 6'h28, 6'h2C: return 32'd0; + 6'h28, 6'h2C: return 32'd0; // AFU host-port drain counters (debug, read-only). 6'h30: return dbg_host_w_counts; 6'h34: return dbg_host_r_counts; diff --git a/hw/rtl/dxa/VX_dxa_unit.sv b/hw/rtl/dxa/VX_dxa_unit.sv index cd3d8c8b13..385d56db7f 100644 --- a/hw/rtl/dxa/VX_dxa_unit.sv +++ b/hw/rtl/dxa/VX_dxa_unit.sv @@ -59,10 +59,8 @@ module VX_dxa_unit import VX_gpu_pkg::*, VX_dxa_pkg::*; #( // Build dxa_req payload dxa_req_data_t dxa_req_data_in; - assign dxa_req_data_in.core_id = NC_WIDTH'(core_id); - if (CORE_ID_WIDTH > NC_WIDTH) begin : g_core_id_hi_unused - `UNUSED_VAR (core_id[CORE_ID_WIDTH-1:NC_WIDTH]) - end + // cluster-local core index + assign dxa_req_data_in.core_id = NC_WIDTH'(core_id % `VX_CFG_NUM_CORES); assign dxa_req_data_in.uuid = execute_if.data.header.uuid; assign dxa_req_data_in.wid = execute_if.data.header.wid; assign dxa_req_data_in.smem_addr = lmem_rel_byte_addr[DXA_SMEM_ADDR_W-1:0]; diff --git a/hw/rtl/interfaces/VX_kmu_bus_if.sv b/hw/rtl/interfaces/VX_kmu_bus_if.sv index a1f1936898..4351a4976c 100644 --- a/hw/rtl/interfaces/VX_kmu_bus_if.sv +++ b/hw/rtl/interfaces/VX_kmu_bus_if.sv @@ -37,7 +37,7 @@ // `dest` is the global core index a FRAGMENT launch must land on. Fragment work // is bin->core affine — that affinity is what keeps same-pixel blend order // correct — so a fragment message cannot be load-balanced the way a compute CTA -// can. Each fan-out level consumes its own slice of `dest` (KMU_DEST_LSB_*); +// can. Each fan-out level consumes its own digit of `dest` (KMU_DEST_DIV_*); // compute launches ignore it and go to any ready output. interface VX_kmu_bus_if import VX_gpu_pkg::*; (); diff --git a/hw/unittest/cp_axil_regfile/main.cpp b/hw/unittest/cp_axil_regfile/main.cpp index 7574b9d6d9..920a9ee3af 100644 --- a/hw/unittest/cp_axil_regfile/main.cpp +++ b/hw/unittest/cp_axil_regfile/main.cpp @@ -9,8 +9,8 @@ // - CP_STATUS reflects the harness-driven cp_busy / cp_error inputs. // - CP_DEV_CAPS returns the configured (NUM_QUEUES, RING_SIZE_LOG2_MAX, // AXI_TID_WIDTH) fields. -// - GPU_DEV_CAPS / GPU_ISA_CAPS read back non-DECERR, decode to sane -// fields, and ignore writes (read-only). +// - GPU_DEV_CAPS / GPU_DEV_CAPS_EXT / GPU_ISA_CAPS read back non-DECERR, +// decode to sane fields, and ignore writes (read-only). // - CP_CYCLE counter actually advances per clock. // - Atomic Q_TAIL commit: writing Q_TAIL_LO alone does NOT advance // q_state.tail; writing Q_TAIL_HI atomically commits both halves. @@ -23,6 +23,7 @@ #include "vl_simulator.h" #include "VVX_cp_axil_regfile_top.h" +#include #include #include #include @@ -152,6 +153,7 @@ template static uint8_t q_profile_en(T* t) { return (uint8_t)read_ static constexpr uint16_t CP_CTRL = 0x000; static constexpr uint16_t CP_STATUS = 0x004; static constexpr uint16_t CP_DEV_CAPS = 0x008; +static constexpr uint16_t GPU_DEV_CAPS_EXT = 0x00C; static constexpr uint16_t CP_CYCLE_LO = 0x010; static constexpr uint16_t CP_CYCLE_HI = 0x014; static constexpr uint16_t GPU_DEV_CAPS_LO = 0x018; @@ -334,8 +336,12 @@ int main(int argc, char** argv) { "T11: caps must not be the DECERR sentinel"); uint64_t dev_caps = ((uint64_t)dhi << 32) | dlo; uint64_t isa_caps = ((uint64_t)ihi << 32) | ilo; - // dev_caps reserved bits [63:42] must be zero. - EXPECT((dev_caps >> 42) == 0, "T11: dev_caps reserved bits set"); + // Exact counts: warps [48:42], clusters [63:57], socket size in the extension word. + EXPECT(((dev_caps >> 42) & 0x7F) == VX_CFG_NUM_WARPS, "T11: dev_caps exact warps"); + EXPECT(((dev_caps >> 57) & 0x7F) == VX_CFG_NUM_CLUSTERS, "T11: dev_caps exact clusters"); + uint64_t de = axil_read(sim, tick, GPU_DEV_CAPS_EXT); + EXPECT((de >> 32) == 0, "T11: GPU_DEV_CAPS_EXT DECERR"); + EXPECT((uint32_t)de == VX_CFG_SOCKET_SIZE, "T11: GPU_DEV_CAPS_EXT socket size"); // isa_caps XLEN field [31:30] = clog2(XLEN)-4, so 1 (RV32) or 2 (RV64). uint32_t xlen_field = (uint32_t)((isa_caps >> 30) & 0x3); EXPECT(xlen_field == 1 || xlen_field == 2, "T11: isa_caps XLEN field"); diff --git a/sim/common/cmd_processor.cpp b/sim/common/cmd_processor.cpp index 0f5d3a80f2..6bb250b452 100644 --- a/sim/common/cmd_processor.cpp +++ b/sim/common/cmd_processor.cpp @@ -15,7 +15,8 @@ namespace vortex { // ============================================================================ // Static GPU capability words // -// GPU_DEV_CAPS / GPU_ISA_CAPS register values exposed as read-only MMIO. +// GPU_DEV_CAPS / GPU_DEV_CAPS_EXT / GPU_ISA_CAPS register values exposed as +// read-only MMIO. // sw/runtime/common/vx_caps.h is the matching decoder. // ============================================================================ namespace { @@ -45,6 +46,10 @@ uint64_t gpu_dev_caps() { | ((uint64_t(cluster_size) & 0xFF) << 49) | ((uint64_t(VX_CFG_NUM_CLUSTERS) & 0x7F) << 57); } +uint32_t gpu_dev_caps_ext() { + static_assert(VX_CFG_SOCKET_SIZE < 256, "GPU_DEV_CAPS_EXT socket size overflows"); + return uint32_t(VX_CFG_SOCKET_SIZE); +} uint64_t gpu_isa_caps() { return (uint64_t(VX_CFG_MISA_EXT) << 32) | ((uint64_t(cp_clog2(VX_CFG_XLEN) - 4) & 0x3) << 30) @@ -78,7 +83,7 @@ void CommandProcessor::mmio_write(uint32_t off, uint32_t value) { case 0x028: satp_ = (satp_ & 0xFFFFFFFF00000000ULL) | uint64_t(value); return; case 0x02C: satp_ = (satp_ & 0x00000000FFFFFFFFULL) | (uint64_t(value) << 32); return; // STATUS / DEV_CAPS / CYCLE / GPU caps are RO; ignore writes. - case 0x004: case 0x008: case 0x010: case 0x014: + case 0x004: case 0x008: case 0x00C: case 0x010: case 0x014: case 0x018: case 0x01C: case 0x020: case 0x024: return; } // Queue 0 (offsets 0x100..0x12F) @@ -130,6 +135,7 @@ uint32_t CommandProcessor::mmio_read(uint32_t off) const { return llc_client | supports_qmd | supports_draw | vm_enabled | fault_report | (uint32_t(6) << 16) | (uint32_t(16) << 8) | uint32_t(1); } + case 0x00C: return gpu_dev_caps_ext(); case 0x010: return uint32_t(cycle_counter_ & 0xFFFFFFFF); case 0x014: return uint32_t(cycle_counter_ >> 32); case 0x018: return uint32_t(gpu_dev_caps() & 0xFFFFFFFF); diff --git a/sim/common/cmd_processor.h b/sim/common/cmd_processor.h index f249b89ca5..de5199979a 100644 --- a/sim/common/cmd_processor.h +++ b/sim/common/cmd_processor.h @@ -26,6 +26,7 @@ // 0x000 CP_CTRL bit0=enable_global, bit1=reset_all // 0x004 CP_STATUS bit0=busy, bit1=error // 0x008 CP_DEV_CAPS {AXI_TID_W:8 | RING_LOG2:8 | NUM_QUEUES:8} +// 0x00C GPU_DEV_CAPS_EXT device-config caps extension (RO) // 0x010 CP_CYCLE_LO // 0x014 CP_CYCLE_HI // 0x018 GPU_DEV_CAPS_LO device-config caps, low 32 bits (RO) diff --git a/sim/simx/constants.h b/sim/simx/constants.h index fabb7e3925..2142e7efd2 100644 --- a/sim/simx/constants.h +++ b/sim/simx/constants.h @@ -81,6 +81,8 @@ inline constexpr uint32_t DCACHE_CHANNELS = __UP((VX_CFG_NUM_LSU_LANES * XLENB) inline constexpr uint32_t VX_CFG_DCACHE_NUM_REQS = (VX_CFG_NUM_LSU_BLOCKS * DCACHE_CHANNELS); inline constexpr uint32_t NUM_SOCKETS = __UP(VX_CFG_NUM_CORES / VX_CFG_SOCKET_SIZE); +static_assert((VX_CFG_NUM_CORES % VX_CFG_SOCKET_SIZE) == 0, + "NUM_CORES must be a multiple of SOCKET_SIZE"); // Demand clients of the cluster L2 (sockets and gfx caches), excluding the // walker's PTE-fetch slot. diff --git a/sim/simx/rtu/rtu_types.h b/sim/simx/rtu/rtu_types.h index 8e84ba11cb..8a20d65e83 100644 --- a/sim/simx/rtu/rtu_types.h +++ b/sim/simx/rtu/rtu_types.h @@ -259,6 +259,8 @@ static_assert(VX_CFG_RTU_MERGE_DEPTH == 0, // Cores served by one RTU, and the per-core slot partition. constexpr uint32_t kRtuCoresPerRtu = VX_CFG_SOCKET_SIZE / VX_CFG_NUM_RTU_CORES; +static_assert(!VX_CFG_EXT_RTU_ENABLED || (kRtuCoresPerRtu * VX_CFG_NUM_RTU_CORES == VX_CFG_SOCKET_SIZE), + "SOCKET_SIZE must be a multiple of NUM_RTU_CORES"); constexpr uint32_t kRtuSlotsPerCore = VX_CFG_RTU_NUM_SLOTS / kRtuCoresPerRtu; static_assert(kRtuSlotsPerCore >= 1, "a core with no slot of its own could never issue a trace"); diff --git a/sw/runtime/common/caps.h b/sw/runtime/common/caps.h index ddb1def115..243fddc820 100644 --- a/sw/runtime/common/caps.h +++ b/sw/runtime/common/caps.h @@ -32,25 +32,34 @@ namespace vortex { // CP regfile offsets for the static capability words. // Pass to a backend's cp_mmio_read(), which applies any platform base offset. +static constexpr uint32_t CP_REG_GPU_DEV_CAPS_EXT = 0x00C; static constexpr uint32_t CP_REG_GPU_DEV_CAPS_LO = 0x018; static constexpr uint32_t CP_REG_GPU_DEV_CAPS_HI = 0x01C; static constexpr uint32_t CP_REG_GPU_ISA_CAPS_LO = 0x020; static constexpr uint32_t CP_REG_GPU_ISA_CAPS_HI = 0x024; -// Warps, sockets per cluster and clusters carry an exact count, so a machine -// that is not a power of two reports truly; the field is zero on devices -// built before it existed, which report only the 3-bit log2 field. -inline uint64_t exact_or_log2(uint64_t dev_caps, uint32_t exact_lsb, uint64_t exact_mask, uint32_t log2_lsb) { - uint64_t exact = (dev_caps >> exact_lsb) & exact_mask; +// Warps, sockets per cluster, clusters and the socket size carry an exact +// count, so a machine that is not a power of two reports truly; the field is +// zero on devices built before it existed, which report only the 3-bit log2 +// field of GPU_DEV_CAPS. +inline uint64_t exact_or_log2(uint64_t exact, uint64_t dev_caps, uint32_t log2_lsb) { return exact ? exact : (1ull << ((dev_caps >> log2_lsb) & 0x7)); } +inline uint64_t exact_or_log2(uint64_t dev_caps, uint32_t exact_lsb, uint64_t exact_mask, uint32_t log2_lsb) { + return exact_or_log2((dev_caps >> exact_lsb) & exact_mask, dev_caps, log2_lsb); +} + // Decode a VX_CAPS_* id from the two CP capability words. Returns true // and writes *out when the id is encoded in the words; returns false // for ids the caller must resolve itself (CACHE_LINE_SIZE, // GLOBAL_MEM_SIZE, CLOCK_RATE, PEAK_MEM_BW). -inline bool decode_caps(uint64_t dev_caps, uint64_t isa_caps, +inline bool decode_caps(uint64_t dev_caps, uint32_t dev_caps_ext, uint64_t isa_caps, uint32_t caps_id, uint64_t* out) { + // A CP without the extension word answers zero, a DECERR sentinel or all + // ones; any reserved bit set means the word is absent. + const uint32_t caps_ext = (dev_caps_ext >> 8) ? 0 : dev_caps_ext; + const uint64_t socket_size = exact_or_log2(caps_ext & 0xFF, dev_caps, 14); switch (caps_id) { case VX_CAPS_VERSION: *out = (dev_caps >> 0) & 0xff; @@ -62,14 +71,13 @@ inline bool decode_caps(uint64_t dev_caps, uint64_t isa_caps, *out = exact_or_log2(dev_caps, 42, 0x7F, 11); return true; case VX_CAPS_NUM_CORES: { - uint64_t socket_size = 1ull << ((dev_caps >> 14) & 0x7); uint64_t cluster_size = exact_or_log2(dev_caps, 49, 0xFF, 17); uint64_t num_clusters = exact_or_log2(dev_caps, 57, 0x7F, 20); *out = num_clusters * cluster_size * socket_size; return true; } case VX_CAPS_SOCKET_SIZE: - *out = 1ull << ((dev_caps >> 14) & 0x7); + *out = socket_size; return true; case VX_CAPS_NUM_CLUSTERS: *out = exact_or_log2(dev_caps, 57, 0x7F, 20); @@ -94,14 +102,15 @@ inline bool decode_caps(uint64_t dev_caps, uint64_t isa_caps, } } -// Read GPU_DEV_CAPS / GPU_ISA_CAPS from the CP regfile through a -// backend's cp_mmio_read primitive. `Reader` is any callable matching -// int(uint32_t off, uint32_t* out) — returns non-zero on failure. +// Read GPU_DEV_CAPS / GPU_DEV_CAPS_EXT / GPU_ISA_CAPS from the CP regfile +// through a backend's cp_mmio_read primitive. `Reader` is any callable +// matching int(uint32_t off, uint32_t* out) — returns non-zero on failure. // Returns 0 on success. template inline int load_caps(Reader&& cp_mmio_read, - uint64_t* dev_caps, uint64_t* isa_caps) { + uint64_t* dev_caps, uint32_t* dev_caps_ext, uint64_t* isa_caps) { uint32_t lo = 0, hi = 0; + if (cp_mmio_read(CP_REG_GPU_DEV_CAPS_EXT, dev_caps_ext)) return -1; if (cp_mmio_read(CP_REG_GPU_DEV_CAPS_LO, &lo)) return -1; if (cp_mmio_read(CP_REG_GPU_DEV_CAPS_HI, &hi)) return -1; *dev_caps = uint64_t(lo) | (uint64_t(hi) << 32); diff --git a/sw/runtime/common/device.cpp b/sw/runtime/common/device.cpp index a32f18e9c2..39009c1927 100644 --- a/sw/runtime/common/device.cpp +++ b/sw/runtime/common/device.cpp @@ -1237,7 +1237,7 @@ vx_result_t Device::memory_info(uint64_t* out_free, uint64_t* out_used) { vx_result_t Device::query_caps(uint32_t caps_id, uint64_t* out_value) { if (!out_value) return VX_ERR_INVALID_VALUE; - // The two static caps words are read once from the CP regfile caps + // The static caps words are read once from the CP regfile caps // window; serialize the load against concurrent cp_reg_* traffic. if (!caps_loaded_) { std::lock_guard g(cp_mu_); @@ -1245,12 +1245,12 @@ vx_result_t Device::query_caps(uint32_t caps_id, uint64_t* out_value) { auto rd = [this](uint32_t off, uint32_t* v) -> int { return platform_->cp_reg_read(off, v) == VX_SUCCESS ? 0 : -1; }; - if (vortex::load_caps(rd, &dev_caps_, &isa_caps_) != 0) + if (vortex::load_caps(rd, &dev_caps_, &dev_caps_ext_, &isa_caps_) != 0) return VX_ERR_DEVICE_LOST; caps_loaded_ = true; } } - if (vortex::decode_caps(dev_caps_, isa_caps_, caps_id, out_value)) + if (vortex::decode_caps(dev_caps_, dev_caps_ext_, isa_caps_, caps_id, out_value)) return VX_SUCCESS; // Caps not encoded in the CP words — platform/runtime constants. switch (caps_id) { diff --git a/sw/runtime/common/vortex2_internal.h b/sw/runtime/common/vortex2_internal.h index 724449d825..f268d9a58d 100644 --- a/sw/runtime/common/vortex2_internal.h +++ b/sw/runtime/common/vortex2_internal.h @@ -435,8 +435,9 @@ class Device : public RefCounted { std::map host_mems_; // Lazily-loaded device/ISA caps words (read once from the CP regfile). - uint64_t dev_caps_ = 0; - uint64_t isa_caps_ = 0; + uint64_t dev_caps_ = 0; + uint32_t dev_caps_ext_ = 0; + uint64_t isa_caps_ = 0; bool caps_loaded_ = false; std::mutex mu_; From 2183b6a0677bb499c6f2eee4819d136a11c76049 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 24 Oct 2026 04:18:44 -0700 Subject: [PATCH 096/124] ci: re-record perf baselines for the MLP Phase 1 timing Regenerated with pytest ci -m perf_gate --update-baselines on fc0a35bb4, where the Phase 1 port moved cycle counts. perf_gate passes exactly against them (49/49) on c5ed76a73. Co-Authored-By: Claude Opus 5.5 --- ci/baselines/perf/core.json | 68 +++++++++++++++---------------- ci/baselines/perf/dxa.json | 40 +++++++++--------- ci/baselines/perf/graphics.json | 34 ++++++++-------- ci/baselines/perf/raytracing.json | 8 ++-- ci/baselines/perf/tensor.json | 20 ++++----- ci/baselines/perf/tensor_mx.json | 4 +- ci/baselines/perf/tensor_sp.json | 4 +- ci/baselines/perf/tensor_wg.json | 12 +++--- 8 files changed, 95 insertions(+), 95 deletions(-) diff --git a/ci/baselines/perf/core.json b/ci/baselines/perf/core.json index 01cd65540c..c9c7dc8a7f 100644 --- a/ci/baselines/perf/core.json +++ b/ci/baselines/perf/core.json @@ -1,177 +1,177 @@ { "core:raycast-nt16:rtlsim": { "32": { - "cycles": 564053, - "instrs": 137486 + "cycles": 11185799, + "instrs": 2241925 }, "app": "raycast", "args": "", - "config_hash": "7a2c92e634460e90", + "config_hash": "ef7e97261abf3c7e", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:raycast-nt4:rtlsim": { "32": { - "cycles": 1498642, - "instrs": 259228 + "cycles": 21106013, + "instrs": 5664693 }, "app": "raycast", "args": "", - "config_hash": "2532cbe6c87ca233", + "config_hash": "4a346b970c5d5f4e", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:sgemm-mc-nt16:rtlsim": { "32": { - "cycles": 720350, + "cycles": 632919, "instrs": 623680 }, "app": "sgemm", "args": "-n128", - "config_hash": "6106872241e50c60", + "config_hash": "12eb01a0cf320bec", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=16 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "core:sgemm-mc-nt4:rtlsim": { "32": { - "cycles": 2227069, + "cycles": 2251795, "instrs": 2494528 }, "app": "sgemm", "args": "-n128", - "config_hash": "ecbd8e8a804188dd", + "config_hash": "a5cc1a944ac4fc92", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "core:sgemm-nt16:rtlsim": { "32": { - "cycles": 1782470, + "cycles": 1665973, "instrs": 623632 }, "app": "sgemm", "args": "-n128", - "config_hash": "6dcefd7e66ed0cc0", + "config_hash": "dcf77cabb080bf39", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:sgemm-nt4:rtlsim": { "32": { - "cycles": 5939731, + "cycles": 5943507, "instrs": 2494480 }, "app": "sgemm", "args": "-n128", - "config_hash": "74a554f940a83c4a", + "config_hash": "9b7d85b091d9be08", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:sgemmx-nt16:rtlsim": { "32": { - "cycles": 1226398, + "cycles": 1225395, "instrs": 266896 }, "app": "sgemmx", "args": "-n128", - "config_hash": "efc73b5e23a33b8f", + "config_hash": "1181ce2dd247a573", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:sgemmx-nt4:rtlsim": { "32": { - "cycles": 4219466, + "cycles": 4218769, "instrs": 1067536 }, "app": "sgemmx", "args": "-n128", - "config_hash": "0f2e76cd4d21f998", + "config_hash": "b8d3bbe9aaa587a9", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:sgemv-nt16:rtlsim": { "32": { - "cycles": 460344, + "cycles": 408992, "instrs": 66800 }, "app": "sgemv", "args": "-m512 -n512", - "config_hash": "bc1d0a1801524d27", + "config_hash": "a2e93b1416c1222b", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:sgemv-nt4:rtlsim": { "32": { - "cycles": 872926, + "cycles": 883260, "instrs": 267152 }, "app": "sgemv", "args": "-m512 -n512", - "config_hash": "7fbf266181d2ec8b", + "config_hash": "212b79b7b9e0df8d", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:softmax-nt16:rtlsim": { "32": { - "cycles": 1271056, + "cycles": 1270578, "instrs": 464089 }, "app": "softmax", "args": "-n64", - "config_hash": "c53ddc9ba2cd96bd", + "config_hash": "4a980cd3436e0491", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:softmax-nt4:rtlsim": { "32": { - "cycles": 4860309, + "cycles": 4860082, "instrs": 1789797 }, "app": "softmax", "args": "-n64", - "config_hash": "6dea102e1b20464c", + "config_hash": "3d84ea77567e034e", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:stencil3d-nt16:rtlsim": { "32": { - "cycles": 2272549, + "cycles": 2283980, "instrs": 892944 }, "app": "stencil3d", "args": "-n32", - "config_hash": "542851d41daf9431", + "config_hash": "5b78acb9a50222ff", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:stencil3d-nt4:rtlsim": { "32": { - "cycles": 4180969, + "cycles": 4185784, "instrs": 1785872 }, "app": "stencil3d", "args": "-n32", - "config_hash": "fa607fa9e1280813", + "config_hash": "dc41c57ce7819a59", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:vecadd-nt16:rtlsim": { "32": { - "cycles": 99953, + "cycles": 95240, "instrs": 29712 }, "app": "vecadd", "args": "-n16384", - "config_hash": "0f4e3087ce474e4b", + "config_hash": "652eaf7c4099617d", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:vecadd-nt4:rtlsim": { "32": { - "cycles": 367765, + "cycles": 368717, "instrs": 118800 }, "app": "vecadd", "args": "-n16384", - "config_hash": "0169f39a14561223", + "config_hash": "c602c9c37a145297", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/dxa.json b/ci/baselines/perf/dxa.json index a331465e29..ec5e66786f 100644 --- a/ci/baselines/perf/dxa.json +++ b/ci/baselines/perf/dxa.json @@ -6,18 +6,18 @@ }, "app": "dxa_copy_mcast", "args": "", - "config_hash": "2be8c5960886f23f", + "config_hash": "28dc0787284f20c8", "configs": "-DVX_CFG_EXT_DXA_ENABLE", "driver": "rtlsim" }, "dxa:perf_gate-copy-nt16:rtlsim": { "32": { - "cycles": 1992, + "cycles": 1953, "instrs": 592 }, "app": "dxa_copy", "args": "-d1", - "config_hash": "37f76aae77fd1afd", + "config_hash": "912948e13624b9a4", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -28,95 +28,95 @@ }, "app": "dxa_copy", "args": "-d1", - "config_hash": "c3b3447f163ec6c3", + "config_hash": "277ced0b15aed8d9", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-mcast-nt16:rtlsim": { "32": { - "cycles": 14023305, + "cycles": 14016579, "instrs": 4962320 }, "app": "sgemm2_dxa_mcast", "args": "-n128", - "config_hash": "62067f11f3ef5af4", + "config_hash": "f247e0e4b7829023", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-mcast-nt4:rtlsim": { "32": { - "cycles": 14063502, + "cycles": 14051140, "instrs": 4962320 }, "app": "sgemm2_dxa_mcast", "args": "-n128", - "config_hash": "98c58db8047bceb9", + "config_hash": "3660e26ec6c4ba10", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-nt16:rtlsim": { "32": { - "cycles": 3921184, + "cycles": 3947991, "instrs": 1263632 }, "app": "sgemm2_dxa", "args": "-n128 -t4 -m1", - "config_hash": "845bebf9fbc108b1", + "config_hash": "55af78d744fa9e46", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-nt4:rtlsim": { "32": { - "cycles": 14352313, + "cycles": 14352123, "instrs": 5017616 }, "app": "sgemm2_dxa", "args": "-n128 -t4 -m1", - "config_hash": "8cf4b0e4e0c7d144", + "config_hash": "8c7ee15d08de3277", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-dxa-mcast-nt16:rtlsim": { "32": { - "cycles": 79131, + "cycles": 78840, "instrs": 32384 }, "app": "sgemm_tcu_wg_dxa_mcast", "args": "-m 128 -n 128 -k 64", - "config_hash": "d3c9c80b16cdf2c2", + "config_hash": "d27df3466a8eed79", "configs": "-DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-dxa-mcast-nt4:rtlsim": { "32": { - "cycles": 360010, + "cycles": 358897, "instrs": 234560 }, "app": "sgemm_tcu_wg_dxa_mcast", "args": "-m 128 -n 128 -k 64", - "config_hash": "38993044a82f32e0", + "config_hash": "776a839834d35db8", "configs": "-DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-dxa:rtlsim": { "32": { - "cycles": 31160, + "cycles": 29487, "instrs": 9160 }, "app": "sgemm_tcu_wg_dxa", "args": "-m 128 -n 128 -k 64", - "config_hash": "699f26ecbecc6bb2", + "config_hash": "56519748fbd3c39b", "configs": "-DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DWGMMA_NRC=16 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_THREADS=32", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-sp-dxa:rtlsim": { "32": { - "cycles": 55512, + "cycles": 48573, "instrs": 9328 }, "app": "sgemm_tcu_wg_sp_dxa", "args": "-m 128 -n 128 -k 128", - "config_hash": "8e607400e57e51f5", + "config_hash": "f1664e1fcc03f8a6", "configs": "-DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_THREADS=32", "driver": "rtlsim" } diff --git a/ci/baselines/perf/graphics.json b/ci/baselines/perf/graphics.json index 489e16a09b..4d13a12aaf 100644 --- a/ci/baselines/perf/graphics.json +++ b/ci/baselines/perf/graphics.json @@ -1,45 +1,45 @@ { "graphics:perf_gate-draw3d-mc:rtlsim": { "32": { - "cycles": 529868, + "cycles": 529816, "instrs": 203136 }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "0777067f98f2730f", + "config_hash": "9b91ab9ac3aa5103", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "graphics:perf_gate-draw3d-nt16:rtlsim": { "32": { - "cycles": 143542, + "cycles": 141632, "instrs": 51652 }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "550e1e3d5ffe23d0", + "config_hash": "1f2de39a01c16b67", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "graphics:perf_gate-draw3d-nt4:rtlsim": { "32": { - "cycles": 529389, + "cycles": 529359, "instrs": 203136 }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "5cc3a870f65d86fa", + "config_hash": "73f771f2643f6527", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "graphics:perf_gate-om-nt16:rtlsim": { "32": { - "cycles": 189356, + "cycles": 189979, "instrs": 68624 }, "app": "gfx_om", "args": "-w 128 -h 128 -r whitebox_128.png", - "config_hash": "f74350c24c97b1f5", + "config_hash": "132840fd850713c9", "configs": "-DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -50,51 +50,51 @@ }, "app": "gfx_om", "args": "-w 128 -h 128 -r whitebox_128.png", - "config_hash": "8a9749c1c0d8264d", + "config_hash": "d8e520e04c9979f4", "configs": "-DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "graphics:perf_gate-raster-nt16:rtlsim": { "32": { - "cycles": 14626, + "cycles": 14487, "instrs": 4708 }, "app": "gfx_raster", "args": "-w 128 -h 128 -t triangle.cgltrace -r triangle_ref_128.png", - "config_hash": "967474845abde936", + "config_hash": "e2f119110f028280", "configs": "-DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "graphics:perf_gate-raster-nt4:rtlsim": { "32": { - "cycles": 55565, + "cycles": 55553, "instrs": 18640 }, "app": "gfx_raster", "args": "-w 128 -h 128 -t triangle.cgltrace -r triangle_ref_128.png", - "config_hash": "40f6d7ca7755b6ca", + "config_hash": "5109e566c9637f1b", "configs": "-DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "graphics:perf_gate-tex-nt16:rtlsim": { "32": { - "cycles": 35297, + "cycles": 35254, "instrs": 12304 }, "app": "gfx_tex", "args": "-i toad.png -r toad_ref_f0.png -f 0 -g 0", - "config_hash": "e44189b53e136920", + "config_hash": "3876028c632f011b", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "graphics:perf_gate-tex-nt4:rtlsim": { "32": { - "cycles": 136487, + "cycles": 136492, "instrs": 49168 }, "app": "gfx_tex", "args": "-i toad.png -r toad_ref_f0.png -f 0 -g 0", - "config_hash": "f0cdf08354c9ae88", + "config_hash": "fe367b2efd48140d", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/raytracing.json b/ci/baselines/perf/raytracing.json index e0ee3a7da4..71ef9f1d16 100644 --- a/ci/baselines/perf/raytracing.json +++ b/ci/baselines/perf/raytracing.json @@ -6,18 +6,18 @@ }, "app": "rt_raycast", "args": "", - "config_hash": "4f78e65dfd2497cb", + "config_hash": "c0bd39f0f980f01d", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "raytracing:perf_gate-rt_raycast-nt16:rtlsim": { "32": { - "cycles": 53443, + "cycles": 52906, "instrs": 11372 }, "app": "rt_raycast", "args": "", - "config_hash": "c0af18c64430f778", + "config_hash": "b899d2a47bcc51ae", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -28,7 +28,7 @@ }, "app": "rt_raycast", "args": "", - "config_hash": "e394fceca77fe6f1", + "config_hash": "47b882de7074c1db", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor.json b/ci/baselines/perf/tensor.json index 975aa8d248..8ce4e9b7b0 100644 --- a/ci/baselines/perf/tensor.json +++ b/ci/baselines/perf/tensor.json @@ -1,56 +1,56 @@ { "tensor:perf_gate-fp16-isw4:rtlsim": { "32": { - "cycles": 197465, + "cycles": 197355, "instrs": 130592 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "9cdc8bb95c856a38", + "config_hash": "b80548e5fa6b84ae", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" }, "tensor:perf_gate-fp16-mc-nt16:rtlsim": { "32": { - "cycles": 36723, + "cycles": 36479, "instrs": 47296 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "153ac3d8001d2be6", + "config_hash": "2935e28e509b5699", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=16 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "tensor:perf_gate-fp16-mc-nt4:rtlsim": { "32": { - "cycles": 221581, + "cycles": 222420, "instrs": 336448 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "21c6d64b10dab7cb", + "config_hash": "d6ccf2e6532a7a88", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "tensor:perf_gate-fp16:rtlsim": { "32": { - "cycles": 256743, + "cycles": 254124, "instrs": 130576 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "04953ed130a8c1a9", + "config_hash": "9c339e61cd76a65c", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32", "driver": "rtlsim" }, "tensor:perf_gate-sgemm2-fp16:rtlsim": { "32": { - "cycles": 590798, + "cycles": 593588, "instrs": 409888 }, "app": "sgemm2_tcu", "args": "-m 128 -n 128 -k 64", - "config_hash": "26d22a59f411cfbe", + "config_hash": "c27f67158e4b632a", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_mx.json b/ci/baselines/perf/tensor_mx.json index a49403e311..3b8f55c440 100644 --- a/ci/baselines/perf/tensor_mx.json +++ b/ci/baselines/perf/tensor_mx.json @@ -1,12 +1,12 @@ { "tensor_mx:perf_gate-mxfp8:rtlsim": { "32": { - "cycles": 76393, + "cycles": 76410, "instrs": 34832 }, "app": "sgemm_tcu_mx", "args": "-m 128 -n 128 -k32", - "config_hash": "7c8765889acb4f32", + "config_hash": "63975cb90eb9477f", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DITYPE=mxfp8 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_sp.json b/ci/baselines/perf/tensor_sp.json index 3ee3f24144..bf7540cbf3 100644 --- a/ci/baselines/perf/tensor_sp.json +++ b/ci/baselines/perf/tensor_sp.json @@ -1,12 +1,12 @@ { "tensor_sp:perf_gate-sparse-fp16:rtlsim": { "32": { - "cycles": 183094, + "cycles": 181992, "instrs": 100880 }, "app": "sgemm_tcu_sp", "args": "-m 128 -n 128 -k 128", - "config_hash": "6e523b9d0f6fe4ee", + "config_hash": "3bb64bb8ac3c1a95", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_wg.json b/ci/baselines/perf/tensor_wg.json index b2af51b8f2..2047a4ef2a 100644 --- a/ci/baselines/perf/tensor_wg.json +++ b/ci/baselines/perf/tensor_wg.json @@ -1,34 +1,34 @@ { "tensor_wg:perf_gate-wgmma-fedp2k-rs:rtlsim": { "32": { - "cycles": 158181, + "cycles": 158197, "instrs": 108512 }, "app": "sgemm_tcu_wg", "args": "-m64 -n64 -k64", - "config_hash": "c65e3b69950519cb", + "config_hash": "6e007939f3dbec76", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_TCU_FEDP2K -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DWGMMA_RS", "driver": "rtlsim" }, "tensor_wg:perf_gate-wgmma-fp16-ss:rtlsim": { "32": { - "cycles": 834873, + "cycles": 834916, "instrs": 525536 }, "app": "sgemm_tcu_wg", "args": "-m 128 -n 128 -k 128", - "config_hash": "0d41f1f51f46d61d", + "config_hash": "76bcef5705d497dc", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=32 -DWGMMA_SS -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" }, "tensor_wg:perf_gate-wgmma-sparse:rtlsim": { "32": { - "cycles": 400023, + "cycles": 406430, "instrs": 258976 }, "app": "sgemm_tcu_wg_sp", "args": "-m 128 -n 128 -k 128", - "config_hash": "b4944cbd79928ac8", + "config_hash": "d983c017c55aa5a7", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" } From afd43315053e226f91c21c6af98c187a955c8465 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sat, 24 Oct 2026 06:19:42 -0700 Subject: [PATCH 097/124] afu: one AXI port per GPU memory port, W forwarded with its AW The AFU memory port count was a per-board constant (PLATFORM_MERGED_ MEMORY_INTERFACE), so it did not follow the GPU configuration and the U280 left channels unmapped. The port count now follows the GPU's memory ports, rounded up to a power of two; the build reads it with afu_mem_ports.sh and generates the HBM/DDR connectivity from it. VX_mem_to_axi without INTERLEAVE truncated the bank index away, sending every bank to the first 1/N of memory; bank i now serves the i-th contiguous 1/N, and xrtsim aborts on an address outside its port's range. Interleaved mode is unchanged. VX_mm_axi_arb (MULTI_OUT) held a W behind the AW grant register and could deadlock with a slave that takes W before AW. W now follows AW order through a 4-entry queue and moves in the AW cycle, which doubles single-beat write throughput. VX_afu_wr_gate and the drain counter keep reset quiesce from splitting a burst, including one whose W leads its AW. xrtsim xrt CI 23/23; afu_reset_seq, mm_axi_arb (old arbiter fails 5/13), cp_llc_port, cp_axi_path and cp_core pass. rtlsim and SimX do not build the changed arbiter mode, so perf_gate and model_parity are unaffected. Not yet run: Vivado timing of the combinational W select, and the U200 DDR address base on a board. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/unittest.yaml | 8 + docs/designs/fpga_afu_shell.md | 35 +- hw/rtl/afu/aved/vortex_afu.v | 23 +- hw/rtl/afu/common/VX_afu_axi_drain.sv | 23 +- hw/rtl/afu/common/VX_afu_wr_gate.sv | 98 +++++ hw/rtl/afu/common/VX_afu_wrap.sv | 196 +++++---- hw/rtl/afu/common/vortex_afu.vh | 63 +++ hw/rtl/afu/xrt/vortex_afu.v | 23 +- hw/rtl/libs/VX_mem_to_axi.sv | 7 +- hw/rtl/libs/VX_mm_axi_arb.sv | 82 ++-- hw/scripts/afu_mem_ports.sh | 89 ++++ hw/syn/xilinx/aved/Makefile | 23 +- hw/syn/xilinx/aved/platforms.mk | 4 +- hw/syn/xilinx/xrt/Makefile | 31 +- hw/syn/xilinx/xrt/package_kernel.tcl | 22 +- hw/syn/xilinx/xrt/platforms.mk | 29 +- hw/unittest/Makefile | 3 + .../afu_reset_seq/VX_afu_reset_seq_top.sv | 29 ++ hw/unittest/afu_reset_seq/main.cpp | 152 ++++++- hw/unittest/mm_axi_arb/Makefile | 23 + hw/unittest/mm_axi_arb/VX_mm_axi_arb_top.sv | 159 +++++++ hw/unittest/mm_axi_arb/main.cpp | 415 ++++++++++++++++++ sim/avedsim/Makefile | 11 + sim/avedsim/vortex_afu_shim.sv | 6 +- sim/xrtsim/Makefile | 12 + sim/xrtsim/vortex_afu_shim.sv | 9 +- sim/xrtsim/xrt_sim.cpp | 254 ++++++----- 27 files changed, 1525 insertions(+), 304 deletions(-) create mode 100644 hw/rtl/afu/common/VX_afu_wr_gate.sv create mode 100755 hw/scripts/afu_mem_ports.sh create mode 100644 hw/unittest/mm_axi_arb/Makefile create mode 100644 hw/unittest/mm_axi_arb/VX_mm_axi_arb_top.sv create mode 100644 hw/unittest/mm_axi_arb/main.cpp diff --git a/ci/testcases/unittest.yaml b/ci/testcases/unittest.yaml index 2f3f33d952..8765f57fac 100644 --- a/ci/testcases/unittest.yaml +++ b/ci/testcases/unittest.yaml @@ -46,6 +46,14 @@ tests: via: script run: "make -C hw/unittest/stream_arb run" touches: [hw/rtl/libs/VX_stream_arb.sv, hw/unittest/stream_arb] + # The ID-routed AXI arbiter shared by the AFU and the command processor: + # write data must reach the slave in address order whichever channel a + # master offers first, and a slave that takes data before the address must + # not deadlock it. xrtsim's always-ready slave exercises neither. + - id: hw-mm-axi-arb + via: script + run: "make -C hw/unittest/mm_axi_arb run" + touches: [hw/rtl/libs/VX_mm_axi_arb.sv, hw/unittest/mm_axi_arb] # Texel blend (VX_tex_lerp, both forms the sampler uses) against the C model # the SW sampler and SimX share, and the exact Vulkan f/256 blend, over every # input. diff --git a/docs/designs/fpga_afu_shell.md b/docs/designs/fpga_afu_shell.md index 413a108129..c4c9760d71 100644 --- a/docs/designs/fpga_afu_shell.md +++ b/docs/designs/fpga_afu_shell.md @@ -26,7 +26,8 @@ DMA-command engine have been removed. | [`VX_afu_wrap.sv`](../../hw/rtl/afu/common/VX_afu_wrap.sv) | The real shell (~1,200 LOC): AXI-Lite bit-12 demux, `VX_cp_core`, `m_axi_host`, the Vortex instance and the device-memory path shared with the CP. | | [`VX_afu_ctrl.sv`](../../hw/rtl/afu/xrt/VX_afu_ctrl.sv) | Slimmed AXI-Lite slave (~322 LOC): `ap_ctrl` stub at 0x00 + a SCOPE serial register pair + SCOPE watchdog. | | [`VX_afu_axil_demux.sv`](../../hw/rtl/afu/common/VX_afu_axil_demux.sv) | AXI-Lite demux splitting the control space on `addr[12]`, one outstanding transaction per direction. | -| [`VX_afu_axi_drain.sv`](../../hw/rtl/afu/common/VX_afu_axi_drain.sv) | Outstanding-transaction tracker per AXI master; reports when a port owes the interconnect nothing. | +| [`VX_afu_axi_drain.sv`](../../hw/rtl/afu/common/VX_afu_axi_drain.sv) | Outstanding-transaction tracker per AXI master; reports when a port owes the interconnect nothing, including a write burst under way ahead of its AW. | +| [`VX_afu_req_gate.sv`](../../hw/rtl/afu/common/VX_afu_req_gate.sv) / [`VX_afu_wr_gate.sv`](../../hw/rtl/afu/common/VX_afu_wr_gate.sv) | Withhold new reads / whole new writes while the reset sequencer quiesces, never withdrawing an offer. | | [`VX_afu_reset_seq.sv`](../../hw/rtl/afu/common/VX_afu_reset_seq.sv) | Quiesce-before-reset sequencer for the soft reset; refuses rather than resetting a master that will not drain. | - **Control.** Host AXI-Lite `addr[12]` splits the slave: `addr[12]=0` → @@ -39,19 +40,43 @@ DMA-command engine have been removed. with its address, and routing W by a register that only updates at the AW handshake sent AW and W to different slaves and deadlocked the interface. Covered by `hw/unittest/afu_axil_demux`. -- **Memory.** Two structures, selected by the number of memory ports. - *One port* (the merged platforms): `Vortex_axi` and the CP's `axi_dev` +- **Memory.** The AFU exposes one AXI master per GPU memory port: + `AFU_MEM_PORTS_OF(VX_MEM_PORTS)` ([`vortex_afu.vh`](../../hw/rtl/afu/common/vortex_afu.vh)), + the last-level cache's memory ports (already capped by the platform's + channel count, `PLATFORM_MEMORY_NUM_BANKS`) rounded up to a power of two. + The kernel top must spell its ports out, so every flow resolves the count + to the literal `AFU_MEM_PORTS` by elaborating a probe against its own flags + ([`afu_mem_ports.sh`](../../hw/scripts/afu_mem_ports.sh)); `VX_afu_wrap` + asserts the two agree, `package_kernel.tcl` checks the packaged kernel's + master count, and the XRT Makefile generates the `--connectivity.sp` lines + from the same value. Lines interleave across the N ports + (`PLATFORM_MEMORY_INTERLEAVE`; without it each port takes a contiguous range), and port *i* owns the *i*-th contiguous + 1/N of device memory: its AXI address is the line index divided by N + within that share. The platform maps the share to its own contiguous group + of `PLATFORM_MEMORY_CHANNELS / N` channels (`HBM[2i:2i+1]` for 16 ports on + a U55C), so ports never contend for a channel. `PLATFORM_MEMORY_OFFSET` is + applied once, at every port. + Two structures, selected by N. + *One port*: `Vortex_axi` and the CP's `axi_dev` share the port through a 2:1 `VX_mm_axi_arb` in its ID-routed multi-outstanding mode (`MULTI_OUT=1`: the top ID bit names the source, Vortex gets the remaining ID bits), and CP bursts reach the platform as - bursts. *Several ports*: Vortex's memory ports and `axi_dev` (bridged by + bursts. Writes keep AW order on the shared W channel, and a write's data + goes with its address, so single-beat writes issue one per cycle. + *Several ports*: Vortex's memory ports and `axi_dev` (bridged by `VX_membus_from_axi`) are the inputs of one bank adapter, `VX_mem_to_axi`, so the CP is merged **upstream** of bank selection and reaches every bank. In both, every port carries as many reads and writes in flight as its masters issue; one outstanding transaction per direction would cap a port at one line per DRAM round trip. CP `axi_host` drives a **dedicated `m_axi_host` AXI master** for the command ring + host DMA. - `PLATFORM_MEMORY_OFFSET` is applied per bank, at the port. + While the reset sequencer quiesces, `VX_afu_req_gate` holds new reads and + `VX_afu_wr_gate` holds new writes on both channels together: AXI4 lets a + master present write data before or with its address, so gating AW alone + would strand data the slave can never pair. An address or data offer + already on the bus obliges the other channel, since a slave may wait for + one before accepting the other. xrtsim aborts on a request outside the + issuing port's share of memory. - **Interrupt.** The AFU `interrupt` pin is driven from `cp_interrupt` ([`:335`](../../hw/rtl/afu/xrt/VX_afu_wrap.sv#L335)). diff --git a/hw/rtl/afu/aved/vortex_afu.v b/hw/rtl/afu/aved/vortex_afu.v index 5335b376bb..6490e304ea 100644 --- a/hw/rtl/afu/aved/vortex_afu.v +++ b/hw/rtl/afu/aved/vortex_afu.v @@ -23,22 +23,14 @@ module vortex_afu #( parameter C_M_AXI_MEM_ID_WIDTH = `PLATFORM_MEMORY_ID_WIDTH, parameter C_M_AXI_MEM_DATA_WIDTH = (`VX_CFG_PLATFORM_MEMORY_DATA_SIZE * 8), parameter C_M_AXI_MEM_ADDR_WIDTH = 64, -`ifdef PLATFORM_MERGED_MEMORY_INTERFACE - parameter C_M_AXI_MEM_NUM_BANKS = 1 -`else - parameter C_M_AXI_MEM_NUM_BANKS = `VX_CFG_PLATFORM_MEMORY_NUM_BANKS -`endif + parameter C_M_AXI_MEM_NUM_BANKS = `AFU_MEM_PORTS ) ( // System signals input wire ap_clk, input wire ap_rst_n, - // AXI4 master interface -`ifdef PLATFORM_MERGED_MEMORY_INTERFACE - `MP_REPEAT (1, GEN_AXI_MEM, MP_COMMA), -`else - `MP_REPEAT (`VX_CFG_PLATFORM_MEMORY_NUM_BANKS, GEN_AXI_MEM, MP_COMMA), -`endif + // AXI4 master interface, one per GPU memory port + `MP_REPEAT (`AFU_MEM_PORTS, GEN_AXI_MEM, MP_COMMA), // AXI4 host-memory master interface (CP command ring + host side of DMA) `GEN_AXI_HOST, @@ -69,6 +61,9 @@ module vortex_afu #( output wire interrupt ); + `AXI_MEM_ARRAYS (C_M_AXI_MEM_NUM_BANKS) + `MP_REPEAT (`AFU_MEM_PORTS, AXI_MEM_TO_ARRAY, MP_SEMI); + VX_afu_wrap #( .C_S_AXI_CTRL_ADDR_WIDTH (C_S_AXI_CONTROL_ADDR_WIDTH), .C_S_AXI_CTRL_DATA_WIDTH (C_S_AXI_CONTROL_DATA_WIDTH), @@ -79,11 +74,7 @@ module vortex_afu #( ) afu_wrap ( .clk (ap_clk), .reset (~ap_rst_n), - `ifdef PLATFORM_MERGED_MEMORY_INTERFACE - `MP_REPEAT (1, AXI_MEM_ARGS, MP_COMMA), - `else - `MP_REPEAT (`VX_CFG_PLATFORM_MEMORY_NUM_BANKS, AXI_MEM_ARGS, MP_COMMA), - `endif + `AXI_MEM_ARRAY_ARGS, `AXI_HOST_ARGS, .s_axi_ctrl_awvalid (s_axi_control_awvalid), .s_axi_ctrl_awready (s_axi_control_awready), diff --git a/hw/rtl/afu/common/VX_afu_axi_drain.sv b/hw/rtl/afu/common/VX_afu_axi_drain.sv index 95e4c05459..b023f7fd0e 100644 --- a/hw/rtl/afu/common/VX_afu_axi_drain.sv +++ b/hw/rtl/afu/common/VX_afu_axi_drain.sv @@ -27,6 +27,10 @@ // directions, so it stays right when AXI4 allows write data to be presented // before its address (which would drive an up/down counter negative). // +// A write burst whose first beats have gone out is in flight too, even when +// its AW has not: the counters only see last beats, so the burst in progress +// is tracked separately. +// // COUNT_WIDTH must exceed the port's maximum outstanding transactions; the // comparison is exact only while fewer than 2**COUNT_WIDTH are in flight. // ============================================================================ @@ -39,6 +43,7 @@ module VX_afu_axi_drain #( // Handshakes observed on the shell side of the request gate. input wire aw_fire, // awvalid && awready + input wire w_fire, // wvalid && wready input wire w_fire_last, // wvalid && wready && wlast input wire b_fire, // bvalid && bready input wire ar_fire, // arvalid && arready @@ -60,15 +65,20 @@ module VX_afu_axi_drain #( reg [COUNT_WIDTH-1:0] b_count; reg [COUNT_WIDTH-1:0] ar_count; reg [COUNT_WIDTH-1:0] r_count; + reg w_in_burst; always @(posedge clk) begin if (reset) begin - aw_count <= '0; - w_count <= '0; - b_count <= '0; - ar_count <= '0; - r_count <= '0; + aw_count <= '0; + w_count <= '0; + b_count <= '0; + ar_count <= '0; + r_count <= '0; + w_in_burst <= 1'b0; end else begin + if (w_fire) begin + w_in_burst <= ~w_fire_last; + end if (aw_fire) begin aw_count <= aw_count + 1'b1; end @@ -94,7 +104,8 @@ module VX_afu_axi_drain #( assign dbg_ar_count = ar_count; assign dbg_r_count = r_count; - assign idle = (aw_count == w_count) + assign idle = ~w_in_burst + && (aw_count == w_count) && (aw_count == b_count) && (ar_count == r_count); diff --git a/hw/rtl/afu/common/VX_afu_wr_gate.sv b/hw/rtl/afu/common/VX_afu_wr_gate.sv new file mode 100644 index 0000000000..821f0526f8 --- /dev/null +++ b/hw/rtl/afu/common/VX_afu_wr_gate.sv @@ -0,0 +1,98 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "vortex_afu.vh" + +// ============================================================================ +// VX_afu_wr_gate — withhold new AXI writes without splitting one. +// +// The write-channel counterpart of VX_afu_req_gate. Gating AW alone is not +// enough: AXI4 lets a master present a write's data before or with its +// address, so a master that does would push W beats of a write whose AW the +// gate then withholds. The slave is left holding data no address will ever +// claim, the drain counters never balance, and the reset sequencer refuses. +// +// A write is the k-th AW paired with the k-th W burst. While stopped, the gate +// lets through only what completes a write already begun on the shell side: +// the AW owed to a W burst that has started or is being offered, the W burst +// owed to an AW that has been accepted or is being offered, the rest of a +// burst in progress, and any offer already presented (VALID is never +// withdrawn, AXI4 §A3.2.1). A slave may hold AWREADY until it sees the data, +// or WREADY until it sees the address, so an offer alone obliges the other +// channel. Everything else waits, so the shell sees whole writes only and the +// drain reaches zero. +// +// LEAD_WIDTH must exceed the number of AWs a master may run ahead of its data. +// ============================================================================ + +module VX_afu_wr_gate #( + parameter LEAD_WIDTH = 16 +) ( + input wire clk, + input wire reset, + + input wire stop_req, // withhold new writes + + input wire in_awvalid, // from the AFU-internal master + output wire in_awready, + output wire out_awvalid, // to the shell + input wire out_awready, + + input wire in_wvalid, + input wire in_wlast, + output wire in_wready, + output wire out_wvalid, + input wire out_wready +); + reg blocked; + reg aw_offered; + reg w_offered; + reg w_in_burst; + reg [LEAD_WIDTH-1:0] aw_lead; // AWs accepted minus W bursts started + + wire aw_fire = out_awvalid && out_awready; + wire w_fire = out_wvalid && out_wready; + wire w_start = w_fire && ~w_in_burst; + + always @(posedge clk) begin + if (reset) begin + blocked <= 1'b0; + aw_offered <= 1'b0; + w_offered <= 1'b0; + w_in_burst <= 1'b0; + aw_lead <= '0; + end else begin + blocked <= stop_req; + aw_offered <= out_awvalid && ~out_awready; + w_offered <= out_wvalid && ~out_wready; + if (w_fire) begin + w_in_burst <= ~in_wlast; + end + aw_lead <= aw_lead + LEAD_WIDTH'(aw_fire) - LEAD_WIDTH'(w_start); + end + end + + wire lead_neg = aw_lead[LEAD_WIDTH-1]; + wire lead_zero = (aw_lead == '0); + wire aw_owed = lead_neg || (lead_zero && w_offered && ~w_in_burst); + wire w_owed = (~lead_neg && ~lead_zero) || (lead_zero && aw_offered); + + wire aw_pass = ~blocked || aw_offered || aw_owed; + wire w_pass = ~blocked || w_offered || w_in_burst || w_owed; + + assign out_awvalid = in_awvalid && aw_pass; + assign in_awready = out_awready && aw_pass; + assign out_wvalid = in_wvalid && w_pass; + assign in_wready = out_wready && w_pass; + +endmodule diff --git a/hw/rtl/afu/common/VX_afu_wrap.sv b/hw/rtl/afu/common/VX_afu_wrap.sv index f19ed26bb8..2d3940170a 100644 --- a/hw/rtl/afu/common/VX_afu_wrap.sv +++ b/hw/rtl/afu/common/VX_afu_wrap.sv @@ -28,7 +28,8 @@ // register at host-offset 0x000. // // Data plane: -// * Memory banks 0..N-1 ride the platform AXI4 master ports. +// * One platform AXI4 master per GPU memory port (AFU_MEM_PORTS_OF); +// port i owns the i-th contiguous 1/N of device memory. // * The CP's device master shares them with Vortex: at the AXI port when // there is one, upstream of bank selection when there are several. // @@ -41,22 +42,43 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( parameter C_M_AXI_MEM_ID_WIDTH = `PLATFORM_MEMORY_ID_WIDTH, parameter C_M_AXI_MEM_DATA_WIDTH = `VX_CFG_PLATFORM_MEMORY_DATA_SIZE * 8, parameter C_M_AXI_MEM_ADDR_WIDTH = 64, -`ifdef PLATFORM_MERGED_MEMORY_INTERFACE - parameter C_M_AXI_MEM_NUM_BANKS = 1 -`else - parameter C_M_AXI_MEM_NUM_BANKS = `VX_CFG_PLATFORM_MEMORY_NUM_BANKS -`endif + parameter C_M_AXI_MEM_NUM_BANKS = `AFU_MEM_PORTS_OF(VX_MEM_PORTS) ) ( // System signals input wire clk, input wire reset, - // AXI4 master interface -`ifdef PLATFORM_MERGED_MEMORY_INTERFACE - `MP_REPEAT (1, GEN_AXI_MEM, MP_COMMA), -`else - `MP_REPEAT (`VX_CFG_PLATFORM_MEMORY_NUM_BANKS, GEN_AXI_MEM, MP_COMMA), -`endif + // AXI4 memory masters + output wire m_axi_mem_awvalid_a [C_M_AXI_MEM_NUM_BANKS], + input wire m_axi_mem_awready_a [C_M_AXI_MEM_NUM_BANKS], + output wire [C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_awaddr_a [C_M_AXI_MEM_NUM_BANKS], + output wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_awid_a [C_M_AXI_MEM_NUM_BANKS], + output wire [7:0] m_axi_mem_awlen_a [C_M_AXI_MEM_NUM_BANKS], + + output wire m_axi_mem_wvalid_a [C_M_AXI_MEM_NUM_BANKS], + input wire m_axi_mem_wready_a [C_M_AXI_MEM_NUM_BANKS], + output wire [C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_wdata_a [C_M_AXI_MEM_NUM_BANKS], + output wire [C_M_AXI_MEM_DATA_WIDTH/8-1:0] m_axi_mem_wstrb_a [C_M_AXI_MEM_NUM_BANKS], + output wire m_axi_mem_wlast_a [C_M_AXI_MEM_NUM_BANKS], + + input wire m_axi_mem_bvalid_a [C_M_AXI_MEM_NUM_BANKS], + output wire m_axi_mem_bready_a [C_M_AXI_MEM_NUM_BANKS], + input wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_bid_a [C_M_AXI_MEM_NUM_BANKS], + input wire [1:0] m_axi_mem_bresp_a [C_M_AXI_MEM_NUM_BANKS], + + output wire m_axi_mem_arvalid_a [C_M_AXI_MEM_NUM_BANKS], + input wire m_axi_mem_arready_a [C_M_AXI_MEM_NUM_BANKS], + output wire [C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_araddr_a [C_M_AXI_MEM_NUM_BANKS], + output wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_arid_a [C_M_AXI_MEM_NUM_BANKS], + output wire [7:0] m_axi_mem_arlen_a [C_M_AXI_MEM_NUM_BANKS], + + input wire m_axi_mem_rvalid_a [C_M_AXI_MEM_NUM_BANKS], + output wire m_axi_mem_rready_a [C_M_AXI_MEM_NUM_BANKS], + input wire [C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_rdata_a [C_M_AXI_MEM_NUM_BANKS], + input wire m_axi_mem_rlast_a [C_M_AXI_MEM_NUM_BANKS], + input wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_rid_a [C_M_AXI_MEM_NUM_BANKS], + input wire [1:0] m_axi_mem_rresp_a [C_M_AXI_MEM_NUM_BANKS], + // AXI4 host-memory master interface (CP command ring + host side of DMA) `GEN_AXI_HOST, // AXI4-Lite slave interface @@ -86,42 +108,10 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( ); localparam M_AXI_MEM_ADDR_WIDTH = `VX_CFG_PLATFORM_MEMORY_ADDR_WIDTH; - wire m_axi_mem_awvalid_a [C_M_AXI_MEM_NUM_BANKS]; - wire m_axi_mem_awready_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_awaddr_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_awid_a [C_M_AXI_MEM_NUM_BANKS]; - wire [7:0] m_axi_mem_awlen_a [C_M_AXI_MEM_NUM_BANKS]; - - wire m_axi_mem_wvalid_a [C_M_AXI_MEM_NUM_BANKS]; - wire m_axi_mem_wready_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_wdata_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_DATA_WIDTH/8-1:0] m_axi_mem_wstrb_a [C_M_AXI_MEM_NUM_BANKS]; - wire m_axi_mem_wlast_a [C_M_AXI_MEM_NUM_BANKS]; - - wire m_axi_mem_bvalid_a [C_M_AXI_MEM_NUM_BANKS]; - wire m_axi_mem_bready_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_bid_a [C_M_AXI_MEM_NUM_BANKS]; - wire [1:0] m_axi_mem_bresp_a [C_M_AXI_MEM_NUM_BANKS]; - - wire m_axi_mem_arvalid_a [C_M_AXI_MEM_NUM_BANKS]; - wire m_axi_mem_arready_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_araddr_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_arid_a [C_M_AXI_MEM_NUM_BANKS]; - wire [7:0] m_axi_mem_arlen_a [C_M_AXI_MEM_NUM_BANKS]; - - wire m_axi_mem_rvalid_a [C_M_AXI_MEM_NUM_BANKS]; - wire m_axi_mem_rready_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_rdata_a [C_M_AXI_MEM_NUM_BANKS]; - wire m_axi_mem_rlast_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_rid_a [C_M_AXI_MEM_NUM_BANKS]; - wire [1:0] m_axi_mem_rresp_a [C_M_AXI_MEM_NUM_BANKS]; - - // convert memory interface to array -`ifdef PLATFORM_MERGED_MEMORY_INTERFACE - `MP_REPEAT (1, AXI_MEM_TO_ARRAY, MP_SEMI); -`else - `MP_REPEAT (`VX_CFG_PLATFORM_MEMORY_NUM_BANKS, AXI_MEM_TO_ARRAY, MP_SEMI); -`endif + // The platform top declares one AXI master per GPU memory port; a count + // that disagrees with the GPU would leave ports idle or requests unrouted. + `STATIC_ASSERT(C_M_AXI_MEM_NUM_BANKS == `AFU_MEM_PORTS_OF(VX_MEM_PORTS), + ("AXI memory port count %0d does not match the GPU's %0d", C_M_AXI_MEM_NUM_BANKS, `AFU_MEM_PORTS_OF(VX_MEM_PORTS))) reg [`VX_CFG_RESET_DELAY-1:0] vx_reset_shift_r; wire vx_reset; @@ -362,26 +352,28 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( // XRT pins m_axi_host to HOST[0]; host addresses pass straight through // (no PLATFORM_MEMORY_OFFSET — that offset is device-memory specific). // Gated like the bank ports: no new host-side request leaves the AFU while - // the reset sequencer is draining. The gate is a VX_afu_req_gate rather - // than a plain AND so an offer already made to the shell is never - // withdrawn -- see the header of that module. - VX_afu_req_gate host_aw_gate ( - .clk (clk), - .reset (reset), - .stop_req (rst_stop_req), - .in_valid (cp_axi_host.awvalid), - .in_ready (cp_axi_host.awready), - .out_valid (m_axi_host_awvalid), - .out_ready (m_axi_host_awready) + // the reset sequencer is draining. The gates never withdraw an offer + // already made to the shell and never split a write -- see their headers. + VX_afu_wr_gate host_wr_gate ( + .clk (clk), + .reset (reset), + .stop_req (rst_stop_req), + .in_awvalid (cp_axi_host.awvalid), + .in_awready (cp_axi_host.awready), + .out_awvalid (m_axi_host_awvalid), + .out_awready (m_axi_host_awready), + .in_wvalid (cp_axi_host.wvalid), + .in_wlast (cp_axi_host.wlast), + .in_wready (cp_axi_host.wready), + .out_wvalid (m_axi_host_wvalid), + .out_wready (m_axi_host_wready) ); assign m_axi_host_awaddr = cp_axi_host.awaddr; assign m_axi_host_awid = {{(C_M_AXI_MEM_ID_WIDTH-`VX_CP_AXI_TID_WIDTH){1'b0}}, cp_axi_host.awid}; assign m_axi_host_awlen = cp_axi_host.awlen; - assign m_axi_host_wvalid = cp_axi_host.wvalid; assign m_axi_host_wdata = cp_axi_host.wdata; assign m_axi_host_wstrb = cp_axi_host.wstrb; assign m_axi_host_wlast = cp_axi_host.wlast; - assign cp_axi_host.wready = m_axi_host_wready; assign cp_axi_host.bvalid = m_axi_host_bvalid; assign cp_axi_host.bid = m_axi_host_bid[`VX_CP_AXI_TID_WIDTH-1:0]; assign cp_axi_host.bresp = m_axi_host_bresp; @@ -442,18 +434,39 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( wire [M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_awaddr_u [C_M_AXI_MEM_NUM_BANKS]; wire [M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_araddr_u [C_M_AXI_MEM_NUM_BANKS]; - // Per-bank XRT BO base offset. Each m_axi_mem_ port targets a different - // xrt::bo (one per DDR/HBM channel) that XRT places at a different virtual - // base address. PLATFORM_MEMORY_OFFSET applies the same synthesis-time - // offset to every bank. - wire [C_M_AXI_MEM_ADDR_WIDTH-1:0] platform_memory_offsets [C_M_AXI_MEM_NUM_BANKS]; - for (genvar i = 0; i < C_M_AXI_MEM_NUM_BANKS; ++i) begin : g_pmo - assign platform_memory_offsets[i] = C_M_AXI_MEM_ADDR_WIDTH'(`PLATFORM_MEMORY_OFFSET); - end + // Port i owns the i-th contiguous 1/N of the device address space, which + // the platform maps to that port's own group of memory channels, so no + // port reaches across another's. Lines interleave across the ports, so + // within the group a line sits at its index divided by N: drop the port + // field from the line address and put the port number on top. + // Without interleaving the bank adapter already sends each port only its + // own share. PLATFORM_MEMORY_OFFSET then rebases every port by the same + // aperture. + `STATIC_ASSERT(`IS_POW2(C_M_AXI_MEM_NUM_BANKS), + ("AXI memory port count %0d is not a power of two", C_M_AXI_MEM_NUM_BANKS)) + localparam MEM_PORT_BITS = `CLOG2(C_M_AXI_MEM_NUM_BANKS); + localparam MEM_LINE_BITS = `CLOG2(C_M_AXI_MEM_DATA_WIDTH / 8); + localparam MEM_GROUP_BITS = M_AXI_MEM_ADDR_WIDTH - MEM_PORT_BITS - MEM_LINE_BITS; + + wire [M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_awaddr_g [C_M_AXI_MEM_NUM_BANKS]; + wire [M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_araddr_g [C_M_AXI_MEM_NUM_BANKS]; for (genvar i = 0; i < C_M_AXI_MEM_NUM_BANKS; ++i) begin : g_addressing - assign m_axi_mem_awaddr_a[i] = C_M_AXI_MEM_ADDR_WIDTH'(m_axi_mem_awaddr_u[i]) + platform_memory_offsets[i]; - assign m_axi_mem_araddr_a[i] = C_M_AXI_MEM_ADDR_WIDTH'(m_axi_mem_araddr_u[i]) + platform_memory_offsets[i]; + if (C_M_AXI_MEM_NUM_BANKS > 1 && `VX_CFG_PLATFORM_MEMORY_INTERLEAVE != 0) begin : g_group + assign m_axi_mem_awaddr_g[i] = {MEM_PORT_BITS'(i), + m_axi_mem_awaddr_u[i][MEM_LINE_BITS + MEM_PORT_BITS +: MEM_GROUP_BITS], + m_axi_mem_awaddr_u[i][MEM_LINE_BITS-1:0]}; + assign m_axi_mem_araddr_g[i] = {MEM_PORT_BITS'(i), + m_axi_mem_araddr_u[i][MEM_LINE_BITS + MEM_PORT_BITS +: MEM_GROUP_BITS], + m_axi_mem_araddr_u[i][MEM_LINE_BITS-1:0]}; + `UNUSED_VAR (m_axi_mem_awaddr_u[i][MEM_LINE_BITS +: MEM_PORT_BITS]) + `UNUSED_VAR (m_axi_mem_araddr_u[i][MEM_LINE_BITS +: MEM_PORT_BITS]) + end else begin : g_linear + assign m_axi_mem_awaddr_g[i] = m_axi_mem_awaddr_u[i]; + assign m_axi_mem_araddr_g[i] = m_axi_mem_araddr_u[i]; + end + assign m_axi_mem_awaddr_a[i] = C_M_AXI_MEM_ADDR_WIDTH'(m_axi_mem_awaddr_g[i]) + C_M_AXI_MEM_ADDR_WIDTH'(`PLATFORM_MEMORY_OFFSET); + assign m_axi_mem_araddr_a[i] = C_M_AXI_MEM_ADDR_WIDTH'(m_axi_mem_araddr_g[i]) + C_M_AXI_MEM_ADDR_WIDTH'(`PLATFORM_MEMORY_OFFSET); end // ======================================================================== @@ -471,9 +484,11 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( // ======================================================================== `SCOPE_IO_SWITCH (2); - // Address requests ahead of the request gate. + // Address and write-data requests ahead of the request gates. wire pre_awvalid_a [C_M_AXI_MEM_NUM_BANKS]; wire pre_awready_a [C_M_AXI_MEM_NUM_BANKS]; + wire pre_wvalid_a [C_M_AXI_MEM_NUM_BANKS]; + wire pre_wready_a [C_M_AXI_MEM_NUM_BANKS]; wire pre_arvalid_a [C_M_AXI_MEM_NUM_BANKS]; wire pre_arready_a [C_M_AXI_MEM_NUM_BANKS]; @@ -667,7 +682,7 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( .m_awvalid (pre_awvalid_a[0]), .m_awready (pre_awready_a[0]), .m_awaddr (m_axi_mem_awaddr_u[0]), .m_awid (m_axi_mem_awid_a[0]), .m_awlen (m_axi_mem_awlen_a[0]), - .m_wvalid (m_axi_mem_wvalid_a[0]), .m_wready (m_axi_mem_wready_a[0]), + .m_wvalid (pre_wvalid_a[0]), .m_wready (pre_wready_a[0]), .m_wdata (m_axi_mem_wdata_a[0]), .m_wstrb (m_axi_mem_wstrb_a[0]), .m_wlast (m_axi_mem_wlast_a[0]), .m_bvalid (m_axi_mem_bvalid_a[0]), .m_bready (m_axi_mem_bready_a[0]), @@ -995,8 +1010,8 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( `UNUSED_PIN (m_axi_awqos), `UNUSED_PIN (m_axi_awregion), - .m_axi_wvalid (m_axi_mem_wvalid_a), - .m_axi_wready (m_axi_mem_wready_a), + .m_axi_wvalid (pre_wvalid_a), + .m_axi_wready (pre_wready_a), .m_axi_wdata (m_axi_mem_wdata_a), .m_axi_wstrb (m_axi_mem_wstrb_a), .m_axi_wlast (m_axi_mem_wlast_a), @@ -1033,22 +1048,27 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( // Request gate + drain tracking. // // Everything upstream drives pre_* instead of the bank port directly. The - // gate below withholds new AW/AR while the reset sequencer is quiescing, - // so the outstanding counts can reach zero in bounded time. W, B and R are - // never gated: a burst whose address the interconnect has already accepted - // must be allowed to finish. + // gates below withhold new reads and writes while the reset sequencer is + // quiescing, so the outstanding counts can reach zero in bounded time. + // A write already begun on either channel completes on both, and B and R + // are never gated: a transaction the interconnect has seen must finish. // ======================================================================== wire mem_idle_a [C_M_AXI_MEM_NUM_BANKS]; for (genvar i = 0; i < C_M_AXI_MEM_NUM_BANKS; ++i) begin : g_axi_gate - VX_afu_req_gate mem_aw_gate ( - .clk (clk), - .reset (reset), - .stop_req (rst_stop_req), - .in_valid (pre_awvalid_a[i]), - .in_ready (pre_awready_a[i]), - .out_valid (m_axi_mem_awvalid_a[i]), - .out_ready (m_axi_mem_awready_a[i]) + VX_afu_wr_gate mem_wr_gate ( + .clk (clk), + .reset (reset), + .stop_req (rst_stop_req), + .in_awvalid (pre_awvalid_a[i]), + .in_awready (pre_awready_a[i]), + .out_awvalid (m_axi_mem_awvalid_a[i]), + .out_awready (m_axi_mem_awready_a[i]), + .in_wvalid (pre_wvalid_a[i]), + .in_wlast (m_axi_mem_wlast_a[i]), + .in_wready (pre_wready_a[i]), + .out_wvalid (m_axi_mem_wvalid_a[i]), + .out_wready (m_axi_mem_wready_a[i]) ); VX_afu_req_gate mem_ar_gate ( @@ -1069,6 +1089,7 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( .clk (clk), .reset (reset), .aw_fire (m_axi_mem_awvalid_a[i] && m_axi_mem_awready_a[i]), + .w_fire (m_axi_mem_wvalid_a[i] && m_axi_mem_wready_a[i]), .w_fire_last (m_axi_mem_wvalid_a[i] && m_axi_mem_wready_a[i] && m_axi_mem_wlast_a[i]), .b_fire (m_axi_mem_bvalid_a[i] && m_axi_mem_bready_a[i]), @@ -1090,6 +1111,7 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( .clk (clk), .reset (reset), .aw_fire (m_axi_host_awvalid && m_axi_host_awready), + .w_fire (m_axi_host_wvalid && m_axi_host_wready), .w_fire_last (m_axi_host_wvalid && m_axi_host_wready && m_axi_host_wlast), .b_fire (m_axi_host_bvalid && m_axi_host_bready), .ar_fire (m_axi_host_arvalid && m_axi_host_arready), diff --git a/hw/rtl/afu/common/vortex_afu.vh b/hw/rtl/afu/common/vortex_afu.vh index 9544345186..4ff28ab33f 100644 --- a/hw/rtl/afu/common/vortex_afu.vh +++ b/hw/rtl/afu/common/vortex_afu.vh @@ -163,6 +163,69 @@ assign m_axi_mem_bresp_a[i] = m_axi_mem_``i``_bresp; \ assign m_axi_mem_bid_a[i] = m_axi_mem_``i``_bid +// AXI memory masters the AFU exposes: one per GPU memory port, rounded up to +// a power of two so that address bits select the port, and at most one per +// platform memory channel. A platform top must spell its ports out, so the +// build resolves this to the literal AFU_MEM_PORTS +// (hw/scripts/afu_mem_ports.sh) and VX_afu_wrap checks that the two agree. +`define AFU_MEM_PORTS_OF(n) `MIN(1 << `CLOG2(n), `VX_CFG_PLATFORM_MEMORY_NUM_BANKS) + +// Per-port arrays connecting a platform top's flat m_axi_mem_ ports to +// VX_afu_wrap (see AXI_MEM_TO_ARRAY). +`define AXI_MEM_ARRAYS(n) \ + wire m_axi_mem_awvalid_a [n]; \ + wire m_axi_mem_awready_a [n]; \ + wire [C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_awaddr_a [n]; \ + wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_awid_a [n]; \ + wire [7:0] m_axi_mem_awlen_a [n]; \ + wire m_axi_mem_wvalid_a [n]; \ + wire m_axi_mem_wready_a [n]; \ + wire [C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_wdata_a [n]; \ + wire [C_M_AXI_MEM_DATA_WIDTH/8-1:0] m_axi_mem_wstrb_a [n]; \ + wire m_axi_mem_wlast_a [n]; \ + wire m_axi_mem_arvalid_a [n]; \ + wire m_axi_mem_arready_a [n]; \ + wire [C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_araddr_a [n]; \ + wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_arid_a [n]; \ + wire [7:0] m_axi_mem_arlen_a [n]; \ + wire m_axi_mem_rvalid_a [n]; \ + wire m_axi_mem_rready_a [n]; \ + wire [C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_rdata_a [n]; \ + wire m_axi_mem_rlast_a [n]; \ + wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_rid_a [n]; \ + wire [1:0] m_axi_mem_rresp_a [n]; \ + wire m_axi_mem_bvalid_a [n]; \ + wire m_axi_mem_bready_a [n]; \ + wire [1:0] m_axi_mem_bresp_a [n]; \ + wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_bid_a [n]; + +`define AXI_MEM_ARRAY_ARGS \ + .m_axi_mem_awvalid_a(m_axi_mem_awvalid_a), \ + .m_axi_mem_awready_a(m_axi_mem_awready_a), \ + .m_axi_mem_awaddr_a(m_axi_mem_awaddr_a), \ + .m_axi_mem_awid_a(m_axi_mem_awid_a), \ + .m_axi_mem_awlen_a(m_axi_mem_awlen_a), \ + .m_axi_mem_wvalid_a(m_axi_mem_wvalid_a), \ + .m_axi_mem_wready_a(m_axi_mem_wready_a), \ + .m_axi_mem_wdata_a(m_axi_mem_wdata_a), \ + .m_axi_mem_wstrb_a(m_axi_mem_wstrb_a), \ + .m_axi_mem_wlast_a(m_axi_mem_wlast_a), \ + .m_axi_mem_arvalid_a(m_axi_mem_arvalid_a), \ + .m_axi_mem_arready_a(m_axi_mem_arready_a), \ + .m_axi_mem_araddr_a(m_axi_mem_araddr_a), \ + .m_axi_mem_arid_a(m_axi_mem_arid_a), \ + .m_axi_mem_arlen_a(m_axi_mem_arlen_a), \ + .m_axi_mem_rvalid_a(m_axi_mem_rvalid_a), \ + .m_axi_mem_rready_a(m_axi_mem_rready_a), \ + .m_axi_mem_rdata_a(m_axi_mem_rdata_a), \ + .m_axi_mem_rlast_a(m_axi_mem_rlast_a), \ + .m_axi_mem_rid_a(m_axi_mem_rid_a), \ + .m_axi_mem_rresp_a(m_axi_mem_rresp_a), \ + .m_axi_mem_bvalid_a(m_axi_mem_bvalid_a), \ + .m_axi_mem_bready_a(m_axi_mem_bready_a), \ + .m_axi_mem_bresp_a(m_axi_mem_bresp_a), \ + .m_axi_mem_bid_a(m_axi_mem_bid_a) + `include "VX_define.vh" `endif // VORTEX_AFU_VH diff --git a/hw/rtl/afu/xrt/vortex_afu.v b/hw/rtl/afu/xrt/vortex_afu.v index 6e3bd65a4d..5e37bd818c 100644 --- a/hw/rtl/afu/xrt/vortex_afu.v +++ b/hw/rtl/afu/xrt/vortex_afu.v @@ -21,22 +21,14 @@ module vortex_afu #( parameter C_M_AXI_MEM_ID_WIDTH = `PLATFORM_MEMORY_ID_WIDTH, parameter C_M_AXI_MEM_DATA_WIDTH = (`VX_CFG_PLATFORM_MEMORY_DATA_SIZE * 8), parameter C_M_AXI_MEM_ADDR_WIDTH = 64, -`ifdef PLATFORM_MERGED_MEMORY_INTERFACE - parameter C_M_AXI_MEM_NUM_BANKS = 1 -`else - parameter C_M_AXI_MEM_NUM_BANKS = `VX_CFG_PLATFORM_MEMORY_NUM_BANKS -`endif + parameter C_M_AXI_MEM_NUM_BANKS = `AFU_MEM_PORTS ) ( // System signals input wire ap_clk, input wire ap_rst_n, - // AXI4 master interface -`ifdef PLATFORM_MERGED_MEMORY_INTERFACE - `MP_REPEAT (1, GEN_AXI_MEM, MP_COMMA), -`else - `MP_REPEAT (`VX_CFG_PLATFORM_MEMORY_NUM_BANKS, GEN_AXI_MEM, MP_COMMA), -`endif + // AXI4 master interface, one per GPU memory port + `MP_REPEAT (`AFU_MEM_PORTS, GEN_AXI_MEM, MP_COMMA), // AXI4 host-memory master interface (CP command ring + host side of DMA) `GEN_AXI_HOST, @@ -75,6 +67,9 @@ module vortex_afu #( reset_r <= ~ap_rst_n; end + `AXI_MEM_ARRAYS (C_M_AXI_MEM_NUM_BANKS) + `MP_REPEAT (`AFU_MEM_PORTS, AXI_MEM_TO_ARRAY, MP_SEMI); + VX_afu_wrap #( .C_S_AXI_CTRL_ADDR_WIDTH (C_S_AXI_CTRL_ADDR_WIDTH), .C_S_AXI_CTRL_DATA_WIDTH (C_S_AXI_CTRL_DATA_WIDTH), @@ -85,11 +80,7 @@ module vortex_afu #( ) afu_wrap ( .clk (ap_clk), .reset (reset_r), - `ifdef PLATFORM_MERGED_MEMORY_INTERFACE - `MP_REPEAT (1, AXI_MEM_ARGS, MP_COMMA), - `else - `MP_REPEAT (`VX_CFG_PLATFORM_MEMORY_NUM_BANKS, AXI_MEM_ARGS, MP_COMMA), - `endif + `AXI_MEM_ARRAY_ARGS, `AXI_HOST_ARGS, .s_axi_ctrl_awvalid (s_axi_ctrl_awvalid), .s_axi_ctrl_awready (s_axi_ctrl_awready), diff --git a/hw/rtl/libs/VX_mem_to_axi.sv b/hw/rtl/libs/VX_mem_to_axi.sv index 8c928152f4..9d0070eb87 100644 --- a/hw/rtl/libs/VX_mem_to_axi.sv +++ b/hw/rtl/libs/VX_mem_to_axi.sv @@ -19,6 +19,11 @@ // (interface-free) so it can sit directly at a platform pin boundary; the // optional AXI sideband (lock/cache/prot/qos/region) is driven to constants. // The inverse of VX_membus_from_axi. +// +// The M masters share one ADDR_WIDTH_OUT byte address space, and each request +// leaves on its bank's master with its own address unchanged. INTERLEAVE +// selects the bank from the low line-address bits; otherwise from the top +// bits, so bank i serves the i-th contiguous 1/M of the space. // ============================================================================ `TRACING_OFF @@ -108,7 +113,7 @@ module VX_mem_to_axi #( localparam LOG2_DATA_SIZE = `CLOG2(DATA_SIZE); localparam BANK_SEL_BITS = `CLOG2(NUM_BANKS_OUT); localparam BANK_SEL_WIDTH = `UP(BANK_SEL_BITS); - localparam DST_ADDR_WDITH = (ADDR_WIDTH_OUT - LOG2_DATA_SIZE) + BANK_SEL_BITS; // convert byte-addressable output addresss to block-addressable input space + localparam DST_ADDR_WDITH = ADDR_WIDTH_OUT - LOG2_DATA_SIZE; // line address width of the output space localparam BANK_ADDR_WIDTH = DST_ADDR_WDITH - BANK_SEL_BITS; localparam NUM_PORTS_IN_BITS = `CLOG2(NUM_PORTS_IN); localparam NUM_PORTS_IN_WIDTH = `UP(NUM_PORTS_IN_BITS); diff --git a/hw/rtl/libs/VX_mm_axi_arb.sv b/hw/rtl/libs/VX_mm_axi_arb.sv index 7bfddde3c2..a45d96375d 100644 --- a/hw/rtl/libs/VX_mm_axi_arb.sv +++ b/hw/rtl/libs/VX_mm_axi_arb.sv @@ -41,8 +41,9 @@ // until WLAST. B/R route back to the owner. IDs pass through untouched. // * MULTI_OUT == 1: ID-routed, multi-outstanding fan-in. The top // LOG2UP(NUM_INPUTS) bits of the AXI ID carry the source index (masters -// must leave them free); reads are fully concurrent, write bursts serialize -// on the shared W channel (AW gated until WLAST) while B stays concurrent. +// must leave them free); reads are fully concurrent, writes share the W +// channel in AW order (a write's W may go with its AW, and later AWs may +// run ahead of earlier data) while B stays concurrent. // Requires ID_WIDTH > LOG2UP(NUM_INPUTS) and NUM_OUTPUTS == 1. // // OUT_REG (0/1): registered AXI stage (VX_mm_axi_slice) on each slave port. @@ -324,10 +325,10 @@ module VX_mm_axi_arb #( // ===== Multi-outstanding, ID-routed (source index in ID high bits) ===== // The top SEL_WIDTH bits of the AXI ID carry the source index; masters - // must leave them free. Reads are fully concurrent; write bursts serialize - // on the shared W channel (AW gated until the prior burst's WLAST) while B - // responses stay concurrent. An offer the slave has not yet taken holds - // the grant, so its addr/id stay stable until the handshake. + // must leave them free. Reads are fully concurrent; write data shares + // the W channel in AW order while B responses stay concurrent. An offer + // the slave has not yet taken holds the grant, so its addr/id stay + // stable until the handshake. localparam LOW_ID = ID_WIDTH - SEL_WIDTH; // source's own low ID bits `STATIC_ASSERT((ID_WIDTH > SEL_WIDTH), ("VX_mm_axi_arb MULTI_OUT: ID_WIDTH must exceed LOG2UP(NUM_INPUTS) to carry the source index")) @@ -382,9 +383,19 @@ module VX_mm_axi_arb #( end assign c_rready = s_rready[r_src]; - // ---- Write: sticky RR grant gated on W-drain, tag awid, demux B by bid ---- - reg w_active; - reg [SEL_WIDTH-1:0] w_route; + // ---- Write: sticky RR grant, tag awid, W in AW order, demux B by bid ---- + // An AW is committed the first cycle it is offered; the held grant then + // keeps it on the bus until the slave takes it, so AWs reach the slave + // in commit order. W bursts follow that order through a small queue of + // committed sources, and the burst of the AW being committed this cycle + // bypasses an empty queue: a single-beat write moves AW and W in the + // same cycle, and W never waits for AWREADY (a slave may accept the + // data first). A full queue holds new AWs back. + localparam WQ_SIZE = 4; + localparam WQ_ADDRW = `LOG2UP(WQ_SIZE); + localparam WQ_CNTW = WQ_ADDRW + 1; + + reg aw_pend; // offered last cycle, not yet taken wire [SEL_WIDTH-1:0] aw_pick; wire aw_any; wire aw_fire = c_awvalid && c_awready; @@ -394,16 +405,10 @@ module VX_mm_axi_arb #( end else begin : g_aw_arb // Once offered, the AW must stay unchanged until the slave takes it // (AXI stability): only the offering master may compete meanwhile. - reg aw_held; reg [SEL_WIDTH-1:0] aw_held_idx; - wire [NUM_INPUTS-1:0] aw_reqs = aw_held ? (s_awvalid & (NUM_INPUTS'(1) << aw_held_idx)) + wire [NUM_INPUTS-1:0] aw_reqs = aw_pend ? (s_awvalid & (NUM_INPUTS'(1) << aw_held_idx)) : s_awvalid; always @(posedge clk) begin - if (reset) begin - aw_held <= 1'b0; - end else begin - aw_held <= c_awvalid && ~c_awready; - end aw_held_idx <= aw_pick; end VX_generic_arbiter #( @@ -416,33 +421,54 @@ module VX_mm_axi_arb #( .grant_valid (aw_any), .grant_ready (aw_fire) ); end - // Gate a new AW until the previous write's W burst drains. - assign c_awvalid = aw_any && !w_active; + + reg [WQ_SIZE-1:0][SEL_WIDTH-1:0] wq_src; + reg [WQ_ADDRW-1:0] wq_rd_ptr, wq_wr_ptr; + reg [WQ_CNTW-1:0] wq_count; + wire wq_empty = (wq_count == '0); + wire wq_full = (wq_count == WQ_CNTW'(WQ_SIZE)); + + assign c_awvalid = aw_any && (aw_pend || ~wq_full); assign c_awaddr = s_awaddr[aw_pick]; assign c_awlen = s_awlen [aw_pick]; assign c_awid = {aw_pick, s_awid[aw_pick][LOW_ID-1:0]}; - assign s_awready = {NUM_INPUTS{!w_active && c_awready}} + assign s_awready = {NUM_INPUTS{(aw_pend || ~wq_full) && c_awready}} & ({{(NUM_INPUTS-1){1'b0}}, aw_any} << aw_pick); + wire aw_commit = c_awvalid && ~aw_pend; + + wire w_route_valid = ~wq_empty || aw_commit; + wire [SEL_WIDTH-1:0] w_route = wq_empty ? aw_pick : wq_src[wq_rd_ptr]; + wire w_last_fire = c_wvalid && c_wready && c_wlast; + wire wq_push = aw_commit && ~(wq_empty && w_last_fire); + wire wq_pop = ~wq_empty && w_last_fire; + always @(posedge clk) begin if (reset) begin - w_active <= 1'b0; - w_route <= '0; + aw_pend <= 1'b0; + wq_rd_ptr <= '0; + wq_wr_ptr <= '0; + wq_count <= '0; end else begin - if (aw_fire) begin - w_active <= 1'b1; - w_route <= aw_pick; - end else if (w_active && c_wvalid && c_wready && c_wlast) begin - w_active <= 1'b0; + aw_pend <= c_awvalid && ~c_awready; + if (wq_push) begin + wq_wr_ptr <= wq_wr_ptr + WQ_ADDRW'(1); + end + if (wq_pop) begin + wq_rd_ptr <= wq_rd_ptr + WQ_ADDRW'(1); end + wq_count <= wq_count + WQ_CNTW'(wq_push) - WQ_CNTW'(wq_pop); + end + if (wq_push) begin + wq_src[wq_wr_ptr] <= aw_pick; end end - assign c_wvalid = w_active && s_wvalid[w_route]; + assign c_wvalid = w_route_valid && s_wvalid[w_route]; assign c_wdata = s_wdata[w_route]; assign c_wstrb = s_wstrb[w_route]; assign c_wlast = s_wlast[w_route]; - assign s_wready = {NUM_INPUTS{w_active && c_wready}} + assign s_wready = {NUM_INPUTS{w_route_valid && c_wready}} & ({{(NUM_INPUTS-1){1'b0}}, 1'b1} << w_route); wire [SEL_WIDTH-1:0] b_src = c_bid[ID_WIDTH-1 -: SEL_WIDTH]; diff --git a/hw/scripts/afu_mem_ports.sh b/hw/scripts/afu_mem_ports.sh new file mode 100755 index 0000000000..df9b7031cd --- /dev/null +++ b/hw/scripts/afu_mem_ports.sh @@ -0,0 +1,89 @@ +#!/bin/bash + +# Copyright © 2019-2023 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Prints how many AXI memory masters the AFU exposes for a configuration. +# +# Usage: afu_mem_ports.sh [-c ] +# +# The count follows the GPU's memory ports, which only the elaborated RTL +# package knows; a platform top must nevertheless spell its ports out +# literally. Elaborating a one-line probe against the same flags as the build +# hands the build the exact value VX_afu_wrap checks it against. +# +# With -c, the result is cached under , keyed on the flags and on +# the contents of every package and configuration header they name, so a +# Makefile can resolve it on every parse without re-running Verilator. + +set -eo pipefail + +cache_dir="" +if [ "$1" = "-c" ]; then + cache_dir=$2 + shift 2 +fi + +VERILATOR=$1 +shift + +if [ -n "$cache_dir" ]; then + key=$({ + printf '%s\n' "$@" + for arg in "$@"; do + case "$arg" in + -I*) for h in VX_config.vh VX_types.vh vortex_afu.vh; do + if [ -f "${arg#-I}/$h" ]; then cat "${arg#-I}/$h"; fi + done ;; + *) if [ -f "$arg" ]; then cat "$arg"; fi ;; + esac + done + } | sha1sum | cut -d' ' -f1) + cache_file="$cache_dir/afu_mem_ports.$key" + if [ -s "$cache_file" ]; then + cat "$cache_file" + exit 0 + fi +fi + +tmp_dir=$(mktemp -d) +trap 'rm -rf "$tmp_dir"' EXIT + +cat > "$tmp_dir/VX_afu_mem_ports_probe.sv" <<'EOF_PROBE' +`include "VX_define.vh" +`include "vortex_afu.vh" +module VX_afu_mem_ports_probe import VX_gpu_pkg::*; (); + $info("AFU_MEM_PORTS=%0d", `AFU_MEM_PORTS_OF(VX_MEM_PORTS)); +endmodule +EOF_PROBE + +if ! out=$("$VERILATOR" --lint-only -Wno-fatal "$@" --top-module VX_afu_mem_ports_probe \ + "$tmp_dir/VX_afu_mem_ports_probe.sv" 2>&1); then + echo "$out" >&2 + echo "afu_mem_ports.sh: the probe did not elaborate" >&2 + exit 1 +fi + +count=$(echo "$out" | grep -o 'AFU_MEM_PORTS=[0-9][0-9]*' | head -1 | cut -d= -f2 || true) +if [ -z "$count" ]; then + echo "$out" >&2 + echo "afu_mem_ports.sh: the probe reported no port count" >&2 + exit 1 +fi + +if [ -n "$cache_dir" ]; then + mkdir -p "$cache_dir" + echo "$count" > "$cache_file.$$" + mv "$cache_file.$$" "$cache_file" +fi +echo "$count" diff --git a/hw/syn/xilinx/aved/Makefile b/hw/syn/xilinx/aved/Makefile index f1471e46d8..0787371146 100644 --- a/hw/syn/xilinx/aved/Makefile +++ b/hw/syn/xilinx/aved/Makefile @@ -172,6 +172,18 @@ CFLAGS += $(RTL_INCLUDE) # ast dump flags XML_CFLAGS = $(filter-out -DSYNTHESIS -DVIVADO, $(CFLAGS)) $(RTL_PKGS) -I$(DPI_DIR) -DSV_DPI +# One AXI memory master per GPU memory port. The kernel top must declare its +# ports literally, so resolve the count from the elaborated configuration. +AFU_MEM_PORTS := $(shell $(SCRIPT_DIR)/afu_mem_ports.sh -c $(BUILD_DIR)/.afu_mem_ports $(VERILATOR_PATH)/bin/verilator $(filter-out -J%,$(XML_CFLAGS))) +ifeq ($(AFU_MEM_PORTS),) +$(error cannot resolve the AFU memory port count for this configuration) +endif +# config.cfg.tmpl binds the single device-memory master to MEM_TAG. +ifneq ($(AFU_MEM_PORTS),1) +$(error the V80 link binds one device-memory master, but this configuration has AFU_MEM_PORTS=$(AFU_MEM_PORTS)) +endif +CFLAGS += -DAFU_MEM_PORTS=$(AFU_MEM_PORTS) + .PHONY: all clean gen-sources gen-ast scope-json package-ip gen-bin check-axil-blocks # Fail before synthesis if any 16-byte block in the CP AXI-Lite regfile is only @@ -188,9 +200,18 @@ check-axil-blocks: all: check-axil-blocks $(VBIN_CONTAINER) gen-sources: $(BUILD_DIR)/sources.txt -$(BUILD_DIR)/sources.txt: +$(BUILD_DIR)/sources.txt: $(BUILD_DIR)/afu_mem_ports.stamp mkdir -p $(BUILD_DIR); cd $(BUILD_DIR); $(SCRIPT_DIR)/gen_sources.sh -P $(CFLAGS) -Csrc -Osources.txt +# The preprocessed sources bake in the kernel's port count: regenerate them +# whenever the configuration changes it. +$(BUILD_DIR)/afu_mem_ports.stamp: force-afu-mem-ports + @mkdir -p $(BUILD_DIR) + @if [ "$$(cat $@ 2>/dev/null)" != "$(AFU_MEM_PORTS)" ]; then echo "$(AFU_MEM_PORTS)" > $@; fi + +.PHONY: force-afu-mem-ports +force-afu-mem-ports: + gen-ast: $(BUILD_DIR)/vortex.xml $(BUILD_DIR)/vortex.xml: mkdir -p $(BUILD_DIR); cd $(BUILD_DIR); $(VERILATOR_PATH)/bin/verilator --xml-only -O0 $(XML_CFLAGS) vortex_afu.v --xml-output vortex.xml diff --git a/hw/syn/xilinx/aved/platforms.mk b/hw/syn/xilinx/aved/platforms.mk index 80ced1dd35..64029f51f0 100644 --- a/hw/syn/xilinx/aved/platforms.mk +++ b/hw/syn/xilinx/aved/platforms.mk @@ -8,10 +8,10 @@ override CONFIGS += -DPLATFORM_MEMORY_DATA_WIDTH=512 # The V80 exposes its HBM stacks through the linker's HBM connectivity tags. # A single wide master fanned across the channels avoids the per-bank base # address problem: the linker assigns each memory range independently, and -# those bases are not known at synthesis time. +# those bases are not known at synthesis time. One memory channel caps the +# AFU at one memory master whatever the GPU's memory port count. override CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=1 override CONFIGS += -DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH=34 -override CONFIGS += -DPLATFORM_MERGED_MEMORY_INTERFACE # Device-memory aperture base. The V80 maps the AFU's m_axi_mem_0 port at # 0x40_0000_0000 in both build flavours -- simulation puts the BRAM model there diff --git a/hw/syn/xilinx/xrt/Makefile b/hw/syn/xilinx/xrt/Makefile index 22dc8986d5..a1ec0c00b2 100644 --- a/hw/syn/xilinx/xrt/Makefile +++ b/hw/syn/xilinx/xrt/Makefile @@ -194,6 +194,26 @@ CFLAGS += $(RTL_INCLUDE) # ast dump flags XML_CFLAGS = $(filter-out -DSYNTHESIS -DVIVADO, $(CFLAGS)) $(RTL_PKGS) -I$(DPI_DIR) -DSV_DPI +# One AXI memory master per GPU memory port. The kernel top must declare its +# ports literally, so resolve the count from the elaborated configuration. +AFU_MEM_PORTS := $(shell $(SCRIPT_DIR)/afu_mem_ports.sh -c $(BUILD_DIR)/.afu_mem_ports $(VERILATOR_PATH)/bin/verilator $(filter-out -J%,$(XML_CFLAGS))) +ifeq ($(AFU_MEM_PORTS),) +$(error cannot resolve the AFU memory port count for this configuration) +endif +CFLAGS += -DAFU_MEM_PORTS=$(AFU_MEM_PORTS) + +# Port i owns the i-th contiguous 1/N of device memory (VX_afu_wrap). Bind it +# to the matching contiguous group of the platform's memory channels, so each +# port reaches only its own channels and no two ports share one. +ifdef PLATFORM_MEMORY_TAG +VPP_FLAGS += $(shell n=$(AFU_MEM_PORTS); g=$$(($(PLATFORM_MEMORY_CHANNELS) / n)); \ + for i in $$(seq 0 $$((n - 1))); do \ + lo=$$((i * g)); hi=$$((lo + g - 1)); \ + if [ $$g -eq 1 ]; then r=$$lo; else r=$$lo:$$hi; fi; \ + echo "--connectivity.sp vortex_afu_1.m_axi_mem_$$i:$(PLATFORM_MEMORY_TAG)[$$r]"; \ + done) +endif + # RTL Kernel only supports Hardware and Hardware Emulation. ifneq ($(TARGET),$(findstring $(TARGET), hw hw_emu)) $(warning WARNING:Application supports only hw hw_emu TARGET. Please use the target for running the application) @@ -204,9 +224,18 @@ endif all: check-devices emconfig $(XCLBIN_CONTAINER) report gen-sources: $(BUILD_DIR)/sources.txt -$(BUILD_DIR)/sources.txt: +$(BUILD_DIR)/sources.txt: $(BUILD_DIR)/afu_mem_ports.stamp mkdir -p $(BUILD_DIR); cd $(BUILD_DIR); $(SCRIPT_DIR)/gen_sources.sh -P $(CFLAGS) -Csrc -Osources.txt +# The preprocessed sources bake in the kernel's port count: regenerate them +# whenever the configuration changes it. +$(BUILD_DIR)/afu_mem_ports.stamp: force-afu-mem-ports + @mkdir -p $(BUILD_DIR) + @if [ "$$(cat $@ 2>/dev/null)" != "$(AFU_MEM_PORTS)" ]; then echo "$(AFU_MEM_PORTS)" > $@; fi + +.PHONY: force-afu-mem-ports +force-afu-mem-ports: + gen-ast: $(BUILD_DIR)/vortex.xml $(BUILD_DIR)/vortex.xml: mkdir -p $(BUILD_DIR); cd $(BUILD_DIR); $(VERILATOR_PATH)/bin/verilator --xml-only -O0 $(XML_CFLAGS) vortex_afu.v --xml-output vortex.xml diff --git a/hw/syn/xilinx/xrt/package_kernel.tcl b/hw/syn/xilinx/xrt/package_kernel.tcl index 2f97354835..d9b9777c72 100644 --- a/hw/syn/xilinx/xrt/package_kernel.tcl +++ b/hw/syn/xilinx/xrt/package_kernel.tcl @@ -45,8 +45,7 @@ set vdefines_list [lindex $vlist 2] #puts ${vdefines_list} set chipscope 0 -set num_banks 1 -set merged_mem_if 0 +set afu_mem_ports -1 # parse vdefines_list for configuration parameters foreach def $vdefines_list { @@ -55,16 +54,9 @@ foreach def $vdefines_list { if { $name == "CHIPSCOPE" } { set chipscope 1 } - if { $name == "PLATFORM_MEMORY_NUM_BANKS" } { - set num_banks [lindex $fields 1] + if { $name == "AFU_MEM_PORTS" } { + set afu_mem_ports [lindex $fields 1] } - if { $name == "PLATFORM_MERGED_MEMORY_INTERFACE" } { - set merged_mem_if 1 - } -} - -if { $merged_mem_if == 1 } { - set num_banks 1 } create_project -force kernel_pack $path_to_tmp_project @@ -177,6 +169,14 @@ foreach m $axi_masters { ipx::associate_bus_interfaces -busif $m -clock ap_clk $core } +# The build binds AFU_MEM_PORTS memory masters to device memory; a kernel +# with any other count would leave ports unbound or bind names it lacks. +set num_mem_masters [llength [lsearch -all -glob $axi_masters "m_axi_mem_*"]] +if { $num_mem_masters != $afu_mem_ports } { + puts "ERROR: packaged kernel has $num_mem_masters m_axi_mem masters, but AFU_MEM_PORTS=$afu_mem_ports" + exit 1 +} + set mem_map [::ipx::add_memory_map -quiet "s_axi_ctrl" $core] set addr_block [::ipx::add_address_block -quiet "reg0" $mem_map] diff --git a/hw/syn/xilinx/xrt/platforms.mk b/hw/syn/xilinx/xrt/platforms.mk index 7b4356bfe0..f50397a122 100644 --- a/hw/syn/xilinx/xrt/platforms.mk +++ b/hw/syn/xilinx/xrt/platforms.mk @@ -18,30 +18,35 @@ else # through the platform slave-bridge / Host Memory Access aperture. All Alveo # XDMA shells expose this as the HOST[0] connectivity tag. VPP_FLAGS += --connectivity.sp vortex_afu_1.m_axi_host:HOST[0] +# PLATFORM_MEMORY_TAG/CHANNELS name the device-memory channels the AFU's +# memory masters bind to; the Makefile gives each master its own group. ifneq ($(findstring xilinx_u55c,$(XSA)),) # 16 GB of HBM2 with 32 channels (512 MB per channel) - CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=32 -DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH=34 - CONFIGS += -DPLATFORM_MERGED_MEMORY_INTERFACE - VPP_FLAGS += --connectivity.sp vortex_afu_1.m_axi_mem_0:HBM[0:31] - #VPP_FLAGS += $(foreach i,$(shell seq 0 31), --connectivity.sp vortex_afu_1.m_axi_mem_$(i):HBM[$(i)]) + PLATFORM_MEMORY_TAG := HBM + PLATFORM_MEMORY_CHANNELS := 32 + CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=$(PLATFORM_MEMORY_CHANNELS) -DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH=34 else ifneq ($(findstring xilinx_u50,$(XSA)),) # 8 GB of HBM2 with 32 channels (256 MB per channel) - CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=32 -DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH=33 - CONFIGS += -DPLATFORM_MERGED_MEMORY_INTERFACE - VPP_FLAGS += --connectivity.sp vortex_afu_1.m_axi_mem_0:HBM[0:31] + PLATFORM_MEMORY_TAG := HBM + PLATFORM_MEMORY_CHANNELS := 32 + CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=$(PLATFORM_MEMORY_CHANNELS) -DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH=33 else ifneq ($(findstring xilinx_u280,$(XSA)),) # 8 GB of HBM2 with 32 channels (256 MB per channel) - CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=32 -DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH=33 - VPP_FLAGS += --connectivity.sp vortex_afu_1.m_axi_mem_0:HBM[0:31] + PLATFORM_MEMORY_TAG := HBM + PLATFORM_MEMORY_CHANNELS := 32 + CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=$(PLATFORM_MEMORY_CHANNELS) -DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH=33 else ifneq ($(findstring xilinx_u250,$(XSA)),) # 16 GB of DDR4 (single channel, bank 0). Multi-bank requires per-bank XRT # VA offsets that aren't known at synthesis time. - CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=1 -DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH=34 - VPP_FLAGS += --connectivity.sp vortex_afu_1.m_axi_mem_0:DDR[0] + PLATFORM_MEMORY_TAG := DDR + PLATFORM_MEMORY_CHANNELS := 1 + CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=$(PLATFORM_MEMORY_CHANNELS) -DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH=34 CONFIGS += -DPLATFORM_MEMORY_OFFSET=40\'h4000000000 else ifneq ($(findstring xilinx_u200,$(XSA)),) # 64 GB of DDR4 with 4 channels (16 GB per channel) - CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=4 -DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH=36 + PLATFORM_MEMORY_TAG := DDR + PLATFORM_MEMORY_CHANNELS := 4 + CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=$(PLATFORM_MEMORY_CHANNELS) -DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH=36 else CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=1 -DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH=32 endif diff --git a/hw/unittest/Makefile b/hw/unittest/Makefile index 87c8e63f85..4f2916de0f 100644 --- a/hw/unittest/Makefile +++ b/hw/unittest/Makefile @@ -19,6 +19,7 @@ all: $(MAKE) -C cp_axil_regfile $(MAKE) -C afu_axil_demux $(MAKE) -C afu_reset_seq + $(MAKE) -C mm_axi_arb $(MAKE) -C cp_axi_path $(MAKE) -C cp_dma $(MAKE) -C cp_llc_port @@ -52,6 +53,7 @@ run: $(MAKE) -C cp_axil_regfile run $(MAKE) -C afu_axil_demux run $(MAKE) -C afu_reset_seq run + $(MAKE) -C mm_axi_arb run $(MAKE) -C cp_axi_path run $(MAKE) -C cp_dma run $(MAKE) -C cp_llc_port run @@ -153,6 +155,7 @@ clean: $(MAKE) -C cp_dma clean $(MAKE) -C cp_llc_port clean $(MAKE) -C stream_arb clean + $(MAKE) -C mm_axi_arb clean $(MAKE) -C cp_core clean $(MAKE) -C fma_unit clean $(MAKE) -C fdiv_unit clean diff --git a/hw/unittest/afu_reset_seq/VX_afu_reset_seq_top.sv b/hw/unittest/afu_reset_seq/VX_afu_reset_seq_top.sv index a0fbbf4d70..c7ad7530af 100644 --- a/hw/unittest/afu_reset_seq/VX_afu_reset_seq_top.sv +++ b/hw/unittest/afu_reset_seq/VX_afu_reset_seq_top.sv @@ -30,6 +30,7 @@ module VX_afu_reset_seq_top #( // AXI handshakes, driven directly by the testbench. input wire aw_fire, + input wire w_fire, input wire w_fire_last, input wire b_fire, input wire ar_fire, @@ -43,6 +44,18 @@ module VX_afu_reset_seq_top #( output wire gate_in_ready, output wire gate_out_valid, + // A real write gate on the same stop_req, the testbench again playing + // both the AFU-internal master (wg_in_*) and the shell (wg_out_*). + input wire wg_in_awvalid, + input wire wg_out_awready, + output wire wg_in_awready, + output wire wg_out_awvalid, + input wire wg_in_wvalid, + input wire wg_in_wlast, + input wire wg_out_wready, + output wire wg_in_wready, + output wire wg_out_wvalid, + output wire stop_req, output wire rst_assert, output wire busy, @@ -60,10 +73,26 @@ module VX_afu_reset_seq_top #( .out_ready (gate_out_ready) ); + VX_afu_wr_gate wr_gate ( + .clk (clk), + .reset (reset), + .stop_req (stop_req), + .in_awvalid (wg_in_awvalid), + .in_awready (wg_in_awready), + .out_awvalid (wg_out_awvalid), + .out_awready (wg_out_awready), + .in_wvalid (wg_in_wvalid), + .in_wlast (wg_in_wlast), + .in_wready (wg_in_wready), + .out_wvalid (wg_out_wvalid), + .out_wready (wg_out_wready) + ); + VX_afu_axi_drain drain ( .clk (clk), .reset (reset), .aw_fire (aw_fire), + .w_fire (w_fire), .w_fire_last (w_fire_last), .b_fire (b_fire), .ar_fire (ar_fire), diff --git a/hw/unittest/afu_reset_seq/main.cpp b/hw/unittest/afu_reset_seq/main.cpp index 6f303d8f3c..3656915e4c 100644 --- a/hw/unittest/afu_reset_seq/main.cpp +++ b/hw/unittest/afu_reset_seq/main.cpp @@ -35,9 +35,11 @@ void tick() { } void clear_fires() { - dut->aw_fire = 0; dut->w_fire_last = 0; dut->b_fire = 0; + dut->aw_fire = 0; dut->w_fire = 0; dut->w_fire_last = 0; dut->b_fire = 0; dut->ar_fire = 0; dut->r_fire_last = 0; dut->gate_in_valid = 0; dut->gate_out_ready = 0; + dut->wg_in_awvalid = 0; dut->wg_out_awready = 0; + dut->wg_in_wvalid = 0; dut->wg_in_wlast = 0; dut->wg_out_wready = 0; } void do_reset() { @@ -203,6 +205,154 @@ int main(int argc, char** argv) { run(8); check(!dut->stop_req, "gate reopens after the sequence"); + // ------------------------------------------------------------------ + // AXI4 lets write data precede its address. Quiescing must neither strand + // data whose AW it then withholds nor withhold the AW of data the shell + // already took: the shell must see whole writes only. + printf("8. quiescing never splits a write\n"); + do_reset(); + dut->ar_fire = 1; tick(); dut->ar_fire = 0; // keep QUIESCE long enough to observe + + // (a) data first: a single-beat W is taken before its AW is presented. + dut->wg_in_wvalid = 1; dut->wg_in_wlast = 1; dut->wg_out_wready = 1; + tick(); + dut->wg_in_wvalid = 0; dut->wg_in_wlast = 0; + pulse_request(); + run(4); + check(dut->stop_req, "sequencer is quiescing"); + dut->wg_in_awvalid = 1; dut->wg_out_awready = 0; + dut->eval(); + check(dut->wg_out_awvalid, "the AW owed to accepted data still goes out"); + tick(); + dut->wg_out_awready = 1; + dut->eval(); + check(dut->wg_out_awvalid && dut->wg_in_awready, "and is accepted"); + tick(); + dut->wg_out_awready = 0; + dut->eval(); + check(!dut->wg_out_awvalid, "the next AW is withheld"); + dut->wg_out_awready = 1; + dut->eval(); + check(!dut->wg_in_awready, "and not reported accepted"); + dut->wg_in_wvalid = 1; dut->wg_in_wlast = 1; + dut->eval(); + check(!dut->wg_out_wvalid, "its data is withheld too"); + check(!dut->wg_in_wready, "and not reported accepted"); + run(4); + check(!dut->wg_out_awvalid && !dut->wg_out_wvalid, "and both stay withheld"); + clear_fires(); + dut->r_fire_last = 1; tick(); dut->r_fire_last = 0; + run(16); + check(saw_assert, "reset proceeds"); + run(8); + + // (b) address first, then a two-beat burst: the data owed to an accepted AW + // goes out in full, and nothing after it. + do_reset(); + dut->ar_fire = 1; tick(); dut->ar_fire = 0; + dut->wg_in_awvalid = 1; dut->wg_out_awready = 1; + tick(); + dut->wg_in_awvalid = 0; + pulse_request(); + run(4); + dut->wg_in_wvalid = 1; dut->wg_in_wlast = 0; dut->wg_out_wready = 1; + dut->eval(); + check(dut->wg_out_wvalid, "the first beat owed to the AW goes out"); + tick(); + dut->wg_in_wlast = 1; + dut->eval(); + check(dut->wg_out_wvalid, "the burst runs to its last beat"); + tick(); + dut->wg_in_wlast = 0; + dut->eval(); + check(!dut->wg_out_wvalid, "data with no accepted AW is withheld"); + dut->wg_in_awvalid = 1; + dut->eval(); + check(!dut->wg_out_awvalid, "a new AW is withheld"); + clear_fires(); + dut->r_fire_last = 1; tick(); dut->r_fire_last = 0; + run(16); + check(saw_assert, "reset proceeds"); + run(8); + check(!dut->stop_req, "gate reopens after the sequence"); + dut->wg_in_awvalid = 1; dut->wg_in_wvalid = 1; dut->wg_in_wlast = 1; + dut->wg_out_awready = 1; dut->wg_out_wready = 1; + dut->eval(); + check(dut->wg_out_awvalid && dut->wg_out_wvalid, "writes flow again once reopened"); + clear_fires(); + + // (c) a slave may hold AWREADY until it has seen the data: an address + // already offered when quiescing begins obliges its data. + do_reset(); + dut->ar_fire = 1; tick(); dut->ar_fire = 0; + dut->wg_in_awvalid = 1; dut->wg_out_awready = 0; + tick(); + pulse_request(); + run(4); + check(dut->stop_req, "sequencer is quiescing"); + dut->wg_in_wvalid = 1; dut->wg_in_wlast = 1; dut->wg_out_wready = 1; + dut->eval(); + check(dut->wg_out_wvalid, "data owed to an offered AW goes out"); + tick(); + dut->wg_in_wvalid = 0; dut->wg_in_wlast = 0; + dut->wg_out_awready = 1; + dut->eval(); + check(dut->wg_out_awvalid, "and the slave then takes the AW"); + tick(); + dut->wg_in_wvalid = 1; dut->wg_in_wlast = 1; + dut->eval(); + check(!dut->wg_out_wvalid && !dut->wg_out_awvalid, "nothing further goes out"); + clear_fires(); + dut->r_fire_last = 1; tick(); dut->r_fire_last = 0; + run(16); + check(saw_assert, "reset proceeds"); + run(8); + + // (d) a slave may hold WREADY until it has seen the address: data already + // offered when quiescing begins obliges its AW. + do_reset(); + dut->ar_fire = 1; tick(); dut->ar_fire = 0; + dut->wg_in_wvalid = 1; dut->wg_in_wlast = 1; dut->wg_out_wready = 0; + tick(); + pulse_request(); + run(4); + dut->wg_in_awvalid = 1; dut->wg_out_awready = 1; + dut->eval(); + check(dut->wg_out_awvalid, "the AW owed to offered data goes out"); + tick(); + dut->wg_in_awvalid = 0; + dut->wg_out_wready = 1; + dut->eval(); + check(dut->wg_out_wvalid, "and the slave then takes the data"); + tick(); + dut->wg_in_awvalid = 1; + dut->eval(); + check(!dut->wg_out_wvalid && !dut->wg_out_awvalid, "nothing further goes out"); + clear_fires(); + dut->r_fire_last = 1; tick(); dut->r_fire_last = 0; + run(16); + check(saw_assert, "reset proceeds"); + run(8); + + // ------------------------------------------------------------------ + // The drain counters see last beats only. A burst whose first beats went + // out ahead of its AW still owes the interconnect the rest of the burst + // and the AW, so the port is not idle and reset must wait. + printf("9. a write burst in progress keeps the port busy\n"); + do_reset(); + dut->w_fire = 1; tick(); dut->w_fire = 0; // beat 1 of 2, no AW yet + tick(); + check(!dut->masters_idle, "not idle with a burst under way"); + pulse_request(); + run(16); + check(!saw_assert, "rst_assert withheld mid-burst"); + dut->w_fire = 1; dut->w_fire_last = 1; tick(); dut->w_fire = 0; dut->w_fire_last = 0; + dut->aw_fire = 1; tick(); dut->aw_fire = 0; + dut->b_fire = 1; tick(); dut->b_fire = 0; + run(8); + check(dut->masters_idle, "idle once the write completes"); + check(saw_assert, "reset proceeds"); + printf("\n%s (%d failure%s)\n", failures ? "FAILED" : "PASSED", failures, failures == 1 ? "" : "s"); delete dut; diff --git a/hw/unittest/mm_axi_arb/Makefile b/hw/unittest/mm_axi_arb/Makefile new file mode 100644 index 0000000000..7ff7e7effb --- /dev/null +++ b/hw/unittest/mm_axi_arb/Makefile @@ -0,0 +1,23 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := mm_axi_arb + +RTL_DIR := $(VORTEX_HOME)/hw/rtl +DPI_DIR := $(VORTEX_HOME)/hw/dpi + +SRC_DIR := $(VORTEX_HOME)/hw/unittest/$(PROJECT) + +CXXFLAGS := -I$(SRC_DIR) -I$(VORTEX_HOME)/hw/unittest/common -I$(SW_COMMON_DIR) + +SRCS := $(SRC_DIR)/main.cpp + +DBG_TRACE_FLAGS := + +RTL_PKGS := + +RTL_INCLUDE := -I$(RTL_DIR) -I$(DPI_DIR) -I$(RTL_DIR)/libs -I$(SRC_DIR) + +TOP := VX_mm_axi_arb_top + +include ../common.mk diff --git a/hw/unittest/mm_axi_arb/VX_mm_axi_arb_top.sv b/hw/unittest/mm_axi_arb/VX_mm_axi_arb_top.sv new file mode 100644 index 0000000000..73f72b6724 --- /dev/null +++ b/hw/unittest/mm_axi_arb/VX_mm_axi_arb_top.sv @@ -0,0 +1,159 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_platform.vh" + +// Harness for the ID-routed, multi-outstanding 2:1 VX_mm_axi_arb; the +// testbench plays both masters and the slave. +module VX_mm_axi_arb_top #( + parameter NUM_INPUTS = 2, + parameter ADDR_WIDTH = 32, + parameter DATA_WIDTH = 32, + parameter ID_WIDTH = 8 +) ( + input wire clk, + input wire reset, + + input wire [NUM_INPUTS-1:0] s_awvalid, + output wire [NUM_INPUTS-1:0] s_awready, + input wire [NUM_INPUTS-1:0][ADDR_WIDTH-1:0] s_awaddr, + input wire [NUM_INPUTS-1:0][ID_WIDTH-1:0] s_awid, + input wire [NUM_INPUTS-1:0][7:0] s_awlen, + + input wire [NUM_INPUTS-1:0] s_wvalid, + output wire [NUM_INPUTS-1:0] s_wready, + input wire [NUM_INPUTS-1:0][DATA_WIDTH-1:0] s_wdata, + input wire [NUM_INPUTS-1:0] s_wlast, + + output wire [NUM_INPUTS-1:0] s_bvalid, + input wire [NUM_INPUTS-1:0] s_bready, + output wire [NUM_INPUTS-1:0][ID_WIDTH-1:0] s_bid, + + input wire [NUM_INPUTS-1:0] s_arvalid, + output wire [NUM_INPUTS-1:0] s_arready, + input wire [NUM_INPUTS-1:0][ADDR_WIDTH-1:0] s_araddr, + input wire [NUM_INPUTS-1:0][ID_WIDTH-1:0] s_arid, + input wire [NUM_INPUTS-1:0][7:0] s_arlen, + + output wire [NUM_INPUTS-1:0] s_rvalid, + input wire [NUM_INPUTS-1:0] s_rready, + output wire [NUM_INPUTS-1:0][DATA_WIDTH-1:0] s_rdata, + output wire [NUM_INPUTS-1:0] s_rlast, + output wire [NUM_INPUTS-1:0][ID_WIDTH-1:0] s_rid, + + output wire m_awvalid, + input wire m_awready, + output wire [ADDR_WIDTH-1:0] m_awaddr, + output wire [ID_WIDTH-1:0] m_awid, + output wire [7:0] m_awlen, + + output wire m_wvalid, + input wire m_wready, + output wire [DATA_WIDTH-1:0] m_wdata, + output wire m_wlast, + + input wire m_bvalid, + output wire m_bready, + input wire [ID_WIDTH-1:0] m_bid, + + output wire m_arvalid, + input wire m_arready, + output wire [ADDR_WIDTH-1:0] m_araddr, + output wire [ID_WIDTH-1:0] m_arid, + output wire [7:0] m_arlen, + + input wire m_rvalid, + output wire m_rready, + input wire [DATA_WIDTH-1:0] m_rdata, + input wire m_rlast, + input wire [ID_WIDTH-1:0] m_rid +); + localparam STRB_WIDTH = DATA_WIDTH / 8; + + wire [NUM_INPUTS-1:0][1:0] s_bresp, s_rresp; + wire [STRB_WIDTH-1:0] m_wstrb; + `UNUSED_VAR ({s_bresp, s_rresp, m_wstrb}) + + VX_mm_axi_arb #( + .NUM_INPUTS (NUM_INPUTS), + .ADDR_WIDTH (ADDR_WIDTH), + .DATA_WIDTH (DATA_WIDTH), + .ID_WIDTH (ID_WIDTH), + .ARBITER ("R"), + .MULTI_OUT (1) + ) arb ( + .clk (clk), + .reset (reset), + + .s_awvalid (s_awvalid), + .s_awready (s_awready), + .s_awaddr (s_awaddr), + .s_awid (s_awid), + .s_awlen (s_awlen), + + .s_wvalid (s_wvalid), + .s_wready (s_wready), + .s_wdata (s_wdata), + .s_wstrb ({NUM_INPUTS{{STRB_WIDTH{1'b1}}}}), + .s_wlast (s_wlast), + + .s_bvalid (s_bvalid), + .s_bready (s_bready), + .s_bid (s_bid), + .s_bresp (s_bresp), + + .s_arvalid (s_arvalid), + .s_arready (s_arready), + .s_araddr (s_araddr), + .s_arid (s_arid), + .s_arlen (s_arlen), + + .s_rvalid (s_rvalid), + .s_rready (s_rready), + .s_rdata (s_rdata), + .s_rlast (s_rlast), + .s_rid (s_rid), + .s_rresp (s_rresp), + + .m_awvalid (m_awvalid), + .m_awready (m_awready), + .m_awaddr (m_awaddr), + .m_awid (m_awid), + .m_awlen (m_awlen), + + .m_wvalid (m_wvalid), + .m_wready (m_wready), + .m_wdata (m_wdata), + .m_wstrb (m_wstrb), + .m_wlast (m_wlast), + + .m_bvalid (m_bvalid), + .m_bready (m_bready), + .m_bid (m_bid), + .m_bresp (2'b00), + + .m_arvalid (m_arvalid), + .m_arready (m_arready), + .m_araddr (m_araddr), + .m_arid (m_arid), + .m_arlen (m_arlen), + + .m_rvalid (m_rvalid), + .m_rready (m_rready), + .m_rdata (m_rdata), + .m_rlast (m_rlast), + .m_rid (m_rid), + .m_rresp (2'b00) + ); + +endmodule diff --git a/hw/unittest/mm_axi_arb/main.cpp b/hw/unittest/mm_axi_arb/main.cpp new file mode 100644 index 0000000000..0dfd3bbc7c --- /dev/null +++ b/hw/unittest/mm_axi_arb/main.cpp @@ -0,0 +1,415 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The properties under test, for the ID-routed (MULTI_OUT) 2:1 arbiter: +// * the slave sees each write's data in the order it saw the addresses, +// whole bursts at a time, whichever channel each master offers first; +// * an offer the slave has not taken stays unchanged (AXI4 A3.2.1); +// * no master waits on AWREADY for its data, so a slave that takes the +// data before the address cannot deadlock it; +// * B and R reach the master that issued them, in order; +// * a write whose address and data are offered together moves in one cycle. + +#include +#include +#include +#include +#include "VVX_mm_axi_arb_top.h" +#include "verilated.h" + +namespace { + +constexpr int NUM_MASTERS = 2; +constexpr int ID_SRC_BIT = 7; // the arbiter's source field (ID_WIDTH-1) + +VVX_mm_axi_arb_top* dut = nullptr; +int failures = 0; +uint32_t rng = 1; + +uint32_t rnd() { + rng = rng * 1103515245u + 12345u; + return rng >> 16; +} + +bool chance(int pct) { return int(rnd() % 100) < pct; } + +void check(bool ok, const char* what) { + printf(" %-62s %s\n", what, ok ? "PASS" : "*** FAIL ***"); + if (!ok) { + ++failures; + } +} + +int errors = 0; +void error(const char* msg, long a = 0, long b = 0) { + if (errors < 10) { + printf(" error: %s (%ld, %ld)\n", msg, a, b); + } + ++errors; +} + +// Packed [N][W] port helpers. +template uint32_t get(T v, int i, int w) { return uint32_t((uint64_t(v) >> (i * w)) & ((1ull << w) - 1)); } +template void put(T& v, int i, int w, uint32_t x) { + uint64_t m = ((1ull << w) - 1) << (i * w); + v = T((uint64_t(v) & ~m) | ((uint64_t(x) << (i * w)) & m)); +} + +uint32_t waddr(int m, int k) { return (uint32_t(m) << 28) | (uint32_t(k) << 8); } +uint32_t wdata(int m, int k, int beat) { return waddr(m, k) | uint32_t(beat); } + +enum Style { + STYLE_FREE, // AW and W offered independently; data may lead its address + STYLE_AWFIRST // data only after its address was taken, one write at a time +}; + +struct Master { + int id; + Style style; + int pct; // offer probability per cycle + int num_writes, num_reads; + std::vector len; // per write, beats - 1 + int aw_next = 0, aw_done = 0; + bool aw_offer = false; + int w_next = 0, w_beat = 0, w_done = 0; + bool w_offer = false; + int b_recv = 0; + int ar_next = 0; + bool ar_offer = false; + std::deque> r_exp; // addr, len + int r_beat = 0, r_done = 0; + bool done() const { return b_recv == num_writes && r_done == num_reads; } +}; + +struct SlaveCfg { + int aw_pct, w_pct, ar_pct, b_pct, r_pct; + bool w_first; // AWREADY only once that write's data has begun +}; + +struct AwRec { uint32_t addr; uint32_t id; int len; }; + +bool run(std::vector& ms, const SlaveCfg& sc, int max_cycles, int* cycles_out, + int* writes_out = nullptr, int* last_write_cycle = nullptr) { + std::deque aw_q; + std::deque> w_bursts; + std::vector w_cur; + int w_started = 0, aw_taken = 0; + std::deque b_q; + std::deque> ar_q; // addr|id, beats + std::deque r_q; + int r_beat = 0; + bool b_hold = false, r_hold = false; + int writes = 0; + // offer-stability tracking on the slave side + bool pend_aw = false, pend_w = false, pend_ar = false; + uint64_t pend_aw_v = 0, pend_w_v = 0, pend_ar_v = 0; + + int cycle = 0; + for (; cycle < max_cycles; ++cycle) { + bool all_done = true; + for (auto& m : ms) { + all_done = all_done && m.done(); + } + if (all_done && aw_q.empty() && w_bursts.empty() && b_q.empty() && r_q.empty()) { + break; + } + + // ---- masters drive ---- + dut->s_awvalid = 0; dut->s_wvalid = 0; dut->s_arvalid = 0; + dut->s_wlast = 0; dut->s_bready = 0; dut->s_rready = 0; + for (int i = 0; i < NUM_MASTERS; ++i) { + auto& m = ms[i]; + if (!m.aw_offer && m.aw_next < m.num_writes) { + bool ok = (m.style == STYLE_FREE) ? (m.aw_next - m.w_done <= 3) + : (m.aw_next == m.w_done); + m.aw_offer = ok && chance(m.pct); + } + if (m.aw_offer) { + dut->s_awvalid |= (1 << i); + put(dut->s_awaddr, i, 32, waddr(i, m.aw_next)); + put(dut->s_awid, i, 8, m.id); + put(dut->s_awlen, i, 8, m.len[m.aw_next]); + } + if (!m.w_offer && m.w_next < m.num_writes) { + bool ok = (m.style == STYLE_FREE) ? (m.w_next <= m.aw_done + 1) + : (m.w_next < m.aw_done); + m.w_offer = ok && (m.w_beat > 0 || chance(m.pct)); + } + if (m.w_offer) { + dut->s_wvalid |= (1 << i); + put(dut->s_wdata, i, 32, wdata(i, m.w_next, m.w_beat)); + if (m.w_beat == m.len[m.w_next]) { + dut->s_wlast |= (1 << i); + } + } + if (!m.ar_offer && m.ar_next < m.num_reads) { + m.ar_offer = chance(m.pct); + } + if (m.ar_offer) { + dut->s_arvalid |= (1 << i); + put(dut->s_araddr, i, 32, (uint32_t(i) << 28) | (uint32_t(m.ar_next) << 8) | 0x80); + put(dut->s_arid, i, 8, m.id); + put(dut->s_arlen, i, 8, m.ar_next % 3); + } + if (chance(70)) { + dut->s_bready |= (1 << i); + } + if (chance(70)) { + dut->s_rready |= (1 << i); + } + } + + // ---- slave drives ---- + dut->m_awready = chance(sc.aw_pct) && (!sc.w_first || w_started > aw_taken); + dut->m_wready = chance(sc.w_pct); + dut->m_arready = chance(sc.ar_pct); + if (!b_hold && !b_q.empty() && chance(sc.b_pct)) { + b_hold = true; + } + dut->m_bvalid = b_hold; + dut->m_bid = b_hold ? b_q.front() : 0; + if (!r_hold && !r_q.empty() && chance(sc.r_pct)) { + r_hold = true; + } + dut->m_rvalid = r_hold; + if (r_hold) { + dut->m_rdata = r_q.front().addr | uint32_t(r_beat); + dut->m_rid = r_q.front().id; + dut->m_rlast = (r_beat == r_q.front().len); + } + + dut->clk = 0; + dut->eval(); + + // ---- offer stability, slave side ---- + uint64_t aw_v = (uint64_t(dut->m_awaddr) << 16) | (uint64_t(dut->m_awid) << 8) | dut->m_awlen; + uint64_t w_v = (uint64_t(dut->m_wdata) << 1) | dut->m_wlast; + uint64_t ar_v = (uint64_t(dut->m_araddr) << 16) | (uint64_t(dut->m_arid) << 8) | dut->m_arlen; + if (pend_aw && (!dut->m_awvalid || aw_v != pend_aw_v)) { + error("AW offer changed before it was taken", cycle); + } + if (pend_w && (!dut->m_wvalid || w_v != pend_w_v)) { + error("W offer changed before it was taken", cycle); + } + if (pend_ar && (!dut->m_arvalid || ar_v != pend_ar_v)) { + error("AR offer changed before it was taken", cycle); + } + pend_aw = dut->m_awvalid && !dut->m_awready; pend_aw_v = aw_v; + pend_w = dut->m_wvalid && !dut->m_wready; pend_w_v = w_v; + pend_ar = dut->m_arvalid && !dut->m_arready; pend_ar_v = ar_v; + + // ---- slave handshakes ---- + if (dut->m_awvalid && dut->m_awready) { + aw_q.push_back({uint32_t(dut->m_awaddr), uint32_t(dut->m_awid), int(dut->m_awlen)}); + ++aw_taken; + } + if (dut->m_wvalid && dut->m_wready) { + if (w_cur.empty()) { + ++w_started; + } + w_cur.push_back(uint32_t(dut->m_wdata)); + if (dut->m_wlast) { + w_bursts.push_back(w_cur); + w_cur.clear(); + } + } + while (!aw_q.empty() && !w_bursts.empty()) { + auto aw = aw_q.front(); auto& bu = w_bursts.front(); + if (int(bu.size()) != aw.len + 1) { + error("burst length differs from its AW", bu.size(), aw.len); + } + for (size_t b = 0; b < bu.size(); ++b) { + if (bu[b] != (aw.addr | uint32_t(b))) { + error("W data does not belong to the AW at its position", bu[b], aw.addr); + } + } + if (((aw.addr >> 28) & 1) != ((aw.id >> ID_SRC_BIT) & 1)) { + error("AW id source field wrong", aw.addr, aw.id); + } + b_q.push_back(aw.id); + aw_q.pop_front(); w_bursts.pop_front(); + ++writes; + if (last_write_cycle) { + *last_write_cycle = cycle; + } + } + if (dut->m_arvalid && dut->m_arready) { + r_q.push_back({uint32_t(dut->m_araddr), uint32_t(dut->m_arid), int(dut->m_arlen)}); + } + bool b_fire = b_hold && dut->m_bready; + bool r_fire = r_hold && dut->m_rready; + + // ---- master handshakes ---- + for (int i = 0; i < NUM_MASTERS; ++i) { + auto& m = ms[i]; + if (m.aw_offer && get(dut->s_awready, i, 1)) { + m.aw_offer = false; + ++m.aw_next; + ++m.aw_done; + } + if (m.w_offer && get(dut->s_wready, i, 1)) { + m.w_offer = false; + if (m.w_beat == m.len[m.w_next]) { + m.w_beat = 0; + ++m.w_next; + ++m.w_done; + } else { + ++m.w_beat; + } + } + if (m.ar_offer && get(dut->s_arready, i, 1)) { + m.ar_offer = false; + m.r_exp.push_back({(uint32_t(i) << 28) | (uint32_t(m.ar_next) << 8) | 0x80, m.ar_next % 3}); + ++m.ar_next; + } + if (get(dut->s_bvalid, i, 1) && get(dut->s_bready, i, 1)) { + if (get(dut->s_bid, i, 8) != uint32_t(m.id)) { + error("B to the wrong master or id", i, get(dut->s_bid, i, 8)); + } + if (m.b_recv >= m.aw_done) { + error("B before its write was issued", i, m.b_recv); + } + ++m.b_recv; + } + if (get(dut->s_rvalid, i, 1) && get(dut->s_rready, i, 1)) { + if (m.r_exp.empty()) { + error("unexpected R", i); + } else { + auto e = m.r_exp.front(); + if (get(dut->s_rid, i, 8) != uint32_t(m.id)) { + error("R to the wrong master or id", i, get(dut->s_rid, i, 8)); + } + if (get(dut->s_rdata, i, 32) != (e.first | uint32_t(m.r_beat))) { + error("R data wrong", i, get(dut->s_rdata, i, 32)); + } + bool last = (m.r_beat == e.second); + if (get(dut->s_rlast, i, 1) != uint32_t(last)) { + error("RLAST wrong", i, m.r_beat); + } + if (last) { + m.r_exp.pop_front(); + m.r_beat = 0; + ++m.r_done; + } else { + ++m.r_beat; + } + } + } + } + + dut->clk = 1; + dut->eval(); + + if (b_fire) { + b_q.pop_front(); + b_hold = false; + } + if (r_fire) { + if (r_beat == r_q.front().len) { + r_q.pop_front(); + r_beat = 0; + } else { + ++r_beat; + } + r_hold = false; + } + } + *cycles_out = cycle; + if (writes_out) { + *writes_out = writes; + } + for (auto& m : ms) { + if (!m.done()) { + return false; + } + } + return true; +} + +void do_reset() { + dut->reset = 1; + for (int i = 0; i < 4; ++i) { + dut->clk = 0; + dut->eval(); + dut->clk = 1; + dut->eval(); + } + dut->reset = 0; + dut->clk = 0; dut->eval(); +} + +Master make_master(int id, Style style, int pct, int writes, int reads, int max_len) { + Master m; + m.id = id; m.style = style; m.pct = pct; + m.num_writes = writes; m.num_reads = reads; + for (int k = 0; k < writes; ++k) { + m.len.push_back(int(rnd() % (max_len + 1))); + } + return m; +} + +} // namespace + +int main(int argc, char** argv) { + Verilated::commandArgs(argc, argv); + dut = new VVX_mm_axi_arb_top; + int cycles = 0; + + printf("1. two masters, random traffic and back-pressure\n"); + for (uint32_t seed = 1; seed <= 8; ++seed) { + rng = seed; + errors = 0; + do_reset(); + std::vector ms = { make_master(5, STYLE_FREE, 60, 400, 300, 3), + make_master(6, STYLE_AWFIRST, 60, 400, 300, 3) }; + SlaveCfg sc = {50, 50, 50, 60, 60, false}; + bool ok = run(ms, sc, 200000, &cycles); + char what[96]; + snprintf(what, sizeof(what), "seed %u: all writes and reads complete in order (%d cycles)", seed, cycles); + check(ok && errors == 0, what); + } + + printf("2. a slave that takes the data before the address\n"); + for (uint32_t seed = 11; seed <= 14; ++seed) { + rng = seed; + errors = 0; + do_reset(); + std::vector ms = { make_master(5, STYLE_FREE, 70, 400, 100, 3), + make_master(6, STYLE_FREE, 70, 400, 100, 3) }; + SlaveCfg sc = {60, 60, 60, 70, 70, true}; + bool ok = run(ms, sc, 200000, &cycles); + char what[96]; + snprintf(what, sizeof(what), "seed %u: no deadlock, data still in address order", seed); + check(ok && errors == 0, what); + } + + printf("3. single-beat writes with address and data offered together\n"); + { + rng = 21; + errors = 0; + do_reset(); + std::vector ms = { make_master(5, STYLE_FREE, 100, 1000, 0, 0), + make_master(6, STYLE_FREE, 0, 0, 0, 0) }; + SlaveCfg sc = {100, 100, 100, 100, 100, false}; + int writes = 0, last = 0; + bool ok = run(ms, sc, 20000, &cycles, &writes, &last); + char what[96]; + snprintf(what, sizeof(what), "one write per cycle (%d writes in %d cycles)", writes, last + 1); + check(ok && errors == 0 && last + 1 <= writes + 16, what); + } + + printf("\n%s (%d failure%s)\n", failures ? "FAILED" : "PASSED", + failures, failures == 1 ? "" : "s"); + delete dut; + return failures ? 1 : 0; +} diff --git a/sim/avedsim/Makefile b/sim/avedsim/Makefile index dfe8ab9bb4..5d0921b125 100644 --- a/sim/avedsim/Makefile +++ b/sim/avedsim/Makefile @@ -141,6 +141,16 @@ RTL_PKGS += $(RTL_DIR)/VX_trace_pkg.sv # Verilator from the -I path above. RTL_PKGS += $(RTL_DIR)/cp/VX_cp_pkg.sv +# One AXI memory master per GPU memory port. The platform top must declare +# its ports literally, so resolve the count from the elaborated configuration. +AFU_MEM_PORTS := $(shell $(SCRIPT_DIR)/afu_mem_ports.sh -c $(DESTDIR)/.afu_mem_ports $(VERILATOR_PATH)/bin/verilator \ + -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) \ + $(CONFIGS) $(RTL_INCLUDE) $(RTL_PKGS)) +ifeq ($(AFU_MEM_PORTS),) +$(error cannot resolve the AFU memory port count for this configuration) +endif +CXXFLAGS += -DAFU_MEM_PORTS=$(AFU_MEM_PORTS) + TOP = vortex_afu_shim VL_FLAGS += --language 1800-2012 --assert -Wall -Wpedantic @@ -149,6 +159,7 @@ VL_FLAGS += --x-initial unique --x-assign unique VL_FLAGS += verilator.vlt VL_FLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) VL_FLAGS += $(CONFIGS) +VL_FLAGS += -DAFU_MEM_PORTS=$(AFU_MEM_PORTS) VL_FLAGS += $(RTL_INCLUDE) VL_FLAGS += $(RTL_PKGS) diff --git a/sim/avedsim/vortex_afu_shim.sv b/sim/avedsim/vortex_afu_shim.sv index ecad4997bc..9600a70718 100644 --- a/sim/avedsim/vortex_afu_shim.sv +++ b/sim/avedsim/vortex_afu_shim.sv @@ -20,14 +20,14 @@ module vortex_afu_shim #( parameter C_M_AXI_MEM_ID_WIDTH = `PLATFORM_MEMORY_ID_WIDTH, parameter C_M_AXI_MEM_DATA_WIDTH = (`VX_CFG_PLATFORM_MEMORY_DATA_SIZE * 8), parameter C_M_AXI_MEM_ADDR_WIDTH = 64, - parameter C_M_AXI_MEM_NUM_BANKS = `VX_CFG_PLATFORM_MEMORY_NUM_BANKS + parameter C_M_AXI_MEM_NUM_BANKS = `AFU_MEM_PORTS ) ( // System signals input wire ap_clk, input wire ap_rst_n, // AXI4 master interface - `MP_REPEAT (`VX_CFG_PLATFORM_MEMORY_NUM_BANKS, GEN_AXI_MEM, MP_COMMA), + `MP_REPEAT (`AFU_MEM_PORTS, GEN_AXI_MEM, MP_COMMA), // AXI4 host-memory master interface (CP command ring + host side of DMA) `GEN_AXI_HOST, @@ -65,7 +65,7 @@ module vortex_afu_shim #( .ap_clk (ap_clk), .ap_rst_n (ap_rst_n), - `MP_REPEAT (`VX_CFG_PLATFORM_MEMORY_NUM_BANKS, AXI_MEM_ARGS, MP_COMMA), + `MP_REPEAT (`AFU_MEM_PORTS, AXI_MEM_ARGS, MP_COMMA), `AXI_HOST_ARGS, diff --git a/sim/xrtsim/Makefile b/sim/xrtsim/Makefile index 4e51883d19..408ce494cd 100644 --- a/sim/xrtsim/Makefile +++ b/sim/xrtsim/Makefile @@ -138,6 +138,17 @@ RTL_PKGS += $(RTL_DIR)/VX_trace_pkg.sv # Verilator from the -I path above. RTL_PKGS += $(RTL_DIR)/cp/VX_cp_pkg.sv +# One AXI memory master per GPU memory port. The platform top must declare +# its ports literally, so resolve the count from the elaborated configuration. +AFU_MEM_PORTS := $(shell $(SCRIPT_DIR)/afu_mem_ports.sh -c $(DESTDIR)/.afu_mem_ports $(VERILATOR_PATH)/bin/verilator \ + -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) \ + $(CONFIGS) $(filter -DVX_CFG_TCU_TYPE_% -DVX_CFG_FPU_TYPE_%,$(XCONFIGS)) \ + $(RTL_INCLUDE) $(RTL_PKGS)) +ifeq ($(AFU_MEM_PORTS),) +$(error cannot resolve the AFU memory port count for this configuration) +endif +CXXFLAGS += -DAFU_MEM_PORTS=$(AFU_MEM_PORTS) + TOP = vortex_afu_shim VL_FLAGS += --language 1800-2012 --assert -Wall -Wpedantic @@ -152,6 +163,7 @@ VL_FLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) # VX_CFG_TCU_TYPE_DPI) unmatched. Add just the enum companions gen_config # resolves, so a CONFIGS enum override actually selects its RTL implementation. VL_FLAGS += $(CONFIGS) $(filter -DVX_CFG_TCU_TYPE_% -DVX_CFG_FPU_TYPE_%,$(XCONFIGS)) +VL_FLAGS += -DAFU_MEM_PORTS=$(AFU_MEM_PORTS) VL_FLAGS += $(RTL_INCLUDE) VL_FLAGS += $(RTL_PKGS) diff --git a/sim/xrtsim/vortex_afu_shim.sv b/sim/xrtsim/vortex_afu_shim.sv index 59ed67bba7..73a8b8f071 100644 --- a/sim/xrtsim/vortex_afu_shim.sv +++ b/sim/xrtsim/vortex_afu_shim.sv @@ -20,14 +20,14 @@ module vortex_afu_shim #( parameter C_M_AXI_MEM_ID_WIDTH = `PLATFORM_MEMORY_ID_WIDTH, parameter C_M_AXI_MEM_DATA_WIDTH = (`VX_CFG_PLATFORM_MEMORY_DATA_SIZE * 8), parameter C_M_AXI_MEM_ADDR_WIDTH = 64, - parameter C_M_AXI_MEM_NUM_BANKS = `VX_CFG_PLATFORM_MEMORY_NUM_BANKS + parameter C_M_AXI_MEM_NUM_BANKS = `AFU_MEM_PORTS ) ( // System signals input wire ap_clk, input wire ap_rst_n, // AXI4 master interface - `MP_REPEAT (`VX_CFG_PLATFORM_MEMORY_NUM_BANKS, GEN_AXI_MEM, MP_COMMA), + `MP_REPEAT (`AFU_MEM_PORTS, GEN_AXI_MEM, MP_COMMA), // AXI4 host-memory master interface (CP command ring + host side of DMA) `GEN_AXI_HOST, @@ -54,6 +54,9 @@ module vortex_afu_shim #( output wire interrupt `IGNORE_WARNINGS_END ); + `AXI_MEM_ARRAYS (C_M_AXI_MEM_NUM_BANKS) + `MP_REPEAT (`AFU_MEM_PORTS, AXI_MEM_TO_ARRAY, MP_SEMI); + VX_afu_wrap #( .C_S_AXI_CTRL_ADDR_WIDTH (C_S_AXI_CTRL_ADDR_WIDTH), .C_S_AXI_CTRL_DATA_WIDTH (C_S_AXI_CTRL_DATA_WIDTH), @@ -65,7 +68,7 @@ module vortex_afu_shim #( .clk (ap_clk), .reset (~ap_rst_n), - `MP_REPEAT (`VX_CFG_PLATFORM_MEMORY_NUM_BANKS, AXI_MEM_ARGS, MP_COMMA), + `AXI_MEM_ARRAY_ARGS, `AXI_HOST_ARGS, diff --git a/sim/xrtsim/xrt_sim.cpp b/sim/xrtsim/xrt_sim.cpp index af948d92dc..d1d0e349fc 100644 --- a/sim/xrtsim/xrt_sim.cpp +++ b/sim/xrtsim/xrt_sim.cpp @@ -161,7 +161,7 @@ class xrt_sim::Impl { "returned ahead of an older ready one\n", rsp_reorder_seed(), (unsigned long)rsp_reordered_); } - for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + for (int b = 0; b < AFU_MEM_PORTS; ++b) { delete mem_alloc_[b]; } for (auto ram : ram_) { @@ -217,21 +217,21 @@ class xrt_sim::Impl { sfp_->open(saif_file ? saif_file : "trace.saif"); #endif - // calculate memory bank size - mem_bank_size_ = (1ull << VX_CFG_PLATFORM_MEMORY_ADDR_WIDTH) / VX_CFG_PLATFORM_MEMORY_NUM_BANKS; + // Each AXI port owns an equal, contiguous share of device memory. + mem_bank_size_ = (1ull << VX_CFG_PLATFORM_MEMORY_ADDR_WIDTH) / AFU_MEM_PORTS; // One memory per AXI port, as on the platform, where each kernel master // is wired to its own memory resource. A request issued on the wrong port // then misses its data instead of finding it in a shared image. - for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + for (int b = 0; b < AFU_MEM_PORTS; ++b) { ram_[b] = new RAM(0, RAM_PAGE_SIZE); } // initialize AXI memory interfaces - MP_M_AXI_MEM(VX_CFG_PLATFORM_MEMORY_NUM_BANKS); + MP_M_AXI_MEM(AFU_MEM_PORTS); // initialize memory allocator - for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + for (int b = 0; b < AFU_MEM_PORTS; ++b) { mem_alloc_[b] = new MemoryAllocator(0, mem_bank_size_, 4096, 64); } @@ -295,13 +295,13 @@ class xrt_sim::Impl { } int mem_alloc(uint64_t size, uint32_t bank_id, uint64_t* addr) { - if (bank_id >= VX_CFG_PLATFORM_MEMORY_NUM_BANKS) + if (bank_id >= AFU_MEM_PORTS) return -1; return mem_alloc_[bank_id]->allocate(size, addr); } int mem_free(uint32_t bank_id, uint64_t addr) { - if (bank_id >= VX_CFG_PLATFORM_MEMORY_NUM_BANKS) + if (bank_id >= AFU_MEM_PORTS) return -1; return mem_alloc_[bank_id]->release(addr); } @@ -309,7 +309,7 @@ class xrt_sim::Impl { int mem_write(uint32_t bank_id, uint64_t addr, uint64_t size, const void* data) { HostLock guard(*this); - if (bank_id >= VX_CFG_PLATFORM_MEMORY_NUM_BANKS) + if (bank_id >= AFU_MEM_PORTS) return -1; uint64_t base_addr = bank_id * mem_bank_size_ + addr; ram_[bank_id]->write(data, base_addr, size); @@ -319,7 +319,7 @@ class xrt_sim::Impl { int mem_read(uint32_t bank_id, uint64_t addr, uint64_t size, void* data) { HostLock guard(*this); - if (bank_id >= VX_CFG_PLATFORM_MEMORY_NUM_BANKS) + if (bank_id >= AFU_MEM_PORTS) return -1; uint64_t base_addr = bank_id * mem_bank_size_ + addr; ram_[bank_id]->read(data, base_addr, size); @@ -328,7 +328,7 @@ class xrt_sim::Impl { int mem_copy(uint32_t bank_id_dest , uint32_t bank_id_src, uint64_t dest_addr, uint64_t src_addr, uint64_t size) { HostLock guard(*this); - if( bank_id_dest >= VX_CFG_PLATFORM_MEMORY_NUM_BANKS || bank_id_src >= VX_CFG_PLATFORM_MEMORY_NUM_BANKS) + if( bank_id_dest >= AFU_MEM_PORTS || bank_id_src >= AFU_MEM_PORTS) return -1; uint64_t dest_base_addr = bank_id_dest * mem_bank_size_ + dest_addr; uint64_t src_base_addr = bank_id_src * mem_bank_size_ + src_addr; @@ -480,10 +480,12 @@ class xrt_sim::Impl { } for (auto& st : m_axi_states_) { + st.aw_pending = {}; st.w_staged.clear(); + st.w_bursts = {}; } - for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + for (int b = 0; b < AFU_MEM_PORTS; ++b) { std::queue empty; std::swap(dram_queues_[b], empty); } @@ -500,7 +502,7 @@ class xrt_sim::Impl { device_->ap_rst_n = 1; // this AXI device is always ready to accept new requests - for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + for (int b = 0; b < AFU_MEM_PORTS; ++b) { *m_axi_mem_[b].arready = 1; *m_axi_mem_[b].awready = 1; *m_axi_mem_[b].wready = 1; @@ -532,10 +534,11 @@ class xrt_sim::Impl { dram_sim_.tick(); - for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { - if (!dram_queues_[b].empty() && dram_sim_.can_accept(b)) { + for (int b = 0; b < AFU_MEM_PORTS; ++b) { + if (!dram_queues_[b].empty() + && dram_sim_.can_accept(dram_channel(dram_queues_[b].front()->addr))) { auto mem_req = dram_queues_[b].front(); - dram_sim_.send_request(b, bank_local_addr(mem_req->addr), mem_req->write, [](void* arg)->bool { + dram_sim_.send_request(dram_channel(mem_req->addr), dram_channel_addr(mem_req->addr), mem_req->write, [](void* arg)->bool { auto orig_req = reinterpret_cast(arg); if (orig_req->ready) { delete orig_req; @@ -553,17 +556,33 @@ class xrt_sim::Impl { #endif } - // Byte address within its bank (DRAM channel) of an AXI bank address: - // VX_mem_to_axi places the bank select above the data-size bits when the - // banks are interleaved. - static uint64_t bank_local_addr(uint64_t axi_addr) { - #if VX_CFG_PLATFORM_MEMORY_INTERLEAVE == 1 - constexpr uint32_t lg2_data_size = log2ceil(VX_CFG_PLATFORM_MEMORY_DATA_SIZE); - constexpr uint32_t bank_sel_bits = log2ceil(VX_CFG_PLATFORM_MEMORY_NUM_BANKS); - return (axi_addr >> (bank_sel_bits + lg2_data_size)) << lg2_data_size; - #else - return axi_addr; - #endif + // Port addresses are physical: the AFU gives each port its own contiguous + // share of device memory, and the platform lays its memory channels out + // linearly across that space, so a port's traffic stays on its own group of + // channels. + static constexpr uint64_t DRAM_CHANNEL_SIZE = + (1ull << VX_CFG_PLATFORM_MEMORY_ADDR_WIDTH) / VX_CFG_PLATFORM_MEMORY_NUM_BANKS; + + static uint32_t dram_channel(uint64_t axi_addr) { + return uint32_t((axi_addr / DRAM_CHANNEL_SIZE) % VX_CFG_PLATFORM_MEMORY_NUM_BANKS); + } + + static uint64_t dram_channel_addr(uint64_t axi_addr) { + return axi_addr % DRAM_CHANNEL_SIZE; + } + + // A port bound to its own channel group cannot reach outside it on the + // platform; a request that would is a routing bug, not a slow access. + void check_port_range(int port, const char* what, uint64_t addr, uint32_t len) { + uint64_t last = addr + uint64_t(len) * VX_CFG_PLATFORM_MEMORY_DATA_SIZE; + uint64_t lo = uint64_t(port) * mem_bank_size_; + uint64_t hi = lo + mem_bank_size_; + if (addr < lo || last >= hi) { + fprintf(stderr, "[XRTSIM] Error: %s at 0x%lx on m_axi_mem_%d, outside its range " + "[0x%lx, 0x%lx)\n", what, (unsigned long)addr, port, + (unsigned long)lo, (unsigned long)hi); + std::abort(); + } } void eval() { @@ -603,7 +622,7 @@ class xrt_sim::Impl { } void axi_mem_bus_reset() { - for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + for (int b = 0; b < AFU_MEM_PORTS; ++b) { // read request address *m_axi_mem_[b].arready = 0; @@ -619,24 +638,24 @@ class xrt_sim::Impl { // write response *m_axi_mem_[b].bvalid = 0; - // states - m_axi_states_[b].aw_active = false; } } void axi_host_bus_reset() { // Ready signals stay constant-high (like the m_axi_mem model): a slave // that toggles *ready races the master, which only samples it the cycle - // after it enters its issue state. Our masters keep one outstanding - // read/write at a time, so always-ready is correct. + // after it enters its issue state. Reads and both write channels queue + // what they accept, so always-ready is correct. *m_axi_host_.arready = 1; *m_axi_host_.awready = 1; *m_axi_host_.wready = 1; *m_axi_host_.rvalid = 0; *m_axi_host_.bvalid = 0; host_rd_active_ = false; - host_wr_active_ = false; - host_wr_b_pending_ = false; + host_aw_pending_ = {}; + host_w_staged_.clear(); + host_w_bursts_ = {}; + host_b_pending_ = {}; host_rd_rsp_ready_ = false; host_wr_rsp_ready_ = false; std::queue().swap(host_rd_pending_); @@ -675,7 +694,7 @@ class xrt_sim::Impl { // B channel — retire the write response. if (*m_axi_host_.bvalid && host_wr_rsp_ready_) { *m_axi_host_.bvalid = 0; - host_wr_b_pending_ = false; + host_b_pending_.pop(); } // Accept a read burst. arready stays high (so the master's handshake // always completes the cycle it issues — no ready-toggle race); a request @@ -706,49 +725,57 @@ class xrt_sim::Impl { *m_axi_host_.rresp = 0; *m_axi_host_.rlast = (host_rd_beat_ >= host_rd_len_); } - // Accept a write burst (one outstanding at a time). - if (*m_axi_host_.awvalid && *m_axi_host_.awready - && !host_wr_active_ && !host_wr_b_pending_) { - host_wr_active_ = true; - host_wr_addr_ = uint64_t(*m_axi_host_.awaddr); - host_wr_id_ = *m_axi_host_.awid; - host_wr_beat_ = 0; + // Write channels: the k-th AW pairs with the k-th W burst, and either may + // arrive first (awready/wready stay high), so both sides queue. A write + // lands in host memory once its pair is complete, in AW order. + if (*m_axi_host_.awvalid && *m_axi_host_.awready) { + host_aw_pending_.push({uint64_t(*m_axi_host_.awaddr), uint32_t(*m_axi_host_.awid)}); } - // Accept a write data beat. - if (host_wr_active_ && *m_axi_host_.wvalid && *m_axi_host_.wready) { - uint64_t a = host_wr_addr_ - + uint64_t(host_wr_beat_) * VX_CFG_PLATFORM_MEMORY_DATA_SIZE; - auto byteen = *m_axi_host_.wstrb; - auto data = (const uint8_t*)m_axi_host_.wdata->data(); - for (int i = 0; i < VX_CFG_PLATFORM_MEMORY_DATA_SIZE; ++i) { - if ((byteen >> i) & 0x1) - reinterpret_cast(a)[i] = data[i]; - } + if (*m_axi_host_.wvalid && *m_axi_host_.wready) { + w_beat_t beat; + beat.addr = 0; + beat.byteen = uint64_t(*m_axi_host_.wstrb); + std::memcpy(beat.data.data(), m_axi_host_.wdata->data(), + VX_CFG_PLATFORM_MEMORY_DATA_SIZE); + host_w_staged_.emplace_back(beat); if (*m_axi_host_.wlast) { - host_wr_active_ = false; - host_wr_b_pending_ = true; - } else { - ++host_wr_beat_; + host_w_bursts_.push(std::move(host_w_staged_)); + host_w_staged_.clear(); + } + } + while (!host_aw_pending_.empty() && !host_w_bursts_.empty()) { + auto aw = host_aw_pending_.front(); + auto& beats = host_w_bursts_.front(); + for (size_t k = 0; k < beats.size(); ++k) { + // Host memory is plain process memory (see the read path above). + auto dst = reinterpret_cast(aw.addr + k * VX_CFG_PLATFORM_MEMORY_DATA_SIZE); + for (int i = 0; i < VX_CFG_PLATFORM_MEMORY_DATA_SIZE; ++i) { + if ((beats[k].byteen >> i) & 0x1) + dst[i] = beats[k].data[i]; + } } + host_b_pending_.push(aw.tag); + host_aw_pending_.pop(); + host_w_bursts_.pop(); } // Present the write response. - if (host_wr_b_pending_ && !*m_axi_host_.bvalid) { + if (!host_b_pending_.empty() && !*m_axi_host_.bvalid) { *m_axi_host_.bvalid = 1; - *m_axi_host_.bid = host_wr_id_; + *m_axi_host_.bid = host_b_pending_.front(); *m_axi_host_.bresp = 0; } } void axi_mem_bus_eval(bool clk) { if (!clk) { - for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + for (int b = 0; b < AFU_MEM_PORTS; ++b) { m_axi_states_[b].read_rsp_ready = *m_axi_mem_[b].rready; m_axi_states_[b].write_rsp_ready = *m_axi_mem_[b].bready; } return; } - for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + for (int b = 0; b < AFU_MEM_PORTS; ++b) { // handle read responses if (*m_axi_mem_[b].rvalid && m_axi_states_[b].read_rsp_ready) { *m_axi_mem_[b].rvalid = 0; @@ -810,6 +837,7 @@ class xrt_sim::Impl { if (*m_axi_mem_[b].arvalid && *m_axi_mem_[b].arready) { uint32_t len = *m_axi_mem_[b].arlen; // beats - 1 uint64_t base = uint64_t(*m_axi_mem_[b].araddr); + check_port_range(b, "read", base, len); for (uint32_t beat = 0; beat <= len; ++beat) { auto mem_req = new mem_req_t(); mem_req->tag = *m_axi_mem_[b].arid; @@ -823,48 +851,51 @@ class xrt_sim::Impl { } } - // handle write address — latch the burst. - if (*m_axi_mem_[b].awvalid && *m_axi_mem_[b].awready - && !m_axi_states_[b].aw_active) { - m_axi_states_[b].aw_active = true; - m_axi_states_[b].aw_addr = uint64_t(*m_axi_mem_[b].awaddr); - m_axi_states_[b].aw_tag = *m_axi_mem_[b].awid; - m_axi_states_[b].aw_beat = 0; + // Write channels. AXI4 lets AW run ahead of its data and data run ahead + // of its address; the k-th AW pairs with the k-th W burst, so both sides + // queue and a write issues once its pair is complete. + auto& st = m_axi_states_[b]; + if (*m_axi_mem_[b].awvalid && *m_axi_mem_[b].awready) { + check_port_range(b, "write", uint64_t(*m_axi_mem_[b].awaddr), *m_axi_mem_[b].awlen); + st.aw_pending.push({uint64_t(*m_axi_mem_[b].awaddr), uint32_t(*m_axi_mem_[b].awid)}); } - // handle write data beats — write one cache line per W beat; the - // last beat (WLAST) queues a single B response for the burst. - if (m_axi_states_[b].aw_active - && *m_axi_mem_[b].wvalid && *m_axi_mem_[b].wready) { - uint64_t byte_addr = m_axi_states_[b].aw_addr - + uint64_t(m_axi_states_[b].aw_beat) - * VX_CFG_PLATFORM_MEMORY_DATA_SIZE; - // The beat is accepted but NOT yet visible in memory: AXI makes a - // write observable only once its B response is returned, and a read to - // the same address issued before that may legally return either value. - // Holding the payload here is what lets a master's own store->fill - // ordering bug appear in simulation instead of only on hardware. + // The beat is accepted but NOT yet visible in memory: AXI makes a write + // observable only once its B response is returned, and a read to the + // same address issued before that may legally return either value. + // Holding the payload here is what lets a master's own store->fill + // ordering bug appear in simulation instead of only on hardware. + if (*m_axi_mem_[b].wvalid && *m_axi_mem_[b].wready) { w_beat_t beat; - beat.addr = byte_addr; + beat.addr = 0; beat.byteen = uint64_t(*m_axi_mem_[b].wstrb); std::memcpy(beat.data.data(), m_axi_mem_[b].wdata->data(), VX_CFG_PLATFORM_MEMORY_DATA_SIZE); - m_axi_states_[b].w_staged.emplace_back(beat); + st.w_staged.emplace_back(beat); if (*m_axi_mem_[b].wlast) { - auto mem_req = new mem_req_t(); - mem_req->tag = m_axi_states_[b].aw_tag; - mem_req->addr = m_axi_states_[b].aw_addr; - mem_req->write = true; - mem_req->ready = false; - mem_req->last = true; - mem_req->w_beats = std::move(m_axi_states_[b].w_staged); - m_axi_states_[b].w_staged.clear(); - pending_mem_reqs_[b].emplace_back(mem_req); - dram_queues_[b].push(mem_req); - m_axi_states_[b].aw_active = false; - } else { - ++m_axi_states_[b].aw_beat; + st.w_bursts.push(std::move(st.w_staged)); + st.w_staged.clear(); + } + } + + // One cache line per W beat (INCR addressing); the burst queues a single + // B response. + while (!st.aw_pending.empty() && !st.w_bursts.empty()) { + auto aw = st.aw_pending.front(); + auto mem_req = new mem_req_t(); + mem_req->tag = aw.tag; + mem_req->addr = aw.addr; + mem_req->write = true; + mem_req->ready = false; + mem_req->last = true; + mem_req->w_beats = std::move(st.w_bursts.front()); + for (size_t i = 0; i < mem_req->w_beats.size(); ++i) { + mem_req->w_beats[i].addr = aw.addr + i * VX_CFG_PLATFORM_MEMORY_DATA_SIZE; } + st.aw_pending.pop(); + st.w_bursts.pop(); + pending_mem_reqs_[b].emplace_back(mem_req); + dram_queues_[b].push(mem_req); } } } @@ -876,15 +907,17 @@ class xrt_sim::Impl { uint64_t byteen; } w_beat_t; + typedef struct { + uint64_t addr; + uint32_t tag; + } aw_req_t; + typedef struct { bool read_rsp_ready; bool write_rsp_ready; - // Write-burst state — latched on AW, advanced one cache line per W beat. - bool aw_active; - uint64_t aw_addr; - uint32_t aw_beat; - uint32_t aw_tag; - std::vector w_staged; // beats accepted so far in this burst + std::queue aw_pending; // accepted AWs awaiting their data + std::vector w_staged; // beats of the burst in progress + std::queue> w_bursts; // whole bursts awaiting their AW } m_axi_state_t; typedef struct { @@ -999,7 +1032,7 @@ class xrt_sim::Impl { } m_axi_mem_t; Vvortex_afu_shim* device_; - RAM* ram_[VX_CFG_PLATFORM_MEMORY_NUM_BANKS]; + RAM* ram_[AFU_MEM_PORTS]; RAM* host_ram_; MemoryAllocator* host_alloc_; DramSim dram_sim_; @@ -1028,17 +1061,17 @@ class xrt_sim::Impl { } }; - std::list pending_mem_reqs_[VX_CFG_PLATFORM_MEMORY_NUM_BANKS]; + std::list pending_mem_reqs_[AFU_MEM_PORTS]; uint32_t rsp_rng_ = rsp_reorder_seed(); uint64_t rsp_reordered_ = 0; - m_axi_mem_t m_axi_mem_[VX_CFG_PLATFORM_MEMORY_NUM_BANKS]; + m_axi_mem_t m_axi_mem_[AFU_MEM_PORTS]; - MemoryAllocator* mem_alloc_[VX_CFG_PLATFORM_MEMORY_NUM_BANKS]; + MemoryAllocator* mem_alloc_[AFU_MEM_PORTS]; - m_axi_state_t m_axi_states_[VX_CFG_PLATFORM_MEMORY_NUM_BANKS]; + m_axi_state_t m_axi_states_[AFU_MEM_PORTS]; - std::queue dram_queues_[VX_CFG_PLATFORM_MEMORY_NUM_BANKS]; + std::queue dram_queues_[AFU_MEM_PORTS]; // m_axi_host AXI slave port + burst state. m_axi_mem_t m_axi_host_; @@ -1048,11 +1081,10 @@ class xrt_sim::Impl { uint32_t host_rd_beat_; uint32_t host_rd_id_; bool host_rd_rsp_ready_; - bool host_wr_active_; - uint64_t host_wr_addr_; - uint32_t host_wr_beat_; - uint32_t host_wr_id_; - bool host_wr_b_pending_; + std::queue host_aw_pending_; + std::vector host_w_staged_; + std::queue> host_w_bursts_; + std::queue host_b_pending_; bool host_wr_rsp_ready_; // Reads accepted while one is already in flight. Several CP masters From 78f2d2ccd7e036771699e10a1069ab9aea1097a2 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 01:27:46 -0700 Subject: [PATCH 098/124] simx: model the miss path's buffers and registered stages SimX ran the two-core L2 sgemm shape and the multi-slice fetch shapes several percent off the RTL because it folded buffers and registers on the miss path into bare channel delays. Each is now modeled as the RTL builds it: - The first dcache and icache port also carries the flush injection, whose arbiter buffers two requests and registers them. A registered two-entry TxBuffer stands in for it, replacing the per-port delay knobs on the coalescer and the lane adapter. - A memory response reaches its bank through the port's response queue, when there is one, and a response network registered at every level. Caches now say whether they have that queue (mem_rsp_queue) or a registered boundary that stands for it (mem_rsp_slice), and the flush taps carry what the bank arbiter's response delay does not. - The socket's L1 memory arbiter is a priority arbiter, icache first, registered in both directions. - The LSU's result crosses one register into the lane gather, so its commit queue holds two entries, and its commit-side elasticity is the response buffer plus the result arbiter (four entries, not six). - With two or more operand collectors, their round-robin merge registers its output. - The lmem switch's dcache response buffer holds one entry when the coalescer or a lane adapter feeds it combinationally. - A private dcache's arbiter can keep port 0 out of read merging (merge_port0), which is how the RTL tagged that port. CI: core:parity-sgemm-fw4 (four fetch slices on a four-port icache); parity-sgemm-fw2 tightens to 3%. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/core.yaml | 11 ++++++++ sim/simx/cluster.cpp | 1 + sim/simx/core.cpp | 44 +++++++++++++++++++---------- sim/simx/lsu_unit.cpp | 6 ++-- sim/simx/mem/cache.cpp | 46 ++++++++++++++++++++----------- sim/simx/mem/cache.h | 3 ++ sim/simx/mem/local_mem_switch.cpp | 5 ++-- sim/simx/mem/local_mem_switch.h | 3 +- sim/simx/mem/lsu_mem_adapter.cpp | 3 +- sim/simx/mem/lsu_mem_adapter.h | 4 --- sim/simx/mem/mem_coalescer.cpp | 3 +- sim/simx/mem/mem_coalescer.h | 4 --- sim/simx/operands.cpp | 3 +- sim/simx/processor.cpp | 1 + sim/simx/socket.cpp | 28 +++++++++++++++---- sim/simx/types.h | 33 ++++++++++++++++++++++ 16 files changed, 143 insertions(+), 55 deletions(-) diff --git a/ci/testcases/core.yaml b/ci/testcases/core.yaml index 1ccfd9dcd7..f449e79c87 100644 --- a/ci/testcases/core.yaml +++ b/ci/testcases/core.yaml @@ -180,10 +180,21 @@ tests: # two-port icache, CTAs as wide as the core, a 16-port dcache. - id: parity-sgemm-fw2 check: model_parity + # Tighter than the default: icache port 0, which carries the flush + # injection, holds four requests, and the shape exercises all of them. + tolerance: 0.03 via: blackbox app: sgemm args: -n128 configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_FETCH_WIDTH=2 +# Four frontend slices on a four-port icache: every slice but the first fetches +# through two request entries, the first through four. +- id: parity-sgemm-fw4 + check: model_parity + via: blackbox + app: sgemm + args: -n128 + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_FETCH_WIDTH=4 # Device printf: per-character IO ring stores, each published behind a fence # that drains, flushes and invalidates the data cache. - id: parity-printf diff --git a/sim/simx/cluster.cpp b/sim/simx/cluster.cpp index a2c83774b9..0f409b8f0e 100644 --- a/sim/simx/cluster.cpp +++ b/sim/simx/cluster.cpp @@ -98,6 +98,7 @@ class Cluster::Impl { 0, // read_merge VX_CFG_L2_MSHR_LINES, // mshr_lines CP_ON_L2 != 0, // write_around + VX_CFG_L2_MRSQ_SIZE != 0, // memory-response queue }); diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index bfc91db066..b926fd3070 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -157,10 +157,14 @@ class Core::Impl { VX_CFG_EXT_DXA_ENABLED + VX_CFG_TCU_WGMMA_ENABLED }); - // create lmem switch + // create lmem switch. A data-cache response reaches its response + // arbiter combinationally through the coalescer or the lane adapter, so + // the switch holds it in one entry, the least a channel holds; without + // either, the channel is the cache's own two-entry output buffer. + const bool dc_rsp_comb = dcache_coalescing || (DCACHE_CHANNELS > 1); for (uint32_t b = 0; b < VX_CFG_NUM_LSU_BLOCKS; ++b) { snprintf(sname, 100, "%s-lmem_switch%d", name.c_str(), b); - lmem_switch_.at(b) = LocalMemSwitch::Create(sname, 1); + lmem_switch_.at(b) = LocalMemSwitch::Create(sname, 1, dc_rsp_comb ? 1 : 2); } // create per-block lmem adapters @@ -188,8 +192,6 @@ class Core::Impl { std::vector*> dc_req_out(VX_CFG_NUM_LSU_BLOCKS * DCACHE_CHANNELS); std::vector*> dc_rsp_in(VX_CFG_NUM_LSU_BLOCKS * DCACHE_CHANNELS); - // The first data-cache port also carries the cache-flush injection, which - // registers its requests once more. if (dcache_coalescing) { // connect memory coalescer; its memory side drives the dcache // channels directly (combinational lane fan-out/fan-in). @@ -201,7 +203,6 @@ class Core::Impl { dc_rsp_in.at(b * DCACHE_CHANNELS + c) = &mem_coalescers_.at(b)->RspIn.at(c); } } - mem_coalescers_.at(0)->set_port_delay(0, 1); } else { // bypass memory coalescer: per-block lane adapter (channel-fused // pass-through when DCACHE_CHANNELS == 1) @@ -213,9 +214,19 @@ class Core::Impl { dc_rsp_in.at(b * DCACHE_CHANNELS + c) = &lsu_dcache_adapter.at(b)->RspIn.at(c); } } - lsu_dcache_adapter.at(0)->set_port_delay(0, 1); } + // The first data-cache port also carries the cache-flush injection, whose + // arbiter buffers and registers its requests once more. + snprintf(sname, 100, "%s-dcache_flush_buf", name.c_str()); + dcache_flush_buf_ = TxBuffer::Create(sname, 2); + dc_req_out.at(0)->bind(&dcache_flush_buf_->In); + dc_req_out.at(0) = &dcache_flush_buf_->Out; + + // The first icache port does the same for the icache. + snprintf(sname, 100, "%s-icache_flush_buf", name.c_str()); + icache_flush_buf_ = TxBuffer::Create(sname, 2); + #ifdef VX_CFG_VM_ENABLE // Per-core L1 TLB stages: hits translate in place, misses park in // the miss station and resolve through the shared cluster TLB/PTW @@ -247,12 +258,14 @@ class Core::Impl { icache_mmu_->ReqOut.at(f).bind(&simobject_->icache_req_out.at(f)); simobject_->icache_rsp_in.at(f).bind(&icache_mmu_->RspIn.at(f)); } + icache_flush_buf_->Out.bind(&icache_mmu_->ReqIn.at(0)); #else // No-VM: direct passthrough. for (uint32_t p = 0; p < VX_CFG_NUM_LSU_BLOCKS * DCACHE_CHANNELS; ++p) { dc_req_out.at(p)->bind(&simobject_->dcache_req_out.at(p)); simobject_->dcache_rsp_in.at(p).bind(dc_rsp_in.at(p)); } + icache_flush_buf_->Out.bind(&simobject_->icache_req_out.at(0)); #endif // initialize execute units @@ -326,9 +339,11 @@ class Core::Impl { for (uint32_t iw = 0; iw < VX_CFG_ISSUE_WIDTH; ++iw) { auto& queues = commit_queues_.emplace_back(); for (uint32_t fu = 0; fu < (uint32_t)FUType::Count; ++fu) { - // Covers the results still crossing into the queue, so a unit can - // retire one result per cycle. - queues.emplace_back(std::make_unique>(simobject_, 3)); + // A unit's lane gather holds two results. The LSU's crosses one + // register into it; the others cross two and take a third slot so + // that they still retire one result per cycle. + uint32_t capacity = (fu == (uint32_t)FUType::LSU) ? 2 : 3; + queues.emplace_back(std::make_unique>(simobject_, capacity)); } } @@ -456,11 +471,12 @@ class Core::Impl { // The MMU translates and forwards to simobject_->icache_req_out // downstream (driven by SimChannel binding in the constructor). auto& icache_rsp = icache_mmu_->RspOut.at(f); - auto& icache_req = icache_mmu_->ReqIn.at(f); + auto& icache_port = icache_mmu_->ReqIn.at(f); #else auto& icache_rsp = simobject_->icache_rsp_in.at(f); - auto& icache_req = simobject_->icache_req_out.at(f); + auto& icache_port = simobject_->icache_req_out.at(f); #endif + auto& icache_req = (f == 0) ? icache_flush_buf_->In : icache_port; // handle icache response — extract instruction word from the TLM line payload if (!icache_rsp.empty()){ @@ -532,9 +548,7 @@ class Core::Impl { mem_req.tag = tag; mem_req.hart_id = trace->cid; mem_req.uuid = trace->uuid; - // The first icache port also carries the cache-flush injection, which - // registers its requests once more. - if (icache_req.try_send(mem_req, (f == 0) ? 2 : 1)) { + if (icache_req.try_send(mem_req, 1)) { DT(3, simobject_->name() << " icache-req: addr=0x" << std::hex << mem_req.addr << ", tag=0x" << mem_req.tag << std::dec << ", " << *trace); #ifdef VX_CFG_EXT_C_ENABLE decompressor->commit_request(from_refetch); @@ -1192,6 +1206,8 @@ class Core::Impl { LocalMem::Ptr local_mem_; std::vector lmem_switch_; std::vector mem_coalescers_; + TxBuffer::Ptr dcache_flush_buf_; + TxBuffer::Ptr icache_flush_buf_; std::vector>> fetch_latch_; std::vector>> decode_latch_; diff --git a/sim/simx/lsu_unit.cpp b/sim/simx/lsu_unit.cpp index eb0b032917..7a887721f7 100644 --- a/sim/simx/lsu_unit.cpp +++ b/sim/simx/lsu_unit.cpp @@ -73,10 +73,8 @@ Instr::Ptr LsuUopGen::get(const Instr& macro_instr, uint32_t uop_index) { } LsuUnit::LsuUnit(const SimContext& ctx, const char* name, Core* core) - // Commit-side elasticity of 6 beats: the writeback path buffers a whole - // coalesced fragment burst, so a denied commit grant queues beats there - // instead of throttling the memory response stream. - : FuncUnit(ctx, name, core, 6) + // the response buffer and the result arbiter, two entries each + : FuncUnit(ctx, name, core, 4) #ifdef TCU_META_ENABLE , TcuReqIn(TCU_AGU_PORTS, this) , TcuRspOut(TCU_AGU_PORTS, this) diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index 67a8e88131..ebeabee7ef 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -2262,7 +2262,7 @@ class CachePortArb : public SimObject { std::vector> IoOut; // non-cacheable requests, per port CachePortArb(const SimContext& ctx, const char* name, const params_t& params, - uint32_t num_inputs, uint32_t num_banks, uint32_t merge_group) + uint32_t num_inputs, uint32_t num_banks, uint32_t merge_group, bool merge_port0) : SimObject(ctx, name) , ReqIn(num_inputs, this) , ReqOut(num_banks, this) @@ -2273,6 +2273,7 @@ class CachePortArb : public SimObject { , delay_((num_inputs > 2) ? 1 : 0) , levels_(arb_tree_levels(num_inputs)) , merge_group_(merge_group) + , merge_port0_(merge_port0) { __assert(merge_group_ <= 1 || (merge_group_ <= 64 && ispow2(merge_group_) && (num_inputs % merge_group_) == 0), "a read-merge group must be a power of two of at most 64 dividing the ports"); @@ -2425,7 +2426,7 @@ class CachePortArb : public SimObject { auto& req_i = ReqIn.at(i).peek(); uint32_t base = (i / merge_group_) * merge_group_; for (uint32_t j = base; j < i; ++j) { - if (followers.at(j) || !this->is_plain_read(j)) { + if (followers.at(j) || !this->is_plain_read(j) || (j == 0 && !merge_port0_)) { continue; } auto& req_j = ReqIn.at(j).peek(); @@ -2531,6 +2532,7 @@ class CachePortArb : public SimObject { std::vector>> tree_stage_; // [level][bank][slice] std::vector> rr_; // [arbiter level][bank][slice] next to grant uint32_t merge_group_; + bool merge_port0_; uint64_t collisions_ = 0; friend class SimObject; @@ -2656,7 +2658,7 @@ class CacheRspMcast : public SimObject { // Sits on one memory port's cache-side response path, ahead of the response // network. While the port's flush token is out the cache is quiescent, so the // port's next response is the token's answer and stops here; every other -// response passes in the same cycle. +// response passes on, crossing the port's registered response stages. class FlushTap : public SimObject { public: using Ptr = std::shared_ptr; @@ -2664,10 +2666,11 @@ class FlushTap : public SimObject { SimChannel RspIn; SimChannel RspOut; - FlushTap(const SimContext& ctx, const char* name) + FlushTap(const SimContext& ctx, const char* name, uint32_t delay) : SimObject(ctx, name) , RspIn(this) , RspOut(this) + , delay_(delay) {} // the port's flush token has been sent @@ -2699,12 +2702,13 @@ class FlushTap : public SimObject { armed_ = false; answered_ = true; RspIn.pop(); - } else if (RspOut.try_send(RspIn.peek(), 0)) { + } else if (RspOut.try_send(RspIn.peek(), delay_)) { RspIn.pop(); } } private: + uint32_t delay_; bool armed_ = false; bool answered_ = false; @@ -2739,12 +2743,30 @@ class Cache::Impl { nc_mem_arbs_.at(i) = MemArbiter::Create(sname, ArbiterType::Priority, 2, 1); } + // Bank memory arbiter delays. Each level of the memory-side request + // network beyond the first is a registered stage; the request path + // buffers the levels the arbiter's own register does not cover in + // mem_req_net_. + uint32_t mem_arb_delay = ((num_banks != config_.mem_ports) && (num_banks > 2 || config_.mem_ports > 2)) ? 1 : 0; + uint32_t mem_net_delay = omega_stages(num_banks, config_.mem_ports) - 1; + uint32_t mem_net_extra = (mem_net_delay > mem_arb_delay) ? (mem_net_delay - mem_arb_delay) : 0; + uint32_t mem_arb_rsp_delay = (num_banks != config_.mem_ports) ? (mem_arb_delay + mem_net_extra) : 0; + __assert(num_banks >= config_.mem_ports, "a cache needs a bank per memory port"); + + // A memory response reaches its bank through the port's response queue, + // when there is one, and a response network registered at every level. + // The taps carry what the bank arbiter's response delay does not. + uint32_t mem_rsp_regs = omega_stages(config_.mem_ports, num_banks) + (config_.mem_rsp_queue ? 1 : 0); + uint32_t mem_rsp_credit = mem_arb_rsp_delay + (config_.mem_rsp_slice ? 1 : 0); + __assert(mem_rsp_regs >= mem_rsp_credit, "memory response path shorter than its arbiter"); + uint32_t tap_delay = mem_rsp_regs - mem_rsp_credit; + // flush-token taps, ticking after the arbiters that feed them and before // the response network they feed flush_taps_.resize(config_.mem_ports); for (uint32_t i = 0; i < config_.mem_ports; ++i) { snprintf(sname, 100, "%s-flush_tap%d", simobject->name().c_str(), i); - flush_taps_.at(i) = FlushTap::Create(sname); + flush_taps_.at(i) = FlushTap::Create(sname, tap_delay); nc_mem_arbs_.at(i)->RspOut.at(0).bind(&flush_taps_.at(i)->RspIn); } @@ -2754,17 +2776,9 @@ class Cache::Impl { simobject->mem_rsp_in.at(i).bind(&nc_mem_arbs_.at(i)->RspIn.at(0)); } - // Create bank's memory arbiter. Each level of the memory-side networks - // beyond the first is a registered stage; the request path buffers the - // levels the arbiter's own register does not cover in mem_req_net_, the - // response path in the bank's memory-response queue. snprintf(sname, 100, "%s-mem_arb", simobject->name().c_str()); - uint32_t mem_arb_delay = ((num_banks != config_.mem_ports) && (num_banks > 2 || config_.mem_ports > 2)) ? 1 : 0; - uint32_t mem_net_delay = omega_stages(num_banks, config_.mem_ports) - 1; - uint32_t mem_net_extra = (mem_net_delay > mem_arb_delay) ? (mem_net_delay - mem_arb_delay) : 0; - __assert(num_banks >= config_.mem_ports, "a cache needs a bank per memory port"); auto bank_mem_arb = MemArbiter::Create(sname, ArbiterType::RoundRobin, num_banks, config_.mem_ports, - mem_arb_delay + mem_net_extra, mem_arb_delay + mem_net_extra); + mem_arb_delay + mem_net_extra, mem_arb_rsp_delay); // Connect bank's memory arbiter to non-cacheable arbiter's input port 0 if (mem_net_extra != 0) { @@ -2788,7 +2802,7 @@ class Cache::Impl { // Create the core-port arbiter ahead of the banks it feeds snprintf(sname, 100, "%s-port_arb", simobject->name().c_str()); uint32_t merge_group = std::min(config_.read_merge, config_.num_inputs); - port_arb_ = CachePortArb::Create(sname, params_, config_.num_inputs, num_banks, merge_group); + port_arb_ = CachePortArb::Create(sname, params_, config_.num_inputs, num_banks, merge_group, config_.merge_port0); for (uint32_t i = 0; i < config_.num_inputs; ++i) { simobject->core_req_in.at(i).bind(&port_arb_->ReqIn.at(i)); } diff --git a/sim/simx/mem/cache.h b/sim/simx/mem/cache.h index ae8b783dd9..70adc78ba5 100644 --- a/sim/simx/mem/cache.h +++ b/sim/simx/mem/cache.h @@ -51,6 +51,9 @@ class Cache : public SimObject { uint8_t read_merge = 0; // merge same-word reads within groups of this many adjacent ports (0: off) uint16_t mshr_lines = 0; // MSHR distinct {line, sector} entries (0: one per request) bool write_around = false; // a no-allocate write miss goes to memory (write-back only) + bool mem_rsp_queue = false; // memory responses enter through a response queue + bool mem_rsp_slice = false; // a registered boundary ahead of the memory-response port stands for that queue's register + bool merge_port0 = true; // port 0's reads match other ports' tags for read merging }; struct PerfStats { diff --git a/sim/simx/mem/local_mem_switch.cpp b/sim/simx/mem/local_mem_switch.cpp index 7cc253575e..a1e64b6c10 100644 --- a/sim/simx/mem/local_mem_switch.cpp +++ b/sim/simx/mem/local_mem_switch.cpp @@ -18,7 +18,8 @@ using namespace vortex; LocalMemSwitch::LocalMemSwitch( const SimContext& ctx, const char* name, - uint32_t delay + uint32_t delay, + uint32_t dc_rsp_size ) : SimObject(ctx, name) // Single-entry ingress: the switch accepts a new request only as its // current one drains, so upstream issue feels downstream cadence @@ -28,7 +29,7 @@ LocalMemSwitch::LocalMemSwitch( , ReqOutLmem(this) , RspInLmem(this) , ReqOutDC(this) - , RspInDC(this) + , RspInDC(this, dc_rsp_size) , delay_(delay) {} diff --git a/sim/simx/mem/local_mem_switch.h b/sim/simx/mem/local_mem_switch.h index 6edbc3f1cb..d765adaead 100644 --- a/sim/simx/mem/local_mem_switch.h +++ b/sim/simx/mem/local_mem_switch.h @@ -33,7 +33,8 @@ class LocalMemSwitch : public SimObject { LocalMemSwitch( const SimContext& ctx, const char* name, - uint32_t delay + uint32_t delay, + uint32_t dc_rsp_size = 2 ); protected: diff --git a/sim/simx/mem/lsu_mem_adapter.cpp b/sim/simx/mem/lsu_mem_adapter.cpp index 60076bb902..e9d5592a8e 100644 --- a/sim/simx/mem/lsu_mem_adapter.cpp +++ b/sim/simx/mem/lsu_mem_adapter.cpp @@ -28,7 +28,6 @@ LsuMemAdapter::LsuMemAdapter( , RspIn(num_inputs, this) , delay_(delay) , rsp_delay_(rsp_delay) - , port_delay_(num_inputs, 0) , pending_mask_(num_inputs) { assert(num_inputs > 0); @@ -146,7 +145,7 @@ void LsuMemAdapter::on_tick() { out_req.flags.local = (t == AddrType::Shared); } - if (ReqOut.at(i).try_send(out_req, delay_ + port_delay_.at(i))) { + if (ReqOut.at(i).try_send(out_req, delay_)) { DT(4, this->name() << " req" << i << ": " << out_req); pending_mask_.reset(i); } diff --git a/sim/simx/mem/lsu_mem_adapter.h b/sim/simx/mem/lsu_mem_adapter.h index 6f6499716f..1864926527 100644 --- a/sim/simx/mem/lsu_mem_adapter.h +++ b/sim/simx/mem/lsu_mem_adapter.h @@ -42,9 +42,6 @@ class LsuMemAdapter : public SimObject { ) : LsuMemAdapter(ctx, name, num_inputs, 0) {} - // Extra request-side registers on one output port. - void set_port_delay(uint32_t port, uint32_t delay) { port_delay_.at(port) = delay; } - protected: void on_reset(); void on_tick(); @@ -52,7 +49,6 @@ class LsuMemAdapter : public SimObject { private: uint32_t delay_; uint32_t rsp_delay_; - std::vector port_delay_; BitVector pending_mask_; friend class SimObject; diff --git a/sim/simx/mem/mem_coalescer.cpp b/sim/simx/mem/mem_coalescer.cpp index bf25a8f05e..4940c9c5db 100644 --- a/sim/simx/mem/mem_coalescer.cpp +++ b/sim/simx/mem/mem_coalescer.cpp @@ -39,7 +39,6 @@ MemCoalescer::MemCoalescer( , sent_mask_(input_size) , line_size_(line_size) , delay_(delay) - , port_delay_(output_size, 0) {} void MemCoalescer::on_reset() { @@ -300,7 +299,7 @@ void MemCoalescer::flush_out_round() { if (!out_round_.lanes.test(o)) { continue; } - if (ReqOut.at(o).try_send(out_round_.reqs.at(o), delay_ + port_delay_.at(o))) { + if (ReqOut.at(o).try_send(out_round_.reqs.at(o), delay_)) { out_round_.lanes.reset(o); } } diff --git a/sim/simx/mem/mem_coalescer.h b/sim/simx/mem/mem_coalescer.h index 3cc6d72c8d..cd9df21223 100644 --- a/sim/simx/mem/mem_coalescer.h +++ b/sim/simx/mem/mem_coalescer.h @@ -46,9 +46,6 @@ class MemCoalescer : public SimObject { const PerfStats& perf_stats() const; - // Extra request-side registers on one output port. - void set_port_delay(uint32_t port, uint32_t delay) { port_delay_.at(port) = delay; } - protected: void on_reset(); void on_tick(); @@ -78,7 +75,6 @@ class MemCoalescer : public SimObject { out_round_t out_round_; uint32_t line_size_; uint32_t delay_; - std::vector port_delay_; PerfStats perf_stats_; friend class SimObject; diff --git a/sim/simx/operands.cpp b/sim/simx/operands.cpp index aaaa018d71..d7230c31ca 100644 --- a/sim/simx/operands.cpp +++ b/sim/simx/operands.cpp @@ -108,8 +108,9 @@ Operands::Operands(const SimContext &ctx, const char* name, Core* core) } if (VX_CFG_NUM_OPCS >= 2) { + // the collectors' round-robin merge registers its output snprintf(sname, 100, "%s-rsp_arb", name); - rsp_arb_ = TraceArbiter::Create(sname, ArbiterType::RoundRobin, VX_CFG_NUM_OPCS, 1); + rsp_arb_ = TraceArbiter::Create(sname, ArbiterType::RoundRobin, VX_CFG_NUM_OPCS, 1, 1); for (uint32_t i = 0; i < VX_CFG_NUM_OPCS; ++i) { opc_units_.at(i)->Output.bind(&rsp_arb_->Inputs.at(i)); } diff --git a/sim/simx/processor.cpp b/sim/simx/processor.cpp index d0a8a8c6e8..b015d165ca 100644 --- a/sim/simx/processor.cpp +++ b/sim/simx/processor.cpp @@ -120,6 +120,7 @@ ProcessorImpl::ProcessorImpl() 0, // read_merge VX_CFG_L3_MSHR_LINES, // mshr_lines CP_ON_L3 != 0, // write_around + VX_CFG_L3_MRSQ_SIZE != 0, // memory-response queue } ); diff --git a/sim/simx/socket.cpp b/sim/simx/socket.cpp index b44fb21845..d2df85bb0c 100644 --- a/sim/simx/socket.cpp +++ b/sim/simx/socket.cpp @@ -75,6 +75,11 @@ class Socket::Impl { false, // shared_llc false, // lazy_fetch true, // core-response register + 0, // read_merge + 0, // mshr_lines + false, // write_around + VX_CFG_ICACHE_MRSQ_SIZE != 0, // memory-response queue + true, // the socket's registered memory port }); snprintf(sname, 100, "%s-dcache", name.c_str()); @@ -103,6 +108,12 @@ class Socket::Impl { true, // core-response register (VX_CFG_DCACHE_READ_MERGE != 0) ? DCACHE_CHANNELS : 0, // read_merge VX_CFG_DCACHE_MSHR_LINES, // mshr_lines + false, // write_around + VX_CFG_DCACHE_MRSQ_SIZE != 0, // memory-response queue + true, // the socket's registered memory port + // A private cache takes the flush injection on port 0 alone, and its + // arbiter re-tags that port's requests apart from the others. + VX_CFG_SOCKET_SIZE != VX_CFG_NUM_DCACHES, // port 0 merges reads }); #ifdef VX_CFG_EXT_TEX_ENABLE @@ -115,7 +126,7 @@ class Socket::Impl { constexpr uint32_t kTcacheWordSize = 4; // = TCACHE_WORD_SIZE constexpr uint32_t kTcacheNumReqs = VX_CFG_TCACHE_NUM_BANKS; constexpr uint32_t kTcacheMemPorts = 1; // = TCACHE_MEM_PORTS - auto tcache = Cache::Create(sname, Cache::Config{ + Cache::Config tcache_config{ false, // bypass log2ceil(VX_CFG_TCACHE_SIZE), // C log2ceil(kTcacheLineSize), // L @@ -132,7 +143,10 @@ class Socket::Impl { 2, // pipeline latency uint8_t(VX_CFG_L2_REPL_POLICY), // replacement policy (use L2 policy as default) false, // is_llc (TCACHE is auxiliary, not LLC) - }); + }; + tcache_config.mem_rsp_queue = (VX_CFG_TCACHE_MRSQ_SIZE != 0); + tcache_config.mem_rsp_slice = true; + auto tcache = Cache::Create(sname, tcache_config); tcache_ = tcache; // tex_core ↔ tcache (per-port). @@ -155,7 +169,7 @@ class Socket::Impl { // VX_CFG_RTCACHE_NUM_BANKS banks. constexpr uint32_t kRtcacheNumInputs = VX_CFG_NUM_RTU_BLOCKS; constexpr uint32_t kRtcacheMemPorts = 1; - auto rtcache = Cache::Create(sname, Cache::Config{ + Cache::Config rtcache_config{ false, // bypass log2ceil(VX_CFG_RTCACHE_SIZE), // C log2ceil(kRtcacheLineSize), // L @@ -172,7 +186,10 @@ class Socket::Impl { 2, // pipeline latency uint8_t(VX_CFG_L2_REPL_POLICY), // replacement policy false, // is_llc - }); + }; + rtcache_config.mem_rsp_queue = (VX_CFG_RTCACHE_MRSQ_SIZE != 0); + rtcache_config.mem_rsp_slice = true; + auto rtcache = Cache::Create(sname, rtcache_config); rtcache_ = rtcache; // RtuCore ↔ rtcache (per memory port). @@ -254,7 +271,8 @@ class Socket::Impl { // connect l1 caches to outgoing memory interfaces for (uint32_t i = 0; i < VX_CFG_L1_MEM_PORTS; ++i) { snprintf(sname, 100, "%s-l1_arb%d", name.c_str(), i); - auto l1_arb = MemArbiter::Create(sname, ArbiterType::RoundRobin, 2 * overlap, overlap); + // icache first; the arbiter registers both directions + auto l1_arb = MemArbiter::Create(sname, ArbiterType::Priority, 2 * overlap, overlap, 1, 1); if (i < overlap) { icaches_->mem_req_out.at(i).bind(&l1_arb->ReqIn.at(i)); diff --git a/sim/simx/types.h b/sim/simx/types.h index 595891d401..a7eda35ac2 100644 --- a/sim/simx/types.h +++ b/sim/simx/types.h @@ -1428,6 +1428,39 @@ class TFifo : public SimObject> { /////////////////////////////////////////////////////////////////////////////// +// A registered elastic buffer: In holds its entries, and the head moves on +// through Out one register later, one per cycle. +template +class TxBuffer : public SimObject> { +public: + using Ptr = std::shared_ptr>; + + SimChannel In; + SimChannel Out; + + TxBuffer(const SimContext& ctx, const char* name, uint32_t size) + : SimObject>(ctx, name) + , In(this, size) + , Out(this) + {} + +protected: + void on_reset() {} + + void on_tick() { + if (!In.empty() && Out.try_send(In.peek(), 1)) { + In.pop(); + } + if (In.size() == 0) { + this->tick_sleep(); + } + } + + friend class SimObject>; +}; + +/////////////////////////////////////////////////////////////////////////////// + template class TxArbiter : public SimObject> { public: From 05d1237ff27039bcd77ed6bc69803d5e36827f58 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 01:27:46 -0700 Subject: [PATCH 099/124] cache: refetch a fully written lazy-fetch sector without a memory read Under lazy fetch a write miss allocates its sector without reading memory and leaves it partial. A later load refetches the sector from memory and merges the stored bytes over it. When stores have already covered every byte, nothing from that read survives the merge, yet it still cost a DRAM round trip. That cost is why the GPU profiles keep L2_LAZY_FETCH off. A refetch root (a partial-miss read, or a parked chain head) whose sector's dirty mask is full now loops back through the lazy queue as a data-less refetch instead of going to memory. It does so only while the queue has the room a new request is admitted with; otherwise it reads memory as before. The RTL (VX_cache_bank) and SimX change together. In SimX a refetch fill now learns whether its sector is still resident at its tag lookup, the tick after it enters, as in the RTL. lazy_fetch -f stores whole lines, or lines with one word per sector left unwritten, loads them back, and compares the two runs' memory reads: every fully written sector must save exactly one read. A chained mode loads while the stores' allocation is still pending, and a mixed mode interleaves whole and partial lines. It needs the memory counters (--perf). CI: cache:lazy_fetch-3 (lazy-fetch dcache) and cache:lazy_fetch-4 (write-back lazy-fetch L2 shared by four cores), on simx and rtlsim; core:parity-lazy-fetch-full. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/cache.yaml | 24 ++++ ci/testcases/core.yaml | 12 ++ hw/rtl/cache/VX_cache_bank.sv | 72 +++++++---- sim/simx/mem/cache.cpp | 112 ++++++++++++----- tests/regression/lazy_fetch/common.h | 49 +++++++- tests/regression/lazy_fetch/kernel.cpp | 65 ++++++++++ tests/regression/lazy_fetch/main.cpp | 160 ++++++++++++++++++++++++- 7 files changed, 433 insertions(+), 61 deletions(-) diff --git a/ci/testcases/cache.yaml b/ci/testcases/cache.yaml index 9b52dab9ee..d2e9377c61 100644 --- a/ci/testcases/cache.yaml +++ b/ci/testcases/cache.yaml @@ -338,6 +338,30 @@ tests: shape: cores: 4 l2cache: true +# A sector every store has covered reads no memory when a load refetches it. +# The app compares the memory reads of fully written lines against lines with +# a word per sector unwritten, so it runs with the memory counters. +- id: lazy_fetch-3 + via: blackbox + drivers: + - simx + - rtlsim + app: lazy_fetch + args: -f -n1 + configs: -DVX_CFG_EXT_A_ENABLE -DVX_CFG_DCACHE_LAZY_FETCH=1 + flags: --perf=7 +- id: lazy_fetch-4 + via: blackbox + drivers: + - simx + - rtlsim + app: lazy_fetch + args: -f -n1 + configs: -DVX_CFG_EXT_A_ENABLE -DVX_CFG_SOCKET_SIZE=1 -DVX_CFG_L2_WRITEBACK=1 -DVX_CFG_L2_LAZY_FETCH=1 + flags: --perf=7 + shape: + cores: 4 + l2cache: true # Both sectors of one sectored L2 line miss together from different warps and # cores: each must get its own fill, never chain behind the other sector's. # Run with the address held per request and with the MSHR line table. diff --git a/ci/testcases/core.yaml b/ci/testcases/core.yaml index f449e79c87..676531abed 100644 --- a/ci/testcases/core.yaml +++ b/ci/testcases/core.yaml @@ -147,6 +147,18 @@ tests: via: blackbox app: lazy_fetch configs: -DVX_CFG_DCACHE_LAZY_FETCH=1 +# Fully written sectors on a lazy-fetch write-back L2 shared by 4 cores: +# partial-miss loads and parked chain heads refetch them through the loopback. +- id: parity-lazy-fetch-full + check: model_parity + via: blackbox + app: lazy_fetch + args: -f -n1 + configs: -DVX_CFG_EXT_A_ENABLE -DVX_CFG_SOCKET_SIZE=1 -DVX_CFG_L2_WRITEBACK=1 -DVX_CFG_L2_LAZY_FETCH=1 + flags: --perf=7 + shape: + cores: 4 + l2cache: true # FPU + LSU locality, multi-warp reuse (~1.29M cycles on the default shape). - id: parity-sgemm check: model_parity diff --git a/hw/rtl/cache/VX_cache_bank.sv b/hw/rtl/cache/VX_cache_bank.sv index ae219feccb..d71663087e 100644 --- a/hw/rtl/cache/VX_cache_bank.sv +++ b/hw/rtl/cache/VX_cache_bank.sv @@ -164,6 +164,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( logic is_refetch; // fill whose root refetches a partial sector logic is_retry; // refetch re-issued after its sector was evicted logic is_park; // replay parked on a partial sector: issues its refetch + logic is_loop; // fill taken from the loopback queue } lazy_req_t; data_t sel_req, st0, dat_in, stD; // request + fill line: sel -> S0 -> stD @@ -374,6 +375,8 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // refetches. A refetch merges into its resident sector one stage late, at // stC; one whose sector was evicted installs nothing and retries as a // plain fill (nothing can dirty an absent sector, so its data is current). + // A refetch whose sector is already fully written has nothing to read: it + // loops back as a data-less refetch instead of going to memory. wire lazyq_valid = ~lazyq_empty; wire [MSHR_ADDR_WIDTH-1:0] lazyq_id; wire lazy_sel = ~mem_rsp_valid && lazyq_valid; @@ -523,10 +526,11 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // tag_matches is computed at S0; left 0 here (overridden at the data bubble). end - assign lz_sel.is_lazy = fill_enable && lazy_sel; - assign lz_sel.is_refetch = fill_enable && ~lazy_sel && mshr_fill_refetch; + assign lz_sel.is_lazy = fill_enable && lazy_sel && ~mshr_fill_refetch; + assign lz_sel.is_refetch = fill_enable && mshr_fill_refetch; assign lz_sel.is_retry = fill_enable && ~lazy_sel && mshr_fill_retry; assign lz_sel.is_park = replay_enable && park_sel; + assign lz_sel.is_loop = fill_enable && lazy_sel; // UUID extraction (debug + MSHR ordering): per stage, from the carried tag. wire [`UP(UUID_WIDTH)-1:0] req_uuid_sel, req_uuid_st0, req_uuid_st1, req_uuid_stc; @@ -780,9 +784,9 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // refetch fill writes no data at stD. wire fill_merge_st0 = refetch_first_st0 && fill_resident_st0; wire fill_nodata_st0 = lazy_st0 || refetch_first_st0; - wire fill_merge_stc, fill_nodata_std, lazy_std; + wire fill_merge_stc, fill_nodata_std, loop_std, partial_miss_stc; if (LAZY_FETCH != 0) begin : g_lazy_pipe - wire fill_merge_st1; + wire fill_merge_st1, partial_miss_st1; VX_pipe_register #( .DATAW ($bits(lazy_req_t)), .RESETW ($bits(lazy_req_t)) @@ -794,25 +798,25 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .data_out (lz_st0) ); VX_pipe_register #( - .DATAW ($bits(lazy_req_t) + 1), - .RESETW ($bits(lazy_req_t) + 1) + .DATAW ($bits(lazy_req_t) + 2), + .RESETW ($bits(lazy_req_t) + 2) ) reg_lz_s1 ( .clk (clk), .reset (reset), .enable (~pipe_stall), - .data_in ({lz_st0, fill_merge_st0}), - .data_out ({lz_st1, fill_merge_st1}) + .data_in ({lz_st0, fill_merge_st0, partial_miss_st0}), + .data_out ({lz_st1, fill_merge_st1, partial_miss_st1}) ); VX_pipe_register #( - .DATAW ($bits(lazy_req_t) + 1), - .RESETW ($bits(lazy_req_t) + 1), + .DATAW ($bits(lazy_req_t) + 2), + .RESETW ($bits(lazy_req_t) + 2), .DEPTH (PIPE_EX) ) reg_lz_cmt ( .clk (clk), .reset (reset), .enable (~pipe_stall), - .data_in ({lz_st1, fill_merge_st1}), - .data_out ({lz_stc, fill_merge_stc}) + .data_in ({lz_st1, fill_merge_st1, partial_miss_st1}), + .data_out ({lz_stc, fill_merge_stc, partial_miss_stc}) ); VX_pipe_register #( .DATAW (2), @@ -822,8 +826,8 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .clk (clk), .reset (reset), .enable (~pipe_stall), - .data_in ({lazy_st0, fill_nodata_st0}), - .data_out ({lazy_std, fill_nodata_std}) + .data_in ({lz_st0.is_loop, fill_nodata_st0}), + .data_out ({loop_std, fill_nodata_std}) ); end else begin : g_no_lazy_pipe assign lz_st0 = '0; @@ -831,8 +835,9 @@ module VX_cache_bank import VX_gpu_pkg::*; #( assign lz_stc = '0; assign fill_merge_stc = 1'b0; assign fill_nodata_std = 1'b0; - assign lazy_std = 1'b0; - `UNUSED_VAR ({lz_sel, fill_merge_st0, fill_nodata_st0}) + assign loop_std = 1'b0; + assign partial_miss_stc = 1'b0; + `UNUSED_VAR ({lz_sel, fill_merge_st0, fill_nodata_st0, lz_st0.is_loop, lz_stc.is_loop}) end // a passthru-AMO replay carries its result word instead of an installed @@ -884,7 +889,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .data_in (fbuf_addr_r), .data_out (fill_addr_std) ); - `RUNTIME_ASSERT (~(do_fill_std && ~stD.req.is_passthru_fill && ~lazy_std) || (stD.req.addr == fill_addr_std), ("%t: %s fill sector misaligned with its array write: addr=0x%0h, staged=0x%0h", $time, INSTANCE_ID, `CS_BANK_TO_FULL_ADDR(stD.req.addr, BANK_ID), `CS_BANK_TO_FULL_ADDR(fill_addr_std, BANK_ID))) + `RUNTIME_ASSERT (~(do_fill_std && ~stD.req.is_passthru_fill && ~loop_std) || (stD.req.addr == fill_addr_std), ("%t: %s fill sector misaligned with its array write: addr=0x%0h, staged=0x%0h", $time, INSTANCE_ID, `CS_BANK_TO_FULL_ADDR(stD.req.addr, BANK_ID), `CS_BANK_TO_FULL_ADDR(fill_addr_std, BANK_ID))) `endif // The fill sector, replicated across the line so each sector's slices see @@ -1351,6 +1356,14 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // a write-miss root under lazy fetch loops its fill back instead. wire lazy_alloc_stc = (LAZY_FETCH != 0) && do_write_stc && ~stC.lk.is_hit && ~stC.lk.mshr_pending && ~write_around_stc; + // the mask of the request's sector, as of every older store + wire [SECTOR_SIZE-1:0] sector_byteen_stc = evict_byteen_stc[sector_idx_stc * SECTOR_SIZE +: SECTOR_SIZE]; + // a refetch root (a partial-miss read or a parked head) whose sector every + // store already covers loops back instead of reading memory, while the + // loopback has the room a new request is admitted with: a parked head is + // not admitted against it, and memory always has room for the refetch. + wire refetch_loop_stc = (LAZY_FETCH != 0) && do_read_stc && ~stC.lk.is_hit && ~stC.lk.mshr_pending + && partial_miss_stc && (& sector_byteen_stc) && ~lazyq_alm_full; wire do_fill_or_flush_stc = stC.req.valid && is_fill_or_flush_stc; // a sector refill keeps the resident line (no eviction), so never writes back. wire do_writeback_stc = do_fill_or_flush_stc && stC.lk.is_dirty && ~stC.lk.is_refill && ~refetch_first_stc; @@ -1455,7 +1468,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // on a dirty eviction. The two are mutually exclusive (a writeback is a // fill/flush commit; a fill request is a creq commit). wire fill_req_push = (do_read_stc || do_write_stc) && ~stC.lk.is_hit && ~stC.lk.mshr_pending - && ~lazy_alloc_stc && ~write_around_stc && ~pipe_stall; + && ~lazy_alloc_stc && ~refetch_loop_stc && ~write_around_stc && ~pipe_stall; wire wa_req_push = write_around_stc && ~pipe_stall; assign mreq_queue_push = fill_req_push || wa_req_push || wb_beat_accept; assign mreq_queue_addr = is_wb_beat ? wb_mreq_addr : rd_mreq_addr; @@ -1539,7 +1552,8 @@ module VX_cache_bank import VX_gpu_pkg::*; #( ); assign mem_req_valid = ~mreq_queue_empty; - // Lazy-fetch loopback: write-miss roots awaiting their data-less fill. + // Lazy-fetch loopback: write-miss roots awaiting their data-less fill, and + // refetch roots whose sector needs no data. // Sized like the memory request queue's reservation: a core request is // admitted only while every request already in the pipe still fits. if (LAZY_FETCH != 0) begin : g_lazyq @@ -1552,7 +1566,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( ) lazy_queue ( .clk (clk), .reset (reset), - .push (lazy_alloc_stc && ~pipe_stall), + .push ((lazy_alloc_stc || refetch_loop_stc) && ~pipe_stall), .pop (lazy_fire), .data_in (stC.req.mshr_id), .data_out (lazyq_id), @@ -1572,7 +1586,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( chain_partial_r <= 1'b0; chain_retry_r <= 1'b0; end else if (fill_fire) begin - chain_partial_r <= lazy_fire; + chain_partial_r <= lazy_fire && ~mshr_fill_refetch; chain_retry_r <= 1'b0; end else if (refetch_hold && ~pipe_stall) begin chain_partial_r <= fill_evicted_st0; @@ -1589,14 +1603,20 @@ module VX_cache_bank import VX_gpu_pkg::*; #( refetch_inflight_r <= 1'b0; end else if (mem_rsp_fire && mshr_fill_refetch) begin refetch_inflight_r <= 1'b1; - end else if (stC.req.valid && stC.req.is_fill && refetch_stc && ~pipe_stall) begin + end else if (stC.req.valid && stC.req.is_fill && refetch_stc && ~lz_stc.is_loop && ~pipe_stall) begin refetch_inflight_r <= 1'b0; end end assign refetch_inflight = refetch_inflight_r; + // A memory refetch keeps its sector staged until its stC merge. + `RUNTIME_ASSERT (~(stC.req.valid && stC.req.is_fill && refetch_stc && ~lz_stc.is_loop) || refetch_inflight_r, + ("%t: %s memory refetch reached its merge unstaged: addr=0x%0h", $time, INSTANCE_ID, `CS_BANK_TO_FULL_ADDR(addr_stc, BANK_ID))) - `RUNTIME_ASSERT (~(lazy_alloc_stc && ~pipe_stall && lazyq_full && ~lazy_fire), + `RUNTIME_ASSERT (~((lazy_alloc_stc || refetch_loop_stc) && ~pipe_stall && lazyq_full && ~lazy_fire), ("%t: %s lazy queue overflow", $time, INSTANCE_ID)) + // A looped-back refetch carries no data: its merge must find every byte written. + `RUNTIME_ASSERT (~(do_merge_stc && lz_stc.is_loop) || (& sector_byteen_stc), + ("%t: %s looped-back refetch of an incompletely written sector: addr=0x%0h", $time, INSTANCE_ID, `CS_BANK_TO_FULL_ADDR(addr_stc, BANK_ID))) `RUNTIME_ASSERT (~(park_fire && fwd_fire), ("%t: %s park and forward in one cycle", $time, INSTANCE_ID)) `RUNTIME_ASSERT (~replay_valid || (mshr_dequeue_reads == (~replay_rw || replay_amo.amo_valid)), ("%t: %s chain head read flag disagrees with its payload", $time, INSTANCE_ID)) @@ -1614,7 +1634,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( assign chain_partial = 1'b0; assign chain_retry = 1'b0; assign refetch_inflight = 1'b0; - `UNUSED_VAR ({lazy_alloc_stc, lazy_fire, refetch_hold, fill_resident_st0, fill_evicted_st0, partial_match_st0}) + `UNUSED_VAR ({lazy_alloc_stc, refetch_loop_stc, sector_byteen_stc, lazy_fire, refetch_hold, fill_resident_st0, fill_evicted_st0, partial_match_st0}) end `UNUSED_VAR (do_lookup_st0) @@ -1652,8 +1672,8 @@ module VX_cache_bank import VX_gpu_pkg::*; #( mem_rsp_full_addr, mem_rsp_id, mshr_fill_refetch, mshr_fill_retry, mem_rsp_data, req_uuid_sel)) end if (lazy_fire) begin - `TRACE(2, ("%t: %s lazy-fill: addr=0x%0h, mshr_id=%0d\n", $time, INSTANCE_ID, - mem_rsp_full_addr, lazyq_id)) + `TRACE(2, ("%t: %s lazy-fill: addr=0x%0h, mshr_id=%0d, refetch=%b\n", $time, INSTANCE_ID, + mem_rsp_full_addr, lazyq_id, mshr_fill_refetch)) end if (park_fire) begin `TRACE(2, ("%t: %s mshr-park: addr=0x%0h, tag=0x%0h, req_idx=%0d (#%0d)\n", $time, INSTANCE_ID, diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index ebeabee7ef..feae96a223 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -220,6 +220,12 @@ static inline bool sector_hit(const sector_t& sec, bool needs_data) { return sec.valid && !(needs_data && sec.partial); } +// Every byte of the sector has been stored: a refetch of it reads nothing. +static inline bool sector_written(const sector_t& sec) { + constexpr uint64_t full_mask = (VX_CFG_MEM_BLOCK_SIZE >= 64) ? ~0ull : ((1ull << VX_CFG_MEM_BLOCK_SIZE) - 1); + return sec.dirty_mask == full_mask; +} + struct line_t { uint64_t tag; uint32_t lru_ctr; @@ -904,6 +910,7 @@ class CacheBank : public SimObject { chain_tag_ = 0; chain_sector_ = 0; refetch_hold_ = false; + refetch_hold_retry_ = false; flushing_ = false; flush_set_idx_ = 0; flush_way_idx_ = 0; @@ -977,15 +984,26 @@ class CacheBank : public SimObject { // // At most one input fires per tick. All inputs flow through pipe_req_. void processInputs(bool fwd_consumed) { + // A refetch fill learns whether its sector is still resident at its tag + // lookup, the tick after it enters: nothing enters behind it meanwhile, + // and its chain then replays, or parks again if the sector was evicted. + if (refetch_hold_) { + refetch_hold_ = false; + bool evicted = false; + if (!refetch_hold_retry_) { + const auto &set = sets_.at(chain_set_); + int resident_id = set.find_resident(chain_tag_); + evicted = (resident_id == -1) || !set.lines.at(resident_id).sectors.at(chain_sector_).valid; + } + chain_partial_ = evicted; + chain_retry_ = evicted; + return; + } + // 1) replay: one chain entry leaves per tick — when the fill-forward // drain completed the head this tick, this slot goes to new core // requests instead if (mshr_.has_ready_reqs() && !fwd_consumed) { - // A refetch fill learns whether its sector is still resident when it - // installs; its chain waits for that answer. - if (refetch_hold_) { - return; - } // A chain whose sector is partial cannot serve a load or AMO: park at // that head and refetch the sector; the rest of the chain waits. if (chain_partial_) { @@ -1072,7 +1090,8 @@ class CacheBank : public SimObject { mshr_.replay(mshr_id); this->setChain(set_id, addr_tag, sector_id, false); if (bank_req.refetch) { - refetch_hold_ = true; + refetch_hold_ = true; + refetch_hold_retry_ = bank_req.refetch_retry; } else { fwd_active_ = true; fwd_set_ = set_id; @@ -1086,8 +1105,9 @@ class CacheBank : public SimObject { return; } } else if (!lazy_q_->empty() && !fwd_active_) { - // Lazy allocation of a write-miss root: a data-less fill through the - // same path, below memory fills in priority. + // A data-less fill through the same path, below memory fills in + // priority: the allocation of a write-miss root, or the refetch of a + // sector every store has covered. uint32_t mshr_id = lazy_q_->peek(); const auto &root_peek = mshr_.peek(mshr_id); const uint32_t set_id = root_peek.set_id; @@ -1106,9 +1126,17 @@ class CacheBank : public SimObject { bank_req.uuid = root_peek.bank_req.uuid; bank_req.mshr_id = mshr_id; bank_req.nodata = true; + bank_req.refetch = root_peek.refetch; + bank_req.refetch_retry = root_peek.refetch_retry; pipe_req_->push(bank_req); mshr_.replay(mshr_id); - this->setChain(set_id, addr_tag, sector_id, true); + // A looped-back refetch completes a fully written sector: like a + // memory refetch, its chain waits for it to install. + this->setChain(set_id, addr_tag, sector_id, !bank_req.refetch); + if (bank_req.refetch) { + refetch_hold_ = true; + refetch_hold_retry_ = bank_req.refetch_retry; + } DT(3, this->name() << " lazy-fill: " << bank_req); lazy_q_->pop(); return; @@ -1188,7 +1216,7 @@ class CacheBank : public SimObject { } // Any admitted request may become a write-miss root: the lazy queue // keeps room for every request already in the pipe. - if (config_.lazy_fetch && lazy_q_->size() >= lazyq_size(config_) - config_.latency) { + if (config_.lazy_fetch && !this->lazyRoom()) { return; } @@ -1322,16 +1350,17 @@ class CacheBank : public SimObject { const bool refetch_first = bank_req.refetch && !bank_req.refetch_retry; assert(!sector_resident || refetch_first); if (refetch_first) { - chain_partial_ = !sector_resident; - chain_retry_ = !sector_resident; - refetch_hold_ = false; if (sector_resident) { // Memory supplies the bytes no store wrote; the sector stays dirty // in exactly the stored bytes, which are not in memory yet. - auto &sec = set.lines.at(resident_id).sectors.at(sector_id); - auto written = sec.data; - sec.data = bank_req.data; - sector_merge(sec, written, sec.dirty_mask); + auto &sec = set.lines.at(resident_id).sectors.at(sector_id); + if (bank_req.nodata) { + assert(sector_written(sec) && "looped-back refetch of an incompletely written sector"); + } else { + auto written = sec.data; + sec.data = bank_req.data; + sector_merge(sec, written, sec.dirty_mask); + } sec.partial = false; } return true; @@ -1405,11 +1434,6 @@ class CacheBank : public SimObject { sec.dirty_mask = 0; sec.data = bank_req.data; } - if (bank_req.refetch) { - chain_partial_ = false; - chain_retry_ = false; - refetch_hold_ = false; - } return true; } @@ -1427,6 +1451,13 @@ class CacheBank : public SimObject { return 1u << log2ceil(config.latency + 1); } + // The loopback still fits every request in the pipe: a new request may be + // admitted, and a refetch with no data to read may loop back; one that + // cannot reads memory instead. + bool lazyRoom() const { + return lazy_q_->size() < lazyq_size(config_) - config_.latency; + } + // Fill-forward drain: complete the oldest ready replay directly when it is // a plain read on the armed chain — no pipeline traversal, and the input // slot stays free for new requests. The first write/AMO head closes the @@ -1705,12 +1736,24 @@ class CacheBank : public SimObject { } break; case bank_req_t::Park: { - if (this->mem_req_out.full()) { - return; // stall - } uint32_t set_id = params_.addr_set_id(bank_req.addr); uint64_t addr_tag = params_.addr_tag(bank_req.addr); uint32_t sector_id = params_.addr_sector_id(bank_req.addr); + auto &set = sets_.at(set_id); + // A parked head needs memory room whichever way its refetch goes. + if (this->mem_req_out.full()) { + return; // stall + } + int resident_id = set.find_resident(addr_tag); + if (resident_id != -1) { + const auto &sec = set.lines.at(resident_id).sectors.at(sector_id); + if (sec.valid && sec.partial && sector_written(sec) && this->lazyRoom()) { + lazy_q_->push(bank_req.mshr_id); + DT(3, this->name() << " refetch-loop: " << bank_req); + pipe_req_->pop(); + return; + } + } MemReq fill; fill.addr = params_.mem_addr_sector(bank_id_, set_id, addr_tag, sector_id); fill.tag = bank_req.mshr_id; @@ -1992,21 +2035,25 @@ class CacheBank : public SimObject { // fill for this fresh miss. bool fill_pending = mshr_.has_pending_fill(set_id, addr_tag, sector_id, -1); // Lazy fetch: a write-miss root allocates through the loopback - // instead of fetching the sector. + // instead of fetching the sector, and so does the refetch of a + // partial sector every store has already covered. const bool lazy_alloc = config_.lazy_fetch && bank_req.write && !fill_pending; - if (lazy_alloc && lazy_q_->full()) { + const bool refetch_loop = partial_resident && !fill_pending + && sector_written(set.lines.at(present_id).sectors.at(sector_id)) + && this->lazyRoom(); + if ((lazy_alloc || refetch_loop) && lazy_q_->full()) { return; } - if (!fill_pending && !lazy_alloc && this->mem_req_out.full()) { + if (!fill_pending && !lazy_alloc && !refetch_loop && this->mem_req_out.full()) { return; } assert(!mshr_.full()); int mshr_id = mshr_.enqueue(bank_req, set_id, addr_tag, sector_id, partial_resident); DT(3, this->name() << " mshr-enqueue: " << bank_req); - if (lazy_alloc) { + if (lazy_alloc || refetch_loop) { lazy_q_->push(mshr_id); - DT(3, this->name() << " lazy-alloc: " << bank_req); + DT(3, this->name() << (lazy_alloc ? " lazy-alloc: " : " refetch-loop: ") << bank_req); } else if (!fill_pending) { MemReq fill; fill.addr = params_.mem_addr_sector(bank_id_, set_id, addr_tag, sector_id); @@ -2186,7 +2233,7 @@ class CacheBank : public SimObject { return n; } TFifo::Ptr pipe_req_; - TFifo::Ptr lazy_q_; // write-miss roots awaiting their data-less fill + TFifo::Ptr lazy_q_; // roots awaiting a data-less fill: write misses and written refetches Cache::PerfStats perf_stats_; @@ -2210,7 +2257,8 @@ class CacheBank : public SimObject { uint32_t chain_set_; uint64_t chain_tag_; uint32_t chain_sector_; - bool refetch_hold_; // a refetch fill has not installed yet: its chain waits + bool refetch_hold_; // a refetch fill is at its tag lookup: its chain waits + bool refetch_hold_retry_; // Flush walk state. bool flushing_; diff --git a/tests/regression/lazy_fetch/common.h b/tests/regression/lazy_fetch/common.h index 308f768331..5b6758319b 100644 --- a/tests/regression/lazy_fetch/common.h +++ b/tests/regression/lazy_fetch/common.h @@ -12,6 +12,19 @@ // Same-set lines: one per thread and round, a page apart, so they compete for // one cache set and a line being refetched is often evicted before its fill. #define SET_STRIDE_WORDS 1024 +// A whole cache line of the largest configured line size: two 64-byte +// sectors, stored before any of it is loaded. +#define FULL_LINE_WORDS 32 +#define SECTOR_WORDS 16 +// Lines a flood launch reads one word of, each a memory block apart. +#define FLOOD_LINE_BYTES 64 + +// How a phase-2 launch stores and loads its lines. +enum { + FULL_SEPARATE = 0, // each thread stores its lines, then loads them all back + FULL_CHAIN = 1, // a block's threads store a sector together, load it at once, store, load + FULL_MIXED = 2 // as FULL_SEPARATE, with every odd line left partial +}; enum { CASE_HALF = 0, // store half the line, then load all of it (same thread) @@ -31,8 +44,13 @@ typedef struct { uint32_t warp_size; uint32_t cross_warp; // blocks hold an even number of warps uint32_t amo; // the device implements the A extension - uint32_t phase; // 0: exercise the cases, 1: re-check every region + uint32_t phase; // 0: exercise the cases, 1: re-check every region, 2: full lines, 3: flood uint32_t rotate; // phase 1: check the regions of thread (gid + rotate) + uint64_t full_addr; + uint32_t full_lines; // phase 2: lines per thread (per warp in FULL_CHAIN); 3: lines it reads + uint32_t full_mode; // phase 2: FULL_* + uint32_t full_skip; // phase 2: leave each sector's last word unwritten + uint32_t num_blocks; } kernel_arg_t; // Pre-pattern the host writes over the whole buffer. @@ -52,6 +70,35 @@ static inline uint32_t store_value(uint32_t region, uint32_t w) { return 0xC0000000u | (region << 4) | w; } +// Word w of a line that a phase-2 launch leaves unwritten. +static inline bool full_skipped(uint32_t line, uint32_t w, const kernel_arg_t* arg) { + bool partial = (arg->full_mode == FULL_MIXED) ? ((line & 1) != 0) : (arg->full_skip != 0); + return partial && (w % SECTOR_WORDS) == (SECTOR_WORDS - 1); +} + +static inline uint32_t full_value(uint32_t line, uint32_t w) { + return 0x50000000u | (line << 5) | w; +} + +// The second store a FULL_CHAIN thread makes to its word of each sector. +static inline uint32_t full_value2(uint32_t line, uint32_t w) { + return 0x60000000u | (line << 5) | w; +} + +// Word `word_idx` of a phase-2 buffer once its launch has run; the host +// pre-patterns the buffer. +static inline uint32_t full_expected(uint32_t word_idx, const kernel_arg_t* arg) { + uint32_t line = word_idx / FULL_LINE_WORDS; + uint32_t w = word_idx % FULL_LINE_WORDS; + if (full_skipped(line, w, arg)) { + return pattern_word(word_idx); + } + if (arg->full_mode == FULL_CHAIN) { + return full_value2(line, w); + } + return full_value(line, w); +} + static inline uint32_t region_index(uint32_t round, uint32_t num_threads, uint32_t gid, uint32_t c) { return (round * num_threads + gid) * NUM_CASES + c; } diff --git a/tests/regression/lazy_fetch/kernel.cpp b/tests/regression/lazy_fetch/kernel.cpp index 5f0dc5e4cd..a704245ec5 100644 --- a/tests/regression/lazy_fetch/kernel.cpp +++ b/tests/regression/lazy_fetch/kernel.cpp @@ -129,6 +129,71 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { uint32_t errors = 0; auto err = reinterpret_cast(arg->err_addr); + if (arg->phase == 3) { + // Read fresh lines from memory, one word each. + auto base = reinterpret_cast(arg->full_addr); + uint32_t sum = 0; + for (uint32_t i = 0; i < arg->full_lines; ++i) { + uint32_t line = i * arg->num_threads + gid; + sum += base[line * (FLOOD_LINE_BYTES / sizeof(uint32_t))]; + } + asm volatile ("" :: "r"(sum)); + err[gid] = 0; + return; + } + + if (arg->phase == 2 && arg->full_mode == FULL_CHAIN) { + // The first SECTOR_WORDS threads of a block store one word each of a + // sector, every warp at once after a barrier, and load it right away: the + // loads reach the cache while the stores' allocation is still pending. + // The second store lands while that chain is being refetched. + auto base = reinterpret_cast(arg->full_addr); + uint32_t t = threadIdx.x; + for (uint32_t i = 0; i < arg->full_lines; ++i) { + uint32_t line = i * arg->num_blocks + blockIdx.x; + for (uint32_t s = 0; s < FULL_LINE_WORDS; s += SECTOR_WORDS) { + uint32_t idx = line * FULL_LINE_WORDS + s + t; + bool skip = full_skipped(line, s + t, arg); + // a skipped word is stored to a scratch word: no branch delays the load + volatile uint32_t* dst = skip ? &err[arg->num_threads + gid] : &base[idx]; + __syncthreads(); + if (t < SECTOR_WORDS) { + *dst = full_value(line, s + t); + uint32_t first = base[idx]; + *dst = full_value2(line, s + t); + uint32_t second = base[idx]; + errors += (first != (skip ? pattern_word(idx) : full_value(line, s + t))) ? 1 : 0; + errors += (second != full_expected(idx, arg)) ? 1 : 0; + } + } + } + err[gid] = errors; + return; + } + + if (arg->phase == 2) { + // Store this thread's lines, all of each sector or all but one word, + // then load them all back. + auto base = reinterpret_cast(arg->full_addr); + for (uint32_t i = 0; i < arg->full_lines; ++i) { + uint32_t line = i * arg->num_threads + gid; + for (uint32_t w = 0; w < FULL_LINE_WORDS; ++w) { + if (!full_skipped(line, w, arg)) { + base[line * FULL_LINE_WORDS + w] = full_value(line, w); + } + } + } + for (uint32_t i = 0; i < arg->full_lines; ++i) { + uint32_t line = i * arg->num_threads + gid; + for (uint32_t w = 0; w < FULL_LINE_WORDS; ++w) { + uint32_t idx = line * FULL_LINE_WORDS + w; + errors += (base[idx] != full_expected(idx, arg)) ? 1 : 0; + } + } + err[gid] = errors; + return; + } + if (arg->phase != 0) { // Every region again, from another thread (another core when there are // several), after the launch boundary flushed the partial lines. diff --git a/tests/regression/lazy_fetch/main.cpp b/tests/regression/lazy_fetch/main.cpp index 60734b7711..413e17bf48 100644 --- a/tests/regression/lazy_fetch/main.cpp +++ b/tests/regression/lazy_fetch/main.cpp @@ -2,7 +2,9 @@ #include #include #include +#include #include +#include #include "common.h" #define RT_CHECK(_expr) \ @@ -19,23 +21,25 @@ const char* kernel_file = "kernel.vxbin"; uint32_t rounds = 16; +bool full_mode = false; vx_device_h device = nullptr; vx_buffer_h buf_buffer = nullptr; vx_buffer_h err_buffer = nullptr; vx_buffer_h set_buffer = nullptr; +std::vector full_buffers; vx_queue_h queue = nullptr; vx_module_h module_ = nullptr; vx_kernel_h kernel = nullptr; static void show_usage() { std::cout << "Vortex Test." << std::endl; - std::cout << "Usage: [-k: kernel] [-n rounds] [-h: help]" << std::endl; + std::cout << "Usage: [-k: kernel] [-n rounds] [-f: fully written lines read no memory] [-h: help]" << std::endl; } static void parse_args(int argc, char **argv) { int c; - while ((c = getopt(argc, argv, "n:k:h")) != -1) { + while ((c = getopt(argc, argv, "n:k:fh")) != -1) { switch (c) { case 'n': rounds = atoi(optarg); @@ -43,6 +47,9 @@ static void parse_args(int argc, char **argv) { case 'k': kernel_file = optarg; break; + case 'f': + full_mode = true; + break; case 'h': show_usage(); exit(0); @@ -59,6 +66,9 @@ void cleanup() { if (buf_buffer) vx_buffer_release(buf_buffer); if (err_buffer) vx_buffer_release(err_buffer); if (set_buffer) vx_buffer_release(set_buffer); + for (auto b : full_buffers) { + vx_buffer_release(b); + } if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); @@ -86,6 +96,139 @@ static int launch(const kernel_arg_t& arg, uint32_t grid, uint32_t block) { return ret; } +static int mem_reads(uint64_t* value) { + return vx_device_mpm_query(device, VX_DCR_MPM_CLASS_MEM, VX_CSR_MPM_MEM_READS, 0, value); +} + +static vx_buffer_h new_buffer(uint32_t size, uint64_t* addr) { + vx_buffer_h buffer = nullptr; + RT_CHECK(vx_buffer_create(device, size, VX_MEM_READ_WRITE, &buffer)); + full_buffers.push_back(buffer); + RT_CHECK(vx_buffer_address(buffer, addr)); + return buffer; +} + +static void read_buffer(void* dst, vx_buffer_h buffer, uint32_t size) { + vx_event_h read_ev = nullptr; + RT_CHECK(vx_enqueue_read(queue, dst, buffer, 0, size, 0, nullptr, &read_ev)); + RT_CHECK(vx_event_wait_value(read_ev, 1, VX_TIMEOUT_INFINITE)); + vx_event_release(read_ev); +} + +static int check_errors(uint32_t num_threads) { + std::vector h_err(num_threads); + read_buffer(h_err.data(), err_buffer, num_threads * sizeof(uint32_t)); + int errors = 0; + for (uint32_t t = 0; t < num_threads; ++t) { + if (h_err[t] != 0) { + if (errors < 16) { + std::cout << "error: thread " << t << " read back 0x" << std::hex << h_err[t] << std::dec + << " wrong words of its lines" << std::endl; + } + ++errors; + } + } + return errors; +} + +static const char* full_name(const kernel_arg_t& arg) { + switch (arg.full_mode) { + case FULL_CHAIN: return arg.full_skip ? "chained, partially written" : "chained, fully written"; + case FULL_MIXED: return "mixed"; + default: return arg.full_skip ? "partially written" : "fully written"; + } +} + +// One phase-2 launch over a fresh pre-patterned buffer, after a launch that +// reads far more. Drivers differ on whether the memory read counter restarts +// at each launch; after the flood, a counter that restarted reads below its +// previous value. Returns the launch's memory reads. +static int64_t run_lines(kernel_arg_t arg, uint32_t grid, uint32_t block, uint32_t lines, int* errors) { + const uint32_t flood_lines = 4096; + uint32_t line_threads = (arg.full_mode == FULL_CHAIN) ? arg.num_blocks : arg.num_threads; + uint32_t words = lines * line_threads * FULL_LINE_WORDS; + vx_buffer_h buffer = new_buffer(words * sizeof(uint32_t), &arg.full_addr); + std::vector h_buf(words); + for (uint32_t i = 0; i < words; ++i) { + h_buf[i] = pattern_word(i); + } + RT_CHECK(vx_enqueue_write(queue, buffer, 0, h_buf.data(), words * sizeof(uint32_t), 0, nullptr, nullptr)); + + kernel_arg_t flood = arg; + flood.phase = 3; + flood.full_lines = flood_lines / arg.num_threads; + new_buffer(flood_lines * FLOOD_LINE_BYTES, &flood.full_addr); + RT_CHECK(launch(flood, grid, block)); + *errors += check_errors(arg.num_threads); + + arg.phase = 2; + arg.full_lines = lines; + uint64_t before = 0, after = 0; + RT_CHECK(mem_reads(&before)); + RT_CHECK(launch(arg, grid, block)); + RT_CHECK(mem_reads(&after)); + int64_t reads = (after < before) ? (int64_t)after : (int64_t)(after - before); + *errors += check_errors(arg.num_threads); + + read_buffer(h_buf.data(), buffer, words * sizeof(uint32_t)); + int bad = 0; + for (uint32_t i = 0; i < words; ++i) { + uint32_t ref = full_expected(i, &arg); + if (h_buf[i] != ref) { + if (bad < 16) { + std::cout << "error: " << full_name(arg) << " word " << i << std::hex << ": actual 0x" << h_buf[i] + << ", expected 0x" << ref << std::dec << std::endl; + } + ++bad; + } + } + *errors += bad; + std::cout << full_name(arg) << " lines, memory reads: " << reads << std::endl; + return reads; +} + +// Lines stored whole and then loaded must not be read from memory. Each pair +// of launches stores and loads the same lines, one leaving a word of each +// sector unwritten: every partial sector is read from memory once, a whole +// one never, and the rest of the two launches' traffic is alike. +static int run_full(kernel_arg_t arg, uint32_t grid, uint32_t block) { + // few enough lines that a 16 KB cache keeps them resident + const uint32_t lines = 2; + int errors = 0; + for (uint32_t mode : {FULL_SEPARATE, FULL_CHAIN}) { + // a chained sector takes a whole block, which needs a word per thread + if (mode == FULL_CHAIN && block < SECTOR_WORDS) { + std::cout << "chained lines skipped: " << block << " threads per block" << std::endl; + continue; + } + arg.full_mode = mode; + uint32_t line_threads = (mode == FULL_CHAIN) ? arg.num_blocks : arg.num_threads; + // a chained sector belongs to a block: give the blocks more lines + uint32_t mode_lines = (mode == FULL_CHAIN) ? lines * 16 : lines; + arg.full_skip = 1; + int64_t partial = run_lines(arg, grid, block, mode_lines, &errors); + arg.full_skip = 0; + int64_t full = run_lines(arg, grid, block, mode_lines, &errors); + if (partial == 0) { + std::cout << "error: memory read counter inactive (run with --perf)" << std::endl; + return errors + 1; + } + int64_t sectors = (int64_t)mode_lines * line_threads * (FULL_LINE_WORDS / SECTOR_WORDS); + int64_t saved = partial - full; + // A sector evicted before its load is read back from memory either way. + int64_t slack = std::max(2, sectors / 16); + std::cout << full_name(arg) << " sectors: " << sectors << ", memory reads saved: " << saved << std::endl; + if (saved < sectors - slack || saved > sectors + slack) { + std::cout << "error: fully written sectors were read from memory" << std::endl; + ++errors; + } + } + // Loads of whole sectors loop back while partial ones read memory. + arg.full_mode = FULL_MIXED; + run_lines(arg, grid, block, lines, &errors); + return errors; +} + int main(int argc, char *argv[]) { parse_args(argc, argv); if (rounds == 0) { @@ -121,6 +264,7 @@ int main(int argc, char *argv[]) { arg.warp_size = num_threads; arg.cross_warp = (num_warps % 2) == 0; arg.amo = (isa_flags & VX_ISA_STD_A) != 0; + arg.num_blocks = grid; std::cout << "threads: " << total_threads << ", amo: " << arg.amo << ", regions: " << num_regions << ", buffer size: " << buf_size << " bytes" << std::endl; @@ -150,6 +294,18 @@ int main(int argc, char *argv[]) { RT_CHECK(vx_module_load_file(device, kernel_file, &module_)); RT_CHECK(vx_module_get_kernel(module_, "main", &kernel)); + if (full_mode) { + int errors = run_full(arg, grid, block); + cleanup(); + if (errors != 0) { + std::cout << "Found " << std::dec << errors << " errors!" << std::endl; + std::cout << "FAILED!" << std::endl; + return 1; + } + std::cout << "PASSED!" << std::endl; + return 0; + } + std::cout << "launch kernel (exercise)" << std::endl; arg.phase = 0; RT_CHECK(launch(arg, grid, block)); From 73db36c78ef81cec89a73f16dff49526b813c813 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 01:27:47 -0700 Subject: [PATCH 100/124] simx: model KMU, CTA retirement, barrier and collector timing SimX ran ahead of, or behind, the RTL at several control points. Each now follows the RTL's registers: - The KMU's lane slice registers a CTA into the core, so the KMU sends it without a delay of its own. The dispatcher returns the lane credit as the CTA is admitted, and the lane buffer offers the freed entry the next cycle. - A CTA's last warp exit reaches the slot table registered and updates it by read-modify-write. The slot frees in the fourth cycle, and the write holds admission off for that cycle. Before, it freed at once. - A transaction-barrier completion crosses the completion queue and a register before it applies (two cycles, not one). - Fetch picks warps by an ibuffer-full flag registered at the end of the previous cycle. - FIFO replacement: the pointer names the victim even when another way is free, and it advances on every installed fill, sector refills included. - With several operand collectors, the scoreboard's skid buffer feeds them through a demux. Only its head is offered, to its own collector, and an entry reaches the head the cycle after the one ahead leaves. Co-Authored-By: Claude Opus 5.5 --- sim/simx/barrier_unit.cpp | 7 ++++--- sim/simx/core.cpp | 15 +++++++++++---- sim/simx/cta_dispatcher.cpp | 27 ++++++++++++++++++++++----- sim/simx/cta_dispatcher.h | 4 +++- sim/simx/kmu/kmu.cpp | 3 ++- sim/simx/mem/cache.cpp | 13 ++++++++++--- sim/simx/operands.cpp | 37 +++++++++++++++++++++++++++---------- sim/simx/operands.h | 4 ++++ 8 files changed, 83 insertions(+), 27 deletions(-) diff --git a/sim/simx/barrier_unit.cpp b/sim/simx/barrier_unit.cpp index 6538afc2a7..968541ad16 100644 --- a/sim/simx/barrier_unit.cpp +++ b/sim/simx/barrier_unit.cpp @@ -24,10 +24,11 @@ using namespace vortex; // Cycles from an SFU barrier op to its effect in the barrier store: the store // is read a cycle before an arrival applies, and with several SFU blocks the // request is also registered out of its block. A transaction-barrier -// completion is registered once before it applies; a global-barrier release -// applies as it arrives. A release then reaches the warp scheduler registered. +// completion crosses the completion queue and a register before it applies; a +// global-barrier release applies as it arrives. A release then reaches the +// warp scheduler registered. static constexpr uint32_t kOpApplyLag = (VX_CFG_NUM_SFU_BLOCKS > 1) ? 2 : 1; -static constexpr uint32_t kEventApplyLag = 1; +static constexpr uint32_t kEventApplyLag = 2; BarrierUnit::BarrierUnit(const SimContext& ctx, const char* name, Core* core, Scheduler* scheduler) : SimObject(ctx, name) diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index b926fd3070..213e9634e9 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -72,6 +72,7 @@ class Core::Impl { , fu_full_(VX_CFG_ISSUE_WIDTH, BitVector<>((uint32_t)FUType::Count, 0)) , fu_full_seen_(VX_CFG_ISSUE_WIDTH, BitVector<>((uint32_t)FUType::Count, 0)) , ibuf_inflight_(VX_CFG_NUM_WARPS, 0) + , ibuf_full_r_(VX_CFG_NUM_WARPS, false) , last_issue_(VX_CFG_NUM_WARPS, 0) , staged_since_(VX_CFG_NUM_WARPS, 0) , head_staged_(VX_CFG_NUM_WARPS, false) @@ -370,6 +371,7 @@ class Core::Impl { pending_ifetches_ = 0; std::fill(ibuf_inflight_.begin(), ibuf_inflight_.end(), 0); + std::fill(ibuf_full_r_.begin(), ibuf_full_r_.end(), false); // perf_stats_ deliberately survives: these are the MPM counters, which // accumulate from device reset. A kernel launch re-enters reset() here but @@ -407,11 +409,12 @@ class Core::Impl { scheduler_->schedule_begin(); // Each frontend slice picks among its own warps whose ibuffer is not full; a - // buffer is full at exactly IBUF_SIZE entries. With an L1, once every warp's - // buffer is full any ready warp may fetch, its instruction waiting at decode. + // buffer is full at exactly IBUF_SIZE entries, a flag registered at the end + // of the previous cycle. With an L1, once every warp's buffer is full any + // ready warp may fetch, its instruction waiting at decode. bool all_full = (VX_CFG_ICACHE_ENABLED || VX_CFG_DCACHE_ENABLED); for (uint32_t w = 0; w < VX_CFG_NUM_WARPS && all_full; ++w) { - all_full = (ibuf_inflight_.at(w) == VX_CFG_IBUF_SIZE); + all_full = ibuf_full_r_.at(w); } // the cycle is idle when no slice has a warp to fetch, its latch free or not bool any_cand = false; @@ -419,7 +422,7 @@ class Core::Impl { auto& fetch_latch = fetch_latch_.at(f); WarpMask warp_mask; for (uint32_t w = f; w < VX_CFG_NUM_WARPS; w += VX_CFG_FETCH_WIDTH) { - if (all_full || ibuf_inflight_.at(w) != VX_CFG_IBUF_SIZE) { + if (all_full || !ibuf_full_r_.at(w)) { warp_mask.set(w); } } @@ -445,6 +448,9 @@ class Core::Impl { if (!any_cand) { ++perf_stats_.sched_idle; } + for (uint32_t w = 0; w < VX_CFG_NUM_WARPS; ++w) { + ibuf_full_r_.at(w) = (ibuf_inflight_.at(w) == VX_CFG_IBUF_SIZE); + } scheduler_->schedule_end(); } @@ -1232,6 +1238,7 @@ class Core::Impl { std::vector> fu_full_seen_; // [iw] that flag one cycle later, as issue sees it std::vector ibuf_inflight_; + std::vector ibuf_full_r_; std::vector last_issue_; // [wid] cycle of the warp's last issue std::vector staged_since_; // [wid] cycle the last issued candidate was staged std::vector head_staged_; // [wid] the buffer head has left for staging diff --git a/sim/simx/cta_dispatcher.cpp b/sim/simx/cta_dispatcher.cpp index 67c82eca72..39502a56e5 100644 --- a/sim/simx/cta_dispatcher.cpp +++ b/sim/simx/cta_dispatcher.cpp @@ -64,6 +64,7 @@ void CtaDispatcher::on_reset() { cta_wids_.reset(); tail_slot_ = 0; init_wait_ = 0; + retiring_.clear(); for (uint32_t i = 0; i < num_warps_; ++i) { slot_rem_warps_[i] = 0; wid_to_slot_[i] = num_warps_; // invalid @@ -103,6 +104,22 @@ uint32_t CtaDispatcher::usable_slots(uint32_t stride) const { } uint32_t CtaDispatcher::step(const WarpMask& active_warps, uint32_t* wids, cta_warp_record_t* recs) { + // A retirement is registered, reads its slot's count from the table and + // writes it back: it updates the slot in its fourth cycle, and the write + // holds admission off for that cycle. + uint64_t now = SimPlatform::instance().cycles(); + bool retire_write = false; + for (size_t i = 0; i < retiring_.size();) { + if (retiring_[i].second > now) { + ++i; + continue; + } + assert(slot_rem_warps_[retiring_[i].first] > 0); + --slot_rem_warps_[retiring_[i].first]; + retire_write |= (retiring_[i].second == now); + retiring_.erase(retiring_.begin() + i); + } + if (cta_done_) { cta_done_ = false; has_cta_ = false; @@ -113,7 +130,7 @@ uint32_t CtaDispatcher::step(const WarpMask& active_warps, uint32_t* wids, cta_w // endpoint holds the one CTA committed to this core. It is popped only on // admission — an unadmitted CTA keeps the lane's credit, so the KMU // cannot over-commit work to a busy core. - if (bus_in.empty()) + if (bus_in.empty() || retire_write) return 0; const kmu_req_t& next = bus_in.peek(); @@ -172,7 +189,8 @@ uint32_t CtaDispatcher::step(const WarpMask& active_warps, uint32_t* wids, cta_w // the admission is reported back to the KMU's launch accounting. cta_ = next; bus_in.pop(); - credit_out.send(1, 1); + // the lane's buffer offers the freed entry to the KMU next cycle + credit_out.send(1, 0); cta_size_ = (cta_.block_size + num_threads_ - 1) / num_threads_; rank_ = 0; @@ -255,10 +273,9 @@ void CtaDispatcher::warp_done(uint32_t wid) { uint32_t slot = wid_to_slot_[wid]; if (slot >= num_warps_) return; // not a CTA-dispatcher warp wid_to_slot_[wid] = num_warps_; // clear assignment - assert(slot_rem_warps_[slot] > 0); // When the last warp of a CTA exits, its slot (and the LMEM region at - // slot × stride) frees immediately for reuse — no in-order reclaim. - --slot_rem_warps_[slot]; + // slot × stride) frees for reuse as the retirement lands — no in-order reclaim. + retiring_.emplace_back(slot, SimPlatform::instance().cycles() + 3); } bool CtaDispatcher::slot_needs_init(uint32_t wid, const ThreadMask& tmask) const { diff --git a/sim/simx/cta_dispatcher.h b/sim/simx/cta_dispatcher.h index 314fd1d4ce..cf1d649b82 100644 --- a/sim/simx/cta_dispatcher.h +++ b/sim/simx/cta_dispatcher.h @@ -106,8 +106,10 @@ class CtaDispatcher : public SimObject { // within a kernel. The occupancy bound floor(capacity/stride) (capped at // NUM_WARPS) replaces the byte ring's free_size accounting, so no wrap or // padding arithmetic is needed. Slots are allocated round-robin via a tail - // pointer and freed immediately when a CTA's last warp exits. + // pointer and freed when the retirement of a CTA's last warp lands. std::vector slot_rem_warps_; // remaining active warps; slot free when 0 + // Warp retirements on their way to the slot table: {slot, cycle it lands}. + std::vector> retiring_; // Reverse lookup: wid → slot index std::vector wid_to_slot_; uint32_t tail_slot_; // next slot to allocate (round-robin over usable slots) diff --git a/sim/simx/kmu/kmu.cpp b/sim/simx/kmu/kmu.cpp index 58734cce0e..731dcdf3ac 100644 --- a/sim/simx/kmu/kmu.cpp +++ b/sim/simx/kmu/kmu.cpp @@ -148,7 +148,8 @@ void Kmu::on_tick() { kmu_req_t req{}; if (!this->next_cta(&req, target)) return; - bus_out.at(target).send(req, 1); + // the lane slice ticks after the KMU and registers the CTA into the core + bus_out.at(target).send(req, 0); ++inflight_[target]; if (!cluster_locked_) { // Message complete: advance past the granted core. diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index feae96a223..ccf1a1d6ba 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -1372,6 +1372,10 @@ class CacheBank : public SimObject { } else { int32_t free_id = -1, repl_id = 0; line_id = set.select_victim(config_.repl_policy, rand_ctr_, &free_id, &repl_id); + // The FIFO pointer names the victim even when another way is free. + if (config_.repl_policy == Cache::FIFO) { + line_id = repl_id; + } auto &victim_line = set.lines.at(line_id); // Count the victim's dirty sectors; the egress queue must accept all @@ -1388,9 +1392,7 @@ class CacheBank : public SimObject { return false; // stall } - if (config_.repl_policy == Cache::FIFO) { - set.fifo_ptr = (set.fifo_ptr + 1) % set.lines.size(); - } else if (config_.repl_policy == Cache::RANDOM) { + if (config_.repl_policy == Cache::RANDOM) { ++rand_ctr_; } @@ -1416,6 +1418,11 @@ class CacheBank : public SimObject { victim_line.reset(); victim_line.tag = addr_tag; } + // Every installed fill advances the set's FIFO pointer, a sector refill + // into the resident way included. + if (config_.repl_policy == Cache::FIFO) { + set.fifo_ptr = (set.fifo_ptr + 1) % set.lines.size(); + } auto &line = set.lines.at(line_id); line.tag = addr_tag; diff --git a/sim/simx/operands.cpp b/sim/simx/operands.cpp index d7230c31ca..56a9d971dd 100644 --- a/sim/simx/operands.cpp +++ b/sim/simx/operands.cpp @@ -115,6 +115,19 @@ Operands::Operands(const SimContext &ctx, const char* name, Core* core) opc_units_.at(i)->Output.bind(&rsp_arb_->Inputs.at(i)); } rsp_arb_->Outputs.at(0).bind(&this->Output); + // The scoreboard's output skid buffer feeds the collectors through a + // demux: its head waits for its own collector, the entries behind it too. + // The head leaves the buffer at its last operand read, when its collector + // pops it, and the next head is offered from the following cycle. + Input.set_registered_ready(true); + Input.tx_callback([this](instr_trace_t* const& trace, uint64_t) { + if (!head_offered_ && this->Input.empty()) { + this->offer_head(trace, 0); + } + }); + for (uint32_t i = 0; i < VX_CFG_NUM_OPCS; ++i) { + opc_units_.at(i)->Input.pop_callback([this]() { this->on_head_consumed(); }); + } } else { // pass-thru this->Input.bind(&opc_units_.at(0)->Input); @@ -127,19 +140,23 @@ Operands::~Operands() { } void Operands::on_reset() { - //-- + head_offered_ = false; } void Operands::on_tick() { - if (VX_CFG_NUM_OPCS < 2) - return; // pass-thru - - // process requests - if (Input.empty()) - return; - auto trace = this->Input.peek(); - if (opc_units_.at(wid_to_opc_idx(trace->wid))->Input.try_send(trace)) { - Input.pop(); + this->tick_sleep(); // the demux runs on Input's delivery and pop events +} + +void Operands::offer_head(instr_trace_t* trace, uint64_t delay) { + opc_units_.at(wid_to_opc_idx(trace->wid))->Input.send(trace, delay); + head_offered_ = true; +} + +void Operands::on_head_consumed() { + this->Input.pop(); + head_offered_ = false; + if (!this->Input.empty()) { + this->offer_head(this->Input.peek(), 1); } } diff --git a/sim/simx/operands.h b/sim/simx/operands.h index c302f7abc3..077b905400 100644 --- a/sim/simx/operands.h +++ b/sim/simx/operands.h @@ -54,6 +54,10 @@ class Operands : public SimObject { Core* core_; std::vector opc_units_; TraceArbiter::Ptr rsp_arb_; + bool head_offered_ = false; + + void offer_head(instr_trace_t* trace, uint64_t delay); + void on_head_consumed(); friend class SimObject; }; From 347c914a6e5c45f740733467b82381091a2a7b4f Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 01:27:48 -0700 Subject: [PATCH 101/124] runtime: stage kernel args in clean slots without a cache flush Every launch writes its argument block into device memory through the CP DMA. While the last-level cache may hold lines the DMA would miss or later overwrite, each such write first appends a full cache flush. A run of short launches therefore paid one full flush per launch for its arguments alone. The args pool now records, per free slot, the flush epoch from which the last-level cache holds none of the slot's lines. A slot a kernel read becomes clean again after the next full flush. Thirty-two slots are allocated when the device opens, before any kernel has run, so they start clean. args_slot_acquire prefers a clean slot, and a MEM_WRITE into one needs no flush ahead of it. Oversized blocks and slots a kernel has read since the last full flush keep the flush. A dcache is also the point of coherence between kernels when it is the only dcache and every core of the one socket shares it. Before, only a single-core device qualified. The RTL (VX_gpu_pkg) and SimX now accept the socket-wide case, so L1-only flush tokens there are answered without a walk. VX_graphics gains a simulation-only assertion that each cluster flush round finds its flush kind latched by a DCR write before it takes the request. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/VX_gpu_pkg.sv | 10 ++-- hw/rtl/VX_graphics.sv | 20 ++++++++ sim/simx/constants.h | 10 ++-- sw/runtime/common/device.cpp | 69 ++++++++++++++++++++-------- sw/runtime/common/queue.cpp | 20 ++++---- sw/runtime/common/vortex2_internal.h | 35 +++++++++++--- 6 files changed, 123 insertions(+), 41 deletions(-) diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index 904bad773c..9e874674d1 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -1747,10 +1747,12 @@ package VX_gpu_pkg; // L1 dcache is the LLC iff neither L2 nor L3 is enabled. localparam DCACHE_IS_LLC = !`VX_CFG_L2_ENABLED && !`VX_CFG_L3_ENABLED; - // It is the point of coherence between kernels only as the one dcache with - // no other memory client beside it (the texture, ray-tracing and DXA ports - // join below it, the raster and OM caches beside it). - localparam DCACHE_SHARED_LLC = DCACHE_IS_LLC && (`VX_CFG_NUM_CORES == 1) && (`VX_CFG_NUM_CLUSTERS == 1) + // It is the point of coherence between kernels only as the one dcache, + // shared by every core of the one socket, with no other memory client + // beside it (the texture, ray-tracing and DXA ports join below it, the + // raster and OM caches beside it). + localparam DCACHE_SHARED_LLC = DCACHE_IS_LLC && (`VX_CFG_NUM_CLUSTERS == 1) + && (`VX_CFG_NUM_CORES == `VX_CFG_SOCKET_SIZE) && (`VX_CFG_NUM_DCACHES == 1) && !`VX_CFG_EXT_TEX_ENABLED && !`VX_CFG_EXT_RTU_ENABLED && !`VX_CFG_EXT_RASTER_ENABLED && !`VX_CFG_EXT_OM_ENABLED && !`VX_CFG_EXT_DXA_ENABLED; diff --git a/hw/rtl/VX_graphics.sv b/hw/rtl/VX_graphics.sv index 37a035506d..2e0fa10725 100644 --- a/hw/rtl/VX_graphics.sv +++ b/hw/rtl/VX_graphics.sv @@ -517,6 +517,26 @@ module VX_graphics import VX_gpu_pkg::* end wire omf_drained = ~(| om_busy_w) && ~(| om_ingress_pending) && ~(| steer_om_pending); +`ifndef SYNTHESIS + // Each cluster flush round must find the latch written for it, strictly + // before the round takes the request. + wire cluster_flush_kind_wr = dcr_bus_if.req_valid && dcr_bus_if.req_data.rw + && (dcr_bus_if.req_data.addr == `VX_DCR_BASE_CACHE_FLUSH); + wire omf_cluster_take = (omf_state_r == OMF_IDLE) && omf_cluster_req; + reg cluster_flush_kind_new_r; + always @(posedge clk) begin + if (reset) begin + cluster_flush_kind_new_r <= 1'b0; + end else if (cluster_flush_kind_wr) begin + cluster_flush_kind_new_r <= 1'b1; + end else if (omf_cluster_take) begin + cluster_flush_kind_new_r <= 1'b0; + end + end + `RUNTIME_ASSERT (~omf_cluster_take || (cluster_flush_kind_new_r && ~cluster_flush_kind_wr), + ("%t: cluster flush taken before its kind was latched", $time)) +`endif + always @(posedge clk) begin if (reset) begin omf_state_r <= OMF_IDLE; diff --git a/sim/simx/constants.h b/sim/simx/constants.h index 2142e7efd2..a1cf857579 100644 --- a/sim/simx/constants.h +++ b/sim/simx/constants.h @@ -40,11 +40,13 @@ namespace vortex { inline constexpr uint32_t XLENB = (VX_CFG_XLEN / 8); // A last-level cache that alone serves every memory client is the point of -// coherence between kernels. The texture, ray-tracing and DXA ports join below -// the dcache and the raster and OM caches beside it, so a dcache with any of -// them is not one. +// coherence between kernels. A dcache is one when it is the only dcache, +// shared by every core of the one socket. The texture, ray-tracing and DXA +// ports join below the dcache and the raster and OM caches beside it, so a +// dcache with any of them is not one. inline constexpr bool DCACHE_SHARED_LLC = (VX_CFG_DCACHE_ENABLED != 0) && (VX_CFG_L2_ENABLED == 0) && (VX_CFG_L3_ENABLED == 0) - && (VX_CFG_NUM_CORES == 1) && (VX_CFG_NUM_CLUSTERS == 1) + && (VX_CFG_NUM_CLUSTERS == 1) && (VX_CFG_NUM_CORES == VX_CFG_SOCKET_SIZE) + && (VX_CFG_NUM_DCACHES == 1) && !VX_CFG_EXT_TEX_ENABLED && !VX_CFG_EXT_RTU_ENABLED && !VX_CFG_EXT_RASTER_ENABLED && !VX_CFG_EXT_OM_ENABLED && !VX_CFG_EXT_DXA_ENABLED; diff --git a/sw/runtime/common/device.cpp b/sw/runtime/common/device.cpp index 39009c1927..957564a3a7 100644 --- a/sw/runtime/common/device.cpp +++ b/sw/runtime/common/device.cpp @@ -157,8 +157,8 @@ Device::~Device() { // Drain the kernel-args scratch pool. { std::lock_guard g(args_pool_mu_); - for (uint64_t addr : args_pool_free_) - this->mem_free(addr); + for (const auto& slot : args_pool_free_) + this->mem_free(slot.addr); args_pool_free_.clear(); } // Park the CP before releasing the buffers it masters into. Ordering is @@ -182,10 +182,13 @@ Device::~Device() { // ============================================================================ vx_result_t Device::args_slot_acquire(uint64_t size, uint64_t* out_addr, - bool* out_pooled) { + bool* out_pooled, bool* out_llc_clean) { if (!out_addr || !out_pooled) return VX_ERR_INVALID_VALUE; + bool llc_clean = false; + if (out_llc_clean) *out_llc_clean = false; if (size > ARGS_SLOT_SIZE) { - // Oversized args block — one-off allocation, not pooled. + // Oversized args block — one-off allocation, not pooled. The range may + // have been another buffer the cache still holds. *out_pooled = false; return this->mem_alloc(size, /*VX_MEM_READ*/ 0x1, out_addr); } @@ -193,8 +196,18 @@ vx_result_t Device::args_slot_acquire(uint64_t size, uint64_t* out_addr, { std::lock_guard g(args_pool_mu_); if (!args_pool_free_.empty()) { - *out_addr = args_pool_free_.back(); - args_pool_free_.pop_back(); + const uint64_t epoch = llc_epoch_.load(); + size_t pick = args_pool_free_.size() - 1; + for (size_t i = 0; i < args_pool_free_.size(); ++i) { + if (args_pool_free_[i].clean_epoch <= epoch) { + pick = i; + break; + } + } + *out_addr = args_pool_free_[pick].addr; + llc_clean = (args_pool_free_[pick].clean_epoch <= epoch); + args_pool_free_.erase(args_pool_free_.begin() + pick); + if (out_llc_clean) *out_llc_clean = llc_clean; return VX_SUCCESS; } } @@ -208,7 +221,7 @@ void Device::args_slot_release(uint64_t addr, bool pooled) { return; } std::lock_guard g(args_pool_mu_); - args_pool_free_.push_back(addr); + args_pool_free_.push_back({addr, llc_epoch_.load() + 1}); } vx_result_t Device::open(uint32_t index, Device** out) { @@ -475,6 +488,17 @@ vx_result_t Device::cp_init() { } } + // No kernel has run yet, so the last-level cache holds none of these. + { + std::lock_guard g(args_pool_mu_); + for (uint32_t i = 0; i < ARGS_POOL_CLEAN; ++i) { + uint64_t addr = 0; + auto r = this->mem_alloc(ARGS_SLOT_SIZE, /*VX_MEM_READ*/ 0x1, &addr); + if (r != VX_SUCCESS) return r; + args_pool_free_.push_back({addr, 0}); + } + } + return VX_SUCCESS; } @@ -551,7 +575,7 @@ bool cp_dma_touches_device_mem(const uint8_t* cl) { } } // namespace -vx_result_t Device::cp_ring_append_(const void* cl) { +vx_result_t Device::cp_ring_append_(const void* cl, bool llc_clean) { // Caller holds cp_mu_. A kernel's results are flushed lazily, by the // next command that needs them: the next launch gets an L1-only flush // (the last-level cache is the point of coherence between kernels), and @@ -560,7 +584,10 @@ vx_result_t Device::cp_ring_append_(const void* cl) { // it is a client of that cache. Appending the flush under the same lock // keeps any other command from slipping between. const uint8_t* b = static_cast(cl); - const bool dma = cp_dma_touches_device_mem(b); + // A write into memory the last-level cache holds no line of has nothing + // to wait for: the cache cannot serve stale data from it or write a + // dirty line back over it. + const bool dma = cp_dma_touches_device_mem(b) && !(llc_clean && b[0] == CP_OPCODE_MEM_WRITE); const bool launch = (b[0] == CP_OPCODE_LAUNCH) || (b[0] == CP_OPCODE_LAUNCH_QMD) || (b[0] == CP_OPCODE_DRAW); @@ -572,6 +599,9 @@ vx_result_t Device::cp_ring_append_(const void* cl) { if (r != VX_SUCCESS) return r; l1_dirty_ = false; llc_dirty_ = llc_dirty_ && !full; + if (full) { + ++llc_epoch_; + } } assert(!(dma && (l1_dirty_ || llc_dirty_))); auto r = cp_ring_write_(cl); @@ -583,6 +613,9 @@ vx_result_t Device::cp_ring_append_(const void* cl) { } else if (b[0] == CP_OPCODE_CACHE_FLUSH) { l1_dirty_ = false; llc_dirty_ = llc_dirty_ && (b[1] & CP_FLUSH_FLAG_L1_ONLY); + if (!(b[1] & CP_FLUSH_FLAG_L1_ONLY)) { + ++llc_epoch_; + } } return VX_SUCCESS; } @@ -655,14 +688,14 @@ vx_result_t Device::cp_batch_end() { return drain_cout(); } -vx_result_t Device::cp_submit_cl_(const void* cl) { +vx_result_t Device::cp_submit_cl_(const void* cl, bool llc_clean) { auto* p = platform(); // Batch mode: append only — cp_mu_ is already held for the batch, and // the single doorbell + poll happen in cp_batch_end. Only the thread that // owns the batch may take this path; anyone else must block on cp_mu_. if (tls_batch_owner == this) { - auto r = cp_ring_append_(cl); + auto r = cp_ring_append_(cl, llc_clean); if (r == VX_SUCCESS) cp_batch_target_ = cp_expected_seqnum_; return r; } @@ -675,7 +708,7 @@ vx_result_t Device::cp_submit_cl_(const void* cl) { std::lock_guard g(cp_mu_); // 1) Write the CL into the ring and reserve its seqnum. - auto r = cp_ring_append_(cl); + auto r = cp_ring_append_(cl, llc_clean); if (r != VX_SUCCESS) return r; target = cp_expected_seqnum_; @@ -1009,7 +1042,7 @@ vx_result_t Device::cp_submit_dcr_read(uint32_t addr, uint32_t tag, vx_result_t Device::cp_submit_mem_(uint8_t opcode, uint64_t arg0, uint64_t arg1, uint64_t arg2, - bool physical) { + bool physical, bool llc_clean) { // CMD_MEM_* on-wire layout (cmd_size=28): // bytes 0..3 header { opcode, flags, reserved=0 } // bytes 4..11 arg0 dst address @@ -1021,7 +1054,7 @@ vx_result_t Device::cp_submit_mem_(uint8_t opcode, uint64_t arg0, std::memcpy(cl + 4, &arg0, sizeof(arg0)); std::memcpy(cl + 12, &arg1, sizeof(arg1)); std::memcpy(cl + 20, &arg2, sizeof(arg2)); - return cp_submit_cl_(cl); + return cp_submit_cl_(cl, llc_clean); } vx_result_t Device::cp_submit_mem_copy(uint64_t dst, uint64_t src, @@ -1031,7 +1064,7 @@ vx_result_t Device::cp_submit_mem_copy(uint64_t dst, uint64_t src, } vx_result_t Device::cp_submit_mem_write(uint64_t dev_dst, const void* host_src, - uint64_t size, bool physical) { + uint64_t size, bool physical, bool llc_clean) { if (size == 0) return VX_SUCCESS; if (!host_src) return VX_ERR_INVALID_VALUE; // Stage through a bounded CP-visible host buffer, one chunk at a time: @@ -1054,7 +1087,7 @@ vx_result_t Device::cp_submit_mem_write(uint64_t dev_dst, const void* host_src, r = platform()->host_mem_push(staging.cp_addr); if (r == VX_SUCCESS) r = cp_submit_mem_(CP_OPCODE_MEM_WRITE, dev_dst + off, - staging.cp_addr, n, physical); + staging.cp_addr, n, physical, llc_clean); } host_free(staging.cp_addr); return r; @@ -1275,8 +1308,8 @@ vx_result_t Device::query_caps(uint32_t caps_id, uint64_t* out_value) { // ============================================================================ vx_result_t Device::dev_write(uint64_t dev_addr, const void* src, - uint64_t size) { - return cp_submit_mem_write(dev_addr, src, size); + uint64_t size, bool llc_clean) { + return cp_submit_mem_write(dev_addr, src, size, /*physical=*/false, llc_clean); } vx_result_t Device::dev_read(void* dst, uint64_t dev_addr, uint64_t size) { diff --git a/sw/runtime/common/queue.cpp b/sw/runtime/common/queue.cpp index 68543c0f60..84d4c22f0d 100644 --- a/sw/runtime/common/queue.cpp +++ b/sw/runtime/common/queue.cpp @@ -413,16 +413,17 @@ vx_result_t Queue::enqueue_launch(const vx_launch_info_t* info, // Empty blob → caller pre-programmed the ARG DCRs (legacy path). uint64_t args_addr = 0; bool args_pooled = false; + bool args_clean = false; bool args_staged = !args_blob.empty(); if (args_staged) { auto r = device_->args_slot_acquire(args_blob.size(), - &args_addr, &args_pooled); + &args_addr, &args_pooled, &args_clean); if (r != VX_SUCCESS) { if (kernel) kernel->release(); *s = *e = now_ns(); return r; } r = device_->dev_write(args_addr, args_blob.data(), - args_blob.size()); + args_blob.size(), args_clean); if (r != VX_SUCCESS) { device_->args_slot_release(args_addr, args_pooled); if (kernel) kernel->release(); @@ -607,11 +608,12 @@ vx_result_t cmd_stage_qmds(Device* device_, const std::vector& recs, // 1) Args blob. if (!rec.args.empty()) { + bool clean = false; r = device_->args_slot_acquire(rec.args.size(), - &st.args_addr, &st.args_pooled); + &st.args_addr, &st.args_pooled, &clean); if (r != VX_SUCCESS) break; st.args_active = true; - r = device_->dev_write(st.args_addr, rec.args.data(), rec.args.size()); + r = device_->dev_write(st.args_addr, rec.args.data(), rec.args.size(), clean); if (r != VX_SUCCESS) break; } @@ -672,10 +674,11 @@ vx_result_t cmd_stage_qmds(Device* device_, const std::vector& recs, st.qmd_words = std::move(qmd); if (device_->cp_supports_qmd()) { const uint64_t qmd_bytes = st.qmd_words.size() * sizeof(uint32_t); - r = device_->args_slot_acquire(qmd_bytes, &st.qmd_addr, &st.qmd_pooled); + bool clean = false; + r = device_->args_slot_acquire(qmd_bytes, &st.qmd_addr, &st.qmd_pooled, &clean); if (r != VX_SUCCESS) break; st.qmd_active = true; - r = device_->dev_write(st.qmd_addr, st.qmd_words.data(), qmd_bytes); + r = device_->dev_write(st.qmd_addr, st.qmd_words.data(), qmd_bytes, clean); if (r != VX_SUCCESS) break; } } @@ -841,10 +844,11 @@ vx_result_t Queue::enqueue_draw(const vx_command_t* commands, } } std::memcpy(desc.data(), &num_steps, sizeof(num_steps)); - r = device_->args_slot_acquire(desc.size(), &desc_addr, &desc_pooled); + bool desc_clean = false; + r = device_->args_slot_acquire(desc.size(), &desc_addr, &desc_pooled, &desc_clean); if (r == VX_SUCCESS) { desc_active = true; - r = device_->dev_write(desc_addr, desc.data(), desc.size()); + r = device_->dev_write(desc_addr, desc.data(), desc.size(), desc_clean); } } diff --git a/sw/runtime/common/vortex2_internal.h b/sw/runtime/common/vortex2_internal.h index f268d9a58d..f5a7e474ff 100644 --- a/sw/runtime/common/vortex2_internal.h +++ b/sw/runtime/common/vortex2_internal.h @@ -300,8 +300,11 @@ class Device : public RefCounted { // cp_submit_mem_read posts CMD_MEM_READ (device->host) into a // VX_MEM_HOST buffer and copies it back to `host_dst`. Synchronous. vx_result_t cp_submit_mem_copy (uint64_t dst, uint64_t src, uint64_t size); + // `llc_clean`: the last-level cache holds no line of the destination, so + // the write needs no cache flush ahead of it. vx_result_t cp_submit_mem_write(uint64_t dev_dst, const void* host_src, - uint64_t size, bool physical = false); + uint64_t size, bool physical = false, + bool llc_clean = false); vx_result_t cp_submit_mem_read (void* host_dst, uint64_t dev_src, uint64_t size, bool physical = false); @@ -312,7 +315,8 @@ class Device : public RefCounted { // engine (CMD_MEM_*) — identically on every backend. The runtime never // touches device memory directly: it only appends commands to the host // ring and the CP executes them. - vx_result_t dev_write(uint64_t dev_addr, const void* src, uint64_t size); + vx_result_t dev_write(uint64_t dev_addr, const void* src, uint64_t size, + bool llc_clean = false); vx_result_t dev_read (void* dst, uint64_t dev_addr, uint64_t size); vx_result_t dev_copy (uint64_t dst, uint64_t src, uint64_t size); @@ -329,8 +333,11 @@ class Device : public RefCounted { // args_slot_release frees correctly. Because vx_enqueue_launch holds // a slot only for the duration of one synchronous CP launch, the // free-list naturally settles at the count of concurrent launches. + // `*out_llc_clean` reports a slot no kernel has read since the last full + // cache flush: the last-level cache holds none of its lines, so staging + // into it needs no flush (see args_pool_free_). vx_result_t args_slot_acquire(uint64_t size, uint64_t* out_addr, - bool* out_pooled); + bool* out_pooled, bool* out_llc_clean = nullptr); void args_slot_release(uint64_t addr, bool pooled); // ----- Device-memory allocation (common-core allocator) ----- @@ -366,7 +373,7 @@ class Device : public RefCounted { // cp_submit_dcr_write / cp_submit_launch — they just build the CL. // While this thread has a batch open it appends only; the single // doorbell + poll are deferred to cp_batch_end. - vx_result_t cp_submit_cl_(const void* cl); + vx_result_t cp_submit_cl_(const void* cl, bool llc_clean = false); // Block until the CP has retired seqnum >= target. // @@ -389,14 +396,15 @@ class Device : public RefCounted { // model) under the same lock: L1-only ahead of a launch while // l1_dirty_, and ahead of a CMD_MEM_* touching device memory while // either flag is set -- full while llc_dirty_. - vx_result_t cp_ring_append_(const void* cl); + vx_result_t cp_ring_append_(const void* cl, bool llc_clean = false); vx_result_t cp_ring_write_(const void* cl); // Build + submit a CMD_MEM_* descriptor (opcode, dst, src, size). // `physical` sets the CMD_MEM header flag so the CP DMA skips VM // translation — used for page-table writes / the PT region. vx_result_t cp_submit_mem_(uint8_t opcode, uint64_t arg0, uint64_t arg1, - uint64_t arg2, bool physical = false); + uint64_t arg2, bool physical = false, + bool llc_clean = false); // ----- CP-visible host memory (command ring + DMA staging) ----- // host_alloc wraps platform()->host_mem_alloc and records the region @@ -464,6 +472,8 @@ class Device : public RefCounted { // may hold lines the CP DMA (when it reaches device memory beneath it) // would miss or later overwrite. Both flags are guarded by cp_mu_. bool llc_dirty_ = false; + // Full cache flushes appended to the ring so far. + std::atomic llc_epoch_{0}; std::mutex cp_mu_; // serialize ring writes // Seqnum the last command appended to the open batch will reach; // cp_batch_end polls for it once. Touched only by the batch owner, which @@ -512,9 +522,20 @@ class Device : public RefCounted { // Kernel-args scratch pool. Free-list of recycled fixed-size device slots; // ARGS_SLOT_SIZE covers typical kernel arg blocks. Drained in ~Device. + // Each free slot records the flush epoch from which the last-level cache + // holds none of its lines: a kernel that read the slot may have left them + // there until the next full flush. The pool starts with ARGS_POOL_CLEAN + // slots allocated at open, when no kernel has run, so a run of launches + // stages its arguments without flushing until it has used them all; the + // next transfer that needs a full flush makes them clean again. static constexpr uint64_t ARGS_SLOT_SIZE = 4096; + static constexpr uint32_t ARGS_POOL_CLEAN = 32; + struct ArgsSlot { + uint64_t addr; + uint64_t clean_epoch; + }; std::mutex args_pool_mu_; - std::vector args_pool_free_; + std::vector args_pool_free_; }; // ============================================================================ From 398197627cdc3b433dbd710d0020c7f5c7a3eff1 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 01:27:48 -0700 Subject: [PATCH 102/124] simx: feed full-bandwidth ALU and FPU straight from the dispatch queue A full-bandwidth ALU or FPU (as many blocks as issue slots and lanes as the SIMD width) has no lane-dispatch buffer: the RTL unit takes the op straight off the dispatch queue, in the cycle the queue releases it. SimX gave it one more register on the way in. Such a unit now holds only the queue's head and is created after its dispatcher, so it ticks after it. It returns the issue credit itself as it accepts the op, and a resolved branch releases its warp through the result register and then the branch register. The SFU, like the LSU, carries its result buffer and response switch in its output channel, so its commit queue holds two entries and its commit hop is one register. A committed result is written back from the commit arbiter's output register, and issue reads a registered readiness, so the scoreboard releases a register two cycles after commit. Co-Authored-By: Claude Opus 5.5 --- sim/simx/alu_unit.cpp | 18 ++++++++++-- sim/simx/alu_unit.h | 3 ++ sim/simx/core.cpp | 61 +++++++++++++++++++++++++++++------------ sim/simx/dispatcher.cpp | 8 ++++-- sim/simx/dispatcher.h | 5 ++++ sim/simx/fpu_unit.cpp | 14 +++++++++- sim/simx/fpu_unit.h | 4 +++ sim/simx/scoreboard.h | 4 ++- 8 files changed, 93 insertions(+), 24 deletions(-) diff --git a/sim/simx/alu_unit.cpp b/sim/simx/alu_unit.cpp index a48fc3ebe5..e30768ac06 100644 --- a/sim/simx/alu_unit.cpp +++ b/sim/simx/alu_unit.cpp @@ -30,7 +30,16 @@ AluUnit::AluUnit(const SimContext& ctx, const char* name, Core* core) // multiplier keeps one result per cycle. : FuncUnit(ctx, name, core, kMulDivLatency + (kGather ? 1 : 0) + 2) , branch_ctl_out(this, VX_CFG_NUM_WARPS) -{} + , ReleaseOut(this, VX_CFG_ISSUE_WIDTH) +{ + if (!kGather) { + // No buffer between the dispatch queue and the unit: the input holds + // only the queue's head. + for (auto& input : Inputs) { + input = SimChannel(this, 1); + } + } +} // Cycles from this unit's execute to its output, beyond the commit-queue hop. // Execute stands for the integer ALU's registered result (the branch_ctl @@ -572,13 +581,16 @@ void AluUnit::on_tick() { auto trace = input.peek(); this->execute(trace); // A branch releases its warp as it resolves, not when its result - // retires. + // retires: the result register, then the branch register. if (std::get_if(&trace->op_type) && trace->eop && trace->resume_warp) { - branch_ctl_out.send(trace->wid, 1); + branch_ctl_out.send(trace->wid, kGather ? 1 : 2); trace->resume_warp = false; } output.send(trace, this->latency_of(trace) + (kGather ? 1 : 0)); input.pop(); + if (!kGather && trace->eop) { + ReleaseOut.send(trace, 0); + } } } idle &= (input.size() == 0); diff --git a/sim/simx/alu_unit.h b/sim/simx/alu_unit.h index 6e2db38121..c7c11e83ad 100644 --- a/sim/simx/alu_unit.h +++ b/sim/simx/alu_unit.h @@ -30,6 +30,9 @@ class AluUnit : public FuncUnit { // Resolved branch of a stalled warp, registered toward the scheduler. SimChannel branch_ctl_out; + // Full bandwidth: the dispatch queue feeds the unit directly, so an op + // returns its issue credit as the unit accepts it. + SimChannel ReleaseOut; protected: void on_tick() override; diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index 213e9634e9..8742972a59 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -269,16 +269,28 @@ class Core::Impl { icache_flush_buf_->Out.bind(&simobject_->icache_req_out.at(0)); #endif - // initialize execute units - snprintf(sname, 100, "%s-alu", name.c_str()); - func_units_.at((int)FUType::ALU) = SimPlatform::instance().create_object(sname, simobject_); - snprintf(sname, 100, "%s-fpu", name.c_str()); - func_units_.at((int)FUType::FPU) = SimPlatform::instance().create_object(sname, simobject_); + // initialize execute units. A full-bandwidth ALU or FPU takes its op + // straight off the dispatch queue, in the cycle the queue releases it, so + // it is created (and ticks) after its dispatcher; the others take their + // input before the dispatcher fills it in the same cycle. + auto create_alu = [&]() { + snprintf(sname, 100, "%s-alu", name.c_str()); + func_units_.at((int)FUType::ALU) = SimPlatform::instance().create_object(sname, simobject_); + }; + auto create_fpu = [&]() { + snprintf(sname, 100, "%s-fpu", name.c_str()); + func_units_.at((int)FUType::FPU) = SimPlatform::instance().create_object(sname, simobject_); + }; + if (AluUnit::kGather) { + create_alu(); + } + if (FpuUnit::kGather) { + create_fpu(); + } snprintf(sname, 100, "%s-lsu", name.c_str()); func_units_.at((int)FUType::LSU) = SimPlatform::instance().create_object(sname, simobject_); snprintf(sname, 100, "%s-sfu", name.c_str()); func_units_.at((int)FUType::SFU) = SimPlatform::instance().create_object(sname, simobject_); - std::static_pointer_cast(func_units_.at((int)FUType::ALU))->branch_ctl_out.bind(&simobject_->branch_ctl_in); std::static_pointer_cast(func_units_.at((int)FUType::SFU))->warp_ctl_out.bind(&simobject_->warp_ctl_in); scheduler_->barrier_unit().unlock_out.bind(&simobject_->bar_unlock_in); #ifdef VX_CFG_EXT_TCU_ENABLE @@ -310,12 +322,13 @@ class Core::Impl { #endif #endif - // initialize dispatchers after the units they feed, so a unit takes its - // input before the dispatcher fills it in the same cycle. + // initialize dispatchers after the buffered units they feed, so such a + // unit takes its input before the dispatcher fills it in the same cycle. { - // A reduced-width ALU or FPU registers the op once more on its way in. - uint32_t alu_dly = AluUnit::kGather ? 2 : 1; - uint32_t fpu_dly = FpuUnit::kGather ? 2 : 1; + // A reduced-width ALU or FPU takes the op through its lane-dispatch + // buffer; a full-bandwidth one takes it as the queue releases it. + uint32_t alu_dly = AluUnit::kGather ? 2 : 0; + uint32_t fpu_dly = FpuUnit::kGather ? 2 : 0; dispatchers_.at((int)FUType::ALU) = SimPlatform::instance().create_object(name.c_str(), simobject_, VX_CFG_DISPATCH_QUEUE_SIZE, VX_CFG_NUM_ALU_BLOCKS, VX_CFG_NUM_ALU_LANES, alu_dly); dispatchers_.at((int)FUType::FPU) = SimPlatform::instance().create_object(name.c_str(), simobject_, VX_CFG_DISPATCH_QUEUE_SIZE, VX_CFG_NUM_FPU_BLOCKS, VX_CFG_NUM_FPU_LANES, fpu_dly); dispatchers_.at((int)FUType::LSU) = SimPlatform::instance().create_object(name.c_str(), simobject_, VX_CFG_DISPATCH_QUEUE_SIZE, VX_CFG_NUM_LSU_BLOCKS, VX_CFG_NUM_LSU_LANES, 1); @@ -323,6 +336,19 @@ class Core::Impl { #ifdef VX_CFG_EXT_TCU_ENABLE dispatchers_.at((int)FUType::TCU) = SimPlatform::instance().create_object(name.c_str(), simobject_, VX_CFG_DISPATCH_QUEUE_SIZE, VX_CFG_NUM_TCU_BLOCKS, VX_CFG_NUM_TCU_LANES, 1); #endif + if (!AluUnit::kGather) { + create_alu(); + auto& dispatch = dispatchers_.at((int)FUType::ALU); + dispatch->set_release_on_accept(); + std::static_pointer_cast(func_units_.at((int)FUType::ALU))->ReleaseOut.bind(&dispatch->ReleaseOut); + } + if (!FpuUnit::kGather) { + create_fpu(); + auto& dispatch = dispatchers_.at((int)FUType::FPU); + dispatch->set_release_on_accept(); + std::static_pointer_cast(func_units_.at((int)FUType::FPU))->ReleaseOut.bind(&dispatch->ReleaseOut); + } + std::static_pointer_cast(func_units_.at((int)FUType::ALU))->branch_ctl_out.bind(&simobject_->branch_ctl_in); for (uint32_t fu = 0; fu < (uint32_t)FUType::Count; ++fu) { auto& dispatch = dispatchers_.at(fu); auto& func_unit = func_units_.at(fu); @@ -340,10 +366,10 @@ class Core::Impl { for (uint32_t iw = 0; iw < VX_CFG_ISSUE_WIDTH; ++iw) { auto& queues = commit_queues_.emplace_back(); for (uint32_t fu = 0; fu < (uint32_t)FUType::Count; ++fu) { - // A unit's lane gather holds two results. The LSU's crosses one - // register into it; the others cross two and take a third slot so - // that they still retire one result per cycle. - uint32_t capacity = (fu == (uint32_t)FUType::LSU) ? 2 : 3; + // A unit's lane gather holds two results. The LSU's and the SFU's + // cross one register into it; the others cross two and take a third + // slot so that they still retire one result per cycle. + uint32_t capacity = (fu == (uint32_t)FUType::LSU || fu == (uint32_t)FUType::SFU) ? 2 : 3; queues.emplace_back(std::make_unique>(simobject_, capacity)); } } @@ -907,8 +933,9 @@ class Core::Impl { auto& arb_in = *commit_queues_.at(iw).at(fu); // The unit's output registers ahead of the commit arbiter that its // output channel does not already carry: the LSU's response buffer and - // result arbiter are in its channel, leaving its lane gather. - uint32_t commit_delay = (fu == (uint32_t)FUType::LSU) ? 1 : 2; + // result arbiter, and the SFU's result buffer and response switch, are + // in their channels, leaving each its lane gather. + uint32_t commit_delay = (fu == (uint32_t)FUType::LSU || fu == (uint32_t)FUType::SFU) ? 1 : 2; if (arb_in.try_send(trace, commit_delay)) { // Release the warp as soon as its stalling instruction's result leaves // the functional unit — the branch target / fence / warp-control is diff --git a/sim/simx/dispatcher.cpp b/sim/simx/dispatcher.cpp index f8a45998dd..3071ae1667 100644 --- a/sim/simx/dispatcher.cpp +++ b/sim/simx/dispatcher.cpp @@ -118,7 +118,9 @@ void Dispatcher::on_tick() { } else { block_pids_.at(b) = -1; // mark block as processed input.pop(); - ReleaseOut.send(trace, 0); + if (!release_on_accept_) { + ReleaseOut.send(trace, 0); + } ++block_sent; } ThreadMask tmask(VX_CFG_NUM_THREADS); @@ -132,7 +134,9 @@ void Dispatcher::on_tick() { } else { // issue the trace input.pop(); - ReleaseOut.send(trace, 0); + if (!release_on_accept_) { + ReleaseOut.send(trace, 0); + } ++block_sent; } DT(3, this->name() << "-pipeline dispatch: " << *new_trace); diff --git a/sim/simx/dispatcher.h b/sim/simx/dispatcher.h index 3fe1eb9468..3f3aad0cba 100644 --- a/sim/simx/dispatcher.h +++ b/sim/simx/dispatcher.h @@ -33,6 +33,10 @@ class Dispatcher : public SimObject { virtual ~Dispatcher(); + // The unit takes ops straight off the slot queue, so the credit returns as + // the unit accepts the op, from the unit itself. + void set_release_on_accept() { release_on_accept_ = true; } + protected: virtual void on_reset(); virtual void on_tick(); @@ -46,6 +50,7 @@ class Dispatcher : public SimObject { uint32_t batch_idx_; uint32_t out_delay_; std::vector block_pids_; + bool release_on_accept_ = false; friend class SimObject; }; diff --git a/sim/simx/fpu_unit.cpp b/sim/simx/fpu_unit.cpp index 270f7b8a3f..56752b4b41 100644 --- a/sim/simx/fpu_unit.cpp +++ b/sim/simx/fpu_unit.cpp @@ -44,7 +44,16 @@ FpuUnit::FpuUnit(const SimContext& ctx, const char* name, Core* core) // until its result is taken off the response port, however deep the // pipelines are. : FuncUnit(ctx, name, core, VX_CFG_FPU_QUEUE_SIZE) -{} + , ReleaseOut(this, VX_CFG_ISSUE_WIDTH) +{ + if (!kGather) { + // No buffer between the dispatch queue and the unit: the input holds + // only the queue's head. + for (auto& input : Inputs) { + input = SimChannel(this, 1); + } + } +} uint32_t FpuUnit::latency_of(const instr_trace_t* trace) const { auto fpu_type = std::get(trace->op_type); @@ -390,6 +399,9 @@ void FpuUnit::on_tick() { uint32_t delay = this->latency_of(trace) + (kGather ? 0 : 1); output.send(trace, delay); input.pop(); + if (!kGather && trace->eop) { + ReleaseOut.send(trace, 0); + } } } idle &= (input.size() == 0); diff --git a/sim/simx/fpu_unit.h b/sim/simx/fpu_unit.h index 4de1f44f72..ef0dcd11a7 100644 --- a/sim/simx/fpu_unit.h +++ b/sim/simx/fpu_unit.h @@ -26,6 +26,10 @@ class FpuUnit : public FuncUnit { static constexpr bool kGather = (VX_CFG_NUM_FPU_BLOCKS != VX_CFG_ISSUE_WIDTH) || (VX_CFG_NUM_FPU_LANES != VX_CFG_SIMD_WIDTH); + // Full bandwidth: the dispatch queue feeds the unit directly, so an op + // returns its issue credit as the unit accepts it. + SimChannel ReleaseOut; + protected: void on_tick() override; diff --git a/sim/simx/scoreboard.h b/sim/simx/scoreboard.h index e05ade6479..d5514bc654 100644 --- a/sim/simx/scoreboard.h +++ b/sim/simx/scoreboard.h @@ -69,7 +69,9 @@ class Scoreboard : public SimObject { return (wid << RegOpd::ID_BITS) | reg.id(); } - static constexpr uint32_t kReleaseDelay = 1; + // A committed result is written back from the commit arbiter's output + // register, and the issue stage reads a registered readiness. + static constexpr uint32_t kReleaseDelay = 2; struct pending_release_t { uint32_t wid; From 03e8304a2507eba9cb6f808ced740bfa476f45ac Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 01:27:49 -0700 Subject: [PATCH 103/124] lsu: split memory fences into acquire and release, order .aq/.rl AMOs Every memory fence sent a full L1 flush token, which both drains earlier writes and invalidates the cache. The fences the RISC-V mappings emit usually need only half of that. A load-acquire's fence r,rw needs only later reads to see other harts' writes (invalidate). A store-release's fence rw,w needs only earlier writes to be visible (drain). Decode now derives the kind from the pred/succ sets. Acquire applies when the successor set reads, release when the predecessor set writes or the successor set does not read. A device access in the predecessor set, or fence.i, does both. The token carries the kind in its attributes (acquire: io with byte enables set; release: noalloc). VX_cache_init: - An acquire walks above the point of coherence and forwards only when the cache below may also hold stale lines (ACQ_FORWARD: an L2 that is not the shared LLC). - A release only quiesces the banks and forwards a release token behind the earlier writes. - Kinds visible together combine to the weakest one that serves them all. Atomics with .aq/.rl were treated as sequentially consistent, which nothing enforced. A new uop expander (VX_uop_amo; LsuUopGen in SimX) places a release fence before an atomic with .rl and an acquire fence after one with .aq. The RTL and SimX change together. fence_mp -m selects how the flag is published and observed: 0 full fences, 1 acquire/release fences, 2 amoswap.rl / amoor.aq. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/VX_gpu_pkg.sv | 14 ++++-- hw/rtl/VX_socket.sv | 1 + hw/rtl/cache/VX_cache.sv | 5 +++ hw/rtl/cache/VX_cache_cluster.sv | 2 + hw/rtl/cache/VX_cache_init.sv | 64 +++++++++++++++++++++------- hw/rtl/cache/VX_cache_wrap.sv | 2 + hw/rtl/core/VX_decode.sv | 7 +++ hw/rtl/core/VX_lsu_block.sv | 11 +++-- hw/rtl/core/VX_uop_amo.sv | 63 +++++++++++++++++++++++++++ hw/rtl/core/VX_uop_sequencer.sv | 19 +++++++++ sim/simx/constants.h | 3 ++ sim/simx/decode.cpp | 26 +++++++++-- sim/simx/lsu_unit.cpp | 38 +++++++++++++++-- sim/simx/mem/cache.cpp | 62 ++++++++++++++++++++------- sim/simx/mem/cache.h | 1 + sim/simx/socket.cpp | 1 + sim/simx/types.h | 11 +++-- tests/regression/fence_mp/common.h | 9 +++- tests/regression/fence_mp/kernel.cpp | 54 ++++++++++++++++++----- tests/regression/fence_mp/main.cpp | 7 ++- 20 files changed, 338 insertions(+), 62 deletions(-) create mode 100644 hw/rtl/core/VX_uop_amo.sv diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index 9e874674d1..db8c1a2e6a 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -98,7 +98,9 @@ package VX_gpu_pkg; // owning one: the two are combinational rewrites of ibuf_in with mutually // exclusive selects (EX_LSU + export_mask vs EX_SFU + INST_SFU_RTUW). localparam UOP_GFX = UOP_TCU + `VX_CFG_EXT_TCU_ENABLED; - localparam UOP_MAX = UOP_GFX + `EXT_GFX_ANY_ENABLED; + // atomics with .aq/.rl expand into fences around the atomic + localparam UOP_AMO = UOP_GFX + `EXT_GFX_ANY_ENABLED; + localparam UOP_MAX = UOP_AMO + `VX_CFG_EXT_A_ENABLED; localparam UOP_CTR_W = 8; localparam CTA_TID_WIDTH = `UP(NW_BITS + NT_BITS); @@ -1120,8 +1122,11 @@ package VX_gpu_pkg; // lsu_args_t carries AMO sideband when EXT_A_ENABLE. // amo_valid / amo_op are inspected only when amo_valid==1; for plain - // loads/stores they're zero. aq/rl are decoded but unused (sequentially - // consistent). amo_unsigned distinguishes the U-variants of MIN/MAX. + // loads/stores they're zero. aq/rl order the AMO with the hart's other + // accesses (fence uops around it); on a memory fence they say what it + // orders: aq = later reads see others' writes (acquire: invalidate), + // rl = earlier writes are visible (release: drain). amo_unsigned + // distinguishes the U-variants of MIN/MAX. // export_mask marks a vx_om_export (OM fragment export) and says which // words of the aperture record the shader is writing: // bit 0 = colour (record + 0), bit 1 = depth (record + 4) @@ -2004,6 +2009,9 @@ package VX_gpu_pkg; localparam L2_IS_LLC = `VX_CFG_L2_ENABLED && !`VX_CFG_L3_ENABLED; // Every memory client of a cluster reaches memory through its L2. localparam L2_SHARED_LLC = L2_IS_LLC && (`VX_CFG_NUM_CLUSTERS == 1); + // An acquire must also invalidate the cache below the dcache unless that + // one is the point of coherence (an L3 always is; memory always is). + localparam DCACHE_ACQ_FORWARD = `VX_CFG_L2_ENABLED && !L2_SHARED_LLC; /////////////////////////////// L3 Parameters ///////////////////////////// diff --git a/hw/rtl/VX_socket.sv b/hw/rtl/VX_socket.sv index 3734c2771d..a8fc37e113 100644 --- a/hw/rtl/VX_socket.sv +++ b/hw/rtl/VX_socket.sv @@ -234,6 +234,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .MEM_OUT_BUF (3), .IS_LLC (DCACHE_IS_LLC), .SHARED_LLC (DCACHE_SHARED_LLC), + .ACQ_FORWARD (DCACHE_ACQ_FORWARD), .BANK_HASH (`VX_CFG_DCACHE_BANK_HASH), .READ_MERGE (`VX_CFG_DCACHE_READ_MERGE ? DCACHE_CHANNELS : 0), .AMO_ENABLE (`VX_CFG_EXT_A_ENABLED) diff --git a/hw/rtl/cache/VX_cache.sv b/hw/rtl/cache/VX_cache.sv index fb54750a87..86814ef6eb 100644 --- a/hw/rtl/cache/VX_cache.sv +++ b/hw/rtl/cache/VX_cache.sv @@ -54,6 +54,7 @@ module VX_cache import VX_gpu_pkg::*; #( parameter IS_LLC = 0, // last-level cache: banks own AMO commit + RVA reservation tracking parameter SHARED_LLC = 0, // the LLC serves every memory client alone (point of coherence) + parameter ACQ_FORWARD = 0, // an acquire flush also invalidates the next level parameter FLUSH_FORWARD = 1, // a flush also flushes the next level @@ -144,6 +145,7 @@ module VX_cache import VX_gpu_pkg::*; #( wire [MEM_PORTS-1:0] mem_flush_valid; wire mem_flush_l1_only; wire mem_flush_io; + wire mem_flush_noalloc; wire [MEM_PORTS-1:0] mem_flush_ready; wire mem_flush_rsp_phase; wire [MEM_PORTS-1:0] mem_flush_rsp_valid; @@ -157,6 +159,7 @@ module VX_cache import VX_gpu_pkg::*; #( .MEM_PORTS (MEM_PORTS), .FLUSH_FORWARD (FLUSH_FORWARD), .SHARED_LLC (SHARED_LLC), + .ACQ_FORWARD (ACQ_FORWARD), .TAG_WIDTH (TAG_WIDTH), .XBAR_SIZE (REQ_XBAR_SIZE), .REQ_SEL_WIDTH (REQ_SEL_WIDTH) @@ -176,6 +179,7 @@ module VX_cache import VX_gpu_pkg::*; #( .mem_flush_valid (mem_flush_valid), .mem_flush_l1_only (mem_flush_l1_only), .mem_flush_io (mem_flush_io), + .mem_flush_noalloc (mem_flush_noalloc), .mem_flush_ready (mem_flush_ready), .mem_flush_rsp_phase (mem_flush_rsp_phase), .mem_flush_rsp_valid (mem_flush_rsp_valid), @@ -675,6 +679,7 @@ module VX_cache import VX_gpu_pkg::*; #( `CS_SECTOR_WIDTH'(0), BANK_MEM_TAG_WIDTH'(0), (`UP(MEM_ATTR_WIDTH)'(1) << MEM_ATTR_FLUSH_OFFS) | (`UP(MEM_ATTR_WIDTH)'(mem_flush_io) << MEM_ATTR_IO_OFFS) + | (`UP(MEM_ATTR_WIDTH)'(mem_flush_noalloc) << MEM_ATTR_NOALLOC_OFFS) }; assign mem_req_xbar_valid[i] = per_bank_mem_req_valid[i] || mem_flush_valid[i]; assign mem_req_xbar_pdata[i] = per_bank_mem_req_valid[i] ? per_bank_mem_req_pdata[i] : mem_flush_pdata; diff --git a/hw/rtl/cache/VX_cache_cluster.sv b/hw/rtl/cache/VX_cache_cluster.sv index a8d0f99a44..c7553e07e9 100644 --- a/hw/rtl/cache/VX_cache_cluster.sv +++ b/hw/rtl/cache/VX_cache_cluster.sv @@ -59,6 +59,7 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( parameter IS_LLC = 0, // Indicates this cache cluster is the LLC (handles AMO completion). parameter SHARED_LLC = 0, // The LLC serves every memory client alone (point of coherence). + parameter ACQ_FORWARD = 0, // An acquire flush also invalidates the next level. parameter FLUSH_FORWARD = 1, // A flush also flushes the next level. parameter BANK_HASH = 0, // Hashed bank select (cache_bank_hash). parameter READ_MERGE = 0, // Merge same-word reads within groups of this many adjacent ports (0: off). @@ -184,6 +185,7 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( .PASSTHRU (PASSTHRU), .IS_LLC (IS_LLC), .SHARED_LLC (SHARED_LLC), + .ACQ_FORWARD (ACQ_FORWARD), .FLUSH_FORWARD (FLUSH_FORWARD), .BANK_HASH (BANK_HASH), .READ_MERGE (READ_MERGE), diff --git a/hw/rtl/cache/VX_cache_init.sv b/hw/rtl/cache/VX_cache_init.sv index 28439232d6..eb265e5ca9 100644 --- a/hw/rtl/cache/VX_cache_init.sv +++ b/hw/rtl/cache/VX_cache_init.sv @@ -38,6 +38,12 @@ // nowhere. Once nothing is held between the cache's inputs and its memory // ports, on the cacheable and the non-cacheable path alike, it goes down every // memory port, so its answer proves the earlier IO writes reached memory. +// The L1-only kind splits by what the fence orders. A release token (noalloc +// attribute) owes no invalidation: the banks only quiesce and the token goes +// down to the point of coherence behind the earlier writes. An acquire token +// (io attribute with byte enables set) owes no drain: the cache walks and +// answers, forwarding only to a cache below that is not the point of +// coherence (ACQ_FORWARD). // Kinds visible together combine to the weakest one that serves them all. module VX_cache_init import VX_gpu_pkg::*; #( parameter NUM_REQS = 4, // Number of Word requests per cycle @@ -45,6 +51,7 @@ module VX_cache_init import VX_gpu_pkg::*; #( parameter MEM_PORTS = 1, // Number of memory ports parameter FLUSH_FORWARD = 1, // Forward the flush to the next level parameter SHARED_LLC = 0, // The LLC serving every memory client alone: answers L1-only tokens without a walk + parameter ACQ_FORWARD = 0, // An acquire token must also invalidate the next level parameter TAG_WIDTH = UUID_WIDTH + 1, // core request tag size parameter XBAR_SIZE = 0, // Requests the request crossbar can hold apart from input order parameter REQ_SEL_WIDTH = `UP(`CLOG2(NUM_REQS)) @@ -65,6 +72,7 @@ module VX_cache_init import VX_gpu_pkg::*; #( output wire [MEM_PORTS-1:0] mem_flush_valid, output wire mem_flush_l1_only, output wire mem_flush_io, + output wire mem_flush_noalloc, input wire [MEM_PORTS-1:0] mem_flush_ready, output wire mem_flush_rsp_phase, input wire [MEM_PORTS-1:0] mem_flush_rsp_valid, @@ -126,18 +134,27 @@ module VX_cache_init import VX_gpu_pkg::*; #( end wire flush_req_enable = (| flush_req_mask); - wire [NUM_REQS-1:0] flush_l1_mask, flush_io_mask; + wire [NUM_REQS-1:0] flush_l1_mask, flush_io_mask, flush_acq_mask, flush_rel_mask; for (genvar i = 0; i < NUM_REQS; ++i) begin : g_flush_kind_mask - assign flush_io_mask[i] = flush_req_mask[i] && core_bus_in_if[i].req_data.attr[MEM_ATTR_IO_OFFS]; - assign flush_l1_mask[i] = flush_req_mask[i] && ~flush_io_mask[i] && (| core_bus_in_if[i].req_data.byteen); + wire tok_io = core_bus_in_if[i].req_data.attr[MEM_ATTR_IO_OFFS]; + wire tok_rel = core_bus_in_if[i].req_data.attr[MEM_ATTR_NOALLOC_OFFS]; + wire tok_l1 = (| core_bus_in_if[i].req_data.byteen); + assign flush_io_mask[i] = flush_req_mask[i] && tok_io && ~tok_l1; + assign flush_acq_mask[i] = flush_req_mask[i] && tok_io && tok_l1; + assign flush_rel_mask[i] = flush_req_mask[i] && ~tok_io && tok_l1 && tok_rel; + assign flush_l1_mask[i] = flush_req_mask[i] && ~tok_io && tok_l1 && ~tok_rel; end - wire any_full = (| (flush_req_mask & ~flush_l1_mask & ~flush_io_mask)); + wire any_full = (| (flush_req_mask & ~flush_l1_mask & ~flush_io_mask & ~flush_acq_mask & ~flush_rel_mask)); wire any_l1 = (| flush_l1_mask); wire any_io = (| flush_io_mask); - // every visible token is L1-only - wire flush_l1_only = any_l1 && ~any_full && ~any_io; - // a walk is owed to a full token, and to an L1-only one above the point of coherence - wire flush_walk = any_full || (any_l1 && (SHARED_LLC == 0)); + wire any_acq = (| flush_acq_mask); + wire any_rel = (| flush_rel_mask); + // a walk is owed to a full token, and to an L1-only or acquire one above + // the point of coherence + wire flush_walk = any_full || ((any_l1 || any_acq) && (SHARED_LLC == 0)); + // a drain with no walk is owed to an IO token, and to a release one above + // the point of coherence + wire flush_drain = any_io || (any_rel && (SHARED_LLC == 0)); reg [NUM_BANKS-1:0] flush_done, flush_done_n; reg fwd_go, fwd_go_n; // the crossbar drained; only tokens enter it now @@ -147,6 +164,7 @@ module VX_cache_init import VX_gpu_pkg::*; #( reg [`UP(UUID_WIDTH)-1:0] flush_uuid_r, flush_uuid_n; reg l1_only_r, l1_only_n; reg io_r, io_n; + reg noalloc_r, noalloc_n; reg skip_r, skip_n; // answer the lowest pending input; its token stays at the input until then @@ -230,7 +248,13 @@ module VX_cache_init import VX_gpu_pkg::*; #( // an IO token also waits for the non-cacheable path, which carries the IO // writes it orders - wire fwd_ready = mem_req_empty && (~io_r || nc_req_empty); + wire fwd_ready = mem_req_empty && (~io_r || l1_only_r || nc_req_empty); + + // The kind forwarded after a walk: full if a full token or an IO one asks + // for more than the point of coherence, L1-only if any token owes a drain + // to it, acquire if only acquire tokens were visible. + wire walk_fwd_acq = any_acq && ~any_full && ~any_io && ~any_l1 && ~any_rel; + wire walk_fwd_l1 = (any_l1 || any_rel) && ~any_full && ~any_io; always @(*) begin state_n = state; @@ -242,6 +266,7 @@ module VX_cache_init import VX_gpu_pkg::*; #( flush_uuid_n = flush_uuid_r; l1_only_n = l1_only_r; io_n = io_r; + noalloc_n = noalloc_r; skip_n = skip_r; case (state) // STATE_IDLE: @@ -249,7 +274,7 @@ module VX_cache_init import VX_gpu_pkg::*; #( if (flush_req_enable) begin if (XBAR_SIZE != 0) begin state_n = STATE_WAIT1; - end else if (~flush_walk && ~any_io) begin + end else if (~flush_walk && ~flush_drain) begin rsp_pending_n = flush_req_mask; state_n = STATE_RESPOND; end else begin @@ -265,7 +290,7 @@ module VX_cache_init import VX_gpu_pkg::*; #( end STATE_WAIT1: begin if (no_inflight_reqs) begin - if (~flush_walk && ~any_io) begin + if (~flush_walk && ~flush_drain) begin rsp_pending_n = flush_req_mask; state_n = STATE_RESPOND; end else begin @@ -284,12 +309,17 @@ module VX_cache_init import VX_gpu_pkg::*; #( if (flush_done_n == {NUM_BANKS{1'b1}}) begin flush_done_n = '0; // answer the tokens visible now; later ones start a new walk. - // A quiesce without a walk serves only the tokens owing none. - rsp_pending_n = skip_r ? (flush_io_mask | (SHARED_LLC ? flush_l1_mask : '0)) + // A quiesce without a walk serves only the tokens owing none, + // and forwards IO if it serves an IO token, else release. + rsp_pending_n = skip_r ? (flush_io_mask | flush_rel_mask + | (SHARED_LLC ? (flush_l1_mask | flush_acq_mask) : '0)) : flush_req_mask; - l1_only_n = flush_l1_only && ~skip_r; - io_n = skip_r; - state_n = FLUSH_FORWARD ? STATE_FORWARD : STATE_RESPOND; + l1_only_n = skip_r ? ~any_io : (walk_fwd_l1 || walk_fwd_acq); + io_n = skip_r ? any_io : walk_fwd_acq; + noalloc_n = skip_r && ~any_io; + // an acquire walk owes the next level nothing unless it too + // may hold stale lines + state_n = (FLUSH_FORWARD && (skip_r || ~walk_fwd_acq || ACQ_FORWARD)) ? STATE_FORWARD : STATE_RESPOND; end end STATE_FORWARD: begin @@ -330,6 +360,7 @@ module VX_cache_init import VX_gpu_pkg::*; #( flush_uuid_r <= flush_uuid_n; l1_only_r <= l1_only_n; io_r <= io_n; + noalloc_r <= noalloc_n; skip_r <= skip_n; end @@ -343,5 +374,6 @@ module VX_cache_init import VX_gpu_pkg::*; #( assign mem_flush_rsp_phase = (state == STATE_FORWARD); assign mem_flush_l1_only = l1_only_r; assign mem_flush_io = io_r; + assign mem_flush_noalloc = noalloc_r; endmodule diff --git a/hw/rtl/cache/VX_cache_wrap.sv b/hw/rtl/cache/VX_cache_wrap.sv index 07b7e2b310..597e63a042 100644 --- a/hw/rtl/cache/VX_cache_wrap.sv +++ b/hw/rtl/cache/VX_cache_wrap.sv @@ -61,6 +61,7 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( parameter IS_LLC = 0, // Set when this cache is the LLC (enables AMO completion at this level). parameter SHARED_LLC = 0, // The LLC serves every memory client alone (point of coherence). + parameter ACQ_FORWARD = 0, // An acquire flush also invalidates the next level. parameter FLUSH_FORWARD = 1, // A flush also flushes the next level. parameter BANK_HASH = 0, // Hashed bank select (cache_bank_hash). parameter READ_MERGE = 0, // Merge same-word reads within groups of this many adjacent ports (0: off). @@ -165,6 +166,7 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( .MEM_OUT_BUF (BYPASS_ENABLE ? 1 : MEM_OUT_BUF), .IS_LLC (IS_LLC), .SHARED_LLC (SHARED_LLC), + .ACQ_FORWARD (ACQ_FORWARD), .FLUSH_FORWARD (FLUSH_FORWARD), .BANK_HASH (BANK_HASH), .READ_MERGE (READ_MERGE), diff --git a/hw/rtl/core/VX_decode.sv b/hw/rtl/core/VX_decode.sv index 538164a70a..a3c30a5b67 100644 --- a/hw/rtl/core/VX_decode.sv +++ b/hw/rtl/core/VX_decode.sv @@ -357,6 +357,13 @@ module VX_decode import op_args.lsu.is_float = 0; op_args.lsu.pack = 0; op_args.lsu.offset = 0; + // A later read must see other harts' data (acquire) when the + // successor set reads; earlier writes must be visible (release) + // when the predecessor set writes. A fence with neither + // releases; one with a device access in its predecessor set, + // or fence.i, does both. + op_args.lsu.amo_aq = (funct3 != 3'b000) || (instr[27:26] != 2'b00) || instr[21]; + op_args.lsu.amo_rl = (funct3 != 3'b000) || (instr[27:26] != 2'b00) || instr[24] || ~instr[21]; end `ifdef VX_CFG_EXT_A_ENABLE INST_AMO: begin diff --git a/hw/rtl/core/VX_lsu_block.sv b/hw/rtl/core/VX_lsu_block.sv index c43935bc21..46412976c2 100644 --- a/hw/rtl/core/VX_lsu_block.sv +++ b/hw/rtl/core/VX_lsu_block.sv @@ -95,6 +95,10 @@ module VX_lsu_block import VX_gpu_pkg::*; #( wire req_is_fence, rsp_is_fence; wire req_fence_io = inst_lsu_is_fence_io(execute_if.data.op_type); + // a memory fence that orders only later reads (acquire) or only earlier + // writes (release) + wire req_fence_acq = ~req_fence_io && ~execute_if.data.op_args.lsu.amo_rl; + wire req_fence_rel = ~req_fence_io && ~execute_if.data.op_args.lsu.amo_aq; wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] full_addr; for (genvar i = 0; i < NUM_LANES; ++i) begin : g_full_addr @@ -119,7 +123,8 @@ module VX_lsu_block import VX_gpu_pkg::*; #( wire [MEM_ADDRW-1:0] io_addr_start = MEM_ADDRW'(`VX_CFG_XLEN'(`VX_MEM_IO_BASE_ADDR) >> MEM_ASHIFT); wire [MEM_ADDRW-1:0] io_addr_end = MEM_ADDRW'(`VX_CFG_XLEN'(`VX_MEM_IO_END_ADDR) >> MEM_ASHIFT); assign mem_req_attr_struct[i].is_flush = req_is_fence; - assign mem_req_attr_struct[i].is_noalloc = 1'b0; + // a release fence's token: no invalidation owed + assign mem_req_attr_struct[i].is_noalloc = req_is_fence && req_fence_rel; `ifdef VX_CFG_EXT_OM_ENABLE // OM fragment-export aperture. // @@ -139,8 +144,8 @@ module VX_lsu_block import VX_gpu_pkg::*; #( assign mem_req_attr_struct[i].is_addr_om = 1'b0; `endif // a fence's token carries its kind: io for a fence ordering only device - // accesses - assign mem_req_attr_struct[i].is_addr_io = req_is_fence ? req_fence_io + // accesses, and for an acquire fence's (byte enables set: no drain owed) + assign mem_req_attr_struct[i].is_addr_io = req_is_fence ? (req_fence_io || req_fence_acq) : (((block_addr >= io_addr_start) && (block_addr < io_addr_end)) || lane_is_om); `ifdef VX_CFG_LMEM_ENABLE diff --git a/hw/rtl/core/VX_uop_amo.sv b/hw/rtl/core/VX_uop_amo.sv new file mode 100644 index 0000000000..c0e54dd216 --- /dev/null +++ b/hw/rtl/core/VX_uop_amo.sv @@ -0,0 +1,63 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// VX_uop_amo: orders an atomic with the hart's other accesses. +// +// An AMO, LR or SC with .rl is preceded by a release fence (earlier writes +// become visible first) and one with .aq is followed by an acquire fence +// (later reads see what the atomic observed): +// [release fence] atomic [acquire fence] +// The fences are plain LSU fence ops whose aq/rl arguments carry their kind; +// they read and write no register. + +module VX_uop_amo import VX_gpu_pkg::*; ( + input clk, + input reset, + + input ibuffer_t ibuf_in, + output ibuffer_t ibuf_out, + + input wire start, + input wire advance, + input wire [UOP_CTR_W-1:0] uop_idx, + output wire [UOP_CTR_W-1:0] uop_count +); + `UNUSED_VAR ({clk, reset, start, advance}) + + wire aq = ibuf_in.op_args.lsu.amo_aq; + wire rl = ibuf_in.op_args.lsu.amo_rl; + assign uop_count = UOP_CTR_W'(1) + UOP_CTR_W'(aq) + UOP_CTR_W'(rl); + + wire [UOP_CTR_W-1:0] amo_idx = UOP_CTR_W'(rl); + wire is_rel = rl && (uop_idx == '0); + wire is_acq = aq && (uop_idx == amo_idx + UOP_CTR_W'(1)); + + ibuffer_t ibuf_r; + always_comb begin + ibuf_r = ibuf_in; + if (is_rel || is_acq) begin + ibuf_r.op_type = INST_OP_BITS'(INST_LSU_FENCE); + ibuf_r.op_args = '0; + ibuf_r.op_args.lsu.amo_aq = is_acq; + ibuf_r.op_args.lsu.amo_rl = is_rel; + ibuf_r.wb = 1'b0; + ibuf_r.rd_xregs = '0; + ibuf_r.wr_xregs = '0; + ibuf_r.used_rs = '0; + end + end + assign ibuf_out = ibuf_r; + +endmodule diff --git a/hw/rtl/core/VX_uop_sequencer.sv b/hw/rtl/core/VX_uop_sequencer.sv index 268450164c..3fb0ffcf49 100644 --- a/hw/rtl/core/VX_uop_sequencer.sv +++ b/hw/rtl/core/VX_uop_sequencer.sv @@ -193,6 +193,25 @@ module VX_uop_sequencer import ); `endif +`ifdef VX_CFG_EXT_A_ENABLE + // ------------------------------------------------------------------ + // Atomic ordering uop expander: fences around an AMO/LR/SC with .aq/.rl + // ------------------------------------------------------------------ + assign uop_in_valid[UOP_AMO] = (uop_in_data.ex_type == EX_LSU) + && uop_in_data.op_args.lsu.amo_valid + && (uop_in_data.op_args.lsu.amo_aq || uop_in_data.op_args.lsu.amo_rl); + VX_uop_amo uop_amo ( + .clk (clk), + .reset (reset), + .ibuf_in (uop_in_data), + .start (uop_in_start[UOP_AMO]), + .advance (uop_in_next[UOP_AMO]), + .uop_idx (uop_ctr), + .ibuf_out (uop_out_data[UOP_AMO]), + .uop_count (uop_out_count[UOP_AMO]) + ); +`endif + wire uop_hold = is_uop_input && ~uop_active; assign output_if.valid = uop_active || (input_if.valid && ~uop_hold); diff --git a/sim/simx/constants.h b/sim/simx/constants.h index a1cf857579..941eeddea2 100644 --- a/sim/simx/constants.h +++ b/sim/simx/constants.h @@ -52,6 +52,9 @@ inline constexpr bool DCACHE_SHARED_LLC = (VX_CFG_DCACHE_ENABLED != 0) && (VX_CF && !VX_CFG_EXT_DXA_ENABLED; inline constexpr bool L2_SHARED_LLC = (VX_CFG_L2_ENABLED != 0) && (VX_CFG_L3_ENABLED == 0) && (VX_CFG_NUM_CLUSTERS == 1); inline constexpr bool L3_SHARED_LLC = (VX_CFG_L3_ENABLED != 0); +// An acquire must also invalidate the cache below the dcache unless that one +// is the point of coherence (an L3 always is; memory always is). +inline constexpr bool DCACHE_ACQ_FORWARD = (VX_CFG_L2_ENABLED != 0) && !L2_SHARED_LLC; // A write-back LLC must be the only instance: replicated ones never make their // dirty lines visible to each other. diff --git a/sim/simx/decode.cpp b/sim/simx/decode.cpp index e0a3d8593e..5a559d4726 100644 --- a/sim/simx/decode.cpp +++ b/sim/simx/decode.cpp @@ -631,10 +631,24 @@ Instr::Ptr Decoder::decode(uint32_t code, uint64_t uuid) { case Opcode::FENCE: { instr->set_fu_type(FUType::LSU); instr->set_op_type(LsuType::FENCE); - // neither the predecessor nor the successor set holds a memory read or - // write (pred = code[27:24], succ = code[23:20], IORW) - uint32_t fence_io = (funct3 == 0) && ((code >> 24) & 0x3) == 0 && ((code >> 20) & 0x3) == 0; - instr->set_args(IntrLsuArgs{fence_io, 0, 0}); + // pred = code[27:24], succ = code[23:20], IORW. A fence with no memory + // read or write in either set orders only device accesses. Otherwise a + // later read must see other harts' data (acquire) when the successor set + // reads, and earlier writes must be visible (release) when the + // predecessor set writes. A fence with neither releases; one with a + // device access in its predecessor set, or fence.i, does both. + uint32_t pred = (code >> 24) & 0xf; + uint32_t succ = (code >> 20) & 0xf; + FenceKind kind; + if (funct3 == 0 && (pred & 0x3) == 0 && (succ & 0x3) == 0) { + kind = FenceKind::IO; + } else { + bool full = (funct3 != 0) || (pred & 0xc) != 0; + bool aq = full || (succ & 0x2) != 0; + bool rl = full || (pred & 0x1) != 0 || (succ & 0x2) == 0; + kind = (aq && rl) ? FenceKind::FULL : (aq ? FenceKind::ACQUIRE : FenceKind::RELEASE); + } + instr->set_args(IntrLsuArgs{static_cast(kind), 0, 0}); } break; case Opcode::AMO: { instr->set_fu_type(FUType::LSU); @@ -657,6 +671,10 @@ Instr::Ptr Decoder::decode(uint32_t code, uint64_t uuid) { std::abort(); } instr->set_args(IntrAmoArgs{funct3, aq, rl}); + // .aq/.rl order the atomic with fences around it (LsuUopGen) + if (aq || rl) { + instr->set_macro_op(); + } instr->set_dest_reg(rd, RegType::Integer); instr->set_src_reg(0, rs1, RegType::Integer); instr->set_src_reg(1, rs2, RegType::Integer); diff --git a/sim/simx/lsu_unit.cpp b/sim/simx/lsu_unit.cpp index 7a887721f7..22b4a9a0ab 100644 --- a/sim/simx/lsu_unit.cpp +++ b/sim/simx/lsu_unit.cpp @@ -29,6 +29,10 @@ using namespace vortex; uint32_t LsuUopGen::uop_count(const Instr& instr) { + // An atomic with .aq/.rl: [release fence] atomic [acquire fence] + if (auto amo = std::get_if(&instr.get_args())) { + return 1 + amo->aq + amo->rl; + } // PACKLB.F: width=0 (LB), 4 byte-elements // PACKLH.F: width=1 (LH), 2 halfword-elements auto args = std::get(instr.get_args()); @@ -46,6 +50,30 @@ Instr::Ptr LsuUopGen::get(const Instr& macro_instr, uint32_t uop_index) { uint32_t steps_shift = (total > 1) ? (32 - log2ceil(total)) : 0; uint64_t uop_uuid = (static_cast(uuid_hi) << 32) | ((uop_index << steps_shift) | uuid_lo); + if (auto amo = std::get_if(¯o_instr.get_args())) { + auto uop_instr = std::allocate_shared(pool_, uop_uuid, FUType::LSU); + uop_instr->set_parent_uuid(parent_uuid); + uint32_t amo_index = amo->rl; + if (uop_index == amo_index) { + uop_instr->set_op_type(macro_instr.get_op_type()); + uop_instr->set_args(*amo); + // an atomic into x0 writes nothing back + auto rd = macro_instr.get_dest_reg(); + if (!(rd.type == RegType::Integer && rd.idx == 0)) { + uop_instr->set_dest_reg(rd.idx, rd.type); + } + for (uint32_t i = 0; i < Instr::MAX_REG_SOURCES; ++i) { + auto rs = macro_instr.get_src_reg(i); + uop_instr->set_src_reg(i, rs.idx, rs.type); + } + } else { + auto kind = (uop_index < amo_index) ? FenceKind::RELEASE : FenceKind::ACQUIRE; + uop_instr->set_op_type(LsuType::FENCE); + uop_instr->set_args(IntrLsuArgs{static_cast(kind), 0, 0}); + } + return uop_instr; + } + auto args = std::get(macro_instr.get_args()); // Macro encodes elem-size as width: 0=byte (PACKLB), 1=halfword (PACKLH). // The uop is a regular unsigned load (LBU/LHU = width + 4) so the LSU @@ -375,10 +403,12 @@ void LsuUnit::send_fence_token(uint32_t b) { LsuReq lsu_req(VX_CFG_NUM_LSU_LANES); lsu_req.op = MemOp::LD; lsu_req.flags.flush = 1; - // an IO fence's token walks nowhere; a memory fence's stops at the shared LLC - auto fence_args = std::get(trace->instr_ptr->get_args()); - lsu_req.flags.io = lsu_fence_is_io(fence_args); - lsu_req.flags.flush_l1 = !lsu_fence_is_io(fence_args); + // an IO fence's token walks nowhere; a memory fence's stops at the shared + // LLC, an acquire one owing no drain and a release one no invalidation + auto kind = lsu_fence_kind(std::get(trace->instr_ptr->get_args())); + lsu_req.flags.io = (kind == FenceKind::IO) || (kind == FenceKind::ACQUIRE); + lsu_req.flags.flush_l1 = (kind != FenceKind::IO); + lsu_req.flags.noalloc = (kind == FenceKind::RELEASE); std::vector lane_entries(VX_CFG_NUM_LSU_LANES); for (uint32_t t = base; t < VX_CFG_NUM_THREADS; ++t) { if (trace->tmask.test(t)) { diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index ccf1a1d6ba..77b3fc79eb 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -2371,8 +2371,8 @@ class CachePortArb : public SimObject { return mask; } - // Tokens owing no walk: IO tokens, and L1-only ones unless they still need - // a walk here. + // Tokens owing no walk: IO and release tokens, and L1-only or acquire ones + // unless they still need a walk here. std::vector tokens(bool l1_needs_walk) const { std::vector mask(ReqIn.size(), false); for (uint32_t i = 0, n = ReqIn.size(); i < n; ++i) { @@ -2380,7 +2380,9 @@ class CachePortArb : public SimObject { continue; } auto &flags = ReqIn.at(i).peek().flags; - mask.at(i) = flags.io || (flags.flush_l1 && !l1_needs_walk); + bool io = flags.io && !flags.flush_l1; + bool rel = !flags.io && flags.flush_l1 && flags.noalloc; + mask.at(i) = io || rel || (flags.flush_l1 && !l1_needs_walk); } return mask; } @@ -2986,13 +2988,13 @@ class Cache::Impl { } auto kinds = this->token_kinds(); bool walk = kinds.walk(config_.shared_llc); - if (!walk && !kinds.any_io) { + if (!walk && !kinds.drain(config_.shared_llc)) { rsp_pending_ = port_arb_->tokens(); flush_state_ = FlushState::Respond; break; } - // with no walk owed, an IO token still quiesces the banks and goes down - // to memory + // with no walk owed, an IO or release token still quiesces the banks + // and goes down walk_skip_ = !walk; for (auto &bank : banks_) { bank->token_flush_begin(walk_skip_); @@ -3007,10 +3009,23 @@ class Cache::Impl { } } auto kinds = this->token_kinds(); - // a quiesce without a walk serves only the tokens owing none + // a quiesce without a walk serves only the tokens owing none, and + // forwards IO if it serves an IO token, else release rsp_pending_ = walk_skip_ ? port_arb_->tokens(!config_.shared_llc) : port_arb_->tokens(); - this->forward_begin(walk_skip_, !walk_skip_ && kinds.any_l1 && !kinds.any_full && !kinds.any_io); - flush_state_ = FlushState::Forward; + // after a walk: full if a full token or an IO one asks for more than the + // point of coherence, L1-only if any token owes a drain to it, acquire + // if only acquire tokens were visible + bool fwd_acq = !walk_skip_ && kinds.any_acq && !kinds.any_full && !kinds.any_io + && !kinds.any_l1 && !kinds.any_rel; + bool fwd_l1 = !walk_skip_ && (kinds.any_l1 || kinds.any_rel) && !kinds.any_full && !kinds.any_io; + if (walk_skip_) { + this->forward_begin(kinds.any_io, !kinds.any_io, !kinds.any_io); + } else { + this->forward_begin(fwd_acq, fwd_l1 || fwd_acq, false); + } + // an acquire walk owes the next level nothing unless it too may hold + // stale lines + flush_state_ = (fwd_acq && !config_.acq_forward) ? FlushState::Respond : FlushState::Forward; break; } case FlushState::Forward: { @@ -3023,7 +3038,7 @@ class Cache::Impl { } // an IO token also waits for the non-cacheable path, which carries the // IO writes it orders - if (fwd_io_ && mem_req_empty) { + if (fwd_io_ && !fwd_l1_ && mem_req_empty) { for (uint32_t i = 0, n = config_.num_inputs; i < n; ++i) { mem_req_empty &= (port_arb_->IoOut.at(i).size() == 0); } @@ -3043,6 +3058,7 @@ class Cache::Impl { token.flags.flush = 1; token.flags.io = fwd_io_; token.flags.flush_l1 = fwd_l1_; + token.flags.noalloc = fwd_noalloc_; token.tag = 0; if (bank_arb_->ReqIn.at(this->bank_lane(i)).try_send(token, 1)) { DT(3, simobject_->name() << " flush-fwd: " << token); @@ -3084,8 +3100,14 @@ class Cache::Impl { bool any_full = false; bool any_l1 = false; bool any_io = false; - // a walk is owed to a full token, and to an L1-only one above the point of coherence - bool walk(bool shared_llc) const { return any_full || (any_l1 && !shared_llc); } + bool any_acq = false; + bool any_rel = false; + // a walk is owed to a full token, and to an L1-only or acquire one above + // the point of coherence + bool walk(bool shared_llc) const { return any_full || ((any_l1 || any_acq) && !shared_llc); } + // a drain with no walk is owed to an IO token, and to a release one above + // the point of coherence + bool drain(bool shared_llc) const { return any_io || (any_rel && !shared_llc); } }; // Kinds of the visible tokens. Several kinds together combine to the @@ -3098,9 +3120,17 @@ class Cache::Impl { } auto &flags = port_arb_->ReqIn.at(i).peek().flags; if (flags.io) { - kinds.any_io = true; + if (flags.flush_l1) { + kinds.any_acq = true; + } else { + kinds.any_io = true; + } } else if (flags.flush_l1) { - kinds.any_l1 = true; + if (flags.noalloc) { + kinds.any_rel = true; + } else { + kinds.any_l1 = true; + } } else { kinds.any_full = true; } @@ -3108,9 +3138,10 @@ class Cache::Impl { return kinds; } - void forward_begin(bool io, bool l1_only) { + void forward_begin(bool io, bool l1_only, bool noalloc) { fwd_io_ = io; fwd_l1_ = l1_only; + fwd_noalloc_ = noalloc; fwd_go_ = false; fwd_sent_ = 0; fwd_done_ = 0; @@ -3200,6 +3231,7 @@ class Cache::Impl { bool fwd_go_ = false; // nothing is left ahead of the forwarded tokens bool fwd_io_ = false; // the forwarded tokens' kind bool fwd_l1_ = false; + bool fwd_noalloc_ = false; bool walk_skip_ = false; // the banks only quiesce: no walk is owed std::vector> mem_req_net_; std::vector flush_taps_; diff --git a/sim/simx/mem/cache.h b/sim/simx/mem/cache.h index 70adc78ba5..92bde10407 100644 --- a/sim/simx/mem/cache.h +++ b/sim/simx/mem/cache.h @@ -54,6 +54,7 @@ class Cache : public SimObject { bool mem_rsp_queue = false; // memory responses enter through a response queue bool mem_rsp_slice = false; // a registered boundary ahead of the memory-response port stands for that queue's register bool merge_port0 = true; // port 0's reads match other ports' tags for read merging + bool acq_forward = false; // an acquire flush also invalidates the next level }; struct PerfStats { diff --git a/sim/simx/socket.cpp b/sim/simx/socket.cpp index d2df85bb0c..083f880cd8 100644 --- a/sim/simx/socket.cpp +++ b/sim/simx/socket.cpp @@ -114,6 +114,7 @@ class Socket::Impl { // A private cache takes the flush injection on port 0 alone, and its // arbiter re-tags that port's requests apart from the others. VX_CFG_SOCKET_SIZE != VX_CFG_NUM_DCACHES, // port 0 merges reads + DCACHE_ACQ_FORWARD, // acq_forward }); #ifdef VX_CFG_EXT_TEX_ENABLE diff --git a/sim/simx/types.h b/sim/simx/types.h index a7eda35ac2..0fb7ab61ea 100644 --- a/sim/simx/types.h +++ b/sim/simx/types.h @@ -386,9 +386,14 @@ struct IntrLsuArgs { int32_t offset; // immediate addr offset (signed); 0 for packLD uops }; -// A fence carries no access width; its width field says it orders only -// device (IO) accesses. -inline bool lsu_fence_is_io(const IntrLsuArgs& args) { return args.width != 0; } +// A fence carries no access width; its width field says what it orders. +enum class FenceKind : uint32_t { + FULL = 0, // earlier writes visible and later reads see others' writes + IO = 1, // only device (IO) accesses + ACQUIRE = 2, // only later reads see others' writes + RELEASE = 3 // only earlier writes visible +}; +inline FenceKind lsu_fence_kind(const IntrLsuArgs& args) { return static_cast(args.width); } inline std::ostream &operator<<(std::ostream &os, const LsuType& type) { switch (type) { diff --git a/tests/regression/fence_mp/common.h b/tests/regression/fence_mp/common.h index 634f577e13..8d2f0a9994 100644 --- a/tests/regression/fence_mp/common.h +++ b/tests/regression/fence_mp/common.h @@ -5,11 +5,18 @@ #define FENCE_MP_MAGIC 0x5a5a0000u +// How the producer publishes the flag and the consumer observes it. +enum { + MP_FENCE = 0, // full fences on both sides + MP_ACQ_REL = 1, // a release fence before the flag store, an acquire fence after each flag load + MP_AMO = 2 // the flag swapped in by an amoswap.rl, read by an amoor.aq +}; + typedef struct { uint32_t num_tasks; // threads launched uint32_t count; // data words passed from producer to consumer uint32_t spin_limit; // consumer polls before giving up - uint32_t pad; + uint32_t mode; // MP_* uint64_t data_addr; // count words, zero on entry uint64_t flag_addr; // one word, zero on entry uint64_t out_addr; // [0] mismatches, [1] 1 if the consumer timed out, [2] pre-read sum diff --git a/tests/regression/fence_mp/kernel.cpp b/tests/regression/fence_mp/kernel.cpp index 80b47e7909..6d716a0aeb 100644 --- a/tests/regression/fence_mp/kernel.cpp +++ b/tests/regression/fence_mp/kernel.cpp @@ -3,11 +3,44 @@ #include "common.h" // Message passing across cores through plain memory: the producer (the first -// thread) writes data, fences, then raises a flag; the consumer (the last -// thread, on the last core) reads the data once beforehand so stale copies sit -// in its caches, polls the flag behind a fence, fences again and checks the -// data. A memory fence must make the data visible across every cache the two -// cores do not share. +// thread) writes data, then raises a flag; the consumer (the last thread, on +// the last core) reads the data once beforehand so stale copies sit in its +// caches, polls the flag, and checks the data. The release must make the data +// visible across every cache the two cores do not share before the flag, and +// the acquire must drop the consumer's stale copies, whether they are full +// fences, acquire/release fences, or the .aq/.rl bits of atomics. +static void publish(volatile uint32_t* flag, uint32_t mode) { + switch (mode) { + case MP_ACQ_REL: + __atomic_thread_fence(__ATOMIC_RELEASE); + *flag = 1; + break; + case MP_AMO: + __atomic_exchange_n(const_cast(flag), 1u, __ATOMIC_RELEASE); + break; + default: + vx_fence(); + *flag = 1; + vx_fence(); + break; + } +} + +static bool observe(volatile uint32_t* flag, uint32_t mode) { + switch (mode) { + case MP_ACQ_REL: { + uint32_t f = *flag; + __atomic_thread_fence(__ATOMIC_ACQUIRE); + return f != 0; + } + case MP_AMO: + return __atomic_fetch_or(const_cast(flag), 0u, __ATOMIC_ACQUIRE) != 0; + default: + vx_fence(); + return *flag != 0; + } +} + __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto data = reinterpret_cast(arg->data_addr); @@ -17,9 +50,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { for (uint32_t k = 0; k < arg->count; ++k) { data[k] = FENCE_MP_MAGIC + k; } - vx_fence(); - *flag = 1; - vx_fence(); + publish(flag, arg->mode); } else if (gid == arg->num_tasks - 1) { uint32_t sum = 0; for (uint32_t k = 0; k < arg->count; ++k) { @@ -28,13 +59,14 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { out[2] = sum; uint32_t timeout = 1; for (uint32_t s = 0; s < arg->spin_limit; ++s) { - vx_fence(); - if (*flag != 0) { + if (observe(flag, arg->mode)) { timeout = 0; break; } } - vx_fence(); + if (arg->mode == MP_FENCE) { + vx_fence(); + } uint32_t errors = 0; for (uint32_t k = 0; k < arg->count; ++k) { errors += (data[k] != FENCE_MP_MAGIC + k); diff --git a/tests/regression/fence_mp/main.cpp b/tests/regression/fence_mp/main.cpp index a8b2c05393..6439256482 100644 --- a/tests/regression/fence_mp/main.cpp +++ b/tests/regression/fence_mp/main.cpp @@ -18,6 +18,7 @@ const char* kernel_file = "kernel.vxbin"; uint32_t count = 64; uint32_t spin_limit = 4096; +uint32_t mode = MP_FENCE; vx_device_h device = nullptr; vx_buffer_h data_buffer = nullptr; @@ -30,15 +31,16 @@ kernel_arg_t kernel_arg = {}; static void show_usage() { std::cout << "Vortex Test." << std::endl; - std::cout << "Usage: [-k: kernel] [-n words] [-s spin limit] [-h: help]" << std::endl; + std::cout << "Usage: [-k: kernel] [-n words] [-s spin limit] [-m mode: 0 fences, 1 acquire/release fences, 2 .aq/.rl atomics] [-h: help]" << std::endl; } static void parse_args(int argc, char **argv) { int c; - while ((c = getopt(argc, argv, "n:s:k:h")) != -1) { + while ((c = getopt(argc, argv, "n:s:m:k:h")) != -1) { switch (c) { case 'n': count = atoi(optarg); break; case 's': spin_limit = atoi(optarg); break; + case 'm': mode = atoi(optarg); break; case 'k': kernel_file = optarg; break; case 'h': show_usage(); exit(0); break; default: show_usage(); exit(-1); @@ -75,6 +77,7 @@ int main(int argc, char *argv[]) { kernel_arg.num_tasks = total_threads; kernel_arg.count = count; kernel_arg.spin_limit = spin_limit; + kernel_arg.mode = mode; std::vector zeros(count, 0); std::vector out(4, 0); From d6be497d75fe0e2cd46cd01fd5f3fd85c38be30d Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 01:27:50 -0700 Subject: [PATCH 104/124] core: keep the dcache tag whole on passthru lanes so port 0 reads merge VX_mem_unit's passthru lanes shifted the LSU tag right by one and cleared its MSB, while lane 0, which also takes the flush injection, kept its select bit at the LSB. One request's tag was therefore encoded differently on port 0 than on the other ports, and the dcache's same-tag read merge never paired port 0 with another port. The tag now passes unchanged; its clear LSB sits where the flush lane's select bit does. SimX drops the merge_port0 knob that modeled the old behavior. SimX's LSU-to-memory path also follows the RTL more closely: - The lmem switch splits combinationally. The LSU ticks first and hands a request over with no delay, and the switch routes it into its output buffers in that cycle. The LSU checks room only in the buffers the beat addresses (req_ready), instead of a one-entry ingress. - The switch's response arbiter has a one-entry output buffer, which the coalescer or a multi-channel lane adapter fills combinationally. - A coalesced batch crosses the coalescer's output register and then the lane adapter's request buffer. The adapter registers its responses. - The dcache's MSHR occupancy is a registered count: a slot freed this cycle admits a request only from the next. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/core/VX_mem_unit.sv | 10 ++++---- sim/simx/core.cpp | 40 ++++++++++++++++++++----------- sim/simx/lsu_unit.cpp | 37 +++++++++++++++++++++------- sim/simx/mem/cache.cpp | 16 ++++++++----- sim/simx/mem/cache.h | 1 - sim/simx/mem/local_mem_switch.cpp | 13 +++++----- sim/simx/mem/local_mem_switch.h | 11 +++++++-- sim/simx/mem/mem_coalescer.cpp | 4 ++-- sim/simx/socket.cpp | 3 --- 9 files changed, 87 insertions(+), 48 deletions(-) diff --git a/hw/rtl/core/VX_mem_unit.sv b/hw/rtl/core/VX_mem_unit.sv index 11a8228b36..6ed8f18c83 100644 --- a/hw/rtl/core/VX_mem_unit.sv +++ b/hw/rtl/core/VX_mem_unit.sv @@ -255,22 +255,22 @@ module VX_mem_unit import VX_gpu_pkg::*; #( .cache_bus_if (dcache_bus_if[i]) ); end else begin : g_passthru_lane - // the cache sees the select bit as a clear tag MSB + // The tag passes unchanged: its clear LSB stands where the flush lane's + // select bit does, so every lane encodes one request's tag alike and + // the cache's read merge compares like with like. assign dcache_bus_if[i].req_valid = lsu_dcache_bus_if[i].req_valid; assign dcache_bus_if[i].req_data.rw = lsu_dcache_bus_if[i].req_data.rw; assign dcache_bus_if[i].req_data.addr = lsu_dcache_bus_if[i].req_data.addr; assign dcache_bus_if[i].req_data.data = lsu_dcache_bus_if[i].req_data.data; assign dcache_bus_if[i].req_data.byteen = lsu_dcache_bus_if[i].req_data.byteen; assign dcache_bus_if[i].req_data.attr = lsu_dcache_bus_if[i].req_data.attr; - assign dcache_bus_if[i].req_data.tag = {1'b0, lsu_dcache_bus_if[i].req_data.tag[DCACHE_TAG_WIDTH-1:1]}; + assign dcache_bus_if[i].req_data.tag = lsu_dcache_bus_if[i].req_data.tag; assign lsu_dcache_bus_if[i].req_ready = dcache_bus_if[i].req_ready; - `UNUSED_VAR (lsu_dcache_bus_if[i].req_data.tag[0]) assign lsu_dcache_bus_if[i].rsp_valid = dcache_bus_if[i].rsp_valid; assign lsu_dcache_bus_if[i].rsp_data.data = dcache_bus_if[i].rsp_data.data; - assign lsu_dcache_bus_if[i].rsp_data.tag = {dcache_bus_if[i].rsp_data.tag[DCACHE_TAG_WIDTH-2:0], 1'b0}; + assign lsu_dcache_bus_if[i].rsp_data.tag = dcache_bus_if[i].rsp_data.tag; assign dcache_bus_if[i].rsp_ready = lsu_dcache_bus_if[i].rsp_ready; - `UNUSED_VAR (dcache_bus_if[i].rsp_data.tag[DCACHE_TAG_WIDTH-1]) end end diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index 8742972a59..8e89758a9b 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -131,17 +131,26 @@ class Core::Impl { ibuffers_.at(i) = TFifo::Create(sname, 1, VX_CFG_IBUF_SIZE + 1); } + // The LSU ticks ahead of the coalescer, the lane adapters and the lmem + // switch. The switch routes a request into its output buffers in the cycle + // the LSU hands it over, as the RTL's combinational split does. The + // switch's one-entry response buffer is drained by the LSU before the + // coalescer or an adapter refills it, its ready being combinational in the + // RTL. + snprintf(sname, 100, "%s-lsu", name.c_str()); + func_units_.at((int)FUType::LSU) = SimPlatform::instance().create_object(sname, simobject_); + // create the memory coalescer, or the lane adapter that replaces it. - // Both tick ahead of the lmem switch: their response paths are - // combinational, and the dcache registers its output. const bool dcache_coalescing = (VX_CFG_NUM_LSU_LANES > 1) && (DCACHE_WORD_SIZE > LSU_WORD_SIZE); std::vector lsu_dcache_adapter(VX_CFG_NUM_LSU_BLOCKS); for (uint32_t b = 0; b < VX_CFG_NUM_LSU_BLOCKS; ++b) { snprintf(sname, 100, "%s-coalescer%d", name.c_str(), b); - mem_coalescers_.at(b) = MemCoalescer::Create(sname, LSU_CHANNELS, DCACHE_CHANNELS, DCACHE_WORD_SIZE, LSU_QUEUE_OUT_SIZE, 1); + // a batch crosses the coalescer's output register, then the lane + // adapter's request buffer, on its way to the data cache + mem_coalescers_.at(b) = MemCoalescer::Create(sname, LSU_CHANNELS, DCACHE_CHANNELS, DCACHE_WORD_SIZE, LSU_QUEUE_OUT_SIZE, 2); if (!dcache_coalescing) { snprintf(sname, 100, "%s-lsu_dcache_adapter%d", name.c_str(), b); - lsu_dcache_adapter.at(b) = LsuMemAdapter::Create(sname, DCACHE_CHANNELS, 1, 0); + lsu_dcache_adapter.at(b) = LsuMemAdapter::Create(sname, DCACHE_CHANNELS, 1, 1); } } @@ -158,14 +167,10 @@ class Core::Impl { VX_CFG_EXT_DXA_ENABLED + VX_CFG_TCU_WGMMA_ENABLED }); - // create lmem switch. A data-cache response reaches its response - // arbiter combinationally through the coalescer or the lane adapter, so - // the switch holds it in one entry, the least a channel holds; without - // either, the channel is the cache's own two-entry output buffer. - const bool dc_rsp_comb = dcache_coalescing || (DCACHE_CHANNELS > 1); + // create lmem switch for (uint32_t b = 0; b < VX_CFG_NUM_LSU_BLOCKS; ++b) { snprintf(sname, 100, "%s-lmem_switch%d", name.c_str(), b); - lmem_switch_.at(b) = LocalMemSwitch::Create(sname, 1, dc_rsp_comb ? 1 : 2); + lmem_switch_.at(b) = LocalMemSwitch::Create(sname, 1); } // create per-block lmem adapters @@ -198,7 +203,9 @@ class Core::Impl { // channels directly (combinational lane fan-out/fan-in). for (uint32_t b = 0; b < VX_CFG_NUM_LSU_BLOCKS; ++b) { lmem_switch_.at(b)->ReqOutDC.bind(&mem_coalescers_.at(b)->ReqIn); - mem_coalescers_.at(b)->RspOut.bind(&lmem_switch_.at(b)->RspInDC); + // the coalescer's response reaches the switch's response buffer + // combinationally, ahead of a local-memory response + mem_coalescers_.at(b)->RspOut.bind(&lmem_switch_.at(b)->RspOut); for (uint32_t c = 0; c < DCACHE_CHANNELS; ++c) { dc_req_out.at(b * DCACHE_CHANNELS + c) = &mem_coalescers_.at(b)->ReqOut.at(c); dc_rsp_in.at(b * DCACHE_CHANNELS + c) = &mem_coalescers_.at(b)->RspIn.at(c); @@ -209,7 +216,14 @@ class Core::Impl { // pass-through when DCACHE_CHANNELS == 1) for (uint32_t b = 0; b < VX_CFG_NUM_LSU_BLOCKS; ++b) { lmem_switch_.at(b)->ReqOutDC.bind(&lsu_dcache_adapter.at(b)->ReqIn); - lsu_dcache_adapter.at(b)->RspOut.bind(&lmem_switch_.at(b)->RspInDC); + // A multi-channel adapter packs its response combinationally into the + // switch's response buffer; a single-channel one passes the cache's + // own output buffer through, which the switch then drains. + if (DCACHE_CHANNELS > 1) { + lsu_dcache_adapter.at(b)->RspOut.bind(&lmem_switch_.at(b)->RspOut); + } else { + lsu_dcache_adapter.at(b)->RspOut.bind(&lmem_switch_.at(b)->RspInDC); + } for (uint32_t c = 0; c < DCACHE_CHANNELS; ++c) { dc_req_out.at(b * DCACHE_CHANNELS + c) = &lsu_dcache_adapter.at(b)->ReqOut.at(c); dc_rsp_in.at(b * DCACHE_CHANNELS + c) = &lsu_dcache_adapter.at(b)->RspIn.at(c); @@ -287,8 +301,6 @@ class Core::Impl { if (FpuUnit::kGather) { create_fpu(); } - snprintf(sname, 100, "%s-lsu", name.c_str()); - func_units_.at((int)FUType::LSU) = SimPlatform::instance().create_object(sname, simobject_); snprintf(sname, 100, "%s-sfu", name.c_str()); func_units_.at((int)FUType::SFU) = SimPlatform::instance().create_object(sname, simobject_); std::static_pointer_cast(func_units_.at((int)FUType::SFU))->warp_ctl_out.bind(&simobject_->warp_ctl_in); diff --git a/sim/simx/lsu_unit.cpp b/sim/simx/lsu_unit.cpp index 22b4a9a0ab..21240dd7b4 100644 --- a/sim/simx/lsu_unit.cpp +++ b/sim/simx/lsu_unit.cpp @@ -395,7 +395,8 @@ void LsuUnit::ingest_inputs(uint32_t b) { // the block until the token's response returns. void LsuUnit::send_fence_token(uint32_t b) { auto& state = states_.at(b); - if (state.pending_reqs.full() || core_->lmem_switch(b)->ReqIn.full()) { + // the token's lanes carry no address: it goes to the data cache + if (state.pending_reqs.full() || !core_->lmem_switch(b)->req_ready(true, false)) { return; } auto trace = state.fence.trace(); @@ -427,7 +428,7 @@ void LsuUnit::send_fence_token(uint32_t b) { lsu_req.cid = trace->cid; lsu_req.wid = trace->wid; lsu_req.uuid = trace->uuid; - core_->lmem_switch(b)->ReqIn.send(lsu_req); + core_->lmem_switch(b)->ReqIn.send(lsu_req, 0); DT(3, this->name() << " fence-token: " << lsu_req); state.fence_token = false; } @@ -459,8 +460,17 @@ void LsuUnit::process_request_step(uint32_t b) { // not gated by the fence lock — its entries still hold fence release via // pending_reqs.empty(). if (b < TCU_AGU_PORTS && !TcuReqIn.at(b).empty()) { - if (!state.pending_reqs.full() && !core_->lmem_switch(b)->ReqIn.full()) { - LsuReq lsu_req = TcuReqIn.at(b).peek(); + const LsuReq& meta_req = TcuReqIn.at(b).peek(); + bool to_dc = false, to_lmem = false; + for (uint32_t i = 0; i < VX_CFG_NUM_LSU_LANES; ++i) { + if (meta_req.mask.test(i)) { + bool local = (get_addr_type(meta_req.addrs.at(i)) == AddrType::Shared); + to_lmem |= local; + to_dc |= !local; + } + } + if (!state.pending_reqs.full() && core_->lmem_switch(b)->req_ready(to_dc, to_lmem)) { + LsuReq lsu_req = meta_req; uint32_t count = lsu_req.mask.count(); std::vector lane_entries(VX_CFG_NUM_LSU_LANES); for (uint32_t i = 0; i < VX_CFG_NUM_LSU_LANES; ++i) { @@ -482,7 +492,7 @@ void LsuUnit::process_request_step(uint32_t b) { entry.client_tag = lsu_req.tag; uint32_t tag = state.pending_reqs.allocate(std::move(entry)); lsu_req.tag = tag; - core_->lmem_switch(b)->ReqIn.send(lsu_req); + core_->lmem_switch(b)->ReqIn.send(lsu_req, 0); DT(3, this->name() << " tcu-meta-req: " << lsu_req); TcuReqIn.at(b).pop(); return; // one request per cycle into the switch @@ -579,8 +589,19 @@ void LsuUnit::process_request_step(uint32_t b) { } if (state.remain_addrs != 0) { - // check lmem switch backpressure - if (core_->lmem_switch(b)->ReqIn.full()) + // check backpressure at the switch buffers this beat addresses + uint32_t beat_t0 = state.addr_list.size() - state.remain_addrs; + uint32_t beat_len = std::min(VX_CFG_NUM_LSU_LANES, state.remain_addrs); + bool to_dc = false, to_lmem = false; + for (uint32_t i = 0; i < beat_len; ++i) { + auto& entry = state.addr_list.at(beat_t0 + i); + if (entry.size == 0) + continue; + bool local = (get_addr_type(entry.addr) == AddrType::Shared); + to_lmem |= local; + to_dc |= !local; + } + if (!core_->lmem_switch(b)->req_ready(to_dc, to_lmem)) return; // stall // setup memory request @@ -657,7 +678,7 @@ void LsuUnit::process_request_step(uint32_t b) { lsu_req.uuid = trace->uuid; // send memory request - core_->lmem_switch(b)->ReqIn.send(lsu_req); + core_->lmem_switch(b)->ReqIn.send(lsu_req, 0); DT(3, this->name() << " mem-req: " << lsu_req); // update stats diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index 77b3fc79eb..4668fcb78b 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -884,6 +884,7 @@ class CacheBank : public SimObject { void on_reset() { perf_stats_ = Cache::PerfStats(); pending_mshr_size_ = 0; + mshr_occupancy_ = 0; pending_amo_probes_ = 0; pipe_core_lines_.clear(); pending_read_reqs_ = 0; @@ -930,6 +931,10 @@ class CacheBank : public SimObject { // priority and the forward drain yields (crsp_sent_). crsp_sent_ = false; + // MSHR occupancy is a registered count: slots a forward, replay or + // release frees this tick admit a new request only from the next one. + mshr_occupancy_ = mshr_.size() + pending_mshr_size_; + // Process next request at the head of the pipeline. if (!pipe_req_->empty()) { this->processRequests(); @@ -1203,7 +1208,7 @@ class CacheBank : public SimObject { // responses from draining — a deadlock. Reserve at admission. bool needs_mshr = !is_amo_passthru; (void)is_amo; - if (needs_mshr && (mshr_.size() + pending_mshr_size_) >= mshr_.capacity()) { + if (needs_mshr && mshr_occupancy_ >= mshr_.capacity()) { ++perf_stats_.mshr_stalls; return; } @@ -2188,6 +2193,7 @@ class CacheBank : public SimObject { std::vector sets_; MSHR mshr_; uint32_t pending_mshr_size_; + uint32_t mshr_occupancy_; uint32_t pending_amo_probes_; // AmoProbe requests in the pipe, each holding a table reservation // {set, tag, sector} of Core requests inside the pipe: their MSHR entries // exist only from pipe exit, so ordering checks against in-flight lines (the @@ -2317,7 +2323,7 @@ class CachePortArb : public SimObject { std::vector> IoOut; // non-cacheable requests, per port CachePortArb(const SimContext& ctx, const char* name, const params_t& params, - uint32_t num_inputs, uint32_t num_banks, uint32_t merge_group, bool merge_port0) + uint32_t num_inputs, uint32_t num_banks, uint32_t merge_group) : SimObject(ctx, name) , ReqIn(num_inputs, this) , ReqOut(num_banks, this) @@ -2328,7 +2334,6 @@ class CachePortArb : public SimObject { , delay_((num_inputs > 2) ? 1 : 0) , levels_(arb_tree_levels(num_inputs)) , merge_group_(merge_group) - , merge_port0_(merge_port0) { __assert(merge_group_ <= 1 || (merge_group_ <= 64 && ispow2(merge_group_) && (num_inputs % merge_group_) == 0), "a read-merge group must be a power of two of at most 64 dividing the ports"); @@ -2483,7 +2488,7 @@ class CachePortArb : public SimObject { auto& req_i = ReqIn.at(i).peek(); uint32_t base = (i / merge_group_) * merge_group_; for (uint32_t j = base; j < i; ++j) { - if (followers.at(j) || !this->is_plain_read(j) || (j == 0 && !merge_port0_)) { + if (followers.at(j) || !this->is_plain_read(j)) { continue; } auto& req_j = ReqIn.at(j).peek(); @@ -2589,7 +2594,6 @@ class CachePortArb : public SimObject { std::vector>> tree_stage_; // [level][bank][slice] std::vector> rr_; // [arbiter level][bank][slice] next to grant uint32_t merge_group_; - bool merge_port0_; uint64_t collisions_ = 0; friend class SimObject; @@ -2859,7 +2863,7 @@ class Cache::Impl { // Create the core-port arbiter ahead of the banks it feeds snprintf(sname, 100, "%s-port_arb", simobject->name().c_str()); uint32_t merge_group = std::min(config_.read_merge, config_.num_inputs); - port_arb_ = CachePortArb::Create(sname, params_, config_.num_inputs, num_banks, merge_group, config_.merge_port0); + port_arb_ = CachePortArb::Create(sname, params_, config_.num_inputs, num_banks, merge_group); for (uint32_t i = 0; i < config_.num_inputs; ++i) { simobject->core_req_in.at(i).bind(&port_arb_->ReqIn.at(i)); } diff --git a/sim/simx/mem/cache.h b/sim/simx/mem/cache.h index 92bde10407..de87d79d01 100644 --- a/sim/simx/mem/cache.h +++ b/sim/simx/mem/cache.h @@ -53,7 +53,6 @@ class Cache : public SimObject { bool write_around = false; // a no-allocate write miss goes to memory (write-back only) bool mem_rsp_queue = false; // memory responses enter through a response queue bool mem_rsp_slice = false; // a registered boundary ahead of the memory-response port stands for that queue's register - bool merge_port0 = true; // port 0's reads match other ports' tags for read merging bool acq_forward = false; // an acquire flush also invalidates the next level }; diff --git a/sim/simx/mem/local_mem_switch.cpp b/sim/simx/mem/local_mem_switch.cpp index a1e64b6c10..29f131a851 100644 --- a/sim/simx/mem/local_mem_switch.cpp +++ b/sim/simx/mem/local_mem_switch.cpp @@ -18,18 +18,17 @@ using namespace vortex; LocalMemSwitch::LocalMemSwitch( const SimContext& ctx, const char* name, - uint32_t delay, - uint32_t dc_rsp_size + uint32_t delay ) : SimObject(ctx, name) - // Single-entry ingress: the switch accepts a new request only as its - // current one drains, so upstream issue feels downstream cadence - // immediately instead of running ahead into queue slack. + // The split is combinational: a request handed over is routed into the + // output buffers in the same cycle, and ReqIn holds nothing across cycles. , ReqIn(this, 1) - , RspOut(this) + // the response arbiter's one-entry output buffer + , RspOut(this, 1) , ReqOutLmem(this) , RspInLmem(this) , ReqOutDC(this) - , RspInDC(this, dc_rsp_size) + , RspInDC(this) , delay_(delay) {} diff --git a/sim/simx/mem/local_mem_switch.h b/sim/simx/mem/local_mem_switch.h index d765adaead..73158544ac 100644 --- a/sim/simx/mem/local_mem_switch.h +++ b/sim/simx/mem/local_mem_switch.h @@ -33,10 +33,17 @@ class LocalMemSwitch : public SimObject { LocalMemSwitch( const SimContext& ctx, const char* name, - uint32_t delay, - uint32_t dc_rsp_size = 2 + uint32_t delay ); + // The output buffers a request addresses can take it this cycle: to the data + // cache, to local memory, or both. + bool req_ready(bool to_dc, bool to_lmem) const { + return ReqIn.size() == 0 + && !(to_dc && ReqOutDC.full()) + && !(to_lmem && ReqOutLmem.full()); + } + protected: void on_tick(); diff --git a/sim/simx/mem/mem_coalescer.cpp b/sim/simx/mem/mem_coalescer.cpp index 4940c9c5db..11bd5af5e3 100644 --- a/sim/simx/mem/mem_coalescer.cpp +++ b/sim/simx/mem/mem_coalescer.cpp @@ -107,8 +107,8 @@ void MemCoalescer::on_tick() { } } - // the response path is combinational; the cache registers its output - RspOut.send(out_rsp, 0); + // packed combinationally into the lmem switch's response buffer + RspOut.send(out_rsp, 1); DT(4, this->name() << " mem-rsp: " << out_rsp); // track remaining responses diff --git a/sim/simx/socket.cpp b/sim/simx/socket.cpp index 083f880cd8..22b0f64820 100644 --- a/sim/simx/socket.cpp +++ b/sim/simx/socket.cpp @@ -111,9 +111,6 @@ class Socket::Impl { false, // write_around VX_CFG_DCACHE_MRSQ_SIZE != 0, // memory-response queue true, // the socket's registered memory port - // A private cache takes the flush injection on port 0 alone, and its - // arbiter re-tags that port's requests apart from the others. - VX_CFG_SOCKET_SIZE != VX_CFG_NUM_DCACHES, // port 0 merges reads DCACHE_ACQ_FORWARD, // acq_forward }); From a010af42d907f2883a9ab6468bf6959bdd072f3d Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 01:27:51 -0700 Subject: [PATCH 105/124] cache: drop .aq/.rl fences at a coherent dcache, add a bank-0 delay knob A dcache that is itself the point of coherence commits every atomic in the one cache all harts use, so the fences around an .aq/.rl atomic order nothing more there. The uop sequencer and SimX decode now leave such atomics unexpanded. DBG_MEM_BANK0_WDELAY is a debug-only knob. A write-through cache holds each write at the head of bank 0's memory queue for 2048 cycles, so writes leaving through the other banks overtake it unless a release orders them. Without a delay, no shape exposes a release fence that fails to drain. fence_mp places its data in even 64-byte lines and its flag in an odd one, so on a two-bank dcache they leave through different banks, and it checks the data newest first. SimX models the knob as the RTL builds it: the delay holds the head of bank 0's in-order memory queue, so everything queued behind the write waits too. Every bank memory send goes through send_mem_req(), which keeps a request from leaving before the one ahead of it, and a launch reset clears the queue's next free departure cycle. A later read-miss fill of the same line therefore cannot overtake the write and install the old word. Without the knob, timing is unchanged. SimX LLC atomics: a store-bearing atomic writes its result back into the line through the core-request slot two cycles after its commit, and holds new core requests off until then. A token flush takes two more cycles after its last write-back before the bank reports done. Both follow the RTL bank. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/cache/VX_cache.sv | 30 +++++++- hw/rtl/core/VX_uop_sequencer.sv | 5 +- sim/simx/decode.cpp | 6 +- sim/simx/mem/cache.cpp | 110 ++++++++++++++++++++++----- tests/regression/fence_mp/common.h | 14 ++++ tests/regression/fence_mp/kernel.cpp | 9 ++- tests/regression/fence_mp/main.cpp | 10 ++- 7 files changed, 153 insertions(+), 31 deletions(-) diff --git a/hw/rtl/cache/VX_cache.sv b/hw/rtl/cache/VX_cache.sv index 86814ef6eb..5184a6f56d 100644 --- a/hw/rtl/cache/VX_cache.sv +++ b/hw/rtl/cache/VX_cache.sv @@ -668,6 +668,30 @@ module VX_cache import VX_gpu_pkg::*; #( wire [NUM_BANKS-1:0] mem_req_xbar_valid; wire [NUM_BANKS-1:0][MEM_REQ_DATAW-1:0] mem_req_xbar_pdata; wire [NUM_BANKS-1:0] mem_req_xbar_ready; + + // Debug: a write-through cache holds each write of bank 0 for a fixed + // delay before it leaves, so requests through the other banks overtake it + // unless something orders them. + wire [NUM_BANKS-1:0] dbg_mem_req_hold; +`ifdef DBG_MEM_BANK0_WDELAY + if (WRITEBACK == 0) begin : g_dbg_wdelay + localparam DBG_WDELAY = 2048; + reg [`CLOG2(DBG_WDELAY+1)-1:0] dbg_wdelay_cnt; + wire dbg_wait = per_bank_mem_req_valid[0] && per_bank_mem_req_rw[0]; + always @(posedge clk) begin + if (reset || ~dbg_wait || (mem_req_xbar_valid[0] && mem_req_xbar_ready[0])) begin + dbg_wdelay_cnt <= '0; + end else if (dbg_wdelay_cnt != DBG_WDELAY) begin + dbg_wdelay_cnt <= dbg_wdelay_cnt + 1; + end + end + assign dbg_mem_req_hold = NUM_BANKS'(dbg_wait && (dbg_wdelay_cnt != DBG_WDELAY)); + end else begin : g_no_dbg_wdelay + assign dbg_mem_req_hold = '0; + end +`else + assign dbg_mem_req_hold = '0; +`endif for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_mem_req_xbar_in if (i < MEM_PORTS) begin : g_flush_req // line 0 hashes to 0, so the raw bank bits are the lane's bank id @@ -681,15 +705,15 @@ module VX_cache import VX_gpu_pkg::*; #( (`UP(MEM_ATTR_WIDTH)'(1) << MEM_ATTR_FLUSH_OFFS) | (`UP(MEM_ATTR_WIDTH)'(mem_flush_io) << MEM_ATTR_IO_OFFS) | (`UP(MEM_ATTR_WIDTH)'(mem_flush_noalloc) << MEM_ATTR_NOALLOC_OFFS) }; - assign mem_req_xbar_valid[i] = per_bank_mem_req_valid[i] || mem_flush_valid[i]; + assign mem_req_xbar_valid[i] = (per_bank_mem_req_valid[i] && ~dbg_mem_req_hold[i]) || mem_flush_valid[i]; assign mem_req_xbar_pdata[i] = per_bank_mem_req_valid[i] ? per_bank_mem_req_pdata[i] : mem_flush_pdata; assign mem_flush_ready[i] = mem_req_xbar_ready[i] && ~per_bank_mem_req_valid[i]; `RUNTIME_ASSERT (~(mem_flush_valid[i] && per_bank_mem_req_valid[i]), ("%t: %s flush token collides with a bank %0d request", $time, INSTANCE_ID, i)) end else begin : g_bank_req - assign mem_req_xbar_valid[i] = per_bank_mem_req_valid[i]; + assign mem_req_xbar_valid[i] = per_bank_mem_req_valid[i] && ~dbg_mem_req_hold[i]; assign mem_req_xbar_pdata[i] = per_bank_mem_req_pdata[i]; end - assign per_bank_mem_req_ready[i] = mem_req_xbar_ready[i]; + assign per_bank_mem_req_ready[i] = mem_req_xbar_ready[i] && ~dbg_mem_req_hold[i]; end // Write-backs from different banks are not ordered with each other inside // the crossbar, so the tokens wait until it has drained. diff --git a/hw/rtl/core/VX_uop_sequencer.sv b/hw/rtl/core/VX_uop_sequencer.sv index 3fb0ffcf49..2b888a8cce 100644 --- a/hw/rtl/core/VX_uop_sequencer.sv +++ b/hw/rtl/core/VX_uop_sequencer.sv @@ -197,7 +197,10 @@ module VX_uop_sequencer import // ------------------------------------------------------------------ // Atomic ordering uop expander: fences around an AMO/LR/SC with .aq/.rl // ------------------------------------------------------------------ - assign uop_in_valid[UOP_AMO] = (uop_in_data.ex_type == EX_LSU) + // A dcache that is itself the point of coherence orders the atomic with + // the hart's other accesses already: no fence is owed. + assign uop_in_valid[UOP_AMO] = !DCACHE_SHARED_LLC + && (uop_in_data.ex_type == EX_LSU) && uop_in_data.op_args.lsu.amo_valid && (uop_in_data.op_args.lsu.amo_aq || uop_in_data.op_args.lsu.amo_rl); VX_uop_amo uop_amo ( diff --git a/sim/simx/decode.cpp b/sim/simx/decode.cpp index 5a559d4726..87e1a98244 100644 --- a/sim/simx/decode.cpp +++ b/sim/simx/decode.cpp @@ -21,6 +21,7 @@ #include #include #include "debug.h" +#include "constants.h" #include "types.h" #include "decode.h" #include "decompressor.h" @@ -671,8 +672,9 @@ Instr::Ptr Decoder::decode(uint32_t code, uint64_t uuid) { std::abort(); } instr->set_args(IntrAmoArgs{funct3, aq, rl}); - // .aq/.rl order the atomic with fences around it (LsuUopGen) - if (aq || rl) { + // .aq/.rl order the atomic with fences around it (LsuUopGen), unless the + // dcache is itself the point of coherence + if ((aq || rl) && !DCACHE_SHARED_LLC) { instr->set_macro_op(); } instr->set_dest_reg(rd, RegType::Integer); diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index 4668fcb78b..d957a056a6 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -416,7 +416,8 @@ struct bank_req_t { Replay = 2, Core = 3, AmoProbe = 4, // non-LLC AMO passthrough: probe-and-invalidate then forward - Park = 5 // chain head parked on a partial sector: issues its refetch + Park = 5, // chain head parked on a partial sector: issues its refetch + AmoWb = 6 // an LLC atomic's result written back through the pipe }; uint64_t addr; @@ -872,6 +873,7 @@ class CacheBank : public SimObject { token_flushing_ = true; token_flush_skip_ = skip; token_flush_idx_ = 0; + token_flush_settle_ = 0; flush_sector_idx_ = 0; this->tick_wake(); } @@ -892,6 +894,9 @@ class CacheBank : public SimObject { pending_fill_reqs_ = 0; rand_ctr_ = 0; adm_seq_ctr_ = 0; + #ifdef DBG_MEM_BANK0_WDELAY + dbg_mem_req_free_ = 0; + #endif if (!keep_lines_) { for (auto &set : sets_) { set.reset(); @@ -912,6 +917,9 @@ class CacheBank : public SimObject { chain_sector_ = 0; refetch_hold_ = false; refetch_hold_retry_ = false; + amo_busy_ = false; + amo_wb_pending_ = false; + amo_wb_tick_ = 0; flushing_ = false; flush_set_idx_ = 0; flush_way_idx_ = 0; @@ -970,7 +978,8 @@ class CacheBank : public SimObject { // any packet reserved toward core_req_in/mem_rsp_in re-arms on arrival. if (this->core_req_in.size() == 0 && this->mem_rsp_in.size() == 0 && pipe_req_->size() == 0 && lazy_q_->size() == 0 && !mshr_.has_ready_reqs() - && !fwd_active_ && !flushing_ && !token_flushing_ && pending_fill_reqs_ == 0) { + && !fwd_active_ && !flushing_ && !token_flushing_ && pending_fill_reqs_ == 0 + && !amo_wb_pending_) { this->tick_sleep(); } } @@ -1148,8 +1157,29 @@ class CacheBank : public SimObject { } } +#if VX_CFG_EXT_A_ENABLED + // An LLC atomic's result goes back into the line as a synthetic write + // through the core-request slot, one compute cycle after its commit. + if (amo_wb_pending_ && SimPlatform::instance().cycles() >= amo_wb_tick_) { + bank_req_t wb_req; + wb_req.reset(); + wb_req.type = bank_req_t::AmoWb; + pipe_req_->push(wb_req); + amo_wb_pending_ = false; + amo_busy_ = false; + return; + } +#endif + // 3) core request if (!this->core_req_in.empty()) { +#if VX_CFG_EXT_A_ENABLED + // A store-bearing atomic holds new core requests off from its lookup + // until its result is written back. + if (amo_busy_) { + return; + } +#endif auto &core_req = this->core_req_in.peek(); if (fill_held && params_.addr_set_id(core_req.addr) == held_set_id @@ -1322,6 +1352,9 @@ class CacheBank : public SimObject { } #if VX_CFG_EXT_A_ENABLED if (is_amo_passthru) ++pending_amo_probes_; + if (is_amo && config_.is_llc && core_req.op != MemOp::AMO_LR) { + amo_busy_ = true; + } #endif if (core_req.is_write()) ++perf_stats_.writes; else ++perf_stats_.reads; @@ -1414,7 +1447,7 @@ class CacheBank : public SimObject { wb.uuid = bank_req.uuid; wb.data = sec.data; wb.byteen = sec.dirty_mask; - this->mem_req_out.send(wb, MEM_REQ_DELAY); + this->send_mem_req(wb); DT(3, this->name() << " writeback: " << wb); ++perf_stats_.evictions; } @@ -1586,13 +1619,22 @@ class CacheBank : public SimObject { w.uuid = bank_req.uuid; w.data = store_block; w.byteen = byteen; - this->mem_req_out.send(w, MEM_REQ_DELAY); + this->send_mem_req(w); DT(3, this->name() << " amo-writethrough: " << w); } // Break other harts' reservations on this line. amo_unit_.invalidate(line_addr, /*except=*/hid); } + // A stored result is written back into the line through the pipe; without + // a store the commit frees the bank at once. + if (do_store) { + amo_wb_pending_ = true; + amo_wb_tick_ = SimPlatform::instance().cycles() + 2; + } else { + amo_busy_ = false; + } + // Send AMO response. MemRsp rsp{bank_req.req_tag, bank_req.hart_id, bank_req.uuid}; rsp.data = rsp_block; @@ -1605,6 +1647,25 @@ class CacheBank : public SimObject { // Pipeline tail: process the head of pipe_req_ — at most one bank_req_t per tick. + // Send a request down the bank's in-order memory queue. + // Debug: a write-through cache holds each write at the head of bank 0's + // queue for a fixed delay, the requests behind it included, so requests + // through the other banks overtake it unless something orders them. + void send_mem_req(const MemReq& req) { + #ifdef DBG_MEM_BANK0_WDELAY + if (bank_id_ == 0 && !config_.write_back) { + constexpr uint64_t DBG_WDELAY = 2048; + uint64_t now = SimPlatform::instance().cycles(); + uint64_t head = std::max(now, dbg_mem_req_free_); + uint64_t leave = head + (req.is_write() ? DBG_WDELAY : 0); + dbg_mem_req_free_ = leave + 1; + this->mem_req_out.send(req, MEM_REQ_DELAY + (leave - now)); + return; + } + #endif + this->mem_req_out.send(req, MEM_REQ_DELAY); + } + // Emit a write-through store downstream, with the optional core response. // All backpressure is checked before any send, so a false return stalls // the caller with no partial effects. @@ -1626,7 +1687,7 @@ class CacheBank : public SimObject { w.uuid = uuid; w.data = data; w.byteen = byteen; - this->mem_req_out.send(w, MEM_REQ_DELAY); + this->send_mem_req(w); DT(3, this->name() << " writethrough: " << w); #if VX_CFG_EXT_A_ENABLED // Committed write: break other harts' reservations. @@ -1656,6 +1717,7 @@ class CacheBank : public SimObject { switch (bank_req.type) { case bank_req_t::None: + case bank_req_t::AmoWb: pipe_req_->pop(); return; @@ -1705,7 +1767,7 @@ class CacheBank : public SimObject { wb.uuid = bank_req.uuid; wb.data = sec.data; wb.byteen = sec.dirty_mask; - this->mem_req_out.send(wb, MEM_REQ_DELAY); + this->send_mem_req(wb); DT(3, this->name() << " amo-probe-wb: " << wb); ++perf_stats_.evictions; } @@ -1731,7 +1793,7 @@ class CacheBank : public SimObject { amo_fwd.amo_cmp = bank_req.amo_cmp; amo_fwd.data = bank_req.data; // carries rhs (extracted at LLC) amo_fwd.flags = bank_req.flags; // carry amo_unsigned (signed vs unsigned MIN/MAX) - this->mem_req_out.send(amo_fwd, MEM_REQ_DELAY); + this->send_mem_req(amo_fwd); DT(3, this->name() << " amo-probe-fwd: " << amo_fwd); ++pending_fill_reqs_; // counts as an outstanding mem-roundtrip for perf --pending_amo_probes_; // reservation transferred to the table entry @@ -1771,7 +1833,7 @@ class CacheBank : public SimObject { fill.tag = bank_req.mshr_id; fill.hart_id = bank_req.hart_id; fill.uuid = bank_req.uuid; - this->mem_req_out.send(fill, MEM_REQ_DELAY); + this->send_mem_req(fill); DT(3, this->name() << " refetch-req: " << fill); ++pending_fill_reqs_; pipe_req_->pop(); @@ -1823,7 +1885,7 @@ class CacheBank : public SimObject { fill.tag = mshr_id; fill.hart_id = bank_req.hart_id; fill.uuid = bank_req.uuid; - this->mem_req_out.send(fill, MEM_REQ_DELAY); + this->send_mem_req(fill); ++pending_fill_reqs_; } pipe_req_->pop(); @@ -1976,7 +2038,7 @@ class CacheBank : public SimObject { w.uuid = bank_req.uuid; w.data = bank_req.data; w.byteen = bank_req.byteen; - this->mem_req_out.send(w, MEM_REQ_DELAY); + this->send_mem_req(w); DT(3, this->name() << " writethrough: " << w); #if VX_CFG_EXT_A_ENABLED // Writethrough commit: break other harts' reservations. @@ -2073,7 +2135,7 @@ class CacheBank : public SimObject { fill.tag = mshr_id; // routes the fill response back here fill.hart_id = bank_req.hart_id; fill.uuid = bank_req.uuid; - this->mem_req_out.send(fill, MEM_REQ_DELAY); + this->send_mem_req(fill); DT(3, this->name() << " fill-req: " << fill); ++pending_fill_reqs_; } @@ -2121,7 +2183,7 @@ class CacheBank : public SimObject { mem_req.op = MemOp::ST; mem_req.data = sec.data; mem_req.byteen = sec.dirty_mask; - this->mem_req_out.send(mem_req, MEM_REQ_DELAY); + this->send_mem_req(mem_req); DT(3, this->name() << " flush-wb: " << mem_req); ++perf_stats_.evictions; sec.dirty = false; @@ -2140,14 +2202,21 @@ class CacheBank : public SimObject { } void processTokenFlush() { + // the bank leaves its write-backs, then signals the end, a cycle each + if (token_flush_settle_ != 0) { + if (--token_flush_settle_ == 0) { + token_flushing_ = false; + DT(3, this->name() << (token_flush_skip_ ? " token-quiesce-done" : " token-flush-done")); + } + return; + } if (pending_fill_reqs_ != 0 || pipe_req_->size() != 0 || !mshr_.empty()) { return; } if (token_flush_skip_) { - token_flushing_ = false; - DT(3, this->name() << " token-quiesce-done"); + token_flush_settle_ = TOKEN_FLUSH_SETTLE; return; } uint32_t ways = config_.write_back ? params_.lines_per_set : 1; @@ -2167,7 +2236,7 @@ class CacheBank : public SimObject { mem_req.op = MemOp::ST; mem_req.data = sec.data; mem_req.byteen = sec.dirty_mask; - this->mem_req_out.send(mem_req, MEM_REQ_DELAY); + this->send_mem_req(mem_req); DT(3, this->name() << " flush-wb: " << mem_req); ++perf_stats_.evictions; } @@ -2181,8 +2250,7 @@ class CacheBank : public SimObject { } } if (++token_flush_idx_ == params_.sets_per_bank * ways) { - token_flushing_ = false; - DT(3, this->name() << " token-flush-done"); + token_flush_settle_ = TOKEN_FLUSH_SETTLE; } } @@ -2272,6 +2340,12 @@ class CacheBank : public SimObject { uint32_t chain_sector_; bool refetch_hold_; // a refetch fill is at its tag lookup: its chain waits bool refetch_hold_retry_; + bool amo_busy_; // a store-bearing LLC atomic holds core requests off + bool amo_wb_pending_; // its result still has to be written back + uint64_t amo_wb_tick_; // ... from this cycle on +#ifdef DBG_MEM_BANK0_WDELAY + uint64_t dbg_mem_req_free_ = 0; // the memory queue's next free departure cycle +#endif // Flush walk state. bool flushing_; @@ -2281,6 +2355,8 @@ class CacheBank : public SimObject { uint32_t flush_sector_idx_; bool token_flushing_; bool token_flush_skip_ = false; + static constexpr uint32_t TOKEN_FLUSH_SETTLE = 2; + uint32_t token_flush_settle_ = 0; uint32_t token_flush_idx_; #if VX_CFG_EXT_A_ENABLED diff --git a/tests/regression/fence_mp/common.h b/tests/regression/fence_mp/common.h index 8d2f0a9994..1b04613e07 100644 --- a/tests/regression/fence_mp/common.h +++ b/tests/regression/fence_mp/common.h @@ -5,6 +5,20 @@ #define FENCE_MP_MAGIC 0x5a5a0000u +// The data lives in the even 64-byte lines of its buffer and the flag in an +// odd one, so on a two-bank dcache they leave through different banks: a test +// can delay one bank's writes and check that only a release orders the flag +// behind them. +#define MP_LINE_WORDS 16 + +static inline uint32_t mp_data_index(uint32_t k) { + return (k / MP_LINE_WORDS) * (2 * MP_LINE_WORDS) + (k % MP_LINE_WORDS); +} + +static inline uint32_t mp_data_words(uint32_t count) { + return ((count + MP_LINE_WORDS - 1) / MP_LINE_WORDS) * (2 * MP_LINE_WORDS); +} + // How the producer publishes the flag and the consumer observes it. enum { MP_FENCE = 0, // full fences on both sides diff --git a/tests/regression/fence_mp/kernel.cpp b/tests/regression/fence_mp/kernel.cpp index 6d716a0aeb..b552343e02 100644 --- a/tests/regression/fence_mp/kernel.cpp +++ b/tests/regression/fence_mp/kernel.cpp @@ -48,13 +48,13 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto out = reinterpret_cast(arg->out_addr); if (gid == 0) { for (uint32_t k = 0; k < arg->count; ++k) { - data[k] = FENCE_MP_MAGIC + k; + data[mp_data_index(k)] = FENCE_MP_MAGIC + k; } publish(flag, arg->mode); } else if (gid == arg->num_tasks - 1) { uint32_t sum = 0; for (uint32_t k = 0; k < arg->count; ++k) { - sum += data[k]; + sum += data[mp_data_index(k)]; } out[2] = sum; uint32_t timeout = 1; @@ -67,9 +67,10 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { if (arg->mode == MP_FENCE) { vx_fence(); } + // newest first: the words written last are the ones still in flight uint32_t errors = 0; - for (uint32_t k = 0; k < arg->count; ++k) { - errors += (data[k] != FENCE_MP_MAGIC + k); + for (uint32_t k = arg->count; k-- > 0;) { + errors += (data[mp_data_index(k)] != FENCE_MP_MAGIC + k); } out[0] = errors; out[1] = timeout; diff --git a/tests/regression/fence_mp/main.cpp b/tests/regression/fence_mp/main.cpp index 6439256482..78f32f09e3 100644 --- a/tests/regression/fence_mp/main.cpp +++ b/tests/regression/fence_mp/main.cpp @@ -79,18 +79,20 @@ int main(int argc, char *argv[]) { kernel_arg.spin_limit = spin_limit; kernel_arg.mode = mode; - std::vector zeros(count, 0); + std::vector zeros(mp_data_words(count), 0); std::vector out(4, 0); - uint32_t data_size = count * sizeof(uint32_t); + uint32_t data_size = mp_data_words(count) * sizeof(uint32_t); uint32_t out_size = out.size() * sizeof(uint32_t); RT_CHECK(vx_buffer_create(device, data_size, VX_MEM_READ_WRITE, &data_buffer)); RT_CHECK(vx_buffer_address(data_buffer, &kernel_arg.data_addr)); - RT_CHECK(vx_buffer_create(device, sizeof(uint32_t), VX_MEM_READ_WRITE, &flag_buffer)); + // the flag sits in the buffer's second (odd) line + RT_CHECK(vx_buffer_create(device, 2 * MP_LINE_WORDS * sizeof(uint32_t), VX_MEM_READ_WRITE, &flag_buffer)); RT_CHECK(vx_buffer_address(flag_buffer, &kernel_arg.flag_addr)); + kernel_arg.flag_addr += MP_LINE_WORDS * sizeof(uint32_t); RT_CHECK(vx_buffer_create(device, out_size, VX_MEM_READ_WRITE, &out_buffer)); RT_CHECK(vx_buffer_address(out_buffer, &kernel_arg.out_addr)); RT_CHECK(vx_enqueue_write(queue, data_buffer, 0, zeros.data(), data_size, 0, nullptr, nullptr)); - RT_CHECK(vx_enqueue_write(queue, flag_buffer, 0, zeros.data(), sizeof(uint32_t), 0, nullptr, nullptr)); + RT_CHECK(vx_enqueue_write(queue, flag_buffer, MP_LINE_WORDS * sizeof(uint32_t), zeros.data(), sizeof(uint32_t), 0, nullptr, nullptr)); RT_CHECK(vx_enqueue_write(queue, out_buffer, 0, out.data(), out_size, 0, nullptr, nullptr)); RT_CHECK(vx_module_load_file(device, kernel_file, &module_)); From b9e7a2480cdd17bca48e6a7931a0fd2b08130140 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 01:27:52 -0700 Subject: [PATCH 106/124] ci: require a debug build to match release cycles (release_parity) A debug build widens the RTL UUID to 44 bits and turns on assertions and traces; none of that may move timing. A debug-only tag width did reach the dcache's read merge and changed cycle counts, and no check noticed. A blackbox case run with --debug can now set release_parity: true. The runner then reruns it built release at xlen 32 and requires identical instructions and cycles, reading the debug run's counts from its --log file. ci lint requires a --debug blackbox case and rejects combining it with a check. debug:sgemm_tcu-2 carries it, reshaped (8 threads, 8 warps, a 4 KB dcache, an L2) so the data cache is under pressure and its two channels' same-word reads merge. SimX: without a registered tree level, the cache's port crossbar is combinational. Each bank is now offered one request per cycle, and the ports are dequeued only when the bank takes it, so a later same-word read can still merge with a request waiting at its port. An offer still pending the next cycle is withdrawn and the ports re-arbitrate. Co-Authored-By: Claude Opus 5.5 --- ci/test_runner.py | 36 +++++++++++ ci/testcase.py | 13 ++++ ci/testcases/debug.yaml | 9 ++- docs/designs/continuous_integration.md | 12 ++++ sim/simx/mem/cache.cpp | 85 ++++++++++++++++++++++++++ 5 files changed, 154 insertions(+), 1 deletion(-) diff --git a/ci/test_runner.py b/ci/test_runner.py index 2cf559d5f1..5f761ba1b2 100644 --- a/ci/test_runner.py +++ b/ci/test_runner.py @@ -44,6 +44,39 @@ def _run_one(case, xlen, driver): return int(instrs), int(cycles) +def _perf_of(text, what): + perf = _PERF_RE.findall(text) + assert perf, "{}: no 'PERF: instrs=…, cycles=…' summary " \ + "(app must call vx_device_dump_perf)".format(what) + instrs, cycles = perf[-1] + return int(instrs), int(cycles) + + +def _release_parity(case, xlen): + """Run the debug case, then (xlen 32) the same run built release; both + must report identical instructions and cycles.""" + argv, env = case.run_command(xlen) + rc = tc.execute(argv, env) + assert rc == 0, "{} failed (exit {}): {}".format(case.id, rc, " ".join(argv)) + if xlen != 32: + return + # a debug run sends its output to the --log file (blackbox.sh) + log = next((a.split("=", 1)[1] for a in argv if a.startswith("--log=")), "run.log") + with open(log, errors="replace") as fh: + dbg_instrs, dbg_cycles = _perf_of(fh.read(), case.id + " [debug]") + rel_argv = [a for a in argv if not a.startswith(("--debug", "--log="))] + rc, out = tc.execute_capture(rel_argv, env) + assert rc == 0, "{} [release] failed (exit {}): {}".format( + case.id, rc, " ".join(rel_argv)) + rel_instrs, rel_cycles = _perf_of(out, case.id + " [release]") + print("RELEASE-PARITY: {}: instrs debug={} release={}, cycles debug={} " + "release={}".format(case.id, dbg_instrs, rel_instrs, dbg_cycles, rel_cycles)) + assert (dbg_instrs, dbg_cycles) == (rel_instrs, rel_cycles), \ + "{}: the debug build changes the run (instrs {} vs {}, cycles {} vs {}) " \ + "-- debug-only state must not reach timing".format( + case.id, dbg_instrs, rel_instrs, dbg_cycles, rel_cycles) + + def _model_parity(case, xlen): simx_instrs, simx_cycles = _run_one(case, xlen, "simx") rtl_instrs, rtl_cycles = _run_one(case, xlen, "rtlsim") @@ -99,6 +132,9 @@ def test_case(case, sim_build, request): if case.check == "perf_gate": _perf_gate(case, xlen, request.config.getoption("--update-baselines")) return + if case.release_parity: + _release_parity(case, xlen) + return argv, env = case.run_command(xlen) rc = tc.execute(argv, env) assert rc == 0, "{} failed (exit {}): {}".format(case.id, rc, " ".join(argv)) diff --git a/ci/testcase.py b/ci/testcase.py index 0e8da16c65..2581b6e61d 100644 --- a/ci/testcase.py +++ b/ci/testcase.py @@ -106,6 +106,12 @@ def __init__(self, category, entry, driver, defaults): # coverage can't creep back in per-file. if self.check in ("model_parity", "perf_gate"): self.xlens = [32] + # release_parity: a debug run (its flags carry --debug) must report + # exactly the cycles of the same run built release. A debug build widens + # the RTL UUID to 44 bits and enables assertions and traces; none of that + # may move timing. The runner adds the release leg itself, at xlen 32 + # only like the cycle gates; other xlens run the debug leg alone. + self.release_parity = bool(entry.get("release_parity", False)) self.tolerance = float(entry.get("tolerance", defaults.get("tolerance", DEFAULT_PARITY_TOLERANCE))) self.authored_drivers = ("driver" in entry) or ("drivers" in entry) @@ -575,6 +581,13 @@ def cmd_lint(args): errors.append("{}: script case missing 'run'".format(c.id)) if any(int(x) not in (32, 64) for x in c.xlens): errors.append("{}: xlen must be 32 and/or 64".format(c.id)) + if c.release_parity: + if c.via != "blackbox" or "--debug" not in c.flags: + errors.append("{}: release_parity needs a blackbox case run " + "with --debug".format(c.id)) + if c.check: + errors.append("{}: release_parity cannot combine with a " + "check".format(c.id)) if c.check: if c.check not in VALID_CHECK: errors.append("{}: invalid check {!r}".format(c.id, c.check)) diff --git a/ci/testcases/debug.yaml b/ci/testcases/debug.yaml index ee8d117b88..21b4ccd8fc 100644 --- a/ci/testcases/debug.yaml +++ b/ci/testcases/debug.yaml @@ -66,13 +66,20 @@ tests: app: sgemm_tcu configs: -DVX_CFG_EXT_TCU_ENABLE flags: --debug=3 --log=run_simx.log +# The debug build (44-bit UUID, assertions, traces) must not move RTL timing: +# the runner reruns this case built release and requires identical cycles. The +# shape keeps the data cache under pressure, with two channels whose same-word +# reads merge, so a debug-only tag width reaching the merge shows in cycles. - id: sgemm_tcu-2 via: blackbox drivers: - rtlsim app: sgemm_tcu - configs: -DVX_CFG_EXT_TCU_ENABLE + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_DCACHE_SIZE=4096 -DVX_CFG_EXT_TCU_ENABLE + shape: + l2cache: true flags: --debug=3 --log=run_rtlsim.log + release_parity: true - id: simx_mt-1 via: blackbox drivers: diff --git a/docs/designs/continuous_integration.md b/docs/designs/continuous_integration.md index c959934e45..20000d7464 100644 --- a/docs/designs/continuous_integration.md +++ b/docs/designs/continuous_integration.md @@ -230,6 +230,18 @@ kernel is all boot and dispatch skew, and its gap ratio is noise. **Never widen a tolerance to absorb a divergence.** Model the behavior, or mark the case `known_issue` while it is investigated. +#### `release_parity` + +A debug build must not change timing. A debug build widens the RTL UUID to 44 +bits and enables assertions and traces. A blackbox case run with `--debug` can +set `release_parity: true`. The runner then reruns it built release, at +xlen 32, and requires identical instructions and cycles; it reads the debug +run's counts from its `--log` file. This is a case field, not a check, so the +case stays in its category cell and costs one extra run there, with no new +cell. `debug:sgemm_tcu-2` carries it. That case is shaped to keep the data +cache under pressure, with two channels whose same-word reads merge; a +debug-only tag width reaching the read merge would show there. + ### 3.4 `perf_gate` rtlsim cycle counts are deterministic and host-independent, so the threshold diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index d957a056a6..abd71bf8af 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -2423,6 +2423,21 @@ class CachePortArb : public SimObject { } } + // Without a registered tree level the hardware crossbar is combinational + // (no request buffer ahead of a bank): a request stays at its port until the + // bank takes it, where a later same-word read can still merge with it. The + // arbiter then offers each bank one request per cycle and dequeues the ports + // only when the bank pops the offer; an offer still pending next cycle is + // withdrawn and the ports are arbitrated afresh. + void attach_bank(uint32_t b, SimChannel* bank_in) { + if (!this->offers_enabled()) { + return; + } + offers_.resize(ReqOut.size()); + offers_.at(b).bank_in = bank_in; + bank_in->pop_callback([this, b]() { this->on_bank_pop(b); }); + } + // Bits of the merged-port mask carried below the request tag. uint32_t merge_bits() const { return (merge_group_ > 1) ? merge_group_ : 0; } @@ -2500,6 +2515,10 @@ class CachePortArb : public SimObject { std::fill(rr.begin(), rr.end(), 0); } collisions_ = 0; + for (auto& offer : offers_) { + offer.valid = false; + offer.withdrawing = false; + } } // Slices of the arbiter level s: level 0 grants the ports, each level above @@ -2524,6 +2543,14 @@ class CachePortArb : public SimObject { } uint32_t nb = ReqOut.size(); uint32_t nl = levels_.size(); + for (auto& offer : offers_) { + if (offer.valid) { + offer.withdrawing = true; + offer.bank_in->pop(); + offer.withdrawing = false; + offer.valid = false; + } + } // the tree's buffered levels, join first: each slice passes one buffer of // the level below per cycle for (uint32_t s = nl; s > 0; --s) { @@ -2622,6 +2649,10 @@ class CachePortArb : public SimObject { while (!(slice_want & (1u << j))) { j = (j + 1) % size; } + if (this->offers_enabled()) { + this->offer(b, sl * slice_ports + j, out, merged, wants.at(b) - 1, &rr, (j + 1) % size); + continue; + } this->grant(sl * slice_ports + j, out, merged); rr = (j + 1) % size; granted = true; @@ -2632,6 +2663,48 @@ class CachePortArb : public SimObject { } } + // Offer port i's request, with the ports merged into it, to bank b; the + // ports are dequeued when the bank takes it (on_bank_pop). + void offer(uint32_t b, uint32_t i, SimChannel& out, const std::vector& merged, + uint32_t collisions, uint32_t* rr, uint32_t rr_next) { + auto req = ReqIn.at(i).peek(); + uint64_t mask = (merge_group_ > 1) ? merged.at(i) : 0; + uint64_t tag = (((uint64_t(req.tag) << this->merge_bits()) | mask) << lg2_inputs_) | i; + __assert((tag >> 32) == 0, "cache bank tag overflow"); + req.tag = uint32_t(tag); + auto& offer = offers_.at(b); + offer.valid = true; + offer.port = i; + offer.mask = mask; + offer.collisions = collisions; + offer.rr = rr; + offer.rr_next = rr_next; + out.send(req, 0); + } + + void on_bank_pop(uint32_t b) { + auto& offer = offers_.at(b); + if (offer.withdrawing) { + return; + } + __assert(offer.valid, "bank took a request the arbiter did not offer"); + uint32_t i = offer.port; + if (offer.mask != 0) { + uint32_t base = (i / merge_group_) * merge_group_; + for (uint64_t f = offer.mask; f != 0; f &= f - 1) { + uint32_t port = base + __builtin_ctzll(f); + DT(3, this->name() << " read-merge: port=" << port << ", leader=" << i << ", " << ReqIn.at(port).peek()); + ReqIn.at(port).pop(); + } + } + ReqIn.at(i).pop(); + *offer.rr = offer.rr_next; + collisions_ += offer.collisions; + offer.valid = false; + } + + bool offers_enabled() const { return levels_.empty() && delay_ == 0; } + // Send port i's request to its bank, with the ports merged into it. void grant(uint32_t i, SimChannel& out, const std::vector& merged) { auto req = ReqIn.at(i).peek(); @@ -2671,6 +2744,17 @@ class CachePortArb : public SimObject { std::vector> rr_; // [arbiter level][bank][slice] next to grant uint32_t merge_group_; uint64_t collisions_ = 0; + struct offer_t { + SimChannel* bank_in = nullptr; + bool valid = false; + bool withdrawing = false; + uint32_t port = 0; + uint64_t mask = 0; + uint32_t collisions = 0; + uint32_t* rr = nullptr; + uint32_t rr_next = 0; + }; + std::vector offers_; // [bank], combinational crossbar only friend class SimObject; }; @@ -2973,6 +3057,7 @@ class Cache::Impl { // bind core ports port_arb_->ReqOut.at(i).bind(&banks_.at(i)->core_req_in); + port_arb_->attach_bank(i, &banks_.at(i)->core_req_in); banks_.at(i)->core_rsp_out.bind(rsp_mcast_ ? &rsp_mcast_->RspIn.at(i) : &bank_core_xbar_->RspIn.at(i)); // bind memory ports From 009a9d55a19038de44de42d8d2f9ac7437615302 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 01:27:53 -0700 Subject: [PATCH 107/124] tcu: fp32-only output, bit-exact TFR numerics in SimX, tcu_unit test The tensor core accumulates in fp32 (int32 for integer inputs) and has no other output format. Narrow fmt_d paths (fp16, bf16, fp8, bf8 and tf32 out, rounded at every step) existed only in SimX and in the test references, and modeled a datapath that does not exist. They are removed. vx_tensor.h rejects another output type at compile time, and VX_tcu_core and SimX assert on an instruction that encodes one. tensor:sgemm_tcu-4 and -6 now produce fp32. SimX on the TFR backend computed dot products as discrete fp32 multiply-adds, which round apart from the datapath's single windowed rounding on subnormal rows and near-cancellations. It now uses the TfrFedp model: products aligned to the largest exponent in a 25-bit window with a sticky bit, summed as integers, rounded once. The model moves from hw/unittest/tcu_fedp to sw/common/fedp.h and gains MX (E8M0) and NVFP4 (UE4M3) scales and fp4 lanes. Other backends keep the discrete fp32 order of the DPI datapath. RTL TFR fixes found by checking the datapath bit for bit: - MXFP4 E8M0 scales were added into the 6-bit term alignment field, so scales far from one (such as 2^-34) wrapped. They now join the lane exponent. - An MX fp8 lane scaled below the exponent range wrapped. The exponent sum is now one bit wider and signed, and such a lane only adds to the sticky bit. - NaN scales (E8M0 0xFF, UE4M3 0x7F) now make their block NaN. - TCU_EXP_BITS is 10 under MX. CI: unittest:hw-tcu-unit runs VX_tcu_unit end to end (dispatch, uops, operand routing, commit) at 0 ULP against TfrFedp on TFR and against the discrete reference on DPI. Directed vectors cover a subnormal fp16 row and near-cancellation. tcu_fedp gains MX/NVFP4 suites with directed scale corners. tcu_unit's format IDs now match VX_tcu_pkg, elements are packed per register, and a packet is dispatched exactly once. The WGMMA design doc gains fmt_d, the MX/NVFP4 scale semantics and TFR rounding. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/tensor.yaml | 4 +- ci/testcases/unittest.yaml | 13 +- docs/designs/tensor_core_wgmma_engine.md | 38 +++- hw/rtl/tcu/VX_tcu_core.sv | 4 + hw/rtl/tcu/VX_tcu_pkg.sv | 3 + hw/rtl/tcu/tfr/VX_tcu_tfr_mul_f4.sv | 33 ++- hw/rtl/tcu/tfr/VX_tcu_tfr_mul_f8.sv | 45 ++-- hw/unittest/Makefile | 44 ++++ hw/unittest/tcu_fedp/Makefile | 8 +- hw/unittest/tcu_fedp/main.cpp | 76 ++++++- hw/unittest/tcu_unit/Makefile | 4 +- hw/unittest/tcu_unit/main.cpp | 213 +++++++++++++----- sim/simx/tcu/tcu_unit.cpp | 152 +++++++------ {hw/unittest/tcu_fedp => sw/common}/fedp.h | 193 ++++++++++++++-- sw/common/tensor_cfg.h | 6 + sw/kernel/include/vx_tensor.h | 6 + tests/regression/sgemm2_tcu/main.cpp | 28 +-- tests/regression/sgemm_tcu/main.cpp | 69 +----- tests/regression/sgemm_tcu_mx/main.cpp | 3 + tests/regression/sgemm_tcu_sp/main.cpp | 61 +---- tests/regression/sgemm_tcu_sp_mx/main.cpp | 3 + tests/regression/sgemm_tcu_wg/main.cpp | 61 +---- tests/regression/sgemm_tcu_wg_dxa/main.cpp | 17 +- .../sgemm_tcu_wg_dxa_mcast/main.cpp | 17 +- tests/regression/sgemm_tcu_wg_mx/main.cpp | 3 + 25 files changed, 687 insertions(+), 417 deletions(-) rename {hw/unittest/tcu_fedp => sw/common}/fedp.h (64%) diff --git a/ci/testcases/tensor.yaml b/ci/testcases/tensor.yaml index 52b0799215..dbcba1d2f5 100644 --- a/ci/testcases/tensor.yaml +++ b/ci/testcases/tensor.yaml @@ -31,7 +31,7 @@ tests: drivers: - simx app: sgemm_tcu - configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=bf16 -DOTYPE=bf16 + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=bf16 -DOTYPE=fp32 - id: sgemm_tcu-5 via: blackbox drivers: @@ -43,7 +43,7 @@ tests: drivers: - simx app: sgemm_tcu - configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DITYPE=bf8 -DOTYPE=bf8 + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DITYPE=bf8 -DOTYPE=fp32 - id: sgemm_tcu-7 via: blackbox drivers: diff --git a/ci/testcases/unittest.yaml b/ci/testcases/unittest.yaml index 8765f57fac..cd3cf35d8c 100644 --- a/ci/testcases/unittest.yaml +++ b/ci/testcases/unittest.yaml @@ -23,14 +23,23 @@ tests: - id: hw-tcu via: script run: "make -C hw/unittest run-tcu" - touches: [hw/rtl/tcu, hw/unittest/tcu_fedp] + touches: [hw/rtl/tcu, hw/unittest/tcu_fedp, sw/common/fedp.h] + # VX_tcu_unit end to end (dispatch, uops, operand routing, commit), bit-exact + # against each datapath's reference: native TFR against the TfrFedp reference + # model, DPI against its discrete mul/add. Then the native FEDP's + # fp8 and block-scaled (MX/NVFP4) formats against the same model. The `hw` + # lane only builds these. + - id: hw-tcu-unit + via: script + run: "make -C hw/unittest run-tcu-unit" + touches: [hw/rtl/tcu, hw/dpi, hw/unittest/tcu_unit, hw/unittest/tcu_fedp, sw/common/fedp.h] # Same native-TFR formats with the multipliers mapped onto DSP48 (USE_DSP=1) — # the FPGA-synthesis path, off by default in sim. Bit-identical to LUT by # contract, so a mismatch is a DSP-packing bug. - id: hw-tcu-dsp via: script run: "make -C hw/unittest run-tcu-dsp" - touches: [hw/rtl/tcu, hw/unittest/tcu_fedp] + touches: [hw/rtl/tcu, hw/unittest/tcu_fedp, sw/common/fedp.h] # The command processor's DMA port into the last-level cache: a launch behind # DMA writes must wait for them to reach the cache. No runtime path launches # right behind a DMA write, so no system test observes this ordering. diff --git a/docs/designs/tensor_core_wgmma_engine.md b/docs/designs/tensor_core_wgmma_engine.md index ad890708d8..eefc1a8a15 100644 --- a/docs/designs/tensor_core_wgmma_engine.md +++ b/docs/designs/tensor_core_wgmma_engine.md @@ -67,6 +67,11 @@ Flags of an MMA instruction: | `rs2[2:1]` | `cd_nregs` — accumulator registers: 0 = 8, 1 = 16, 2 = 32 | | `rs2[3]` | `a_from_smem` — A comes from shared memory; B always does for WGMMA | +`fmt_d` is fp32 for float and MX inputs and int32 for integer inputs; the +tensor core has no other output format. The kernel API rejects any other output +type at compile time (`static_assert` in `vx_tensor.h`). An instruction that +encodes one anyway trips an assertion in the RTL (`VX_tcu_core`) and in SimX. + | `op_type` | value | operation | |---|---|---| | `INST_TCU_WMMA` | 0 | warp MMA | @@ -92,6 +97,16 @@ micro-op arguments do not carry a sparse bit. An MX format has bit 3 of the id set, which is how decode recognizes that the instruction depends on scale metadata. +Scales follow the OCP MX and NVFP4 definitions: + +- **mxfp8, mxbf8, mxfp4** take an E8M0 scale per 32 elements, worth + 2^(e-127). A product of two scales spans 2^-254 to 2^254. 0xFF is NaN. +- **nvfp4** takes a UE4M3 scale per 16 elements: unsigned, so bit 7 is + padding, and 0x7F is NaN (PTX `.ue4m3`). A zero exponent field is read as a + normal; subnormal scales are an open question. +- **Out-of-range products.** A NaN scale makes its block NaN. A product scaled + below the datapath's exponent range only reaches the sticky bit. + ### 2.2 Metadata dependencies Metadata is not an architectural register, so the scoreboard tracks it with @@ -398,9 +413,10 @@ MMA (§2.2). ## 8. FEDP backends -`VX_CFG_TCU_TYPE` selects one implementation of the dot product. All compute -the same function; each asserts the configured latency against its own -stage structure. +`VX_CFG_TCU_TYPE` selects one implementation of the dot product. Each asserts +the configured latency against its own stage structure. They agree up to +rounding, not bit for bit: `TFR` rounds once over a fixed window, the others +round each multiply and add. | backend | implementation | latency | |---|---|---| @@ -412,8 +428,13 @@ stage structure. `TFR` is the universal default. It is the only backend that does not pay a full floating-point rounding per tree level: it aligns every product to one -exponent and adds them as integers, rounding once. On an FPGA its -multipliers map onto DSP blocks through `VX_CFG_TCU_USE_DSP`. +exponent and adds them as integers, rounding once. The alignment keeps a +25-bit window below the largest product's exponent field, with a sticky bit +for what falls out of it. A row of subnormals, or products that nearly cancel, +can therefore round a few ULP away from the exact sum. +[`sw/common/fedp.h`](../../sw/common/fedp.h) (`TfrFedp`) models it bit for +bit. On an FPGA its multipliers map onto DSP blocks through +`VX_CFG_TCU_USE_DSP`. --- @@ -432,6 +453,13 @@ multipliers map onto DSP blocks through `VX_CFG_TCU_USE_DSP`. Operand loads go through simulation channels; the tensor path never reads memory directly. +Numerics follow the configured backend for fp32 results from float and +block-scaled (MX, NVFP4) sources. Under `TFR`, each dot product is the +`TfrFedp` model, the same one the RTL is checked against bit for bit in +`hw-tcu-unit`. Otherwise it is a discrete fp32 multiply-add: in the DPI +backend's order for unscaled formats, and per element for block-scaled ones. +C and D are fp32 (int32 for integer inputs) in both models; see §2. + Perf: MPM class `TCU` — `TCU_TBUF_STALLS` (0xB03, cycles a WGMMA micro-op waited on tile-buffer data), `TCU_TBUF_CACHE_HITS` (0xB04, B rows reused), `TCU_LMEM_READS` (0xB05). diff --git a/hw/rtl/tcu/VX_tcu_core.sv b/hw/rtl/tcu/VX_tcu_core.sv index b70a8e3a50..035186b074 100644 --- a/hw/rtl/tcu/VX_tcu_core.sv +++ b/hw/rtl/tcu/VX_tcu_core.sv @@ -153,6 +153,10 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( wire setup_enqueue = execute_fire && is_wgmma_setup; wire fedp_enqueue = execute_fire && !is_wgmma_setup; + // The accumulator is fp32 (int32 for integer inputs); no other fmt_d exists. + `RUNTIME_ASSERT (~fedp_enqueue || (fmt_d == (tcu_fmt_is_int(fmt_s) ? TCU_FMT_WIDTH'(TCU_I32_ID) : TCU_FMT_WIDTH'(TCU_FP32_ID))), + ("%t: *** %s unsupported TCU output format: fmt_s=%0d, fmt_d=%0d (fp32 only)", $time, INSTANCE_ID, fmt_s, fmt_d)) + // ----------------------------------------------------------------------- // Sparse metadata: VX_tcu_sp_meta (for WMMA_SP) + optional tile-buffer mux // ----------------------------------------------------------------------- diff --git a/hw/rtl/tcu/VX_tcu_pkg.sv b/hw/rtl/tcu/VX_tcu_pkg.sv index 7c0c76d7fa..10d4940158 100644 --- a/hw/rtl/tcu/VX_tcu_pkg.sv +++ b/hw/rtl/tcu/VX_tcu_pkg.sv @@ -216,10 +216,13 @@ package VX_tcu_pkg; localparam TCU_MX_MAX_SF = `MAX(`MAX(TCU_MX_SF_FP8, TCU_MX_SF_MXFP4), TCU_MX_SF_NVFP4); + // MX block scales span 2^-254 .. 2^256 on top of the elements' own range. `ifdef VX_CFG_TCU_TF32_ENABLE localparam TCU_EXP_BITS = 10; `elsif VX_CFG_TCU_FP16_ENABLE localparam TCU_EXP_BITS = 10; + `elsif VX_CFG_TCU_MX_ENABLE + localparam TCU_EXP_BITS = 10; `else localparam TCU_EXP_BITS = 9; `endif diff --git a/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_f4.sv b/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_f4.sv index bdd2b2354e..8f65a751ac 100644 --- a/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_f4.sv +++ b/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_f4.sv @@ -94,6 +94,7 @@ module VX_tcu_tfr_mul_f4 import VX_tcu_pkg::*; wire [3:0][23:0] term_mag_shifted; wire [3:0][EXP_TERM_W_MXFP4-1:0] term_exp_biased; wire [3:0] term_valid; + wire [3:0] term_on; wire [3:0] term_sign; wire [3:0][1:0] a_man, b_man; wire [3:0][3:0] f4_man_prod; @@ -102,6 +103,7 @@ module VX_tcu_tfr_mul_f4 import VX_tcu_pkg::*; localparam OFF = (i % 2) * 16 + j * 4; wire lane_valid = vld_mask[i * 4 + j]; + assign term_on[j] = lane_valid; wire [3:0] raw_a = a_row[K_WORD][OFF +: 4]; wire [3:0] raw_b = b_col[K_WORD][OFF +: 4]; @@ -119,13 +121,11 @@ module VX_tcu_tfr_mul_f4 import VX_tcu_pkg::*; assign b_exp[0] = raw_b[2] & ~raw_b[1]; assign b_exp[1] = raw_b[2] & raw_b[1]; - wire signed [9:0] sf_exp_a = $signed({1'b0, sf_a}) - 10'sd127; - wire signed [9:0] sf_exp_b = $signed({1'b0, sf_b}) - 10'sd127; - wire signed [5:0] exp_biased_raw = 6'(10'(EXP_ADJ_MXFP4) - + sf_exp_a - + sf_exp_b - + 10'(a_exp) - + 10'(b_exp)); + // the block scales are common to the lane's terms: they join its + // exponent, not this alignment field + wire [EXP_TERM_W_MXFP4-1:0] exp_biased_raw = EXP_ADJ_MXFP4 + + EXP_TERM_W_MXFP4'(a_exp) + + EXP_TERM_W_MXFP4'(b_exp); assign term_exp_biased[j] = term_valid[j] ? exp_biased_raw : '0; end @@ -250,10 +250,16 @@ module VX_tcu_tfr_mul_f4 import VX_tcu_pkg::*; assign result_sig_mxfp4[i] = {sum_sign & ~is_zero_out, abs_sum[23:0]}; assign sig_zero_mxfp4[i] = is_zero_out; - assign result_exp_mxfp4[i] = EXP_W'(max_exp_biased) + EXP_W'(EXP_BASE_BIASED_MXFP4); + + // The E8M0 scales (2^-254 .. 2^256 together) join the lane exponent. + // A lane with a term stays within [15, 529]; an empty one wraps, but + // its exponent is cleared with its zero significand. + assign result_exp_mxfp4[i] = EXP_W'(max_exp_biased) + EXP_BASE_BIASED_MXFP4 + + EXP_W'(sf_a) + EXP_W'(sf_b) - EXP_W'(254); // The sign field is only consumed for infinity lanes downstream. - assign exceptions_mxfp4[i].is_nan = 1'b0; + // An E8M0 scale of 0xFF is NaN, and so is every element it scales. + assign exceptions_mxfp4[i].is_nan = ((sf_a == 8'hFF) || (sf_b == 8'hFF)) && (|term_on); assign exceptions_mxfp4[i].is_inf = 1'b0; assign exceptions_mxfp4[i].sign = 1'b0; end @@ -284,13 +290,14 @@ module VX_tcu_tfr_mul_f4 import VX_tcu_pkg::*; for (genvar i = 0; i < TCK; ++i) begin : g_lane_nvfp4 localparam K_WORD = i / 2; - `UNUSED_VAR ({sf_a[7], sf_b[7]}) - // e4m3 scale factor mantissa mul + // UE4M3 scale factors (unsigned: bit 7 is padding); 0x7F is NaN. + `UNUSED_VAR ({sf_a[7], sf_b[7]}) wire [3:0] sf_man_a = {1'b1, sf_a[2:0]}; wire [3:0] sf_man_b = {1'b1, sf_b[2:0]}; wire [3:0] sf_exp_a = sf_a[6:3]; wire [3:0] sf_exp_b = sf_b[6:3]; + wire sf_nan = (sf_a[6:0] == 7'h7F) || (sf_b[6:0] == 7'h7F); wire [7:0] sf_man_prod; VX_tcu_tfr_wmul #( @@ -307,6 +314,7 @@ module VX_tcu_tfr_mul_f4 import VX_tcu_pkg::*; wire [3:0][23:0] term_mag_shifted; wire [3:0][5:0] term_exp_biased; wire [3:0] term_valid; + wire [3:0] term_on; wire [3:0] term_sign; wire [3:0][1:0] a_man, b_man; wire [3:0][3:0] f4_man_prod; @@ -316,6 +324,7 @@ module VX_tcu_tfr_mul_f4 import VX_tcu_pkg::*; localparam OFF = (i % 2) * 16 + j * 4; wire lane_valid = vld_mask[i * 4 + j]; + assign term_on[j] = lane_valid; wire [3:0] raw_a = a_row[K_WORD][OFF +: 4]; wire [3:0] raw_b = b_col[K_WORD][OFF +: 4]; @@ -511,7 +520,7 @@ module VX_tcu_tfr_mul_f4 import VX_tcu_pkg::*; assign result_exp_nvfp4[i] = EXP_W'(max_exp_biased) + EXP_W'(EXP_BASE_BIASED); // The sign field is only consumed for infinity lanes downstream. - assign exceptions_nvfp4[i].is_nan = 1'b0; + assign exceptions_nvfp4[i].is_nan = sf_nan && (|term_on); assign exceptions_nvfp4[i].is_inf = 1'b0; assign exceptions_nvfp4[i].sign = 1'b0; end diff --git a/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_f8.sv b/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_f8.sv index 60d3c41503..1c8c3cbbc6 100644 --- a/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_f8.sv +++ b/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_f8.sv @@ -194,24 +194,28 @@ module VX_tcu_tfr_mul_f8 import VX_tcu_pkg::*; wire [7:0] bias_sel = is_bfloat ? BIAS_CONST_BF8 : BIAS_CONST_FP8; - wire [EXP_W-1:0] max_pre_sum_cpa, bias_sel_cpa; - wire [EXP_W-1:0] final_exp; + // One bit wider than the exponent, signed: the E8M0 scales (2^-254 .. + // 2^256 together) can take a lane below the exponent range, where it + // only adds to the sticky bit. + localparam SX = EXP_W + 1; + wire [SX-1:0] max_pre_sum_cpa, bias_sel_cpa; + wire [SX-1:0] final_exp_s; `ifdef VX_CFG_TCU_MX_ENABLE - wire [EXP_W-1:0] bias_cpa_sum, bias_cpa_carry; - wire [3*EXP_W-1:0] sf_comp = fmt_f[3] ? {EXP_W'(sf_a), EXP_W'(sf_b), -EXP_W'(254)} : (3*EXP_W)'(0); + wire [SX-1:0] bias_cpa_sum, bias_cpa_carry; + wire [3*SX-1:0] sf_comp = fmt_f[3] ? {SX'(sf_a), SX'(sf_b), -SX'(254)} : (3*SX)'(0); VX_csa_tree #( .N(4), - .W(EXP_W), - .S(EXP_W) + .W(SX), + .S(SX) ) exp_sf_csa ( - .operands ({EXP_W'(bias_sel), sf_comp}), + .operands ({SX'(bias_sel), sf_comp}), .sum (bias_cpa_sum), .carry (bias_cpa_carry) ); VX_ks_adder #( - .N(EXP_W), - .BYPASS(`FORCE_BUILTIN_ADDER(EXP_W)) + .N(SX), + .BYPASS(`FORCE_BUILTIN_ADDER(SX)) ) exp_bias_add ( .dataa(bias_cpa_sum), .datab(bias_cpa_carry), @@ -220,19 +224,20 @@ module VX_tcu_tfr_mul_f8 import VX_tcu_pkg::*; `UNUSED_PIN(cout) ); `else - assign bias_sel_cpa = EXP_W'(bias_sel); + assign bias_sel_cpa = SX'(bias_sel); `endif - assign max_pre_sum_cpa = EXP_W'(max_pre_sum); + assign max_pre_sum_cpa = SX'(max_pre_sum); VX_ks_adder #( - .N(EXP_W), - .BYPASS(`FORCE_BUILTIN_ADDER(EXP_W)) + .N(SX), + .BYPASS(`FORCE_BUILTIN_ADDER(SX)) ) exp_final_add ( - .dataa(EXP_W'(max_pre_sum_cpa)), - .datab(EXP_W'(bias_sel_cpa)), + .dataa(max_pre_sum_cpa), + .datab(bias_sel_cpa), .cin(1'b0), - .sum(final_exp), + .sum(final_exp_s), `UNUSED_PIN(cout) ); + wire [EXP_W-1:0] final_exp = final_exp_s[SX-1] ? '0 : final_exp_s[EXP_W-1:0]; // The two fp8 mantissa products in a lane are independent unsigned 4x4 // multiplies — small enough to pack into a single DSP48 (USE_DSP), @@ -335,7 +340,13 @@ module VX_tcu_tfr_mul_f8 import VX_tcu_pkg::*; wire add_nan = (pos_inf_0 && neg_inf_1) || (neg_inf_0 && pos_inf_1); - wire any_nan = (|(nan_sel & lane_valid)) || add_nan; + `ifdef VX_CFG_TCU_MX_ENABLE + // an E8M0 scale of 0xFF is NaN, and so is every element it scales + wire sf_nan = fmt_f[3] && ((sf_a == 8'hFF) || (sf_b == 8'hFF)) && (|lane_valid); + `else + wire sf_nan = 1'b0; + `endif + wire any_nan = (|(nan_sel & lane_valid)) || add_nan || sf_nan; wire any_inf = (|(inf_sel & lane_valid)) && ~any_nan; wire final_sign_inf = inf_sel[0] ? sign_sel[0] : sign_sel[1]; diff --git a/hw/unittest/Makefile b/hw/unittest/Makefile index 4f2916de0f..a53bf2b353 100644 --- a/hw/unittest/Makefile +++ b/hw/unittest/Makefile @@ -125,6 +125,50 @@ run-tcu: $(call tcu_int_pair,DPI,INT8,17,18) $(call tcu_int_pair,DPI,INT4,19,20) +# VX_tcu_unit end to end (dispatch, uops, operand routing, commit), bit for bit +# (--ulp=0) against the reference of each datapath: the native TFR one against +# the TfrFedp reference model, the DPI one against its discrete fp32 mul/add. +# One build per tile shape and datapath: 4 threads (2x2) with tf32/fp16/bf16, +# 8 threads (4x2) fp16 on DPI. +TCU_UNIT_TFR := $(TCU_BASE) -DVX_CFG_TCU_TYPE_TFR -DVX_CFG_TCU_TF32_ENABLE +TCU_UNIT_DPI := $(TCU_BASE) -DVX_CFG_TCU_TYPE_DPI +run-tcu-unit: + $(MAKE) -C tcu_unit clean + $(MAKE) -C tcu_unit CONFIGS="$(TCU_UNIT_TFR)" + $(MAKE) -C tcu_unit run CONFIGS="$(TCU_UNIT_TFR)" OPTS="--fmt=1 --ulp=0 --no-trace" + $(MAKE) -C tcu_unit run CONFIGS="$(TCU_UNIT_TFR)" OPTS="--fmt=2 --ulp=0 --no-trace" + $(MAKE) -C tcu_unit run CONFIGS="$(TCU_UNIT_TFR)" OPTS="--fmt=3 --ulp=0 --no-trace" + $(MAKE) -C tcu_unit clean + $(MAKE) -C tcu_unit CONFIGS="$(TCU_UNIT_DPI) -DVX_CFG_TCU_TF32_ENABLE" + $(MAKE) -C tcu_unit run CONFIGS="$(TCU_UNIT_DPI) -DVX_CFG_TCU_TF32_ENABLE" OPTS="--fmt=1 --ulp=0 --no-trace" + $(MAKE) -C tcu_unit run CONFIGS="$(TCU_UNIT_DPI) -DVX_CFG_TCU_TF32_ENABLE" OPTS="--fmt=2 --ulp=0 --no-trace" + $(MAKE) -C tcu_unit run CONFIGS="$(TCU_UNIT_DPI) -DVX_CFG_TCU_TF32_ENABLE" OPTS="--fmt=3 --ulp=0 --no-trace" + $(MAKE) -C tcu_unit clean + $(MAKE) -C tcu_unit CONFIGS="$(TCU_UNIT_DPI) -DVX_CFG_NUM_THREADS=8" + $(MAKE) -C tcu_unit run CONFIGS="$(TCU_UNIT_DPI) -DVX_CFG_NUM_THREADS=8" OPTS="--fmt=2 --ulp=0 --no-trace" + $(call tcu_mx_suite,2,1) + $(call tcu_mx_suite,4,2) + +# The native datapath's fp8 and block-scaled formats against the TfrFedp +# reference model, bit for bit: fp8/bf8, MXFP8/MXBF8 (E8M0 scales), MXFP4 (E8M0) +# and NVFP4 (E4M3). VX_tcu_unit takes its block scales from the metadata path, +# so these run on the FEDP. Two shapes: 2 registers on one scale slot, 4 on two. +# Directed legs pin zero products in an fp8 lane, MXFP4 scales far from one +# (2^-34), MXFP8/MXBF8 scales so small the lane underflows, and NaN scales. +TCU_MX := $(TCU_BASE) -DVX_CFG_TCU_TYPE_TFR -DUSE_FEDP -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP8_ENABLE \ + -DVX_CFG_TCU_FP4_ENABLE -DVX_CFG_TCU_MXFP4_ENABLE -DVX_CFG_TCU_NVFP4_ENABLE +# format:sf_a:sf_b +TCU_MX_DIRECTED := 10:110:110 8:0:3 9:4:2 8:0xff:127 11:0x7f:0x38 +tcu_mx_opts = --fmt=$(word 1,$(subst :, ,$(1))) --sfa=$(word 2,$(subst :, ,$(1))) --sfb=$(word 3,$(subst :, ,$(1))) +# $(1)=NUM_REGS $(2)=SF +define tcu_mx_suite + $(MAKE) -C tcu_fedp clean + $(MAKE) -C tcu_fedp NUM_REGS=$(1) SF=$(2) CONFIGS="$(TCU_MX)" + $(foreach f,4 5 8 9 10 11,$(MAKE) -C tcu_fedp run NUM_REGS=$(1) SF=$(2) CONFIGS="$(TCU_MX)" OPTS="--fmt=$(f) --W=25 --ulp=0 --no-trace" &&) true + $(foreach f,4 8 9,$(MAKE) -C tcu_fedp run NUM_REGS=$(1) SF=$(2) CONFIGS="$(TCU_MX)" OPTS="--fmt=$(f) --W=25 --ulp=0 --features=zeros --tests=20000 --no-trace" &&) true + $(foreach o,$(TCU_MX_DIRECTED),$(MAKE) -C tcu_fedp run NUM_REGS=$(1) SF=$(2) CONFIGS="$(TCU_MX)" OPTS="$(call tcu_mx_opts,$(o)) --W=25 --ulp=0 --tests=20000 --no-trace" &&) true +endef + # Same native-TFR formats as run-tcu, but with the multipliers mapped onto DSP48 # (USE_DSP=1). USE_DSP is bit-identical to the LUT path by contract, so the same # references apply — a mismatch here is a DSP-packing bug. diff --git a/hw/unittest/tcu_fedp/Makefile b/hw/unittest/tcu_fedp/Makefile index 7f095276fe..507bbee44c 100644 --- a/hw/unittest/tcu_fedp/Makefile +++ b/hw/unittest/tcu_fedp/Makefile @@ -26,8 +26,14 @@ XCONFIGS := $(shell python3 $(ROOT_DIR)/ci/gen_config.py --config=$(VORTEX_HOME) # configured VX_CFG_TCU_LATENCY. LATENCY ?= $(patsubst -DVX_CFG_TCU_LATENCY=%,%,$(filter -DVX_CFG_TCU_LATENCY=%, $(XCONFIGS))) -CXXFLAGS += -DNUM_REGS=$(NUM_REGS) -DLATENCY=$(LATENCY) +# Block-scale slots of an MX datapath: the lanes split evenly across them. +SF ?= 1 + +CXXFLAGS += -DNUM_REGS=$(NUM_REGS) -DLATENCY=$(LATENCY) -DSF=$(SF) PARAMS := -GN=$(NUM_REGS) -GLATENCY=$(LATENCY) +ifneq (,$(filter -DVX_CFG_TCU_MX_ENABLE, $(XCONFIGS))) +PARAMS += -GSF=$(SF) +endif SRCS += $(DPI_DIR)/dpi_util.cpp SRCS += $(DPI_DIR)/dpi_float.cpp diff --git a/hw/unittest/tcu_fedp/main.cpp b/hw/unittest/tcu_fedp/main.cpp index 3e11bd1dde..5e60fbcaa1 100644 --- a/hw/unittest/tcu_fedp/main.cpp +++ b/hw/unittest/tcu_fedp/main.cpp @@ -158,6 +158,19 @@ void init_default_fp_format(uint32_t fmt, int *exp_bits, int *sig_bits) { *exp_bits = 5; *sig_bits = 2; break; + case 8: // mxfp8 (E4M3, E8M0 block scale) + *exp_bits = 4; + *sig_bits = 3; + break; + case 9: // mxbf8 (E5M2, E8M0 block scale) + *exp_bits = 5; + *sig_bits = 2; + break; + case 10: // mxfp4 (E2M1, E8M0 block scale) + case 11: // nvfp4 (E2M1, E4M3 block scale) + *exp_bits = 2; + *sig_bits = 1; + break; default: break; } @@ -187,6 +200,7 @@ struct TestConfig { int ulp = 1; // float precision error bits int sparsity = 0; // % of zero inputs int32_t test_id = -1; // Specific test ID to run (for debugging) + int sf_a = -1, sf_b = -1; // fixed block scales (debugging), random when negative }; std::ostream &operator<<(std::ostream &os, const TestConfig &config) { @@ -255,9 +269,17 @@ static void print_format(const std::string& prefix, std::vector& value static int float_fmt_width(int exp_bits, int sig_bits) { int element_bits = 1 + exp_bits + sig_bits; + if (element_bits <= 4) return 4; // fp4 packs eight to a register return 8 * ((element_bits + 7) / 8); } +// Block-scaled (MX/NV) formats carry the 2nd MSB of the format code. +static bool is_mx_fmt(uint32_t fmt) { return (fmt & 0x18) == 0x8; } + +#ifndef SF +#define SF 1 +#endif + static int int_fmt_width(int fmt) { switch (fmt) { case 16: return 32; // int32 @@ -486,6 +508,20 @@ class Testbench { return value; } + // A block scale: E8M0 (MX) or UE4M3 (NVFP4, NaN at 0x7F). Half the draws + // stay within 2^+-8 of one, the rest span the whole encoding. + uint8_t generate_scale(uint32_t fmt) { + std::uniform_int_distribution coin(0, 1); + if (fmt == 11) { + std::uniform_int_distribution any(0, 0x7f); + std::uniform_int_distribution near(0x30, 0x48); + return uint8_t(coin(rng_) ? near(rng_) : any(rng_)); + } + std::uniform_int_distribution any(0, 255); + std::uniform_int_distribution near(127 - 8, 127 + 8); + return uint8_t(coin(rng_) ? near(rng_) : any(rng_)); + } + uint32_t generate_fp_value(const std::string &feature, uint32_t exp_bits, uint32_t sig_bits, uint32_t test_id) { const uint32_t all_exp = (exp_bits == 32 ? 0xFFFFFFFFu : ((1u << exp_bits) - 1u)); const uint32_t max_frac = (sig_bits == 32 ? 0xFFFFFFFFu : ((1u << sig_bits) - 1u)); @@ -739,7 +775,7 @@ class Testbench { const uint32_t tests_per_feature = (NT + NF - 1) / NF; #ifdef USE_FEDP - FEDP fedp(config_.exp_bits, config_.sig_bits, (int)config_.frm, config_.W, config_.renorm); + TfrFedp fedp(config_.exp_bits, config_.sig_bits, (int)config_.frm, config_.W, config_.renorm); #endif uint32_t skipped = 0; @@ -780,6 +816,24 @@ class Testbench { uint32_t c_value_hex; std::memcpy(&c_value_hex, &c_value_float, sizeof(float)); + // Block scales, one per scale slot; lane l (two per register) takes + // slot l * SF / lanes, as the datapath maps them. + const bool mx = is_mx_fmt(config_.fmt_s); + const int lanes = 2 * NUM_REGS; + uint8_t sf_slot_a[SF] = {}, sf_slot_b[SF] = {}; + std::vector sf_lane_a(lanes), sf_lane_b(lanes); + if (mx) { + for (int k = 0; k < SF; ++k) { + sf_slot_a[k] = generate_scale(config_.fmt_s); + sf_slot_b[k] = generate_scale(config_.fmt_s); + if (config_.sf_a >= 0) sf_slot_a[k] = uint8_t(config_.sf_a); + if (config_.sf_b >= 0) sf_slot_b[k] = uint8_t(config_.sf_b); + } + for (int l = 0; l < lanes; ++l) { + sf_lane_a[l] = sf_slot_a[(l * SF) / lanes]; + sf_lane_b[l] = sf_slot_b[(l * SF) / lanes]; + } + } // skip if not in selected test id if (config_.test_id >= 0 && test_id != config_.test_id) continue; @@ -793,6 +847,11 @@ class Testbench { pack_elements(a_value_hex, element_bits, NUM_REGS, a_packed.data()); pack_elements(b_value_hex, element_bits, NUM_REGS, b_packed.data()); + #ifdef VX_CFG_TCU_MX_ENABLE + std::memcpy(&dut_->sf_a, sf_slot_a, SF); + std::memcpy(&dut_->sf_b, sf_slot_b, SF); + #endif + // Apply to DUT for (int i = 0; i < NUM_REGS; i++) { WRITE_WDATA(dut_->a_row, i, a_packed[i]); @@ -821,7 +880,9 @@ class Testbench { std::memcpy(&dut_result, &dut_result_bits, sizeof(float)); #ifdef USE_FEDP - float expected = fedp(a_packed.data(), b_packed.data(), c_value_float, NUM_REGS); + float expected = mx ? fedp(a_packed.data(), b_packed.data(), c_value_float, NUM_REGS, + sf_lane_a.data(), sf_lane_b.data(), config_.fmt_s == 11) + : fedp(a_packed.data(), b_packed.data(), c_value_float, NUM_REGS); #else float expected = dot_product(a_value_hex.data(), b_value_hex.data(), c_value_hex, total_elements, config_.exp_bits, config_.sig_bits, config_.fused); @@ -848,6 +909,13 @@ class Testbench { print_float(" bf_values=", b_values_float, true); print_format(" bx_values=", b_value_hex, true); print_float(" c_value=", c_value_float, true); + if (mx) { + std::cout << " sf_a=" << std::hex; + for (int k = 0; k < SF; ++k) std::cout << " 0x" << int(sf_slot_a[k]); + std::cout << ", sf_b="; + for (int k = 0; k < SF; ++k) std::cout << " 0x" << int(sf_slot_b[k]); + std::cout << std::dec << std::endl; + } print_float(" expected=", expected, true); print_float(" actual=", dut_result, true); std::cout << " delta=" << delta << std::endl; @@ -927,6 +995,10 @@ TestConfig parse_args(int argc, char **argv) { config_.num_tests = std::stoi(arg.substr(8)); } else if (arg.substr(0, 7) == "--test=") { config_.test_id = std::stoi(arg.substr(7)); + } else if (arg.substr(0, 6) == "--sfa=") { + config_.sf_a = std::stoi(arg.substr(6), nullptr, 0); + } else if (arg.substr(0, 6) == "--sfb=") { + config_.sf_b = std::stoi(arg.substr(6), nullptr, 0); } else if (arg.substr(0, 7) == "--seed=") { config_.random_seed = std::stoi(arg.substr(7)); } else { diff --git a/hw/unittest/tcu_unit/Makefile b/hw/unittest/tcu_unit/Makefile index 25a468a2e5..936f24d4ea 100644 --- a/hw/unittest/tcu_unit/Makefile +++ b/hw/unittest/tcu_unit/Makefile @@ -56,12 +56,14 @@ RTL_INCLUDE += -I$(THIRD_PARTY_DIR)/hardfloat/source/RISCV TOP := VX_tcu_fedp_bhf endif +# the native TFR datapath is checked against tcu_fedp's windowed FEDP model ifneq (,$(filter -DVX_CFG_TCU_TYPE_TFR, $(XCONFIGS))) RTL_INCLUDE += -I$(RTL_DIR)/tcu/tfr +CXXFLAGS += -DUSE_FEDP endif # default test: fp16 -OPTS ?= --fmt=1 +OPTS ?= --fmt=2 TOP := VX_tcu_unit_top diff --git a/hw/unittest/tcu_unit/main.cpp b/hw/unittest/tcu_unit/main.cpp index 9b52e631e2..847e57b3fc 100644 --- a/hw/unittest/tcu_unit/main.cpp +++ b/hw/unittest/tcu_unit/main.cpp @@ -43,6 +43,9 @@ #include "VX_config.h" #include "softfloat_ext.h" #include +#ifdef USE_FEDP +#include "fedp.h" +#endif bool sim_trace_enabled() { return true; } @@ -74,9 +77,9 @@ constexpr int LANE_WORDS = VX_CFG_XLEN / 32; // TCU format IDs (must match VX_tcu_pkg.sv) constexpr uint32_t TCU_FP32_ID = 0; -constexpr uint32_t TCU_FP16_ID = 1; -constexpr uint32_t TCU_BF16_ID = 2; -constexpr uint32_t TCU_TF32_ID = 5; +constexpr uint32_t TCU_TF32_ID = 1; +constexpr uint32_t TCU_FP16_ID = 2; +constexpr uint32_t TCU_BF16_ID = 3; // WMMA op_type (INST_TCU_WMMA = 4'h0 from VX_gpu_pkg.sv) constexpr uint32_t INST_TCU_WMMA = 0; @@ -114,6 +117,8 @@ struct FmtInfo { int exp_bits; int sig_bits; int width_bits; // 16, 19 (tf32), or 32 + // a 32-bit register packs this many elements, element e in bits [e*width +: width] + int per_word() const { return 32 / width_bits; } }; static FmtInfo get_fmt_info(uint32_t fmt_id) { @@ -129,41 +134,54 @@ static FmtInfo get_fmt_info(uint32_t fmt_id) { } // Convert a packed source-format element to float (for software reference). -static float elem_to_float(uint32_t raw, const FmtInfo &fi) { +[[maybe_unused]] static float elem_to_float(uint32_t raw, const FmtInfo &fi) { return cvt_custom_to_f32(raw, fi.exp_bits, fi.sig_bits, 0 /*RNE*/, nullptr); } // ---- Software WMMA reference ----------------------------------------------- -// Computes D[t] = sum_k(A[i][k] * B[k][j]) + C[t] -// where t = i*TC_N + j, A in row-major, B in col-major (each column consecutive). -// -// rs1_data layout: lane t -> A[i][k] for t = i*TC_K + k (A in row-major) -// BUT each OUTPUT thread t=(i,j) shares the same A row. -// a_row[k] = rs1_data[i * TC_K + k] (all TC_N output threads in the same row share this) -// rs2_data layout: lane t -> B[k][j] for t = j*TC_K + k (B^T in row-major, i.e. B columns packed) -// b_col[k] = rs2_data[j * TC_K + k] -// rs3_data layout: lane t -> C[i][j] where t = i*TC_N + j -// +// D[t] = sum_k(A[i][k] * B[k][j]) + C[t], t = i*TC_N + j. Each register lane +// packs per_word() elements: +// rs1 lane i*TC_K + w holds A[i][w*per_word() ...] (row i of A) +// rs2 lane j*TC_K + w holds B[w*per_word() ...][j] (column j of B) +// rs3 lane t holds C[t] (fp32) +// The native TFR datapath is checked against the windowed FEDP model +// (USE_FEDP, as tcu_fedp); the DPI datapath against its discrete fp32 mul/add. static std::vector wmma_ref( - const std::vector &rs1, // VX_CFG_NUM_THREADS elements (A, row-major) - const std::vector &rs2, // VX_CFG_NUM_THREADS elements (B columns, col-major) - const std::vector &rs3, // VX_CFG_NUM_THREADS elements (C, row-major) - const FmtInfo &fi_s) // source format (for A and B) + const std::vector &rs1, + const std::vector &rs2, + const std::vector &rs3, + const FmtInfo &fi_s, + int window) { std::vector result(VX_CFG_NUM_THREADS); - +#ifdef USE_FEDP + TfrFedp fedp(fi_s.exp_bits, fi_s.sig_bits, 0 /*RNE*/, window); +#else + (void)window; +#endif for (int i = 0; i < TCU_TC_M; i++) { for (int j = 0; j < TCU_TC_N; j++) { int t = i * TCU_TC_N + j; - // Accumulate with fp32 rounding at each multiply-add step (matches DPI model). - float prod = 0.0f; + float c = bit_cast(rs3[t]); +#ifdef USE_FEDP + result[t] = fedp(&rs1[i * TCU_TC_K], &rs2[j * TCU_TC_K], c, TCU_TC_K); +#else + const int epw = fi_s.per_word(); + const uint32_t emask = (fi_s.width_bits == 32) ? ~0u : ((1u << fi_s.width_bits) - 1u); + // the DPI datapath's order: each register's products summed in + // fp32, then the register sums in order, then C + float sum = 0.0f; for (int k = 0; k < TCU_TC_K; k++) { - float a = elem_to_float(rs1[i * TCU_TC_K + k], fi_s); - float b = elem_to_float(rs2[j * TCU_TC_K + k], fi_s); - prod += a * b; + float word_sum = 0.0f; + for (int e = 0; e < epw; e++) { + uint32_t ea = (rs1[i * TCU_TC_K + k] >> (e * fi_s.width_bits)) & emask; + uint32_t eb = (rs2[j * TCU_TC_K + k] >> (e * fi_s.width_bits)) & emask; + word_sum += elem_to_float(ea, fi_s) * elem_to_float(eb, fi_s); + } + sum += word_sum; } - float c = bit_cast(rs3[t]); // C is always fp32 - result[t] = prod + c; + result[t] = sum + c; +#endif } } return result; @@ -208,7 +226,9 @@ struct TestConfig { RoundingMode frm = RoundingMode::RNE; uint32_t num_tests = 1000; int ulp = 1; + int window = 25; // TFR accumulator window (FEDP model) int32_t test_id = -1; + bool directed_only = false; std::vector features = {"infinities","nans","subnormals","normals","zeros"}; }; @@ -250,6 +270,30 @@ static uint32_t gen_fp_value(const std::string &feat, uint32_t eb, uint32_t sb, std::abort(); } +// ---- Directed vectors ------------------------------------------------------- +// Register contents (rs1 = A rows, rs2 = B columns, rs3 = C) on which the native +// datapath's windowed reduction and a discrete fp32 dot product round apart; +// each runs ahead of the random tests for its format and shape. +struct DirectedVec { + const char *name; + uint32_t fmt; + int num_threads; + uint32_t rs1[4], rs2[4], rs3[4]; +}; +static const DirectedVec kDirected[] = { + // fp16 rows of subnormals only: the window aligns on the products' + // exponent fields, so their few significant bits sit low in it + {"fp16 subnormal row", TCU_FP16_ID, 4, + {0x015e005e, 0x024080e2, 0x81fb0228, 0x820c8134}, + {0x41389f40, 0x96d5bc23, 0x285ac58a, 0x7044a57d}, + {0x4feb0a36, 0x4ca6e55a, 0x9a3df577, 0xfa232341}}, + // fp16 products that nearly cancel: the window truncates the residue + {"fp16 cancellation", TCU_FP16_ID, 4, + {0x2f13b5fb, 0x1f98ae04, 0xbdc6632b, 0xc171dfc8}, + {0xb03832e2, 0x2f380ce8, 0x609b8940, 0x7507d768}, + {0x387492a6, 0x2b2843b3, 0xad72f4f6, 0x858ca727}}, +}; + // ---- Testbench class ------------------------------------------------------- class Testbench { public: @@ -300,6 +344,25 @@ class Testbench { << " VX_CFG_NUM_THREADS=" << VX_CFG_NUM_THREADS << "\n"; + // directed vectors for this format and tile shape + uint32_t directed_done = 0; + for (const auto &v : kDirected) { + if (v.fmt != cfg_.fmt_s || v.num_threads != VX_CFG_NUM_THREADS) + continue; + std::vector rs1(v.rs1, v.rs1 + VX_CFG_NUM_THREADS); + std::vector rs2(v.rs2, v.rs2 + VX_CFG_NUM_THREADS); + std::vector rs3(v.rs3, v.rs3 + VX_CFG_NUM_THREADS); + if (!this->check(rs1, rs2, rs3, fi, std::string("Directed (") + v.name + ")")) + return false; + ++directed_done; + } + if (directed_done != 0) { + std::cout << directed_done << " directed test(s) PASSED!\n"; + } + if (cfg_.directed_only) { + return true; + } + uint32_t tests_done = 0; for (uint32_t test_id = 0; test_id < cfg_.num_tests; test_id++) { if (cfg_.test_id >= 0 && (int)test_id != cfg_.test_id) @@ -309,37 +372,30 @@ class Testbench { size_t feat_idx = test_id % cfg_.features.size(); const std::string &feat = cfg_.features[feat_idx]; - // Generate source elements for A and B (VX_CFG_NUM_THREADS values each) - std::vector rs1(VX_CFG_NUM_THREADS), rs2(VX_CFG_NUM_THREADS), rs3(VX_CFG_NUM_THREADS); - for (int t = 0; t < VX_CFG_NUM_THREADS; t++) { - rs1[t] = gen_fp_value(feat, fi.exp_bits, fi.sig_bits, test_id * VX_CFG_NUM_THREADS + t, rng_); - rs2[t] = gen_fp_value("normals", fi.exp_bits, fi.sig_bits, test_id * VX_CFG_NUM_THREADS + t, rng_); - float c = bit_cast(gen_fp_value("normals", 8, 23, test_id * VX_CFG_NUM_THREADS + t + 1, rng_)); - rs3[t] = bit_cast(c); + // A carries the feature, B and C are normals; each register lane + // packs per_word() source elements. + const int epw = fi.per_word(); + std::vector rs1(VX_CFG_NUM_THREADS, 0), rs2(VX_CFG_NUM_THREADS, 0), rs3(VX_CFG_NUM_THREADS); + for (int t = 0; t < TCU_TC_M * TCU_TC_K; t++) { + for (int e = 0; e < epw; e++) { + uint32_t elem_id = (test_id * VX_CFG_NUM_THREADS + t) * epw + e; + rs1[t] |= gen_fp_value(feat, fi.exp_bits, fi.sig_bits, elem_id, rng_) << (e * fi.width_bits); + } } - - // Compute software reference - std::vector expected = wmma_ref(rs1, rs2, rs3, fi); - // Drive DUT - std::vector result = drive_and_collect(rs1, rs2, rs3, fi); - - bool test_pass = true; - for (int t = 0; t < VX_CFG_NUM_THREADS; t++) { - float act = bit_cast(result[t]); - int delta = approx_equal(act, expected[t]); - if (std::abs(delta) > cfg_.ulp) { - if (test_pass) { - // First failure for this test - std::cout << "Test #" << test_id << " (" << feat << ") FAILED:\n"; - } - std::cout << " thread=" << t - << " expected="; print_float("", expected[t], false); - std::cout << " actual="; print_float("", act, false); - std::cout << " delta=" << delta << "\n"; - test_pass = false; + for (int t = 0; t < TCU_TC_N * TCU_TC_K; t++) { + for (int e = 0; e < epw; e++) { + uint32_t elem_id = (test_id * VX_CFG_NUM_THREADS + t) * epw + e; + rs2[t] |= gen_fp_value("normals", fi.exp_bits, fi.sig_bits, elem_id, rng_) << (e * fi.width_bits); } } - if (!test_pass) return false; + for (int t = 0; t < VX_CFG_NUM_THREADS; t++) { + rs3[t] = gen_fp_value("normals", 8, 23, test_id * VX_CFG_NUM_THREADS + t + 1, rng_); + } + + std::ostringstream label; + label << "Test #" << test_id << " (" << feat << ")"; + if (!this->check(rs1, rs2, rs3, fi, label.str())) + return false; tests_done++; } @@ -354,6 +410,38 @@ class Testbench { } private: + // Drive one packet and compare every thread against the reference. + bool check(const std::vector &rs1, const std::vector &rs2, + const std::vector &rs3, const FmtInfo &fi, const std::string &label) { + std::vector expected = wmma_ref(rs1, rs2, rs3, fi, cfg_.window); + std::vector result = drive_and_collect(rs1, rs2, rs3, fi); + bool pass = true; + for (int t = 0; t < VX_CFG_NUM_THREADS; t++) { + float act = bit_cast(result[t]); + int delta = approx_equal(act, expected[t]); + if (std::abs(delta) > cfg_.ulp) { + if (pass) { + std::cout << label << " FAILED:\n"; + } + std::cout << " thread=" << t + << " expected="; print_float("", expected[t], false); + std::cout << " actual="; print_float("", act, false); + std::cout << " delta=" << delta << "\n"; + pass = false; + } + } + if (!pass) { + std::cout << std::hex << " rs1="; + for (int t = 0; t < VX_CFG_NUM_THREADS; t++) std::cout << " 0x" << rs1[t]; + std::cout << "\n rs2="; + for (int t = 0; t < VX_CFG_NUM_THREADS; t++) std::cout << " 0x" << rs2[t]; + std::cout << "\n rs3="; + for (int t = 0; t < VX_CFG_NUM_THREADS; t++) std::cout << " 0x" << rs3[t]; + std::cout << std::dec << "\n"; + } + return pass; + } + TestConfig cfg_; uint64_t cycle_; std::mt19937 rng_; @@ -454,7 +542,8 @@ class Testbench { write_lane(&dut_->dispatch_rs3_data, t, rs3[t]); } - // Clock until dispatch fires (ready goes high) + // The packet fires on the edge that samples valid with ready: test + // ready ahead of each edge, so it is sent exactly once. dut_->commit_ready = 1; uint64_t start_cycle = cycle_; while (true) { @@ -462,10 +551,12 @@ class Testbench { std::cerr << "TIMEOUT waiting for dispatch_ready\n"; std::abort(); } + dut_->clk = 0; + dut_->eval(); + bool fire = dut_->dispatch_ready; tick(); - if (dut_->dispatch_ready) break; + if (fire) break; } - // Dispatch fired: de-assert valid idle_dispatch(); // Wait for commit result @@ -508,6 +599,10 @@ static TestConfig parse_args(int argc, char **argv) { cfg.random_seed = std::stoi(arg.substr(7)); } else if (arg.substr(0, 6) == "--ulp=") { cfg.ulp = std::stoi(arg.substr(6)); + } else if (arg == "--directed") { + cfg.directed_only = true; + } else if (arg.substr(0, 4) == "--W=") { + cfg.window = std::stoi(arg.substr(4)); } else if (arg.substr(0, 14) == "--trace-start=") { cfg.trace_start = std::stoull(arg.substr(14)); } else if (arg.substr(0, 12) == "--trace-end=") { @@ -519,12 +614,14 @@ static TestConfig parse_args(int argc, char **argv) { while (std::getline(ss, item, ',')) cfg.features.push_back(item); } else { std::cout << "Usage: " << argv[0] << " [options]\n" - << " --fmt= Source format (0=fp32,1=fp16,2=bf16,5=tf32)\n" + << " --fmt= Source format (1=tf32,2=fp16,3=bf16)\n" << " --frm= Rounding mode (RNE,RTZ,RDN,RUP,RMM)\n" << " --tests= Number of test cases (default 1000)\n" << " --test= Run a single test by id\n" << " --seed= Random seed\n" << " --ulp= ULP tolerance (default 1)\n" + << " --W= TFR accumulator window of the FEDP model (default 25)\n" + << " --directed Run only the directed vectors\n" << " --no-trace Disable VCD output\n" << " --features=LIST Comma-separated list of features to test\n" << " --trace-start=N Start VCD trace at timestamp N\n" diff --git a/sim/simx/tcu/tcu_unit.cpp b/sim/simx/tcu/tcu_unit.cpp index f5e33c6984..1d4a9a1880 100644 --- a/sim/simx/tcu/tcu_unit.cpp +++ b/sim/simx/tcu/tcu_unit.cpp @@ -16,6 +16,7 @@ #include "tcu_unit.h" #include "tensor_cfg.h" #include +#include #include "core.h" #include "scheduler.h" #include "local_mem.h" @@ -75,8 +76,8 @@ static inline uint8_t unpack_u8(uint32_t word, uint32_t idx) { return (word >> (idx * 8)) & 0xffu; } -// FMA: fused multiply-add returning an Ot-typed accumulator (bit-packed in uint32). -// Widens narrow inputs/accumulator to fp32, performs mul+add, rounds once to Ot. +// FMA: one multiply-add step into an Ot-typed accumulator (bit-packed +// in uint32): fp32 for float inputs, int32 for integer inputs. template struct FMA { using itype = typename It::dtype; @@ -97,15 +98,6 @@ template <> struct FMA { return rv_fadd_s(rv_fmul_s(fa, fb, 0, nullptr), c, 0, nullptr); } }; -template <> struct FMA { - static uint32_t eval(uint16_t a, uint16_t b, uint32_t c) { - auto fa = rv_htof_s(a, 0, nullptr); - auto fb = rv_htof_s(b, 0, nullptr); - auto fc = rv_htof_s(uint16_t(c), 0, nullptr); - return rv_ftoh_s(rv_fmadd_s(fa, fb, fc, 0, nullptr), 0, nullptr); - } -}; - // -- bf16 inputs -- template <> struct FMA { static uint32_t eval(uint16_t a, uint16_t b, uint32_t c) { @@ -114,15 +106,6 @@ template <> struct FMA { return rv_fadd_s(rv_fmul_s(fa, fb, 0, nullptr), c, 0, nullptr); } }; -template <> struct FMA { - static uint32_t eval(uint16_t a, uint16_t b, uint32_t c) { - auto fa = rv_btof_s(a, 0, nullptr); - auto fb = rv_btof_s(b, 0, nullptr); - auto fc = rv_btof_s(uint16_t(c), 0, nullptr); - return rv_ftob_s(rv_fmadd_s(fa, fb, fc, 0, nullptr), 0, nullptr); - } -}; - // -- fp8 inputs -- template <> struct FMA { static uint32_t eval(uint8_t a, uint8_t b, uint32_t c) { @@ -131,15 +114,6 @@ template <> struct FMA { return rv_fadd_s(rv_fmul_s(fa, fb, 0, nullptr), c, 0, nullptr); } }; -template <> struct FMA { - static uint32_t eval(uint8_t a, uint8_t b, uint32_t c) { - auto fa = rv_e4m3tof_s(a, 0, nullptr); - auto fb = rv_e4m3tof_s(b, 0, nullptr); - auto fc = rv_e4m3tof_s(uint8_t(c), 0, nullptr); - return rv_ftoe4m3_s(rv_fmadd_s(fa, fb, fc, 0, nullptr), 0, nullptr); - } -}; - // -- bf8 inputs -- template <> struct FMA { static uint32_t eval(uint8_t a, uint8_t b, uint32_t c) { @@ -148,15 +122,6 @@ template <> struct FMA { return rv_fadd_s(rv_fmul_s(fa, fb, 0, nullptr), c, 0, nullptr); } }; -template <> struct FMA { - static uint32_t eval(uint8_t a, uint8_t b, uint32_t c) { - auto fa = rv_e5m2tof_s(a, 0, nullptr); - auto fb = rv_e5m2tof_s(b, 0, nullptr); - auto fc = rv_e5m2tof_s(uint8_t(c), 0, nullptr); - return rv_ftoe5m2_s(rv_fmadd_s(fa, fb, fc, 0, nullptr), 0, nullptr); - } -}; - // -- tf32 inputs -- template <> struct FMA { static uint32_t eval(uint32_t a, uint32_t b, uint32_t c) { @@ -165,19 +130,21 @@ template <> struct FMA { return rv_fadd_s(rv_fmul_s(fa, fb, 0, nullptr), c, 0, nullptr); } }; -template <> struct FMA { - static uint32_t eval(uint32_t a, uint32_t b, uint32_t c) { - auto fa = rv_tf32tof_s(a, 0, nullptr); - auto fb = rv_tf32tof_s(b, 0, nullptr); - auto fc = rv_tf32tof_s(c, 0, nullptr); - return rv_ftotf32_s(rv_fmadd_s(fa, fb, fc, 0, nullptr), 0, nullptr); - } -}; - -// Generic FEDP: universal rule keyed on output width. -// * Wide Ot (fp32): accumulate Σ(a_k*b_k) in fp32, add c_val last. -// * Narrow Ot (fp16/bf16/fp8/bf8/…): chain FMA so the accumulator is -// rounded to Ot each step. +// Float source formats the native TFR datapath reduces (exponent, mantissa bits). +template struct TfrFormat { static constexpr bool valid = false; }; +template <> struct TfrFormat { static constexpr bool valid = true; static constexpr int eb = 5, sb = 10; }; +template <> struct TfrFormat { static constexpr bool valid = true; static constexpr int eb = 8, sb = 7; }; +template <> struct TfrFormat { static constexpr bool valid = true; static constexpr int eb = 4, sb = 3; }; +template <> struct TfrFormat { static constexpr bool valid = true; static constexpr int eb = 5, sb = 2; }; +template <> struct TfrFormat { static constexpr bool valid = true; static constexpr int eb = 8, sb = 10; }; + +// Generic FEDP, keyed on the output format. +// * fp32 Ot on the native TFR datapath: its fixed-point reduction — +// products aligned to the largest exponent in a 25-bit window, summed as +// integers, rounded once (the TfrFedp model the RTL is checked against). +// * fp32 Ot otherwise: accumulate Σ(a_k*b_k) in fp32 per register, +// the register sums in order, c_val last (the DPI datapath's order). +// * int32 Ot (integer inputs): exact integer accumulation. template struct FEDP { using itype = typename It::dtype; @@ -188,6 +155,22 @@ struct FEDP { static uint32_t eval_n(const reg_data_t *a_row, const reg_data_t *b_col, uint32_t c_val, uint32_t k_words) { constexpr uint32_t i_ratio = sizeof(uint32_t) / sizeof(itype); static_assert(i_ratio * sizeof(itype) == sizeof(uint32_t), "FEDP: tcK * i_ratio must be <= 32"); +#ifdef VX_CFG_TCU_TYPE_TFR + if constexpr (std::is_same_v && TfrFormat::valid) { + std::vector a(k_words), b(k_words); + for (uint32_t z = 0; z < k_words; ++z) { + a[z] = a_row[z].u32; + b[z] = b_col[z].u32; + } + TfrFedp fedp(TfrFormat::eb, TfrFormat::sb); + float c; + std::memcpy(&c, &c_val, sizeof(c)); + float d = fedp(a.data(), b.data(), c, k_words); + uint32_t d_bits; + std::memcpy(&d_bits, &d, sizeof(d_bits)); + return d_bits; + } else +#endif if constexpr (std::is_same_v) { uint32_t acc = 0; for (uint32_t z = 0; z < k_words; ++z) { @@ -201,6 +184,7 @@ struct FEDP { } return rv_fadd_s(c_val, acc, 0, nullptr); } else { + static_assert(std::is_same_v, "TCU output type must be fp32 (int32 for integer inputs)"); uint32_t acc = c_val; for (uint32_t z = 0; z < k_words; ++z) { auto a = reinterpret_cast(&a_row[z].u32); @@ -282,26 +266,6 @@ static PFN_FEDP_N select_FEDP_N(uint32_t IT, uint32_t OT) { break; } break; - case vt::fp16::id: - if (IT == vt::fp16::id) - return FEDP::eval_n; - break; - case vt::bf16::id: - if (IT == vt::bf16::id) - return FEDP::eval_n; - break; - case vt::fp8::id: - if (IT == vt::fp8::id) - return FEDP::eval_n; - break; - case vt::bf8::id: - if (IT == vt::bf8::id) - return FEDP::eval_n; - break; - case vt::tf32::id: - if (IT == vt::tf32::id) - return FEDP::eval_n; - break; case vt::int32::id: switch (IT) { case vt::int8::id: @@ -620,6 +584,12 @@ class TcuUnit::Impl { trace->dst_data.assign(num_threads, reg_data_t{}); auto& rd_data = trace->dst_data; + if (tcu_type != TcuType::TCU_LD && !vt::tcu_output_valid(tpuArgs.fmt_s, tpuArgs.fmt_d)) { + std::cerr << "*** TCU output format " << tpuArgs.fmt_d << " for input format " << tpuArgs.fmt_s + << " is not supported: the TCU returns fp32 (int32 for integer inputs)" << std::endl; + std::abort(); + } + switch (tcu_type) { case TcuType::WMMA: case TcuType::WMMA_SP: @@ -1371,6 +1341,46 @@ class TcuUnit::Impl { }; if (fmt_d == vt::fp32::id) { +#ifdef VX_CFG_TCU_TYPE_TFR + // The native datapath: the TfrFedp model, with each lane's (16 bits of + // a register) scales taken at its first element. + { + int eb = 0, sb = 0; + bool e4m3_scale = false; + if (fmt_s == vt::mxfp8::id) { + eb = 4; sb = 3; + } else if (fmt_s == vt::mxbf8::id) { + eb = 5; sb = 2; + } else if (fmt_s == vt::mxfp4::id) { + eb = 2; sb = 1; + } else if (fmt_s == vt::nvfp4::id) { + eb = 2; sb = 1; e4m3_scale = true; + } else { + std::abort(); + } + uint32_t sparse_ratio = is_sparse ? 2 : 1; + std::vector a(fedp_words), b(fedp_words); + std::vector sf_a(2 * fedp_words), sf_b(2 * fedp_words); + for (uint32_t z = 0; z < fedp_words; ++z) { + a[z] = a_row[z].u32; + b[z] = b_col[z].u32; + } + for (uint32_t lane = 0; lane < 2 * fedp_words; ++lane) { + uint32_t z = lane / 2; + uint32_t e = (lane % 2) * (ratio / 2); + uint32_t elem_k = ((step_k * fedp_words + z) * ratio + e) * sparse_ratio; + sf_a[lane] = uint8_t(scale_a(elem_k)); + sf_b[lane] = uint8_t(scale_b(elem_k)); + } + TfrFedp fedp(eb, sb); + float c; + std::memcpy(&c, &c_val, sizeof(c)); + float d = fedp(a.data(), b.data(), c, fedp_words, sf_a.data(), sf_b.data(), e4m3_scale); + uint32_t d_bits; + std::memcpy(&d_bits, &d, sizeof(d_bits)); + return d_bits; + } +#endif uint32_t acc = c_val; for (uint32_t z = 0; z < fedp_words; ++z) { for (uint32_t e = 0; e < ratio; ++e) { diff --git a/hw/unittest/tcu_fedp/fedp.h b/sw/common/fedp.h similarity index 64% rename from hw/unittest/tcu_fedp/fedp.h rename to sw/common/fedp.h index d93beefb23..486128c3a0 100644 --- a/hw/unittest/tcu_fedp/fedp.h +++ b/sw/common/fedp.h @@ -25,9 +25,9 @@ #include #ifdef FEDP_TRACE -#define LOG(...) std::fprintf(stderr, __VA_ARGS__); +#define FEDP_LOG(...) std::fprintf(stderr, __VA_ARGS__); #else -#define LOG(...) +#define FEDP_LOG(...) #endif #ifdef FEDP_RTL_WATCH @@ -36,26 +36,42 @@ #define RTL_WATCH(...) #endif -class FEDP { +class TfrFedp { public: - explicit FEDP(int exp_bits = 5, int sig_bits = 10, int frm = 0, int W = 25, bool renorm = false, bool no_window = false) + explicit TfrFedp(int exp_bits = 5, int sig_bits = 10, int frm = 0, int W = 25, bool renorm = false, bool no_window = false) : exp_bits_(exp_bits), sig_bits_(sig_bits), frm_(frm), W_(W), renorm_(renorm), no_window_(no_window) { - LOG("[ctor] fmt=e%dm%d, frm=%d, W=%d, renorm_=%s, no_window_=%s\n", + FEDP_LOG("[ctor] fmt=e%dm%d, frm=%d, W=%d, renorm_=%s, no_window_=%s\n", exp_bits_, sig_bits_, frm_, W_, (renorm_ ? "true" : "false"), (no_window_ ? "true" : "false")); assert(exp_bits_ > 0 && exp_bits_ <= 8); assert(sig_bits_ > 0 && sig_bits_ <= 10); assert(frm_ >= 0 && frm_ <= 4); } + // Block-scaled formats: one scale byte per lane for A and B, two lanes per + // 32-bit register (a lane is 16 bits of it: two fp8 or four fp4 elements). + // fp8/bf8 (MXFP8/MXBF8) and fp4 (MXFP4): E8M0 scales; + // fp4 with e4m3_scale (NVFP4): E4M3 scales. + float operator()(const uint32_t *a, const uint32_t *b, float c, uint32_t n, + const uint8_t *sf_a, const uint8_t *sf_b, bool e4m3_scale = false) { + sf_a_ = sf_a; + sf_b_ = sf_b; + e4m3_scale_ = e4m3_scale; + float d = (*this)(a, b, c, n); + sf_a_ = sf_b_ = nullptr; + e4m3_scale_ = false; + return d; + } + float operator()(const uint32_t *a, const uint32_t *b, float c, uint32_t n) { const auto c_enc = bitsFromF32(c); rtl_watch_s0(a, b, c_enc, n); - const auto terms = decode_inputs(a, b, n); const auto c_dec = decode_input(c_enc, 8, 23); const auto c_term = decodeC_to_common(c_dec); - const auto mul_res = multiply_to_common(terms, c_term); + const auto mul_res = (exp_bits_ == 2 && sig_bits_ == 1) + ? multiply_f4(a, b, n, c_term) + : multiply_to_common(decode_inputs(a, b, n), c_term); rtl_watch_s1(mul_res, c_term, c_enc); @@ -157,7 +173,7 @@ class FEDP { } // -------- [RTL-WATCH] helpers -------- - inline void rtl_watch_s0(const uint32_t* a, const uint32_t* b, uint32_t c_enc, uint32_t n) { + inline void rtl_watch_s0([[maybe_unused]] const uint32_t* a, [[maybe_unused]] const uint32_t* b, [[maybe_unused]] uint32_t c_enc, [[maybe_unused]] uint32_t n) { #ifdef FEDP_RTL_WATCH RTL_WATCH("[RTL_WATCH] FEDP-S0(%lu): a_row=", req_id_); RTL_WATCH("{"); @@ -174,7 +190,7 @@ class FEDP { #endif } - inline void rtl_watch_s1(const mul_res_t& res, const cterm_t& c_term, uint32_t c_enc) { + inline void rtl_watch_s1([[maybe_unused]] const mul_res_t& res, [[maybe_unused]] const cterm_t& c_term, [[maybe_unused]] uint32_t c_enc) { #ifdef FEDP_RTL_WATCH RTL_WATCH("[RTL_WATCH] FEDP-S1(%lu): max_exp=0x%x, shift_amt=", req_id_, res.L); RTL_WATCH("{"); @@ -202,7 +218,7 @@ class FEDP { #endif } - inline void rtl_watch_s2(const align_t& aln, const mul_res_t& res, const cterm_t& c_term, uint32_t c_enc) { + inline void rtl_watch_s2([[maybe_unused]] const align_t& aln, [[maybe_unused]] const mul_res_t& res, [[maybe_unused]] const cterm_t& c_term, [[maybe_unused]] uint32_t c_enc) { #ifdef FEDP_RTL_WATCH RTL_WATCH("[RTL_WATCH] FEDP-S2(%lu): max_exp=0x%x, aln_sig=", req_id_, res.L); RTL_WATCH("{"); @@ -228,7 +244,7 @@ class FEDP { #endif } - inline void rtl_watch_s3(const acc_t& acc, const mul_res_t& res, const cterm_t& c_term, uint32_t c_enc) { + inline void rtl_watch_s3([[maybe_unused]] const acc_t& acc, [[maybe_unused]] const mul_res_t& res, [[maybe_unused]] const cterm_t& c_term, [[maybe_unused]] uint32_t c_enc) { #ifdef FEDP_RTL_WATCH uint32_t sigs_sign = 0; for (size_t i = 0; i < res.terms.size() - 1; ++i) { @@ -239,7 +255,7 @@ class FEDP { #endif } - inline void rtl_watch_norm(const nrm_t& nrm, int flags, int max_exp) { + inline void rtl_watch_norm([[maybe_unused]] const nrm_t& nrm, [[maybe_unused]] int flags, [[maybe_unused]] int max_exp) { #ifdef FEDP_RTL_WATCH int lsb = nrm.kept & 1; RTL_WATCH("[RTL_WATCH] FEDP-NORM(%lu): abs_sum=0x%x, L=%d, G=%d, R=%d, S=%d, Rup=%d\n", @@ -248,12 +264,19 @@ class FEDP { #endif } - inline void rtl_watch_s4(uint32_t result) { + inline void rtl_watch_s4([[maybe_unused]] uint32_t result) { #ifdef FEDP_RTL_WATCH RTL_WATCH("[RTL_WATCH] FEDP-S4(%lu): result=0x%08x\n", req_id_, result); #endif } + // Exponent of a zero term: it stands for the datapath's exponent 0, which + // add_c_term places WA - 1 below this model's exponent origin. + static constexpr int kZeroE = -(1 << 20); + + // The datapath's accumulator width over `lanes` product terms and C. + int acc_width(uint32_t lanes) const { return W_ + 2 + (32 - lzcN(lanes, 32)); } + // -------- logic -------- static inline std::pair csa(uint32_t a, uint32_t b, uint32_t c) { uint32_t s = a ^ b ^ c; @@ -303,10 +326,11 @@ class FEDP { const int sb = sig_bits_, eb = exp_bits_; std::vector out; std::vector eps; // per-term *window LSB exponent* (not raw Ep/Ec) + std::vector sel_eps; + std::vector nonzero; out.reserve(v.size() + 1); eps.reserve(v.size() + 1); - bool c_is_zero = (c_term.cls == 0 && c_term.sp == 0); uint8_t c_nan = (c_term.sp == 3) ? FL_NAN : 0; uint8_t c_pinf = (c_term.sp == 1) ? FL_PINF : 0; @@ -354,8 +378,14 @@ class FEDP { Ep -= lzc_prod; } + // the fp8 lane pairs its two products by their exponent fields even + // when one is zero (a zero field counts as 1, as for a subnormal) + sel_eps.push_back((std::max(a.exp, 1) - bias(eb)) + (std::max(b.exp, 1) - bias(eb)) + + F32_BIAS + EXP_POS_BIAS + 1 - W_); + nonzero.push_back(!p_is_zero); + if (p_is_zero) { - Ep = 0; + Ep = kZeroE; Mp = 0; } @@ -372,7 +402,22 @@ class FEDP { for (size_t j = 0; j < out.size(); j += 2) { if (j + 1 < out.size()) { - auto r = reduce_f8(out[j], eps[j], out[j + 1], eps[j + 1]); + auto r = reduce_f8(out[j], sel_eps[j], nonzero[j], out[j + 1], sel_eps[j + 1], nonzero[j + 1]); + // MX: the lane's E8M0 scales add to its exponent field, which is + // 10 bits wide and sits WA - 1 above this model's exponents + if (sf_a_) { + // an E8M0 scale of 0xFF is NaN, and so is every element it scales + if (sf_a_[j / 2] == 0xff || sf_b_[j / 2] == 0xff) { + flags |= FL_NAN; + } + // a lane scaled below the exponent range sits at exponent 0, + // where it only adds to the sticky bit + if (r.second != kZeroE) { + const int WA = acc_width(uint32_t(out.size() / 2)); + int lane_exp = r.second + (WA - 1) + int(sf_a_[j / 2]) + int(sf_b_[j / 2]) - 254; + r.second = std::max(lane_exp, 0) - (WA - 1); + } + } red_out.push_back(r.first); red_eps.push_back(r.second); } else { @@ -385,13 +430,23 @@ class FEDP { eps.swap(red_eps); } + return add_c_term(std::move(out), std::move(eps), c_term, flags); + } + + // Append the C term and take every term's shift to the largest exponent. + mul_res_t add_c_term(std::vector out, std::vector eps, const cterm_t &c_term, uint8_t flags) { + bool c_is_zero = (c_term.cls == 0 && c_term.sp == 0); int Ec = c_term.Ec + F32_BIAS + EXP_POS_BIAS + 24 - W_; if (c_is_zero) { - Ec = 0; + Ec = kZeroE; } out.push_back({c_term.sign, c_term.Mc}); eps.push_back(Ec); + const int WA = acc_width(uint32_t(out.size() - 1)); + for (auto &e : eps) { + if (e == kZeroE) e = -(WA - 1); + } int L = *std::max_element(eps.begin(), eps.end()); @@ -404,12 +459,99 @@ class FEDP { return {out, L , shifts, flags}; } - std::pair reduce_f8(const pterm_t& t1, int e1, const pterm_t& t2, int e2) { + // Block-scaled fp4 (E2M1), one lane per 16 bits: its four products are + // aligned to their largest exponent in a 6-bit lane field, summed in 27-bit + // two's complement, and leave the datapath as one term. E2M1 has no + // infinity or NaN. MXFP4 adds the E8M0 scales to the lane exponent; NVFP4 + // folds the E4M3 scales' exponent fields into the 6-bit field (they fit) and + // multiplies each product by the scales' mantissa product. + mul_res_t multiply_f4(const uint32_t *a, const uint32_t *b, uint32_t n, const cterm_t &c_term) { + assert(sf_a_ && sf_b_ && "fp4 needs block scales"); + const int SIG_SHIFT = 11; + const int EXP_TERM_BIAS = 32; + const int exp_comp = e4m3_scale_ ? -(2 * 1 + 2 * (7 + 3)) : -(2 * 1 + 10); + const int exp_adj = (EXP_TERM_BIAS + exp_comp) & 63; + // the lane exponent the datapath emits is 10 bits wide; this + // model's lane exponents sit WA - 1 below it + const int WA = acc_width(2 * n); + const int bias_base = F32_BIAS + 2 * (23 - 22) - W_ + WA - 1 + EXP_POS_BIAS; + auto f4_man = [](uint32_t x) -> uint32_t { return ((x >> 1) & 3) == 0 ? 1u : (2u | (x & 1)); }; + auto f4_exp = [](uint32_t x) -> uint32_t { + uint32_t e2 = (x >> 2) & 1, e1 = (x >> 1) & 1; + return (e2 & ~e1 & 1) | ((e2 & e1) << 1); + }; + std::vector out; + std::vector eps; + uint8_t flags = 0; + if (c_term.sp == 3) flags |= FL_NAN; + if (c_term.sp == 1) flags |= FL_PINF; + if (c_term.sp == 2) flags |= FL_NINF; + for (uint32_t lane = 0; lane < 2 * n; ++lane) { + const uint32_t word = lane / 2, off = (lane % 2) * 16; + const uint32_t sfa = sf_a_[lane], sfb = sf_b_[lane]; + // NVFP4 UE4M3 scales (bit 7 is padding): 0x7F is NaN. MXFP4 E8M0 + // scales join the lane exponent; 0xFF is NaN. + uint32_t sf_man_prod = 1; + int sf_exp = 0; // into the 6-bit term exponent (NVFP4) + int sf_lane_exp = 0; // into the lane exponent (MXFP4) + if (e4m3_scale_) { + sf_man_prod = (8u | (sfa & 7)) * (8u | (sfb & 7)); + sf_exp = int((sfa >> 3) & 15) + int((sfb >> 3) & 15); + if ((sfa & 0x7f) == 0x7f || (sfb & 0x7f) == 0x7f) { + flags |= FL_NAN; + } + } else { + sf_lane_exp = int(sfa) + int(sfb) - 254; + if (sfa == 0xff || sfb == 0xff) { + flags |= FL_NAN; + } + } + uint32_t texp[4], tprod[4], tsign[4]; + bool tvalid[4]; + uint32_t max_exp = 0; + for (int j = 0; j < 4; ++j) { + uint32_t ra = (a[word] >> (off + 4 * j)) & 0xf; + uint32_t rb = (b[word] >> (off + 4 * j)) & 0xf; + tvalid[j] = (ra & 7) != 0 && (rb & 7) != 0; + tsign[j] = ((ra ^ rb) >> 3) & 1; + uint32_t prod = f4_man(ra) * f4_man(rb); + tprod[j] = e4m3_scale_ ? ((prod * sf_man_prod) & 0x7ff) : prod; + texp[j] = tvalid[j] ? (uint32_t(exp_adj + sf_exp + int(f4_exp(ra)) + int(f4_exp(rb))) & 63) : 0; + max_exp = std::max(max_exp, texp[j]); + } + uint32_t sum = 0; // 27-bit two's complement + for (int j = 0; j < 4; ++j) { + uint32_t shift = (max_exp - texp[j]) & 63; + uint32_t mag = tvalid[j] ? ((tprod[j] << SIG_SHIFT) & 0xffffff) : 0; + uint32_t aligned = (shift >= 24) ? 0 : (mag >> shift); + sum = (sum + (tsign[j] ? (0u - aligned) : aligned)) & 0x7ffffff; + } + uint32_t sum_sign = (sum >> 26) & 1; + uint32_t abs_sum = (sum_sign ? (0u - sum) : sum) & 0x3ffffff; + if (abs_sum == 0) { + out.push_back({0, 0}); + eps.push_back(kZeroE); + continue; + } + const int lane_exp = (int(max_exp) + bias_base + exp_comp + sf_lane_exp) & 1023; + out.push_back({int(sum_sign), int(abs_sum & 0xffffff)}); + eps.push_back(lane_exp - (WA - 1)); + } + return add_c_term(std::move(out), std::move(eps), c_term, flags); + } + + // One fp8 lane: its two products, aligned to the one the datapath picks as + // larger. A zero product still competes by its exponent fields when the + // other is nonzero and listed second, as the datapath compares them. + std::pair reduce_f8(const pterm_t& t1, int e1, bool v1, const pterm_t& t2, int e2, bool v2) { // Pick max exponent term int max_E, max_s, min_E, min_s; int64_t max_P, min_P; - if (e1 >= e2) { + if (!v1 && !v2) { + return {{0, 0}, kZeroE}; + } + if ((v1 && !v2) || (v2 && e1 >= e2)) { max_E = e1; max_P = t1.Mp; max_s = t1.sign; min_E = e2; min_P = t2.Mp; min_s = t2.sign; } else { @@ -435,7 +577,11 @@ class FEDP { int64_t val = val1 + val2; if (val == 0) { - return {{0, 0}, 0}; + // Only an exact cancellation of two equal-exponent products clears the + // lane exponent; a product aligned out of the lane keeps it, so the + // empty lane still sets the largest exponent. + bool cancel = (max_s != min_s) && (e1 == e2) && (t1.Mp == t2.Mp); + return {{0, 0}, cancel ? kZeroE : (max_E + 1)}; } int s = (val < 0) ? 1 : 0; @@ -446,7 +592,7 @@ class FEDP { P >>= 1; E += 1; - LOG("[reduce_f8 (%ld)] t1=(%d, %d, %d), t2=(%d, %d, %d) | diff=%d max_P=0x%lx min_P=0x%lx aligned=0x%lx val=0x%lx -> s=%d P=0x%lx E=%d\n", + FEDP_LOG("[reduce_f8 (%ld)] t1=(%d, %d, %d), t2=(%d, %d, %d) | diff=%d max_P=0x%lx min_P=0x%lx aligned=0x%lx val=0x%lx -> s=%d P=0x%lx E=%d\n", req_id_, t1.sign, t1.Mp, e1, t2.sign, t2.Mp, e2, diff, max_P, min_P, aligned, val, s, P, E); return {{s, (int)P}, E}; @@ -569,6 +715,8 @@ class FEDP { uint32_t kept_rounded = r.kept + (round_up ? 1 : 0); int be = r.e; if (kept_rounded & (1u << 24)) { kept_rounded >>= 1; be += 1; } + // the datapath's result exponent is a signed 10-bit quantity + be = ((be & 1023) ^ 512) - 512; if (be >= 255) return s_bit | 0x7f800000u; if (be <= 0) return s_bit; return s_bit | ((be & 0xff) << 23) | (kept_rounded & 0x7FFFFFu); @@ -577,5 +725,8 @@ class FEDP { // members -------- int exp_bits_, sig_bits_, frm_, W_, HR_; bool renorm_, no_window_; + const uint8_t *sf_a_ = nullptr; // per-lane block scales, null when unscaled + const uint8_t *sf_b_ = nullptr; + bool e4m3_scale_ = false; uint64_t req_id_ = 0; }; \ No newline at end of file diff --git a/sw/common/tensor_cfg.h b/sw/common/tensor_cfg.h index 6ca343bb12..e980bc5c77 100644 --- a/sw/common/tensor_cfg.h +++ b/sw/common/tensor_cfg.h @@ -156,6 +156,12 @@ inline const char* fmt_string(uint32_t fmt) { } } +// The TCU accumulates and returns fp32 (int32 for integer inputs); it has no +// other output format. +inline constexpr bool tcu_output_valid(uint32_t it, uint32_t ot) { + return (it >= int32::id) ? (ot == int32::id) : (ot == fp32::id); +} + inline constexpr bool mx_scale_format(uint32_t fmt) { switch (fmt) { case mxfp8::id: diff --git a/sw/kernel/include/vx_tensor.h b/sw/kernel/include/vx_tensor.h index 618a95b030..277b6ecc8b 100644 --- a/sw/kernel/include/vx_tensor.h +++ b/sw/kernel/include/vx_tensor.h @@ -173,6 +173,9 @@ struct wmma_context { public: + static_assert(tcu_output_valid(It::id, Ot::id), + "TCU output type must be fp32 (int32 for integer inputs)"); + using input_t = typename It::dtype; using output_t = typename Ot::dtype; @@ -675,6 +678,9 @@ template struct wgmma_context { + static_assert(tcu_output_valid(It::id, Ot::id), + "TCU output type must be fp32 (int32 for integer inputs)"); + private: static constexpr uint32_t NRA = 4; diff --git a/tests/regression/sgemm2_tcu/main.cpp b/tests/regression/sgemm2_tcu/main.cpp index 400cc84793..a38b6c40f4 100644 --- a/tests/regression/sgemm2_tcu/main.cpp +++ b/tests/regression/sgemm2_tcu/main.cpp @@ -255,17 +255,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint16_t eval(uint16_t a, uint16_t b, uint16_t c) { - auto fa = bit_cast(rv_htof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_htof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_htof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftoh_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static float eval(uint16_t a, uint16_t b, float c) { @@ -275,17 +264,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint16_t eval(uint16_t a, uint16_t b, uint16_t c) { - auto fa = bit_cast(rv_btof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_btof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_btof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftob_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static float eval(uint8_t a, uint8_t b, float c) { @@ -330,6 +308,9 @@ using cfg = vt::wmma_config_t; using itype_t = typename vt::ITYPE::dtype; using otype_t = typename vt::OTYPE::dtype; +static_assert(vt::tcu_output_valid(vt::ITYPE::id, vt::OTYPE::id), + "TCU output type must be fp32 (int32 for integer inputs)"); + static void matmul_cpu(otype_t *C, const itype_t *A, const itype_t *B, uint32_t M, uint32_t N, uint32_t K) { for (uint32_t m = 0; m < M; ++m) { for (uint32_t n = 0; n < N; ++n) { @@ -346,8 +327,7 @@ static void matmul_cpu(otype_t *C, const itype_t *A, const itype_t *B, uint32_t } data_accessor_t::write(C, m * N + n, static_cast(acc)); } else { - // Narrow outputs re-round the accumulator to the output type every - // step, matching the hardware chain exactly. + // int32 output (integer inputs): exact accumulation. otype_t sum(0); for (uint32_t k = 0; k < K; ++k) { auto a = data_accessor_t::read(A, m * K + k); diff --git a/tests/regression/sgemm_tcu/main.cpp b/tests/regression/sgemm_tcu/main.cpp index 1815228882..0c9446a561 100644 --- a/tests/regression/sgemm_tcu/main.cpp +++ b/tests/regression/sgemm_tcu/main.cpp @@ -368,17 +368,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint16_t eval(uint16_t a, uint16_t b, uint16_t c) { - auto fa = bit_cast(rv_htof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_htof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_htof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftoh_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static float eval(uint16_t a, uint16_t b, float c) { @@ -388,17 +377,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint16_t eval(uint16_t a, uint16_t b, uint16_t c) { - auto fa = bit_cast(rv_btof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_btof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_btof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftob_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static float eval(uint8_t a, uint8_t b, float c) { @@ -408,17 +386,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint8_t eval(uint8_t a, uint8_t b, uint8_t c) { - auto fa = bit_cast(rv_e4m3tof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_e4m3tof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_e4m3tof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftoe4m3_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static float eval(uint8_t a, uint8_t b, float c) { @@ -428,17 +395,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint8_t eval(uint8_t a, uint8_t b, uint8_t c) { - auto fa = bit_cast(rv_e5m2tof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_e5m2tof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_e5m2tof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftoe5m2_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static float eval(uint32_t a, uint32_t b, float c) { @@ -448,17 +404,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint32_t eval(uint32_t a, uint32_t b, uint32_t c) { - auto fa = bit_cast(rv_tf32tof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_tf32tof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_tf32tof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftotf32_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static int32_t eval(uint8_t a, uint8_t b, int32_t c) { @@ -495,6 +440,9 @@ inline typename T::dtype generate_B_value() { /////////////////////////////////////////////////////////////////////////////// +static_assert(vt::tcu_output_valid(vt::ITYPE::id, vt::OTYPE::id), + "TCU output type must be fp32 (int32 for integer inputs)"); + using cfg = vt::wmma_config_t; using itype_t = typename vt::ITYPE::dtype; @@ -507,10 +455,10 @@ static void matmul_cpu(otype_t *C, const itype_t *A, const itype_t *B, uint32_t for (uint32_t m = 0; m < M; ++m) { for (uint32_t n = 0; n < N; ++n) { if constexpr (std::is_same::value) { - // fp32 output: the tensor core accumulates the K products in a wide - // accumulator and rounds to fp32 once; a per-step-rounded reference - // drifts by several ULP over K. Each product is exact in fp32, so a - // double accumulation reproduces the single-rounding dot product. + // The tensor core accumulates the K products in a wide accumulator + // and rounds to fp32 once; a per-step-rounded reference drifts by + // several ULP over K. Each product is exact in fp32, so a double + // accumulation reproduces the single-rounding dot product. double acc = 0.0; for (uint32_t k = 0; k < KS; ++k) { auto a = data_accessor_t::read(A, m * KS + k); @@ -519,8 +467,7 @@ static void matmul_cpu(otype_t *C, const itype_t *A, const itype_t *B, uint32_t } data_accessor_t::write(C, m * N + n, static_cast(acc)); } else { - // Narrow outputs re-round the accumulator to the output type every - // step, matching the hardware chain exactly. + // int32 output (integer inputs): exact accumulation. otype_t sum(0); for (uint32_t k = 0; k < KS; ++k) { auto a = data_accessor_t::read(A, m * KS + k); diff --git a/tests/regression/sgemm_tcu_mx/main.cpp b/tests/regression/sgemm_tcu_mx/main.cpp index a0cff63253..f3d10d3b77 100644 --- a/tests/regression/sgemm_tcu_mx/main.cpp +++ b/tests/regression/sgemm_tcu_mx/main.cpp @@ -33,6 +33,9 @@ using cfg = vt::wmma_config_t; using itype_t = typename vt::ITYPE::dtype; using otype_t = typename vt::OTYPE::dtype; +static_assert(vt::tcu_output_valid(vt::ITYPE::id, vt::OTYPE::id), + "TCU output type must be fp32 (int32 for integer inputs)"); + static constexpr uint32_t kElemsPerByte = (vt::ITYPE::bits < 8) ? (8 / vt::ITYPE::bits) : 1; static constexpr uint32_t kScalePack = 4; diff --git a/tests/regression/sgemm_tcu_sp/main.cpp b/tests/regression/sgemm_tcu_sp/main.cpp index c3237eea4b..dd357fe7a1 100644 --- a/tests/regression/sgemm_tcu_sp/main.cpp +++ b/tests/regression/sgemm_tcu_sp/main.cpp @@ -361,17 +361,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint16_t eval(uint16_t a, uint16_t b, uint16_t c) { - auto fa = bit_cast(rv_htof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_htof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_htof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftoh_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static float eval(uint16_t a, uint16_t b, float c) { @@ -381,17 +370,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint16_t eval(uint16_t a, uint16_t b, uint16_t c) { - auto fa = bit_cast(rv_btof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_btof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_btof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftob_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static float eval(uint8_t a, uint8_t b, float c) { @@ -401,17 +379,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint8_t eval(uint8_t a, uint8_t b, uint8_t c) { - auto fa = bit_cast(rv_e4m3tof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_e4m3tof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_e4m3tof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftoe4m3_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static float eval(uint8_t a, uint8_t b, float c) { @@ -421,17 +388,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint8_t eval(uint8_t a, uint8_t b, uint8_t c) { - auto fa = bit_cast(rv_e5m2tof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_e5m2tof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_e5m2tof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftoe5m2_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static float eval(uint32_t a, uint32_t b, float c) { @@ -441,17 +397,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint32_t eval(uint32_t a, uint32_t b, uint32_t c) { - auto fa = bit_cast(rv_tf32tof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_tf32tof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_tf32tof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftotf32_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static int32_t eval(uint8_t a, uint8_t b, int32_t c) { @@ -493,6 +438,9 @@ using cfg = vt::wmma_config_t; using itype_t = typename vt::ITYPE::dtype; using otype_t = typename vt::OTYPE::dtype; +static_assert(vt::tcu_output_valid(vt::ITYPE::id, vt::OTYPE::id), + "TCU output type must be fp32 (int32 for integer inputs)"); + // Dense CPU reference matmul (pruned A has zeros at masked positions) static void matmul_cpu(otype_t *C, const itype_t *A, const itype_t *B, uint32_t M, uint32_t N, uint32_t K) { @@ -513,8 +461,7 @@ static void matmul_cpu(otype_t *C, const itype_t *A, const itype_t *B, uint32_t } data_accessor_t::write(C, m * N + n, static_cast(acc)); } else { - // Narrow outputs re-round the accumulator to the output type every - // step, matching the hardware chain exactly. + // int32 output (integer inputs): exact accumulation. otype_t sum(0); for (uint32_t k = 0; k < KS; ++k) { auto a = data_accessor_t::read(A, m * KS + k); diff --git a/tests/regression/sgemm_tcu_sp_mx/main.cpp b/tests/regression/sgemm_tcu_sp_mx/main.cpp index 8dabef023d..b7b4e9048c 100644 --- a/tests/regression/sgemm_tcu_sp_mx/main.cpp +++ b/tests/regression/sgemm_tcu_sp_mx/main.cpp @@ -34,6 +34,9 @@ using cfg = vt::wmma_config_t; using itype_t = typename vt::ITYPE::dtype; using otype_t = typename vt::OTYPE::dtype; +static_assert(vt::tcu_output_valid(vt::ITYPE::id, vt::OTYPE::id), + "TCU output type must be fp32 (int32 for integer inputs)"); + static constexpr uint32_t kElemsPerByte = (vt::ITYPE::bits < 8) ? (8 / vt::ITYPE::bits) : 1; static constexpr uint32_t kScalePack = 4; diff --git a/tests/regression/sgemm_tcu_wg/main.cpp b/tests/regression/sgemm_tcu_wg/main.cpp index 18b07262a1..2085b13249 100644 --- a/tests/regression/sgemm_tcu_wg/main.cpp +++ b/tests/regression/sgemm_tcu_wg/main.cpp @@ -366,17 +366,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint16_t eval(uint16_t a, uint16_t b, uint16_t c) { - auto fa = bit_cast(rv_htof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_htof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_htof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftoh_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static float eval(uint16_t a, uint16_t b, float c) { @@ -386,17 +375,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint16_t eval(uint16_t a, uint16_t b, uint16_t c) { - auto fa = bit_cast(rv_btof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_btof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_btof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftob_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static float eval(uint8_t a, uint8_t b, float c) { @@ -406,17 +384,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint8_t eval(uint8_t a, uint8_t b, uint8_t c) { - auto fa = bit_cast(rv_e4m3tof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_e4m3tof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_e4m3tof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftoe4m3_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static float eval(uint8_t a, uint8_t b, float c) { @@ -426,17 +393,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint8_t eval(uint8_t a, uint8_t b, uint8_t c) { - auto fa = bit_cast(rv_e5m2tof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_e5m2tof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_e5m2tof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftoe5m2_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static float eval(uint32_t a, uint32_t b, float c) { @@ -446,17 +402,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint32_t eval(uint32_t a, uint32_t b, uint32_t c) { - auto fa = bit_cast(rv_tf32tof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_tf32tof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_tf32tof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftotf32_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static int32_t eval(uint8_t a, uint8_t b, int32_t c) { @@ -501,6 +446,9 @@ static constexpr uint32_t per_warp_N = wg_cfg::xtileN; using itype_t = typename vt::ITYPE::dtype; using otype_t = typename vt::OTYPE::dtype; +static_assert(vt::tcu_output_valid(vt::ITYPE::id, vt::OTYPE::id), + "TCU output type must be fp32 (int32 for integer inputs)"); + static void matmul_cpu(otype_t *C, const itype_t *A, const itype_t *B, uint32_t M, uint32_t N, uint32_t K) { uint32_t subbytes = 8 / vt::ITYPE::bits; @@ -520,8 +468,7 @@ static void matmul_cpu(otype_t *C, const itype_t *A, const itype_t *B, uint32_t } data_accessor_t::write(C, m * N + n, static_cast(acc)); } else { - // Narrow outputs re-round the accumulator to the output type every - // step, matching the hardware chain exactly. + // int32 output (integer inputs): exact accumulation. otype_t sum(0); for (uint32_t k = 0; k < KS; ++k) { auto a = data_accessor_t::read(A, m * KS + k); diff --git a/tests/regression/sgemm_tcu_wg_dxa/main.cpp b/tests/regression/sgemm_tcu_wg_dxa/main.cpp index a651a7bda6..f9175cdf33 100644 --- a/tests/regression/sgemm_tcu_wg_dxa/main.cpp +++ b/tests/regression/sgemm_tcu_wg_dxa/main.cpp @@ -174,17 +174,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint32_t eval(uint32_t a, uint32_t b, uint32_t c) { - auto fa = bit_cast(rv_tf32tof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_tf32tof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_tf32tof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftotf32_s(bit_cast(fd), 0, nullptr); - } -}; - /////////////////////////////////////////////////////////////////////////////// using cfg = vt::wgmma_config_t; @@ -192,6 +181,9 @@ using cfg = vt::wgmma_config_t(acc); } else { - // Narrow outputs re-round the accumulator to the output type every - // step, matching the hardware chain exactly. + // int32 output (integer inputs): exact accumulation. otype_t sum(0); for (uint32_t k = 0; k < K; ++k) { sum = muladd_t::eval(A[m * K + k], B[k * N + n], sum); diff --git a/tests/regression/sgemm_tcu_wg_dxa_mcast/main.cpp b/tests/regression/sgemm_tcu_wg_dxa_mcast/main.cpp index f32fbd3da4..aca8b0962f 100644 --- a/tests/regression/sgemm_tcu_wg_dxa_mcast/main.cpp +++ b/tests/regression/sgemm_tcu_wg_dxa_mcast/main.cpp @@ -174,17 +174,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint32_t eval(uint32_t a, uint32_t b, uint32_t c) { - auto fa = bit_cast(rv_tf32tof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_tf32tof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_tf32tof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftotf32_s(bit_cast(fd), 0, nullptr); - } -}; - /////////////////////////////////////////////////////////////////////////////// using cfg = vt::wgmma_config_t; @@ -192,6 +181,9 @@ using cfg = vt::wgmma_config_t(acc); } else { - // Narrow outputs re-round the accumulator to the output type every - // step, matching the hardware chain exactly. + // int32 output (integer inputs): exact accumulation. otype_t sum(0); for (uint32_t k = 0; k < K; ++k) { sum = muladd_t::eval(A[m * K + k], B[k * N + n], sum); diff --git a/tests/regression/sgemm_tcu_wg_mx/main.cpp b/tests/regression/sgemm_tcu_wg_mx/main.cpp index f149b1dffc..ac8b594628 100644 --- a/tests/regression/sgemm_tcu_wg_mx/main.cpp +++ b/tests/regression/sgemm_tcu_wg_mx/main.cpp @@ -35,6 +35,9 @@ static constexpr uint32_t kTileN = cfg::xtileN; using itype_t = typename vt::ITYPE::dtype; using otype_t = typename vt::OTYPE::dtype; +static_assert(vt::tcu_output_valid(vt::ITYPE::id, vt::OTYPE::id), + "TCU output type must be fp32 (int32 for integer inputs)"); + static constexpr uint32_t kElemsPerByte = (vt::ITYPE::bits < 8) ? (8 / vt::ITYPE::bits) : 1; static constexpr uint32_t kScalePack = 4; From 8206c7248dbdaf5e313c4cf4abc2fbd159398d89 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 01:27:55 -0700 Subject: [PATCH 108/124] ci: check fence_mp's release fence and .rl atomic behind a slow bank 0 fence_mp-5 (-m1, release/acquire fences) and fence_mp-6 (-m2, .rl/.aq atomics) run two cores with an L2 under DBG_MEM_BANK0_WDELAY, on simx and rtlsim. Bank 0's writes are held back, and the flag, in a bank-1 line, overtakes the data in bank-0 lines unless the release waits for them. Without the delay, no shape would fail a release that does not drain. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/regression.yaml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/ci/testcases/regression.yaml b/ci/testcases/regression.yaml index d2d6c695d6..c9693b26a6 100644 --- a/ci/testcases/regression.yaml +++ b/ci/testcases/regression.yaml @@ -288,6 +288,31 @@ tests: cores: 2 l2cache: true l3cache: true +# A release must drain: bank 0's writes are held back by the debug delay, and +# the flag, in a bank-1 line, overtakes the data in bank-0 lines unless the +# release fence (-m1) or the .rl atomic (-m2) waits for them. +- id: fence_mp-5 + via: blackbox + drivers: + - simx + - rtlsim + app: fence_mp + args: -n64 -m1 + configs: -DVX_CFG_EXT_A_ENABLE -DDBG_MEM_BANK0_WDELAY + shape: + cores: 2 + l2cache: true +- id: fence_mp-6 + via: blackbox + drivers: + - simx + - rtlsim + app: fence_mp + args: -n64 -m2 + configs: -DVX_CFG_EXT_A_ENABLE -DDBG_MEM_BANK0_WDELAY + shape: + cores: 2 + l2cache: true # Kernel to kernel with no host transfer between (one command batch): the # launch flushes are L1-only, so the last-level cache alone carries each # round's writes to the next round, and the full flush runs before readback. From c9b78161c406f367ecf42ea1eb2212c390b08e17 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 01:27:56 -0700 Subject: [PATCH 109/124] cache: merge same-line icache fetches from a core's fetch slices With FETCH_WIDTH > 1 each fetch slice has its own icache port. Slices of one core often fetch different words of the same line in the same cycle (sgemm's three-line loop does almost every cycle), and each took its own bank access, serialized behind the others. READ_MERGE_LINE extends read merging from same-word to same-line reads. Plain reads of one line from a merge group in the same cycle share one bank access. The request carries every group port's {tag, word select} above the merged-port mask, the bank returns one word per slot, and the response crossbar hands each port its own word and tag. Only read-only caches may use it (static assert). The icache enables it through VX_CFG_ICACHE_READ_MERGE, default 1, which changes nothing at the default FETCH_WIDTH of 1. SimX merges by line in the port arbiter. The merged ports' tags ride the leader's request through the bank and back on its response, as in the RTL tag, and the response multicast hands each port its own. Under line merging the bank reads the whole line: decoding every slot's word select would sit on the data array's enable path. CI: config1:icache_merge-1 (sgemm, four fetch slices) and icache_merge-2 (raycast, four fetch slices on 16 icache banks), on simx and rtlsim. Co-Authored-By: Claude Opus 5.5 --- VX_config.toml | 3 ++ ci/testcases/config1.yaml | 18 +++++++++ hw/rtl/VX_socket.sv | 3 ++ hw/rtl/cache/VX_cache.sv | 68 ++++++++++++++++++++++++++------ hw/rtl/cache/VX_cache_bank.sv | 47 ++++++++++++++++++++-- hw/rtl/cache/VX_cache_cluster.sv | 2 + hw/rtl/cache/VX_cache_data.sv | 6 ++- hw/rtl/cache/VX_cache_wrap.sv | 2 + sim/simx/mem/cache.cpp | 54 +++++++++++++++++++++---- sim/simx/mem/cache.h | 1 + sim/simx/socket.cpp | 4 +- sim/simx/types.h | 5 +++ 12 files changed, 186 insertions(+), 27 deletions(-) diff --git a/VX_config.toml b/VX_config.toml index ab8a07a5dc..3abeb0d774 100644 --- a/VX_config.toml +++ b/VX_config.toml @@ -218,6 +218,9 @@ VX_CFG_ICACHE_CRSQ_SIZE = 0 VX_CFG_ICACHE_MEM_PORTS = 1 # banks behind the fetch-slice request ports VX_CFG_ICACHE_NUM_BANKS = "expr: pow(2, clog2($VX_CFG_FETCH_WIDTH))" +# Merge same-line fetches from the core's slices in the same cycle into one +# bank access; the response goes to every merged slice, each with its word. +VX_CFG_ICACHE_READ_MERGE = 1 VX_CFG_ICACHE_LATENCY = "expr: 2 + max(0, clog2($VX_CFG_ICACHE_SIZE) - clog2(16384))" VX_CFG_DCACHE_SIZE = 16384 diff --git a/ci/testcases/config1.yaml b/ci/testcases/config1.yaml index f8ef17bbcb..a5d8bdcf45 100644 --- a/ci/testcases/config1.yaml +++ b/ci/testcases/config1.yaml @@ -401,3 +401,21 @@ tests: - simx app: regression/vecadd configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_NUM_LSU_BLOCKS=4 -DVX_CFG_NUM_LSU_LANES=4 +# Fetch slices reading one line in the same cycle share one icache access, and +# each slice must get its own word: sgemm's three-line loop merges almost every +# cycle. The second case puts the merged responses behind 16 banks. +- id: icache_merge-1 + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm + args: -n64 + configs: -DVX_CFG_NUM_WARPS=16 -DVX_CFG_NUM_THREADS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_FETCH_WIDTH=4 +- id: icache_merge-2 + via: blackbox + drivers: + - simx + - rtlsim + app: raycast + configs: -DVX_CFG_NUM_WARPS=16 -DVX_CFG_NUM_THREADS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_FETCH_WIDTH=4 -DVX_CFG_ICACHE_NUM_BANKS=16 diff --git a/hw/rtl/VX_socket.sv b/hw/rtl/VX_socket.sv index a8fc37e113..0552c7e28e 100644 --- a/hw/rtl/VX_socket.sv +++ b/hw/rtl/VX_socket.sv @@ -171,6 +171,9 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .NC_ENABLE (0), // read-only: no dirty data, and the dcache's flush walks the shared levels .FLUSH_FORWARD (0), + // a core's fetch slices read different words of the same lines + .READ_MERGE (`VX_CFG_ICACHE_READ_MERGE ? ICACHE_NUM_REQS : 0), + .READ_MERGE_LINE (1), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3) ) icache ( diff --git a/hw/rtl/cache/VX_cache.sv b/hw/rtl/cache/VX_cache.sv index 5184a6f56d..4b5b17c86e 100644 --- a/hw/rtl/cache/VX_cache.sv +++ b/hw/rtl/cache/VX_cache.sv @@ -61,6 +61,7 @@ module VX_cache import VX_gpu_pkg::*; #( parameter BANK_HASH = 0, // hashed bank select (cache_bank_hash) parameter READ_MERGE = 0, // merge same-word reads within groups of this many adjacent ports (0: off) + parameter READ_MERGE_LINE = 0, // merge any reads of the same line instead, each port keeping its own tag and word parameter AMO_ENABLE = 0 // synthesize atomic-op logic in the banks ) ( @@ -95,7 +96,13 @@ module VX_cache import VX_gpu_pkg::*; #( // leader's group) below the request tag through the bank. localparam MERGE_GROUP = (READ_MERGE != 0) ? `MIN(READ_MERGE, NUM_REQS) : 1; localparam DO_READ_MERGE = (MERGE_GROUP > 1); - localparam BANK_TAG_WIDTH = TAG_WIDTH + (DO_READ_MERGE ? MERGE_GROUP : 0); + // Line merging also carries every group port's {tag, word select}, one + // slot per port above the mask, and the bank answers with one word per slot. + localparam MERGE_LINE = DO_READ_MERGE && (READ_MERGE_LINE != 0); + localparam SLOT_WIDTH = TAG_WIDTH + WORD_SEL_WIDTH; + localparam SLOTS_WIDTH = MERGE_LINE ? (MERGE_GROUP * SLOT_WIDTH) : 0; + localparam RSP_WORDS = MERGE_LINE ? MERGE_GROUP : 1; + localparam BANK_TAG_WIDTH = TAG_WIDTH + SLOTS_WIDTH + (DO_READ_MERGE ? MERGE_GROUP : 0); localparam CORE_REQ_DATAW = LINE_ADDR_WIDTH + 1 + WORD_SEL_WIDTH + WORD_SIZE + WORD_WIDTH + BANK_TAG_WIDTH + `UP(MEM_ATTR_WIDTH); localparam CORE_RSP_DATAW = WORD_WIDTH + TAG_WIDTH; localparam BANK_MEM_TAG_WIDTH = UUID_WIDTH + MSHR_ADDR_WIDTH; @@ -288,7 +295,7 @@ module VX_cache import VX_gpu_pkg::*; #( wire [NUM_BANKS-1:0][`UP(MEM_ATTR_WIDTH)-1:0] per_bank_core_req_attr; wire [NUM_BANKS-1:0] per_bank_core_req_ready; wire [NUM_BANKS-1:0] per_bank_core_rsp_valid; - wire [NUM_BANKS-1:0][`CS_WORD_WIDTH-1:0] per_bank_core_rsp_data; + wire [NUM_BANKS-1:0][RSP_WORDS-1:0][`CS_WORD_WIDTH-1:0] per_bank_core_rsp_data; wire [NUM_BANKS-1:0][BANK_TAG_WIDTH-1:0] per_bank_core_rsp_tag; wire [NUM_BANKS-1:0][REQ_SEL_WIDTH-1:0] per_bank_core_rsp_idx; wire [NUM_BANKS-1:0] per_bank_core_rsp_ready; @@ -345,11 +352,11 @@ module VX_cache import VX_gpu_pkg::*; #( assign core_req_bid[i] = '0; end end - // Read merging: plain reads of the same word with the same tag from ports - // of one group in the same cycle are served by one bank access. The lowest - // such port leads; the others stay out of the request crossbar and are - // accepted in the cycle their leader is, so every port keeps its own - // request order into the bank. + // Read merging: plain reads of the same word with the same tag (or, under + // line merging, of the same line) from ports of one group in the same cycle + // are served by one bank access. The lowest such port leads; the others + // stay out of the request crossbar and are accepted in the cycle their + // leader is, so every port keeps its own request order into the bank. wire [NUM_REQS-1:0] core_req_follower; wire [NUM_REQS-1:0] core_req_xbar_ready; wire [NUM_REQS-1:0][BANK_TAG_WIDTH-1:0] core_req_bank_tag; @@ -369,9 +376,15 @@ module VX_cache import VX_gpu_pkg::*; #( for (genvar i = 0; i < NUM_REQS; ++i) begin : g_same_i for (genvar j = 0; j < NUM_REQS; ++j) begin : g_same_j if (j < i && (j / MERGE_GROUP) == (i / MERGE_GROUP)) begin : g_cmp - assign same_word[i][j] = plain_read[i] && plain_read[j] - && (core_req_addr[i] == core_req_addr[j]) - && (core_req_tag[i] == core_req_tag[j]); + if (MERGE_LINE) begin : g_line + assign same_word[i][j] = plain_read[i] && plain_read[j] + && (core_req_line_addr[i] == core_req_line_addr[j]) + && (core_req_bid[i] == core_req_bid[j]); + end else begin : g_word + assign same_word[i][j] = plain_read[i] && plain_read[j] + && (core_req_addr[i] == core_req_addr[j]) + && (core_req_tag[i] == core_req_tag[j]); + end end else begin : g_none assign same_word[i][j] = 1'b0; end @@ -387,8 +400,18 @@ module VX_cache import VX_gpu_pkg::*; #( for (genvar j = 0; j < NUM_REQS; ++j) begin : g_leader_fire assign leader_fire[j] = same_word[i][j] && ~core_req_follower[j] && core_req_xbar_ready[j]; end + wire [`UP(SLOTS_WIDTH)-1:0] slots; + if (MERGE_LINE) begin : g_slots + for (genvar k = 0; k < MERGE_GROUP; ++k) begin : g_slot + assign slots[k * SLOT_WIDTH +: SLOT_WIDTH] = {core_req_tag[GROUP_BASE + k], core_req_wsel[GROUP_BASE + k]}; + end + assign core_req_bank_tag[i] = {core_req_tag[i], slots, merged_mask}; + end else begin : g_no_slots + assign slots = '0; + `UNUSED_VAR (slots) + assign core_req_bank_tag[i] = {core_req_tag[i], merged_mask}; + end assign core_req_follower[i] = (| same_word[i]); - assign core_req_bank_tag[i] = {core_req_tag[i], merged_mask}; assign core_req_ready[i] = core_req_follower[i] ? (| leader_fire) : core_req_xbar_ready[i]; end end else begin : g_no_read_merge @@ -473,6 +496,9 @@ module VX_cache import VX_gpu_pkg::*; #( .MREQ_SIZE (MREQ_SIZE), .LATENCY (LATENCY), .TAG_WIDTH (BANK_TAG_WIDTH), + .RSP_WORDS (RSP_WORDS), + .RSP_WSEL_OFFS (DO_READ_MERGE ? MERGE_GROUP : 0), + .RSP_WSEL_STRIDE (SLOT_WIDTH), .CORE_OUT_BUF (CORE_RSP_BUF_ENABLE ? 3 : 0), .MEM_OUT_BUF (MEM_REQ_BUF_ENABLE ? 3 : 0), .IS_LLC (IS_LLC), @@ -524,7 +550,7 @@ module VX_cache import VX_gpu_pkg::*; #( wire [NUM_REQS-1:0] core_rsp_ready; wire [NUM_BANKS-1:0][CORE_RSP_DATAW-1:0] per_bank_core_rsp_pdata; for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_per_bank_core_rsp_pdata - assign per_bank_core_rsp_pdata[i] = {per_bank_core_rsp_data[i], per_bank_core_rsp_tag[i][BANK_TAG_WIDTH-1 -: TAG_WIDTH]}; + assign per_bank_core_rsp_pdata[i] = {per_bank_core_rsp_data[i][0], per_bank_core_rsp_tag[i][BANK_TAG_WIDTH-1 -: TAG_WIDTH]}; end // A flush token's answer takes bank 0's lane into the response crossbar // when the bank has no response of its own, so it leaves through the same @@ -557,6 +583,22 @@ module VX_cache import VX_gpu_pkg::*; #( wire [NUM_BANKS-1:0][NUM_REQS-1:0] lane_taken; wire [NUM_REQS-1:0][NUM_BANKS-1:0] per_output_valid; wire [NUM_REQS-1:0][NUM_BANKS-1:0] per_output_ready; + wire [NUM_REQS-1:0][NUM_BANKS-1:0][CORE_RSP_DATAW-1:0] per_output_pdata; + // Under line merging each port takes its own slot's word and tag; the + // leader keeps the lane's tag, which a flush answer also uses. + for (genvar j = 0; j < NUM_REQS; ++j) begin : g_output_pdata + for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_lane + if (MERGE_LINE) begin : g_slot + localparam K = j % MERGE_GROUP; + wire [TAG_WIDTH-1:0] lane_tag = core_rsp_xbar_pdata[i][TAG_WIDTH-1:0]; + wire [TAG_WIDTH-1:0] slot_tag = per_bank_core_rsp_tag[i][MERGE_GROUP + K * SLOT_WIDTH + WORD_SEL_WIDTH +: TAG_WIDTH]; + wire is_leader = (core_rsp_xbar_idx[i] == REQ_SEL_WIDTH'(j)); + assign per_output_pdata[j][i] = {per_bank_core_rsp_data[i][K], (is_leader ? lane_tag : slot_tag)}; + end else begin : g_word + assign per_output_pdata[j][i] = core_rsp_xbar_pdata[i]; + end + end + end for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_lanes wire [MERGE_GROUP-1:0] merged_mask; if (i == 0) begin : g_flush_mask @@ -593,7 +635,7 @@ module VX_cache import VX_gpu_pkg::*; #( .clk (clk), .reset (reset), .valid_in (per_output_valid[j]), - .data_in (core_rsp_xbar_pdata), + .data_in (per_output_pdata[j]), .ready_in (per_output_ready[j]), .valid_out (core_rsp_valid[j]), .data_out (core_rsp_pdata[j]), diff --git a/hw/rtl/cache/VX_cache_bank.sv b/hw/rtl/cache/VX_cache_bank.sv index d71663087e..893809a06c 100644 --- a/hw/rtl/cache/VX_cache_bank.sv +++ b/hw/rtl/cache/VX_cache_bank.sv @@ -41,6 +41,11 @@ module VX_cache_bank import VX_gpu_pkg::*; #( parameter IS_LLC = 0, // last-level cache: AMOs commit locally here parameter AMO_ENABLE = 0, // synthesize atomic-op logic parameter LATENCY = 2, // pipeline depth; >2 defers the data array to break the tag->data path + // Words per core response. Above 1, response word k is the line's word + // whose select rides in the request tag at RSP_WSEL_OFFS + k * RSP_WSEL_STRIDE. + parameter RSP_WORDS = 1, + parameter RSP_WSEL_OFFS = 0, + parameter RSP_WSEL_STRIDE = 0, parameter MSHR_ADDR_WIDTH = `LOG2UP(MSHR_SIZE), parameter MEM_TAG_WIDTH = UUID_WIDTH + MSHR_ADDR_WIDTH, parameter REQ_SEL_WIDTH = `UP(`CS_REQ_SEL_BITS), @@ -70,7 +75,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // Core response output wire core_rsp_valid, - output wire [`CS_WORD_WIDTH-1:0] core_rsp_data, + output wire [RSP_WORDS-1:0][`CS_WORD_WIDTH-1:0] core_rsp_data, output wire [TAG_WIDTH-1:0] core_rsp_tag, output wire [REQ_SEL_WIDTH-1:0] core_rsp_idx, input wire core_rsp_ready, @@ -108,6 +113,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( `STATIC_ASSERT(LATENCY >= 2, ("invalid parameter: cache bank LATENCY must be >= 2")) `STATIC_ASSERT(!LAZY_FETCH || (WRITEBACK && DIRTY_BYTES && IS_LLC), ("invalid parameter: lazy fetch requires a write-back LLC with dirty bytes")) `STATIC_ASSERT(!WRITE_AROUND || (WRITEBACK && WRITE_ENABLE), ("invalid parameter: write-around requires a write-back cache")) + `STATIC_ASSERT(RSP_WORDS == 1 || (!WRITE_ENABLE && !AMO_ENABLE), ("invalid parameter: multi-word responses are read-only")) `UNUSED_PARAM (MRSQ_SIZE) // ======================================================================== @@ -909,6 +915,16 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire mask_clr_line_stc = (mask_fill_stc && ~stC.lk.is_refill) || (stC.req.valid && stC.req.is_flush); wire mask_clr_sec_stc = mask_fill_stc && stC.lk.is_refill; + // The data slices a read returns: its word's, or under multi-word + // responses the whole line, since a merged read may take any word (and + // decoding the slots' selects here would sit on the array's enable path). + wire [`CS_WORDS_PER_LINE-1:0] read_words_std; + if (RSP_WORDS > 1) begin : g_read_words + assign read_words_std = '1; + end else begin : g_read_word + assign read_words_std = `CS_WORDS_PER_LINE'(1) << stD.req.word_idx; + end + VX_cache_data #( .CACHE_SIZE (CACHE_SIZE), .LINE_SIZE (LINE_SIZE), @@ -944,6 +960,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .fill_data (fbuf_line), .write_word (stD.wdata), .word_idx (stD.req.word_idx), + .read_words (read_words_std), .sector_idx (sector_idx_std), .write_byteen (stD.req.byteen), .read_data (read_data_stc), @@ -1311,11 +1328,33 @@ module VX_cache_bank import VX_gpu_pkg::*; #( assign fwd_word = fbuf_data_r[`CS_WORD_WIDTH-1:0]; end + // A multi-word response takes each word its tag selects, from the line at + // commit or from the staged sector on a forward. + wire [RSP_WORDS-1:0][`CS_WORD_WIDTH-1:0] crsp_words, fwd_words; + if (RSP_WORDS > 1) begin : g_rsp_words + for (genvar k = 0; k < RSP_WORDS; ++k) begin : g_word + wire [WORD_SEL_WIDTH-1:0] wsel_stc = stC.req.tag[RSP_WSEL_OFFS + k * RSP_WSEL_STRIDE +: WORD_SEL_WIDTH]; + wire [WORD_SEL_WIDTH-1:0] wsel_fwd = replay_tag[RSP_WSEL_OFFS + k * RSP_WSEL_STRIDE +: WORD_SEL_WIDTH]; + assign crsp_words[k] = read_data_stc[wsel_stc]; + if (`CS_WORDS_PER_SECTOR > 1) begin : g_fwd + wire [`CLOG2(`CS_WORDS_PER_SECTOR)-1:0] fwd_k = wsel_fwd[`CLOG2(`CS_WORDS_PER_SECTOR)-1:0]; + assign fwd_words[k] = fbuf_data_r[fwd_k * `CS_WORD_WIDTH +: `CS_WORD_WIDTH]; + end else begin : g_fwd_1 + assign fwd_words[k] = fbuf_data_r[`CS_WORD_WIDTH-1:0]; + `UNUSED_VAR (wsel_fwd) + end + end + `UNUSED_VAR ({crsp_queue_data, fwd_word}) + end else begin : g_rsp_word + assign crsp_words = crsp_queue_data; + assign fwd_words = fwd_word; + end + `RUNTIME_ASSERT (~fwd_fire || (replay_addr == fbuf_addr_r), ("%t: %s fill-forward address mismatch: addr=0x%0h, staged=0x%0h", $time, INSTANCE_ID, `CS_BANK_TO_FULL_ADDR(replay_addr, BANK_ID), `CS_BANK_TO_FULL_ADDR(fbuf_addr_r, BANK_ID))) `RUNTIME_ASSERT (~(flush_fire && fwd_pending), ("%t: %s flush during fill-forward drain", $time, INSTANCE_ID)) VX_elastic_buffer #( - .DATAW (TAG_WIDTH + `CS_WORD_WIDTH + REQ_SEL_WIDTH), + .DATAW (TAG_WIDTH + RSP_WORDS * `CS_WORD_WIDTH + REQ_SEL_WIDTH), .SIZE (CRSQ_QUEUE_SIZE), .OUT_REG (`TO_OUT_BUF_REG(CORE_OUT_BUF)) ) core_rsp_queue ( @@ -1323,8 +1362,8 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .reset (reset), .valid_in (crsp_queue_valid || fwd_head), .ready_in (crsp_queue_ready), - .data_in (crsp_queue_valid ? {stC.req.tag, crsp_queue_data, stC.req.req_idx} - : {replay_tag, fwd_word, replay_idx}), + .data_in (crsp_queue_valid ? {stC.req.tag, crsp_words, stC.req.req_idx} + : {replay_tag, fwd_words, replay_idx}), .data_out ({core_rsp_tag, core_rsp_data, core_rsp_idx}), .valid_out (core_rsp_valid), .ready_out (core_rsp_ready) diff --git a/hw/rtl/cache/VX_cache_cluster.sv b/hw/rtl/cache/VX_cache_cluster.sv index c7553e07e9..6065ef0f5e 100644 --- a/hw/rtl/cache/VX_cache_cluster.sv +++ b/hw/rtl/cache/VX_cache_cluster.sv @@ -63,6 +63,7 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( parameter FLUSH_FORWARD = 1, // A flush also flushes the next level. parameter BANK_HASH = 0, // Hashed bank select (cache_bank_hash). parameter READ_MERGE = 0, // Merge same-word reads within groups of this many adjacent ports (0: off). + parameter READ_MERGE_LINE = 0, // Merge any reads of the same line instead, each port keeping its own tag and word. parameter AMO_ENABLE = 0 // Enable atomic memory operations. ) ( @@ -189,6 +190,7 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( .FLUSH_FORWARD (FLUSH_FORWARD), .BANK_HASH (BANK_HASH), .READ_MERGE (READ_MERGE), + .READ_MERGE_LINE (READ_MERGE_LINE), .AMO_ENABLE (AMO_ENABLE) ) cache_wrap ( `ifdef PERF_ENABLE diff --git a/hw/rtl/cache/VX_cache_data.sv b/hw/rtl/cache/VX_cache_data.sv index bc5bf12385..16d79555b3 100644 --- a/hw/rtl/cache/VX_cache_data.sv +++ b/hw/rtl/cache/VX_cache_data.sv @@ -71,6 +71,7 @@ module VX_cache_data import VX_gpu_pkg::*; #( input wire [`CS_WORD_WIDTH-1:0] write_word, input wire [WORD_SIZE-1:0] write_byteen, input wire [`UP(`CS_WORD_SEL_BITS)-1:0] word_idx, + input wire [`CS_WORDS_PER_LINE-1:0] read_words, // slices a read returns (word_idx's, plus a merged read's) input wire [`UP(`CS_SECTOR_SEL_BITS)-1:0] sector_idx, // sector being filled // outputs output wire [`CS_LINE_WIDTH-1:0] read_data, @@ -180,8 +181,8 @@ module VX_cache_data import VX_gpu_pkg::*; #( wire fill_sec_en = (`CS_SECTORS_PER_LINE == 1) || (sector_idx == `UP(`CS_SECTOR_SEL_BITS)'(SLICE_SECTOR)); - // load reads the selected slice; writeback/flush reads all slices. - wire slice_read = (read && word_en) || ((fill || flush) && WRITEBACK); + // a load reads its words' slices; writeback/flush reads all slices. + wire slice_read = (read && read_words[s]) || ((fill || flush) && WRITEBACK); wire slice_write; wire [WRENW-1:0] slice_wren; @@ -202,6 +203,7 @@ module VX_cache_data import VX_gpu_pkg::*; #( `UNUSED_VAR (write_byteen) `UNUSED_VAR (hit_any) `UNUSED_VAR ({fill_write, merge_write}) + `UNUSED_VAR (word_en) assign slice_write = fill && fill_sec_en; assign slice_wren = 1'b1; assign slice_wdata = fill_data[s]; diff --git a/hw/rtl/cache/VX_cache_wrap.sv b/hw/rtl/cache/VX_cache_wrap.sv index 597e63a042..f71dbea3b2 100644 --- a/hw/rtl/cache/VX_cache_wrap.sv +++ b/hw/rtl/cache/VX_cache_wrap.sv @@ -65,6 +65,7 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( parameter FLUSH_FORWARD = 1, // A flush also flushes the next level. parameter BANK_HASH = 0, // Hashed bank select (cache_bank_hash). parameter READ_MERGE = 0, // Merge same-word reads within groups of this many adjacent ports (0: off). + parameter READ_MERGE_LINE = 0, // Merge any reads of the same line instead, each port keeping its own tag and word. parameter AMO_ENABLE = 0 // Enable atomic operations in cache banks. ) ( @@ -170,6 +171,7 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( .FLUSH_FORWARD (FLUSH_FORWARD), .BANK_HASH (BANK_HASH), .READ_MERGE (READ_MERGE), + .READ_MERGE_LINE (READ_MERGE_LINE), .AMO_ENABLE (AMO_ENABLE) ) cache ( .clk (clk), diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index abd71bf8af..6f09d976f1 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -423,6 +423,7 @@ struct bank_req_t { uint64_t addr; uint32_t hart_id; uint64_t req_tag; + std::shared_ptr> merged_tags; // ride to the response uint64_t uuid; uint32_t mshr_id; ReqType type; @@ -474,6 +475,7 @@ struct bank_req_t { addr = 0; hart_id = 0; req_tag = 0; + merged_tags.reset(); uuid = 0; mshr_id = 0; type = ReqType::None; @@ -1290,6 +1292,7 @@ class CacheBank : public SimObject { coalesced.hart_id = core_req.hart_id; coalesced.uuid = core_req.uuid; coalesced.req_tag = core_req.tag; + coalesced.merged_tags = core_req.merged_tags; coalesced.write = (core_req.op == MemOp::ST); coalesced.op = core_req.op; coalesced.data = core_req.data; @@ -1328,6 +1331,7 @@ class CacheBank : public SimObject { bank_req.hart_id = core_req.hart_id; bank_req.uuid = core_req.uuid; bank_req.req_tag = core_req.tag; + bank_req.merged_tags = core_req.merged_tags; // bank_req.write means "route through the cache write path" — true only // for plain stores. AMOs (op == AMO_SC / AMO RMW) are write-bearing // semantically (memop_is_write returns true) but use the dedicated AMO @@ -1537,6 +1541,7 @@ class CacheBank : public SimObject { mshr_.dequeue(&req); MemRsp rsp{req.req_tag, req.hart_id, req.uuid}; rsp.data = fwd_data_; + rsp.merged_tags = req.merged_tags; this->core_rsp_out.send(rsp); DT(3, this->name() << " fwd-rsp: " << rsp); return true; @@ -1934,6 +1939,7 @@ class CacheBank : public SimObject { MemRsp rsp{bank_req.req_tag, bank_req.hart_id, bank_req.uuid}; if (!bank_req.write) rsp.data = hit_sec.data; + rsp.merged_tags = bank_req.merged_tags; this->core_rsp_out.send(rsp); crsp_sent_ = true; DT(3, this->name() << " replay-rsp: " << rsp); @@ -2060,6 +2066,7 @@ class CacheBank : public SimObject { MemRsp rsp{bank_req.req_tag, bank_req.hart_id, bank_req.uuid}; if (!bank_req.write) rsp.data = hit_sec.data; + rsp.merged_tags = bank_req.merged_tags; this->core_rsp_out.send(rsp); crsp_sent_ = true; DT(3, this->name() << " core-rsp: " << rsp); @@ -2391,7 +2398,9 @@ class CacheBank : public SimObject { // With read merging, plain reads of the same word with the same tag from ports // of one group in the same cycle are served by one bank access: the lowest // such port leads, the others stay out of arbitration and are accepted with -// it. The merged ports' mask rides below the tag through the bank. +// it. The merged ports' mask rides below the tag through the bank. Line +// merging takes any plain read of the same line instead, whatever its word +// and tag. class CachePortArb : public SimObject { public: std::vector> ReqIn; // core ports @@ -2399,7 +2408,8 @@ class CachePortArb : public SimObject { std::vector> IoOut; // non-cacheable requests, per port CachePortArb(const SimContext& ctx, const char* name, const params_t& params, - uint32_t num_inputs, uint32_t num_banks, uint32_t merge_group) + uint32_t num_inputs, uint32_t num_banks, uint32_t merge_group, + bool merge_line) : SimObject(ctx, name) , ReqIn(num_inputs, this) , ReqOut(num_banks, this) @@ -2410,6 +2420,7 @@ class CachePortArb : public SimObject { , delay_((num_inputs > 2) ? 1 : 0) , levels_(arb_tree_levels(num_inputs)) , merge_group_(merge_group) + , merge_line_(merge_line) { __assert(merge_group_ <= 1 || (merge_group_ <= 64 && ispow2(merge_group_) && (num_inputs % merge_group_) == 0), "a read-merge group must be a power of two of at most 64 dividing the ports"); @@ -2595,8 +2606,11 @@ class CachePortArb : public SimObject { continue; } auto& req_j = ReqIn.at(j).peek(); - if ((req_j.addr >> params_.word_select_addr_start) == (req_i.addr >> params_.word_select_addr_start) - && req_j.tag == req_i.tag) { + bool same = merge_line_ + ? ((req_j.addr >> params_.bank_select_addr_start) == (req_i.addr >> params_.bank_select_addr_start)) + : ((req_j.addr >> params_.word_select_addr_start) == (req_i.addr >> params_.word_select_addr_start) + && req_j.tag == req_i.tag); + if (same) { merged.at(j) |= 1ull << (i - base); followers.at(i) = true; break; @@ -2672,6 +2686,7 @@ class CachePortArb : public SimObject { uint64_t tag = (((uint64_t(req.tag) << this->merge_bits()) | mask) << lg2_inputs_) | i; __assert((tag >> 32) == 0, "cache bank tag overflow"); req.tag = uint32_t(tag); + req.merged_tags = this->merged_tags(i, mask); auto& offer = offers_.at(b); offer.valid = true; offer.port = i; @@ -2705,6 +2720,21 @@ class CachePortArb : public SimObject { bool offers_enabled() const { return levels_.empty() && delay_ == 0; } + // Under line merging, the merged ports' own tags, which ride the leader's + // bank access and its response. + std::shared_ptr> merged_tags(uint32_t i, uint64_t mask) const { + if (!merge_line_ || mask == 0) { + return nullptr; + } + uint32_t base = (i / merge_group_) * merge_group_; + auto tags = std::make_shared>(merge_group_, 0); + for (uint64_t f = mask; f != 0; f &= f - 1) { + uint32_t k = __builtin_ctzll(f); + tags->at(k) = ReqIn.at(base + k).peek().tag; + } + return tags; + } + // Send port i's request to its bank, with the ports merged into it. void grant(uint32_t i, SimChannel& out, const std::vector& merged) { auto req = ReqIn.at(i).peek(); @@ -2712,6 +2742,7 @@ class CachePortArb : public SimObject { if (merge_group_ > 1) { uint32_t base = (i / merge_group_) * merge_group_; mask = merged.at(i); + req.merged_tags = this->merged_tags(i, mask); for (uint64_t f = mask; f != 0; f &= f - 1) { uint32_t port = base + __builtin_ctzll(f); DT(3, this->name() << " read-merge: port=" << port << ", leader=" << i << ", " << req); @@ -2743,6 +2774,7 @@ class CachePortArb : public SimObject { std::vector>> tree_stage_; // [level][bank][slice] std::vector> rr_; // [arbiter level][bank][slice] next to grant uint32_t merge_group_; + bool merge_line_; uint64_t collisions_ = 0; struct offer_t { SimChannel* bank_in = nullptr; @@ -2773,12 +2805,13 @@ class CacheRspMcast : public SimObject { std::vector> RspOut; // core ports CacheRspMcast(const SimContext& ctx, const char* name, uint32_t num_banks, - uint32_t num_outputs, uint32_t merge_group) + uint32_t num_outputs, uint32_t merge_group, bool merge_line) : SimObject(ctx, name) , RspIn(num_banks, this) , RspOut(num_outputs, this) , lg2_outputs_(log2ceil(num_outputs)) , merge_group_(merge_group) + , merge_line_(merge_line) , slices_(arb_tree_slices(num_banks)) , sent_(num_banks, 0) , last_grant_(num_outputs * slices_, num_banks - 1) @@ -2846,7 +2879,12 @@ class CacheRspMcast : public SimObject { continue; } auto rsp = RspIn.at(b).peek(); + uint32_t lane_tag = rsp.tag; rsp.tag >>= (lg2_outputs_ + merge_group_); + if (merge_line_ && o != (lane_tag & ((1u << lg2_outputs_) - 1))) { + __assert(rsp.merged_tags, "a line-merged response lost its merged tags"); + rsp.tag = rsp.merged_tags->at(o % merge_group_); + } if (out.try_send(rsp, (slices_ > 1) ? 1 : 0)) { DT(4, this->name() << " rsp" << b << "_" << o << ": " << rsp); sent_.at(b) |= 1ull << o; @@ -2867,6 +2905,7 @@ class CacheRspMcast : public SimObject { private: uint32_t lg2_outputs_; uint32_t merge_group_; + bool merge_line_; uint32_t slices_; std::vector sent_; // [bank] ports that took the lane's response std::vector last_grant_; // [port][slice] round-robin pointer @@ -3023,7 +3062,8 @@ class Cache::Impl { // Create the core-port arbiter ahead of the banks it feeds snprintf(sname, 100, "%s-port_arb", simobject->name().c_str()); uint32_t merge_group = std::min(config_.read_merge, config_.num_inputs); - port_arb_ = CachePortArb::Create(sname, params_, config_.num_inputs, num_banks, merge_group); + const bool merge_line = config_.merge_line && merge_group > 1; + port_arb_ = CachePortArb::Create(sname, params_, config_.num_inputs, num_banks, merge_group, merge_line); for (uint32_t i = 0; i < config_.num_inputs; ++i) { simobject->core_req_in.at(i).bind(&port_arb_->ReqIn.at(i)); } @@ -3038,7 +3078,7 @@ class Cache::Impl { if (port_arb_->merge_bits() != 0) { // one buffered arbiter per core port over the bank lanes __assert(config_.core_out_buf, "read merging needs a registered core-response port"); - rsp_mcast_ = CacheRspMcast::Create(sname, num_banks, config_.num_inputs, port_arb_->merge_bits()); + rsp_mcast_ = CacheRspMcast::Create(sname, num_banks, config_.num_inputs, port_arb_->merge_bits(), merge_line); core_rsp_delay_ = 1; } else if (config_.core_out_buf) { bank_core_xbar_ = MemCrossBar::Create(sname, ArbiterType::RoundRobin, config_.num_inputs, num_banks, bank_sel, 0, 0); diff --git a/sim/simx/mem/cache.h b/sim/simx/mem/cache.h index de87d79d01..a65ee53196 100644 --- a/sim/simx/mem/cache.h +++ b/sim/simx/mem/cache.h @@ -54,6 +54,7 @@ class Cache : public SimObject { bool mem_rsp_queue = false; // memory responses enter through a response queue bool mem_rsp_slice = false; // a registered boundary ahead of the memory-response port stands for that queue's register bool acq_forward = false; // an acquire flush also invalidates the next level + bool merge_line = false; // read merging takes any read of the same line, each merged port keeping its own tag and word }; struct PerfStats { diff --git a/sim/simx/socket.cpp b/sim/simx/socket.cpp index 22b0f64820..b5b32226f3 100644 --- a/sim/simx/socket.cpp +++ b/sim/simx/socket.cpp @@ -75,11 +75,13 @@ class Socket::Impl { false, // shared_llc false, // lazy_fetch true, // core-response register - 0, // read_merge + (VX_CFG_ICACHE_READ_MERGE != 0) ? ICACHE_NUM_REQS : 0, // read_merge: the core's fetch slices 0, // mshr_lines false, // write_around VX_CFG_ICACHE_MRSQ_SIZE != 0, // memory-response queue true, // the socket's registered memory port + false, // acq_forward + true, // merge_line: slices fetch different words of a line }); snprintf(sname, 100, "%s-dcache", name.c_str()); diff --git a/sim/simx/types.h b/sim/simx/types.h index 0fb7ab61ea..b18936dbd3 100644 --- a/sim/simx/types.h +++ b/sim/simx/types.h @@ -1245,6 +1245,10 @@ struct MemReq { uint32_t hart_id; uint64_t uuid; MemFlags flags; + // Under cache line merging: the tags of the ports merged into this read, by + // offset in the leader's merge group. They ride the bank access and its + // response, as in the hardware's bank tag. + std::shared_ptr> merged_tags; MemReq(MemOp _op = MemOp::LD, uint64_t _addr = 0, @@ -1284,6 +1288,7 @@ struct MemRsp { uint32_t hart_id; uint64_t uuid; std::shared_ptr data; + std::shared_ptr> merged_tags; // see MemReq MemRsp(uint64_t _tag = 0, uint32_t _hart_id = 0, From 1babe84fbff329b803dd126a8fee8c0bb95bdeca Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 01:27:57 -0700 Subject: [PATCH 110/124] cache: keep a draining write-through chain pending, retime bank paths On a write-through cache the MSHR's pending test left out write entries, because their data had already gone to memory. But a chained store reaches the array only when it replays behind the fill. Once the chain's read head had replayed, a same-word load arriving before the store's replay found the line resident and hit, returning the value from before the thread's own store. Writes now count as pending on either policy, so the load links behind the store. SimX already chained such a hit. Several compare cones move off the bank's critical paths. Cycles do not change, and each carries a runtime assertion against the original combinational result: - VX_cache_amo registers its lane hits in every configuration and takes its coalesce keys a cycle early. - The dirty-byte mask is read one stage ahead of commit from a dual-port RAM, with write forwarding (MASK_SYNC, deferred-commit banks only), so commit takes it from a register and a large mask fits block RAM. - The MSHR takes its allocate match a cycle early against the request entering S0. The order-hazard probes drop the replay-dequeue exclusion, which the bank's link hold makes redundant, taking the S0 address compare out of the data array's write-enable cone. raw_order issues load, store, store, store, delay, load per warp and sweeps the delay across the chain's drain, then checks that every load returned its own store. CI: cache:raw_order-wt (write-through L1 under an L2, simx and rtlsim). Co-Authored-By: Claude Opus 5.5 --- ci/testcases/cache.yaml | 15 +++ docs/designs/cache_subsystem.md | 5 +- hw/rtl/cache/VX_cache_amo.sv | 80 ++++++++----- hw/rtl/cache/VX_cache_bank.sv | 48 ++++++-- hw/rtl/cache/VX_cache_data.sv | 110 +++++++++++++----- hw/rtl/cache/VX_cache_mshr.sv | 64 +++++++++-- tests/regression/Makefile | 2 +- tests/regression/raw_order/Makefile | 16 +++ tests/regression/raw_order/common.h | 19 ++++ tests/regression/raw_order/kernel.cpp | 61 ++++++++++ tests/regression/raw_order/main.cpp | 158 ++++++++++++++++++++++++++ 11 files changed, 503 insertions(+), 75 deletions(-) create mode 100644 tests/regression/raw_order/Makefile create mode 100644 tests/regression/raw_order/common.h create mode 100644 tests/regression/raw_order/kernel.cpp create mode 100644 tests/regression/raw_order/main.cpp diff --git a/ci/testcases/cache.yaml b/ci/testcases/cache.yaml index d2e9377c61..12bd2c56b4 100644 --- a/ci/testcases/cache.yaml +++ b/ci/testcases/cache.yaml @@ -385,3 +385,18 @@ tests: shape: cores: 4 l2cache: true +# A thread's load must return its own earlier store to the same word. On the +# write-through L1 under an L2, the stores of a miss chain reach the array only +# as they replay behind the fill: a load landing after the chain's read head +# has replayed, and before the store has, must link behind the store rather +# than hit the line ahead of it. The delay sweep lands the load across that +# window. +- id: raw_order-wt + via: blackbox + drivers: + - simx + - rtlsim + app: raw_order + args: -n16384 -s64 -d2 + shape: + l2cache: true diff --git a/docs/designs/cache_subsystem.md b/docs/designs/cache_subsystem.md index 80c5038263..8fc8e61c06 100644 --- a/docs/designs/cache_subsystem.md +++ b/docs/designs/cache_subsystem.md @@ -411,7 +411,10 @@ to new requests. This removes the read-after-fill round trip from miss latency. The window closes on the **first write or AMO head**. That head and every later chain entry replay through the pipeline as usual, preserving program order: an -older store must merge before a younger same-line read responds. +older store must merge before a younger same-line read responds. Stores still +waiting in the chain keep its line pending, on a write-through cache too, where +they have already been written through: a same-word access that arrives while +they drain links behind them instead of hitting the array ahead of them. Two details make it correct: diff --git a/hw/rtl/cache/VX_cache_amo.sv b/hw/rtl/cache/VX_cache_amo.sv index 63a857d330..a1d3338c8f 100644 --- a/hw/rtl/cache/VX_cache_amo.sv +++ b/hw/rtl/cache/VX_cache_amo.sv @@ -322,17 +322,13 @@ module VX_cache_amo import VX_gpu_pkg::*; #( wire [WBQ_SIZE-1:0][NUM_LANES-1:0] wbq_lane_hit; wire [NUM_LANES-1:0] post_lane_hit; - if ((PIPE_EX != 0) && (NUM_LANES > 1)) begin : g_hit_reg - // A deferred-commit bank has the next commit-stage key a cycle - // early. Where the word spans several lanes the forward merge is - // wide enough for the {line, word} compare to set the response - // path, so the lane hits are registered and the forward network - // and the response start from flops: each is taken between the - // queue's next contents and the request entering the commit stage - // next cycle, with the slot's occupancy folded in. The queue moves - // during a stall while the commit request holds, so both keys are - // compared and the stall selects. A single-lane word keeps the - // live compare, which is cheaper there. + // The lane hits are registered so the forward network and the + // response start from flops: each is taken between the queue's next + // contents and the request entering the commit stage next cycle (S0 + // itself when the commit is not deferred), with the slot's occupancy + // folded in. The queue moves during a stall while the commit request + // holds, so both keys are compared and the stall selects. + if (1) begin : g_hit_reg wire [WBQ_SIZE-1:0] wbq_key_n; for (genvar i = 0; i < WBQ_SIZE; ++i) begin : g_wbq_key_n assign wbq_key_n[i] = (wbq_addr[i] == res_addr_n) && (wbq_wsel[i] == word_idx_n); @@ -386,17 +382,6 @@ module VX_cache_amo import VX_gpu_pkg::*; #( end assign wbq_lane_hit = wbq_lane_hit_r; assign post_lane_hit = post_lane_hit_r; - end else begin : g_hit_live - for (genvar i = 0; i < WBQ_SIZE; ++i) begin : g_wbq - for (genvar l = 0; l < NUM_LANES; ++l) begin : g_lane - assign wbq_lane_hit[i][l] = (WBQ_CNTW'(i) < wbq_count) && wbq_key_st1[i] - && (wbq_lane[i] == LANE_IDXW'(l)); - end - end - for (genvar l = 0; l < NUM_LANES; ++l) begin : g_post_lane - assign post_lane_hit[l] = post_wb_valid && post_key_st1 && (post_wb_lane == LANE_IDXW'(l)); - end - `UNUSED_VAR (word_idx_n) end wire [WBQ_SIZE-1:0] wbq_word_hit; @@ -585,13 +570,54 @@ module VX_cache_amo import VX_gpu_pkg::*; #( // independent of the drain: the highest match is the target either way // unless it is the draining head, so wb_fire, which arrives late with // the bank's stall, only picks between the two precomputed outcomes. + // The {line, word, lane} keys against the compute stage are taken a + // cycle early, when it loads from the commit stage, against each slot + // and the settling entry as they stand after this cycle's update, so + // the target select and the merge start from flops. A result pushes + // only the cycle after its load, the one cycle these keys are used. + wire [WBQ_SIZE-1:0] coal_eq_same, coal_eq_up; + for (genvar i = 0; i < WBQ_SIZE; ++i) begin : g_coal_eq + assign coal_eq_same[i] = (wbq_addr[i] == addr_st1) && (wbq_wsel[i] == word_idx_st1) + && (wbq_lane[i] == lane_st1); + if (i < WBQ_SIZE-1) begin : g_up + assign coal_eq_up[i] = (wbq_addr[i+1] == addr_st1) && (wbq_wsel[i+1] == word_idx_st1) + && (wbq_lane[i+1] == lane_st1); + end else begin : g_last + assign coal_eq_up[i] = coal_eq_same[i]; + end + end + wire coal_eq_cmp = (cmp_addr == addr_st1) && (cmp_wsel == word_idx_st1) && (cmp_lane == lane_st1); + wire coal_eq_post = (post_wb_addr == addr_st1) && (post_wb_wsel == word_idx_st1) + && (post_wb_lane == lane_st1); + reg [WBQ_SIZE-1:0] coal_key_r; + reg post_coal_key_r; + always @(posedge clk) begin + for (integer i = 0; i < WBQ_SIZE; ++i) begin + if (wb_push && (wb_slot == WBQ_IDXW'(i))) begin + coal_key_r[i] <= coal_eq_cmp; + end else if (wb_fire) begin + coal_key_r[i] <= coal_eq_up[i]; + end else begin + coal_key_r[i] <= coal_eq_same[i]; + end + end + post_coal_key_r <= wb_fire ? coal_eq_same[0] : coal_eq_post; + end + for (genvar i = 0; i < WBQ_SIZE; ++i) begin : g_coal_check + `RUNTIME_ASSERT (~wb_push || (coal_key_r[i] == ((wbq_addr[i] == cmp_addr) && (wbq_wsel[i] == cmp_wsel) + && (wbq_lane[i] == cmp_lane))), + ("%t: AMO coalesce key out of sync (slot=%0d)", $time, i)) + end + `RUNTIME_ASSERT (~wb_push || (post_coal_key_r == ((post_wb_addr == cmp_addr) && (post_wb_wsel == cmp_wsel) + && (post_wb_lane == cmp_lane))), + ("%t: AMO settling-entry coalesce key out of sync", $time)) + reg [WBQ_SIZE-1:0] wb_coal_hit; reg [WBQ_IDXW-1:0] wb_coal_idx; // pre-shift index of the coalesce target always @(*) begin wb_coal_idx = '0; for (integer i = 0; i < WBQ_SIZE; ++i) begin - wb_coal_hit[i] = (WBQ_CNTW'(i) < wbq_count) && (wbq_addr[i] == cmp_addr) - && (wbq_wsel[i] == cmp_wsel) && (wbq_lane[i] == cmp_lane); + wb_coal_hit[i] = (WBQ_CNTW'(i) < wbq_count) && coal_key_r[i]; if (wb_coal_hit[i]) begin wb_coal_idx = WBQ_IDXW'(i); end @@ -613,10 +639,8 @@ module VX_cache_amo import VX_gpu_pkg::*; #( // than it: drop those bytes from the settling entry, so the forward // never has two writers for one byte. On a drain the settling entry // is reloaded from the head, which a same-key push never coalesces into. - wire post_push_post = wb_push && (post_wb_addr == cmp_addr) && (post_wb_wsel == cmp_wsel) - && (post_wb_lane == cmp_lane); - wire post_push_head = wb_push && (wbq_addr[0] == cmp_addr) && (wbq_wsel[0] == cmp_wsel) - && (wbq_lane[0] == cmp_lane); + wire post_push_post = wb_push && post_coal_key_r; + wire post_push_head = wb_push && coal_key_r[0]; always @(posedge clk) begin if (reset) begin diff --git a/hw/rtl/cache/VX_cache_bank.sv b/hw/rtl/cache/VX_cache_bank.sv index 893809a06c..2b90f0e660 100644 --- a/hw/rtl/cache/VX_cache_bank.sv +++ b/hw/rtl/cache/VX_cache_bank.sv @@ -500,6 +500,17 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // pending wb (chained AMO replays from MSHR after a fill) and must not be // cleared by amo_wb_pending; the synthetic writeback carries amo.valid=0 so // it never re-commits at S1. + // The core-request slot's address: a pending LLC AMO writeback or the core + // request. The MSHR takes its allocate match against it a cycle early. + wire [`CS_LINE_ADDR_WIDTH-1:0] creq_addr_sel = amo_wb_pending ? amo_wb_addr : core_req_addr; + wire [WORD_SEL_WIDTH-1:0] creq_wsel_sel = amo_wb_pending ? amo_wb_word_idx : core_req_wsel; + wire [`UP(`CS_SECTOR_SEL_BITS)-1:0] creq_sector_sel; + if (`CS_SECTOR_SEL_BITS != 0) begin : g_creq_sector + assign creq_sector_sel = creq_wsel_sel[`CS_WORD_SEL_BITS-1 -: `CS_SECTOR_SEL_BITS]; + end else begin : g_creq_sector_0 + assign creq_sector_sel = '0; + end + always @(*) begin sel_req = '0; sel_req.req.valid = init_fire || replay_fire || fill_fire || flush_fire || core_req_fire; @@ -513,14 +524,13 @@ module VX_cache_bank import VX_gpu_pkg::*; #( sel_req.req.attr = amo_wb_pending ? amo_wb_attr : (core_req_valid ? core_req_attr : '0); sel_req.req.way_idx = flush_way; sel_req.req.addr = (init_valid | flush_valid) ? `CS_LINE_ADDR_WIDTH'(flush_sel) - : (replay_mux ? replay_addr : (fill_mux ? mem_rsp_addr - : (amo_wb_pending ? amo_wb_addr : core_req_addr))); + : (replay_mux ? replay_addr : (fill_mux ? mem_rsp_addr : creq_addr_sel)); sel_req.req.byteen = replay_mux ? replay_byteen : (amo_wb_pending ? amo_wb_byteen : core_req_byteen); // a fill carries the installed sector in its word_idx high bits so the // tag/data stages mark/write the right sector (0 when 1 sector/line). sel_req.req.word_idx = replay_mux ? replay_wsel : (fill_mux ? fill_word_idx - : (amo_wb_pending ? amo_wb_word_idx : core_req_wsel)); + : creq_wsel_sel); sel_req.req.req_idx = replay_mux ? replay_idx : (amo_wb_pending ? amo_wb_idx : core_req_idx); sel_req.req.tag = (init_valid | flush_valid) ? (flush_valid ? flush_tag : '0) : (replay_mux ? replay_tag : (fill_mux ? (lazy_sel ? '0 : mem_rsp_tag_s) @@ -699,8 +709,8 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // disjoint words — the common case in streaming, where the whole warp's // beats land on one sector — keep the fast release path, as does // read-after-read on the same word. + // (the probes carry their own pending test; see VX_cache_mshr) assign creq_hit_order_hazard_st0 = st0.req.is_creq && ~st0.req.is_replay - && mshr_pending_raw && (mshr_pending_wr_raw || (st0.req.rw && mshr_pending_ovl_raw)) && ~is_amo_fwd_st0 && ~((AMO_ENABLE != 0) && st0.req.amo.amo_valid); @@ -774,15 +784,30 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .data_out (st1) ); + // stP is the stage that feeds stC: the dirty-byte mask is read there, so + // stC gets it from a register. + commit_t stP; VX_pipe_register #( .DATAW ($bits(commit_t)), .RESETW (1), - .DEPTH (PIPE_EX) - ) reg_cmt ( + .DEPTH ((PIPE_EX != 0) ? (PIPE_EX - 1) : 0) + ) reg_cmt_p ( .clk (clk), .reset (reset), .enable (~pipe_stall), .data_in (st1), + .data_out (stP) + ); + + VX_pipe_register #( + .DATAW ($bits(commit_t)), + .RESETW (1), + .DEPTH ((PIPE_EX != 0) ? 1 : 0) + ) reg_cmt ( + .clk (clk), + .reset (reset), + .enable (~pipe_stall), + .data_in (stP), .data_out (stC) ); @@ -935,7 +960,8 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .WRITE_ENABLE (WRITE_ENABLE), .WRITEBACK (WRITEBACK), .DIRTY_BYTES (DIRTY_BYTES), - .LAZY_FETCH (LAZY_FETCH) + .LAZY_FETCH (LAZY_FETCH), + .MASK_SYNC (PIPE_EX != 0) ) cache_data ( .clk (clk), .reset (reset), @@ -950,6 +976,9 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .mask_word (stC.req.word_idx), .mask_byteen (stC.req.byteen), .mask_sector (sector_idx_stc), + .mask_read (~pipe_stall), + .mask_rway (stP.req.way_idx), + .mask_rline (stP.req.addr[`CS_LINE_SEL_BITS-1:0]), .merge_write (do_merge_stc && ~pipe_stall), .flush (do_flush_std && ~pipe_stall), .read (do_read_std && ~pipe_stall), @@ -1045,7 +1074,6 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .NUM_BANKS (NUM_BANKS), .MSHR_SIZE (MSHR_SIZE), .MSHR_LINES (MSHR_LINES), - .WRITEBACK (WRITEBACK), .AMO_ENABLE (AMO_ENABLE != 0), .AMO_PASSTHRU ((AMO_ENABLE != 0) && (IS_LLC == 0)), .WORD_SEL_WIDTH (WORD_SEL_WIDTH), @@ -1077,12 +1105,16 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .dequeue_data ({replay_wsel, replay_byteen, replay_data, replay_tag, replay_idx, replay_amo}), .dequeue_id (replay_id), .dequeue_ready (replay_ready || fwd_fire), + .dequeue_fwd (fwd_fire), .dequeue_park (park_sel), .dequeue_park_retry (chain_retry), .dequeue_block (replay_block), .allocate_valid (mshr_allocate_st0 && ~pipe_stall), .allocate_addr (st0.req.addr), .allocate_sector (sector_idx_st0), + .allocate_addr_n (creq_addr_sel), + .allocate_sector_n (creq_sector_sel), + .allocate_adv (~pipe_stall), .allocate_word_idx (st0.req.word_idx), .allocate_rw (st0.req.rw), // Only non-LLC AMOs must not coalesce; at the LLC same-line AMOs coalesce diff --git a/hw/rtl/cache/VX_cache_data.sv b/hw/rtl/cache/VX_cache_data.sv index 16d79555b3..247a847dd1 100644 --- a/hw/rtl/cache/VX_cache_data.sv +++ b/hw/rtl/cache/VX_cache_data.sv @@ -40,7 +40,8 @@ module VX_cache_data import VX_gpu_pkg::*; #( parameter WRITE_ENABLE = 1, // Enable cache writeable parameter WRITEBACK = 0, // Enable cache writeback parameter DIRTY_BYTES = 0, // Enable dirty bytes on writeback - parameter LAZY_FETCH = 0 // Fills may be data-less or merge under the dirty bytes + parameter LAZY_FETCH = 0, // Fills may be data-less or merge under the dirty bytes + parameter MASK_SYNC = 0 // Dirty-byte mask read one stage ahead of the commit stage ) ( input wire clk, input wire reset, @@ -58,6 +59,11 @@ module VX_cache_data import VX_gpu_pkg::*; #( input wire [`UP(`CS_WORD_SEL_BITS)-1:0] mask_word, input wire [WORD_SIZE-1:0] mask_byteen, input wire [`UP(`CS_SECTOR_SEL_BITS)-1:0] mask_sector, + // MASK_SYNC: the {way, line} that enters the commit stage next, and its + // advance enable + input wire mask_read, + input wire [`CS_WAY_SEL_WIDTH-1:0] mask_rway, + input wire [`CS_LINE_SEL_BITS-1:0] mask_rline, // merge of a refetched sector into its resident copy at the commit stage // (the stage behind it is empty): only the clean bytes input wire merge_write, @@ -112,8 +118,8 @@ module VX_cache_data import VX_gpu_pkg::*; #( end // Per-byte dirty mask (writeback only), one mask word per {way, line}. It - // is read and updated only at the commit stage, so each access sees every - // older update with no bypass. A line's mask is cleared when it is + // is updated only at the commit stage, in pipeline order, so each access + // sees every older update. A line's mask is cleared when it is // installed, which leaves nothing for the reset walk to clear. if (DIRTY_BYTES != 0) begin : g_dirty_bytes wire [`CS_WORDS_PER_LINE-1:0][WORD_SIZE-1:0] set_word; @@ -131,39 +137,89 @@ module VX_cache_data import VX_gpu_pkg::*; #( wire [LINE_SIZE-1:0] set_mask = set_word; wire [LINE_SIZE-1:0] clr_mask = clr_sector; - wire [DATA_ADDR_WIDTH-1:0] mask_addr; + wire [DATA_ADDR_WIDTH-1:0] mask_addr, mask_raddr; if (NUM_WAYS > 1) begin : g_mask_way_addr - assign mask_addr = DATA_ADDR_WIDTH'({mask_way[WAY_SEL_BITS-1:0], mask_line}); + assign mask_addr = DATA_ADDR_WIDTH'({mask_way[WAY_SEL_BITS-1:0], mask_line}); + assign mask_raddr = DATA_ADDR_WIDTH'({mask_rway[WAY_SEL_BITS-1:0], mask_rline}); end else begin : g_mask_line_addr - assign mask_addr = mask_line; + `UNUSED_VAR (mask_rway) + assign mask_addr = mask_line; + assign mask_raddr = mask_rline; end - // A whole-word read-modify-write shares one write enable across the - // mask, so it maps onto wide distributed RAM rather than one cell per - // byte (or a shattered block RAM). wire [LINE_SIZE-1:0] mask_rdata; - VX_sp_ram #( - .DATAW (LINE_SIZE), - .SIZE (DATA_RAM_DEPTH), - .OUT_REG (0), - .LUTRAM (1), - .RDW_MODE ("W"), - .INIT_ENABLE (1), - .INIT_VALUE (0) - ) byteen_store ( - .clk (clk), - .reset (reset), - .read (1'b1), - .write (mask_write), - .wren (1'b1), - .addr (mask_addr), - .wdata ((mask_rdata & ~clr_mask) | set_mask), - .rdata (mask_rdata) - ); + wire [LINE_SIZE-1:0] mask_wdata = (mask_rdata & ~clr_mask) | set_mask; + + if (MASK_SYNC != 0) begin : g_mask_sync + // Read at the stage ahead, so the commit stage takes the mask from + // a register rather than through a deep distributed-RAM read mux + // (and a large mask fits block RAM). A write by the request now + // committing to the address read behind it is forwarded. The + // pipeline holds as a whole, so nothing writes while the read is + // held. + wire [LINE_SIZE-1:0] mask_ram_rdata; + VX_dp_ram #( + .DATAW (LINE_SIZE), + .SIZE (DATA_RAM_DEPTH), + .OUT_REG (1), + .RDW_MODE ("R"), + .INIT_ENABLE (1), + .INIT_VALUE (0) + ) byteen_store ( + .clk (clk), + .reset (reset), + .read (mask_read), + .write (mask_write), + .wren (1'b1), + .waddr (mask_addr), + .wdata (mask_wdata), + .raddr (mask_raddr), + .rdata (mask_ram_rdata) + ); + + reg mask_fwd_r; + reg [LINE_SIZE-1:0] mask_fwd_data_r; + reg [DATA_ADDR_WIDTH-1:0] mask_raddr_r; + always @(posedge clk) begin + if (mask_read) begin + mask_fwd_r <= mask_write && (mask_addr == mask_raddr); + mask_fwd_data_r <= mask_wdata; + mask_raddr_r <= mask_raddr; + end + end + assign mask_rdata = mask_fwd_r ? mask_fwd_data_r : mask_ram_rdata; + + `RUNTIME_ASSERT (~mask_write || mask_read, ("%t: dirty-byte mask written while its read is held", $time)) + `RUNTIME_ASSERT (~mask_write || (mask_addr == mask_raddr_r), ("%t: dirty-byte mask read for another line: 0x%0h, 0x%0h", $time, mask_addr, mask_raddr_r)) + end else begin : g_mask_async + `UNUSED_VAR ({mask_read, mask_raddr}) + // A whole-word read-modify-write shares one write enable across the + // mask, so it maps onto wide distributed RAM rather than one cell per + // byte (or a shattered block RAM). + VX_sp_ram #( + .DATAW (LINE_SIZE), + .SIZE (DATA_RAM_DEPTH), + .OUT_REG (0), + .LUTRAM (1), + .RDW_MODE ("W"), + .INIT_ENABLE (1), + .INIT_VALUE (0) + ) byteen_store ( + .clk (clk), + .reset (reset), + .read (1'b1), + .write (mask_write), + .wren (1'b1), + .addr (mask_addr), + .wdata (mask_wdata), + .rdata (mask_rdata) + ); + end assign evict_byteen = mask_rdata; end else begin : g_no_dirty_bytes `UNUSED_VAR ({mask_write, mask_set, mask_clr_line, mask_clr_sector}) `UNUSED_VAR ({mask_line, mask_word, mask_byteen, mask_sector}) + `UNUSED_VAR ({mask_read, mask_rway, mask_rline}) assign evict_byteen = '1; // update whole line end diff --git a/hw/rtl/cache/VX_cache_mshr.sv b/hw/rtl/cache/VX_cache_mshr.sv index c314df553c..05616d0bdd 100644 --- a/hw/rtl/cache/VX_cache_mshr.sv +++ b/hw/rtl/cache/VX_cache_mshr.sv @@ -61,7 +61,6 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( parameter MSHR_SIZE = 4, // Miss Reserv Queue Knob parameter MSHR_LINES = 0, // distinct line entries (0 = address held per request) parameter DATA_WIDTH = 1, // MSHR parameters - parameter WRITEBACK = 0, // Enable cache writeback parameter AMO_ENABLE = 0, // Per-entry AMO tracking (probe outputs) parameter AMO_PASSTHRU = 0, // Non-LLC passthrough: AMO entries never coalesce parameter WORD_SEL_WIDTH = 1, // word-in-line select width (order-hazard granularity) @@ -115,6 +114,8 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( output wire [DATA_WIDTH-1:0] dequeue_data, output wire [MSHR_ADDR_WIDTH-1:0] dequeue_id, input wire dequeue_ready, + // the dequeue this cycle is the bank's fill-forward port, not a replay + input wire dequeue_fwd, // park the chain at its head instead of consuming it: the entry stays // allocated (it becomes the root of a refetch) and the walk stops. input wire dequeue_park, @@ -126,6 +127,10 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( input wire allocate_valid, input wire [`CS_LINE_ADDR_WIDTH-1:0] allocate_addr, input wire [`UP(`CS_SECTOR_SEL_BITS)-1:0] allocate_sector, + // the core request entering S0 next cycle, and the S0 advance enable + input wire [`CS_LINE_ADDR_WIDTH-1:0] allocate_addr_n, + input wire [`UP(`CS_SECTOR_SEL_BITS)-1:0] allocate_sector_n, + input wire allocate_adv, input wire [WORD_SEL_WIDTH-1:0] allocate_word_idx, input wire allocate_rw, input wire allocate_is_amo, // AMO: never coalesce @@ -321,6 +326,7 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( if (MSHR_LINES != 0) begin : g_line_table localparam LINE_ID_W = `LOG2UP(MSHR_LINES); + `UNUSED_VAR ({allocate_addr_n, allocate_sector_n, allocate_adv}) localparam LCNT_W = `CLOG2(MSHR_SIZE + 1); reg [`CS_LINE_ADDR_WIDTH-1:0] line_addr [0:MSHR_LINES-1]; @@ -440,10 +446,27 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( end end + // The allocate match is taken a cycle early, against the core request + // entering S0 and the table as it stands after this cycle's allocate, + // so S0 starts it from flops: it gates the data array's write enable + // through the order-hazard probes. A request of another kind entering + // S0 does not allocate, so its match is never used. Nothing allocates + // while S0 holds. + wire alloc_self_match = (allocate_addr == allocate_addr_n) && (allocate_sector == allocate_sector_n); + reg [MSHR_SIZE-1:0] entry_match_r; for (genvar i = 0; i < MSHR_SIZE; ++i) begin : g_entry_match - assign entry_match[i] = (addr_table[i] == allocate_addr) && (sector_table[i] == allocate_sector); + wire match_n = (allocate_fire && (allocate_id == MSHR_ADDR_WIDTH'(i))) ? alloc_self_match + : ((addr_table[i] == allocate_addr_n) && (sector_table[i] == allocate_sector_n)); + always @(posedge clk) begin + if (allocate_adv) begin + entry_match_r[i] <= match_n; + end + end + assign entry_match[i] = entry_match_r[i]; assign entry_probe[i] = (addr_table[i] == probe_addr); assign entry_addr[i] = addr_table[i]; + `RUNTIME_ASSERT (~allocate_valid || (entry_match_r[i] == ((addr_table[i] == allocate_addr) && (sector_table[i] == allocate_sector))), + ("%t: *** %s allocate match out of sync: id=%0d", $time, INSTANCE_ID, i)) end assign fill_addr = addr_table[fill_id]; @@ -619,12 +642,12 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( assign allocate_id = allocate_id_r; assign allocate_previd = prev_idx; - if (WRITEBACK) begin : g_pending_wb - assign allocate_pending = |addr_matches; - end else begin : g_pending_wt - // exclude write requests if writethrough - assign allocate_pending = |(addr_matches & ~write_table); - end + // Any undrained entry of the line is pending, writes included. On a + // write-through cache a write entry only lives behind a read root, so once + // the root has replayed the writes still waiting are a chain mid-drain: a + // same-word access must link behind them, not overtake them through the + // array. + assign allocate_pending = |addr_matches; // Word-granular order-hazard probes. A chained entry only constrains a // requester that touches the SAME word: the entry's replay reads or @@ -634,14 +657,35 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( for (genvar i = 0; i < MSHR_SIZE; ++i) begin : g_word_matches assign word_matches[i] = (word_table[i] == allocate_word_idx); end + // These probes gate the data array's write enable at S0, so they leave the + // replay dequeue out of their exclusion: a replay that could change them (a + // same-word entry, with a write on either side, against an allocating core + // request) is held that cycle by the bank's link hold, so the exclusion + // never alters them, and dropping it takes the S0 address compare behind + // that hold out of the write-enable cone. The forward port is not held, so + // its dequeue stays excluded, as does a release. + wire [MSHR_SIZE-1:0] hazard_matches; + for (genvar i = 0; i < MSHR_SIZE; ++i) begin : g_hazard_matches + assign hazard_matches[i] = valid_table[i] && entry_match[i] && ~amo_mask[i] + && ~(dequeue_fwd && ~((LAZY_FETCH != 0) && dequeue_park) && (dequeue_id == MSHR_ADDR_WIDTH'(i))) + && ~(finalize_valid && finalize_is_release && (finalize_id == MSHR_ADDR_WIDTH'(i))); + end + wire hazard_pending = |hazard_matches; // The probed word's chain holds a WRITE that has not reached the data array // yet. A same-word hit completing ahead of it would read pre-store data (or // a write-hit would be overwritten by the older store's replay), inverting // arrival order — the bank demotes such a hit to a chained miss. - assign allocate_pending_wr = |(addr_matches & write_table & word_matches); + assign allocate_pending_wr = hazard_pending && |(hazard_matches & write_table & word_matches); // Any undrained same-word entry: an incoming WRITE hit must not land ahead // of an older chained read of the same word. - assign allocate_pending_ovl = |(addr_matches & word_matches); + assign allocate_pending_ovl = hazard_pending && |(hazard_matches & word_matches); + // the order hazard the bank derives from the two probes is the one the full + // exclusion gives + `RUNTIME_ASSERT (~allocate_valid + || ((allocate_pending_wr || (allocate_rw && allocate_pending_ovl)) + == (allocate_pending && (|(addr_matches & write_table & word_matches) + || (allocate_rw && |(addr_matches & word_matches))))), + ("%t: *** %s order-hazard probe diverged from the full exclusion", $time, INSTANCE_ID)) assign dequeue_valid = dequeue_val && ~((LAZY_FETCH != 0) && dequeue_block); // Sampled with dequeue_id_n, alongside the payload in mshr_store, rather diff --git a/tests/regression/Makefile b/tests/regression/Makefile index a11c131c2f..c6f6777677 100644 --- a/tests/regression/Makefile +++ b/tests/regression/Makefile @@ -15,7 +15,7 @@ TESTS := \ dxa_copy dxa_copy_mcast dxa_kmajor_check \ async_barrier async_gbarrier gbar_phase bar_slot_phase packld wgather wsync quad_lod \ occupancy multikernel reopen module_reload single_cta \ - flush_coherence lazy_fetch sector_merge \ + flush_coherence lazy_fetch sector_merge raw_order \ vm_test vm_fault vm_stress # --- common exclude list --------------------------------------------- diff --git a/tests/regression/raw_order/Makefile b/tests/regression/raw_order/Makefile new file mode 100644 index 0000000000..48e56c03b6 --- /dev/null +++ b/tests/regression/raw_order/Makefile @@ -0,0 +1,16 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := raw_order + +SRC_DIR := $(VORTEX_HOME)/tests/regression/$(PROJECT) + +SRCS := $(SRC_DIR)/main.cpp + +VX_SRCS := $(SRC_DIR)/kernel.cpp + +OPTS ?= -n2048 + +KERNEL_LIB := vortex2 + +include ../common.mk diff --git a/tests/regression/raw_order/common.h b/tests/regression/raw_order/common.h new file mode 100644 index 0000000000..dde800c43d --- /dev/null +++ b/tests/regression/raw_order/common.h @@ -0,0 +1,19 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include + +#define RAW_MARK 0x80000000u + +typedef struct { + uint32_t num_words; // words in the data buffer + uint32_t num_threads; // total threads launched + uint32_t warp_size; // threads per warp + uint32_t delay_sweep; // iteration k delays the checked load (k % delay_sweep) * delay_step steps + uint32_t delay_step; + uint64_t data_addr; // in: initial words; out: regions 0, 1 and 3 overwritten with RAW_MARK + index + uint64_t new_addr; // out: the value each region-0 load returned + uint64_t sink_addr; // out: per-thread sum of the region-2 loads (keeps them alive) +} kernel_arg_t; + +#endif diff --git a/tests/regression/raw_order/kernel.cpp b/tests/regression/raw_order/kernel.cpp new file mode 100644 index 0000000000..f704172ba3 --- /dev/null +++ b/tests/regression/raw_order/kernel.cpp @@ -0,0 +1,61 @@ +#include +#include "common.h" + +// Each warp owns a group of four regions (one word per lane each, one cache +// line per group) and, per iteration, issues in program order: +// +// load r2 - misses: heads the group's miss chain (answered from the fill) +// store r3 - chained behind it +// store r1 - chained behind it +// store r0 - chained behind it +// delay - warp-uniform countdown, swept per iteration +// load r0 - the checked load: must return the store to r0 above +// +// On a write-through cache the stores leave for memory at once and update +// the array when they replay behind the fill. The sweep lands the checked +// load at varying points of the chain's drain, including after the head has +// replayed and before the store to r0 has. The sequence is inline assembly +// so no compiler scheduling moves the requests relative to each other. +__kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { + auto data = reinterpret_cast(arg->data_addr); + auto nval = reinterpret_cast(arg->new_addr); + auto sink = reinterpret_cast(arg->sink_addr); + const uint32_t W = arg->warp_size; + const uint32_t tid = blockIdx.x * blockDim.x + threadIdx.x; + const uint32_t lane = tid % W; + const uint32_t warp = tid / W; + const uint32_t num_warps = arg->num_threads / W; + const uint32_t group_words = 4 * W; + uint32_t acc = 0; + uint32_t k = 0; + for (uint32_t g = warp; (g + 1) * group_words <= arg->num_words; g += num_warps, ++k) { + uint32_t* r0 = data + g * group_words + lane; + uint32_t* r1 = r0 + W; + uint32_t* r2 = r0 + 2 * W; + uint32_t* r3 = r0 + 3 * W; + uint32_t i0 = g * group_words + lane; + uint32_t mark0 = RAW_MARK + i0; + uint32_t mark1 = RAW_MARK + i0 + W; + uint32_t mark3 = RAW_MARK + i0 + 3 * W; + uint32_t delay = (k % arg->delay_sweep) * arg->delay_step; + uint32_t a, v; + __asm__ volatile( + "lw %[a], 0(%[r2])\n\t" + "sw %[m3], 0(%[r3])\n\t" + "sw %[m1], 0(%[r1])\n\t" + "sw %[m0], 0(%[r0])\n\t" + "beqz %[n], 2f\n\t" + "1:\n\t" + "addi %[n], %[n], -1\n\t" + "bnez %[n], 1b\n\t" + "2:\n\t" + "lw %[v], 0(%[r0])\n\t" + : [a] "=&r"(a), [v] "=&r"(v), [n] "+r"(delay) + : [r0] "r"(r0), [r1] "r"(r1), [r2] "r"(r2), [r3] "r"(r3), + [m0] "r"(mark0), [m1] "r"(mark1), [m3] "r"(mark3) + : "memory"); + acc += a; + nval[i0] = v; + } + sink[tid] = acc; +} diff --git a/tests/regression/raw_order/main.cpp b/tests/regression/raw_order/main.cpp new file mode 100644 index 0000000000..24a9c44404 --- /dev/null +++ b/tests/regression/raw_order/main.cpp @@ -0,0 +1,158 @@ +// raw_order — a thread's load must observe its own earlier store to the same +// word, even while the line is still draining its miss chain. + +#include +#include "common.h" + +#include +#include +#include +#include +#include +#include + +#define CHECK(expr) do { \ + vx_result_t _r = (expr); \ + if (_r != VX_SUCCESS) { \ + std::fprintf(stderr, "FAIL %s:%d: '%s' returned %s\n", \ + __FILE__, __LINE__, #expr, vx_result_string(_r)); \ + std::exit(1); \ + } \ +} while (0) + +namespace { +const char* kernel_file = "kernel.vxbin"; +uint32_t num_words = 1024; +uint32_t delay_sweep = 4; +uint32_t delay_step = 1; + +void parse_args(int argc, char** argv) { + int c; + while ((c = getopt(argc, argv, "n:s:d:k:h")) != -1) { + switch (c) { + case 'n': num_words = std::atoi(optarg); break; + case 's': delay_sweep = std::atoi(optarg); break; + case 'd': delay_step = std::atoi(optarg); break; + case 'k': kernel_file = optarg; break; + default: + std::cout << "Usage: [-n words] [-s delay sweep] [-d delay step] [-k kernel] [-h]" << std::endl; + std::exit(c == 'h' ? 0 : -1); + } + } +} + +uint32_t init_word(uint32_t i) { return i * 7 + 1; } +} // namespace + +int main(int argc, char** argv) { + parse_args(argc, argv); + if (delay_sweep == 0) delay_sweep = 1; + + vx_device_h dev = nullptr; + CHECK(vx_device_open(0, &dev)); + + // Fill every warp of every core. + uint64_t nt = 0, nw = 0, nc = 0; + CHECK(vx_device_query(dev, VX_CAPS_NUM_THREADS, &nt)); + CHECK(vx_device_query(dev, VX_CAPS_NUM_WARPS, &nw)); + CHECK(vx_device_query(dev, VX_CAPS_NUM_CORES, &nc)); + const uint32_t num_threads = static_cast(nt * nw * nc); + std::cout << "raw_order: words=" << num_words << " threads=" << num_threads + << " delay=" << delay_sweep << "x" << delay_step << std::endl; + + vx_queue_info_t qi = { sizeof(qi), nullptr, VX_QUEUE_PRIORITY_NORMAL, 0 }; + vx_queue_h q = nullptr; + CHECK(vx_queue_create(dev, &qi, &q)); + + const uint64_t data_size = uint64_t(num_words) * sizeof(uint32_t); + const uint64_t sink_size = uint64_t(num_threads) * sizeof(uint32_t); + vx_buffer_h data_buf = nullptr, new_buf = nullptr, sink_buf = nullptr; + CHECK(vx_buffer_create(dev, data_size, VX_MEM_READ_WRITE, &data_buf)); + CHECK(vx_buffer_create(dev, data_size, VX_MEM_WRITE, &new_buf)); + CHECK(vx_buffer_create(dev, sink_size, VX_MEM_WRITE, &sink_buf)); + + vx_module_h mod = nullptr; + vx_kernel_h kern = nullptr; + CHECK(vx_module_load_file(dev, kernel_file, &mod)); + CHECK(vx_module_get_kernel(mod, "main", &kern)); + + kernel_arg_t kernel_arg{}; + kernel_arg.num_words = num_words; + kernel_arg.num_threads = num_threads; + kernel_arg.warp_size = static_cast(nt); + kernel_arg.delay_sweep = delay_sweep; + kernel_arg.delay_step = delay_step; + CHECK(vx_buffer_address(data_buf, &kernel_arg.data_addr)); + CHECK(vx_buffer_address(new_buf, &kernel_arg.new_addr)); + CHECK(vx_buffer_address(sink_buf, &kernel_arg.sink_addr)); + + std::vector h_data(num_words), h_new(num_words); + for (uint32_t i = 0; i < num_words; ++i) h_data[i] = init_word(i); + CHECK(vx_enqueue_write(q, data_buf, 0, h_data.data(), data_size, 0, nullptr, nullptr)); + + uint32_t grid[1], block[1]; + CHECK(vx_device_max_occupancy_grid(dev, 1, &num_threads, grid, block)); + + vx_launch_info_t li{}; + li.struct_size = sizeof(li); + li.kernel = kern; + li.args_host = &kernel_arg; + li.args_size = sizeof(kernel_arg); + li.ndim = 1; + li.grid_dim[0] = grid[0]; + li.block_dim[0]= block[0]; + + vx_event_h launch_ev = nullptr, read_ev = nullptr; + CHECK(vx_enqueue_launch(q, &li, 0, nullptr, &launch_ev)); + CHECK(vx_enqueue_read(q, h_new.data(), new_buf, 0, data_size, 1, &launch_ev, nullptr)); + CHECK(vx_enqueue_read(q, h_data.data(), data_buf, 0, data_size, 1, &launch_ev, &read_ev)); + CHECK(vx_event_wait_value(read_ev, 1, VX_TIMEOUT_INFINITE)); + + // Each warp owns groups of four regions of warp_size words: regions 0, 1 + // and 3 are overwritten, region 0 is then loaded back, region 2 is only + // read. + const uint32_t W = static_cast(nt); + const uint32_t group_words = 4 * W; + const uint32_t groups = num_words / group_words; + if (groups < num_threads / W) { + std::cout << "words (" << num_words << ") must cover one group per warp (" + << (num_threads / W) * group_words << ")" << std::endl; + return 1; + } + int errors = 0; + auto expect = [&](const char* what, uint32_t i, uint32_t actual, uint32_t ref) { + if (actual == ref) return; + if (errors < 16) + std::printf("*** %s [%u] expected=0x%x actual=0x%x%s\n", what, i, ref, actual, + actual == init_word(i) ? " (before its own earlier store)" : ""); + ++errors; + }; + for (uint32_t g = 0; g < groups; ++g) { + for (uint32_t l = 0; l < W; ++l) { + uint32_t i0 = g * group_words + l; + expect("load r0", i0, h_new[i0], RAW_MARK + i0); + expect("data r0", i0, h_data[i0], RAW_MARK + i0); + expect("data r1", i0 + W, h_data[i0 + W], RAW_MARK + i0 + W); + expect("data r2", i0 + 2 * W, h_data[i0 + 2 * W], init_word(i0 + 2 * W)); + expect("data r3", i0 + 3 * W, h_data[i0 + 3 * W], RAW_MARK + i0 + 3 * W); + } + } + + vx_event_release(read_ev); + vx_event_release(launch_ev); + vx_buffer_release(sink_buf); + vx_buffer_release(new_buf); + vx_buffer_release(data_buf); + vx_kernel_release(kern); + vx_module_release(mod); + vx_queue_release(q); + vx_device_dump_perf(dev, stdout); + vx_device_release(dev); + + if (errors) { + std::cout << "Found " << errors << " errors!\nFAILED!" << std::endl; + return 1; + } + std::cout << "PASSED!" << std::endl; + return 0; +} From 32432bb9d9af5406a42c504d1b484383faf6a3d5 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 01:27:59 -0700 Subject: [PATCH 111/124] simx: match LSU ingress, SFU result buffers and lmem crossbar to RTL Diffing per-core issue order against the RTL on sgemm n64 with two cores and an L2 found three structures SimX did not model: - The LSU's lane-dispatch skid buffer, the scheduler's request queue and its pending table all register their full flags, so a slot freed in one cycle takes a new request in the next. SimX freed the entry, sent the next request and ingested a new trace in one tick. The LSU now admits against room sampled at the start of the tick, and its inputs are registered-ready. - The WCTL and CSR units each hold two results in their own buffer ahead of the SFU's shared response switch, which holds two more. SimX let one six-entry output channel absorb them, so up to ten SFU ops sat between dispatch and commit, where the RTL holds eight. A WCTL or CSR op is now admitted only while fewer than two of that unit's results wait beyond the switch's entries. Results reach commit after different latencies, so each is tracked at its arrival cycle, in the order commit pops them. - The local-memory request crossbar registers its output toward the banks. With it, a shared-memory load's minimum latency matches the RTL's 14 cycles. CI: core:parity-sgemm-mc64 (n64, two cores, L2, two operand collectors per issue slice, 3% tolerance) runs at +1.72% against the RTL. parity-sgemmx moves from -0.62% to -3.40%, still within its 5%: the registered LSU input is RTL-faithful, so another SimX error on that local-memory kernel had been offsetting it. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/core.yaml | 12 ++++++++ sim/simx/lsu_unit.cpp | 20 +++++++++---- sim/simx/lsu_unit.h | 2 +- sim/simx/mem/local_mem.cpp | 14 ++++----- sim/simx/sfu_unit.cpp | 60 +++++++++++++++++++++++++++++++++----- sim/simx/sfu_unit.h | 19 ++++++++++++ 6 files changed, 106 insertions(+), 21 deletions(-) diff --git a/ci/testcases/core.yaml b/ci/testcases/core.yaml index 676531abed..e6210b28a3 100644 --- a/ci/testcases/core.yaml +++ b/ci/testcases/core.yaml @@ -188,6 +188,18 @@ tests: args: -n128 configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 shape: {cores: 2, l2cache: true} +# The same shape at n64: L2 hits on a miss-heavy working set, two operand +# collectors per issue slice, and the load-response path's exact entry counts. +- id: parity-sgemm-mc64 + check: model_parity + # Tighter than the default: the shape exercises those entries and the + # collectors' output register exactly. + tolerance: 0.03 + via: blackbox + app: sgemm + args: -n64 + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 + shape: {cores: 2, l2cache: true} # SGEMM on two frontend slices feeding four issue slices: per-slice fetch on a # two-port icache, CTAs as wide as the core, a 16-port dcache. - id: parity-sgemm-fw2 diff --git a/sim/simx/lsu_unit.cpp b/sim/simx/lsu_unit.cpp index 21240dd7b4..93252a9eb8 100644 --- a/sim/simx/lsu_unit.cpp +++ b/sim/simx/lsu_unit.cpp @@ -108,7 +108,12 @@ LsuUnit::LsuUnit(const SimContext& ctx, const char* name, Core* core) , TcuRspOut(TCU_AGU_PORTS, this) #endif , pending_loads_(0) -{} +{ + // the lane-dispatch skid buffer's ready is registered + for (auto& input : Inputs) { + input.set_registered_ready(true); + } +} LsuUnit::~LsuUnit() {} @@ -354,10 +359,10 @@ void LsuUnit::process_response_step(uint32_t b) { // At most one transaction/cycle: the trace is not dispatchable until the next // cycle, so req_queue adds a real cycle of latency. A while-loop here would // fabricate write bandwidth and collapse that latency to zero. -void LsuUnit::ingest_inputs(uint32_t b) { +void LsuUnit::ingest_inputs(uint32_t b, bool queue_room, uint32_t reads_held) { auto& state = states_.at(b); auto& input = Inputs.at(b); - if (input.empty() || state.req_queue.full()) + if (input.empty() || !queue_room) return; auto trace = input.peek(); // Hold a fence at the input head until req_queue is empty so the @@ -368,7 +373,7 @@ void LsuUnit::ingest_inputs(uint32_t b) { return; const bool is_read = !lsu_type_tag || *lsu_type_tag == LsuType::LOAD; if (is_read) { - if (state.pending_reqs.size() + state.staged_reads >= VX_CFG_LSU_PENDING_SIZE) { + if (reads_held >= VX_CFG_LSU_PENDING_SIZE) { return; } ++state.staged_reads; @@ -715,8 +720,13 @@ void LsuUnit::on_tick() { // as it stood at the start of this cycle, THEN ingest one new trace for // next cycle. Dispatching before ingest makes req_queue a 1-cycle stage // rather than a same-tick passthrough. + // The scheduler's request queue and its pending table register their + // full flags: a slot freed this cycle takes a new request next cycle. + auto& state = states_.at(b); + bool queue_room = !state.req_queue.full(); + uint32_t reads_held = state.pending_reqs.size() + state.staged_reads; this->process_response_step(b); this->process_request_step(b); - this->ingest_inputs(b); + this->ingest_inputs(b, queue_room, reads_held); } } diff --git a/sim/simx/lsu_unit.h b/sim/simx/lsu_unit.h index cb81581e53..045ddc9880 100644 --- a/sim/simx/lsu_unit.h +++ b/sim/simx/lsu_unit.h @@ -111,7 +111,7 @@ class LsuUnit : public FuncUnit { void send_fence_token(uint32_t b); // Drain Inputs[b] into req_queue. - void ingest_inputs(uint32_t b); + void ingest_inputs(uint32_t b, bool queue_room, uint32_t reads_held); struct mem_addr_size_t { uint64_t addr; diff --git a/sim/simx/mem/local_mem.cpp b/sim/simx/mem/local_mem.cpp index 211f46c5d7..5d25807a80 100644 --- a/sim/simx/mem/local_mem.cpp +++ b/sim/simx/mem/local_mem.cpp @@ -88,12 +88,12 @@ class LocalMem::Impl { snprintf(sname, 100, "%s-xbar", simobject->name().c_str()); uint32_t lg2_line_size = log2ceil(config_.line_size); uint32_t num_banks = 1 << config.B; - // The crossbar's registered request and response outputs are taken by - // the switch and bank stages around it, so its delays here are only the - // extra tree level: an arbiter over more than 1.5x the fan-out limit is - // split into a buffered tree, one cycle deeper. The request side - // arbitrates the lanes at each bank, the response side the banks at - // each lane. + // The request crossbar registers its output toward the banks, one cycle + // ahead of the SRAM read. The response crossbar's register is the + // adapter's response stage. An arbiter over more than 1.5x the fan-out + // limit is split into a buffered tree, one cycle deeper. The request + // side arbitrates the lanes at each bank, the response side the banks + // at each lane. auto tree_delay = [](uint32_t requesters) -> uint32_t { constexpr uint32_t kMaxFanout = 8; return (requesters > kMaxFanout + kMaxFanout / 2) ? 1 : 0; @@ -102,7 +102,7 @@ class LocalMem::Impl { [lg2_line_size, num_banks](const MemCrossBar::ReqType& req) { // Custom logic to calculate the output index using bank interleaving return (uint32_t)((req.addr >> lg2_line_size) & (num_banks-1)); - }, tree_delay(config.num_reqs), tree_delay(num_banks)); + }, 1 + tree_delay(config.num_reqs), tree_delay(num_banks)); for (uint32_t i = 0; i < config.num_reqs; ++i) { simobject->Inputs.at(i).bind(&mem_xbar_->ReqIn.at(i)); mem_xbar_->RspOut.at(i).bind(&simobject->Outputs.at(i)); diff --git a/sim/simx/sfu_unit.cpp b/sim/simx/sfu_unit.cpp index 4c57d1ab8b..14d76713c8 100644 --- a/sim/simx/sfu_unit.cpp +++ b/sim/simx/sfu_unit.cpp @@ -96,6 +96,40 @@ bool SfuUnit::rtu_trace2_reserve_slot(uint32_t wid) { #endif +// Results sent toward commit with the unit that produced them, in the order +// they reach the output channel (arrival cycle, then send order), which is the +// order commit pops them in. The oldest PE_SWITCH_SIZE stand in the shared +// response switch; the rest still wait in their unit's own result buffer. +void SfuUnit::sync_out(uint32_t b) { + auto& q = out_pe_.at(b); + uint32_t n = Outputs.at(b).size(); + while (q.size() > n) { + q.pop_front(); + } +} + +bool SfuUnit::pe_buf_full(uint32_t b, PeClass pe) { + this->sync_out(b); + auto& q = out_pe_.at(b); + uint32_t held = 0; + for (size_t i = PE_SWITCH_SIZE; i < q.size(); ++i) { + held += (q[i].pe == pe); + } + return held >= PE_BUF_SIZE; +} + +void SfuUnit::send_out(uint32_t b, instr_trace_t* trace, uint32_t delay, PeClass pe) { + this->sync_out(b); + Outputs.at(b).send(trace, delay); + auto& q = out_pe_.at(b); + uint64_t arrive = SimPlatform::instance().cycles() + delay; + auto it = q.end(); + while (it != q.begin() && std::prev(it)->arrive > arrive) { + --it; + } + q.insert(it, OutPe{arrive, pe}); +} + void SfuUnit::on_tick() { #ifdef VX_CFG_EXT_RTU_ENABLE // Drain RTU rsps. Two flavors, both completing the warp's parked WAIT @@ -125,7 +159,7 @@ void SfuUnit::on_tick() { auto wb = is_candidate ? rtu_unit_->on_candidate_rsp(rsp) : rtu_unit_->on_terminal_rsp(rsp); if (wb.trace) { - Outputs.at(wb.block_id).send(wb.trace, this->latency_of(wb.trace)); + this->send_out(wb.block_id, wb.trace, this->latency_of(wb.trace), PE_OTHER); DT(3, "rtu-rsp deliver: core=" << core_->id() << ", wid=" << wb.trace->wid << ", cand=" << is_candidate); } else { @@ -153,7 +187,7 @@ void SfuUnit::on_tick() { { // Unit latency is already modeled by the TEX pipeline; charge only // the gather/writeback hop. - output.send(trace, 2); + this->send_out(rsp.block_id, trace, 2, PE_OTHER); DT(3, "tex-rsp deliver: core=" << core_->id() << ", wid=" << trace->wid); } tex_rsp_in.pop(); @@ -337,7 +371,7 @@ void SfuUnit::on_tick() { continue; // OM back-pressure — nothing was sent; retry } } - output.send(trace, 1); + this->send_out(b, trace, 1, PE_OTHER); input.pop(); continue; } @@ -369,7 +403,7 @@ void SfuUnit::on_tick() { if (output.full()) continue; if (!rtu_unit_->process_cb_ret(trace, b)) continue; // bus backpressure — nothing was sent - output.send(trace, this->latency_of(trace)); + this->send_out(b, trace, this->latency_of(trace), PE_OTHER); input.pop(); continue; } @@ -384,7 +418,7 @@ void SfuUnit::on_tick() { if (output.full()) continue; if (!rtu_unit_->process_trace_uop(trace, b, args.uop)) continue; - output.send(trace, this->latency_of(trace)); + this->send_out(b, trace, this->latency_of(trace), PE_OTHER); input.pop(); continue; } @@ -400,7 +434,7 @@ void SfuUnit::on_tick() { } instr_trace_t* wb = rtu_unit_->process_wait(trace, b); if (wb) { - output.send(wb, this->latency_of(wb)); + this->send_out(b, wb, this->latency_of(wb), PE_OTHER); } input.pop(); continue; @@ -414,7 +448,7 @@ void SfuUnit::on_tick() { auto args = std::get(trace->instr_ptr->get_args()); if (output.full()) continue; rtu_window_.process_getw_uop(trace, args.uop, *rtu_p == GfxwType::GETWF); - output.send(trace, this->latency_of(trace)); + this->send_out(b, trace, this->latency_of(trace), PE_OTHER); input.pop(); continue; } @@ -465,6 +499,13 @@ void SfuUnit::on_tick() { if (output.full()) continue; // stall — no side effects this tick + // The warp-control and CSR units each answer through their own + // two-entry result buffer ahead of the shared response switch. + const bool is_wctl = std::get_if(&trace->op_type) != nullptr; + const PeClass pe = is_csr ? PE_CSR : (is_wctl ? PE_WCTL : PE_OTHER); + if (pe != PE_OTHER && this->pe_buf_full(b, pe)) + continue; + // WSYNC has a structural gate: cannot complete until prior insts retire. // BAR (vx_barrier and vx_barrier_arrive) drains LSU before continuing — // implements CUDA __syncthreads / OpenCL barrier(CLK_LOCAL_MEM_FENCE) semantic. @@ -502,7 +543,7 @@ void SfuUnit::on_tick() { } uint32_t delay = this->latency_of(trace); - output.send(trace, delay); + this->send_out(b, trace, delay, pe); // Warp-control refines the default (fetch_stall) release decision: a // sync-barrier, a not-yet-last barrier arrival, a deferred wspawn, or a // warp that disabled itself (tmask=0) keeps the warp parked — it is @@ -536,6 +577,9 @@ void SfuUnit::on_reset() { std::fill(std::begin(res_rr_), std::end(res_rr_), 0); csr_read_done_.fill(nullptr); bar_arrive_.fill(BarArrive{}); + for (auto& q : out_pe_) { + q.clear(); + } } SfuUnit::SharedRes SfuUnit::shared_res(uint32_t block) { diff --git a/sim/simx/sfu_unit.h b/sim/simx/sfu_unit.h index 5e208b614e..7ea94a6b8d 100644 --- a/sim/simx/sfu_unit.h +++ b/sim/simx/sfu_unit.h @@ -13,6 +13,8 @@ #pragma once +#include + #include #include #include "func_unit.h" @@ -133,6 +135,23 @@ class SfuUnit : public FuncUnit { std::unique_ptr wctl_unit_; std::unique_ptr csr_unit_; + // The unit a result in the output channel came from: the warp-control and + // CSR units each hold PE_BUF_SIZE results ahead of the shared response + // switch, which holds PE_SWITCH_SIZE. Results reach the channel after + // different delays, so each is kept at its arrival cycle, in the channel's + // own order. + enum PeClass : uint8_t { PE_OTHER = 0, PE_WCTL, PE_CSR }; + static constexpr uint32_t PE_BUF_SIZE = 2; + static constexpr uint32_t PE_SWITCH_SIZE = 2; + struct OutPe { + uint64_t arrive; + PeClass pe; + }; + std::array, VX_CFG_NUM_SFU_BLOCKS> out_pe_; + void sync_out(uint32_t b); + bool pe_buf_full(uint32_t b, PeClass pe); + void send_out(uint32_t b, instr_trace_t* trace, uint32_t delay, PeClass pe); + // Per block, the CSR op that has spent its one-cycle CTA-context read. std::array csr_read_done_{}; // Per block, the barrier arrive waiting for the shared barrier unit's From d0cb49c70df9cd4ff339c3230c24547c4e314749 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 01:28:00 -0700 Subject: [PATCH 112/124] cache: fix a lazy-fetch dcache deadlock between AMO write-back and fills On a lazy-fetch dcache with a deferred commit stage (latency 3), a queued AMO result drains through the core-request slot. Like a core request, it waited for the lazy queue's pipeline reservation. Meanwhile a full AMO write-back queue held off the fills that drain the lazy queue, and the bank deadlocked. With the pipe empty, only the write-back itself can push, so it now needs only a free slot (lazyq_wb_block). The lazy queue is now twice the pipe depth, so a few queued entries leave admission and the refetch loopback open. SimX sizes its loopback the same way. CI: cache:lazy_fetch-5 (latency-3 lazy-fetch dcache with atomics, simx and rtlsim). Co-Authored-By: Claude Opus 5.5 --- ci/testcases/cache.yaml | 10 ++++++++++ hw/rtl/cache/VX_cache_bank.sv | 21 +++++++++++++++------ sim/simx/mem/cache.cpp | 5 +++-- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/ci/testcases/cache.yaml b/ci/testcases/cache.yaml index 12bd2c56b4..c4326f73c0 100644 --- a/ci/testcases/cache.yaml +++ b/ci/testcases/cache.yaml @@ -362,6 +362,16 @@ tests: shape: cores: 4 l2cache: true +# The same on a dcache with a deferred commit stage (latency 3): a queued AMO +# result drains through the core-request slot while the lazy queue holds its +# pipeline reservation, and fully written sectors still loop back data-less. +- id: lazy_fetch-5 + via: blackbox + drivers: + - simx + - rtlsim + app: lazy_fetch + configs: -DVX_CFG_EXT_A_ENABLE -DVX_CFG_DCACHE_LAZY_FETCH=1 -DVX_CFG_DCACHE_LATENCY=3 # Both sectors of one sectored L2 line miss together from different warps and # cores: each must get its own fill, never chain behind the other sector's. # Run with the address held per request and with the MSHR line table. diff --git a/hw/rtl/cache/VX_cache_bank.sv b/hw/rtl/cache/VX_cache_bank.sv index 2b90f0e660..c9150cc112 100644 --- a/hw/rtl/cache/VX_cache_bank.sv +++ b/hw/rtl/cache/VX_cache_bank.sv @@ -197,7 +197,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire [`UP(`CS_SECTOR_SEL_BITS)-1:0] mem_rsp_sector; // sector this fill installs wire [MSHR_ADDR_WIDTH-1:0] mshr_alloc_id, mshr_previd; wire mshr_fill_refetch, mshr_fill_retry, mshr_dequeue_reads; - wire lazyq_empty, lazyq_alm_full; + wire lazyq_empty, lazyq_alm_full, lazyq_full; wire mshr_pending_raw; wire mshr_pending_wr_raw; wire mshr_pending_ovl_raw; @@ -433,6 +433,13 @@ module VX_cache_bank import VX_gpu_pkg::*; #( assign flush_ready = flush_grant && ~(WRITEBACK && mreq_queue_alm_full) && ~pipe_stall; // a core request may become a write-miss root, which pushes to lazyq. wire creq_alm_full = mreq_queue_alm_full || lazyq_alm_full || replay_block; + // A queued AMO result drains through the core-request slot, while a full + // AMO writeback queue holds off the fills that drain the lazy queue. So + // the writeback is not held to the lazy queue's pipeline reservation when + // the pipe is empty: then only the writeback itself can push, and a free + // slot is room enough. + wire lazyq_wb_block = lazyq_alm_full && (pipe_inflight || lazyq_full); + wire amo_wb_alm_full = mreq_queue_alm_full || lazyq_wb_block || replay_block; assign core_req_ready = creq_grant && ~creq_alm_full && ~mshr_alm_full && ~pipe_stall && ~amo_commit_busy && ~req_input_defer; @@ -445,9 +452,9 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire flush_fire = flush_valid && flush_ready; // amo_wb_path already excludes the cycle a fresh AMO commits at S1, so the // writeback never races the chain update. - wire amo_wb_fire = amo_wb_path && creq_grant && ~creq_alm_full && ~mshr_alm_full && ~pipe_stall; - wire core_req_fire = (amo_creq_path || amo_wb_path) && creq_grant - && ~creq_alm_full && ~mshr_alm_full && ~pipe_stall; + wire amo_wb_fire = amo_wb_path && creq_grant && ~amo_wb_alm_full && ~mshr_alm_full && ~pipe_stall; + wire core_req_fire = ((amo_creq_path && ~creq_alm_full) || (amo_wb_path && ~amo_wb_alm_full)) + && creq_grant && ~mshr_alm_full && ~pipe_stall; wire [MSHR_ADDR_WIDTH-1:0] mem_rsp_id = mem_rsp_tag[MSHR_ADDR_WIDTH-1:0]; wire [MSHR_ADDR_WIDTH-1:0] fill_id = lazy_sel ? lazyq_id : mem_rsp_id; @@ -1628,8 +1635,9 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // Sized like the memory request queue's reservation: a core request is // admitted only while every request already in the pipe still fits. if (LAZY_FETCH != 0) begin : g_lazyq - wire lazyq_full; - localparam LAZYQ_SIZE = 1 << `CLOG2(PIPELINE_STAGES + 1); + // Room for twice the pipe, so a few queued entries leave the + // reservation (and the refetch loopback, which needs it) open. + localparam LAZYQ_SIZE = 1 << `CLOG2(2 * PIPELINE_STAGES); VX_fifo_queue #( .DATAW (MSHR_ADDR_WIDTH), .DEPTH (LAZYQ_SIZE), @@ -1702,6 +1710,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( assign lazyq_id = '0; assign lazyq_empty = 1'b1; assign lazyq_alm_full = 1'b0; + assign lazyq_full = 1'b0; assign chain_partial = 1'b0; assign chain_retry = 1'b0; assign refetch_inflight = 1'b0; diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index 6f09d976f1..b2a7d24945 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -1495,9 +1495,10 @@ class CacheBank : public SimObject { chain_sector_ = sector_id; } - // Lazy-fetch loopback depth: room for every request in the pipe plus one. + // Lazy-fetch loopback depth: room for twice the pipe, so a few queued + // entries leave admission and the refetch loopback open. static uint32_t lazyq_size(const Cache::Config &config) { - return 1u << log2ceil(config.latency + 1); + return 1u << log2ceil(2 * config.latency); } // The loopback still fits every request in the pipe: a new request may be From 5391be583ddf75ff05f2dff9564ed90da942b856 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 01:28:02 -0700 Subject: [PATCH 113/124] syn: read vortex_afu.v as SystemVerilog in the Vivado flows The platform tops pass unpacked per-port arrays to VX_afu_wrap, which Vivado accepts only from SystemVerilog sources. The elaboration check, the per-block DUT project, the XRT kernel packager and the AVED IP packager now set vortex_afu.v's file type accordingly. The DUT `top` build also resolves AFU_MEM_PORTS with afu_mem_ports.sh, as the XRT flow does. The unused `unittest` DUT, which had no top module, leaves the Xilinx and Altera synthesis catalogs. Co-Authored-By: Claude Opus 5.5 --- hw/scripts/xilinx_elab_check.tcl | 6 ++++++ hw/syn/altera/dut/catalog.mk | 6 +----- hw/syn/xilinx/README | 2 +- hw/syn/xilinx/aved/package_ip.tcl | 7 +++++++ hw/syn/xilinx/dut/build.mk | 11 +++++++++++ hw/syn/xilinx/dut/catalog.mk | 6 ++---- hw/syn/xilinx/dut/project.tcl | 6 ++++++ hw/syn/xilinx/xrt/package_kernel.tcl | 6 ++++++ 8 files changed, 40 insertions(+), 10 deletions(-) diff --git a/hw/scripts/xilinx_elab_check.tcl b/hw/scripts/xilinx_elab_check.tcl index 8933471fbf..39ce791404 100644 --- a/hw/scripts/xilinx_elab_check.tcl +++ b/hw/scripts/xilinx_elab_check.tcl @@ -46,6 +46,12 @@ if {$mic_mode eq "off" || $mic_mode eq "0"} { create_project -in_memory add_files -norecurse ${mic_sources} + # The platform tops (vortex_afu.v) pass unpacked per-port arrays to + # VX_afu_wrap, which needs SystemVerilog. + set mic_afu_tops [get_files -quiet */vortex_afu.v] + if {[llength $mic_afu_tops] > 0} { + set_property file_type SystemVerilog $mic_afu_tops + } # Black-box every generated vendor IP via its BLOCK_STUB so a -rtl elaboration # resolves the IP instances (FPU FMA/DIV/SQRT/MUL/ADD, etc.) without pulling diff --git a/hw/syn/altera/dut/catalog.mk b/hw/syn/altera/dut/catalog.mk index 2bca1ee61b..74563d021f 100644 --- a/hw/syn/altera/dut/catalog.mk +++ b/hw/syn/altera/dut/catalog.mk @@ -9,7 +9,7 @@ # Add a DUT: append its name to DUTS and define its variables. No new directory, # no dispatcher edit. -DUTS := cache core fpu issue lmem mem_unit scope top unittest vortex +DUTS := cache core fpu issue lmem mem_unit scope top vortex # --------------------------------------------------------------------------- # shared include fragments @@ -75,10 +75,6 @@ top_INC = $(BASE_INC) -I$(RTL_DIR)/core -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm -I$(RT -I$(AFU_DIR) -I$(AFU_DIR)/ccip -I$(IP_CACHE_DIR) $(FPU_INC) $(TCU_INC) $(DXA_INC) top_PKG = $(TCU_PKG) $(DXA_PKG) -unittest_PRJ := Unittest -unittest_INC = $(BASE_INC) -I$(RTL_DIR)/core -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm -I$(RTL_DIR)/cache \ - -I$(IP_CACHE_DIR) $(FPU_INC) - vortex_PRJ := Vortex vortex_INC = $(BASE_INC) -I$(RTL_DIR)/core -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm -I$(RTL_DIR)/cache \ -I$(IP_CACHE_DIR) $(FPU_INC) $(TCU_INC) $(DXA_INC) diff --git a/hw/syn/xilinx/README b/hw/syn/xilinx/README index 0ae222a173..d698de9fde 100644 --- a/hw/syn/xilinx/README +++ b/hw/syn/xilinx/README @@ -9,7 +9,7 @@ vivado -mode batch -source xilinx_ip_gen.tcl -tclargs ip/xilinx_u50_gen3x16_xdma # build per-block (dut): Vivado project-mode synthesis of an individual block. # Self-backgrounds; writes //build.log. Default DEVICE: xcu55c. -# Targets: core vortex top issue cache fpu tcu dxa tex raster om lmem mem_unit scope unittest +# Targets: core vortex top issue cache fpu tcu dxa tex raster om lmem mem_unit scope cd hw/syn/xilinx/dut PREFIX=smoke1 make core PREFIX=smoke1 make vortex diff --git a/hw/syn/xilinx/aved/package_ip.tcl b/hw/syn/xilinx/aved/package_ip.tcl index 0e6bdb33cc..4df28092a9 100644 --- a/hw/syn/xilinx/aved/package_ip.tcl +++ b/hw/syn/xilinx/aved/package_ip.tcl @@ -63,6 +63,13 @@ foreach line [split $data "\n"] { } } +# The platform tops (vortex_afu.v) pass unpacked per-port arrays to +# VX_afu_wrap, which needs SystemVerilog. +set afu_tops [get_files -quiet */vortex_afu.v] +if {[llength $afu_tops] > 0} { + set_property file_type SystemVerilog $afu_tops +} + # Pull in the floating-point cores generated above so they are packaged # alongside the RTL. The set matches the XRT flow's package_kernel.tcl. set ip_files [list] diff --git a/hw/syn/xilinx/dut/build.mk b/hw/syn/xilinx/dut/build.mk index 878af652d0..dbff6f5ae3 100644 --- a/hw/syn/xilinx/dut/build.mk +++ b/hw/syn/xilinx/dut/build.mk @@ -32,3 +32,14 @@ RTL_PKGS += $($(DUT)_PKG) ifeq ($($(DUT)_EXT),1) include $(VORTEX_HOME)/hw/syn/extensions.mk endif + +# An AFU build reads the platform tops too, which declare one AXI master per +# GPU memory port literally: resolve the count from the elaborated +# configuration, as the xrt flow does. +ifeq ($($(DUT)_AFU),1) +AFU_MEM_PORTS := $(shell $(SCRIPT_DIR)/afu_mem_ports.sh -c .afu_mem_ports $(VERILATOR_PATH)/bin/verilator $(filter-out -J% -DSYNTHESIS -DVIVADO,$(CFLAGS)) $(RTL_PKGS)) +ifeq ($(AFU_MEM_PORTS),) +$(error cannot resolve the AFU memory port count for this configuration) +endif +CFLAGS += -DAFU_MEM_PORTS=$(AFU_MEM_PORTS) +endif diff --git a/hw/syn/xilinx/dut/catalog.mk b/hw/syn/xilinx/dut/catalog.mk index 1bb5edaa98..abdef0dae8 100644 --- a/hw/syn/xilinx/dut/catalog.mk +++ b/hw/syn/xilinx/dut/catalog.mk @@ -11,7 +11,7 @@ # directory, no dispatcher edit. DUTS := cache core cp dxa fpu gfx issue lmem mem_unit om raster rtu \ - scope tcu tensor tex top unittest vm vortex + scope tcu tensor tex top vm vortex # --------------------------------------------------------------------------- # shared include fragments @@ -131,12 +131,10 @@ tex_PKG = $(RTL_DIR)/tex/VX_tex_pkg.sv top_PRJ := VX_afu_wrap top_IP := 1 top_EXT := 1 +top_AFU := 1 top_INC = $(BASE_INC) -I$(RTL_DIR)/core -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm -I$(RTL_DIR)/cache \ -I$(RTL_DIR)/cp -I$(AFU_DIR) -I$(AFU_COMMON_DIR) $(FPU_INC) -unittest_PRJ := Unittest -unittest_INC = $(BASE_INC) -I$(RTL_DIR)/core -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm -I$(RTL_DIR)/cache $(FPU_INC) - vm_PRJ := VX_vm_top vm_CFG := -DVX_CFG_VM_ENABLE -DVX_CFG_EXT_A_ENABLE -DVX_CFG_NUM_CORES=2 \ -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 \ diff --git a/hw/syn/xilinx/dut/project.tcl b/hw/syn/xilinx/dut/project.tcl index 56339544ef..9f512e3f94 100644 --- a/hw/syn/xilinx/dut/project.tcl +++ b/hw/syn/xilinx/dut/project.tcl @@ -173,6 +173,12 @@ proc run_setup {} { # Add the design sources add_files -norecurse -verbose $vsources_list + # The platform tops (vortex_afu.v) pass unpacked per-port arrays to + # VX_afu_wrap, which needs SystemVerilog. + set afu_tops [get_files -quiet */vortex_afu.v] + if {[llength $afu_tops] > 0} { + set_property file_type SystemVerilog $afu_tops + } # process defines set_property verilog_define ${vdefines_list} [current_fileset] diff --git a/hw/syn/xilinx/xrt/package_kernel.tcl b/hw/syn/xilinx/xrt/package_kernel.tcl index d9b9777c72..c9b0ab20cb 100644 --- a/hw/syn/xilinx/xrt/package_kernel.tcl +++ b/hw/syn/xilinx/xrt/package_kernel.tcl @@ -62,6 +62,12 @@ foreach def $vdefines_list { create_project -force kernel_pack $path_to_tmp_project add_files -norecurse ${vsources_list} +# The platform tops (vortex_afu.v) pass unpacked per-port arrays to +# VX_afu_wrap, which needs SystemVerilog. +set afu_tops [get_files -quiet */vortex_afu.v] +if {[llength $afu_tops] > 0} { + set_property file_type SystemVerilog $afu_tops +} set obj [get_filesets sources_1] set ip_files [list \ From 7820027c0f35bdf2326bd60885d668b806b1159c Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 01:28:03 -0700 Subject: [PATCH 114/124] simx: keep the last-level cache across a launch reset A launch after CP DMA writes into the last-level cache drained the whole model before its reset, including the transfer's line fills. The hardware does not reset between a transfer and the launch that follows it. The last-level cache, the memory below it and the CP port are now built under a LaunchKeepScope and survive reset_launch(), so the kernel meets fills still in flight. The crossbar's 64-input bound, which its per-output round-robin state never needed, is dropped. Co-Authored-By: Claude Opus 5.5 --- sim/common/simobject.h | 42 +++++++++++++- sim/common/simobject_functional.h | 42 +++++++++++++- sim/simx/cluster.cpp | 59 +++++++++++--------- sim/simx/processor.cpp | 93 ++++++++++++++++++------------- sim/simx/processor_impl.h | 5 +- sim/simx/types.h | 2 +- 6 files changed, 172 insertions(+), 71 deletions(-) diff --git a/sim/common/simobject.h b/sim/common/simobject.h index 225dab157a..5cc55e36ec 100644 --- a/sim/common/simobject.h +++ b/sim/common/simobject.h @@ -88,6 +88,7 @@ class SimObjectBase { std::string name_; bool tick_active_ = true; uint32_t domain_ = 0; + bool launch_kept_ = false; // built under a LaunchKeepScope virtual void do_reset() = 0; virtual void do_tick() = 0; friend class SimPlatform; @@ -265,6 +266,7 @@ class SimPlatform { uint32_t alloc_domain() { return num_domains_++; } uint32_t num_domains() const { return num_domains_; } uint32_t build_domain() const { return build_domain_; } + bool build_launch_kept() const { return build_launch_kept_; } // Cap the parallel thread count (must be called before the first tick). // A limit of 1 forces serial execution — required while the configuration @@ -280,6 +282,25 @@ class SimPlatform { req_workers_ = std::max(1u, num_workers); } + // Objects built under a LaunchKeepScope (the last-level cache and the + // memory below it, when the command processor writes into that cache) + // carry their state and in-flight work across reset_launch(). + class LaunchKeepScope { + public: + explicit LaunchKeepScope(bool keep) { + auto& p = SimPlatform::instance(); + saved_ = p.build_launch_kept_; + p.build_launch_kept_ = saved_ || keep; + } + ~LaunchKeepScope() { + SimPlatform::instance().build_launch_kept_ = saved_; + } + LaunchKeepScope(const LaunchKeepScope&) = delete; + LaunchKeepScope& operator=(const LaunchKeepScope&) = delete; + private: + bool saved_; + }; + class DomainScope { public: explicit DomainScope(uint32_t domain) { @@ -331,6 +352,7 @@ class SimPlatform { void schedule(Func&& func, const Pkt& pkt, uint64_t delay, uint32_t domain); void reset(); + void reset_launch(); void tick(); private: @@ -395,6 +417,7 @@ class SimPlatform { uint32_t num_domains_ = 1; uint32_t build_domain_ = 0; + bool build_launch_kept_ = false; static inline thread_local uint32_t tl_exec_domain_ = HOST_DOMAIN; static inline thread_local SimObjectBase* tl_exec_object_ = nullptr; static inline thread_local uint64_t tl_cross_seq_ = 0; @@ -1183,6 +1206,22 @@ inline void SimPlatform::validate_topology() { } } +// A kernel launch resets every object but those built under a +// LaunchKeepScope: the hardware does not reset between a transfer and the +// launch that follows it, so the transfer's line fills may still be in +// flight in the kept memory side when the kernel starts. Their events stay +// scheduled and time keeps counting; the rest of the model is quiescent. +inline void SimPlatform::reset_launch() { + for (auto* object : active_reset_) { + if (!object->launch_kept_) { + object->do_reset(); + } + } + for (auto& object : objects_) { + object->tick_active_ = true; + } +} + inline void SimPlatform::reset() { this->validate_topology(); @@ -1361,7 +1400,8 @@ void SimChannelEvent::fire() { inline SimObjectBase::SimObjectBase(const SimContext&, const std::string& name) : name_(name) - , domain_(SimPlatform::instance().build_domain()) {} + , domain_(SimPlatform::instance().build_domain()) + , launch_kept_(SimPlatform::instance().build_launch_kept()) {} inline SimChannelBase::SimChannelBase(SimObjectBase* module) : module_(module) diff --git a/sim/common/simobject_functional.h b/sim/common/simobject_functional.h index 924f09ba56..11668176e9 100644 --- a/sim/common/simobject_functional.h +++ b/sim/common/simobject_functional.h @@ -103,6 +103,7 @@ class SimObjectBase { std::string name_; bool tick_active_ = true; uint32_t domain_ = 0; + bool launch_kept_ = false; // built under a LaunchKeepScope virtual void do_reset() = 0; virtual void do_tick() = 0; friend class SimPlatform; @@ -276,6 +277,7 @@ class SimPlatform { uint32_t alloc_domain() { return num_domains_++; } uint32_t num_domains() const { return num_domains_; } uint32_t build_domain() const { return build_domain_; } + bool build_launch_kept() const { return build_launch_kept_; } // Cap the parallel thread count (must be called before the first tick). // A limit of 1 forces serial execution — required while the configuration @@ -291,6 +293,25 @@ class SimPlatform { req_workers_ = std::max(1u, num_workers); } + // Objects built under a LaunchKeepScope (the last-level cache and the + // memory below it, when the command processor writes into that cache) + // carry their state and in-flight work across reset_launch(). + class LaunchKeepScope { + public: + explicit LaunchKeepScope(bool keep) { + auto& p = SimPlatform::instance(); + saved_ = p.build_launch_kept_; + p.build_launch_kept_ = saved_ || keep; + } + ~LaunchKeepScope() { + SimPlatform::instance().build_launch_kept_ = saved_; + } + LaunchKeepScope(const LaunchKeepScope&) = delete; + LaunchKeepScope& operator=(const LaunchKeepScope&) = delete; + private: + bool saved_; + }; + class DomainScope { public: explicit DomainScope(uint32_t domain) { @@ -342,6 +363,7 @@ class SimPlatform { void schedule(Func&& func, const Pkt& pkt, uint64_t delay, uint32_t domain); void reset(); + void reset_launch(); void tick(); private: @@ -408,6 +430,7 @@ class SimPlatform { uint32_t num_domains_ = 1; uint32_t build_domain_ = 0; + bool build_launch_kept_ = false; static inline thread_local uint32_t tl_exec_domain_ = HOST_DOMAIN; static inline thread_local SimObjectBase* tl_exec_object_ = nullptr; static inline thread_local uint64_t tl_cross_seq_ = 0; @@ -1171,6 +1194,22 @@ inline void SimPlatform::validate_topology() { } } +// A kernel launch resets every object but those built under a +// LaunchKeepScope: the hardware does not reset between a transfer and the +// launch that follows it, so the transfer's line fills may still be in +// flight in the kept memory side when the kernel starts. Their events stay +// scheduled and time keeps counting; the rest of the model is quiescent. +inline void SimPlatform::reset_launch() { + for (auto* object : active_reset_) { + if (!object->launch_kept_) { + object->do_reset(); + } + } + for (auto& object : objects_) { + object->tick_active_ = true; + } +} + inline void SimPlatform::reset() { this->validate_topology(); @@ -1348,7 +1387,8 @@ void SimChannelEvent::fire() { inline SimObjectBase::SimObjectBase(const SimContext&, const std::string& name) : name_(name) - , domain_(SimPlatform::instance().build_domain()) {} + , domain_(SimPlatform::instance().build_domain()) + , launch_kept_(SimPlatform::instance().build_launch_kept()) {} inline SimChannelBase::SimChannelBase(SimObjectBase* module) : module_(module) diff --git a/sim/simx/cluster.cpp b/sim/simx/cluster.cpp index 0f409b8f0e..8ecebf429d 100644 --- a/sim/simx/cluster.cpp +++ b/sim/simx/cluster.cpp @@ -73,33 +73,38 @@ class Cluster::Impl { snprintf(sname, 100, "%s-l2cache", name.c_str()); // L2 is the LLC iff L2 is enabled and L3 is not. - l2cache_ = Cache::Create(sname, Cache::Config{ - !VX_CFG_L2_ENABLED, - log2ceil(VX_CFG_L2_SIZE),// C - log2ceil(VX_CFG_L2_LINE_SIZE),// L - log2ceil(VX_CFG_L2_SECTOR_SIZE),// S - log2ceil(VX_CFG_L1_LINE_SIZE), // W - log2ceil(VX_CFG_L2_NUM_WAYS), // A - log2ceil(VX_CFG_L2_NUM_BANKS), // B - VX_CFG_XLEN, // address bits - L2_CACHE_REQS, // request size - VX_CFG_L2_MEM_PORTS, // memory ports - VX_CFG_L2_WRITEBACK, // write-back - false, // write response - VX_CFG_L2_MSHR_SIZE, // mshr size - VX_CFG_L2_LATENCY, // pipeline latency - VX_CFG_L2_REPL_POLICY, // replacement policy - (VX_CFG_L2_ENABLED != 0) && (VX_CFG_L3_ENABLED == 0), // is_llc - true, // non-cacheable bypass - (VX_CFG_L2_BANK_HASH != 0) && (VX_CFG_L2_NUM_BANKS > 1), // bank_hash - L2_SHARED_LLC, // shared_llc - VX_CFG_L2_LAZY_FETCH != 0, // lazy_fetch - true, // core-response register (behind the bypass layer) - 0, // read_merge - VX_CFG_L2_MSHR_LINES, // mshr_lines - CP_ON_L2 != 0, // write_around - VX_CFG_L2_MRSQ_SIZE != 0, // memory-response queue - }); + { + // the CP writes into this cache when it is the last level: its state + // and a transfer's fills carry across the launch reset + SimPlatform::LaunchKeepScope keep(CP_ON_L2 != 0); + l2cache_ = Cache::Create(sname, Cache::Config{ + !VX_CFG_L2_ENABLED, + log2ceil(VX_CFG_L2_SIZE),// C + log2ceil(VX_CFG_L2_LINE_SIZE),// L + log2ceil(VX_CFG_L2_SECTOR_SIZE),// S + log2ceil(VX_CFG_L1_LINE_SIZE), // W + log2ceil(VX_CFG_L2_NUM_WAYS), // A + log2ceil(VX_CFG_L2_NUM_BANKS), // B + VX_CFG_XLEN, // address bits + L2_CACHE_REQS, // request size + VX_CFG_L2_MEM_PORTS, // memory ports + VX_CFG_L2_WRITEBACK, // write-back + false, // write response + VX_CFG_L2_MSHR_SIZE, // mshr size + VX_CFG_L2_LATENCY, // pipeline latency + VX_CFG_L2_REPL_POLICY, // replacement policy + (VX_CFG_L2_ENABLED != 0) && (VX_CFG_L3_ENABLED == 0), // is_llc + true, // non-cacheable bypass + (VX_CFG_L2_BANK_HASH != 0) && (VX_CFG_L2_NUM_BANKS > 1), // bank_hash + L2_SHARED_LLC, // shared_llc + VX_CFG_L2_LAZY_FETCH != 0, // lazy_fetch + true, // core-response register (behind the bypass layer) + 0, // read_merge + VX_CFG_L2_MSHR_LINES, // mshr_lines + CP_ON_L2 != 0, // write_around + VX_CFG_L2_MRSQ_SIZE != 0, // memory-response queue + }); + } // connect l2cache memory interface diff --git a/sim/simx/processor.cpp b/sim/simx/processor.cpp index b015d165ca..7f74d987af 100644 --- a/sim/simx/processor.cpp +++ b/sim/simx/processor.cpp @@ -50,10 +50,14 @@ ProcessorImpl::ProcessorImpl() constexpr uint32_t total_cores = VX_CFG_NUM_CLUSTERS * NUM_SOCKETS * VX_CFG_SOCKET_SIZE; kmu_ = Kmu::Create("kmu", total_cores); - // create memory simulator + // create memory simulator; with the CP writing into the last-level cache, + // the memory side carries a transfer's fills across the launch reset auto dram_config = DramSim::platform_config(); dram_config.block_size = VX_CFG_MEM_BLOCK_SIZE; - memsim_ = Memory::Create("dram", Memory::Config{VX_CFG_L3_MEM_PORTS, dram_config}); + { + SimPlatform::LaunchKeepScope keep(CP_ON_L2 != 0 || CP_ON_L3 != 0); + memsim_ = Memory::Create("dram", Memory::Config{VX_CFG_L3_MEM_PORTS, dram_config}); + } char sname[100]; @@ -95,34 +99,37 @@ ProcessorImpl::ProcessorImpl() // create L3 cache; when L3 is enabled it is the LLC, otherwise it is a // transparent bypass arbiter and the L2 (or L1) is the LLC. - l3cache_ = Cache::Create("l3cache", Cache::Config{ - !VX_CFG_L3_ENABLED, - log2ceil(VX_CFG_L3_SIZE), // C - log2ceil(VX_CFG_L3_LINE_SIZE), // L - log2ceil(VX_CFG_L3_SECTOR_SIZE), // S - log2ceil(VX_CFG_L2_SECTOR_SIZE), // W - log2ceil(VX_CFG_L3_NUM_WAYS), // A - log2ceil(VX_CFG_L3_NUM_BANKS), // B - VX_CFG_XLEN, // address bits - L3_CACHE_REQS, // request size - VX_CFG_L3_MEM_PORTS, // memory ports - VX_CFG_L3_WRITEBACK, // write-back - false, // write response - VX_CFG_L3_MSHR_SIZE, // mshr size - VX_CFG_L3_LATENCY, // pipeline latency - VX_CFG_L3_REPL_POLICY, // replacement policy - VX_CFG_L3_ENABLED != 0, // is_llc when L3 is the LLC - true, // non-cacheable bypass - false, // bank_hash - L3_SHARED_LLC, // shared_llc - VX_CFG_L3_LAZY_FETCH != 0, // lazy_fetch - true, // core-response register (behind the bypass layer) - 0, // read_merge - VX_CFG_L3_MSHR_LINES, // mshr_lines - CP_ON_L3 != 0, // write_around - VX_CFG_L3_MRSQ_SIZE != 0, // memory-response queue - } - ); + { + SimPlatform::LaunchKeepScope keep(CP_ON_L2 != 0 || CP_ON_L3 != 0); + l3cache_ = Cache::Create("l3cache", Cache::Config{ + !VX_CFG_L3_ENABLED, + log2ceil(VX_CFG_L3_SIZE), // C + log2ceil(VX_CFG_L3_LINE_SIZE), // L + log2ceil(VX_CFG_L3_SECTOR_SIZE), // S + log2ceil(VX_CFG_L2_SECTOR_SIZE), // W + log2ceil(VX_CFG_L3_NUM_WAYS), // A + log2ceil(VX_CFG_L3_NUM_BANKS), // B + VX_CFG_XLEN, // address bits + L3_CACHE_REQS, // request size + VX_CFG_L3_MEM_PORTS, // memory ports + VX_CFG_L3_WRITEBACK, // write-back + false, // write response + VX_CFG_L3_MSHR_SIZE, // mshr size + VX_CFG_L3_LATENCY, // pipeline latency + VX_CFG_L3_REPL_POLICY, // replacement policy + VX_CFG_L3_ENABLED != 0, // is_llc when L3 is the LLC + true, // non-cacheable bypass + false, // bank_hash + L3_SHARED_LLC, // shared_llc + VX_CFG_L3_LAZY_FETCH != 0, // lazy_fetch + true, // core-response register (behind the bypass layer) + 0, // read_merge + VX_CFG_L3_MSHR_LINES, // mshr_lines + CP_ON_L3 != 0, // write_around + VX_CFG_L3_MRSQ_SIZE != 0, // memory-response queue + } + ); + } #if VX_CFG_EXT_A_ENABLED // Build-time invariant: every cache above the LLC must be write-through. @@ -175,6 +182,7 @@ ProcessorImpl::ProcessorImpl() // The CP DMA's client slot on the shared last-level cache. if constexpr (CP_ON_L2 != 0 || CP_ON_L3 != 0) { + SimPlatform::LaunchKeepScope keep(true); cp_llc_port_ = CpLlcPort::Create("cp-llc-port"); if constexpr (CP_ON_L2 != 0) { cp_llc_port_->MemReqOut.bind(&clusters_.at(0)->cp_mem_req_in()); @@ -301,8 +309,7 @@ void ProcessorImpl::flush_caches(bool l1_only) { } int ProcessorImpl::run() { - this->cp_mem_fence(); - this->reset(); + this->launch_reset(); kmu_->start(); this->forward_delegated_launch(); @@ -373,19 +380,27 @@ void ProcessorImpl::cp_mem_transfer(uint64_t addr, const uint8_t* src, uint8_t* this->llc_keep_lines(); } -void ProcessorImpl::cp_mem_fence() { +void ProcessorImpl::launch_reset() { if (!cp_llc_port_) { + this->reset(); return; } + // The launch fence follows the CP's DMA writes, and the kernel starts once + // the last-level cache has taken them. A write that missed still has its + // line fill in flight, as on the hardware, so the memory side keeps its + // state across the reset and the kernel meets those fills. if (cp_llc_port_->writes_pending()) { cp_llc_port_->fence(); } - // The launch resets the model, so a transfer's fills land first: memory - // holds a read in flight without a channel packet. - while (!cp_llc_port_->idle() || !SimPlatform::instance().idle() || perf_mem_pending_reads_ != 0) { + while (!cp_llc_port_->idle()) { SimPlatform::instance().tick(); } - this->llc_keep_lines(); + SimPlatform::instance().reset_launch(); + platform_reset_ = true; + perf_mem_reads_ = 0; + perf_mem_writes_ = 0; + perf_mem_latency_ = 0; + is_cycle_initialized_ = false; } void ProcessorImpl::llc_keep_lines() { @@ -427,10 +442,10 @@ void ProcessorImpl::reset() { bool ProcessorImpl::cycle() { // Lazy first-call init mirrors run()'s top-of-loop sequence so the // external driver doesn't need to choreograph reset + kmu start - // separately. reset() clears is_cycle_initialized_ so a back-to-back + // separately. A reset clears is_cycle_initialized_ so a back-to-back // kernel launch re-dispatches. if (!is_cycle_initialized_) { - this->reset(); + this->launch_reset(); kmu_->start(); this->forward_delegated_launch(); is_cycle_initialized_ = true; diff --git a/sim/simx/processor_impl.h b/sim/simx/processor_impl.h index 052d06f8f4..f2ae7212db 100644 --- a/sim/simx/processor_impl.h +++ b/sim/simx/processor_impl.h @@ -92,8 +92,9 @@ class ProcessorImpl { // read returned; the last-level cache keeps what it took into the next launch. void cp_mem_transfer(uint64_t addr, const uint8_t* src, uint8_t* dst, uint64_t size, bool noalloc); - // A launch that follows DMA writes waits for the port's fence. - void cp_mem_fence(); + // Reset the model for a launch. After DMA writes it waits for the port's + // fence and keeps the memory side, whose line fills may still be in flight. + void launch_reset(); void llc_keep_lines(); diff --git a/sim/simx/types.h b/sim/simx/types.h index b18936dbd3..75acbf2080 100644 --- a/sim/simx/types.h +++ b/sim/simx/types.h @@ -1637,7 +1637,7 @@ class TxCrossBar : public SimObject> { , lg2_outputs_(log2ceil(num_outputs)) , output_sel_(output_sel) , collisions_(0) { - assert(num_inputs <= 64); + // per-output round-robin pointers only: no bound on the input count assert(num_outputs <= 64); assert(ispow2(num_outputs)); assert(output_sel != nullptr); From b6717056f725de2bd76bdf9613f1d164f5d6a656 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 01:28:03 -0700 Subject: [PATCH 115/124] ci: re-record the perf_gate baselines for the MLP changes Regenerated with pytest ci -m perf_gate --update-baselines on the CI toolchain (LLVM v3.0.1) after the MLP series. Every case's config hash moves with the new defaults; retired instruction counts are unchanged. Cycles move with the miss-path, fence, lazy-fetch and tensor-core changes in the preceding commits: the 49 cases are 3.4% faster than master at the geometric mean, led by sgemv-nt16 (-44%), fp16-mc-nt16 (-8.6%) and stencil3d-nt16 (-3.1%). The same tree passed model_parity and the regression suites. Co-Authored-By: Claude Opus 5.5 --- ci/baselines/perf/core.json | 64 +++++++++++++++---------------- ci/baselines/perf/dxa.json | 42 ++++++++++---------- ci/baselines/perf/graphics.json | 36 ++++++++--------- ci/baselines/perf/raytracing.json | 12 +++--- ci/baselines/perf/tensor.json | 20 +++++----- ci/baselines/perf/tensor_mx.json | 4 +- ci/baselines/perf/tensor_sp.json | 4 +- ci/baselines/perf/tensor_wg.json | 12 +++--- 8 files changed, 97 insertions(+), 97 deletions(-) diff --git a/ci/baselines/perf/core.json b/ci/baselines/perf/core.json index c9c7dc8a7f..5c239ea9bc 100644 --- a/ci/baselines/perf/core.json +++ b/ci/baselines/perf/core.json @@ -1,177 +1,177 @@ { "core:raycast-nt16:rtlsim": { "32": { - "cycles": 11185799, + "cycles": 11182301, "instrs": 2241925 }, "app": "raycast", "args": "", - "config_hash": "ef7e97261abf3c7e", + "config_hash": "969509e046ab06ca", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:raycast-nt4:rtlsim": { "32": { - "cycles": 21106013, + "cycles": 21106989, "instrs": 5664693 }, "app": "raycast", "args": "", - "config_hash": "4a346b970c5d5f4e", + "config_hash": "9fde6e6f7f37e2d4", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:sgemm-mc-nt16:rtlsim": { "32": { - "cycles": 632919, + "cycles": 635244, "instrs": 623680 }, "app": "sgemm", "args": "-n128", - "config_hash": "12eb01a0cf320bec", + "config_hash": "d063e6340ecd4137", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=16 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "core:sgemm-mc-nt4:rtlsim": { "32": { - "cycles": 2251795, + "cycles": 2204469, "instrs": 2494528 }, "app": "sgemm", "args": "-n128", - "config_hash": "a5cc1a944ac4fc92", + "config_hash": "15237d49dceccaee", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "core:sgemm-nt16:rtlsim": { "32": { - "cycles": 1665973, + "cycles": 1661746, "instrs": 623632 }, "app": "sgemm", "args": "-n128", - "config_hash": "dcf77cabb080bf39", + "config_hash": "aa47b9962f16edb2", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:sgemm-nt4:rtlsim": { "32": { - "cycles": 5943507, + "cycles": 5942823, "instrs": 2494480 }, "app": "sgemm", "args": "-n128", - "config_hash": "9b7d85b091d9be08", + "config_hash": "5ad71cacfd524b08", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:sgemmx-nt16:rtlsim": { "32": { - "cycles": 1225395, + "cycles": 1224642, "instrs": 266896 }, "app": "sgemmx", "args": "-n128", - "config_hash": "1181ce2dd247a573", + "config_hash": "22fd9b73cb7d2ba8", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:sgemmx-nt4:rtlsim": { "32": { - "cycles": 4218769, + "cycles": 4217077, "instrs": 1067536 }, "app": "sgemmx", "args": "-n128", - "config_hash": "b8d3bbe9aaa587a9", + "config_hash": "a14ffd5b6f36fa72", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:sgemv-nt16:rtlsim": { "32": { - "cycles": 408992, + "cycles": 381483, "instrs": 66800 }, "app": "sgemv", "args": "-m512 -n512", - "config_hash": "a2e93b1416c1222b", + "config_hash": "df3b981f27e8e320", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:sgemv-nt4:rtlsim": { "32": { - "cycles": 883260, + "cycles": 866110, "instrs": 267152 }, "app": "sgemv", "args": "-m512 -n512", - "config_hash": "212b79b7b9e0df8d", + "config_hash": "3297af456dd6427a", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:softmax-nt16:rtlsim": { "32": { - "cycles": 1270578, + "cycles": 1270624, "instrs": 464089 }, "app": "softmax", "args": "-n64", - "config_hash": "4a980cd3436e0491", + "config_hash": "558d5f9fda5e3deb", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:softmax-nt4:rtlsim": { "32": { - "cycles": 4860082, + "cycles": 4860368, "instrs": 1789797 }, "app": "softmax", "args": "-n64", - "config_hash": "3d84ea77567e034e", + "config_hash": "79a0a9910744e8d5", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:stencil3d-nt16:rtlsim": { "32": { - "cycles": 2283980, + "cycles": 2235935, "instrs": 892944 }, "app": "stencil3d", "args": "-n32", - "config_hash": "5b78acb9a50222ff", + "config_hash": "461212f296e5def6", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:stencil3d-nt4:rtlsim": { "32": { - "cycles": 4185784, + "cycles": 4180121, "instrs": 1785872 }, "app": "stencil3d", "args": "-n32", - "config_hash": "dc41c57ce7819a59", + "config_hash": "0556d0ca4d4eee68", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:vecadd-nt16:rtlsim": { "32": { - "cycles": 95240, + "cycles": 96002, "instrs": 29712 }, "app": "vecadd", "args": "-n16384", - "config_hash": "652eaf7c4099617d", + "config_hash": "36eb32e6e90447f5", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:vecadd-nt4:rtlsim": { "32": { - "cycles": 368717, + "cycles": 368700, "instrs": 118800 }, "app": "vecadd", "args": "-n16384", - "config_hash": "c602c9c37a145297", + "config_hash": "36ba610d0e7114cb", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/dxa.json b/ci/baselines/perf/dxa.json index ec5e66786f..35d5419be0 100644 --- a/ci/baselines/perf/dxa.json +++ b/ci/baselines/perf/dxa.json @@ -1,23 +1,23 @@ { "dxa:perf_gate-copy-mcast:rtlsim": { "32": { - "cycles": 3381, + "cycles": 3389, "instrs": 961 }, "app": "dxa_copy_mcast", "args": "", - "config_hash": "28dc0787284f20c8", + "config_hash": "c269fb880dec8b2c", "configs": "-DVX_CFG_EXT_DXA_ENABLE", "driver": "rtlsim" }, "dxa:perf_gate-copy-nt16:rtlsim": { "32": { - "cycles": 1953, + "cycles": 1961, "instrs": 592 }, "app": "dxa_copy", "args": "-d1", - "config_hash": "912948e13624b9a4", + "config_hash": "ae2903a3d445d89b", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -28,95 +28,95 @@ }, "app": "dxa_copy", "args": "-d1", - "config_hash": "277ced0b15aed8d9", + "config_hash": "3bf4cf26dbfc656b", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-mcast-nt16:rtlsim": { "32": { - "cycles": 14016579, + "cycles": 14016558, "instrs": 4962320 }, "app": "sgemm2_dxa_mcast", "args": "-n128", - "config_hash": "f247e0e4b7829023", + "config_hash": "662e5c2b0c7b8fc1", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-mcast-nt4:rtlsim": { "32": { - "cycles": 14051140, + "cycles": 14051366, "instrs": 4962320 }, "app": "sgemm2_dxa_mcast", "args": "-n128", - "config_hash": "3660e26ec6c4ba10", + "config_hash": "d6b2042d190ac5ce", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-nt16:rtlsim": { "32": { - "cycles": 3947991, + "cycles": 3935575, "instrs": 1263632 }, "app": "sgemm2_dxa", "args": "-n128 -t4 -m1", - "config_hash": "55af78d744fa9e46", + "config_hash": "f292acf98eaaf8e3", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-nt4:rtlsim": { "32": { - "cycles": 14352123, + "cycles": 14351025, "instrs": 5017616 }, "app": "sgemm2_dxa", "args": "-n128 -t4 -m1", - "config_hash": "8c7ee15d08de3277", + "config_hash": "604991b30fdc7c3a", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-dxa-mcast-nt16:rtlsim": { "32": { - "cycles": 78840, + "cycles": 78350, "instrs": 32384 }, "app": "sgemm_tcu_wg_dxa_mcast", "args": "-m 128 -n 128 -k 64", - "config_hash": "d27df3466a8eed79", + "config_hash": "2cc358de1155b70b", "configs": "-DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-dxa-mcast-nt4:rtlsim": { "32": { - "cycles": 358897, + "cycles": 358377, "instrs": 234560 }, "app": "sgemm_tcu_wg_dxa_mcast", "args": "-m 128 -n 128 -k 64", - "config_hash": "776a839834d35db8", + "config_hash": "93e2a7ee4a7b218d", "configs": "-DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-dxa:rtlsim": { "32": { - "cycles": 29487, + "cycles": 29643, "instrs": 9160 }, "app": "sgemm_tcu_wg_dxa", "args": "-m 128 -n 128 -k 64", - "config_hash": "56519748fbd3c39b", + "config_hash": "9106c858ba91e5ae", "configs": "-DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DWGMMA_NRC=16 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_THREADS=32", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-sp-dxa:rtlsim": { "32": { - "cycles": 48573, + "cycles": 48722, "instrs": 9328 }, "app": "sgemm_tcu_wg_sp_dxa", "args": "-m 128 -n 128 -k 128", - "config_hash": "f1664e1fcc03f8a6", + "config_hash": "2091dd383d7ad57e", "configs": "-DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_THREADS=32", "driver": "rtlsim" } diff --git a/ci/baselines/perf/graphics.json b/ci/baselines/perf/graphics.json index 4d13a12aaf..32a26f3bb2 100644 --- a/ci/baselines/perf/graphics.json +++ b/ci/baselines/perf/graphics.json @@ -1,100 +1,100 @@ { "graphics:perf_gate-draw3d-mc:rtlsim": { "32": { - "cycles": 529816, + "cycles": 529951, "instrs": 203136 }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "9b91ab9ac3aa5103", + "config_hash": "b05e6b5e6b9ffc06", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "graphics:perf_gate-draw3d-nt16:rtlsim": { "32": { - "cycles": 141632, + "cycles": 141529, "instrs": 51652 }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "1f2de39a01c16b67", + "config_hash": "6e98bbba0a7822d3", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "graphics:perf_gate-draw3d-nt4:rtlsim": { "32": { - "cycles": 529359, + "cycles": 529326, "instrs": 203136 }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "73f771f2643f6527", + "config_hash": "208358458841bcde", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "graphics:perf_gate-om-nt16:rtlsim": { "32": { - "cycles": 189979, + "cycles": 189991, "instrs": 68624 }, "app": "gfx_om", "args": "-w 128 -h 128 -r whitebox_128.png", - "config_hash": "132840fd850713c9", + "config_hash": "9d0a1a59914e4ff0", "configs": "-DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "graphics:perf_gate-om-nt4:rtlsim": { "32": { - "cycles": 740701, + "cycles": 740707, "instrs": 274448 }, "app": "gfx_om", "args": "-w 128 -h 128 -r whitebox_128.png", - "config_hash": "d8e520e04c9979f4", + "config_hash": "bbaf6fac9da36d0f", "configs": "-DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "graphics:perf_gate-raster-nt16:rtlsim": { "32": { - "cycles": 14487, + "cycles": 14481, "instrs": 4708 }, "app": "gfx_raster", "args": "-w 128 -h 128 -t triangle.cgltrace -r triangle_ref_128.png", - "config_hash": "e2f119110f028280", + "config_hash": "6f2b1e9c2df8bb37", "configs": "-DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "graphics:perf_gate-raster-nt4:rtlsim": { "32": { - "cycles": 55553, + "cycles": 55551, "instrs": 18640 }, "app": "gfx_raster", "args": "-w 128 -h 128 -t triangle.cgltrace -r triangle_ref_128.png", - "config_hash": "5109e566c9637f1b", + "config_hash": "e3036cf5282b4118", "configs": "-DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "graphics:perf_gate-tex-nt16:rtlsim": { "32": { - "cycles": 35254, + "cycles": 35278, "instrs": 12304 }, "app": "gfx_tex", "args": "-i toad.png -r toad_ref_f0.png -f 0 -g 0", - "config_hash": "3876028c632f011b", + "config_hash": "82f630774395560e", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "graphics:perf_gate-tex-nt4:rtlsim": { "32": { - "cycles": 136492, + "cycles": 136502, "instrs": 49168 }, "app": "gfx_tex", "args": "-i toad.png -r toad_ref_f0.png -f 0 -g 0", - "config_hash": "fe367b2efd48140d", + "config_hash": "3a44bdd6d3900bb7", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/raytracing.json b/ci/baselines/perf/raytracing.json index 71ef9f1d16..ae04bd9c87 100644 --- a/ci/baselines/perf/raytracing.json +++ b/ci/baselines/perf/raytracing.json @@ -1,34 +1,34 @@ { "raytracing:perf_gate-rt_raycast-mc:rtlsim": { "32": { - "cycles": 85040, + "cycles": 85069, "instrs": 39346 }, "app": "rt_raycast", "args": "", - "config_hash": "c0bd39f0f980f01d", + "config_hash": "68b742a1457ab4a3", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "raytracing:perf_gate-rt_raycast-nt16:rtlsim": { "32": { - "cycles": 52906, + "cycles": 52944, "instrs": 11372 }, "app": "rt_raycast", "args": "", - "config_hash": "b899d2a47bcc51ae", + "config_hash": "53b9672f25769532", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "raytracing:perf_gate-rt_raycast-nt4:rtlsim": { "32": { - "cycles": 159655, + "cycles": 159677, "instrs": 39330 }, "app": "rt_raycast", "args": "", - "config_hash": "47b882de7074c1db", + "config_hash": "afe227c6459fe67e", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor.json b/ci/baselines/perf/tensor.json index 8ce4e9b7b0..6678ef084d 100644 --- a/ci/baselines/perf/tensor.json +++ b/ci/baselines/perf/tensor.json @@ -1,56 +1,56 @@ { "tensor:perf_gate-fp16-isw4:rtlsim": { "32": { - "cycles": 197355, + "cycles": 196204, "instrs": 130592 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "b80548e5fa6b84ae", + "config_hash": "e7b2b07afe27a336", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" }, "tensor:perf_gate-fp16-mc-nt16:rtlsim": { "32": { - "cycles": 36479, + "cycles": 38764, "instrs": 47296 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "2935e28e509b5699", + "config_hash": "60e7762f69f5cccf", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=16 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "tensor:perf_gate-fp16-mc-nt4:rtlsim": { "32": { - "cycles": 222420, + "cycles": 221653, "instrs": 336448 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "d6ccf2e6532a7a88", + "config_hash": "a70b10177d28a4d9", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "tensor:perf_gate-fp16:rtlsim": { "32": { - "cycles": 254124, + "cycles": 252792, "instrs": 130576 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "9c339e61cd76a65c", + "config_hash": "55ac763cadf62490", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32", "driver": "rtlsim" }, "tensor:perf_gate-sgemm2-fp16:rtlsim": { "32": { - "cycles": 593588, + "cycles": 594162, "instrs": 409888 }, "app": "sgemm2_tcu", "args": "-m 128 -n 128 -k 64", - "config_hash": "c27f67158e4b632a", + "config_hash": "d9980e4be1d6630d", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_mx.json b/ci/baselines/perf/tensor_mx.json index 3b8f55c440..5d8ca88fab 100644 --- a/ci/baselines/perf/tensor_mx.json +++ b/ci/baselines/perf/tensor_mx.json @@ -1,12 +1,12 @@ { "tensor_mx:perf_gate-mxfp8:rtlsim": { "32": { - "cycles": 76410, + "cycles": 76658, "instrs": 34832 }, "app": "sgemm_tcu_mx", "args": "-m 128 -n 128 -k32", - "config_hash": "63975cb90eb9477f", + "config_hash": "433197d2eaf189c5", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DITYPE=mxfp8 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_sp.json b/ci/baselines/perf/tensor_sp.json index bf7540cbf3..257076acad 100644 --- a/ci/baselines/perf/tensor_sp.json +++ b/ci/baselines/perf/tensor_sp.json @@ -1,12 +1,12 @@ { "tensor_sp:perf_gate-sparse-fp16:rtlsim": { "32": { - "cycles": 181992, + "cycles": 183077, "instrs": 100880 }, "app": "sgemm_tcu_sp", "args": "-m 128 -n 128 -k 128", - "config_hash": "3bb64bb8ac3c1a95", + "config_hash": "200d2f36d7530327", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_wg.json b/ci/baselines/perf/tensor_wg.json index 2047a4ef2a..00f45bea82 100644 --- a/ci/baselines/perf/tensor_wg.json +++ b/ci/baselines/perf/tensor_wg.json @@ -1,34 +1,34 @@ { "tensor_wg:perf_gate-wgmma-fedp2k-rs:rtlsim": { "32": { - "cycles": 158197, + "cycles": 158241, "instrs": 108512 }, "app": "sgemm_tcu_wg", "args": "-m64 -n64 -k64", - "config_hash": "6e007939f3dbec76", + "config_hash": "9a317f76414111c9", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_TCU_FEDP2K -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DWGMMA_RS", "driver": "rtlsim" }, "tensor_wg:perf_gate-wgmma-fp16-ss:rtlsim": { "32": { - "cycles": 834916, + "cycles": 838095, "instrs": 525536 }, "app": "sgemm_tcu_wg", "args": "-m 128 -n 128 -k 128", - "config_hash": "76bcef5705d497dc", + "config_hash": "a747dcd3d6a89d6b", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=32 -DWGMMA_SS -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" }, "tensor_wg:perf_gate-wgmma-sparse:rtlsim": { "32": { - "cycles": 406430, + "cycles": 399994, "instrs": 258976 }, "app": "sgemm_tcu_wg_sp", "args": "-m 128 -n 128 -k 128", - "config_hash": "d983c017c55aa5a7", + "config_hash": "1e905ff223e20367", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" } From c08c24521991dfe7d4d9430b4954a2253e6b899d Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 01:28:04 -0700 Subject: [PATCH 116/124] docs: add the memory-level parallelism design document docs/designs/mlp_optimizations.md describes the memory-level parallelism features of the cache, memory, DMA and LSU paths: what each one fixes, how it works, the cycles it saves on the measured shapes, and the VX_CFG knobs that control it. Each feature has a diagram under docs/assets/img/mlp_*.svg. The index links it under the memory-system designs. Co-Authored-By: Claude Opus 5.5 --- docs/assets/img/mlp_afu_inflight.svg | 34 ++ docs/assets/img/mlp_afu_ports.svg | 57 ++++ docs/assets/img/mlp_arg_slots.svg | 36 +++ docs/assets/img/mlp_cp_dma_llc.svg | 46 +++ docs/assets/img/mlp_fence_split.svg | 40 +++ docs/assets/img/mlp_fences.svg | 41 +++ docs/assets/img/mlp_fill_forward.svg | 45 +++ docs/assets/img/mlp_fill_staging.svg | 40 +++ docs/assets/img/mlp_flush_tokens.svg | 45 +++ docs/assets/img/mlp_hashed_banks.svg | 53 +++ docs/assets/img/mlp_icache_merge.svg | 54 ++++ docs/assets/img/mlp_launch_flush.svg | 30 ++ docs/assets/img/mlp_lazy_fetch.svg | 48 +++ docs/assets/img/mlp_llc_atomics.svg | 33 ++ docs/assets/img/mlp_lmem_round_robin.svg | 39 +++ docs/assets/img/mlp_lsu_paths.svg | 42 +++ docs/assets/img/mlp_mshr_line_table.svg | 51 +++ docs/assets/img/mlp_read_merge.svg | 54 ++++ docs/assets/img/mlp_sectoring.svg | 46 +++ docs/assets/img/mlp_wt_order.svg | 41 +++ docs/designs/mlp_optimizations.md | 389 +++++++++++++++++++++++ docs/index.md | 1 + 22 files changed, 1265 insertions(+) create mode 100644 docs/assets/img/mlp_afu_inflight.svg create mode 100644 docs/assets/img/mlp_afu_ports.svg create mode 100644 docs/assets/img/mlp_arg_slots.svg create mode 100644 docs/assets/img/mlp_cp_dma_llc.svg create mode 100644 docs/assets/img/mlp_fence_split.svg create mode 100644 docs/assets/img/mlp_fences.svg create mode 100644 docs/assets/img/mlp_fill_forward.svg create mode 100644 docs/assets/img/mlp_fill_staging.svg create mode 100644 docs/assets/img/mlp_flush_tokens.svg create mode 100644 docs/assets/img/mlp_hashed_banks.svg create mode 100644 docs/assets/img/mlp_icache_merge.svg create mode 100644 docs/assets/img/mlp_launch_flush.svg create mode 100644 docs/assets/img/mlp_lazy_fetch.svg create mode 100644 docs/assets/img/mlp_llc_atomics.svg create mode 100644 docs/assets/img/mlp_lmem_round_robin.svg create mode 100644 docs/assets/img/mlp_lsu_paths.svg create mode 100644 docs/assets/img/mlp_mshr_line_table.svg create mode 100644 docs/assets/img/mlp_read_merge.svg create mode 100644 docs/assets/img/mlp_sectoring.svg create mode 100644 docs/assets/img/mlp_wt_order.svg create mode 100644 docs/designs/mlp_optimizations.md diff --git a/docs/assets/img/mlp_afu_inflight.svg b/docs/assets/img/mlp_afu_inflight.svg new file mode 100644 index 0000000000..ffc14da73f --- /dev/null +++ b/docs/assets/img/mlp_afu_inflight.svg @@ -0,0 +1,34 @@ + + Before: each request waits a full 100-cycle round trip before the next is sent. After: four requests overlap and finish in about the time of one round trip plus three cycles. + + + + + Before: one in flight + R1 · 100 cyc + R2 + R3 + R4 + ≈ 400 cyc + After: many in flight + + R1–R4 overlap: ≈ 104 cyc + time → + + diff --git a/docs/assets/img/mlp_afu_ports.svg b/docs/assets/img/mlp_afu_ports.svg new file mode 100644 index 0000000000..1a51bb3d18 --- /dev/null +++ b/docs/assets/img/mlp_afu_ports.svg @@ -0,0 +1,57 @@ + + The AFU now has one AXI port per GPU memory port, each mapped to its own HBM group, and a write's data now leaves in the same cycle as its address. + + + +Before: fixed per board +After: follows the GPU + +GPU port 0 +GPU port 1 +GPU port 2 +GPU port 3 +1 AXI port +HBM + + + +GPU port 0 +GPU port 1 +GPU port 2 +GPU port 3 +AXI 0 +AXI 1 +AXI 2 +AXI 3 +HBM group 0 +HBM group 1 +HBM group 2 +HBM group 3 + + +Write fall-through +before +AW +W +2 cycles per write +after +AW + W +1 cycle per write + + diff --git a/docs/assets/img/mlp_arg_slots.svg b/docs/assets/img/mlp_arg_slots.svg new file mode 100644 index 0000000000..a770b2c019 --- /dev/null +++ b/docs/assets/img/mlp_arg_slots.svg @@ -0,0 +1,36 @@ + + Before, uploading a launch's arguments forced a full LLC flush; now arguments go into a slot no kernel has read since the last full flush, so the launch follows right away. + + + + +Before +upload args +full LLC flush +launch + + +After +upload args to a clean slot +launch + +no kernel has read this slot +since the last full flush + + diff --git a/docs/assets/img/mlp_cp_dma_llc.svg b/docs/assets/img/mlp_cp_dma_llc.svg new file mode 100644 index 0000000000..58c436f523 --- /dev/null +++ b/docs/assets/img/mlp_cp_dma_llc.svg @@ -0,0 +1,46 @@ + + Before: CP DMA writes DRAM directly, leaving stale lines in L2 that force a full LLC walk. After: CP DMA enters L2 on its own client slot, so only the L1s are flushed. + + + +Before: DMA beside the LLC + After: DMA as an LLC client + + host + CP DMA + core L1s + L2: stale lines + DRAM + + + writes DRAM + + needs full walk + + host + CP DMA + core L1s + L2: coherent + DRAM + + L1-only flush + client slot + + + diff --git a/docs/assets/img/mlp_fence_split.svg b/docs/assets/img/mlp_fence_split.svg new file mode 100644 index 0000000000..2027ea9bf6 --- /dev/null +++ b/docs/assets/img/mlp_fence_split.svg @@ -0,0 +1,40 @@ + + An acquire only invalidates afterward, a release only drains beforehand, and only a full fence does both. + + + + +Acquire +Release +Full +fence r,rw · amo.aq +fence rw,w · amo.rl +fence rw,rw · amo.aqrl +before +nothing +drain writes, no walk +drain writes, no walk +after +invalidate L1 +nothing +invalidate L1 +No L1 action at all where the dcache is itself the coherence point +(one socket, one dcache, no other memory client) + + diff --git a/docs/assets/img/mlp_fences.svg b/docs/assets/img/mlp_fences.svg new file mode 100644 index 0000000000..5c4b1d5056 --- /dev/null +++ b/docs/assets/img/mlp_fences.svg @@ -0,0 +1,41 @@ + + Before: a fence token walks every line in L1 and in the L2 and writes back to DRAM. After: L1 writes back to L2, and the shared L2 acknowledges the token without a walk; DRAM is not touched. + + + +Before: full flush per fence + After: L1-only token + + core: fence + L1: walk all lines + L2 (LLC): walk all lines + DRAM: write-backs + token + token + dirty + token + dirty + + core: fence + L1: write back dirty + L2 (LLC): ack, no walk + DRAM: not touched + L1-only token + dirty lines + ack + + diff --git a/docs/assets/img/mlp_fill_forward.svg b/docs/assets/img/mlp_fill_forward.svg new file mode 100644 index 0000000000..1923b1c975 --- /dev/null +++ b/docs/assets/img/mlp_fill_forward.svg @@ -0,0 +1,45 @@ + + Before, every waiting request replayed through the bank pipeline after a fill. With fill forwarding, the waiting reads take their data from the staged fill, and only the first write replays. + + + +Before: every waiter replays +After: reads forward from the fill + +fill: line L +ld L+0 (waiting) +ld L+4 (waiting) +st L+8 (waiting) +bank pipeline3 replay passes + + + +new requests wait + +fill: line L +ld L+0: done +ld L+4: done +st L+8 (waiting) + + +forward +bank pipeline1 replay, keeps order + + + diff --git a/docs/assets/img/mlp_fill_staging.svg b/docs/assets/img/mlp_fill_staging.svg new file mode 100644 index 0000000000..ac92df36e8 --- /dev/null +++ b/docs/assets/img/mlp_fill_staging.svg @@ -0,0 +1,40 @@ + + Before, a bank took a new fill every other cycle; now fills enter back to back, one per cycle. + + + + +Before +fill 1 +busy +fill 2 +busy +fill 3 +busy +fill 4 +4 fills in 7 cycles +After +fill 1 +fill 2 +fill 3 +fill 4 +4 fills in 4 cycles +cycles → + + diff --git a/docs/assets/img/mlp_flush_tokens.svg b/docs/assets/img/mlp_flush_tokens.svg new file mode 100644 index 0000000000..7a77a0177c --- /dev/null +++ b/docs/assets/img/mlp_flush_tokens.svg @@ -0,0 +1,45 @@ + + A broadcast flush overlaps every core's flush, and a flush token on every memory port makes sure no write arrives after the next level has finished its walk. + + + +Before: cores flush in turn +After: one broadcast, all overlap + +core 0 +core 1 +core 2 +core 3 + +core 0 +core 1 +core 2 +core 3 +Before: token on port 0 only +After: a token on every port +L1 (2 memory ports) +next level: line left dirty +token +write,late +L1 (2 memory ports) +next level: answers after both +token +token + + diff --git a/docs/assets/img/mlp_hashed_banks.svg b/docs/assets/img/mlp_hashed_banks.svg new file mode 100644 index 0000000000..259a3d19a4 --- /dev/null +++ b/docs/assets/img/mlp_hashed_banks.svg @@ -0,0 +1,53 @@ + + Four requests with a power-of-two stride all queue on bank 0 with plain bank select; with the hash they spread over banks 0 to 3. + + + +Before: bank = line[1:0] + After: bank = line[1:0] XOR hash(line) + + line 0x000 + line 0x100 + line 0x200 + line 0x300 + bank 0 · 4 queued + bank 1 idle + bank 2 idle + bank 3 idle + + + + + + line 0x000 + line 0x100 + line 0x200 + line 0x300 + bank 0 + bank 1 + bank 2 + bank 3 + + + + + serialized: 4 cycles + parallel: 1 cycle + + diff --git a/docs/assets/img/mlp_icache_merge.svg b/docs/assets/img/mlp_icache_merge.svg new file mode 100644 index 0000000000..da923d6298 --- /dev/null +++ b/docs/assets/img/mlp_icache_merge.svg @@ -0,0 +1,54 @@ + + Before: four fetch slices request the same icache line and are served one per cycle. After: one access returns word 0 to 3 to slices 0 to 3 in the same cycle. + + + +Before: slices take turns + After: one access, a word per slice + + slice 0: line L + slice 1: line L + slice 2: line L + slice 3: line L + icache + + + + + S0 + S1 + S2 + S3 + 4 cycles + + slice 0: line L + slice 1: line L + slice 2: line L + slice 3: line L + icache + + + + + + w0w1w2w3 + S0–3 + 1 cycle + + diff --git a/docs/assets/img/mlp_launch_flush.svg b/docs/assets/img/mlp_launch_flush.svg new file mode 100644 index 0000000000..5d10af1f49 --- /dev/null +++ b/docs/assets/img/mlp_launch_flush.svg @@ -0,0 +1,30 @@ + + Flush cycles per launch, drawn to scale: a full flush walks the L2; an L1-only flush skips it. + + + + +1 core, full flush422 +1 core, L1-only152 +4 cores + L2, full flush2,750 +4 cores + L2, L1-only180 + +02,750 cycles per launch + + diff --git a/docs/assets/img/mlp_lazy_fetch.svg b/docs/assets/img/mlp_lazy_fetch.svg new file mode 100644 index 0000000000..4a58bc1d2b --- /dev/null +++ b/docs/assets/img/mlp_lazy_fetch.svg @@ -0,0 +1,48 @@ + + Stores fill the sector's byte mask without a DRAM read. When a load arrives on a fully written sector, the old design reads DRAM anyway; design G loops back inside the bank with no DRAM read. + + + + + 1 · store miss + 2 · more stores + 3 · load arrives + + sector installed, partial = 1 + no DRAM read + + byte mask full + partial still = 1 + ld → partial line + + + + + + + Without the mask check + With the mask check + refetch from DRAM + merge under 0 bytes + + mask full → loop back + clear partial, no DRAM + + + diff --git a/docs/assets/img/mlp_llc_atomics.svg b/docs/assets/img/mlp_llc_atomics.svg new file mode 100644 index 0000000000..250253c402 --- /dev/null +++ b/docs/assets/img/mlp_llc_atomics.svg @@ -0,0 +1,33 @@ + + An atomic executes once at the last-level cache, and every L1 it passes through drops its copy of the line. + + + + +core 0: amoadd X +core 1: ld X +L1: pass, drop X +L1: miss on X +L2 (LLC): read-modify-write X + +AMO + +sees new X + + diff --git a/docs/assets/img/mlp_lmem_round_robin.svg b/docs/assets/img/mlp_lmem_round_robin.svg new file mode 100644 index 0000000000..6009b161f7 --- /dev/null +++ b/docs/assets/img/mlp_lmem_round_robin.svg @@ -0,0 +1,39 @@ + + Before: DXA holds the LMEM port for about 540 cycles while TCU reads wait. After: grants alternate between DXA and TCU every cycle. + + + + + Before: fixed priority + DXA writes + holds the port ~540 cycles + TCU reads + stalled ~510 cycles + go + After: round-robin + DXA writes + TCU reads + + + + + grants alternate + time → + + diff --git a/docs/assets/img/mlp_lsu_paths.svg b/docs/assets/img/mlp_lsu_paths.svg new file mode 100644 index 0000000000..641b348ff8 --- /dev/null +++ b/docs/assets/img/mlp_lsu_paths.svg @@ -0,0 +1,42 @@ + + Each LSU block now owns its path to its dcache ports, so the core no longer funnels every block through one shared memory path. + + + +Before: one shared path +After: a path per LSU block + +LSU block 0 +LSU block 1 +LSU block 2 +LSU block 3 +core memorypath +dcache + + + +LSU block 0 +LSU block 1 +LSU block 2 +LSU block 3 +dcacheports 0–3 + +own path, own status + + diff --git a/docs/assets/img/mlp_mshr_line_table.svg b/docs/assets/img/mlp_mshr_line_table.svg new file mode 100644 index 0000000000..0097fe146b --- /dev/null +++ b/docs/assets/img/mlp_mshr_line_table.svg @@ -0,0 +1,51 @@ + + Before: a flat MSHR where each of 16 entries stores a full line address. After: 64 small request entries point by index into a 16-entry line table of full addresses. + + + +Before: flat MSHR (16) + After: requests (64) → line table (16) + + full line addr | request + full line addr | request + full line addr | request + ⋮ + full line addr | request + 16 wide comparators + full at 16 requests + + idx 0 | request + idx 0 | request + idx 1 | request + idx 0 | request + ⋮ + idx 2 | request + 64 narrow entries + 0: line A · refs 3 + 1: line B · refs 1 + 2: line C · refs 1 + ⋮ 16 lines + + + + + + 16 wide comparators + + diff --git a/docs/assets/img/mlp_read_merge.svg b/docs/assets/img/mlp_read_merge.svg new file mode 100644 index 0000000000..fad93e4e00 --- /dev/null +++ b/docs/assets/img/mlp_read_merge.svg @@ -0,0 +1,54 @@ + + Before: four ports load word A and the bank serves them in four cycles. After: one access at cycle 1, with the reply multicast to all four ports. + + + +Before: one access per port + After: one access, multicast + + port 0: ld A + port 1: ld A + port 2: ld A + port 3: ld A + bank + + + + + P0 + P1 + P2 + P3 + 4 cycles + + port 0: ld A + port 1: ld A + port 2: ld A + port 3: ld A + bank + + + + + + P0–3 + 1 cycle + dashed: one reply to every port + + diff --git a/docs/assets/img/mlp_sectoring.svg b/docs/assets/img/mlp_sectoring.svg new file mode 100644 index 0000000000..228691552e --- /dev/null +++ b/docs/assets/img/mlp_sectoring.svg @@ -0,0 +1,46 @@ + + L2 and L3 lines carry one tag over two 64-byte sectors; dcache banks scale with the thread count. + + + +L2/L3: one tag, two sectors +Dcache: banks follow the lanes + +before +tag +64 B line +tag +64 B line +2 tags per 128 B +after +tag +sector 0 · 64 B +sector 1 · 64 B +1 tag per 128 B: half the tags +fills and evictions move one 64 B sector + +4 threads → 4 banks + +8 threads → 8 banks + +16 threads → 16 banks + +banks = line footprint / word size, line size unchanged + + diff --git a/docs/assets/img/mlp_wt_order.svg b/docs/assets/img/mlp_wt_order.svg new file mode 100644 index 0000000000..1abc739ba6 --- /dev/null +++ b/docs/assets/img/mlp_wt_order.svg @@ -0,0 +1,41 @@ + + In a write-through cache, a load to a word with an older store still waiting must wait behind that store, not hit the old data. + + + +Before: the load hits stale data +After: the load waits for the store + +read miss, line L +store X (waiting) + +load X (new) +hit: returns old X + + +read miss, line L +store X (waiting) +load X: linked behind + + +returns new X + +a waiting storenow counts asa hazard + + diff --git a/docs/designs/mlp_optimizations.md b/docs/designs/mlp_optimizations.md new file mode 100644 index 0000000000..e93fc29890 --- /dev/null +++ b/docs/designs/mlp_optimizations.md @@ -0,0 +1,389 @@ +# Memory-Level Parallelism Optimizations + +A GPU hides memory latency by keeping many memory requests in flight at once. This is memory-level parallelism (MLP). It runs from the lanes of one warp, through the caches and the command processor's DMA, out to the DRAM channels. This document describes the features that raise Vortex's MLP: +- how the cache hierarchy is organized; +- how misses are handled; +- how same-address requests are merged; +- how coherence and ordering are kept cheap; +- how the DMA engines share the memory system; +- how the FPGA shell reaches HBM. + +Each section says what problem the feature removes, how it works, and what it gains. + +## Reading the numbers + +- Gains are changes in total kernel cycles, so −10% means 10% faster. Each gain is measured against the same configuration with the feature off. +- **rtlsim** is the Verilator model of the RTL. **SimX** is the cycle-level C++ model. **xrtsim** is the RTL plus the FPGA shell. +- The GPU shapes (`a100`, `h100`, `rtx5080`, …) are hardware profiles. A profile sets cores, warps, threads, cache sizes, and the DRAM type and timing (HBM2, HBM3, GDDR6) from data-sheet values. See [simulation.md](../simulation.md#hardware-profiles). + +## Summary + +| Feature | Level | What it removes | Best measured gain | Default | +|---|---|---|---|---| +| [Sectored L2/L3](#sectored-l2-and-l3-caches) | L2/L3 | Half of the LLC tags | Tag area halved | On | +| [Word-size dcache banking](#dcache-banks-follow-the-thread-count) | L1D | Fixed bank count as threads grow | Bank count scales with threads | On | +| [Hashed bank select](#hashed-bank-select) | L1D, L2 | Power-of-two strides on one bank | sgemv −32.8% | On (L2; dcache ≥ 4 banks) | +| [Fill forwarding](#fill-forwarding) | All caches | Replaying waiters after a fill | sgemv −5.0% | On | +| [Back-to-back fills](#back-to-back-fills) | All caches | A bubble after each fill | rtx5080 stream −6.2% | On | +| [MSHR line table](#mshr-line-table) | All caches | One wide address compare per waiting request | sgemm −15.3% at 400-cycle latency | Off | +| [Write-allocate without fetch](#write-allocate-without-fetch-lazy-fetch) | Write-back caches | Reading a line only to overwrite it | DRAM reads −33% | Off | +| [Full-sector refetch skip](#full-sector-refetch-skip) | Write-back caches | Refetching a fully written sector | rtx5080 sgemm −43.5% with lazy L2 | Off (with lazy fetch) | +| [Dcache read merge](#dcache-read-merge) | L1D | Same-word loads serialized on one bank | sgemm −17.2% | On | +| [Icache same-line merge](#icache-same-line-merge) | L1I | Fetch slices serialized on one line | rtx5080 sgemm −25.8% | On | +| [Broadcast flush, per-port tokens](#broadcast-flush-and-per-port-flush-tokens) | Whole hierarchy | Serial per-core flushes; lost writes | Flush 2.76M → 16.5k cycles per launch | On | +| [L1-only launch flush](#l1-only-flush-between-launches) | Whole hierarchy | Walking the LLC at every launch | Flush 2,750 → 180 cycles per launch | On | +| [Argument upload slots](#launch-argument-slots) | Runtime + LLC | A full flush after each argument upload | multikernel −7.5% | On | +| [Targeted fences](#targeted-fences) | L1D, LLC | A full flush per fence | printf 13.6× faster | On | +| [Acquire/release fences](#acquire-and-release-fences) | L1D | Doing both halves of a fence | Ordering-correct atomics at minimal cost | On | +| [Atomics at the LLC](#atomics-at-the-last-level-cache) | L1D, LLC | Atomics racing across private caches | Correct atomics across the hierarchy | With the A extension | +| [Write-through ordering](#write-through-ordering) | L1D | A load returning data older than its own store | Correctness | On | +| [CP DMA through the LLC](#command-processor-dma-through-the-llc) | CP, LLC | An LLC walk for every host transfer | multikernel −42.7% | Off (on in GPU profiles) | +| [LMEM DMA round-robin](#local-memory-dma-round-robin) | Local memory | DXA starving tensor-core reads | WGMMA with DXA −10.2% | On | +| [Per-block LSU memory paths](#per-block-lsu-memory-paths) | Core | One shared core memory path | Scales with LSU blocks | On | +| [Many AXI transactions in flight](#many-axi-transactions-in-flight) | FPGA shell | One outstanding request per port | 4-core sgemv 1.30× | On | +| [Ports follow the GPU; write fall-through](#memory-ports-follow-the-gpu-and-write-fall-through) | FPGA shell | A fixed port count; a cycle per write | Write throughput 2× | On | + +## Configuration knobs + +Some of these features can be set with a `VX_CFG_*` knob (see [Build Configuration System](build_configuration_system.md)). The rest are always on, or are selected automatically from the cache layout. + +| Feature | Knob | Default | +|---|---|---| +| Hashed bank select | `VX_CFG_DCACHE_BANK_HASH`, `VX_CFG_L2_BANK_HASH` | dcache: on with 4 or more banks; L2: on. The L3 has no bank hash. | +| MSHR line table | `VX_CFG_{DCACHE,L2,L3}_MSHR_LINES` | 0 (off): flat MSHR | +| Write-allocate without fetch | `VX_CFG_{DCACHE,L2,L3}_LAZY_FETCH` | 0 (off). Enabling it also enables the dirty-byte masks (`*_DIRTYBYTES`). | +| Full-sector refetch skip | Part of lazy fetch | Follows `*_LAZY_FETCH` | +| Dcache read merge | `VX_CFG_DCACHE_READ_MERGE` | 1 (on) | +| Icache same-line merge | `VX_CFG_ICACHE_READ_MERGE` | 1 (on). No effect with one fetch slice. | +| CP DMA through the LLC | `VX_CFG_CP_LLC_CLIENT` | 0 (off); 1 in the GPU profiles. Takes effect only when one LLC serves every core. | +| Atomics at the LLC | `VX_CFG_EXT_A_ENABLE` | Off | +| Sectored L2/L3 | `VX_CFG_{L2,L3}_SECTOR_SIZE` (a size) | One bus granule. Setting it to the line size removes sectoring. | +| Dcache word-size banking | `VX_CFG_DCACHE_WORD_SIZE` (a size) | Derived from the lane count and XLEN | + +**Always on:** fill forwarding, back-to-back fills, broadcast flush and per-port flush tokens, launch argument slots, targeted fences, acquire/release fences, write-through ordering, local-memory DMA round-robin, per-block LSU memory paths, many AXI transactions in flight, and write fall-through. + +**Selected automatically:** +- The L1-only launch flush and the fence shortcuts apply whenever a single LLC is the point of coherence. +- The FPGA shell's memory port count follows the GPU's memory ports. + +--- + +## Cache organization + +### Sectored L2 and L3 caches + +![L2/L3 sectoring and dcache banking](../assets/img/mlp_sectoring.svg) + +**Problem.** An LLC line as small as the memory bus granule needs one tag per 64 bytes. At GPU LLC sizes (megabytes), that is a large tag array. + +**How it works.** L2 and L3 lines are twice the bus granule. One tag covers the line, and the line is split into two sectors of one bus granule each, with a valid bit per sector. Fills and evictions move one sector at a time, so memory traffic stays at bus granularity, while the tag array is half the size. Sector size is set with `VX_CFG_{L2,L3}_SECTOR_SIZE`. + +**Gain.** The LLC tag area is halved at the same capacity and the same memory traffic. + +### Dcache banks follow the thread count + +**Problem.** A dcache with a fixed bank count serializes a warp's lanes once the thread count outgrows it. + +**How it works.** The coalescer's word size sets the dcache banking: banks = line footprint / word size, and there is one bank per request lane. Wider warps get more banks without a smaller line. See the right half of the figure above. + +**Gain.** Dcache parallelism tracks `NUM_THREADS`, so a 16- or 32-thread warp is not throttled by a 4-bank cache. + +### Hashed bank select + +![Hashed bank select](../assets/img/mlp_hashed_banks.svg) + +**Problem.** A cache picks its bank from the lowest line-address bits. GPU kernels walk arrays with power-of-two strides: matrix rows, tiles, per-thread blocks. Every request in such a stride has the same low bits, so all of them queue on one bank while the others sit idle. + +**How it works.** The bank index is the low line-address bits XORed with a hash of the higher bits. The hash is an IPOLY polynomial: IPOLY(5) for 16 banks, IPOLY(37) for 32, and an XOR fold for other counts. Strided requests spread over all banks. Each line still has exactly one bank, so the rest of the cache is unchanged. It is controlled by `VX_CFG_{DCACHE,L2}_BANK_HASH`: on for the L2, and on for a dcache with 4 or more banks. Below 4 banks, the extra queuing outweighs the spreading. + +**Gain (rtlsim):** +- sgemv 512×512 at 16 threads: −32.8% +- fp16 tensor sgemm on 2 cores + L2: −13.8% +- WGMMA with DXA: −13.5% +- multi-core sgemm: −8.2% +- SimX, rtx5080, N=256: −34% + +--- + +## Miss handling + +### Fill forwarding + +![Fill forwarding](../assets/img/mlp_fill_forward.svg) + +**Problem.** When a miss returns, every request waiting on that line (the MSHR chain) used to replay through the bank pipeline one at a time. Each replay is a full pipeline pass, and new requests are blocked while the chain drains. + +**How it works.** The reads at the head of the chain take their data straight from the staged fill sector, which is fill-buffer forwarding, standard in GPU L1s. The first write or atomic in the chain, and everything after it, still replays, so same-line accesses keep program order. The input arbiter stays open to new requests while the chain completes. Always on. + +**Gain (rtlsim):** sgemm −2.7%, sgemv −5.0%, multi-core sgemm −2.2%. + +### Back-to-back fills + +![Back-to-back fills](../assets/img/mlp_fill_staging.svg) + +**Problem.** A bank could not take a fill in the cycle right after another fill. Under heavy miss traffic, fills arrived in bursts and the bank accepted only one every other cycle. + +**How it works.** Fills follow each other into the bank pipeline back to back. Each staged fill keeps its own sector until it installs, so consecutive fills don't conflict. + +**Gain:** +- rtx5080 profile, 4 MiB stream: −6.2% +- the stock configuration (shallow L2): within ±0.5% +- the cache also closed timing faster: Fmax up by 3% + +### MSHR line table + +![MSHR line table](../assets/img/mlp_mshr_line_table.svg) + +**Problem.** Each MSHR entry holds a full line address, and every new miss compares against all of them. To keep more misses in flight, the MSHR needs more entries, and each one adds a wide comparator. At high memory latency, a 16-entry MSHR was full a fifth of the time, often with only one or two distinct lines in it. + +**How it works.** Distinct {line, sector} pairs go into a small reference-counted line table. Each request entry keeps only an index into that table. The cache holds many more waiting requests (for example 64) while comparing addresses against only the line table (for example 16 entries). Controlled by `VX_CFG_{DCACHE,L2,L3}_MSHR_LINES`. + +**Gain (rtlsim at 400-cycle memory latency, MSHR 16 → 64 requests over 16 lines):** sgemm −15.3%, raycast −8.1%. Without the dcache read merge, the same sgemm gains 24.8%. At the stock latency the MSHR does not limit, the gain is under 0.5%, and the feature is off by default. + +### Write-allocate without fetch (lazy fetch) + +**Problem.** A write-back cache that misses on a store normally reads the whole line from memory, then overwrites part or all of it. Streaming writes therefore cost a read for every line they write. + +**How it works.** A store miss installs the sector at once, with a per-byte dirty mask and a *partial* bit, without reading memory. A later load or atomic on a partial sector refetches it and merges memory data only under the bytes that were never written. An eviction writes back only the dirty bytes. A refill of one sector keeps the dirty bytes of the line's other sectors. Controlled by `VX_CFG_{DCACHE,L2,L3}_LAZY_FETCH`. + +**Gain:** 1-core vecadd DRAM reads −33% and memory requests −25%. Cycles are unchanged because that kernel is latency-bound. Off by default. + +### Full-sector refetch skip + +![Full-sector refetch skip](../assets/img/mlp_lazy_fetch.svg) + +**Problem.** With lazy fetch, once stores have written every byte of a sector, the partial bit still triggered a refetch on the next load. The cache read memory only to merge it under zero unwritten bytes. On L2-heavy kernels, these useless reads made lazy fetch slower than leaving it off. + +**How it works.** When the refetch commits, the bank checks the byte mask. If the mask is full, the refetch loops back through the bank as a fill with no data, clears the partial bit, and reads nothing from memory. If the loopback queue has no room, the refetch falls back to memory as before. + +**Gain:** +- DRAM reads: 145 → 17 at the dcache and 539 → 27 at the L2, on a fully written stream. +- SimX, rtx5080 profile, 4 cores, sgemm -n192 with L2 lazy fetch: 285,006 → 161,121 cycles (−43.5%). Lazy fetch is now slightly faster than having it off. + +--- + +## Request merging + +### Dcache read merge + +![Dcache read merge](../assets/img/mlp_read_merge.svg) + +**Problem.** Uniform loads make every lane of a warp read the same word in the same cycle. Kernel arguments and the shared operand of a matrix multiply are typical. Each LSU port sent its own request to the same bank, and the bank served them one per cycle. + +**How it works.** Plain reads of the same word from the ports of one LSU block in the same cycle are served by one bank access, and the reply goes to every port that asked. Each port's order into the bank is unchanged, so loads stay ordered against stores. The merge compares request tags in the same form on every port, so it doesn't depend on how wide the tag is. Controlled by `VX_CFG_DCACHE_READ_MERGE`. + +**Gain (rtlsim sgemm -n128):** +- 16 threads with an 8-byte dcache word: −17.2% +- 2 banks: −9.8% +- 32 threads: −7.9% +- 16 threads: −6.5% +- 4 cores + L2: −6.5% + +At 16 threads, dcache reads fall 25% and bank stalls 55%. + +### Icache same-line merge + +![Icache same-line merge](../assets/img/mlp_icache_merge.svg) + +**Problem.** With several fetch slices, the front end fetches up to one instruction per slice per cycle. A dense kernel's inner loop often spans only a few cache lines (sgemm: 3). So the slices keep requesting the same line and take turns at its bank, losing 1 to 2 of 4 fetch slots every cycle even with a 100% hit rate. A warp has only one fetch in flight, so each cycle of waiting lowers its fetch rate directly. + +**How it works.** Reads of the same icache line from different fetch slices in the same cycle become one bank access. The bank returns each slice the word it asked for. This differs from the dcache merge: the slices want different words of one line, not the same word. Controlled by `VX_CFG_ICACHE_READ_MERGE`. It has no effect with one fetch slice. + +**Gain (rtlsim / SimX):** + +| Shape | sgemm | vecadd | conv3 | raycast | +|---|---|---|---|---| +| Issue and fetch width 4 | −12.1% / −12.4% | | | −4.7% / −4.7% | +| rtx5080 core | −25.8% / −20.2% | −15.7% / −20.1% | −3.4% / −4.6% | | + +With the merge, 16 icache banks are no longer slower than 4. + +--- + +## Coherence and ordering + +### Broadcast flush and per-port flush tokens + +![Broadcast flush and per-port tokens](../assets/img/mlp_flush_tokens.svg) + +**Problem:** +- The command processor flushed the cores one after another, so a launch boundary cost the sum of every core's flush. +- A cache forwarded its flush token on memory port 0 only. A write still queued on another port could reach the next level after that level had finished its walk, and stay dirty there. + +**How it works:** +- One broadcast `CACHE_FLUSH` write starts every core's flush at once. The command processor then collects the completions, so the flushes overlap. +- A cache walks its banks and sends a flush token down **every** memory port. It answers only once all ports have answered. +- Every independently buffered write path carries its own token. +- Read-only caches (icache, texture, ray-tracing caches) no longer forward flushes at all. + +**Gain:** flush cost per launch dropped from 2.76M to 16.5k cycles on the measured shape. It also fixed the lost-write case above. + +### L1-only flush between launches + +![L1-only launch flush](../assets/img/mlp_launch_flush.svg) + +**Problem.** Every kernel launch flushed the whole hierarchy, including a walk of the LLC. Back-to-back launches paid for it each time. + +**How it works.** When a single LLC serves every memory client, it is the point of coherence: launches only need the L1s cleaned. The launch flush becomes an L1-only token, which the shared LLC answers without walking. LLCs that are replicated per core or per cluster, and LLCs with peer clients beside them (texture, ray tracing, raster, output merger, DXA), still walk. A dcache counts as the LLC only when it is the single dcache of a single socket with no other memory client. The runtime posts flushes lazily and flushes fully before any device-memory transfer that needs it. The device reports busy until every core write, including uncached I/O stores, is visible in memory. + +**Gain:** flush cycles per launch 422 → 152 at 1 core and 2,750 → 180 at 4 cores + L2. Chained launches at 4 cores + L2: −11%. + +### Launch argument slots + +![Launch argument slots](../assets/img/mlp_arg_slots.svg) + +**Problem.** Each launch uploads its arguments and launch descriptors to device memory. That upload made the LLC dirty, so the next launch needed a full flush. Workloads that launch many small kernels behaved as if the L1-only flush didn't exist. + +**How it works.** Arguments and descriptors go into a pool of slots. A slot is reused only if no kernel has read it since the last full flush, so no cache can hold a stale copy and the upload needs no flush. This is the same dirty tracking AMD's driver does for its argument buffers. + +**Gain:** multikernel at 4 cores + L2: −7.5% on rtlsim, −8.5% on SimX. A workload that reads results back after every launch still needs the full flush and is unchanged. + +### Targeted fences + +![Targeted fences](../assets/img/mlp_fences.svg) + +**Problem.** A memory fence flushed the whole hierarchy: every level walked all of its lines and wrote dirty data to DRAM. `printf` fences after every device write, so printf-heavy kernels spent most of their time walking caches. + +**How it works.** +- A memory fence sends an L1-only token. The L1s write their dirty lines back to the shared LLC, and the LLC answers without walking. DRAM is not touched. +- A fence that orders only device (I/O) accesses becomes its own LSU operation. Every level drains its in-flight writes to its memory ports, with no walk anywhere. + +**Gain (rtlsim):** +- printf at 2 cores + L2: 4,006,822 → 295,165 cycles (13.6×) +- printf at the default configuration: −42.9% +- fence test at 2 cores + L2: −55% +- async global barrier: −13.3% + +### Acquire and release fences + +![Acquire and release fences](../assets/img/mlp_fence_split.svg) + +**Problem.** A fence did both halves of a full barrier, even when the program asked for only one. Atomics marked acquire or release (`.aq`/`.rl`) were not ordered at all. + +**How it works.** The fence's predecessor and successor sets choose the action: +- **Acquire** (`fence r,rw`, `amo.aq`): invalidates clean L1 lines after the operation. Nothing written back. +- **Release** (`fence rw,w`, `amo.rl`): drains pending writes before the operation, without a walk. Nothing invalidated. +- **Full** (`fence rw,rw`, `amo.aqrl`): does both. + +Atomics with `.aq`/`.rl` get these steps around them. All of it is skipped when the dcache is itself the point of coherence. + +**Gain:** correct acquire/release ordering for atomics, at the minimum cost the memory model allows. Fences, printf, global barriers and sgemm are unchanged; their kernels use full fences. A test that makes every atomic sequentially consistent pays for the ordering it now gets. + +### Atomics at the last-level cache + +![Atomics at the LLC](../assets/img/mlp_llc_atomics.svg) + +**How it works.** RISC-V `A`-extension atomics run once, as a read-modify-write at the last-level cache. Each private L1 they pass through drops its copy of the line, so the next load of that word from any core misses and sees the new value. `LR`/`SC` use a reservation table. This keeps atomics correct across L1, L2 and L3 without a coherence protocol between the L1s. Controlled by `VX_CFG_EXT_A_ENABLE`. + +### Write-through ordering + +![Write-through ordering](../assets/img/mlp_wt_order.svg) + +**Problem.** In a write-through L1 (every configuration with more than one core, or with an L2 or L3), a store can be waiting in the MSHR behind an earlier read miss to the same line. A load of the same word that arrived then could hit the cache and return the value from before the store. + +**How it works.** A waiting store counts as a hazard. A later access to the same word links behind it in the MSHR chain and sees the stored value. The same rule keeps a newer write from overtaking an older one. + +**Gain:** correctness. The ordering test fails without the rule and passes with it. + +--- + +## DMA + +### Command-processor DMA through the LLC + +![CP DMA through the LLC](../assets/img/mlp_cp_dma_llc.svg) + +**Problem.** The command processor copied host data straight into DRAM, beside the L2. The L2 could hold stale copies of those lines, so every host transfer needed a full flush and invalidate of the LLC. Launches with uploads between them paid this every time. + +**How it works:** +- When one LLC serves every core, the command processor's DMA becomes a client of it on its own slot. A page-table walker attaches the same way. +- Host data flows through the LLC and keeps it coherent, so a transfer needs only an L1-only flush. +- Transfers of at least half the LLC write around it, so they don't evict the working set. +- A launch issued after DMA writes waits for an L1-only flush token sent on the DMA's slot. +- The runtime detects the feature through the `DMA_LLC_CLIENT` capability. +- Controlled by `VX_CFG_CP_LLC_CLIENT`: off by default, on in the GPU profiles. + +**Gain (rtlsim):** + +| Shape | dogfood | multikernel | sgemm -n32 | +|---|---|---|---| +| 2 clusters + L2 + L3 | −30.0% | −42.7% | −14.1% | +| 4 cores + L2 | −9.1% | −23.4% | | +| 8 MB L3 | −64% | | | + +**Virtual memory.** SimX's command processor translates DMA addresses before they enter the LLC. The RTL command processor has no MMU, so host DMA with virtual memory is a SimX-only configuration for now. + +### Local-memory DMA round-robin + +![LMEM DMA round-robin](../assets/img/mlp_lmem_round_robin.svg) + +**Problem.** Two clients share the local-memory DMA port: the DXA engine, which writes tiles into shared memory, and the tensor core, which reads WGMMA operands from it. The DXA had fixed priority. One tiled-scatter transfer held the port for about 540 cycles, and a tensor-core batch caught behind it stalled for about 510. That stall delayed the warpgroup barrier, and the barrier delayed the next DXA request. + +**How it works.** The port grants round-robin, so a tensor-core read waits at most one DXA write. It costs one bit of state, and a single-client configuration is unchanged. + +**Gain (rtlsim):** WGMMA with DXA −10.2%, sparse WGMMA with DXA −11.6%. + +--- + +## Load/store unit + +### Per-block LSU memory paths + +![Per-block LSU memory paths](../assets/img/mlp_lsu_paths.svg) + +**Problem.** With several LSU blocks per core (one per issue slice), all of them went through one shared core memory path. Both bandwidth and floorplanning were limited by that single funnel. + +**How it works.** Each LSU block owns its path to its own dcache ports, along with its own status signals. The tensor-core blocks also get their own metadata ports into their slice's LSU. This is the processing-block organization of NVIDIA SMs: each partition has its own load/store path. + +**Gain:** cycle-identical at today's shapes. It is the structure that lets memory bandwidth scale with the number of LSU blocks, and it shortens the wires in the core. + +--- + +## FPGA memory interface + +### Many AXI transactions in flight + +![Many AXI transactions in flight](../assets/img/mlp_afu_inflight.svg) + +**Problem.** The FPGA shell limited each AXI memory port to very few outstanding transactions. By Little's law, a port's throughput is the number of requests in flight divided by latency. One request in flight at 100 cycles of latency delivers one line per 100 cycles, however much bandwidth the HBM has. + +**How it works.** The limit is gone. Requests are arbitrated with their AXI IDs, and responses are routed back by ID, so many transactions overlap their latency. A grant is held while a request is being offered, which keeps the AXI rule that a valid request stays stable. + +**Gain (xrtsim):** + +| Case | Speedup | +|---|---| +| One port, 100-cycle latency | 1.25–1.27× | +| 4-core sgemv | 1.30× | +| Several ports | 1.06–1.08× | +| Default timing | 1.00–1.13× | + +### Memory ports follow the GPU, and write fall-through + +![Memory ports and write fall-through](../assets/img/mlp_afu_ports.svg) + +**Problem.** +- The number of AXI memory ports was fixed per board, not set by the GPU configuration. A larger GPU could not use more HBM channels, and some boards left channels unmapped. +- A write's data waited a cycle behind its address. + +**How it works.** +- The shell gets one AXI port per GPU memory port, rounded up to a power of two. +- Each port owns a contiguous part of memory, mapped to its own HBM channel group, and the board connectivity is generated from the port count. +- A write's data now leaves in the same cycle as its address. Data stays in address order through a small queue. +- Reset waits for any write already in progress, so it never splits a burst. + +**Gain:** +- Single-beat writes: twice the throughput (1,000 writes in 1,000 cycles instead of 2,000). +- Small GPUs (1–4 cores) are not port-bound and see within ±1.2%. The port scaling is what lets larger GPUs use the HBM. + +--- + +## How the gains were measured + +- **DRAM model.** All simulators share one DRAM model: + - independent channels, refresh, and open-page bank interleave; + - JEDEC-style HBM2, HBM3 and GDDR6 timing from data-sheet values; + - per-channel response queues. + + The A100 profile's idle latency (216/257 core cycles open-row/conflict) matches GPGPU-Sim's SM80 model (about 215/255). +- **Model agreement.** SimX models every feature here with the same structure as the RTL, and CI checks SimX against rtlsim on each feature's shapes. So gains measured on SimX at full GPU scale reflect the hardware. Each feature has a test that fails when it is broken or turned off. diff --git a/docs/index.md b/docs/index.md index 82631b4f45..103fbc2c2b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -47,6 +47,7 @@ Detailed architectural specifications for Vortex subsystems live under - [Custom Accelerator ISA Extensions](designs/custom_accelerator_isa_extensions.md) - **Memory system** - [Cache Subsystem](designs/cache_subsystem.md) + - [Memory-Level Parallelism Optimizations](designs/mlp_optimizations.md) - [Atomic Memory Operations](designs/atomic_memory_operations.md) - [Multi-Cache AMO Coherence](designs/multicache_amo_coherence.md) - [Memory Fabric Attributes](designs/memory_fabric_attributes.md) From 2339de0b996b01475bf2de38a9ca79473136e330 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 06:27:41 -0700 Subject: [PATCH 117/124] syn: leave the XRT platform top out of the yosys AFU DUT The yosys top DUT synthesizes VX_afu_wrap, but its include path also pulled in hw/rtl/afu/xrt, so sv2v read the XRT platform top too. That top now spells out one AXI master per GPU memory port through AFU_MEM_PORTS, a count only the Xilinx flows resolve from the elaborated configuration, so asic_gate's top build stopped at source conversion on the undefined macro. The platform top is not part of the synthesized design: drop its directory from the include path. asic_gate -b top: 618.3 MHz against the 620.9 MHz baseline, cell area +0.5%, PASS. The other eleven asic_gate DUTs pass on the same tree. Co-Authored-By: Claude Opus 5.5 --- hw/syn/yosys/dut/catalog.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hw/syn/yosys/dut/catalog.mk b/hw/syn/yosys/dut/catalog.mk index d67a94c75b..6e28351a77 100644 --- a/hw/syn/yosys/dut/catalog.mk +++ b/hw/syn/yosys/dut/catalog.mk @@ -83,7 +83,7 @@ tensor_CFG := -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_EXT_TCU_ENA # Multi-core system top behind the AFU control/memory wrapper, as in the Xilinx # catalog: 2 cores with atomics and virtual memory. top_TOP := VX_afu_wrap -top_INC := -I$(VORTEX_HOME)/hw/rtl/cp -I$(AFU_RTL_DIR)/xrt -I$(AFU_RTL_DIR)/common +top_INC := -I$(VORTEX_HOME)/hw/rtl/cp -I$(AFU_RTL_DIR)/common top_CFG := -DVX_CFG_NUM_THREADS=4 -DVX_CFG_NUM_WARPS=4 -DVX_CFG_NUM_CORES=2 \ -DVX_CFG_EXT_A_ENABLE -DVX_CFG_VM_ENABLE -DVX_CFG_DCACHE_LATENCY=3 From 1578b79cb5ae8ce71bf7634f6984b01fe39285b5 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 11:04:10 -0700 Subject: [PATCH 118/124] ci: split the model_parity and perf_gate cells into hour-long shards Each cell's two shards ran about 2h25 on the CI runners, close to the tests job's timeout and the longest jobs of a run. Their cases are long rtlsim runs, so these two cells get a 60-minute shard budget instead of the default 150 minutes: both now plan four shards, balanced on the recorded runtimes (model_parity 54-58 min, perf_gate 50 min each). The shard count still comes from the planner and reaches pytest through --shard, so every shard derives the same split. Co-Authored-By: Claude Opus 5.5 --- ci/testcase.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/ci/testcase.py b/ci/testcase.py index f5e6a10717..3998238844 100644 --- a/ci/testcase.py +++ b/ci/testcase.py @@ -61,6 +61,8 @@ # into the tests job's timeout. RUNTIMES_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "runtimes.json") SHARD_BUDGET_S = 150 * 60 +# Cells of long rtlsim runs split finer: each shard of these stays near an hour. +CELL_SHARD_BUDGET_S = {"model_parity": 60 * 60, "perf_gate": 60 * 60} # Weight of a case with no recorded time (new, or not yet run on that xlen). DEFAULT_CASE_S = 120 @@ -421,19 +423,19 @@ def cell_name(case): return case.check or case.category -def shard_plan(case_ids, xlen, shards=None): +def shard_plan(case_ids, xlen, shards=None, budget=SHARD_BUDGET_S): """Split a cell's cases into shards balanced on their recorded runtimes. Longest case first onto the least-loaded shard, with ties broken on case id and shard index, so the planner and every shard derive the same split from the same ids. Unless `shards` is given, the count is the fewest that keep - the cell's recorded total within SHARD_BUDGET_S per shard. + the cell's recorded total within `budget` per shard. Returns (shards, {case id: 0-based shard}). """ runtimes = load_runtimes(xlen) weight = {i: runtimes.get(i, DEFAULT_CASE_S) for i in case_ids} if shards is None: - shards = max(1, math.ceil(sum(weight.values()) / SHARD_BUDGET_S)) + shards = max(1, math.ceil(sum(weight.values()) / budget)) load = [0.0] * shards plan = {} for i in sorted(weight, key=lambda i: (-weight[i], i)): @@ -472,7 +474,7 @@ def cmd_matrix(args): # the CHECK, not on a category that happens to share its name -- a check is a # marker across suites, and no category name may control whether it runs. # - # A cell whose recorded runtime exceeds SHARD_BUDGET_S is emitted as `shards` + # A cell whose recorded runtime exceeds its budget is emitted as `shards` # cells, each running shard `shard` of the split (shard_plan). The split is # over every case the cell's -m expression selects, not only those this # event's filters planned the cell for, because that is what the cell runs. @@ -499,7 +501,8 @@ def cmd_matrix(args): out = [] for key, cell in cells.items(): cell["needs"] = sorted(cell["needs"]) - shards, _ = shard_plan(members.get(key, []), key[2]) + shards, _ = shard_plan(members.get(key, []), key[2], + budget=CELL_SHARD_BUDGET_S.get(key[0], SHARD_BUDGET_S)) if shards == 1: out.append(cell) continue From 3500f0aa9d022ae5c414527a10e5f802b360a3ad Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 12:55:56 -0700 Subject: [PATCH 119/124] simx: keep a straddling instruction's low half until decode takes it The RVC decompressor joined a 32-bit instruction that straddles a word from the buffered low half and the refetched high half, and cleared the buffer as it did. When the decode latch was full the response stayed and was offered again, but the cleared buffer sent the retry down the fresh fetch path, which read the wrong half-word: at XLEN 64, raycast with RVC decoded a c.srli in place of an fmul.s, jumped to a wild PC and spun until the CI cell's 3-hour timeout. The RTL decompressor holds its state under backpressure. The buffer is now freed when the instruction enters the decode latch, so a retry sees the same state. Decode backpressure while a response waits arises on this branch once every warp's ibuffer is full and ready warps still fetch. rvc and simx: 5/5 at XLEN 64 and 32; raycast -w 64 -h 64 with RVC at XLEN 64 passes with master's instruction count (5,676,110). Non-RVC SimX cycles are unchanged (sgemm 796,238, raycast 21,106,995). Co-Authored-By: Claude Opus 5.5 --- sim/simx/core.cpp | 3 +++ sim/simx/decompressor.cpp | 5 ++++- sim/simx/decompressor.h | 6 ++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index 8e89758a9b..9aed139299 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -533,6 +533,9 @@ class Core::Impl { // queues a refetch internally for cross-word 32-bit completion. bool emit = decompressor->on_icache_rsp(trace, *mem_rsp.data); bool progressed = !emit || decode_latch.try_push(trace); + if (emit && progressed) { + decompressor->on_emit(trace); + } #else bool emit = true; bool progressed = false; diff --git a/sim/simx/decompressor.cpp b/sim/simx/decompressor.cpp index 0de1b35499..2c98736fc2 100644 --- a/sim/simx/decompressor.cpp +++ b/sim/simx/decompressor.cpp @@ -444,6 +444,10 @@ void Decompressor::commit_request(bool from_refetch) { } } +void Decompressor::on_emit(const instr_trace_t* trace) { + state_.at(trace->wid) = RvcSlot{}; +} + Word Decompressor::fetch_addr(const instr_trace_t* trace) const { const auto& rvc = state_.at(trace->wid); if (rvc.needs_second) { @@ -466,7 +470,6 @@ bool Decompressor::on_icache_rsp(instr_trace_t* trace, const mem_block_t& line) // Combine: low half from buffer, high half = low16 of new word. uint16_t high = word & 0xFFFFu; trace->code = (uint32_t(high) << 16) | rvc.low_half; - rvc = RvcSlot{}; return true; } diff --git a/sim/simx/decompressor.h b/sim/simx/decompressor.h index ea5ad0508d..2aed4892d1 100644 --- a/sim/simx/decompressor.h +++ b/sim/simx/decompressor.h @@ -72,8 +72,14 @@ class Decompressor : public SimObject { // 32-bit. The decoder later detects RVC from code[1:0] and // decompresses internally; // - queues `trace` for refetch internally and returns false. + // An emitted response may be refused by a full decode latch and offered + // again: the warp's buffered low half is kept until on_emit(). bool on_icache_rsp(instr_trace_t* trace, const mem_block_t& line); + // Notify that the instruction emitted for `trace` entered the decode + // latch, which frees the warp's half-word buffer. + void on_emit(const instr_trace_t* trace); + protected: void on_reset(); friend class SimObject; From 29c250d9a71fee2a71970afdb0860c511ca9a3f2 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 13:10:18 -0700 Subject: [PATCH 120/124] core: drop the MUFU instance the master merge left in VX_sfu_unit Merging master auto-merged its VX_mufu_unit instance into VX_sfu_unit, but on this branch the SFU's processing elements live in VX_sfu_block, which already instantiates MUFU. VX_sfu_unit has none of the PE interfaces the instance names, so any build with VX_CFG_EXT_MUFU_ENABLE failed to elaborate: parity-mufu on rtlsim and the mufu app in the rtlsim regression could not build. VX_sfu_unit is back to its pre-merge form, and the mufu cases now run when VX_sfu_block changes. mufu on simx, rtlsim and xrtsim: 9/9. parity-mufu 0.01%, -nt8 0.14%, -nt16 0.15%. mufu and zfhmin regression apps pass on rtlsim. Builds without MUFU are unchanged: the removed block was entirely under VX_CFG_EXT_MUFU_ENABLE. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/mufu.yaml | 2 +- hw/rtl/core/VX_sfu_unit.sv | 12 ------------ 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/ci/testcases/mufu.yaml b/ci/testcases/mufu.yaml index c7f70873b9..42f075d0a1 100644 --- a/ci/testcases/mufu.yaml +++ b/ci/testcases/mufu.yaml @@ -8,7 +8,7 @@ defaults: - 32 - 64 tier: smoke - touches: [hw/rtl/mufu, hw/rtl/core/VX_sfu_unit.sv, sim/simx/mufu_unit.cpp, sim/simx/sfu_unit.cpp, sw/common/mufu.cpp, sw/common/mufu_tables.h, sw/kernel/include/vx_mufu.h] + touches: [hw/rtl/mufu, hw/rtl/core/VX_sfu_unit.sv, hw/rtl/core/VX_sfu_block.sv, sim/simx/mufu_unit.cpp, sim/simx/sfu_unit.cpp, sw/common/mufu.cpp, sw/common/mufu_tables.h, sw/kernel/include/vx_mufu.h] tests: - id: mufu-1 via: blackbox diff --git a/hw/rtl/core/VX_sfu_unit.sv b/hw/rtl/core/VX_sfu_unit.sv index 820be2b1a1..59ba4c8dc3 100644 --- a/hw/rtl/core/VX_sfu_unit.sv +++ b/hw/rtl/core/VX_sfu_unit.sv @@ -132,18 +132,6 @@ module VX_sfu_unit import VX_gpu_pkg::*; #( ); `endif -`ifdef VX_CFG_EXT_MUFU_ENABLE - VX_mufu_unit #( - .INSTANCE_ID (`SFORMATF(("%s-mufu", INSTANCE_ID))), - .NUM_LANES (NUM_LANES) - ) mufu_unit ( - .clk (clk), - .reset (reset), - .execute_if (pe_execute_if[PE_IDX_MUFU]), - .result_if (pe_result_if[PE_IDX_MUFU]) - ); -`endif - VX_lane_gather #( .BLOCK_SIZE (BLOCK_SIZE), .NUM_LANES (NUM_LANES), From 48f4e425510a2ad94567b4ec8bddf3421fe51e3e Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 13:46:06 -0700 Subject: [PATCH 121/124] ci: re-record the perf_gate baselines after the master merge Regenerated with pytest ci -m perf_gate --update-baselines on the merged tree (CI toolchain, LLVM v3.0.1), in four shards whose records were combined case by case. Every case's config hash changes with the configuration keys master added; retired instructions and cycles are identical to the pre-merge baselines in all 49 cases. Against master's baselines the 47 comparable cases are 3.5% faster at the geometric mean. Co-Authored-By: Claude Opus 5.5 --- ci/baselines/perf/core.json | 32 +++++++++++++++---------------- ci/baselines/perf/dxa.json | 22 ++++++++++----------- ci/baselines/perf/graphics.json | 18 ++++++++--------- ci/baselines/perf/raytracing.json | 6 +++--- ci/baselines/perf/tensor.json | 10 +++++----- ci/baselines/perf/tensor_mx.json | 2 +- ci/baselines/perf/tensor_sp.json | 2 +- ci/baselines/perf/tensor_wg.json | 6 +++--- 8 files changed, 49 insertions(+), 49 deletions(-) diff --git a/ci/baselines/perf/core.json b/ci/baselines/perf/core.json index 5c239ea9bc..8257cb1e42 100644 --- a/ci/baselines/perf/core.json +++ b/ci/baselines/perf/core.json @@ -6,7 +6,7 @@ }, "app": "raycast", "args": "", - "config_hash": "969509e046ab06ca", + "config_hash": "b9d9ee09370158c4", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -17,7 +17,7 @@ }, "app": "raycast", "args": "", - "config_hash": "9fde6e6f7f37e2d4", + "config_hash": "d0bce0794d7bfdbc", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -28,7 +28,7 @@ }, "app": "sgemm", "args": "-n128", - "config_hash": "d063e6340ecd4137", + "config_hash": "14d91b803afc0b16", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=16 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, @@ -39,7 +39,7 @@ }, "app": "sgemm", "args": "-n128", - "config_hash": "15237d49dceccaee", + "config_hash": "ea174630d2a4f571", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, @@ -50,7 +50,7 @@ }, "app": "sgemm", "args": "-n128", - "config_hash": "aa47b9962f16edb2", + "config_hash": "912a20e26fc4d53b", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -61,7 +61,7 @@ }, "app": "sgemm", "args": "-n128", - "config_hash": "5ad71cacfd524b08", + "config_hash": "23fd0d2a32b65356", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -72,7 +72,7 @@ }, "app": "sgemmx", "args": "-n128", - "config_hash": "22fd9b73cb7d2ba8", + "config_hash": "0dae7e6b06c6facd", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -83,7 +83,7 @@ }, "app": "sgemmx", "args": "-n128", - "config_hash": "a14ffd5b6f36fa72", + "config_hash": "e2fe551b6ed5319a", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -94,7 +94,7 @@ }, "app": "sgemv", "args": "-m512 -n512", - "config_hash": "df3b981f27e8e320", + "config_hash": "6075eb2ef46a71f7", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -105,7 +105,7 @@ }, "app": "sgemv", "args": "-m512 -n512", - "config_hash": "3297af456dd6427a", + "config_hash": "800b8c9f7edc548a", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -116,7 +116,7 @@ }, "app": "softmax", "args": "-n64", - "config_hash": "558d5f9fda5e3deb", + "config_hash": "3580b572151bfcde", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -127,7 +127,7 @@ }, "app": "softmax", "args": "-n64", - "config_hash": "79a0a9910744e8d5", + "config_hash": "15f1137753d108a7", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -138,7 +138,7 @@ }, "app": "stencil3d", "args": "-n32", - "config_hash": "461212f296e5def6", + "config_hash": "a6ddac00bf4be79b", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -149,7 +149,7 @@ }, "app": "stencil3d", "args": "-n32", - "config_hash": "0556d0ca4d4eee68", + "config_hash": "63352c381701eb2a", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -160,7 +160,7 @@ }, "app": "vecadd", "args": "-n16384", - "config_hash": "36eb32e6e90447f5", + "config_hash": "d14a86b1d94d9967", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -171,7 +171,7 @@ }, "app": "vecadd", "args": "-n16384", - "config_hash": "36ba610d0e7114cb", + "config_hash": "dcca7a6e7c88ad28", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/dxa.json b/ci/baselines/perf/dxa.json index 35d5419be0..89fa130f36 100644 --- a/ci/baselines/perf/dxa.json +++ b/ci/baselines/perf/dxa.json @@ -6,7 +6,7 @@ }, "app": "dxa_copy_mcast", "args": "", - "config_hash": "c269fb880dec8b2c", + "config_hash": "b6b95c0822edf48b", "configs": "-DVX_CFG_EXT_DXA_ENABLE", "driver": "rtlsim" }, @@ -17,7 +17,7 @@ }, "app": "dxa_copy", "args": "-d1", - "config_hash": "ae2903a3d445d89b", + "config_hash": "a6138216544c603d", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -28,7 +28,7 @@ }, "app": "dxa_copy", "args": "-d1", - "config_hash": "3bf4cf26dbfc656b", + "config_hash": "461ebf20c7b39b93", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -39,7 +39,7 @@ }, "app": "sgemm2_dxa_mcast", "args": "-n128", - "config_hash": "662e5c2b0c7b8fc1", + "config_hash": "6d3b0c78bc6bce82", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -50,7 +50,7 @@ }, "app": "sgemm2_dxa_mcast", "args": "-n128", - "config_hash": "d6b2042d190ac5ce", + "config_hash": "44732317d093efc0", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -61,7 +61,7 @@ }, "app": "sgemm2_dxa", "args": "-n128 -t4 -m1", - "config_hash": "f292acf98eaaf8e3", + "config_hash": "ad5d3a1ba925762f", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -72,7 +72,7 @@ }, "app": "sgemm2_dxa", "args": "-n128 -t4 -m1", - "config_hash": "604991b30fdc7c3a", + "config_hash": "84141dd86fe75797", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -83,7 +83,7 @@ }, "app": "sgemm_tcu_wg_dxa_mcast", "args": "-m 128 -n 128 -k 64", - "config_hash": "2cc358de1155b70b", + "config_hash": "cb07e8565e928aa5", "configs": "-DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -94,7 +94,7 @@ }, "app": "sgemm_tcu_wg_dxa_mcast", "args": "-m 128 -n 128 -k 64", - "config_hash": "93e2a7ee4a7b218d", + "config_hash": "0f396d75e95e93fe", "configs": "-DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -105,7 +105,7 @@ }, "app": "sgemm_tcu_wg_dxa", "args": "-m 128 -n 128 -k 64", - "config_hash": "9106c858ba91e5ae", + "config_hash": "93664ea8afb18300", "configs": "-DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DWGMMA_NRC=16 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_THREADS=32", "driver": "rtlsim" }, @@ -116,7 +116,7 @@ }, "app": "sgemm_tcu_wg_sp_dxa", "args": "-m 128 -n 128 -k 128", - "config_hash": "2091dd383d7ad57e", + "config_hash": "73097ce78c7e35b3", "configs": "-DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_THREADS=32", "driver": "rtlsim" } diff --git a/ci/baselines/perf/graphics.json b/ci/baselines/perf/graphics.json index 32a26f3bb2..e0e39c0977 100644 --- a/ci/baselines/perf/graphics.json +++ b/ci/baselines/perf/graphics.json @@ -6,7 +6,7 @@ }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "b05e6b5e6b9ffc06", + "config_hash": "02ff3eb8cc7c891c", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, @@ -17,7 +17,7 @@ }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "6e98bbba0a7822d3", + "config_hash": "33bc329a5c31e08d", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -28,7 +28,7 @@ }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "208358458841bcde", + "config_hash": "bd063408e042748e", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -39,7 +39,7 @@ }, "app": "gfx_om", "args": "-w 128 -h 128 -r whitebox_128.png", - "config_hash": "9d0a1a59914e4ff0", + "config_hash": "a283b22b5fc238d4", "configs": "-DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -50,7 +50,7 @@ }, "app": "gfx_om", "args": "-w 128 -h 128 -r whitebox_128.png", - "config_hash": "bbaf6fac9da36d0f", + "config_hash": "5b99293235b7c2fa", "configs": "-DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -61,7 +61,7 @@ }, "app": "gfx_raster", "args": "-w 128 -h 128 -t triangle.cgltrace -r triangle_ref_128.png", - "config_hash": "6f2b1e9c2df8bb37", + "config_hash": "e6d78d84f54ee283", "configs": "-DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -72,7 +72,7 @@ }, "app": "gfx_raster", "args": "-w 128 -h 128 -t triangle.cgltrace -r triangle_ref_128.png", - "config_hash": "e3036cf5282b4118", + "config_hash": "15f7ab48d7b2edce", "configs": "-DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -83,7 +83,7 @@ }, "app": "gfx_tex", "args": "-i toad.png -r toad_ref_f0.png -f 0 -g 0", - "config_hash": "82f630774395560e", + "config_hash": "c808376069d5b819", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -94,7 +94,7 @@ }, "app": "gfx_tex", "args": "-i toad.png -r toad_ref_f0.png -f 0 -g 0", - "config_hash": "3a44bdd6d3900bb7", + "config_hash": "9d3f7a8db252c33d", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/raytracing.json b/ci/baselines/perf/raytracing.json index ae04bd9c87..a9a61b8666 100644 --- a/ci/baselines/perf/raytracing.json +++ b/ci/baselines/perf/raytracing.json @@ -6,7 +6,7 @@ }, "app": "rt_raycast", "args": "", - "config_hash": "68b742a1457ab4a3", + "config_hash": "75e25538007e3fd7", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, @@ -17,7 +17,7 @@ }, "app": "rt_raycast", "args": "", - "config_hash": "53b9672f25769532", + "config_hash": "4d66c19f8df23f2e", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -28,7 +28,7 @@ }, "app": "rt_raycast", "args": "", - "config_hash": "afe227c6459fe67e", + "config_hash": "fb0b2247eb1c36e6", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor.json b/ci/baselines/perf/tensor.json index 6678ef084d..89bfffe2f7 100644 --- a/ci/baselines/perf/tensor.json +++ b/ci/baselines/perf/tensor.json @@ -6,7 +6,7 @@ }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "e7b2b07afe27a336", + "config_hash": "51141703c93beb72", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" }, @@ -17,7 +17,7 @@ }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "60e7762f69f5cccf", + "config_hash": "8afdd549e2e91df6", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=16 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, @@ -28,7 +28,7 @@ }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "a70b10177d28a4d9", + "config_hash": "33b3e7a8d1fd702d", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, @@ -39,7 +39,7 @@ }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "55ac763cadf62490", + "config_hash": "0371a3688be32c73", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32", "driver": "rtlsim" }, @@ -50,7 +50,7 @@ }, "app": "sgemm2_tcu", "args": "-m 128 -n 128 -k 64", - "config_hash": "d9980e4be1d6630d", + "config_hash": "c410f503b55870cf", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_mx.json b/ci/baselines/perf/tensor_mx.json index 5d8ca88fab..c10ac69fc3 100644 --- a/ci/baselines/perf/tensor_mx.json +++ b/ci/baselines/perf/tensor_mx.json @@ -6,7 +6,7 @@ }, "app": "sgemm_tcu_mx", "args": "-m 128 -n 128 -k32", - "config_hash": "433197d2eaf189c5", + "config_hash": "31c2447f43720b96", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DITYPE=mxfp8 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_sp.json b/ci/baselines/perf/tensor_sp.json index 257076acad..9e03868a83 100644 --- a/ci/baselines/perf/tensor_sp.json +++ b/ci/baselines/perf/tensor_sp.json @@ -6,7 +6,7 @@ }, "app": "sgemm_tcu_sp", "args": "-m 128 -n 128 -k 128", - "config_hash": "200d2f36d7530327", + "config_hash": "069b267054841b9f", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_wg.json b/ci/baselines/perf/tensor_wg.json index 00f45bea82..690531b2c5 100644 --- a/ci/baselines/perf/tensor_wg.json +++ b/ci/baselines/perf/tensor_wg.json @@ -6,7 +6,7 @@ }, "app": "sgemm_tcu_wg", "args": "-m64 -n64 -k64", - "config_hash": "9a317f76414111c9", + "config_hash": "5a6929889f5a9768", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_TCU_FEDP2K -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DWGMMA_RS", "driver": "rtlsim" }, @@ -17,7 +17,7 @@ }, "app": "sgemm_tcu_wg", "args": "-m 128 -n 128 -k 128", - "config_hash": "a747dcd3d6a89d6b", + "config_hash": "531700427d640d57", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=32 -DWGMMA_SS -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" }, @@ -28,7 +28,7 @@ }, "app": "sgemm_tcu_wg_sp", "args": "-m 128 -n 128 -k 128", - "config_hash": "1e905ff223e20367", + "config_hash": "1bece58b69db57f0", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" } From 40ec5f4c989aec0f8451a688a34afa6199043ee8 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 14:38:40 -0700 Subject: [PATCH 122/124] core: read the RTU perf counters through the MPM DCR window The master merge brought in the RTU perf counters (#435), but the RTU class case in VX_csr_mpm was lost while resolving the branch's MPM rework. With PERF_ENABLE and the RTU on, sysmem_perf.rtu/rtcache went unread and lint failed (UNUSEDSIGNAL), which broke model_parity-rt_raycast. SimX and the runtime already read this class. Co-Authored-By: Claude Opus 5.5 --- hw/rtl/core/VX_csr_mpm.sv | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/hw/rtl/core/VX_csr_mpm.sv b/hw/rtl/core/VX_csr_mpm.sv index 9105e11851..4acdd07c1c 100644 --- a/hw/rtl/core/VX_csr_mpm.sv +++ b/hw/rtl/core/VX_csr_mpm.sv @@ -235,6 +235,21 @@ module VX_csr_mpm import VX_gpu_pkg::*; #( default:; endcase end + `endif + `ifdef VX_CFG_EXT_RTU_ENABLE + `VX_DCR_MPM_CLASS_RTU: begin + case (dcr_addr) + `CSR_READ_64(`VX_CSR_MPM_RTU_RAYS, dcr_data_w, sysmem_perf.rtu.rays); + `CSR_READ_64(`VX_CSR_MPM_RTU_HITS, dcr_data_w, sysmem_perf.rtu.hits); + `CSR_READ_64(`VX_CSR_MPM_RTU_BOX_TESTS, dcr_data_w, sysmem_perf.rtu.box_tests); + `CSR_READ_64(`VX_CSR_MPM_RTU_TRI_TESTS, dcr_data_w, sysmem_perf.rtu.tri_tests); + `CSR_READ_64(`VX_CSR_MPM_RTU_CALLBACKS, dcr_data_w, sysmem_perf.rtu.callbacks); + `CSR_READ_64(`VX_CSR_MPM_RTU_BUSY, dcr_data_w, sysmem_perf.rtu.busy); + `CSR_READ_64(`VX_CSR_MPM_RTCACHE_READS, dcr_data_w, sysmem_perf.rtcache.reads); + `CSR_READ_64(`VX_CSR_MPM_RTCACHE_MISS_R, dcr_data_w, sysmem_perf.rtcache.read_misses); + default:; + endcase + end `endif default:; endcase From ffd8a8666189d8950db8f8ba36808a82b8c11303 Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 17:41:17 -0700 Subject: [PATCH 123/124] AGENTS: drop the proposals section Proposals are working documents that are never committed, so docs/proposals/ has no tracked files and the rule sent every agent to write untracked drafts into its own worktree, where they scattered. Where contributors keep drafts is their own workflow, not a repo rule. The documentation map keeps docs/designs/ for accepted designs. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 1324ebf732..59757bf89d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -34,8 +34,7 @@ This is the canonical entry point for **both human contributors and AI coding ag - [CONTRIBUTING.md](CONTRIBUTING.md) — public fork/PR contribution flow - [docs/bug_fixes.md](docs/bug_fixes.md) — bug-fix discipline (root-cause vs patch) - [docs/continuous_integration.md](docs/continuous_integration.md) — CI pipeline -- [docs/proposals/](docs/proposals/) — design and migration proposals (drafts and in-progress) -- [docs/designs/](docs/designs/) — accepted designs (post-proposal, post-implementation) +- [docs/designs/](docs/designs/) — accepted designs (implemented) --- @@ -154,19 +153,11 @@ See [docs/coding_guidelines_cpp.md](docs/coding_guidelines_cpp.md) and [docs/cod --- -## 7. Proposals - -Design and migration proposals belong in [`docs/proposals/`](docs/proposals/) — never in build dirs or the repo root. A proposal captures the *why* of a non-trivial change before the code lands. - -Accepted designs that outlive their proposal phase live in [`docs/designs/`](docs/designs/). - ---- - -## 8. Living Document Policy +## 7. Living Document Policy This file is versioned alongside the code and evolves with it. Both humans and agents are expected to update it: - **Add a rule** when you discover a footgun the next contributor will hit. Include the *why*. - **Update or remove a rule** that's gone stale. A stale rule is worse than no rule. - **Don't duplicate** content from the topic docs in `docs/`. AGENTS.md links; the topic doc explains. -- **Branch-specific addenda are forbidden.** Branch context goes in a `docs/proposals/_proposal.md`, not in branch-local `AGENTS.md` variants. If the rule is general, promote it here; if it's a project-tracking detail, keep it in the proposal. +- **Branch-specific addenda are forbidden.** No branch-local `AGENTS.md` variants: if a rule is general, promote it here; project-tracking details stay out of the repo docs. From 3f25c838ca87a28318cc9c3f07181580743a7ecc Mon Sep 17 00:00:00 2001 From: tinebp Date: Mon, 26 Oct 2026 18:43:22 -0700 Subject: [PATCH 124/124] afu: pass the AXI memory ports to VX_afu_wrap as flat buses VX_afu_wrap took its AXI memory ports as SystemVerilog unpacked arrays, which plain Verilog cannot connect, so the platform tops (vortex_afu.v) were read as SystemVerilog. The Vitis kernel packager rejects an SystemVerilog top: it then refuses every SV interface with struct ports in the design, and no XRT bitstream could be packaged. Each AXI field now crosses into VX_afu_wrap as one flat vector with port i in slice i, and the wrapper splits it back into the per-port arrays its logic uses. The platform tops and the xrtsim shim are plain Verilog again, and the file-type overrides in the Vivado flows are gone. The port count still follows C_M_AXI_MEM_NUM_BANKS, so VX_afu_wrap builds standalone as the synthesis `top` DUT as before. Verified: xrtsim CI xrt cases 24/24; U55C configs on xrtsim (4c socket 4 + AMO + VM; 2c + TCU fp16 sparse; 1c IW4 + TCU sparse + WGMMA + DXA) pass vecadd, sgemm, amo, vm_test, sgemm_tcu(_sp), sgemm_tcu_wg_sp(_dxa) and dxa_copy; Vivado elaboration check and vortex_afu.xo packaging pass for the 4-core config (they failed before). Co-Authored-By: Claude Opus 5.5 --- hw/rtl/afu/aved/vortex_afu.v | 6 +- hw/rtl/afu/common/VX_afu_wrap.sv | 114 ++++++++++++++++------ hw/rtl/afu/common/vortex_afu.vh | 136 ++++++++++++++++----------- hw/rtl/afu/xrt/vortex_afu.v | 6 +- hw/scripts/xilinx_elab_check.tcl | 6 -- hw/syn/xilinx/aved/package_ip.tcl | 7 -- hw/syn/xilinx/dut/project.tcl | 6 -- hw/syn/xilinx/xrt/package_kernel.tcl | 6 -- sim/xrtsim/vortex_afu_shim.sv | 6 +- 9 files changed, 175 insertions(+), 118 deletions(-) diff --git a/hw/rtl/afu/aved/vortex_afu.v b/hw/rtl/afu/aved/vortex_afu.v index 6490e304ea..54f5a5f288 100644 --- a/hw/rtl/afu/aved/vortex_afu.v +++ b/hw/rtl/afu/aved/vortex_afu.v @@ -61,8 +61,8 @@ module vortex_afu #( output wire interrupt ); - `AXI_MEM_ARRAYS (C_M_AXI_MEM_NUM_BANKS) - `MP_REPEAT (`AFU_MEM_PORTS, AXI_MEM_TO_ARRAY, MP_SEMI); + `AXI_MEM_BUSES (C_M_AXI_MEM_NUM_BANKS) + `MP_REPEAT (`AFU_MEM_PORTS, AXI_MEM_TO_BUS, MP_SEMI); VX_afu_wrap #( .C_S_AXI_CTRL_ADDR_WIDTH (C_S_AXI_CONTROL_ADDR_WIDTH), @@ -74,7 +74,7 @@ module vortex_afu #( ) afu_wrap ( .clk (ap_clk), .reset (~ap_rst_n), - `AXI_MEM_ARRAY_ARGS, + `AXI_MEM_BUS_ARGS, `AXI_HOST_ARGS, .s_axi_ctrl_awvalid (s_axi_control_awvalid), .s_axi_ctrl_awready (s_axi_control_awready), diff --git a/hw/rtl/afu/common/VX_afu_wrap.sv b/hw/rtl/afu/common/VX_afu_wrap.sv index 2d3940170a..208d8b4542 100644 --- a/hw/rtl/afu/common/VX_afu_wrap.sv +++ b/hw/rtl/afu/common/VX_afu_wrap.sv @@ -48,36 +48,36 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( input wire clk, input wire reset, - // AXI4 memory masters - output wire m_axi_mem_awvalid_a [C_M_AXI_MEM_NUM_BANKS], - input wire m_axi_mem_awready_a [C_M_AXI_MEM_NUM_BANKS], - output wire [C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_awaddr_a [C_M_AXI_MEM_NUM_BANKS], - output wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_awid_a [C_M_AXI_MEM_NUM_BANKS], - output wire [7:0] m_axi_mem_awlen_a [C_M_AXI_MEM_NUM_BANKS], - - output wire m_axi_mem_wvalid_a [C_M_AXI_MEM_NUM_BANKS], - input wire m_axi_mem_wready_a [C_M_AXI_MEM_NUM_BANKS], - output wire [C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_wdata_a [C_M_AXI_MEM_NUM_BANKS], - output wire [C_M_AXI_MEM_DATA_WIDTH/8-1:0] m_axi_mem_wstrb_a [C_M_AXI_MEM_NUM_BANKS], - output wire m_axi_mem_wlast_a [C_M_AXI_MEM_NUM_BANKS], - - input wire m_axi_mem_bvalid_a [C_M_AXI_MEM_NUM_BANKS], - output wire m_axi_mem_bready_a [C_M_AXI_MEM_NUM_BANKS], - input wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_bid_a [C_M_AXI_MEM_NUM_BANKS], - input wire [1:0] m_axi_mem_bresp_a [C_M_AXI_MEM_NUM_BANKS], - - output wire m_axi_mem_arvalid_a [C_M_AXI_MEM_NUM_BANKS], - input wire m_axi_mem_arready_a [C_M_AXI_MEM_NUM_BANKS], - output wire [C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_araddr_a [C_M_AXI_MEM_NUM_BANKS], - output wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_arid_a [C_M_AXI_MEM_NUM_BANKS], - output wire [7:0] m_axi_mem_arlen_a [C_M_AXI_MEM_NUM_BANKS], - - input wire m_axi_mem_rvalid_a [C_M_AXI_MEM_NUM_BANKS], - output wire m_axi_mem_rready_a [C_M_AXI_MEM_NUM_BANKS], - input wire [C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_rdata_a [C_M_AXI_MEM_NUM_BANKS], - input wire m_axi_mem_rlast_a [C_M_AXI_MEM_NUM_BANKS], - input wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_rid_a [C_M_AXI_MEM_NUM_BANKS], - input wire [1:0] m_axi_mem_rresp_a [C_M_AXI_MEM_NUM_BANKS], + // AXI4 memory masters, port i in slice i of each bus + output wire [C_M_AXI_MEM_NUM_BANKS-1:0] m_axi_mem_awvalid_bus, + input wire [C_M_AXI_MEM_NUM_BANKS-1:0] m_axi_mem_awready_bus, + output wire [C_M_AXI_MEM_NUM_BANKS*C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_awaddr_bus, + output wire [C_M_AXI_MEM_NUM_BANKS*C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_awid_bus, + output wire [C_M_AXI_MEM_NUM_BANKS*8-1:0] m_axi_mem_awlen_bus, + + output wire [C_M_AXI_MEM_NUM_BANKS-1:0] m_axi_mem_wvalid_bus, + input wire [C_M_AXI_MEM_NUM_BANKS-1:0] m_axi_mem_wready_bus, + output wire [C_M_AXI_MEM_NUM_BANKS*C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_wdata_bus, + output wire [C_M_AXI_MEM_NUM_BANKS*(C_M_AXI_MEM_DATA_WIDTH/8)-1:0] m_axi_mem_wstrb_bus, + output wire [C_M_AXI_MEM_NUM_BANKS-1:0] m_axi_mem_wlast_bus, + + input wire [C_M_AXI_MEM_NUM_BANKS-1:0] m_axi_mem_bvalid_bus, + output wire [C_M_AXI_MEM_NUM_BANKS-1:0] m_axi_mem_bready_bus, + input wire [C_M_AXI_MEM_NUM_BANKS*C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_bid_bus, + input wire [C_M_AXI_MEM_NUM_BANKS*2-1:0] m_axi_mem_bresp_bus, + + output wire [C_M_AXI_MEM_NUM_BANKS-1:0] m_axi_mem_arvalid_bus, + input wire [C_M_AXI_MEM_NUM_BANKS-1:0] m_axi_mem_arready_bus, + output wire [C_M_AXI_MEM_NUM_BANKS*C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_araddr_bus, + output wire [C_M_AXI_MEM_NUM_BANKS*C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_arid_bus, + output wire [C_M_AXI_MEM_NUM_BANKS*8-1:0] m_axi_mem_arlen_bus, + + input wire [C_M_AXI_MEM_NUM_BANKS-1:0] m_axi_mem_rvalid_bus, + output wire [C_M_AXI_MEM_NUM_BANKS-1:0] m_axi_mem_rready_bus, + input wire [C_M_AXI_MEM_NUM_BANKS*C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_rdata_bus, + input wire [C_M_AXI_MEM_NUM_BANKS-1:0] m_axi_mem_rlast_bus, + input wire [C_M_AXI_MEM_NUM_BANKS*C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_rid_bus, + input wire [C_M_AXI_MEM_NUM_BANKS*2-1:0] m_axi_mem_rresp_bus, // AXI4 host-memory master interface (CP command ring + host side of DMA) `GEN_AXI_HOST, @@ -113,6 +113,60 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( `STATIC_ASSERT(C_M_AXI_MEM_NUM_BANKS == `AFU_MEM_PORTS_OF(VX_MEM_PORTS), ("AXI memory port count %0d does not match the GPU's %0d", C_M_AXI_MEM_NUM_BANKS, `AFU_MEM_PORTS_OF(VX_MEM_PORTS))) + wire m_axi_mem_awvalid_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_awready_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_awaddr_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_awid_a [C_M_AXI_MEM_NUM_BANKS]; + wire [7:0] m_axi_mem_awlen_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_wvalid_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_wready_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_wdata_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_DATA_WIDTH/8-1:0] m_axi_mem_wstrb_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_wlast_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_bvalid_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_bready_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_bid_a [C_M_AXI_MEM_NUM_BANKS]; + wire [1:0] m_axi_mem_bresp_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_arvalid_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_arready_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_araddr_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_arid_a [C_M_AXI_MEM_NUM_BANKS]; + wire [7:0] m_axi_mem_arlen_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_rvalid_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_rready_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_rdata_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_rlast_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_rid_a [C_M_AXI_MEM_NUM_BANKS]; + wire [1:0] m_axi_mem_rresp_a [C_M_AXI_MEM_NUM_BANKS]; + + for (genvar i = 0; i < C_M_AXI_MEM_NUM_BANKS; ++i) begin : g_axi_mem_bus + assign m_axi_mem_awvalid_bus[i] = m_axi_mem_awvalid_a[i]; + assign m_axi_mem_awready_a[i] = m_axi_mem_awready_bus[i]; + assign m_axi_mem_awaddr_bus[i * C_M_AXI_MEM_ADDR_WIDTH +: C_M_AXI_MEM_ADDR_WIDTH] = m_axi_mem_awaddr_a[i]; + assign m_axi_mem_awid_bus[i * C_M_AXI_MEM_ID_WIDTH +: C_M_AXI_MEM_ID_WIDTH] = m_axi_mem_awid_a[i]; + assign m_axi_mem_awlen_bus[i * 8 +: 8] = m_axi_mem_awlen_a[i]; + assign m_axi_mem_wvalid_bus[i] = m_axi_mem_wvalid_a[i]; + assign m_axi_mem_wready_a[i] = m_axi_mem_wready_bus[i]; + assign m_axi_mem_wdata_bus[i * C_M_AXI_MEM_DATA_WIDTH +: C_M_AXI_MEM_DATA_WIDTH] = m_axi_mem_wdata_a[i]; + assign m_axi_mem_wstrb_bus[i * (C_M_AXI_MEM_DATA_WIDTH/8) +: (C_M_AXI_MEM_DATA_WIDTH/8)] = m_axi_mem_wstrb_a[i]; + assign m_axi_mem_wlast_bus[i] = m_axi_mem_wlast_a[i]; + assign m_axi_mem_bvalid_a[i] = m_axi_mem_bvalid_bus[i]; + assign m_axi_mem_bready_bus[i] = m_axi_mem_bready_a[i]; + assign m_axi_mem_bid_a[i] = m_axi_mem_bid_bus[i * C_M_AXI_MEM_ID_WIDTH +: C_M_AXI_MEM_ID_WIDTH]; + assign m_axi_mem_bresp_a[i] = m_axi_mem_bresp_bus[i * 2 +: 2]; + assign m_axi_mem_arvalid_bus[i] = m_axi_mem_arvalid_a[i]; + assign m_axi_mem_arready_a[i] = m_axi_mem_arready_bus[i]; + assign m_axi_mem_araddr_bus[i * C_M_AXI_MEM_ADDR_WIDTH +: C_M_AXI_MEM_ADDR_WIDTH] = m_axi_mem_araddr_a[i]; + assign m_axi_mem_arid_bus[i * C_M_AXI_MEM_ID_WIDTH +: C_M_AXI_MEM_ID_WIDTH] = m_axi_mem_arid_a[i]; + assign m_axi_mem_arlen_bus[i * 8 +: 8] = m_axi_mem_arlen_a[i]; + assign m_axi_mem_rvalid_a[i] = m_axi_mem_rvalid_bus[i]; + assign m_axi_mem_rready_bus[i] = m_axi_mem_rready_a[i]; + assign m_axi_mem_rdata_a[i] = m_axi_mem_rdata_bus[i * C_M_AXI_MEM_DATA_WIDTH +: C_M_AXI_MEM_DATA_WIDTH]; + assign m_axi_mem_rlast_a[i] = m_axi_mem_rlast_bus[i]; + assign m_axi_mem_rid_a[i] = m_axi_mem_rid_bus[i * C_M_AXI_MEM_ID_WIDTH +: C_M_AXI_MEM_ID_WIDTH]; + assign m_axi_mem_rresp_a[i] = m_axi_mem_rresp_bus[i * 2 +: 2]; + end + reg [`VX_CFG_RESET_DELAY-1:0] vx_reset_shift_r; wire vx_reset; wire vx_start; diff --git a/hw/rtl/afu/common/vortex_afu.vh b/hw/rtl/afu/common/vortex_afu.vh index 4ff28ab33f..c9a6c5075e 100644 --- a/hw/rtl/afu/common/vortex_afu.vh +++ b/hw/rtl/afu/common/vortex_afu.vh @@ -163,6 +163,33 @@ assign m_axi_mem_bresp_a[i] = m_axi_mem_``i``_bresp; \ assign m_axi_mem_bid_a[i] = m_axi_mem_``i``_bid +`define AXI_MEM_TO_BUS(i) \ + assign m_axi_mem_``i``_awvalid = m_axi_mem_awvalid_bus[i]; \ + assign m_axi_mem_awready_bus[i] = m_axi_mem_``i``_awready; \ + assign m_axi_mem_``i``_awaddr = m_axi_mem_awaddr_bus[(i)*C_M_AXI_MEM_ADDR_WIDTH +: C_M_AXI_MEM_ADDR_WIDTH]; \ + assign m_axi_mem_``i``_awid = m_axi_mem_awid_bus[(i)*C_M_AXI_MEM_ID_WIDTH +: C_M_AXI_MEM_ID_WIDTH]; \ + assign m_axi_mem_``i``_awlen = m_axi_mem_awlen_bus[(i)*8 +: 8]; \ + assign m_axi_mem_``i``_wvalid = m_axi_mem_wvalid_bus[i]; \ + assign m_axi_mem_wready_bus[i] = m_axi_mem_``i``_wready; \ + assign m_axi_mem_``i``_wdata = m_axi_mem_wdata_bus[(i)*C_M_AXI_MEM_DATA_WIDTH +: C_M_AXI_MEM_DATA_WIDTH]; \ + assign m_axi_mem_``i``_wstrb = m_axi_mem_wstrb_bus[(i)*(C_M_AXI_MEM_DATA_WIDTH/8) +: (C_M_AXI_MEM_DATA_WIDTH/8)]; \ + assign m_axi_mem_``i``_wlast = m_axi_mem_wlast_bus[i]; \ + assign m_axi_mem_``i``_arvalid = m_axi_mem_arvalid_bus[i]; \ + assign m_axi_mem_arready_bus[i] = m_axi_mem_``i``_arready; \ + assign m_axi_mem_``i``_araddr = m_axi_mem_araddr_bus[(i)*C_M_AXI_MEM_ADDR_WIDTH +: C_M_AXI_MEM_ADDR_WIDTH]; \ + assign m_axi_mem_``i``_arid = m_axi_mem_arid_bus[(i)*C_M_AXI_MEM_ID_WIDTH +: C_M_AXI_MEM_ID_WIDTH]; \ + assign m_axi_mem_``i``_arlen = m_axi_mem_arlen_bus[(i)*8 +: 8]; \ + assign m_axi_mem_rvalid_bus[i] = m_axi_mem_``i``_rvalid; \ + assign m_axi_mem_``i``_rready = m_axi_mem_rready_bus[i]; \ + assign m_axi_mem_rdata_bus[(i)*C_M_AXI_MEM_DATA_WIDTH +: C_M_AXI_MEM_DATA_WIDTH] = m_axi_mem_``i``_rdata; \ + assign m_axi_mem_rlast_bus[i] = m_axi_mem_``i``_rlast; \ + assign m_axi_mem_rid_bus[(i)*C_M_AXI_MEM_ID_WIDTH +: C_M_AXI_MEM_ID_WIDTH] = m_axi_mem_``i``_rid; \ + assign m_axi_mem_rresp_bus[(i)*2 +: 2] = m_axi_mem_``i``_rresp; \ + assign m_axi_mem_bvalid_bus[i] = m_axi_mem_``i``_bvalid; \ + assign m_axi_mem_``i``_bready = m_axi_mem_bready_bus[i]; \ + assign m_axi_mem_bresp_bus[(i)*2 +: 2] = m_axi_mem_``i``_bresp; \ + assign m_axi_mem_bid_bus[(i)*C_M_AXI_MEM_ID_WIDTH +: C_M_AXI_MEM_ID_WIDTH] = m_axi_mem_``i``_bid + // AXI memory masters the AFU exposes: one per GPU memory port, rounded up to // a power of two so that address bits select the port, and at most one per // platform memory channel. A platform top must spell its ports out, so the @@ -170,61 +197,62 @@ // (hw/scripts/afu_mem_ports.sh) and VX_afu_wrap checks that the two agree. `define AFU_MEM_PORTS_OF(n) `MIN(1 << `CLOG2(n), `VX_CFG_PLATFORM_MEMORY_NUM_BANKS) -// Per-port arrays connecting a platform top's flat m_axi_mem_ ports to -// VX_afu_wrap (see AXI_MEM_TO_ARRAY). -`define AXI_MEM_ARRAYS(n) \ - wire m_axi_mem_awvalid_a [n]; \ - wire m_axi_mem_awready_a [n]; \ - wire [C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_awaddr_a [n]; \ - wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_awid_a [n]; \ - wire [7:0] m_axi_mem_awlen_a [n]; \ - wire m_axi_mem_wvalid_a [n]; \ - wire m_axi_mem_wready_a [n]; \ - wire [C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_wdata_a [n]; \ - wire [C_M_AXI_MEM_DATA_WIDTH/8-1:0] m_axi_mem_wstrb_a [n]; \ - wire m_axi_mem_wlast_a [n]; \ - wire m_axi_mem_arvalid_a [n]; \ - wire m_axi_mem_arready_a [n]; \ - wire [C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_araddr_a [n]; \ - wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_arid_a [n]; \ - wire [7:0] m_axi_mem_arlen_a [n]; \ - wire m_axi_mem_rvalid_a [n]; \ - wire m_axi_mem_rready_a [n]; \ - wire [C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_rdata_a [n]; \ - wire m_axi_mem_rlast_a [n]; \ - wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_rid_a [n]; \ - wire [1:0] m_axi_mem_rresp_a [n]; \ - wire m_axi_mem_bvalid_a [n]; \ - wire m_axi_mem_bready_a [n]; \ - wire [1:0] m_axi_mem_bresp_a [n]; \ - wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_bid_a [n]; +// Flat per-field buses connecting a platform top's m_axi_mem_ ports to +// VX_afu_wrap, port i in slice i. They are plain Verilog, so the platform +// tops stay Verilog, which the Vitis and AVED kernel packagers require. +`define AXI_MEM_BUSES(n) \ + wire [(n)-1:0] m_axi_mem_awvalid_bus; \ + wire [(n)-1:0] m_axi_mem_awready_bus; \ + wire [(n)*C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_awaddr_bus; \ + wire [(n)*C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_awid_bus; \ + wire [(n)*8-1:0] m_axi_mem_awlen_bus; \ + wire [(n)-1:0] m_axi_mem_wvalid_bus; \ + wire [(n)-1:0] m_axi_mem_wready_bus; \ + wire [(n)*C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_wdata_bus; \ + wire [(n)*(C_M_AXI_MEM_DATA_WIDTH/8)-1:0] m_axi_mem_wstrb_bus; \ + wire [(n)-1:0] m_axi_mem_wlast_bus; \ + wire [(n)-1:0] m_axi_mem_arvalid_bus; \ + wire [(n)-1:0] m_axi_mem_arready_bus; \ + wire [(n)*C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_araddr_bus; \ + wire [(n)*C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_arid_bus; \ + wire [(n)*8-1:0] m_axi_mem_arlen_bus; \ + wire [(n)-1:0] m_axi_mem_rvalid_bus; \ + wire [(n)-1:0] m_axi_mem_rready_bus; \ + wire [(n)*C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_rdata_bus; \ + wire [(n)-1:0] m_axi_mem_rlast_bus; \ + wire [(n)*C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_rid_bus; \ + wire [(n)*2-1:0] m_axi_mem_rresp_bus; \ + wire [(n)-1:0] m_axi_mem_bvalid_bus; \ + wire [(n)-1:0] m_axi_mem_bready_bus; \ + wire [(n)*2-1:0] m_axi_mem_bresp_bus; \ + wire [(n)*C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_bid_bus; -`define AXI_MEM_ARRAY_ARGS \ - .m_axi_mem_awvalid_a(m_axi_mem_awvalid_a), \ - .m_axi_mem_awready_a(m_axi_mem_awready_a), \ - .m_axi_mem_awaddr_a(m_axi_mem_awaddr_a), \ - .m_axi_mem_awid_a(m_axi_mem_awid_a), \ - .m_axi_mem_awlen_a(m_axi_mem_awlen_a), \ - .m_axi_mem_wvalid_a(m_axi_mem_wvalid_a), \ - .m_axi_mem_wready_a(m_axi_mem_wready_a), \ - .m_axi_mem_wdata_a(m_axi_mem_wdata_a), \ - .m_axi_mem_wstrb_a(m_axi_mem_wstrb_a), \ - .m_axi_mem_wlast_a(m_axi_mem_wlast_a), \ - .m_axi_mem_arvalid_a(m_axi_mem_arvalid_a), \ - .m_axi_mem_arready_a(m_axi_mem_arready_a), \ - .m_axi_mem_araddr_a(m_axi_mem_araddr_a), \ - .m_axi_mem_arid_a(m_axi_mem_arid_a), \ - .m_axi_mem_arlen_a(m_axi_mem_arlen_a), \ - .m_axi_mem_rvalid_a(m_axi_mem_rvalid_a), \ - .m_axi_mem_rready_a(m_axi_mem_rready_a), \ - .m_axi_mem_rdata_a(m_axi_mem_rdata_a), \ - .m_axi_mem_rlast_a(m_axi_mem_rlast_a), \ - .m_axi_mem_rid_a(m_axi_mem_rid_a), \ - .m_axi_mem_rresp_a(m_axi_mem_rresp_a), \ - .m_axi_mem_bvalid_a(m_axi_mem_bvalid_a), \ - .m_axi_mem_bready_a(m_axi_mem_bready_a), \ - .m_axi_mem_bresp_a(m_axi_mem_bresp_a), \ - .m_axi_mem_bid_a(m_axi_mem_bid_a) +`define AXI_MEM_BUS_ARGS \ + .m_axi_mem_awvalid_bus(m_axi_mem_awvalid_bus), \ + .m_axi_mem_awready_bus(m_axi_mem_awready_bus), \ + .m_axi_mem_awaddr_bus(m_axi_mem_awaddr_bus), \ + .m_axi_mem_awid_bus(m_axi_mem_awid_bus), \ + .m_axi_mem_awlen_bus(m_axi_mem_awlen_bus), \ + .m_axi_mem_wvalid_bus(m_axi_mem_wvalid_bus), \ + .m_axi_mem_wready_bus(m_axi_mem_wready_bus), \ + .m_axi_mem_wdata_bus(m_axi_mem_wdata_bus), \ + .m_axi_mem_wstrb_bus(m_axi_mem_wstrb_bus), \ + .m_axi_mem_wlast_bus(m_axi_mem_wlast_bus), \ + .m_axi_mem_arvalid_bus(m_axi_mem_arvalid_bus), \ + .m_axi_mem_arready_bus(m_axi_mem_arready_bus), \ + .m_axi_mem_araddr_bus(m_axi_mem_araddr_bus), \ + .m_axi_mem_arid_bus(m_axi_mem_arid_bus), \ + .m_axi_mem_arlen_bus(m_axi_mem_arlen_bus), \ + .m_axi_mem_rvalid_bus(m_axi_mem_rvalid_bus), \ + .m_axi_mem_rready_bus(m_axi_mem_rready_bus), \ + .m_axi_mem_rdata_bus(m_axi_mem_rdata_bus), \ + .m_axi_mem_rlast_bus(m_axi_mem_rlast_bus), \ + .m_axi_mem_rid_bus(m_axi_mem_rid_bus), \ + .m_axi_mem_rresp_bus(m_axi_mem_rresp_bus), \ + .m_axi_mem_bvalid_bus(m_axi_mem_bvalid_bus), \ + .m_axi_mem_bready_bus(m_axi_mem_bready_bus), \ + .m_axi_mem_bresp_bus(m_axi_mem_bresp_bus), \ + .m_axi_mem_bid_bus(m_axi_mem_bid_bus) `include "VX_define.vh" diff --git a/hw/rtl/afu/xrt/vortex_afu.v b/hw/rtl/afu/xrt/vortex_afu.v index 5e37bd818c..05863d8791 100644 --- a/hw/rtl/afu/xrt/vortex_afu.v +++ b/hw/rtl/afu/xrt/vortex_afu.v @@ -67,8 +67,8 @@ module vortex_afu #( reset_r <= ~ap_rst_n; end - `AXI_MEM_ARRAYS (C_M_AXI_MEM_NUM_BANKS) - `MP_REPEAT (`AFU_MEM_PORTS, AXI_MEM_TO_ARRAY, MP_SEMI); + `AXI_MEM_BUSES (C_M_AXI_MEM_NUM_BANKS) + `MP_REPEAT (`AFU_MEM_PORTS, AXI_MEM_TO_BUS, MP_SEMI); VX_afu_wrap #( .C_S_AXI_CTRL_ADDR_WIDTH (C_S_AXI_CTRL_ADDR_WIDTH), @@ -80,7 +80,7 @@ module vortex_afu #( ) afu_wrap ( .clk (ap_clk), .reset (reset_r), - `AXI_MEM_ARRAY_ARGS, + `AXI_MEM_BUS_ARGS, `AXI_HOST_ARGS, .s_axi_ctrl_awvalid (s_axi_ctrl_awvalid), .s_axi_ctrl_awready (s_axi_ctrl_awready), diff --git a/hw/scripts/xilinx_elab_check.tcl b/hw/scripts/xilinx_elab_check.tcl index 39ce791404..8933471fbf 100644 --- a/hw/scripts/xilinx_elab_check.tcl +++ b/hw/scripts/xilinx_elab_check.tcl @@ -46,12 +46,6 @@ if {$mic_mode eq "off" || $mic_mode eq "0"} { create_project -in_memory add_files -norecurse ${mic_sources} - # The platform tops (vortex_afu.v) pass unpacked per-port arrays to - # VX_afu_wrap, which needs SystemVerilog. - set mic_afu_tops [get_files -quiet */vortex_afu.v] - if {[llength $mic_afu_tops] > 0} { - set_property file_type SystemVerilog $mic_afu_tops - } # Black-box every generated vendor IP via its BLOCK_STUB so a -rtl elaboration # resolves the IP instances (FPU FMA/DIV/SQRT/MUL/ADD, etc.) without pulling diff --git a/hw/syn/xilinx/aved/package_ip.tcl b/hw/syn/xilinx/aved/package_ip.tcl index 4df28092a9..0e6bdb33cc 100644 --- a/hw/syn/xilinx/aved/package_ip.tcl +++ b/hw/syn/xilinx/aved/package_ip.tcl @@ -63,13 +63,6 @@ foreach line [split $data "\n"] { } } -# The platform tops (vortex_afu.v) pass unpacked per-port arrays to -# VX_afu_wrap, which needs SystemVerilog. -set afu_tops [get_files -quiet */vortex_afu.v] -if {[llength $afu_tops] > 0} { - set_property file_type SystemVerilog $afu_tops -} - # Pull in the floating-point cores generated above so they are packaged # alongside the RTL. The set matches the XRT flow's package_kernel.tcl. set ip_files [list] diff --git a/hw/syn/xilinx/dut/project.tcl b/hw/syn/xilinx/dut/project.tcl index 9f512e3f94..56339544ef 100644 --- a/hw/syn/xilinx/dut/project.tcl +++ b/hw/syn/xilinx/dut/project.tcl @@ -173,12 +173,6 @@ proc run_setup {} { # Add the design sources add_files -norecurse -verbose $vsources_list - # The platform tops (vortex_afu.v) pass unpacked per-port arrays to - # VX_afu_wrap, which needs SystemVerilog. - set afu_tops [get_files -quiet */vortex_afu.v] - if {[llength $afu_tops] > 0} { - set_property file_type SystemVerilog $afu_tops - } # process defines set_property verilog_define ${vdefines_list} [current_fileset] diff --git a/hw/syn/xilinx/xrt/package_kernel.tcl b/hw/syn/xilinx/xrt/package_kernel.tcl index c9b0ab20cb..d9b9777c72 100644 --- a/hw/syn/xilinx/xrt/package_kernel.tcl +++ b/hw/syn/xilinx/xrt/package_kernel.tcl @@ -62,12 +62,6 @@ foreach def $vdefines_list { create_project -force kernel_pack $path_to_tmp_project add_files -norecurse ${vsources_list} -# The platform tops (vortex_afu.v) pass unpacked per-port arrays to -# VX_afu_wrap, which needs SystemVerilog. -set afu_tops [get_files -quiet */vortex_afu.v] -if {[llength $afu_tops] > 0} { - set_property file_type SystemVerilog $afu_tops -} set obj [get_filesets sources_1] set ip_files [list \ diff --git a/sim/xrtsim/vortex_afu_shim.sv b/sim/xrtsim/vortex_afu_shim.sv index 73a8b8f071..d6a453507d 100644 --- a/sim/xrtsim/vortex_afu_shim.sv +++ b/sim/xrtsim/vortex_afu_shim.sv @@ -54,8 +54,8 @@ module vortex_afu_shim #( output wire interrupt `IGNORE_WARNINGS_END ); - `AXI_MEM_ARRAYS (C_M_AXI_MEM_NUM_BANKS) - `MP_REPEAT (`AFU_MEM_PORTS, AXI_MEM_TO_ARRAY, MP_SEMI); + `AXI_MEM_BUSES (C_M_AXI_MEM_NUM_BANKS) + `MP_REPEAT (`AFU_MEM_PORTS, AXI_MEM_TO_BUS, MP_SEMI); VX_afu_wrap #( .C_S_AXI_CTRL_ADDR_WIDTH (C_S_AXI_CTRL_ADDR_WIDTH), @@ -68,7 +68,7 @@ module vortex_afu_shim #( .clk (ap_clk), .reset (~ap_rst_n), - `AXI_MEM_ARRAY_ARGS, + `AXI_MEM_BUS_ARGS, `AXI_HOST_ARGS,