From 190caf44ceac3b7ad63961218d1e9bb657565c2e Mon Sep 17 00:00:00 2001 From: Daniel Byrne Date: Wed, 21 Oct 2026 09:49:18 -0700 Subject: [PATCH] get_wq: round robin per thread instead of through a shared counter get_wq() picked a work queue with next_wq++ on a global atomic (and on a per-device atomic in NUMA-aware mode), so every DSA submit on every thread did a locked read-modify-write on one shared cache line. Each thread now keeps its own round-robin position. A thread takes its starting position once from a shared counter, so concurrent threads still spread across the queues, and from then on picking a queue touches only thread-local state. The position is 32 bits, so the rotation no longer skews each time the old atomic_uchar counter wrapped at 256 with a queue count that does not divide it. Measured on a Xeon 6787P with four DSA devices on one node, one shared WQ each: N pinned threads each doing back-to-back memcpy of one size through LD_PRELOAD, auto-tuning off, ns per call (median of 3): 4 KiB 16 threads 2295 -> 1432 32 threads 6629 -> 1549 16 KiB 32 threads 5920 -> 2991 64 KiB 32 threads 16970 -> 12124 Single-thread and device-bound cases are unchanged within noise, and the descriptors stay evenly spread: DSA perfmon counted the same number on each of the four devices in every NUMA mode, before and after. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Mv7xKtQPkGoXQUvvuuhAEz --- dto.c | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/dto.c b/dto.c index 5eb211e..458b607 100644 --- a/dto.c +++ b/dto.c @@ -91,7 +91,6 @@ struct dto_wq { struct dto_device { struct dto_wq* wqs[MAX_WQS]; uint8_t num_wqs; - atomic_uchar next_wq; }; #endif @@ -127,7 +126,11 @@ static struct dto_wq wqs[MAX_WQS]; static struct dto_device* devices[MAX_NUMA_NODES]; #endif static uint8_t num_wqs; -static atomic_uchar next_wq; +/* round-robin start position for the next thread that picks a WQ (get_wq) */ +static atomic_uint next_thread_wq; +/* this thread's round-robin position */ +static __thread unsigned int thr_wq_rr; +static __thread bool thr_wq_rr_set; static atomic_uchar dto_initialized; static atomic_uchar dto_initializing; static uint8_t use_std_lib_calls; @@ -1603,6 +1606,21 @@ static void cleanup_dto(void) #endif } +/* Next round-robin position for this thread. Each thread keeps its own, so + * picking a WQ writes no cache line shared with other threads; a counter + * incremented by every submit on every thread was measurable on small + * operations. Threads start at successive positions, so concurrent threads + * still spread across the WQs. */ +static __always_inline unsigned int thread_wq_rr(void) +{ + if (unlikely(!thr_wq_rr_set)) { + thr_wq_rr = atomic_fetch_add_explicit(&next_thread_wq, 1, + memory_order_relaxed); + thr_wq_rr_set = true; + } + return thr_wq_rr++; +} + static __always_inline struct dto_wq *get_wq(void* buf) { struct dto_wq* wq = NULL; @@ -1617,14 +1635,14 @@ static __always_inline struct dto_wq *get_wq(void* buf) struct dto_device* dev = devices[numa_node]; if (dev != NULL && dev->num_wqs > 0) { - wq = dev->wqs[dev->next_wq++ % dev->num_wqs]; + wq = dev->wqs[thread_wq_rr() % dev->num_wqs]; } } } #endif if (wq == NULL) { - wq = &wqs[next_wq++ % num_wqs]; + wq = &wqs[thread_wq_rr() % num_wqs]; } return wq;