From 6d837bb70ed979a7b0f494d4edac1ad10a9caf32 Mon Sep 17 00:00:00 2001 From: sb32445 Date: Mon, 16 Nov 2026 14:14:43 +0100 Subject: [PATCH] cuda: fuse SwiGLU + sign flip + Hadamard transform for PTQ1_0 prompt batches, and quantize in the same launch for the MMQ The FFN down input of a Hadamard-folded model is SWIGLU(gate, up) -> MUL(signs) -> RESHAPE -> MUL_MAT(Hadamard hint). For more than 8 tokens (decode and verify batches keep ggml_cuda_try_fwht_q8) the CUDA backend ran this as the GLU kernel, then fwht_cuda_block. fwht_cuda_block_swiglu reads gate and up, forms silu(gate)*up with the expression of the GLU kernel, applies scale, signs and the same butterfly, and writes the transformed rows: one launch less and one full pass over the activation less. The output may sit exactly on top of the gate buffer (the allocator reuses it); the kernel reads a whole transform before it writes it, any other overlap refuses the fusion. GGML_CUDA_GLU_FWHT_FUSION=0 disables it. When the only reader of the transformed rows is a PTQ1_0 mul_mat_q (prompt batches), the same launch also quantizes the rows to block_q8_1_mmq (D4) and the F32 rows are never written (swiglu_fwht_quantize_mmq reproduces unary_gated_op_kernel, the signed fwht_cuda_block and quantize_mmq_q8_1 in order, including its 127/amax form, so the rows are bit-identical to the three-launch version). The rows go into a pool block held for the graph and are registered in ctx.fwht_q8() with the new mmq flag; ggml_cuda_mul_mat hands them to ggml_cuda_mul_mat_q as external_q8 without quantizing. The fusion is refused if the transform output has any other reader, is a graph output, or the consumer would not take the MMQ path. GGML_CUDA_GLU_FWHT_Q8=0 keeps only the F32-output version. Ternary-Bonsai-2-27B PTQ1_0, RTX 4070, llama-bench pp2048, q4_0 K/V, against prism 6447906bf (mean of two runs): -ub 512 1354.8 -> 1411.6 tok/s (+4.2 %), -ub 2048 1279.2 -> 1343.0 (+5.0 %); at depth 32768 +3.1 % and +3.5 %. Decode (MTP, 8 interleaved pairs) is unchanged (+0.01 %, outputs identical). Perplexity and the KL against the unfused build are unchanged (mean KLD 0.000000, same top token 100 %). tests: MUL_MAT_HADAMARD cases with swiglu (9, 33, 64 and 17 tokens) and with a PTQ1_0 ffn_down consumer (9, 33, 512, 64 and 17 tokens). Co-Authored-By: Claude Sonnet 5.5 --- ggml/src/ggml-cuda/common.cuh | 1 + ggml/src/ggml-cuda/fwht.cu | 101 +++++++++++++++++++++++++++++ ggml/src/ggml-cuda/fwht.cuh | 6 ++ ggml/src/ggml-cuda/ggml-cuda.cu | 110 ++++++++++++++++++++++++++++++++ ggml/src/ggml-cuda/quantize.cu | 98 ++++++++++++++++++++++++++++ ggml/src/ggml-cuda/quantize.cuh | 6 ++ tests/test-backend-ops.cpp | 27 +++++++- 7 files changed, 346 insertions(+), 3 deletions(-) diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index aac4b7bf298f..537c86af135b 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -1678,6 +1678,7 @@ struct ggml_cuda_fwht_q8 { int64_t ncols = 0; // rows quantized (src1->ne[1] of every consumer) const void * data = nullptr; // the q8_1 rows: the transform's own output buffer, or a pool block // held for the rest of the graph when that buffer aliases the input + bool mmq = false; // the rows are block_q8_1_mmq (D4, for a PTQ1_0 mul_mat_q), not a mat-vec layout }; struct ggml_cuda_fwht_q8_context { diff --git a/ggml/src/ggml-cuda/fwht.cu b/ggml/src/ggml-cuda/fwht.cu index 24a4f4d69202..4ee6c7ca64cf 100644 --- a/ggml/src/ggml-cuda/fwht.cu +++ b/ggml/src/ggml-cuda/fwht.cu @@ -1,5 +1,6 @@ #include "common.cuh" #include "fwht.cuh" +#include "unary.cuh" #include @@ -312,6 +313,106 @@ bool ggml_cuda_op_mul_mat_use_fwht(const struct ggml_tensor * op) { ggml_are_same_shape(b, op); } +// silu(gate) * up folded into the load of fwht_cuda_block (always with the sign flip): v = silu(a)*b is the GLU node's value +// (unary_gated_op_kernel), then the same "* scale", "* sign" and butterfly as fwht_cuda_block. +template +__launch_bounds__(NT, 1) +__global__ void fwht_cuda_block_swiglu(const float * gate, const float * up, float * dst, const int64_t n_rows, + const float scale, const float * signs, const int n_blk) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int NE = N / NT; + static_assert(NE >= 1 && N % NT == 0 && NT % warp_size == 0, "bad FWHT block shape"); + + __shared__ float s[N]; + + const int64_t r = blockIdx.x; + if (r >= n_rows) { + return; + } + + gate += r * N; + up += r * N; + dst += r * N; + + const int tid = threadIdx.x; + const int lane = tid % warp_size; + + ggml_cuda_pdl_sync(); + const float * signs_row = signs + (r % n_blk) * N; + + float reg[NE]; +#pragma unroll + for (int i = 0; i < NE; ++i) { + const float v = ggml_cuda_op_silu_single(gate[i * NT + tid]) * up[i * NT + tid]; + reg[i] = v * scale; + reg[i] *= signs_row[i * NT + tid]; + } + + ggml_cuda_fwht_block_butterfly(reg, s, tid, lane); + +#pragma unroll + for (int i = 0; i < NE; ++i) { + dst[i * NT + tid] = reg[i]; + } +} + +bool ggml_cuda_op_swiglu_fwht_signed(ggml_backend_cuda_context & ctx, const ggml_tensor * gate, const ggml_tensor * up, + const ggml_tensor * signs_t, ggml_tensor * dst) { + static const bool off = getenv("GGML_CUDA_GLU_FWHT_FUSION") != nullptr && atoi(getenv("GGML_CUDA_GLU_FWHT_FUSION")) == 0; + static const bool legacy = getenv("GGML_CUDA_FWHT_LEGACY") != nullptr; + if (off || legacy) { + return false; + } + if (gate->type != GGML_TYPE_F32 || up->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || signs_t->type != GGML_TYPE_F32 || + !ggml_is_contiguous(gate) || !ggml_is_contiguous(up) || !ggml_is_contiguous(dst) || !ggml_is_contiguous(signs_t) || + !ggml_are_same_shape(gate, up) || ggml_nelements(gate) != ggml_nelements(dst)) { + return false; + } + const int n = dst->ne[0]; + const int64_t rows = ggml_nelements(dst) / n; + if (signs_t->ne[0] % n != 0 || rows == 0) { + return false; + } + const int n_blk = signs_t->ne[0] / n; + const float * signs = (const float *) signs_t->data; + const float scale = 1 / sqrtf(n); + cudaStream_t stream = ctx.stream(); + const float * g_d = (const float *) gate->data; + const float * u_d = (const float *) up->data; + float * d_d = (float *) dst->data; + const int nt = ggml_cuda_fwht_block_threads(n, rows); + +#define FWHT_GLU_LAUNCH(NN, NTT) { \ + const dim3 g((unsigned) rows, 1, 1), b(NTT, 1, 1); \ + const ggml_cuda_kernel_launch_params lp = ggml_cuda_kernel_launch_params(g, b, 0, stream); \ + ggml_cuda_kernel_launch(fwht_cuda_block_swiglu, lp, g_d, u_d, d_d, rows, scale, signs, n_blk); \ + return true; \ + } +#define FWHT_GLU_CASE(NN) \ + case NN: { \ + if constexpr (NN >= 1024) { \ + if (nt == 1024) FWHT_GLU_LAUNCH(NN, 1024) \ + } \ + if constexpr (NN >= 512) { \ + if (nt == 512) FWHT_GLU_LAUNCH(NN, 512) \ + } \ + FWHT_GLU_LAUNCH(NN, FWHT_BLOCK_THREADS) \ + } + switch (n) { + FWHT_GLU_CASE(512) + FWHT_GLU_CASE(1024) + FWHT_GLU_CASE(2048) + FWHT_GLU_CASE(4096) +#if !defined(GGML_USE_MUSA) + FWHT_GLU_CASE(8192) +#endif // !defined(GGML_USE_MUSA) + default: + return false; + } +#undef FWHT_GLU_CASE +#undef FWHT_GLU_LAUNCH +} + bool ggml_cuda_op_fwht(ggml_backend_cuda_context & ctx, const ggml_tensor * src, ggml_tensor * dst) { GGML_ASSERT(ggml_are_same_shape(src, dst)); return fwht_dispatch(ctx, src, dst, nullptr); diff --git a/ggml/src/ggml-cuda/fwht.cuh b/ggml/src/ggml-cuda/fwht.cuh index a964fba6673c..f4c19218eeb9 100644 --- a/ggml/src/ggml-cuda/fwht.cuh +++ b/ggml/src/ggml-cuda/fwht.cuh @@ -65,6 +65,12 @@ bool ggml_cuda_op_fwht(ggml_backend_cuda_context & ctx, const ggml_tensor * src, bool ggml_cuda_op_fwht_signed(ggml_backend_cuda_context & ctx, const ggml_tensor * src, const ggml_tensor * signs, ggml_tensor * dst); +// silu(gate) * up, the sign flip and the transform in one launch (the GLU node, the MUL by the signs and the Hadamard +// MUL_MAT of an FFN down input). Same arithmetic and order as the three nodes, so the result is bit-identical. +// Returns false (nothing launched) when the shapes or the width are not covered. +bool ggml_cuda_op_swiglu_fwht_signed(ggml_backend_cuda_context & ctx, const ggml_tensor * gate, const ggml_tensor * up, + const ggml_tensor * signs, ggml_tensor * dst); + // Threads per block for the one-block-per-transform FWHT kernels (fwht_cuda_block, fwht_quantize_q8_1) at width n. // 256 by default; with few transforms in flight (decode and speculative verify: 64 or fewer) min(n/2, 1024), so each // thread keeps at least two elements and the few blocks there are use more of the SM. The butterfly runs its stages diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index d2350d92e23b..33675687c5d1 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -2185,6 +2185,13 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor return; } if (ggml_cuda_should_use_mmq(src0->type, cc, ne11, /*n_experts =*/ 0)) { + // activation rows already quantized for the MMQ by the SwiGLU + Hadamard fusion (the F32 rows of src1 were not written) + const ggml_cuda_fwht_q8 * pre = src0->type == GGML_TYPE_PTQ1_0 ? ctx.fwht_q8().find(src1) : nullptr; + if (pre != nullptr && pre->mmq) { + GGML_ASSERT(pre->ncols == ne11 && pre->ne0 == ne10); + ggml_cuda_mul_mat_q(ctx, src0, src1, nullptr, dst, nullptr, nullptr, nullptr, (void *) pre->data, /*quantize_external =*/ false); + return; + } ggml_cuda_mul_mat_q(ctx, src0, src1, nullptr, dst); return; } @@ -5631,6 +5638,109 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph } } + // SWIGLU (separate gate and up) + Hadamard sign flip + reshape + FWHT-hint matmul: the FFN down input of a + // Hadamard-folded model. One launch reads gate and up and writes the transformed rows; bit-identical to the four nodes. + if (node->op == GGML_OP_GLU && ggml_can_fuse_subgraph(cgraph, i, { GGML_OP_GLU, GGML_OP_MUL, GGML_OP_RESHAPE, GGML_OP_MUL_MAT }, { i + 3 })) { + const ggml_tensor * glu = node; + const ggml_tensor * mul = cgraph->nodes[i + 1]; + const ggml_tensor * reshape = cgraph->nodes[i + 2]; + ggml_tensor * mm = cgraph->nodes[i + 3]; + const ggml_tensor * signs = mul->src[1]; + + const bool pattern_ok = ggml_get_glu_op(glu) == GGML_GLU_OP_SWIGLU && !ggml_get_op_params_i32(glu, 1) && + glu->src[1] != nullptr && mul->src[0] == glu && ggml_get_op_params_i32(mm, 1) == GGML_HINT_SRC0_IS_HADAMARD && + mm->src[1] == reshape && reshape->src[0] == mul && glu->type == GGML_TYPE_F32 && mul->type == GGML_TYPE_F32 && + signs->ne[1] == 1 && signs->ne[2] == 1 && signs->ne[3] == 1 && signs->ne[0] == glu->ne[0] && + signs->ne[0] % mm->src[0]->ne[0] == 0 && ggml_is_contiguous(glu) && + glu->ne[1] > MMVQ_MAX_BATCH_SIZE; // decode and verify batches keep ggml_cuda_try_fwht_q8 (FWHT + q8_1 for the mat-vec) + // The kernel reads a whole transform (N consecutive floats of gate and up) before it writes it, so the output may sit exactly + // on top of gate or up (the allocator reuses the gate buffer; the shapes differ, so the generic alias check rejects it). + // Any other overlap with gate, up or the signs is refused. + auto overlaps = [&](const ggml_tensor * a, const ggml_tensor * b, bool exact_ok) { + const char * a0 = (const char *) a->data; const char * a1 = a0 + ggml_nbytes(a); + const char * b0 = (const char *) b->data; const char * b1 = b0 + ggml_nbytes(b); + if (a1 <= b0 || b1 <= a0) { + return false; + } + return !(exact_ok && a0 == b0 && a1 == b1); + }; + const bool mem_ok = pattern_ok && !overlaps(mm, glu->src[0], true) && !overlaps(mm, glu->src[1], true) && !overlaps(mm, signs, false); + // Prompt batches whose only reader is a PTQ1_0 mul_mat_q (the ffn_down): the same launch also quantizes the transformed rows + // for the MMQ and the F32 rows are never written. ggml_cuda_mul_mat finds the rows through ctx.fwht_q8(). + // GGML_CUDA_GLU_FWHT_Q8=0 keeps the F32 output (SwiGLU + transform only). + static const bool q8_off = (getenv("GGML_CUDA_GLU_FWHT_Q8") != nullptr && atoi(getenv("GGML_CUDA_GLU_FWHT_Q8")) == 0) || + (getenv("GGML_CUDA_GLU_FWHT_FUSION") != nullptr && atoi(getenv("GGML_CUDA_GLU_FWHT_FUSION")) == 0); + if (mem_ok && !q8_off) { + const int cc = ggml_cuda_info().devices[cuda_ctx->device].cc; + const ggml_tensor * aliases[4] = { mm, nullptr, nullptr, nullptr }; + int n_aliases = 1; + int uses_expected = ggml_node_get_use_count(cgraph, i + 3); + int uses_found = 0; + const ggml_tensor * cons = nullptr; + bool q8_ok = GGML_CUDA_CC_IS_NVIDIA(cc) && cc >= GGML_CUDA_CC_TURING && !(mm->flags & GGML_TENSOR_FLAG_OUTPUT); + for (int j = i + 4; j < cgraph->n_nodes && q8_ok; ++j) { + const ggml_tensor * t = cgraph->nodes[j]; + for (int sidx = 0; sidx < GGML_MAX_SRC && q8_ok; ++sidx) { + const ggml_tensor * src = t->src[sidx]; + bool is_alias = false; + for (int a = 0; a < n_aliases; ++a) { + is_alias = is_alias || src == aliases[a]; + } + if (!is_alias) { + continue; + } + uses_found++; + if (t->op == GGML_OP_RESHAPE && t->view_src == mm && t->data == mm->data && + ggml_nelements(t) == ggml_nelements(mm) && ggml_is_contiguous(t) && !(t->flags & GGML_TENSOR_FLAG_OUTPUT)) { + if (n_aliases == 4) { + q8_ok = false; + } else { + aliases[n_aliases++] = t; + uses_expected += ggml_node_get_use_count(cgraph, j); + } + continue; + } + const ggml_tensor * w = t->src[0]; + if (cons != nullptr || t->op != GGML_OP_MUL_MAT || sidx != 1 || !w || w->type != GGML_TYPE_PTQ1_0 || + t->type != GGML_TYPE_F32 || src->type != GGML_TYPE_F32 || !ggml_is_contiguous(src) || + src->ne[2] != 1 || src->ne[3] != 1 || src->ne[0] != glu->ne[0] || src->ne[1] != glu->ne[1] || + ggml_get_op_params_i32(t, 1) == GGML_HINT_SRC0_IS_HADAMARD || + ggml_cuda_should_use_mmvq(w->type, cc, src->ne[1]) || + !ggml_cuda_should_use_mmq(w->type, cc, src->ne[1], 0) || + (ggml_backend_buffer_get_usage(w->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE && + ggml_nbytes(w) != ggml_backend_buffer_get_alloc_size(w->buffer, w) && w->view_src)) { + q8_ok = false; + } else { + cons = t; + } + } + } + if (q8_ok && cons != nullptr && uses_found == uses_expected && glu->ne[0] % MATRIX_ROW_PADDING == 0 && + glu->ne[2] == 1 && glu->ne[3] == 1) { + const int n = (int) mm->src[0]->ne[0]; + const int n_blk = (int) (signs->ne[0] / n); + const size_t bytes = ggml_cuda_mul_mat_q_q8_size(cons->src[0], cons->src[1]); + auto blk = std::make_unique>(cuda_ctx->pool(), bytes); + if (ggml_cuda_swiglu_fwht_quantize_mmq((const float *) glu->src[0]->data, (const float *) glu->src[1]->data, + (const float *) signs->data, blk->get(), n, n_blk, glu->ne[1], cuda_ctx->stream())) { + ggml_cuda_fwht_q8 e; + e.mmq = true; + e.ne0 = glu->ne[0]; + e.ncols = glu->ne[1]; + e.data = blk->get(); + for (int a = 0; a < n_aliases; ++a) { + cuda_ctx->fwht_q8().set(aliases[a], e); + } + cuda_ctx->fwht_q8().held.push_back(std::move(blk)); + return 3; + } + } + } + if (mem_ok && ggml_cuda_op_swiglu_fwht_signed(*cuda_ctx, glu->src[0], glu->src[1], signs, mm)) { + return 3; + } + } + // Hadamard sign flip + reshape + FWHT-hint matmul: multiply the sign // vector during the transform's load instead of a separate full pass if (ggml_can_fuse_subgraph(cgraph, i, { GGML_OP_MUL, GGML_OP_RESHAPE, GGML_OP_MUL_MAT }, { i + 2 })) { diff --git a/ggml/src/ggml-cuda/quantize.cu b/ggml/src/ggml-cuda/quantize.cu index 3199043831a6..b01aec45c0d4 100644 --- a/ggml/src/ggml-cuda/quantize.cu +++ b/ggml/src/ggml-cuda/quantize.cu @@ -965,6 +965,104 @@ void quantize_row_q8_1_cuda( } } +// SwiGLU + sign flip + Hadamard transform + MMQ activation quantization in one launch: the FFN down input of a Hadamard-folded +// model when the mat-mul is a PTQ1_0 MMQ (prompt batches). Same arithmetic as unary_gated_op_kernel, the signed +// fwht_cuda_block and quantize_mmq_q8_1 in sequence (note the MMQ quantizer's d_inv = 127/amax form, +// not the amax/127 of fwht_quantize_q8_1), so the rows are bit-identical to the three-launch version. +// One CUDA block per transform r = token*n_blk + blk of width N; the 32-wide scale groups lie inside one warp. +template +__launch_bounds__(NT, 1) +static __global__ void swiglu_fwht_quantize_mmq( + const float * gate, const float * up, const float * signs, block_q8_1_mmq * y, + const int n_blk, const int64_t ntok, const float scale) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int NE = N / NT; + static_assert(NE >= 1 && N % NT == 0 && NT % warp_size == 0 && NT % QK8_1 == 0, "bad fused FWHT shape"); + static_assert(warp_size % QK8_1 == 0, "a 32-block must not span two warps"); + + __shared__ float s[N]; + + const int64_t r = blockIdx.x; + const int64_t t = r / n_blk; // token (row of the mat-mul's src1) + const int blk = (int) (r - t * n_blk); // transform block within the row + const int tid = threadIdx.x; + const int lane = tid % warp_size; + + ggml_cuda_pdl_sync(); + const float * signs_row = signs + (int64_t) blk * N; + const float * g = gate + r * N; + const float * u = up + r * N; + + float reg[NE]; +#pragma unroll + for (int i = 0; i < NE; ++i) { + const float v = ggml_cuda_op_silu_single(g[i * NT + tid]) * u[i * NT + tid]; + reg[i] = v * scale; + reg[i] *= signs_row[i * NT + tid]; + } + ggml_cuda_fwht_block_butterfly(reg, s, tid, lane); + +#pragma unroll + for (int i = 0; i < NE; ++i) { + const float xi = reg[i]; + const int64_t col = (int64_t) blk * N + i * NT + tid; // column within the row of K = n_blk*N values + float amax = fabsf(xi); + amax = warp_reduce_max(amax); + + const float d_inv = 127.0f / amax; + const int8_t q = roundf(xi * d_inv); + const float d = 1.0f / d_inv; + + const int64_t k_block = col / QK8_1_MMQ; + const int iqs = (int) (col % QK8_1_MMQ); + block_q8_1_mmq * yb = y + (k_block * ntok + t); + yb->qs[iqs] = q; + if (iqs % QK8_1 == 0) { + yb->d4[iqs / QK8_1] = d; + } + } +} + +bool ggml_cuda_swiglu_fwht_quantize_mmq( + const float * gate, const float * up, const float * signs, void * vy, + const int n, const int n_blk, const int64_t ntok, cudaStream_t stream) { + const int64_t rows = ntok * n_blk; + if (rows == 0 || ((int64_t) n * n_blk) % MATRIX_ROW_PADDING != 0) { + return false; + } + const float scale = 1 / sqrtf(n); + const int nt = ggml_cuda_fwht_block_threads(n, rows); +#define SWQ_LAUNCH(NN, NTT) { \ + const dim3 g((unsigned) rows, 1, 1), b(NTT, 1, 1); \ + const ggml_cuda_kernel_launch_params lp = ggml_cuda_kernel_launch_params(g, b, 0, stream); \ + ggml_cuda_kernel_launch(swiglu_fwht_quantize_mmq, lp, gate, up, signs, (block_q8_1_mmq *) vy, n_blk, ntok, scale); \ + return true; \ + } +#define SWQ_CASE(NN) \ + case NN: { \ + if constexpr (NN >= 1024) { \ + if (nt == 1024) SWQ_LAUNCH(NN, 1024) \ + } \ + if constexpr (NN >= 512) { \ + if (nt == 512) SWQ_LAUNCH(NN, 512) \ + } \ + SWQ_LAUNCH(NN, 256) \ + } + switch (n) { + SWQ_CASE(512) + SWQ_CASE(1024) + SWQ_CASE(2048) + SWQ_CASE(4096) +#if !defined(GGML_USE_MUSA) + SWQ_CASE(8192) +#endif // !defined(GGML_USE_MUSA) + default: + return false; + } +#undef SWQ_CASE +#undef SWQ_LAUNCH +} + void quantize_mmq_q8_1_cuda( const float * x, const int32_t * ids, void * vy, const ggml_type type_src0, const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03, diff --git a/ggml/src/ggml-cuda/quantize.cuh b/ggml/src/ggml-cuda/quantize.cuh index f9a80f2f03d4..2373e8e8fb8a 100644 --- a/ggml/src/ggml-cuda/quantize.cuh +++ b/ggml/src/ggml-cuda/quantize.cuh @@ -88,3 +88,9 @@ void quantize_scatter_mmq_q8_1_cuda(const float * x, int64_t nrows_dst, int n_expert_used, cudaStream_t stream); + +// SwiGLU(gate, up) * signs -> block-wise Hadamard transform (width n, n_blk blocks per row) -> MMQ q8_1 quantization (D4 layout, as +// quantize_mmq_q8_1 writes for PTQ1_0), ntok rows of K = n*n_blk values. Returns false when the width is not covered. +bool ggml_cuda_swiglu_fwht_quantize_mmq( + const float * gate, const float * up, const float * signs, void * vy, + int n, int n_blk, int64_t ntok, cudaStream_t stream); diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 6026c296ef85..aa452bcaf6cf 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -6107,15 +6107,19 @@ struct test_fwht_signed : public test_case { const int64_t n_tokens; const ggml_type type_x; const bool swiglu; // x is the SwiGLU of a gate and an up projection (the ffn_down input) + const ggml_type type_w; // != COUNT: the transform feeds a mat-mul with a weight of this type (the ffn_down; the CUDA backend quantizes the rows in the same launch) test_fwht_signed(int64_t blk = 1024, int64_t width = 5120, int64_t n_tokens = 7, - ggml_type type_x = GGML_TYPE_F32, bool swiglu = false) - : blk(blk), width(width), n_tokens(n_tokens), type_x(type_x), swiglu(swiglu) {} + ggml_type type_x = GGML_TYPE_F32, bool swiglu = false, ggml_type type_w = GGML_TYPE_COUNT) + : blk(blk), width(width), n_tokens(n_tokens), type_x(type_x), swiglu(swiglu), type_w(type_w) {} std::string vars() override { - return VARS_TO_STR5(blk, width, n_tokens, type_x, swiglu); + return type_w == GGML_TYPE_COUNT ? VARS_TO_STR5(blk, width, n_tokens, type_x, swiglu) + : std::string(VARS_TO_STR5(blk, width, n_tokens, type_x, swiglu)) + "," + VAR_TO_STR(type_w); } + double max_nmse_err() override { return type_w == GGML_TYPE_COUNT ? 1e-7 : 5e-4; } + std::string op_desc(ggml_tensor * t) override { GGML_UNUSED(t); return "MUL_MAT_HADAMARD"; @@ -6146,6 +6150,12 @@ struct test_fwht_signed : public test_case { cur = ggml_reshape_2d(ctx, cur, blk, width / blk * n_tokens); ggml_tensor * out = ggml_mul_mat(ctx, a, cur); ggml_mul_mat_set_hint(out, GGML_HINT_SRC0_IS_HADAMARD); + if (type_w != GGML_TYPE_COUNT) { + out = ggml_reshape_2d(ctx, out, width, n_tokens); + ggml_tensor * w = ggml_new_tensor_2d(ctx, type_w, width, 96); + ggml_set_name(w, "w_down"); + out = ggml_mul_mat(ctx, w, out); + } ggml_set_name(out, "out"); return out; } @@ -12334,6 +12344,17 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_fwht_signed(1024, 17408, 1, GGML_TYPE_F32, true)); test_cases.emplace_back(new test_fwht_signed(1024, 17408, 8, GGML_TYPE_F32, true)); test_cases.emplace_back(new test_fwht_signed(1024, 17408, 512, GGML_TYPE_F32, true)); + // prompt batches (more than 8 tokens) take the fused SwiGLU + sign flip + transform launch on CUDA + test_cases.emplace_back(new test_fwht_signed(1024, 17408, 9, GGML_TYPE_F32, true)); + test_cases.emplace_back(new test_fwht_signed(1024, 17408, 33, GGML_TYPE_F32, true)); + test_cases.emplace_back(new test_fwht_signed(1024, 5120, 64, GGML_TYPE_F32, true)); + test_cases.emplace_back(new test_fwht_signed(512, 6144, 17, GGML_TYPE_F32, true)); + // ... and the MMQ activation quantization of a PTQ1_0 ffn_down in the same launch + test_cases.emplace_back(new test_fwht_signed(1024, 17408, 9, GGML_TYPE_F32, true, GGML_TYPE_PTQ1_0)); + test_cases.emplace_back(new test_fwht_signed(1024, 17408, 33, GGML_TYPE_F32, true, GGML_TYPE_PTQ1_0)); + test_cases.emplace_back(new test_fwht_signed(1024, 17408, 512, GGML_TYPE_F32, true, GGML_TYPE_PTQ1_0)); + test_cases.emplace_back(new test_fwht_signed(1024, 5120, 64, GGML_TYPE_F32, true, GGML_TYPE_PTQ1_0)); + test_cases.emplace_back(new test_fwht_signed(512, 6144, 17, GGML_TYPE_F32, true, GGML_TYPE_PTQ1_0)); test_cases.emplace_back(new test_fwht_signed(128, 1024, 5, GGML_TYPE_F32, true)); test_cases.emplace_back(new test_fwht_signed(2048, 8192, 3, GGML_TYPE_F32, true)); test_cases.emplace_back(new test_fwht_signed(1024, 5120, 4, GGML_TYPE_F16, true));