Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions ggml/src/ggml-cuda/common.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -1678,6 +1678,7 @@ struct ggml_cuda_fwht_q8 {
int64_t ncols = 0; // rows quantized (src1->ne[1] of every consumer)
const void * data = nullptr; // the q8_1 rows: the transform's own output buffer, or a pool block
// held for the rest of the graph when that buffer aliases the input
bool mmq = false; // the rows are block_q8_1_mmq (D4, for a PTQ1_0 mul_mat_q), not a mat-vec layout
};

struct ggml_cuda_fwht_q8_context {
Expand Down
101 changes: 101 additions & 0 deletions ggml/src/ggml-cuda/fwht.cu
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#include "common.cuh"
#include "fwht.cuh"
#include "unary.cuh"

#include <cstdlib>

Expand Down Expand Up @@ -312,6 +313,106 @@ bool ggml_cuda_op_mul_mat_use_fwht(const struct ggml_tensor * op) {
ggml_are_same_shape(b, op);
}

// silu(gate) * up folded into the load of fwht_cuda_block (always with the sign flip): v = silu(a)*b is the GLU node's value
// (unary_gated_op_kernel<op_silu, float>), then the same "* scale", "* sign" and butterfly as fwht_cuda_block.
template <int N, int NT>
__launch_bounds__(NT, 1)
__global__ void fwht_cuda_block_swiglu(const float * gate, const float * up, float * dst, const int64_t n_rows,
const float scale, const float * signs, const int n_blk) {
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
constexpr int NE = N / NT;
static_assert(NE >= 1 && N % NT == 0 && NT % warp_size == 0, "bad FWHT block shape");

__shared__ float s[N];

const int64_t r = blockIdx.x;
if (r >= n_rows) {
return;
}

gate += r * N;
up += r * N;
dst += r * N;

const int tid = threadIdx.x;
const int lane = tid % warp_size;

ggml_cuda_pdl_sync();
const float * signs_row = signs + (r % n_blk) * N;

float reg[NE];
#pragma unroll
for (int i = 0; i < NE; ++i) {
const float v = ggml_cuda_op_silu_single(gate[i * NT + tid]) * up[i * NT + tid];
reg[i] = v * scale;
reg[i] *= signs_row[i * NT + tid];
}

ggml_cuda_fwht_block_butterfly<N, NT>(reg, s, tid, lane);

#pragma unroll
for (int i = 0; i < NE; ++i) {
dst[i * NT + tid] = reg[i];
}
}

bool ggml_cuda_op_swiglu_fwht_signed(ggml_backend_cuda_context & ctx, const ggml_tensor * gate, const ggml_tensor * up,
const ggml_tensor * signs_t, ggml_tensor * dst) {
static const bool off = getenv("GGML_CUDA_GLU_FWHT_FUSION") != nullptr && atoi(getenv("GGML_CUDA_GLU_FWHT_FUSION")) == 0;
static const bool legacy = getenv("GGML_CUDA_FWHT_LEGACY") != nullptr;
if (off || legacy) {
return false;
}
if (gate->type != GGML_TYPE_F32 || up->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || signs_t->type != GGML_TYPE_F32 ||
!ggml_is_contiguous(gate) || !ggml_is_contiguous(up) || !ggml_is_contiguous(dst) || !ggml_is_contiguous(signs_t) ||
!ggml_are_same_shape(gate, up) || ggml_nelements(gate) != ggml_nelements(dst)) {
return false;
}
const int n = dst->ne[0];
const int64_t rows = ggml_nelements(dst) / n;
if (signs_t->ne[0] % n != 0 || rows == 0) {
return false;
}
const int n_blk = signs_t->ne[0] / n;
const float * signs = (const float *) signs_t->data;
const float scale = 1 / sqrtf(n);
cudaStream_t stream = ctx.stream();
const float * g_d = (const float *) gate->data;
const float * u_d = (const float *) up->data;
float * d_d = (float *) dst->data;
const int nt = ggml_cuda_fwht_block_threads(n, rows);

#define FWHT_GLU_LAUNCH(NN, NTT) { \
const dim3 g((unsigned) rows, 1, 1), b(NTT, 1, 1); \
const ggml_cuda_kernel_launch_params lp = ggml_cuda_kernel_launch_params(g, b, 0, stream); \
ggml_cuda_kernel_launch(fwht_cuda_block_swiglu<NN, NTT>, lp, g_d, u_d, d_d, rows, scale, signs, n_blk); \
return true; \
}
#define FWHT_GLU_CASE(NN) \
case NN: { \
if constexpr (NN >= 1024) { \
if (nt == 1024) FWHT_GLU_LAUNCH(NN, 1024) \
} \
if constexpr (NN >= 512) { \
if (nt == 512) FWHT_GLU_LAUNCH(NN, 512) \
} \
FWHT_GLU_LAUNCH(NN, FWHT_BLOCK_THREADS) \
}
switch (n) {
FWHT_GLU_CASE(512)
FWHT_GLU_CASE(1024)
FWHT_GLU_CASE(2048)
FWHT_GLU_CASE(4096)
#if !defined(GGML_USE_MUSA)
FWHT_GLU_CASE(8192)
#endif // !defined(GGML_USE_MUSA)
default:
return false;
}
#undef FWHT_GLU_CASE
#undef FWHT_GLU_LAUNCH
}

bool ggml_cuda_op_fwht(ggml_backend_cuda_context & ctx, const ggml_tensor * src, ggml_tensor * dst) {
GGML_ASSERT(ggml_are_same_shape(src, dst));
return fwht_dispatch(ctx, src, dst, nullptr);
Expand Down
6 changes: 6 additions & 0 deletions ggml/src/ggml-cuda/fwht.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,12 @@ bool ggml_cuda_op_fwht(ggml_backend_cuda_context & ctx, const ggml_tensor * src,
bool ggml_cuda_op_fwht_signed(ggml_backend_cuda_context & ctx, const ggml_tensor * src,
const ggml_tensor * signs, ggml_tensor * dst);

// silu(gate) * up, the sign flip and the transform in one launch (the GLU node, the MUL by the signs and the Hadamard
// MUL_MAT of an FFN down input). Same arithmetic and order as the three nodes, so the result is bit-identical.
// Returns false (nothing launched) when the shapes or the width are not covered.
bool ggml_cuda_op_swiglu_fwht_signed(ggml_backend_cuda_context & ctx, const ggml_tensor * gate, const ggml_tensor * up,
const ggml_tensor * signs, ggml_tensor * dst);

// Threads per block for the one-block-per-transform FWHT kernels (fwht_cuda_block, fwht_quantize_q8_1) at width n.
// 256 by default; with few transforms in flight (decode and speculative verify: 64 or fewer) min(n/2, 1024), so each
// thread keeps at least two elements and the few blocks there are use more of the SM. The butterfly runs its stages
Expand Down
110 changes: 110 additions & 0 deletions ggml/src/ggml-cuda/ggml-cuda.cu
Original file line number Diff line number Diff line change
Expand Up @@ -2185,6 +2185,13 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor
return;
}
if (ggml_cuda_should_use_mmq(src0->type, cc, ne11, /*n_experts =*/ 0)) {
// activation rows already quantized for the MMQ by the SwiGLU + Hadamard fusion (the F32 rows of src1 were not written)
const ggml_cuda_fwht_q8 * pre = src0->type == GGML_TYPE_PTQ1_0 ? ctx.fwht_q8().find(src1) : nullptr;
if (pre != nullptr && pre->mmq) {
GGML_ASSERT(pre->ncols == ne11 && pre->ne0 == ne10);
ggml_cuda_mul_mat_q(ctx, src0, src1, nullptr, dst, nullptr, nullptr, nullptr, (void *) pre->data, /*quantize_external =*/ false);
return;
}
ggml_cuda_mul_mat_q(ctx, src0, src1, nullptr, dst);
return;
}
Expand Down Expand Up @@ -5631,6 +5638,109 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph
}
}

// SWIGLU (separate gate and up) + Hadamard sign flip + reshape + FWHT-hint matmul: the FFN down input of a
// Hadamard-folded model. One launch reads gate and up and writes the transformed rows; bit-identical to the four nodes.
if (node->op == GGML_OP_GLU && ggml_can_fuse_subgraph(cgraph, i, { GGML_OP_GLU, GGML_OP_MUL, GGML_OP_RESHAPE, GGML_OP_MUL_MAT }, { i + 3 })) {
const ggml_tensor * glu = node;
const ggml_tensor * mul = cgraph->nodes[i + 1];
const ggml_tensor * reshape = cgraph->nodes[i + 2];
ggml_tensor * mm = cgraph->nodes[i + 3];
const ggml_tensor * signs = mul->src[1];

const bool pattern_ok = ggml_get_glu_op(glu) == GGML_GLU_OP_SWIGLU && !ggml_get_op_params_i32(glu, 1) &&
glu->src[1] != nullptr && mul->src[0] == glu && ggml_get_op_params_i32(mm, 1) == GGML_HINT_SRC0_IS_HADAMARD &&
mm->src[1] == reshape && reshape->src[0] == mul && glu->type == GGML_TYPE_F32 && mul->type == GGML_TYPE_F32 &&
signs->ne[1] == 1 && signs->ne[2] == 1 && signs->ne[3] == 1 && signs->ne[0] == glu->ne[0] &&
signs->ne[0] % mm->src[0]->ne[0] == 0 && ggml_is_contiguous(glu) &&
glu->ne[1] > MMVQ_MAX_BATCH_SIZE; // decode and verify batches keep ggml_cuda_try_fwht_q8 (FWHT + q8_1 for the mat-vec)
// The kernel reads a whole transform (N consecutive floats of gate and up) before it writes it, so the output may sit exactly
// on top of gate or up (the allocator reuses the gate buffer; the shapes differ, so the generic alias check rejects it).
// Any other overlap with gate, up or the signs is refused.
auto overlaps = [&](const ggml_tensor * a, const ggml_tensor * b, bool exact_ok) {
const char * a0 = (const char *) a->data; const char * a1 = a0 + ggml_nbytes(a);
const char * b0 = (const char *) b->data; const char * b1 = b0 + ggml_nbytes(b);
if (a1 <= b0 || b1 <= a0) {
return false;
}
return !(exact_ok && a0 == b0 && a1 == b1);
};
const bool mem_ok = pattern_ok && !overlaps(mm, glu->src[0], true) && !overlaps(mm, glu->src[1], true) && !overlaps(mm, signs, false);
// Prompt batches whose only reader is a PTQ1_0 mul_mat_q (the ffn_down): the same launch also quantizes the transformed rows
// for the MMQ and the F32 rows are never written. ggml_cuda_mul_mat finds the rows through ctx.fwht_q8().
// GGML_CUDA_GLU_FWHT_Q8=0 keeps the F32 output (SwiGLU + transform only).
static const bool q8_off = (getenv("GGML_CUDA_GLU_FWHT_Q8") != nullptr && atoi(getenv("GGML_CUDA_GLU_FWHT_Q8")) == 0) ||
(getenv("GGML_CUDA_GLU_FWHT_FUSION") != nullptr && atoi(getenv("GGML_CUDA_GLU_FWHT_FUSION")) == 0);
if (mem_ok && !q8_off) {
const int cc = ggml_cuda_info().devices[cuda_ctx->device].cc;
const ggml_tensor * aliases[4] = { mm, nullptr, nullptr, nullptr };
int n_aliases = 1;
int uses_expected = ggml_node_get_use_count(cgraph, i + 3);
int uses_found = 0;
const ggml_tensor * cons = nullptr;
bool q8_ok = GGML_CUDA_CC_IS_NVIDIA(cc) && cc >= GGML_CUDA_CC_TURING && !(mm->flags & GGML_TENSOR_FLAG_OUTPUT);
for (int j = i + 4; j < cgraph->n_nodes && q8_ok; ++j) {
const ggml_tensor * t = cgraph->nodes[j];
for (int sidx = 0; sidx < GGML_MAX_SRC && q8_ok; ++sidx) {
const ggml_tensor * src = t->src[sidx];
bool is_alias = false;
for (int a = 0; a < n_aliases; ++a) {
is_alias = is_alias || src == aliases[a];
}
if (!is_alias) {
continue;
}
uses_found++;
if (t->op == GGML_OP_RESHAPE && t->view_src == mm && t->data == mm->data &&
ggml_nelements(t) == ggml_nelements(mm) && ggml_is_contiguous(t) && !(t->flags & GGML_TENSOR_FLAG_OUTPUT)) {
if (n_aliases == 4) {
q8_ok = false;
} else {
aliases[n_aliases++] = t;
uses_expected += ggml_node_get_use_count(cgraph, j);
}
continue;
}
const ggml_tensor * w = t->src[0];
if (cons != nullptr || t->op != GGML_OP_MUL_MAT || sidx != 1 || !w || w->type != GGML_TYPE_PTQ1_0 ||
t->type != GGML_TYPE_F32 || src->type != GGML_TYPE_F32 || !ggml_is_contiguous(src) ||
src->ne[2] != 1 || src->ne[3] != 1 || src->ne[0] != glu->ne[0] || src->ne[1] != glu->ne[1] ||
ggml_get_op_params_i32(t, 1) == GGML_HINT_SRC0_IS_HADAMARD ||
ggml_cuda_should_use_mmvq(w->type, cc, src->ne[1]) ||
!ggml_cuda_should_use_mmq(w->type, cc, src->ne[1], 0) ||
(ggml_backend_buffer_get_usage(w->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE &&
ggml_nbytes(w) != ggml_backend_buffer_get_alloc_size(w->buffer, w) && w->view_src)) {
q8_ok = false;
} else {
cons = t;
}
}
}
if (q8_ok && cons != nullptr && uses_found == uses_expected && glu->ne[0] % MATRIX_ROW_PADDING == 0 &&
glu->ne[2] == 1 && glu->ne[3] == 1) {
const int n = (int) mm->src[0]->ne[0];
const int n_blk = (int) (signs->ne[0] / n);
const size_t bytes = ggml_cuda_mul_mat_q_q8_size(cons->src[0], cons->src[1]);
auto blk = std::make_unique<ggml_cuda_pool_alloc<char>>(cuda_ctx->pool(), bytes);
if (ggml_cuda_swiglu_fwht_quantize_mmq((const float *) glu->src[0]->data, (const float *) glu->src[1]->data,
(const float *) signs->data, blk->get(), n, n_blk, glu->ne[1], cuda_ctx->stream())) {
ggml_cuda_fwht_q8 e;
e.mmq = true;
e.ne0 = glu->ne[0];
e.ncols = glu->ne[1];
e.data = blk->get();
for (int a = 0; a < n_aliases; ++a) {
cuda_ctx->fwht_q8().set(aliases[a], e);
}
cuda_ctx->fwht_q8().held.push_back(std::move(blk));
return 3;
}
}
}
if (mem_ok && ggml_cuda_op_swiglu_fwht_signed(*cuda_ctx, glu->src[0], glu->src[1], signs, mm)) {
return 3;
}
}

// Hadamard sign flip + reshape + FWHT-hint matmul: multiply the sign
// vector during the transform's load instead of a separate full pass
if (ggml_can_fuse_subgraph(cgraph, i, { GGML_OP_MUL, GGML_OP_RESHAPE, GGML_OP_MUL_MAT }, { i + 2 })) {
Expand Down
98 changes: 98 additions & 0 deletions ggml/src/ggml-cuda/quantize.cu
Original file line number Diff line number Diff line change
Expand Up @@ -965,6 +965,104 @@ void quantize_row_q8_1_cuda(
}
}

// SwiGLU + sign flip + Hadamard transform + MMQ activation quantization in one launch: the FFN down input of a Hadamard-folded
// model when the mat-mul is a PTQ1_0 MMQ (prompt batches). Same arithmetic as unary_gated_op_kernel<op_silu>, the signed
// fwht_cuda_block and quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_D4> in sequence (note the MMQ quantizer's d_inv = 127/amax form,
// not the amax/127 of fwht_quantize_q8_1), so the rows are bit-identical to the three-launch version.
// One CUDA block per transform r = token*n_blk + blk of width N; the 32-wide scale groups lie inside one warp.
template <int N, int NT>
__launch_bounds__(NT, 1)
static __global__ void swiglu_fwht_quantize_mmq(
const float * gate, const float * up, const float * signs, block_q8_1_mmq * y,
const int n_blk, const int64_t ntok, const float scale) {
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
constexpr int NE = N / NT;
static_assert(NE >= 1 && N % NT == 0 && NT % warp_size == 0 && NT % QK8_1 == 0, "bad fused FWHT shape");
static_assert(warp_size % QK8_1 == 0, "a 32-block must not span two warps");

__shared__ float s[N];

const int64_t r = blockIdx.x;
const int64_t t = r / n_blk; // token (row of the mat-mul's src1)
const int blk = (int) (r - t * n_blk); // transform block within the row
const int tid = threadIdx.x;
const int lane = tid % warp_size;

ggml_cuda_pdl_sync();
const float * signs_row = signs + (int64_t) blk * N;
const float * g = gate + r * N;
const float * u = up + r * N;

float reg[NE];
#pragma unroll
for (int i = 0; i < NE; ++i) {
const float v = ggml_cuda_op_silu_single(g[i * NT + tid]) * u[i * NT + tid];
reg[i] = v * scale;
reg[i] *= signs_row[i * NT + tid];
}
ggml_cuda_fwht_block_butterfly<N, NT>(reg, s, tid, lane);

#pragma unroll
for (int i = 0; i < NE; ++i) {
const float xi = reg[i];
const int64_t col = (int64_t) blk * N + i * NT + tid; // column within the row of K = n_blk*N values
float amax = fabsf(xi);
amax = warp_reduce_max<QK8_1>(amax);

const float d_inv = 127.0f / amax;
const int8_t q = roundf(xi * d_inv);
const float d = 1.0f / d_inv;

const int64_t k_block = col / QK8_1_MMQ;
const int iqs = (int) (col % QK8_1_MMQ);
block_q8_1_mmq * yb = y + (k_block * ntok + t);
yb->qs[iqs] = q;
if (iqs % QK8_1 == 0) {
yb->d4[iqs / QK8_1] = d;
}
}
}

bool ggml_cuda_swiglu_fwht_quantize_mmq(
const float * gate, const float * up, const float * signs, void * vy,
const int n, const int n_blk, const int64_t ntok, cudaStream_t stream) {
const int64_t rows = ntok * n_blk;
if (rows == 0 || ((int64_t) n * n_blk) % MATRIX_ROW_PADDING != 0) {
return false;
}
const float scale = 1 / sqrtf(n);
const int nt = ggml_cuda_fwht_block_threads(n, rows);
#define SWQ_LAUNCH(NN, NTT) { \
const dim3 g((unsigned) rows, 1, 1), b(NTT, 1, 1); \
const ggml_cuda_kernel_launch_params lp = ggml_cuda_kernel_launch_params(g, b, 0, stream); \
ggml_cuda_kernel_launch(swiglu_fwht_quantize_mmq<NN, NTT>, lp, gate, up, signs, (block_q8_1_mmq *) vy, n_blk, ntok, scale); \
return true; \
}
#define SWQ_CASE(NN) \
case NN: { \
if constexpr (NN >= 1024) { \
if (nt == 1024) SWQ_LAUNCH(NN, 1024) \
} \
if constexpr (NN >= 512) { \
if (nt == 512) SWQ_LAUNCH(NN, 512) \
} \
SWQ_LAUNCH(NN, 256) \
}
switch (n) {
SWQ_CASE(512)
SWQ_CASE(1024)
SWQ_CASE(2048)
SWQ_CASE(4096)
#if !defined(GGML_USE_MUSA)
SWQ_CASE(8192)
#endif // !defined(GGML_USE_MUSA)
default:
return false;
}
#undef SWQ_CASE
#undef SWQ_LAUNCH
}

void quantize_mmq_q8_1_cuda(
const float * x, const int32_t * ids, void * vy, const ggml_type type_src0,
const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03,
Expand Down
6 changes: 6 additions & 0 deletions ggml/src/ggml-cuda/quantize.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -88,3 +88,9 @@ void quantize_scatter_mmq_q8_1_cuda(const float * x,
int64_t nrows_dst,
int n_expert_used,
cudaStream_t stream);

// SwiGLU(gate, up) * signs -> block-wise Hadamard transform (width n, n_blk blocks per row) -> MMQ q8_1 quantization (D4 layout, as
// quantize_mmq_q8_1 writes for PTQ1_0), ntok rows of K = n*n_blk values. Returns false when the width is not covered.
bool ggml_cuda_swiglu_fwht_quantize_mmq(
const float * gate, const float * up, const float * signs, void * vy,
int n, int n_blk, int64_t ntok, cudaStream_t stream);
Loading