Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ggml
Submodule ggml updated 382 files
25 changes: 9 additions & 16 deletions src/core/ggml_extend.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -344,22 +344,15 @@ ggml_tensor* ggml_ext_pad(ggml_context* ctx,
}

static ggml_tensor* conv_1d(ggml_context* ctx, ggml_tensor* x, ggml_tensor* w, int s0, int p0, int d0, bool force_prec_f32) {
ggml_tensor* result;
if (force_prec_f32) {
ggml_tensor* patches = ggml_im2col(ctx, w, x, s0, 0, p0, 0, d0, 0, false, GGML_TYPE_F32);
result = ggml_mul_mat(ctx,
ggml_reshape_2d(ctx, patches, patches->ne[0], patches->ne[2] * patches->ne[1]),
ggml_reshape_2d(ctx, w, w->ne[0] * w->ne[1], w->ne[2]));
result = ggml_reshape_3d(ctx, result, patches->ne[1], w->ne[2], patches->ne[2]);
} else {
result = ggml_conv_1d(ctx, w, x, s0, p0, d0);
}
if (x->ne[2] > 1) {
// mul_mat packs positions and batches before output channels: [OL, N, OC].
result = ggml_reshape_3d(ctx, result, result->ne[0], x->ne[2], w->ne[2]);
result = ggml_cont(ctx, ggml_permute(ctx, result, 0, 2, 1, 3));
}
return result;
if (!force_prec_f32) {
return ggml_conv_1d(ctx, w, x, s0, p0, d0);
}
ggml_tensor* patches = ggml_im2col(ctx, w, x, s0, 0, p0, 0, d0, 0, false, GGML_TYPE_F32);
ggml_tensor* result = ggml_mul_mat(ctx,
ggml_reshape_2d(ctx, patches, patches->ne[0], patches->ne[2] * patches->ne[1]),
ggml_reshape_2d(ctx, w, w->ne[0] * w->ne[1], w->ne[2]));
result = ggml_reshape_3d(ctx, result, patches->ne[1], patches->ne[2], w->ne[2]);
return ggml_cont(ctx, ggml_permute(ctx, result, 0, 2, 1, 3));
}

ggml_tensor* ggml_ext_conv_1d(ggml_context* ctx,
Expand Down
2 changes: 1 addition & 1 deletion src/model/common/ggml_block.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@
#include <vector>

#include "core/ggml_extend.h"
#include "core/ggml_runner.h"
#include "core/ggml_extend_backend.h"
#include "core/ggml_runner.h"
#include "model.h"

class GGMLBlock {
Expand Down
31 changes: 10 additions & 21 deletions src/model/vae/minimax_h3_audio_vae.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -363,33 +363,22 @@ namespace MiniMaxH3 {
auto pre = std::dynamic_pointer_cast<AudioAttentionProjection>(blocks["pre_block"]);
auto mean_proj = std::dynamic_pointer_cast<LTXV::Conv1D>(blocks["mean_proj"]);

// GGML's batched conv1d storage interleaves the stream dimension
// with output channels. Subsequent layers then read stereo samples
// as adjacent feature channels. Run each mono stream independently,
// matching PyTorch's reshape(B*S, 1, samples), and concatenate only
// the completed normalized latents.
const int64_t streams = waveform->ne[2] * waveform->ne[3];
waveform = ggml_reshape_3d(ctx->ggml_ctx,
waveform,
waveform->ne[0],
1,
streams);
ggml_tensor* stereo_z = nullptr;
for (int64_t stream = 0; stream < streams; ++stream) {
auto mono = ggml_ext_slice(ctx->ggml_ctx, waveform, 2, stream, stream + 1);
auto x = encoder->forward(ctx, mono);
x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3));
x = pre->forward(ctx, x);
x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3));
auto z = mean_proj->forward(ctx, x);

auto mean = ggml_reshape_4d(ctx->ggml_ctx, params["latents_mean"], 1, kLatentChannels, 1, 1);
auto std = ggml_reshape_4d(ctx->ggml_ctx, params["latents_std"], 1, kLatentChannels, 1, 1);
z = ggml_div(ctx->ggml_ctx, ggml_sub(ctx->ggml_ctx, z, mean), std);
z = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, z, 0, 2, 1, 3));
stereo_z = stereo_z == nullptr ? z : ggml_concat(ctx->ggml_ctx, stereo_z, z, 1);
}
return stereo_z;
auto x = encoder->forward(ctx, waveform);
x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3));
x = pre->forward(ctx, x);
x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3));
auto z = mean_proj->forward(ctx, x);

auto mean = ggml_reshape_4d(ctx->ggml_ctx, params["latents_mean"], 1, kLatentChannels, 1, 1);
auto std = ggml_reshape_4d(ctx->ggml_ctx, params["latents_std"], 1, kLatentChannels, 1, 1);
z = ggml_div(ctx->ggml_ctx, ggml_sub(ctx->ggml_ctx, z, mean), std);
return ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, z, 0, 2, 1, 3));
}

ggml_tensor* decode(GGMLRunnerContext* ctx, ggml_tensor* latent) {
Expand Down
Loading