Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
137 changes: 118 additions & 19 deletions ggml/src/ggml-openvino/ggml-decoder.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,65 @@ bool is_moe_expert_sum_add(const ggml_tensor * node) {
return base != nullptr && base->ne[1] > 1 && plane_indices.size() == static_cast<size_t>(base->ne[1]);
}

bool is_mmproj_flash_attn_pattern(const ggml_tensor * node) {
if (node == nullptr || node->op != GGML_OP_FLASH_ATTN_EXT) {
return false;
}
if (node->src[0] == nullptr || node->src[1] == nullptr || node->src[2] == nullptr) {
return false;
}

auto unwrap_cast = [](const ggml_tensor * t) -> const ggml_tensor * {
if (t != nullptr && t->op == GGML_OP_CPY && t->src[0] != nullptr) {
return t->src[0];
}
return t;
};

const ggml_tensor * q = unwrap_cast(node->src[0]);
const ggml_tensor * k = unwrap_cast(node->src[1]);
const ggml_tensor * v = unwrap_cast(node->src[2]);

if (q->op != GGML_OP_PERMUTE || k->op != GGML_OP_PERMUTE || v->op != GGML_OP_PERMUTE) {
return false;
}

if (q->src[0] == nullptr || k->src[0] == nullptr || v->src[0] == nullptr) {
return false;
}

// mmproj attention tensors are direct projections without KV cache
if (k->src[0]->op == GGML_OP_VIEW || v->src[0]->op == GGML_OP_VIEW ||
GgmlOvDecoder::is_cache(k->src[0], nullptr) || GgmlOvDecoder::is_cache(v->src[0], nullptr) ||
GgmlOvDecoder::is_cache(k->src[0]->view_src, nullptr) || GgmlOvDecoder::is_cache(v->src[0]->view_src, nullptr)) {
return false;
}

// Permutation must swap axis 1 and 2: (0, 2, 1, 3)
const int32_t expected_perm[4] = {0, 2, 1, 3};
for (const auto * t : {q, k, v}) {
if (memcmp(t->op_params, expected_perm, sizeof(expected_perm)) != 0) {
return false;
}
}

// Shapes: [d_head, n_pos, n_head, B]
if (q->ne[0] != k->ne[0] || k->ne[0] != v->ne[0]) {
return false;
}
if (k->ne[1] != v->ne[1] || k->ne[2] != v->ne[2] || k->ne[2] == 0) {
return false;
}
if (q->ne[2] % k->ne[2] != 0) {
return false;
}
if (q->ne[3] != k->ne[3] || k->ne[3] != v->ne[3]) {
return false;
}

return true;
}

std::string GgmlOvDecoder::get_tensor_name(const ggml_cgraph * cgraph, const ggml_tensor * tensor) {
if (tensor == nullptr) {
return "";
Expand Down Expand Up @@ -347,8 +406,9 @@ void GgmlOvDecoder::set_input_output() {
current_node_info.node_inputs[src_name] = src;
current_node_info.node_inputs_names.push_back(src_name);

if (src->op == GGML_OP_VIEW) {
// Traverse upward through nested VIEW operations
if (src->op == GGML_OP_VIEW || src->op == GGML_OP_RESHAPE || src->op == GGML_OP_TRANSPOSE ||
src->op == GGML_OP_PERMUTE) {
// Traverse upward through nested VIEW/RESHAPE operations
std::remove_reference_t<decltype(current_node_info.node_inputs_views[src_name])> view_chain;
auto * current = src;

Expand All @@ -358,8 +418,9 @@ void GgmlOvDecoder::set_input_output() {
current_name = get_tensor_graph_input_ov_name(this, m_cgraph, current, node);
}
view_chain.emplace_back(current_name, current);
// If current src is also a VIEW, continue traversing
if (current->src[0] != nullptr && current->src[0]->op == GGML_OP_VIEW) {
// If current src is also a VIEW or RESHAPE, continue traversing
if (current->src[0] != nullptr &&
(current->src[0]->op == GGML_OP_VIEW || current->src[0]->op == GGML_OP_RESHAPE)) {
current = current->src[0];
} else {
break;
Expand Down Expand Up @@ -586,9 +647,11 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const {
op_case = 6;
}
}
} else {
} else if (node->src[0]->src[0]->op == GGML_OP_ROPE) {
// rope'ed query tensor
op_case = 2;
} else {
op_case = 1;
}
break;
}
Expand Down Expand Up @@ -948,7 +1011,7 @@ std::pair<ModelParams, ComputeParams> GgmlOvDecoder::compute_llm_params(ggml_cgr

switch (node->op) {
case GGML_OP_FLASH_ATTN_EXT:
if (node->src[0] == nullptr || node->src[1] == nullptr) {
if (node->src[0] == nullptr || node->src[1] == nullptr || is_mmproj_flash_attn_pattern(node)) {
return -1;
}
switch (node->src[1]->op) {
Expand Down Expand Up @@ -1190,7 +1253,11 @@ std::pair<ModelParams, ComputeParams> GgmlOvDecoder::compute_llm_params(ggml_cgr
}

ggml_tensor * cache_k = cache_k_view->src[0];
int layer = extract_layer_from_name(cache_k->name).value();
auto layer_opt = extract_layer_from_name(cache_k->name);
if (!layer_opt.has_value()) {
continue;
}
int layer = layer_opt.value();

// Classified by the pre-pass above, which groups layers by mask tensor identity. The
// mask NAME cannot be used: build_attn_inp_kq_mask() gives both masks the same name.
Expand Down Expand Up @@ -1904,7 +1971,8 @@ void GgmlOvDecoder::compute_model_outputs() {
for (int node_n = 0; node_n < m_cgraph->n_nodes; node_n++) {
auto * cur_node = m_cgraph->nodes[node_n];
// if the node op is NONE means this node is not used at all, we can skip it directly without adding to model outputs.
if (cur_node->op == GGML_OP_NONE || cur_node->op == GGML_OP_VIEW || cur_node->op == GGML_OP_RESHAPE) {
if (cur_node->op == GGML_OP_NONE || cur_node->op == GGML_OP_VIEW || cur_node->op == GGML_OP_RESHAPE ||
cur_node->op == GGML_OP_TRANSPOSE || cur_node->op == GGML_OP_PERMUTE) {
continue;
}
const bool runtime_recurrent_write = !m_naive && is_recurrent_cache(cur_node->view_src) &&
Expand Down Expand Up @@ -2464,8 +2532,9 @@ size_t GgmlOvDecoder::get_view_input_src_offset(int node_idx, const std::string
if (it != m_node_info_list[node_idx].node_inputs_views.end()) {
if (view_index < it->second.size()) {
auto * view_tensor = it->second[view_index].second;
if (view_tensor && view_tensor->src[0]) {
return view_tensor->src[0]->view_offs;
auto * src_tensor = view_tensor ? (view_tensor->src[0] ? view_tensor->src[0] : view_tensor->view_src) : nullptr;
if (src_tensor) {
return src_tensor->view_offs;
}
}
}
Expand All @@ -2491,8 +2560,9 @@ std::vector<size_t> GgmlOvDecoder::get_view_input_src_stride(int node_idx,
if (it != m_node_info_list[node_idx].node_inputs_views.end()) {
if (view_index < it->second.size()) {
auto * view_tensor = it->second[view_index].second;
if (view_tensor && view_tensor->src[0]) {
return get_stride(view_tensor->src[0]);
auto * src_tensor = view_tensor ? (view_tensor->src[0] ? view_tensor->src[0] : view_tensor->view_src) : nullptr;
if (src_tensor) {
return get_stride(src_tensor);
}
}
}
Expand All @@ -2516,8 +2586,9 @@ ov::Shape GgmlOvDecoder::get_view_input_src_ggml_shape(int node_idx,
if (it != m_node_info_list[node_idx].node_inputs_views.end()) {
if (view_index < it->second.size()) {
auto * view_tensor = it->second[view_index].second;
if (view_tensor && view_tensor->src[0]) {
return get_shape(view_tensor->src[0]);
auto * src_tensor = view_tensor ? (view_tensor->src[0] ? view_tensor->src[0] : view_tensor->view_src) : nullptr;
if (src_tensor) {
return get_shape(src_tensor);
}
}
}
Expand Down Expand Up @@ -2554,8 +2625,8 @@ ov::PartialShape GgmlOvDecoder::get_view_input_src_ov_shape(int node_idx,
if (it != m_node_info_list[node_idx].node_inputs_views.end()) {
if (view_index < it->second.size()) {
auto * view_tensor = it->second[view_index].second;
if (view_tensor && view_tensor->src[0]) {
auto * src_tensor = view_tensor->src[0];
auto * src_tensor = view_tensor ? (view_tensor->src[0] ? view_tensor->src[0] : view_tensor->view_src) : nullptr;
if (src_tensor) {
ov::PartialShape shape = ov::PartialShape{get_shape(src_tensor)};

// Check if this tensor has a dynamic dimension
Expand Down Expand Up @@ -2588,9 +2659,36 @@ std::string GgmlOvDecoder::get_view_input_src_name(int node_idx, const std::stri
if (it != m_node_info_list[node_idx].node_inputs_views.end()) {
if (view_index < it->second.size()) {
auto * view_tensor = it->second[view_index].second;
if (view_tensor && view_tensor->src[0]) {
return get_tensor_ov_name(m_cgraph, view_tensor->src[0]);
if (view_tensor) {
auto * base = view_tensor->src[0] ? view_tensor->src[0] : view_tensor->view_src;
if (base) {
if (is_rope_freqs_weight(base->view_src ? base->view_src : base, m_node_info_list[node_idx].node)) {
return "rope_freqs.weight";
}
if (base->flags & GGML_TENSOR_FLAG_INPUT) {
return get_tensor_graph_input_ov_name(this, m_cgraph, base, m_node_info_list[node_idx].node);
}
return get_tensor_ov_name(m_cgraph, base);
}
}
}
}
return "";
}

std::string GgmlOvDecoder::get_input_view_src_name(int node_idx, const std::string & name) const {
auto it = m_node_info_list[node_idx].node_inputs.find(name);
if (it != m_node_info_list[node_idx].node_inputs.end()) {
auto * src = it->second;
auto * base = src->view_src ? src->view_src : (src->src[0] ? src->src[0] : nullptr);
if (base) {
if (is_rope_freqs_weight(base->view_src ? base->view_src : base, m_node_info_list[node_idx].node)) {
return "rope_freqs.weight";
}
if (base->flags & GGML_TENSOR_FLAG_INPUT) {
return get_tensor_graph_input_ov_name(this, m_cgraph, base, m_node_info_list[node_idx].node);
}
return get_tensor_ov_name(m_cgraph, base);
}
}
return "";
Expand Down Expand Up @@ -2690,7 +2788,8 @@ bool GgmlOvDecoder::is_view_like_alias_of(int node_idx, const std::string & view
if (node->view_src == nullptr || get_tensor_ov_name(m_cgraph, node->view_src) != view_src_name) {
return false;
}
return node->op == GGML_OP_RESHAPE || node->op == GGML_OP_VIEW;
return node->op == GGML_OP_RESHAPE || node->op == GGML_OP_VIEW || node->op == GGML_OP_TRANSPOSE ||
node->op == GGML_OP_PERMUTE;
}

const std::string & GgmlOvDecoder::get_op_name() const {
Expand Down
5 changes: 5 additions & 0 deletions ggml/src/ggml-openvino/ggml-decoder.h
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,9 @@ std::optional<int> extract_layer_from_name(const std::string & name);
// detects the MoE expert-plane-sum ADD chain (see definition); used by supports_op too
bool is_moe_expert_sum_add(const ggml_tensor * node);

// detects the mmproj / vision attention pattern; used by supports_op and compute_llm_params
bool is_mmproj_flash_attn_pattern(const ggml_tensor * node);

class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder {
public:
static std::string get_tensor_name(const ggml_cgraph * cgraph, const ggml_tensor * tensor);
Expand Down Expand Up @@ -251,6 +254,8 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder {

virtual bool is_view_like_alias_of(int node_idx, const std::string & view_src_name) const override;

virtual std::string get_input_view_src_name(int node_idx, const std::string & name) const override;

virtual const std::string & get_op_type() const override;

virtual const std::string & get_op_type(int node_idx) const override;
Expand Down
26 changes: 15 additions & 11 deletions ggml/src/ggml-openvino/ggml-openvino.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -415,19 +415,16 @@ static enum ggml_status ggml_backend_openvino_buffer_init_tensor(ggml_backend_bu
}
}

// Views share the extra from view_src
// Views share the extra from view_src only when shape and data match
if (tensor->view_src != nullptr) {
GGML_ASSERT(tensor->view_src->buffer->buft == buffer->buft);
if (tensor->view_src->extra != nullptr) {
// The cached ov::Tensor carries the shape it was built with, so sharing view_src's
// extra hands out the wrong shape for a reshaping view (e.g. Vcur reshaped from
// [n_embd, n_tokens] to [head_size, n_heads_kv, n_tokens]). When such a view is a
// graph input, binding it fails the shape check. Give it its own extra instead;
// ggml_openvino_create_tensor_extra reads ne and data off the view, so the offset is
// handled too. Only safe for a contiguous view - the ov::Tensor assumes dense strides.
// Skip empty views: they have no data, and on GPU one can sit at the end of the USM buffer.
if (!ggml_are_same_shape(tensor, tensor->view_src) && ggml_is_contiguous(tensor) &&
!ggml_is_quantized(tensor->type) && tensor->data != nullptr && ggml_nbytes(tensor) > 0) {
if (ggml_are_same_shape(tensor, tensor->view_src) && tensor->data == tensor->view_src->data) {
tensor->extra = tensor->view_src->extra;
return GGML_STATUS_SUCCESS;
}
if (ggml_is_contiguous(tensor) && !ggml_is_quantized(tensor->type) &&
tensor->data != nullptr && ggml_nbytes(tensor) > 0) {
if (ggml_openvino_tensor_extra * extra =
ggml_openvino_create_tensor_extra(tensor, ctx->is_remote,
ggml_openvino_get_available_devices()[ctx->device])) {
Expand All @@ -440,7 +437,6 @@ static enum ggml_status ggml_backend_openvino_buffer_init_tensor(ggml_backend_bu
return GGML_STATUS_SUCCESS;
}
}
tensor->extra = tensor->view_src->extra;
}
return GGML_STATUS_SUCCESS;
}
Expand Down Expand Up @@ -1517,6 +1513,10 @@ static bool has_non_contiguous_view_input(const ggml_tensor * op) {
}

static bool is_supported_flash_attn_pattern(const ggml_tensor * op) {
if (is_mmproj_flash_attn_pattern(op)) {
return true;
}

// Each Q/K/V input must follow one of:
// PERMUTE -> VIEW -> base (view_src==nullptr) (llama KV-cache path)
// PERMUTE -> RESHAPE -> base (view_src==nullptr) (whisper Q)
Expand Down Expand Up @@ -1564,6 +1564,10 @@ static bool is_supported_flash_attn_pattern(const ggml_tensor * op) {
}

static bool is_gemma3n_flash_attn_pattern(const ggml_tensor * op) {
if (is_mmproj_flash_attn_pattern(op)) {
return false;
}

if (!is_supported_flash_attn_pattern(op)) {
return false;
}
Expand Down
2 changes: 2 additions & 0 deletions ggml/src/ggml-openvino/openvino/decoder.h
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,8 @@ class GgmlDecoder : public DecoderBase {

virtual bool is_view_like_alias_of(int node_idx, const std::string & view_src_name) const = 0;

virtual std::string get_input_view_src_name(int /*node_idx*/, const std::string & /*name*/) const { return ""; }

virtual const std::string & get_op_type() const = 0;

virtual const std::string & get_op_type(int node_idx) const = 0;
Expand Down
25 changes: 20 additions & 5 deletions ggml/src/ggml-openvino/openvino/node_context.h
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,10 @@ class NodeContext : public frontend::NodeContext {
return m_decoder->get_view_input_size(m_node_idx, m_input_names[index]);
}

std::string get_input_view_src_name(size_t index) const {
return m_decoder->get_input_view_src_name(m_node_idx, m_input_names[index]);
}

size_t get_view_input_offset(size_t index, size_t view_index) const {
return m_decoder->get_view_input_offset(m_node_idx, m_input_names[index], view_index);
}
Expand Down Expand Up @@ -132,18 +136,29 @@ class NodeContext : public frontend::NodeContext {
return base_it->second;
}
if (!base_name.empty()) {
return m_tensor_map->at(base_name);
auto base_it = m_tensor_map->find(base_name);
if (base_it != m_tensor_map->end()) {
return base_it->second;
}
}
}
// Not a VIEW or failed to get base name, use the original logic
return m_tensor_map->at(m_input_names[idx]);
return get_input(m_input_names[idx]);
}

Output<Node> get_input(const std::string & name) const override {
if (m_tensor_map->find(name) == m_tensor_map->end()) {
throw std::runtime_error("'" + name + "' not found in tensor map.");
auto it = m_tensor_map->find(name);
if (it != m_tensor_map->end()) {
return it->second;
}
std::string view_src_name = m_decoder->get_input_view_src_name(m_node_idx, name);
if (!view_src_name.empty()) {
auto src_it = m_tensor_map->find(view_src_name);
if (src_it != m_tensor_map->end()) {
return src_it->second;
}
}
return m_tensor_map->at(name);
throw std::runtime_error("'" + name + "' not found in tensor map.");
}

bool has_input(const std::string & name) const { return m_tensor_map->find(name) != m_tensor_map->end(); }
Expand Down
3 changes: 2 additions & 1 deletion ggml/src/ggml-openvino/openvino/op/view.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,8 @@ OutputVector translate_view(const NodeContext & context) {
// single-expert slice here. Gated by name (ffn_moe_weighted...view) so it can't
// affect any other view.
const std::string & vname = context.get_name();
if (vname.find("ffn_moe_weighted") != std::string::npos) {
if (vname.find("ffn_moe_weighted") != std::string::npos ||
vname.find("per_layer") != std::string::npos) {
auto src_ps = context.get_input_shape(0);
auto dst_ps = context.get_output_shape();
if (src_ps.rank().is_static() && dst_ps.rank().is_static() && src_ps.rank() == dst_ps.rank() &&
Expand Down
9 changes: 7 additions & 2 deletions ggml/src/ggml-openvino/openvino/translate_session.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -360,8 +360,13 @@ std::shared_ptr<Model> TranslateSession::translate_graph(const frontend::InputMo
#ifdef GGML_OPENVINO_HAS_CACHE_UPDATE
converted_outputs = recurrent_fusion.translate(node_idx, node_context);
#endif
if (converted_outputs.empty()) {
converted_outputs = it->second(node_context);
try {
if (converted_outputs.empty()) {
converted_outputs = it->second(node_context);
}
} catch (const std::exception & e) {
OPENVINO_THROW("Error in op translator for '", operation_type, "' while translating node '",
decoder->get_op_name(node_idx), "' (node_idx: ", node_idx, "): ", e.what());
}

FRONT_END_OP_CONVERSION_CHECK(node_output_names.size() == converted_outputs.size(), "Number of ",
Expand Down
Loading