From b4c959b54a4509df670ed0d14d33290ee70b0607 Mon Sep 17 00:00:00 2001 From: Mostafa Faheem Date: Thu, 8 Oct 2026 05:33:40 +0300 Subject: [PATCH 1/2] ggml-openvino: initial MTMD support --- ggml/src/ggml-openvino/ggml-decoder.cpp | 132 +++++++++++++++--- ggml/src/ggml-openvino/ggml-decoder.h | 5 + ggml/src/ggml-openvino/ggml-openvino.cpp | 26 ++-- ggml/src/ggml-openvino/openvino/decoder.h | 2 + .../src/ggml-openvino/openvino/node_context.h | 36 ++++- .../openvino/op/flash_attn_ext.cpp | 6 + ggml/src/ggml-openvino/openvino/op/view.cpp | 3 +- .../openvino/translate_session.cpp | 9 +- ggml/src/ggml-openvino/openvino/utils.cpp | 11 ++ ggml/src/ggml-openvino/utils.cpp | 9 +- 10 files changed, 199 insertions(+), 40 deletions(-) diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index 38db895c532f..bd02f54d7f4f 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -245,6 +245,65 @@ bool is_moe_expert_sum_add(const ggml_tensor * node) { return base != nullptr && base->ne[1] > 1 && plane_indices.size() == static_cast(base->ne[1]); } +bool is_mmproj_flash_attn_pattern(const ggml_tensor * node) { + if (node == nullptr || node->op != GGML_OP_FLASH_ATTN_EXT) { + return false; + } + if (node->src[0] == nullptr || node->src[1] == nullptr || node->src[2] == nullptr) { + return false; + } + + auto unwrap_cast = [](const ggml_tensor * t) -> const ggml_tensor * { + if (t != nullptr && t->op == GGML_OP_CPY && t->src[0] != nullptr) { + return t->src[0]; + } + return t; + }; + + const ggml_tensor * q = unwrap_cast(node->src[0]); + const ggml_tensor * k = unwrap_cast(node->src[1]); + const ggml_tensor * v = unwrap_cast(node->src[2]); + + if (q->op != GGML_OP_PERMUTE || k->op != GGML_OP_PERMUTE || v->op != GGML_OP_PERMUTE) { + return false; + } + + if (q->src[0] == nullptr || k->src[0] == nullptr || v->src[0] == nullptr) { + return false; + } + + // mmproj attention tensors are direct projections without KV cache + if (k->src[0]->op == GGML_OP_VIEW || v->src[0]->op == GGML_OP_VIEW || + GgmlOvDecoder::is_cache(k->src[0], nullptr) || GgmlOvDecoder::is_cache(v->src[0], nullptr) || + GgmlOvDecoder::is_cache(k->src[0]->view_src, nullptr) || GgmlOvDecoder::is_cache(v->src[0]->view_src, nullptr)) { + return false; + } + + // Permutation must swap axis 1 and 2: (0, 2, 1, 3) + const int32_t expected_perm[4] = {0, 2, 1, 3}; + for (const auto * t : {q, k, v}) { + if (memcmp(t->op_params, expected_perm, sizeof(expected_perm)) != 0) { + return false; + } + } + + // Shapes: [d_head, n_pos, n_head, B] + if (q->ne[0] != k->ne[0] || k->ne[0] != v->ne[0]) { + return false; + } + if (k->ne[1] != v->ne[1] || k->ne[2] != v->ne[2] || k->ne[2] == 0) { + return false; + } + if (q->ne[2] % k->ne[2] != 0) { + return false; + } + if (q->ne[3] != k->ne[3] || k->ne[3] != v->ne[3]) { + return false; + } + + return true; +} + std::string GgmlOvDecoder::get_tensor_name(const ggml_cgraph * cgraph, const ggml_tensor * tensor) { if (tensor == nullptr) { return ""; @@ -347,8 +406,8 @@ void GgmlOvDecoder::set_input_output() { current_node_info.node_inputs[src_name] = src; current_node_info.node_inputs_names.push_back(src_name); - if (src->op == GGML_OP_VIEW) { - // Traverse upward through nested VIEW operations + if (src->op == GGML_OP_VIEW || src->op == GGML_OP_RESHAPE) { + // Traverse upward through nested VIEW/RESHAPE operations std::remove_reference_t view_chain; auto * current = src; @@ -358,8 +417,9 @@ void GgmlOvDecoder::set_input_output() { current_name = get_tensor_graph_input_ov_name(this, m_cgraph, current, node); } view_chain.emplace_back(current_name, current); - // If current src is also a VIEW, continue traversing - if (current->src[0] != nullptr && current->src[0]->op == GGML_OP_VIEW) { + // If current src is also a VIEW or RESHAPE, continue traversing + if (current->src[0] != nullptr && + (current->src[0]->op == GGML_OP_VIEW || current->src[0]->op == GGML_OP_RESHAPE)) { current = current->src[0]; } else { break; @@ -492,7 +552,7 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const { if (src->ne[2] * src->ne[3] == node->ne[1]) { op_case = 5; } - } else if (node->ne[0] == 1 && src->ne[0] * src->ne[1] * src->ne[2] == node->ne[1]) { + } else if (node->ne[0] == 1 && src->ne[0] * src->ne[1] * src->ne[2] == node->ne[1] /*&& is_kvcache(src, node)*/) { op_case = 3; } else if (name.find("linear_attn_qkv_mixed") == 0 || name.find("alpha") == 0) { op_case = 6; @@ -586,9 +646,11 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const { op_case = 6; } } - } else { + } else if (node->src[0]->src[0]->op == GGML_OP_ROPE) { // rope'ed query tensor op_case = 2; + } else { + op_case = 1; } break; } @@ -948,7 +1010,7 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr switch (node->op) { case GGML_OP_FLASH_ATTN_EXT: - if (node->src[0] == nullptr || node->src[1] == nullptr) { + if (node->src[0] == nullptr || node->src[1] == nullptr || is_mmproj_flash_attn_pattern(node)) { return -1; } switch (node->src[1]->op) { @@ -1190,7 +1252,11 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr } ggml_tensor * cache_k = cache_k_view->src[0]; - int layer = extract_layer_from_name(cache_k->name).value(); + auto layer_opt = extract_layer_from_name(cache_k->name); + if (!layer_opt.has_value()) { + continue; + } + int layer = layer_opt.value(); // Classified by the pre-pass above, which groups layers by mask tensor identity. The // mask NAME cannot be used: build_attn_inp_kq_mask() gives both masks the same name. @@ -2464,8 +2530,9 @@ size_t GgmlOvDecoder::get_view_input_src_offset(int node_idx, const std::string if (it != m_node_info_list[node_idx].node_inputs_views.end()) { if (view_index < it->second.size()) { auto * view_tensor = it->second[view_index].second; - if (view_tensor && view_tensor->src[0]) { - return view_tensor->src[0]->view_offs; + auto * src_tensor = view_tensor ? (view_tensor->src[0] ? view_tensor->src[0] : view_tensor->view_src) : nullptr; + if (src_tensor) { + return src_tensor->view_offs; } } } @@ -2491,8 +2558,9 @@ std::vector GgmlOvDecoder::get_view_input_src_stride(int node_idx, if (it != m_node_info_list[node_idx].node_inputs_views.end()) { if (view_index < it->second.size()) { auto * view_tensor = it->second[view_index].second; - if (view_tensor && view_tensor->src[0]) { - return get_stride(view_tensor->src[0]); + auto * src_tensor = view_tensor ? (view_tensor->src[0] ? view_tensor->src[0] : view_tensor->view_src) : nullptr; + if (src_tensor) { + return get_stride(src_tensor); } } } @@ -2516,8 +2584,9 @@ ov::Shape GgmlOvDecoder::get_view_input_src_ggml_shape(int node_idx, if (it != m_node_info_list[node_idx].node_inputs_views.end()) { if (view_index < it->second.size()) { auto * view_tensor = it->second[view_index].second; - if (view_tensor && view_tensor->src[0]) { - return get_shape(view_tensor->src[0]); + auto * src_tensor = view_tensor ? (view_tensor->src[0] ? view_tensor->src[0] : view_tensor->view_src) : nullptr; + if (src_tensor) { + return get_shape(src_tensor); } } } @@ -2554,8 +2623,8 @@ ov::PartialShape GgmlOvDecoder::get_view_input_src_ov_shape(int node_idx, if (it != m_node_info_list[node_idx].node_inputs_views.end()) { if (view_index < it->second.size()) { auto * view_tensor = it->second[view_index].second; - if (view_tensor && view_tensor->src[0]) { - auto * src_tensor = view_tensor->src[0]; + auto * src_tensor = view_tensor ? (view_tensor->src[0] ? view_tensor->src[0] : view_tensor->view_src) : nullptr; + if (src_tensor) { ov::PartialShape shape = ov::PartialShape{get_shape(src_tensor)}; // Check if this tensor has a dynamic dimension @@ -2588,9 +2657,36 @@ std::string GgmlOvDecoder::get_view_input_src_name(int node_idx, const std::stri if (it != m_node_info_list[node_idx].node_inputs_views.end()) { if (view_index < it->second.size()) { auto * view_tensor = it->second[view_index].second; - if (view_tensor && view_tensor->src[0]) { - return get_tensor_ov_name(m_cgraph, view_tensor->src[0]); + if (view_tensor) { + auto * base = view_tensor->src[0] ? view_tensor->src[0] : view_tensor->view_src; + if (base) { + if (is_rope_freqs_weight(base->view_src ? base->view_src : base, m_node_info_list[node_idx].node)) { + return "rope_freqs.weight"; + } + if (base->flags & GGML_TENSOR_FLAG_INPUT) { + return get_tensor_graph_input_ov_name(this, m_cgraph, base, m_node_info_list[node_idx].node); + } + return get_tensor_ov_name(m_cgraph, base); + } + } + } + } + return ""; +} + +std::string GgmlOvDecoder::get_input_view_src_name(int node_idx, const std::string & name) const { + auto it = m_node_info_list[node_idx].node_inputs.find(name); + if (it != m_node_info_list[node_idx].node_inputs.end()) { + auto * src = it->second; + auto * base = src->view_src ? src->view_src : (src->src[0] ? src->src[0] : nullptr); + if (base) { + if (is_rope_freqs_weight(base->view_src ? base->view_src : base, m_node_info_list[node_idx].node)) { + return "rope_freqs.weight"; + } + if (base->flags & GGML_TENSOR_FLAG_INPUT) { + return get_tensor_graph_input_ov_name(this, m_cgraph, base, m_node_info_list[node_idx].node); } + return get_tensor_ov_name(m_cgraph, base); } } return ""; diff --git a/ggml/src/ggml-openvino/ggml-decoder.h b/ggml/src/ggml-openvino/ggml-decoder.h index b7ff4b715f06..0ba2d0db055c 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.h +++ b/ggml/src/ggml-openvino/ggml-decoder.h @@ -140,6 +140,9 @@ std::optional extract_layer_from_name(const std::string & name); // detects the MoE expert-plane-sum ADD chain (see definition); used by supports_op too bool is_moe_expert_sum_add(const ggml_tensor * node); +// detects the mmproj / vision attention pattern; used by supports_op and compute_llm_params +bool is_mmproj_flash_attn_pattern(const ggml_tensor * node); + class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { public: static std::string get_tensor_name(const ggml_cgraph * cgraph, const ggml_tensor * tensor); @@ -251,6 +254,8 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { virtual bool is_view_like_alias_of(int node_idx, const std::string & view_src_name) const override; + virtual std::string get_input_view_src_name(int node_idx, const std::string & name) const override; + virtual const std::string & get_op_type() const override; virtual const std::string & get_op_type(int node_idx) const override; diff --git a/ggml/src/ggml-openvino/ggml-openvino.cpp b/ggml/src/ggml-openvino/ggml-openvino.cpp index a7f5d177f527..35504ba8f913 100644 --- a/ggml/src/ggml-openvino/ggml-openvino.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino.cpp @@ -415,19 +415,16 @@ static enum ggml_status ggml_backend_openvino_buffer_init_tensor(ggml_backend_bu } } - // Views share the extra from view_src + // Views share the extra from view_src only when shape and data match if (tensor->view_src != nullptr) { GGML_ASSERT(tensor->view_src->buffer->buft == buffer->buft); if (tensor->view_src->extra != nullptr) { - // The cached ov::Tensor carries the shape it was built with, so sharing view_src's - // extra hands out the wrong shape for a reshaping view (e.g. Vcur reshaped from - // [n_embd, n_tokens] to [head_size, n_heads_kv, n_tokens]). When such a view is a - // graph input, binding it fails the shape check. Give it its own extra instead; - // ggml_openvino_create_tensor_extra reads ne and data off the view, so the offset is - // handled too. Only safe for a contiguous view - the ov::Tensor assumes dense strides. - // Skip empty views: they have no data, and on GPU one can sit at the end of the USM buffer. - if (!ggml_are_same_shape(tensor, tensor->view_src) && ggml_is_contiguous(tensor) && - !ggml_is_quantized(tensor->type) && tensor->data != nullptr && ggml_nbytes(tensor) > 0) { + if (ggml_are_same_shape(tensor, tensor->view_src) && tensor->data == tensor->view_src->data) { + tensor->extra = tensor->view_src->extra; + return GGML_STATUS_SUCCESS; + } + if (ggml_is_contiguous(tensor) && !ggml_is_quantized(tensor->type) && + tensor->data != nullptr && ggml_nbytes(tensor) > 0) { if (ggml_openvino_tensor_extra * extra = ggml_openvino_create_tensor_extra(tensor, ctx->is_remote, ggml_openvino_get_available_devices()[ctx->device])) { @@ -440,7 +437,6 @@ static enum ggml_status ggml_backend_openvino_buffer_init_tensor(ggml_backend_bu return GGML_STATUS_SUCCESS; } } - tensor->extra = tensor->view_src->extra; } return GGML_STATUS_SUCCESS; } @@ -1517,6 +1513,10 @@ static bool has_non_contiguous_view_input(const ggml_tensor * op) { } static bool is_supported_flash_attn_pattern(const ggml_tensor * op) { + if (is_mmproj_flash_attn_pattern(op)) { + return true; + } + // Each Q/K/V input must follow one of: // PERMUTE -> VIEW -> base (view_src==nullptr) (llama KV-cache path) // PERMUTE -> RESHAPE -> base (view_src==nullptr) (whisper Q) @@ -1564,6 +1564,10 @@ static bool is_supported_flash_attn_pattern(const ggml_tensor * op) { } static bool is_gemma3n_flash_attn_pattern(const ggml_tensor * op) { + if (is_mmproj_flash_attn_pattern(op)) { + return false; + } + if (!is_supported_flash_attn_pattern(op)) { return false; } diff --git a/ggml/src/ggml-openvino/openvino/decoder.h b/ggml/src/ggml-openvino/openvino/decoder.h index 980f0b88dc86..de670211ccb5 100644 --- a/ggml/src/ggml-openvino/openvino/decoder.h +++ b/ggml/src/ggml-openvino/openvino/decoder.h @@ -95,6 +95,8 @@ class GgmlDecoder : public DecoderBase { virtual bool is_view_like_alias_of(int node_idx, const std::string & view_src_name) const = 0; + virtual std::string get_input_view_src_name(int /*node_idx*/, const std::string & /*name*/) const { return ""; } + virtual const std::string & get_op_type() const = 0; virtual const std::string & get_op_type(int node_idx) const = 0; diff --git a/ggml/src/ggml-openvino/openvino/node_context.h b/ggml/src/ggml-openvino/openvino/node_context.h index 948405745b95..78ae6f98431d 100644 --- a/ggml/src/ggml-openvino/openvino/node_context.h +++ b/ggml/src/ggml-openvino/openvino/node_context.h @@ -61,6 +61,10 @@ class NodeContext : public frontend::NodeContext { return m_decoder->get_view_input_size(m_node_idx, m_input_names[index]); } + std::string get_input_view_src_name(size_t index) const { + return m_decoder->get_input_view_src_name(m_node_idx, m_input_names[index]); + } + size_t get_view_input_offset(size_t index, size_t view_index) const { return m_decoder->get_view_input_offset(m_node_idx, m_input_names[index], view_index); } @@ -132,18 +136,40 @@ class NodeContext : public frontend::NodeContext { return base_it->second; } if (!base_name.empty()) { - return m_tensor_map->at(base_name); + auto base_it = m_tensor_map->find(base_name); + if (base_it != m_tensor_map->end()) { + return base_it->second; + } } } // Not a VIEW or failed to get base name, use the original logic - return m_tensor_map->at(m_input_names[idx]); + auto it = m_tensor_map->find(m_input_names[idx]); + if (it != m_tensor_map->end()) { + return it->second; + } + std::string view_src_name = m_decoder->get_input_view_src_name(m_node_idx, m_input_names[idx]); + if (!view_src_name.empty()) { + auto src_it = m_tensor_map->find(view_src_name); + if (src_it != m_tensor_map->end()) { + return src_it->second; + } + } + throw std::runtime_error("'" + m_input_names[idx] + "' not found in tensor map."); } Output get_input(const std::string & name) const override { - if (m_tensor_map->find(name) == m_tensor_map->end()) { - throw std::runtime_error("'" + name + "' not found in tensor map."); + auto it = m_tensor_map->find(name); + if (it != m_tensor_map->end()) { + return it->second; + } + std::string view_src_name = m_decoder->get_input_view_src_name(m_node_idx, name); + if (!view_src_name.empty()) { + auto src_it = m_tensor_map->find(view_src_name); + if (src_it != m_tensor_map->end()) { + return src_it->second; + } } - return m_tensor_map->at(name); + throw std::runtime_error("'" + name + "' not found in tensor map."); } bool has_input(const std::string & name) const { return m_tensor_map->find(name) != m_tensor_map->end(); } diff --git a/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp b/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp index 065e92fcddcc..fb32b643f760 100644 --- a/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp +++ b/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp @@ -143,6 +143,12 @@ OutputVector translate_flash_attn_ext(const NodeContext & context) { const float logit_softcap = params[2]; auto q = std::make_shared(q_f32, ov::element::f16); + // if (k.get_element_type() != ov::element::f16) { + // k = std::make_shared(k, ov::element::f16); + // } + // if (v.get_element_type() != ov::element::f16) { + // v = std::make_shared(v, ov::element::f16); + // } auto scale_node = std::make_shared(ov::element::f16, ov::Shape{}, std::vector{scale}); ov::Output res; diff --git a/ggml/src/ggml-openvino/openvino/op/view.cpp b/ggml/src/ggml-openvino/openvino/op/view.cpp index 4ebe9c4b483a..80511cf4c9f2 100644 --- a/ggml/src/ggml-openvino/openvino/op/view.cpp +++ b/ggml/src/ggml-openvino/openvino/op/view.cpp @@ -53,7 +53,8 @@ OutputVector translate_view(const NodeContext & context) { // single-expert slice here. Gated by name (ffn_moe_weighted...view) so it can't // affect any other view. const std::string & vname = context.get_name(); - if (vname.find("ffn_moe_weighted") != std::string::npos) { + if (vname.find("ffn_moe_weighted") != std::string::npos || + vname.find("per_layer") != std::string::npos) { auto src_ps = context.get_input_shape(0); auto dst_ps = context.get_output_shape(); if (src_ps.rank().is_static() && dst_ps.rank().is_static() && src_ps.rank() == dst_ps.rank() && diff --git a/ggml/src/ggml-openvino/openvino/translate_session.cpp b/ggml/src/ggml-openvino/openvino/translate_session.cpp index 3da712f72bdf..5e1a30338580 100644 --- a/ggml/src/ggml-openvino/openvino/translate_session.cpp +++ b/ggml/src/ggml-openvino/openvino/translate_session.cpp @@ -360,8 +360,13 @@ std::shared_ptr TranslateSession::translate_graph(const frontend::InputMo #ifdef GGML_OPENVINO_HAS_CACHE_UPDATE converted_outputs = recurrent_fusion.translate(node_idx, node_context); #endif - if (converted_outputs.empty()) { - converted_outputs = it->second(node_context); + try { + if (converted_outputs.empty()) { + converted_outputs = it->second(node_context); + } + } catch (const std::exception & e) { + OPENVINO_THROW("Error in op translator for '", operation_type, "' while translating node '", + decoder->get_op_name(node_idx), "' (node_idx: ", node_idx, "): ", e.what()); } FRONT_END_OP_CONVERSION_CHECK(node_output_names.size() == converted_outputs.size(), "Number of ", diff --git a/ggml/src/ggml-openvino/openvino/utils.cpp b/ggml/src/ggml-openvino/openvino/utils.cpp index ec7513951c01..3423b21c5737 100644 --- a/ggml/src/ggml-openvino/openvino/utils.cpp +++ b/ggml/src/ggml-openvino/openvino/utils.cpp @@ -461,6 +461,17 @@ ov::Output process_view_input_new(const NodeContext & context, int inp // Check if this input has view inputs size_t view_input_size = context.get_view_input_size(input_index); if (view_input_size == 0) { + auto exp_shape = context.get_input_shape(input_index); + if (exp_shape.is_static() && input.get_partial_shape().is_static() && + exp_shape != input.get_partial_shape() && + ov::shape_size(exp_shape.to_shape()) == ov::shape_size(input.get_shape())) { + auto target_shape = exp_shape.to_shape(); + return std::make_shared( + input, + ov::op::v0::Constant::create(ov::element::i64, {target_shape.size()}, + std::vector(target_shape.begin(), target_shape.end())), + false); + } // No view inputs, return the input as is return input; } diff --git a/ggml/src/ggml-openvino/utils.cpp b/ggml/src/ggml-openvino/utils.cpp index 7c5f6b7567b3..117f86158d11 100644 --- a/ggml/src/ggml-openvino/utils.cpp +++ b/ggml/src/ggml-openvino/utils.cpp @@ -669,9 +669,12 @@ ov::Tensor convert_ggml_input_to_ov(const std::shared_ptr & ggml_ if (ggml_tensor->extra != nullptr && !ggml_decoder->is_splited_model()) { auto * extra_base = static_cast(ggml_tensor->extra); if (extra_base->type == ggml_openvino_extra_base::Type::TENSOR) { - // GGML_LOG_DEBUG("Using ggml_tensor->extra as ov::Tensor for input: %s\n", name.c_str()); auto * tensor_extra = static_cast(extra_base); - return *tensor_extra->tensor; + const auto expected_shape = GgmlOvDecoder::get_shape(ggml_tensor); + if (tensor_extra->tensor && tensor_extra->tensor->get_shape() == expected_shape && + tensor_extra->data == ggml_tensor->data) { + return *tensor_extra->tensor; + } } } @@ -685,7 +688,7 @@ ov::Tensor convert_ggml_input_to_ov(const std::shared_ptr & ggml_ input_shape = GgmlOvDecoder::get_shape(ggml_tensor); } - if (ggml_decoder->is_splited_model() && !ggml_is_contiguous(ggml_tensor)) { + if (!ggml_is_contiguous(ggml_tensor) && (ggml_decoder->is_splited_model() || ggml_tensor->op != GGML_OP_VIEW)) { return make_contiguous_split_input_tensor(ggml_tensor, input_shape); } From e7374045a2bca1a3aafb1b65155a5616f54c00e1 Mon Sep 17 00:00:00 2001 From: Mostafa Faheem Date: Wed, 18 Nov 2026 03:23:34 +0200 Subject: [PATCH 2/2] remove unnecessary changes --- ggml/src/ggml-openvino/ggml-decoder.cpp | 11 +++++++---- ggml/src/ggml-openvino/openvino/node_context.h | 13 +------------ .../ggml-openvino/openvino/op/flash_attn_ext.cpp | 6 ------ 3 files changed, 8 insertions(+), 22 deletions(-) diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index bd02f54d7f4f..d85d765d3a9f 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -406,7 +406,8 @@ void GgmlOvDecoder::set_input_output() { current_node_info.node_inputs[src_name] = src; current_node_info.node_inputs_names.push_back(src_name); - if (src->op == GGML_OP_VIEW || src->op == GGML_OP_RESHAPE) { + if (src->op == GGML_OP_VIEW || src->op == GGML_OP_RESHAPE || src->op == GGML_OP_TRANSPOSE || + src->op == GGML_OP_PERMUTE) { // Traverse upward through nested VIEW/RESHAPE operations std::remove_reference_t view_chain; auto * current = src; @@ -552,7 +553,7 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const { if (src->ne[2] * src->ne[3] == node->ne[1]) { op_case = 5; } - } else if (node->ne[0] == 1 && src->ne[0] * src->ne[1] * src->ne[2] == node->ne[1] /*&& is_kvcache(src, node)*/) { + } else if (node->ne[0] == 1 && src->ne[0] * src->ne[1] * src->ne[2] == node->ne[1]) { op_case = 3; } else if (name.find("linear_attn_qkv_mixed") == 0 || name.find("alpha") == 0) { op_case = 6; @@ -1970,7 +1971,8 @@ void GgmlOvDecoder::compute_model_outputs() { for (int node_n = 0; node_n < m_cgraph->n_nodes; node_n++) { auto * cur_node = m_cgraph->nodes[node_n]; // if the node op is NONE means this node is not used at all, we can skip it directly without adding to model outputs. - if (cur_node->op == GGML_OP_NONE || cur_node->op == GGML_OP_VIEW || cur_node->op == GGML_OP_RESHAPE) { + if (cur_node->op == GGML_OP_NONE || cur_node->op == GGML_OP_VIEW || cur_node->op == GGML_OP_RESHAPE || + cur_node->op == GGML_OP_TRANSPOSE || cur_node->op == GGML_OP_PERMUTE) { continue; } const bool runtime_recurrent_write = !m_naive && is_recurrent_cache(cur_node->view_src) && @@ -2786,7 +2788,8 @@ bool GgmlOvDecoder::is_view_like_alias_of(int node_idx, const std::string & view if (node->view_src == nullptr || get_tensor_ov_name(m_cgraph, node->view_src) != view_src_name) { return false; } - return node->op == GGML_OP_RESHAPE || node->op == GGML_OP_VIEW; + return node->op == GGML_OP_RESHAPE || node->op == GGML_OP_VIEW || node->op == GGML_OP_TRANSPOSE || + node->op == GGML_OP_PERMUTE; } const std::string & GgmlOvDecoder::get_op_name() const { diff --git a/ggml/src/ggml-openvino/openvino/node_context.h b/ggml/src/ggml-openvino/openvino/node_context.h index 78ae6f98431d..9d7aa32666aa 100644 --- a/ggml/src/ggml-openvino/openvino/node_context.h +++ b/ggml/src/ggml-openvino/openvino/node_context.h @@ -143,18 +143,7 @@ class NodeContext : public frontend::NodeContext { } } // Not a VIEW or failed to get base name, use the original logic - auto it = m_tensor_map->find(m_input_names[idx]); - if (it != m_tensor_map->end()) { - return it->second; - } - std::string view_src_name = m_decoder->get_input_view_src_name(m_node_idx, m_input_names[idx]); - if (!view_src_name.empty()) { - auto src_it = m_tensor_map->find(view_src_name); - if (src_it != m_tensor_map->end()) { - return src_it->second; - } - } - throw std::runtime_error("'" + m_input_names[idx] + "' not found in tensor map."); + return get_input(m_input_names[idx]); } Output get_input(const std::string & name) const override { diff --git a/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp b/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp index fb32b643f760..065e92fcddcc 100644 --- a/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp +++ b/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp @@ -143,12 +143,6 @@ OutputVector translate_flash_attn_ext(const NodeContext & context) { const float logit_softcap = params[2]; auto q = std::make_shared(q_f32, ov::element::f16); - // if (k.get_element_type() != ov::element::f16) { - // k = std::make_shared(k, ov::element::f16); - // } - // if (v.get_element_type() != ov::element::f16) { - // v = std::make_shared(v, ov::element::f16); - // } auto scale_node = std::make_shared(ov::element::f16, ov::Shape{}, std::vector{scale}); ov::Output res;