diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index d88cd81a0a50..332a17f89ff3 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -2591,7 +2591,7 @@ void GgmlOvDecoder::compute_node_dynamic_dims() { // root_src = src->org_src; // } if (root_src) { - if (is_inp_tok(root_src, node) || is_inp_pos(root_src, node) || is_output_idx(root_src, node)) { + if (is_inp_tok(root_src, node) || is_inp_pos(root_src, node) || is_get_rows_input_idx(root_src, node)) { m_node_dynamic_dims[root_src] = 0; m_node_dynamic_dims[src] = m_node_dynamic_dims[root_src]; continue; @@ -2599,7 +2599,7 @@ void GgmlOvDecoder::compute_node_dynamic_dims() { self(self, root_src); m_node_dynamic_dims[src] = m_node_dynamic_dims[root_src]; } else { - if (is_inp_tok(src, node) || is_inp_pos(src, node) || is_output_idx(src, node)) { + if (is_inp_tok(src, node) || is_inp_pos(src, node) || is_get_rows_input_idx(src, node)) { m_node_dynamic_dims[src] = 0; continue; } @@ -2723,8 +2723,16 @@ void GgmlOvDecoder::compute_node_dynamic_dims() { if (m_node_dynamic_dims[node->src[0]] != -1) { auto dynamic_dim_idx = m_node_dynamic_dims[node->src[0]]; auto dynamic_dim_stride = node->src[0]->nb[dynamic_dim_idx]; + // Size-1 dims repeat a stride (e.g. a single KV head in [head_size, 1, n_tokens]), so prefer + // the match that keeps the dynamic size and fall back to the first stride match. for (int i = 0; i < GGML_MAX_DIMS; i++) { - if (node->nb[i] == dynamic_dim_stride) { + if (node->nb[i] != dynamic_dim_stride) { + continue; + } + if (m_node_dynamic_dims[node] == -1) { + m_node_dynamic_dims[node] = i; + } + if (node->ne[i] == node->src[0]->ne[dynamic_dim_idx]) { m_node_dynamic_dims[node] = i; break; } diff --git a/ggml/src/ggml-openvino/ggml-decoder.h b/ggml/src/ggml-openvino/ggml-decoder.h index 503ce24de80d..67399c0555d3 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.h +++ b/ggml/src/ggml-openvino/ggml-decoder.h @@ -608,11 +608,25 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { return m_model_params.swa_mask != nullptr && tensor == m_model_params.swa_mask; } - static bool is_output_idx(const ggml_tensor * tensor, const ggml_tensor * op) { + // A graph-input index list of GET_ROWS: out_ids, or the inp->s_copy leaf when build_rs gathers + // every recurrent state at once. + static bool is_get_rows_input_idx(const ggml_tensor * tensor, const ggml_tensor * op) { return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] && op->src[0]->op != GGML_OP_NONE && op->src[1]->op == GGML_OP_NONE; } + // out_ids, which selects the output rows. The recurrent state gather reads a state cache instead. + static bool is_output_idx(const ggml_tensor * tensor, const ggml_tensor * op) { + if (!is_get_rows_input_idx(tensor, op)) { + return false; + } + const ggml_tensor * data = op->src[0]; + while (data != nullptr && (data->op == GGML_OP_VIEW || data->op == GGML_OP_RESHAPE)) { + data = data->src[0]; + } + return !is_recurrent_cache(data); + } + // the state permutation index input used in SSM/DeltaNet models (inp->s_copy in llama-graph.cpp) static bool is_inp_s_copy(const ggml_tensor * tensor, const ggml_tensor * op) { return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] &&