From 7f5e243c4e72d4caaa861e16709afbb2301bfc71 Mon Sep 17 00:00:00 2001 From: ravi9 Date: Thu, 5 Nov 2026 03:48:55 +0530 Subject: [PATCH 1/2] ggml-openvino: do not treat the recurrent state gather as out_ids The output_len scan now runs for every model and takes the first GET_ROWS whose index is a graph input. On Qwen3.5 that is the recurrent state gather of build_rs, which indexes the inp->s_copy leaf directly, so output_len became 1. In static (NPU) mode the prompt model then returned one logits row per chunk, and llama-perplexity read stale rows (PPL about 240000 on Qwen3.5-2B). The same predicate also gave the s_copy input the out_ids shape and data in static mode. is_output_idx now skips GET_ROWS that read a recurrent state cache. The old predicate stays as is_get_rows_input_idx for the dynamic-dim bookkeeping, so the s_copy leaf keeps its dynamic length there. --- ggml/src/ggml-openvino/ggml-decoder.cpp | 4 ++-- ggml/src/ggml-openvino/ggml-decoder.h | 16 +++++++++++++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index d88cd81a0a50..1933a1afb066 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -2591,7 +2591,7 @@ void GgmlOvDecoder::compute_node_dynamic_dims() { // root_src = src->org_src; // } if (root_src) { - if (is_inp_tok(root_src, node) || is_inp_pos(root_src, node) || is_output_idx(root_src, node)) { + if (is_inp_tok(root_src, node) || is_inp_pos(root_src, node) || is_get_rows_input_idx(root_src, node)) { m_node_dynamic_dims[root_src] = 0; m_node_dynamic_dims[src] = m_node_dynamic_dims[root_src]; continue; @@ -2599,7 +2599,7 @@ void GgmlOvDecoder::compute_node_dynamic_dims() { self(self, root_src); m_node_dynamic_dims[src] = m_node_dynamic_dims[root_src]; } else { - if (is_inp_tok(src, node) || is_inp_pos(src, node) || is_output_idx(src, node)) { + if (is_inp_tok(src, node) || is_inp_pos(src, node) || is_get_rows_input_idx(src, node)) { m_node_dynamic_dims[src] = 0; continue; } diff --git a/ggml/src/ggml-openvino/ggml-decoder.h b/ggml/src/ggml-openvino/ggml-decoder.h index 503ce24de80d..67399c0555d3 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.h +++ b/ggml/src/ggml-openvino/ggml-decoder.h @@ -608,11 +608,25 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { return m_model_params.swa_mask != nullptr && tensor == m_model_params.swa_mask; } - static bool is_output_idx(const ggml_tensor * tensor, const ggml_tensor * op) { + // A graph-input index list of GET_ROWS: out_ids, or the inp->s_copy leaf when build_rs gathers + // every recurrent state at once. + static bool is_get_rows_input_idx(const ggml_tensor * tensor, const ggml_tensor * op) { return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] && op->src[0]->op != GGML_OP_NONE && op->src[1]->op == GGML_OP_NONE; } + // out_ids, which selects the output rows. The recurrent state gather reads a state cache instead. + static bool is_output_idx(const ggml_tensor * tensor, const ggml_tensor * op) { + if (!is_get_rows_input_idx(tensor, op)) { + return false; + } + const ggml_tensor * data = op->src[0]; + while (data != nullptr && (data->op == GGML_OP_VIEW || data->op == GGML_OP_RESHAPE)) { + data = data->src[0]; + } + return !is_recurrent_cache(data); + } + // the state permutation index input used in SSM/DeltaNet models (inp->s_copy in llama-graph.cpp) static bool is_inp_s_copy(const ggml_tensor * tensor, const ggml_tensor * op) { return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] && From cdc9f86bb401bf515244b33e2b8bcebfb0ef3024 Mon Sep 17 00:00:00 2001 From: ravi9 Date: Thu, 5 Nov 2026 03:49:08 +0530 Subject: [PATCH 2/2] ggml-openvino: prefer the size-preserving dim when a RESHAPE repeats a stride The RESHAPE dynamic-dim rule matches the source's dynamic stride against the result's strides. A size-1 dim repeats the next dim's stride: Gemma 4 E2B has one KV head, so [head_size, n_tokens] -> [head_size, 1, n_tokens] has the same stride on dims 1 and 2. The first match picked the size-1 head dim, so the K/V views lost their dynamic token dim and became static reshapes with the first graph's token count. Every later graph with another token count failed. Prefer the stride match that keeps the dynamic size, and fall back to the first stride match for reshapes that change it. --- ggml/src/ggml-openvino/ggml-decoder.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index 1933a1afb066..332a17f89ff3 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -2723,8 +2723,16 @@ void GgmlOvDecoder::compute_node_dynamic_dims() { if (m_node_dynamic_dims[node->src[0]] != -1) { auto dynamic_dim_idx = m_node_dynamic_dims[node->src[0]]; auto dynamic_dim_stride = node->src[0]->nb[dynamic_dim_idx]; + // Size-1 dims repeat a stride (e.g. a single KV head in [head_size, 1, n_tokens]), so prefer + // the match that keeps the dynamic size and fall back to the first stride match. for (int i = 0; i < GGML_MAX_DIMS; i++) { - if (node->nb[i] == dynamic_dim_stride) { + if (node->nb[i] != dynamic_dim_stride) { + continue; + } + if (m_node_dynamic_dims[node] == -1) { + m_node_dynamic_dims[node] = i; + } + if (node->ne[i] == node->src[0]->ne[dynamic_dim_idx]) { m_node_dynamic_dims[node] = i; break; }