Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 11 additions & 3 deletions ggml/src/ggml-openvino/ggml-decoder.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2591,15 +2591,15 @@ void GgmlOvDecoder::compute_node_dynamic_dims() {
// root_src = src->org_src;
// }
if (root_src) {
if (is_inp_tok(root_src, node) || is_inp_pos(root_src, node) || is_output_idx(root_src, node)) {
if (is_inp_tok(root_src, node) || is_inp_pos(root_src, node) || is_get_rows_input_idx(root_src, node)) {
m_node_dynamic_dims[root_src] = 0;
m_node_dynamic_dims[src] = m_node_dynamic_dims[root_src];
continue;
}
self(self, root_src);
m_node_dynamic_dims[src] = m_node_dynamic_dims[root_src];
} else {
if (is_inp_tok(src, node) || is_inp_pos(src, node) || is_output_idx(src, node)) {
if (is_inp_tok(src, node) || is_inp_pos(src, node) || is_get_rows_input_idx(src, node)) {
m_node_dynamic_dims[src] = 0;
continue;
}
Expand Down Expand Up @@ -2723,8 +2723,16 @@ void GgmlOvDecoder::compute_node_dynamic_dims() {
if (m_node_dynamic_dims[node->src[0]] != -1) {
auto dynamic_dim_idx = m_node_dynamic_dims[node->src[0]];
auto dynamic_dim_stride = node->src[0]->nb[dynamic_dim_idx];
// Size-1 dims repeat a stride (e.g. a single KV head in [head_size, 1, n_tokens]), so prefer
// the match that keeps the dynamic size and fall back to the first stride match.
for (int i = 0; i < GGML_MAX_DIMS; i++) {
if (node->nb[i] == dynamic_dim_stride) {
if (node->nb[i] != dynamic_dim_stride) {
continue;
}
if (m_node_dynamic_dims[node] == -1) {
m_node_dynamic_dims[node] = i;
}
if (node->ne[i] == node->src[0]->ne[dynamic_dim_idx]) {
m_node_dynamic_dims[node] = i;
break;
}
Expand Down
16 changes: 15 additions & 1 deletion ggml/src/ggml-openvino/ggml-decoder.h
Original file line number Diff line number Diff line change
Expand Up @@ -608,11 +608,25 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder {
return m_model_params.swa_mask != nullptr && tensor == m_model_params.swa_mask;
}

static bool is_output_idx(const ggml_tensor * tensor, const ggml_tensor * op) {
// A graph-input index list of GET_ROWS: out_ids, or the inp->s_copy leaf when build_rs gathers
// every recurrent state at once.
static bool is_get_rows_input_idx(const ggml_tensor * tensor, const ggml_tensor * op) {
return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] && op->src[0]->op != GGML_OP_NONE &&
op->src[1]->op == GGML_OP_NONE;
}

// out_ids, which selects the output rows. The recurrent state gather reads a state cache instead.
static bool is_output_idx(const ggml_tensor * tensor, const ggml_tensor * op) {
if (!is_get_rows_input_idx(tensor, op)) {
return false;
}
const ggml_tensor * data = op->src[0];
while (data != nullptr && (data->op == GGML_OP_VIEW || data->op == GGML_OP_RESHAPE)) {
data = data->src[0];
}
return !is_recurrent_cache(data);
}

// the state permutation index input used in SSM/DeltaNet models (inp->s_copy in llama-graph.cpp)
static bool is_inp_s_copy(const ggml_tensor * tensor, const ggml_tensor * op) {
return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] &&
Expand Down