diff --git a/ggml/src/ggml-openvino/utils.cpp b/ggml/src/ggml-openvino/utils.cpp index 7c5f6b7567b3..05d9acd9acdd 100644 --- a/ggml/src/ggml-openvino/utils.cpp +++ b/ggml/src/ggml-openvino/utils.cpp @@ -2192,8 +2192,21 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, const std::shared old_m_params = ggml_decoder->get_model_params(); cache_hit = old_m_params.can_reuse_statically(m_params); if (cache_hit && is_prefill && ggml_openvino_kv_slice_enabled()) { - cache_hit = entry->prefill_attention_size == c_params.attention_size && - entry->prefill_attention_size_swa == c_params.attention_size_swa; + // A larger compiled prefill also serves a smaller attended length: the extra KV rows are masked, as with + // the power-of-two rounding. Without this, a new prompt walks up the sizes again, and since only one + // prefill model is kept, every prompt longer than one chunk recompiles. Head-major KV is bound at the + // storage size, so it keeps the exact match. + // The output count is part of the static prefill shape (more than 1 only for llama-perplexity). + const bool reuse_larger = !ggml_openvino_npu_kv_head_major_enabled(); + cache_hit = entry->prefill_output_len == c_params.output_len && + (reuse_larger ? entry->prefill_attention_size >= c_params.attention_size && + entry->prefill_attention_size_swa >= c_params.attention_size_swa : + entry->prefill_attention_size == c_params.attention_size && + entry->prefill_attention_size_swa == c_params.attention_size_swa); + if (cache_hit) { + c_params.attention_size = entry->prefill_attention_size; + c_params.attention_size_swa = entry->prefill_attention_size_swa; + } prefill_size_changed = !cache_hit; } } @@ -2260,6 +2273,7 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, const std::shared entry->ptr = ggml_decoder; entry->prefill_attention_size = c_params.attention_size; entry->prefill_attention_size_swa = c_params.attention_size_swa; + entry->prefill_output_len = c_params.output_len; entry->decode_attention_size = compiled.decode_attention_size; entry->shared_key = shared_key; entry->decode_window_requests.clear(); @@ -2274,6 +2288,10 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, const std::shared r_ctx->ov_output_names_cache_prefill[key] = compiled.prefill_output_names; decoder_end_time = conversion_end_time = compile_end_time = ggml_time_us(); GGML_LOG_DEBUG("ggml-openvino: shared compiled model HIT (static)\n"); + if (prefill_size_changed) { + GGML_LOG_INFO("ggml-openvino: reuse the compiled prefill model of attention size %d\n", + c_params.attention_size); + } } else { auto decode_c_params = c_params; if (ggml_openvino_kv_slice_enabled()) { @@ -2600,6 +2618,7 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, const std::shared entry->ptr = ggml_decoder; entry->prefill_attention_size = c_params.attention_size; entry->prefill_attention_size_swa = c_params.attention_size_swa; + entry->prefill_output_len = c_params.output_len; entry->decode_attention_size = decode_c_params.attention_size; entry->shared_key = shared_key; entry->decode_window_requests.clear(); diff --git a/ggml/src/ggml-openvino/utils.h b/ggml/src/ggml-openvino/utils.h index 31fa7abc2435..98cfff803c91 100644 --- a/ggml/src/ggml-openvino/utils.h +++ b/ggml/src/ggml-openvino/utils.h @@ -95,9 +95,11 @@ struct decoder_runtime_ctx { std::shared_ptr mutex; std::shared_ptr ptr; - // Attention sizes the cached prefill model was built with; decode calls overwrite the decoder's. + // Attention sizes and output count the cached prefill model was built with; decode calls overwrite the + // decoder's. int prefill_attention_size = -1; int prefill_attention_size_swa = -1; + int prefill_output_len = -1; // KV window of the decode model in the compiled pair (GGML_OPENVINO_NPU_BLOB_PREBUILD). int decode_attention_size = -1; std::string shared_key;