Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 21 additions & 2 deletions ggml/src/ggml-openvino/utils.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2192,8 +2192,21 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, const std::shared
old_m_params = ggml_decoder->get_model_params();
cache_hit = old_m_params.can_reuse_statically(m_params);
if (cache_hit && is_prefill && ggml_openvino_kv_slice_enabled()) {
cache_hit = entry->prefill_attention_size == c_params.attention_size &&
entry->prefill_attention_size_swa == c_params.attention_size_swa;
// A larger compiled prefill also serves a smaller attended length: the extra KV rows are masked, as with
// the power-of-two rounding. Without this, a new prompt walks up the sizes again, and since only one
// prefill model is kept, every prompt longer than one chunk recompiles. Head-major KV is bound at the
// storage size, so it keeps the exact match.
// The output count is part of the static prefill shape (more than 1 only for llama-perplexity).
const bool reuse_larger = !ggml_openvino_npu_kv_head_major_enabled();
cache_hit = entry->prefill_output_len == c_params.output_len &&
(reuse_larger ? entry->prefill_attention_size >= c_params.attention_size &&
entry->prefill_attention_size_swa >= c_params.attention_size_swa :
entry->prefill_attention_size == c_params.attention_size &&
entry->prefill_attention_size_swa == c_params.attention_size_swa);
if (cache_hit) {
c_params.attention_size = entry->prefill_attention_size;
c_params.attention_size_swa = entry->prefill_attention_size_swa;
}
prefill_size_changed = !cache_hit;
}
}
Expand Down Expand Up @@ -2260,6 +2273,7 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, const std::shared
entry->ptr = ggml_decoder;
entry->prefill_attention_size = c_params.attention_size;
entry->prefill_attention_size_swa = c_params.attention_size_swa;
entry->prefill_output_len = c_params.output_len;
entry->decode_attention_size = compiled.decode_attention_size;
entry->shared_key = shared_key;
entry->decode_window_requests.clear();
Expand All @@ -2274,6 +2288,10 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, const std::shared
r_ctx->ov_output_names_cache_prefill[key] = compiled.prefill_output_names;
decoder_end_time = conversion_end_time = compile_end_time = ggml_time_us();
GGML_LOG_DEBUG("ggml-openvino: shared compiled model HIT (static)\n");
if (prefill_size_changed) {
GGML_LOG_INFO("ggml-openvino: reuse the compiled prefill model of attention size %d\n",
c_params.attention_size);
}
} else {
auto decode_c_params = c_params;
if (ggml_openvino_kv_slice_enabled()) {
Expand Down Expand Up @@ -2600,6 +2618,7 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, const std::shared
entry->ptr = ggml_decoder;
entry->prefill_attention_size = c_params.attention_size;
entry->prefill_attention_size_swa = c_params.attention_size_swa;
entry->prefill_output_len = c_params.output_len;
entry->decode_attention_size = decode_c_params.attention_size;
entry->shared_key = shared_key;
entry->decode_window_requests.clear();
Expand Down
4 changes: 3 additions & 1 deletion ggml/src/ggml-openvino/utils.h
Original file line number Diff line number Diff line change
Expand Up @@ -95,9 +95,11 @@ struct decoder_runtime_ctx {

std::shared_ptr<std::mutex> mutex;
std::shared_ptr<GgmlOvDecoder> ptr;
// Attention sizes the cached prefill model was built with; decode calls overwrite the decoder's.
// Attention sizes and output count the cached prefill model was built with; decode calls overwrite the
// decoder's.
int prefill_attention_size = -1;
int prefill_attention_size_swa = -1;
int prefill_output_len = -1;
// KV window of the decode model in the compiled pair (GGML_OPENVINO_NPU_BLOB_PREBUILD).
int decode_attention_size = -1;
std::string shared_key;
Expand Down