From 4490dfef8ad9f3c9a2fae7450e21f01a55b77e36 Mon Sep 17 00:00:00 2001 From: jileil Date: Mon, 16 Nov 2026 10:51:42 +0800 Subject: [PATCH 1/2] feat(oga): serve gemma4-12b on oga-hipep with shared KV and images Register gemma4-12b on the oga-hipep backend and publish the public Gemma 4 12B GPU package as gemma4-12b-oga-hipep:12b. gemma4_unified sessions reuse the preallocated present buffer and leave graph capture off. JPEG and PNG go through the package image processor. Co-authored-by: Cursor --- src/common/AutoModel/builtin_backends.cpp | 5 + src/common/AutoModel/modeling_gemma4_12b.cpp | 100 ++++++++++++- src/common/oga/oga_engine.cpp | 108 ++++++++++++++ src/common/oga/oga_hipep_backend.cpp | 36 ++++- src/include/oga/oga_api.hpp | 13 +- src/include/oga/oga_engine.hpp | 23 ++- src/model_list.json | 48 +++++++ src/test/oga_hipep/test_oga_hipep.cpp | 144 +++++++++++++++++++ 8 files changed, 470 insertions(+), 7 deletions(-) diff --git a/src/common/AutoModel/builtin_backends.cpp b/src/common/AutoModel/builtin_backends.cpp index 183824689..ff8184946 100644 --- a/src/common/AutoModel/builtin_backends.cpp +++ b/src/common/AutoModel/builtin_backends.cpp @@ -135,6 +135,11 @@ void register_builtin_backends(BackendRegistry& registry) { registry.register_backend("gemma4e", flm::backend::kOgaHipepBackendId, flm::oga::hipep_factory(), flm::oga::hipep_traits()); + // The Gemma 4 12B GPU package uses the gemma4-12b frontend. The tag + // gemma4-12b-oga-hipep selects this backend; details.family stays gemma4-12b. + registry.register_backend("gemma4-12b", flm::backend::kOgaHipepBackendId, + flm::oga::hipep_factory(), + flm::oga::hipep_traits()); registry.register_backend("gpt-oss", flm::backend::kOgaHipepBackendId, flm::oga::hipep_factory(), flm::oga::hipep_traits()); diff --git a/src/common/AutoModel/modeling_gemma4_12b.cpp b/src/common/AutoModel/modeling_gemma4_12b.cpp index 6a5f43c30..7c7bfc6fa 100644 --- a/src/common/AutoModel/modeling_gemma4_12b.cpp +++ b/src/common/AutoModel/modeling_gemma4_12b.cpp @@ -10,8 +10,37 @@ #include "metrices.hpp" #include "error_measure.hpp" #include "models/gemma4_12b/flm/aie2p/gemma4_12b_npu.hpp" +#if defined(FLM_ENABLE_OGA_HIPEP) +#include "base64.hpp" +#include "oga/oga_engine.hpp" +#include +#include +#endif #include +#if defined(FLM_ENABLE_OGA_HIPEP) +namespace { +std::string ImageBytesFromBase64(std::string payload) { + const auto marker = payload.find("base64,"); + if (marker != std::string::npos) payload.erase(0, marker + 7); + std::string bytes = base64::from_base64(payload); + if (bytes.size() < 8) + throw ModelRequestError(400, false, "image payload is not a valid encoded image", "invalid_request_error"); + return bytes; +} + +std::string ImageBytesFromFile(const std::string& path) { + std::ifstream in(path, std::ios::binary); + if (!in) + throw ModelRequestError(400, false, "cannot read image " + path, "invalid_request_error"); + std::string bytes((std::istreambuf_iterator(in)), std::istreambuf_iterator()); + if (bytes.size() < 8) + throw ModelRequestError(400, false, "image file is empty: " + path, "invalid_request_error"); + return bytes; +} +} // namespace +#endif + /************ Gemma4_12B family **************/ Gemma4_12B::Gemma4_12B(flm_rt::device* npu_device_inst) : AutoModel(npu_device_inst, "Gemma4_12B") {} @@ -44,8 +73,11 @@ void Gemma4_12B::load_model(std::string model_path, json model_info, int default for (size_t i = 0; i < PROFILER_TYPE_NUM; i++) { this->profiler_list[i].reset(); } - // Text-only prompts only (see insert), so no image placeholder id is given. - _prefix_attach(lm_engine, gemma4_12b_npu_state_api(lm_engine)); + // The OGA engine has no NPU prefix-slot table. NPU prompts keep theirs. + const flm_state_api* state = this->backend_id() == flm::backend::kOgaHipepBackendId + ? nullptr + : gemma4_12b_npu_state_api(lm_engine); + _prefix_attach(lm_engine, state); } void Gemma4_12B::setup_tokenizer(std::string model_path) { @@ -102,6 +134,16 @@ bool Gemma4_12B::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, } +#if defined(FLM_ENABLE_OGA_HIPEP) + const bool oga = this->backend_id() == flm::backend::kOgaHipepBackendId; + std::vector oga_image_bytes; + if (oga && !input.audios.empty()) { + throw ModelRequestError(400, false, + "gemma4-12b on oga-hipep does not accept audio input", + "invalid_request_error"); + } +#endif + gemma4_12b_image_payload_t image_payload; gemma4_12b_audio_payload_t audio_payload; audio_payload.num_audios = 0; @@ -125,7 +167,14 @@ bool Gemma4_12B::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, if (message.contains("images")) { for (auto& img : message["images"]) { std::string img_str = img.get(); - +#if defined(FLM_ENABLE_OGA_HIPEP) + if (oga) { + oga_image_bytes.push_back(ImageBytesFromBase64(std::move(img_str))); + newContent.push_back({{"type", "image"}, {"image", img}}); + total_images++; + continue; + } +#endif gemma4_12b_image_t image = this->load_image_base64(img_str); if (image.width <= 0 || image.height <= 0) { header_print("ERROR", "Skipping invalid base64 image; prefilling language only for this item"); @@ -153,6 +202,13 @@ bool Gemma4_12B::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, newContent.push_back({{"type", "text"}, {"text", message.value("content", "")}}); // Process Audios if (message.contains("audios")) { +#if defined(FLM_ENABLE_OGA_HIPEP) + if (oga && !message["audios"].empty()) { + throw ModelRequestError(400, false, + "gemma4-12b on oga-hipep does not accept audio input", + "invalid_request_error"); + } +#endif gemma4_12b_npu *gemma4e_engine = dynamic_cast(this->lm_engine); if (!gemma4e_engine) { throw std::runtime_error( @@ -263,6 +319,12 @@ bool Gemma4_12B::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, } if(input.images.size() > 0){ +#if defined(FLM_ENABLE_OGA_HIPEP) + if (oga) { + for (const auto& img_str : input.images) + oga_image_bytes.push_back(ImageBytesFromFile(img_str)); + } else +#endif for (const auto& img_str : input.images) { gemma4_12b_image_t image = this->load_image(img_str); @@ -313,6 +375,38 @@ bool Gemma4_12B::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, templated_text = this->apply_chat_template(messages); } +#if defined(FLM_ENABLE_OGA_HIPEP) + if (oga && !oga_image_bytes.empty()) { + // generate() feeds this empty thought channel itself when thinking is off. + // The processor must not run it, or the four tokens would be in the KV cache twice. + const std::string thought = "<|channel>thought\n"; + if (!this->enable_think && !meta_info.continue_final_message && + templated_text.size() >= thought.size() && + templated_text.compare(templated_text.size() - thought.size(), thought.size(), thought) == 0) { + templated_text.resize(templated_text.size() - thought.size()); + } + auto* engine = dynamic_cast(this->lm_engine); + if (!engine) + throw std::runtime_error("oga-hipep engine is not an OgaEngine"); + std::vector image_tokens = engine->StagePromptImages(templated_text, oga_image_bytes); + this->profiler_list[TKOEN_ENCODE_TIME].stop(image_tokens.size()); + // The processor already ran the whole prompt, so this prefill must not be chunked. + const int saved_prefill = meta_info.max_prefill_len; + meta_info.max_prefill_len = 0; + bool success = false; + try { + success = this->_shared_insert(meta_info, image_tokens, is_cancelled, nullptr); + } catch (...) { + meta_info.max_prefill_len = saved_prefill; + throw; + } + meta_info.max_prefill_len = saved_prefill; + this->_checkpoint_history(); + this->lm_engine->checkpoint(); + return success; + } +#endif + // update the tokens to include the image tokens std::vector tokens; std::vector tokens_init = this->tokenizer->encode(templated_text); diff --git a/src/common/oga/oga_engine.cpp b/src/common/oga/oga_engine.cpp index 2415eaeb2..e2da3beaf 100644 --- a/src/common/oga/oga_engine.cpp +++ b/src/common/oga/oga_engine.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -19,6 +20,43 @@ namespace flm::oga { namespace { static_assert(sizeof(int) == sizeof(std::int32_t), "token ids are passed to OGA as int32_t"); +void Require(const void* function, const char* symbol) { + if (!function) + throw std::runtime_error(std::string("the hip-ep onnxruntime-genai is missing ") + symbol); +} + +std::vector ReadInputIds(const OgaApi& api, OgaTensor* tensor) { + const OgaFunctions& f = api.functions(); + OgaElementType type = OgaElementType_undefined; + api.Check(f.tensor_get_type(tensor, &type), "OgaTensorGetType(input_ids)"); + std::size_t rank = 0; + api.Check(f.tensor_get_shape_rank(tensor, &rank), "OgaTensorGetShapeRank(input_ids)"); + std::vector shape(rank); + if (rank) api.Check(f.tensor_get_shape(tensor, shape.data(), rank), "OgaTensorGetShape(input_ids)"); + if (rank == 2 && shape[0] == 1) shape.erase(shape.begin()); + if (shape.size() != 1 || shape[0] <= 0) + throw std::runtime_error("processor input_ids are not a single token row"); + const auto count = static_cast(shape[0]); + void* data = nullptr; + api.Check(f.tensor_get_data(tensor, &data), "OgaTensorGetData(input_ids)"); + if (!data) throw std::runtime_error("processor input_ids have no data"); + std::vector ids(count); + if (type == OgaElementType_int32) { + const auto* from = static_cast(data); + std::copy(from, from + count, ids.begin()); + } else if (type == OgaElementType_int64) { + const auto* from = static_cast(data); + for (std::size_t i = 0; i < count; ++i) { + if (from[i] < 0 || from[i] > static_cast(std::numeric_limits::max())) + throw std::runtime_error("processor input id does not fit in int"); + ids[i] = static_cast(from[i]); + } + } else { + throw std::runtime_error("processor input_ids are not int32 or int64"); + } + return ids; +} + float HalfToFloat(std::uint16_t half) { const std::uint32_t sign = static_cast(half & 0x8000u) << 16; std::uint32_t exponent = (half >> 10) & 0x1fu; @@ -106,6 +144,24 @@ OgaEngine::~OgaEngine() = default; buffer OgaEngine::forward(int id) { return Run({id}); } buffer OgaEngine::prefill(std::vector& ids, void* payload) { + if (staged_prompt_) { + if (payload) + throw std::invalid_argument(display_name_ + " staged image prompt takes no vision payload"); + std::unique_ptr staged = std::move(staged_prompt_); + if (ids != staged->ids) + throw std::runtime_error(display_name_ + + " image prompt must be prefilled as the processor's whole sequence"); + const OgaFunctions& f = api_->functions(); + live_.reset(); + OgaApi::Owned generator = NewGenerator(max_length_); + api_->Check(f.generator_set_inputs(generator.get(), staged->inputs.get()), + "OgaGenerator_SetInputs"); + logical_ = staged->ids; + images_.clear(); + saved_.reset(); + live_.emplace(Live{std::move(generator), max_length_, logical_, std::nullopt, {}}); + return Logits(live_->generator.get()); + } if (payload && !vision_) throw std::invalid_argument(display_name_ + " on OGA does not run a vision payload"); if (ids.empty()) throw std::invalid_argument(display_name_ + " request contains no token IDs"); @@ -142,6 +198,8 @@ void OgaEngine::clear_context() { logical_.clear(); images_.clear(); saved_.reset(); + // staged_prompt_ stays. The caller clears a divergent history and then + // prefills the image sequence that was just staged. } buffer OgaEngine::get_k_cache(int, int) { @@ -476,6 +534,56 @@ buffer OgaEngine::Logits(OgaGenerator* generator) const { return out; } +std::vector OgaEngine::StagePromptImages(const std::string& prompt, + const std::vector& encoded_images) { + if (encoded_images.empty()) + throw std::invalid_argument(display_name_ + " image prompt has no images"); + if (prompt.empty()) + throw std::invalid_argument(display_name_ + " image prompt is empty"); + const OgaFunctions& f = api_->functions(); + Require(reinterpret_cast(f.load_images_from_buffers), "OgaLoadImagesFromBuffers"); + Require(reinterpret_cast(f.destroy_images), "OgaDestroyImages"); + Require(reinterpret_cast(f.create_multimodal_processor), "OgaCreateMultiModalProcessor"); + Require(reinterpret_cast(f.destroy_multimodal_processor), "OgaDestroyMultiModalProcessor"); + Require(reinterpret_cast(f.processor_process_images), "OgaProcessorProcessImages"); + Require(reinterpret_cast(f.named_tensors_get), "OgaNamedTensorsGet"); + + std::vector buffers; + std::vector sizes; + buffers.reserve(encoded_images.size()); + sizes.reserve(encoded_images.size()); + for (const std::string& image : encoded_images) { + if (image.size() < 8) + throw std::invalid_argument(display_name_ + " image is not an encoded file"); + buffers.push_back(image.data()); + sizes.push_back(image.size()); + } + OgaImages* raw_images = nullptr; + api_->Check(f.load_images_from_buffers(buffers.data(), sizes.data(), buffers.size(), &raw_images), + "OgaLoadImagesFromBuffers"); + OgaApi::Owned images(raw_images, {f.destroy_images}); + OgaMultiModalProcessor* raw_processor = nullptr; + api_->Check(f.create_multimodal_processor(model_.get(), &raw_processor), + "OgaCreateMultiModalProcessor"); + OgaApi::Owned processor(raw_processor, {f.destroy_multimodal_processor}); + OgaNamedTensors* raw_inputs = nullptr; + api_->Check(f.processor_process_images(processor.get(), prompt.c_str(), images.get(), &raw_inputs), + "OgaProcessorProcessImages"); + OgaApi::Owned inputs(raw_inputs, {f.destroy_named_tensors}); + OgaTensor* raw_ids = nullptr; + api_->Check(f.named_tensors_get(inputs.get(), "input_ids", &raw_ids), + "OgaNamedTensorsGet(input_ids)"); + std::vector ids = ReadInputIds(*api_, raw_ids); + CheckIds(ids); + if (ids.size() > max_length_) + throw std::out_of_range(display_name_ + " image prompt has " + std::to_string(ids.size()) + + " tokens, past the context length of " + std::to_string(max_length_)); + staged_prompt_ = std::make_unique(StagedPrompt{std::move(inputs), ids}); + header_print("FLM", "OGA image processor: " << encoded_images.size() << " image(s), " + << ids.size() << " tokens"); + return ids; +} + void OgaEngine::CheckIds(const std::vector& ids) const { for (const int id : ids) if (id < 0 || static_cast(id) >= vocab_size_) diff --git a/src/common/oga/oga_hipep_backend.cpp b/src/common/oga/oga_hipep_backend.cpp index 7967b66a3..56c14306d 100644 --- a/src/common/oga/oga_hipep_backend.cpp +++ b/src/common/oga/oga_hipep_backend.cpp @@ -38,6 +38,12 @@ std::string Text(const std::filesystem::path& path) { constexpr const char* kSessionOverlay = R"({"model":{"decoder":{"session_options":{"session.disable_cpu_ep_fallback":"1"}}}})"; +/// \brief gemma4_unified keeps the preallocated present buffer and does not capture a graph. +/// \note With past_present_share_buffer the present OrtValue is allocated to max_length. +/// A captured graph is baked for one sequence length, so capture stays off. +constexpr const char* kGemmaSearchOverlay = + R"({"model":{"decoder":{"session_options":{"provider_options":[{"AMDGPU":{"profile":"hip","enable_graph_capture":"0","hip_graph_enable":"0"}}]}}},"search":{"past_present_share_buffer":true}})"; + nlohmann::json ReadJson(const std::filesystem::path& path) { flm::file_access::ObserveOpen(path); std::ifstream input(path, std::ios::binary); @@ -93,7 +99,8 @@ struct GenaiConfig { /// \brief whether the engine runs the vision model of a package of this model type /// \note The engine feeds OGA the inputs of its Qwen3.5 vision pipeline (pixel_values per /// patch, image_grid_thw, num_image_tokens). A package of another type that ships a -/// vision model, such as gemma3 or gemma4, runs text only. +/// vision model, such as gemma3 or gemma4, runs text on that pipeline. gemma4_unified +/// images go through the package processor (OgaEngine::StagePromptImages) instead. bool RunsVisionOf(const std::string& type) { return type == "qwen3_5" || type == "qwen3_5_moe"; } @@ -141,6 +148,20 @@ std::int64_t PatchValues(const nlohmann::json& vision_config) { cfg_get(vision_config, "temporal_patch_size", 1) * patch * patch; } +/// \brief whether genai_config.json names a gemma4_unified package +/// \note A missing or unreadable file is not that package. OpenModel is also called +/// on paths that have no package yet. +bool IsGemma4Unified(const std::filesystem::path& root) { + try { + const auto genai = ReadJson(root / "genai_config.json"); + const auto model = genai.find("model"); + return model != genai.end() && model->is_object() && + model->value("type", std::string()) == "gemma4_unified"; + } catch (const std::exception&) { + return false; + } +} + } // namespace OgaApi::Owned OpenModel(const OgaApi& api, const std::filesystem::path& root) { @@ -153,6 +174,19 @@ OgaApi::Owned OpenModel(const OgaApi& api, const std::filesystem::path api.Check(f.config_clear_providers(config.get()), "OgaConfigClearProviders"); api.Check(f.config_append_provider(config.get(), kAmdGpuProviderName), "OgaConfigAppendProvider"); api.Check(f.config_overlay(config.get(), kSessionOverlay), "OgaConfigOverlay"); + if (IsGemma4Unified(root)) { + // ClearProviders drops the package's AMDGPU options. This export's KV + // grows by concatenation, which graph capture rejects, so capture stays off. + for (const char* provider : {"AMDGPU", kAmdGpuProviderName}) { + api.Check(f.config_set_provider_option(config.get(), provider, "profile", "hip"), + "OgaConfigSetProviderOption"); + api.Check(f.config_set_provider_option(config.get(), provider, "enable_graph_capture", "0"), + "OgaConfigSetProviderOption"); + api.Check(f.config_set_provider_option(config.get(), provider, "hip_graph_enable", "0"), + "OgaConfigSetProviderOption"); + } + api.Check(f.config_overlay(config.get(), kGemmaSearchOverlay), "OgaConfigOverlay"); + } OgaModel* raw_model = nullptr; api.Check(f.create_model_from_config(config.get(), &raw_model), "OgaCreateModelFromConfig"); OgaApi::Owned model(raw_model, {f.destroy_model}); diff --git a/src/include/oga/oga_api.hpp b/src/include/oga/oga_api.hpp index ef22bbf1c..19e567f99 100644 --- a/src/include/oga/oga_api.hpp +++ b/src/include/oga/oga_api.hpp @@ -28,6 +28,7 @@ X(create_config, OgaCreateConfig) \ X(config_clear_providers, OgaConfigClearProviders) \ X(config_append_provider, OgaConfigAppendProvider) \ + X(config_set_provider_option, OgaConfigSetProviderOption) \ X(config_overlay, OgaConfigOverlay) \ X(destroy_config, OgaDestroyConfig) \ X(create_model_from_config, OgaCreateModelFromConfig) \ @@ -52,10 +53,18 @@ X(generator_set_inputs, OgaGenerator_SetInputs) /// \brief entry points an OGA may lack; OgaFunctions holds null for each one it does not export -/// \note OGA 0.17 exports no OgaGenerator_SnapshotState. +/// \note OGA 0.17 exports no OgaGenerator_SnapshotState. The image-processor +/// entries are optional so a text-only API still resolves; a prompt that +/// stages images checks them and names the one that is missing. #define FLM_OGA_OPTIONAL_FUNCTIONS(X) \ X(generator_snapshot_state, OgaGenerator_SnapshotState) \ - X(generator_rewind_to, OgaGenerator_RewindTo) + X(generator_rewind_to, OgaGenerator_RewindTo) \ + X(load_images_from_buffers, OgaLoadImagesFromBuffers) \ + X(destroy_images, OgaDestroyImages) \ + X(create_multimodal_processor, OgaCreateMultiModalProcessor) \ + X(destroy_multimodal_processor, OgaDestroyMultiModalProcessor) \ + X(processor_process_images, OgaProcessorProcessImages) \ + X(named_tensors_get, OgaNamedTensorsGet) namespace flm::oga { diff --git a/src/include/oga/oga_engine.hpp b/src/include/oga/oga_engine.hpp index 405f4cf86..ea3eb774b 100644 --- a/src/include/oga/oga_engine.hpp +++ b/src/include/oga/oga_engine.hpp @@ -63,8 +63,18 @@ class OgaEngine final : public causal_lm { std::uint64_t replayed_tokens() const noexcept { return replayed_; } buffer forward(int id) override; + /// \brief Run the package processor on `prompt` and the encoded image bytes, + /// and hold the inputs until the next prefill(). + /// \note Gemma 4 has no vision-start token, so the patch-grid path does not + /// apply. `prompt` has one image placeholder per image. The next + /// prefill() must be given exactly the token ids this returns, as one + /// sequence: the processor already ran all of them. + /// \return the expanded input ids, including each image's soft tokens + std::vector StagePromptImages(const std::string& prompt, + const std::vector& encoded_images); /// \param payload an OgaVisionInput with the images of the runs of image - /// tokens in `ids`, or null when it has none + /// tokens in `ids`, or null when it has none. A staged image prompt + /// ignores this and consumes the processor inputs instead. buffer prefill(std::vector& ids, void* payload = nullptr) override; void set_context_length(int length) override; void load_weights(Q4NX&) override; @@ -99,6 +109,14 @@ class OgaEngine final : public causal_lm { std::uint64_t hash = 0; }; + /// \brief processor inputs waiting for the next prefill of `ids` + /// \note clear_context() does not drop this. A new image prompt diverges + /// from the text history, and that clear happens before prefill. + struct StagedPrompt { + OgaApi::Owned inputs; + std::vector ids; + }; + /// \brief `logical` and its images, as checkpoint() left them struct Saved { std::vector tokens; @@ -157,6 +175,9 @@ class OgaEngine final : public causal_lm { std::optional saved_; // The generator is destroyed before the model it reads. OgaApi::Owned model_; + /// \brief set by StagePromptImages; prefill() takes it. Not cleared with the conversation. + /// \note Declared after the model so processor inputs are released first. + std::unique_ptr staged_prompt_; std::optional live_; }; diff --git a/src/model_list.json b/src/model_list.json index f7ed0ed34..c0bf95d0e 100644 --- a/src/model_list.json +++ b/src/model_list.json @@ -2574,6 +2574,54 @@ } } }, + "gemma4-12b-oga-hipep": { + "12b": { + "name": "Gemma-4-12B-it_gpuep_1.0_gpu_128", + "url": "https://huggingface.co/amd/Gemma-4-12B-it_gpuep_1.0_gpu_128/resolve/adc217f3ae0f54bd55aa049845f132c6706ba9d9", + "file_url": "https://huggingface.co/api/models/amd/Gemma-4-12B-it_gpuep_1.0_gpu_128/tree/adc217f3ae0f54bd55aa049845f132c6706ba9d9", + "size": 9446605606, + "default_context_length": 32768, + "max_prefill_len": 4096, + "details": { + "family": "gemma4-12b", + "think": true, + "think_toggleable": true, + "parameter_size": "12B", + "quantization_level": "INT4 K-quant group-32, fp16 activations" + }, + "flm_min_version": "1.0.14", + "vlm": true, + "files": [ + "audio_encoder/model.onnx", + "audio_encoder/model.onnx.data", + "audio_feature_extraction.json", + "chat_template.jinja", + "config.json", + "decoder/model.onnx", + "decoder/model.onnx.data", + "embedding/model.onnx", + "embedding/model.onnx.data", + "genai_config.json", + "image_processor.json", + "tokenizer.json", + "tokenizer_config.json", + "vision_encoder/model.onnx", + "vision_encoder/model.onnx.data" + ], + "file_sources": { + "config.json": { + "url": "https://huggingface.co/google/gemma-4-12B-it", + "revision": "43c9d014b231474fa7212dbb48a762566c2b255d" + } + }, + "label": [ + "text", + "image", + "reasoning", + "tool-calling" + ] + } + }, "gemma4-it-oga-hipep": { "12b": { "name": "Gemma-4-12B-it_gpuep_1.0_gpu_128", diff --git a/src/test/oga_hipep/test_oga_hipep.cpp b/src/test/oga_hipep/test_oga_hipep.cpp index 0b56f0a06..64d174d4b 100644 --- a/src/test/oga_hipep/test_oga_hipep.cpp +++ b/src/test/oga_hipep/test_oga_hipep.cpp @@ -45,6 +45,10 @@ struct OgaGenerator { /// \brief the length of the last OgaGenerator_SnapshotState std::optional snapshot; }; +struct OgaImages { + std::size_t count = 0; +}; +struct OgaMultiModalProcessor {}; struct OgaTensor { OgaElementType type = OgaElementType_float32; std::vector shape; @@ -99,6 +103,17 @@ struct FakeOga { std::string device = "AMDGPU"; /// \brief each OgaRegisterExecutionProviderLibrary call, as name and path std::vector> registrations; + /// \brief each OgaConfigSetProviderOption call + struct ProviderOption { + std::string provider; + std::string key; + std::string value; + }; + std::vector provider_options; + /// \brief prompts handed to OgaProcessorProcessImages + std::vector processed_prompts; + /// \brief image counts handed to OgaLoadImagesFromBuffers + std::vector loaded_images; std::optional fail_registration; }; @@ -141,6 +156,11 @@ OgaResult* OGA_API_CALL FakeConfigAppendProvider(OgaConfig*, const char* provide g_fake.providers.emplace_back(provider); return nullptr; } +OgaResult* OGA_API_CALL FakeConfigSetProviderOption(OgaConfig*, const char* provider, const char* key, + const char* value) { + g_fake.provider_options.push_back({provider, key, value}); + return nullptr; +} OgaResult* OGA_API_CALL FakeConfigOverlay(OgaConfig*, const char* json) { g_fake.overlays.emplace_back(json); return nullptr; @@ -340,6 +360,42 @@ OgaResult* OGA_API_CALL FakeRewindTo(OgaGenerator* generator, size_t length) { return nullptr; } +OgaResult* OGA_API_CALL FakeLoadImages(const void**, const size_t*, size_t count, OgaImages** out) { + g_fake.loaded_images.push_back(count); + *out = new OgaImages{count}; + return nullptr; +} +void OGA_API_CALL FakeDestroyImages(OgaImages* images) { delete images; } +OgaResult* OGA_API_CALL FakeCreateProcessor(const OgaModel*, OgaMultiModalProcessor** out) { + *out = new OgaMultiModalProcessor; + return nullptr; +} +void OGA_API_CALL FakeDestroyProcessor(OgaMultiModalProcessor* processor) { delete processor; } +OgaResult* OGA_API_CALL FakeProcessImages(const OgaMultiModalProcessor*, const char* prompt, const OgaImages*, + OgaNamedTensors** out) { + g_fake.processed_prompts.emplace_back(prompt); + auto* named = new OgaNamedTensors; + OgaTensor ids; + ids.type = OgaElementType_int32; + ids.shape = {1, 3}; + const std::int32_t tokens[] = {7, 8, 9}; + ids.data.resize(sizeof(tokens)); + std::memcpy(ids.data.data(), tokens, sizeof(tokens)); + named->tensors.emplace("input_ids", std::move(ids)); + OgaTensor pixels; + pixels.type = OgaElementType_float16; + pixels.shape = {1, 0, 0}; + named->tensors.emplace("pixel_values", std::move(pixels)); + *out = named; + return nullptr; +} +OgaResult* OGA_API_CALL FakeNamedTensorsGet(OgaNamedTensors* named, const char* name, OgaTensor** out) { + const auto it = named->tensors.find(name); + if (it == named->tensors.end()) return Error(std::string("no ") + name); + *out = &it->second; + return nullptr; +} + std::map FakeSymbols() { return { {"OgaResultGetError", reinterpret_cast(&FakeResultGetError)}, @@ -349,6 +405,7 @@ std::map FakeSymbols() { {"OgaCreateConfig", reinterpret_cast(&FakeCreateConfig)}, {"OgaConfigClearProviders", reinterpret_cast(&FakeConfigClearProviders)}, {"OgaConfigAppendProvider", reinterpret_cast(&FakeConfigAppendProvider)}, + {"OgaConfigSetProviderOption", reinterpret_cast(&FakeConfigSetProviderOption)}, {"OgaConfigOverlay", reinterpret_cast(&FakeConfigOverlay)}, {"OgaDestroyConfig", reinterpret_cast(&FakeDestroyConfig)}, {"OgaCreateModelFromConfig", reinterpret_cast(&FakeCreateModelFromConfig)}, @@ -373,6 +430,12 @@ std::map FakeSymbols() { {"OgaGenerator_SetInputs", reinterpret_cast(&FakeSetInputs)}, {"OgaGenerator_SnapshotState", reinterpret_cast(&FakeSnapshotState)}, {"OgaGenerator_RewindTo", reinterpret_cast(&FakeRewindTo)}, + {"OgaLoadImagesFromBuffers", reinterpret_cast(&FakeLoadImages)}, + {"OgaDestroyImages", reinterpret_cast(&FakeDestroyImages)}, + {"OgaCreateMultiModalProcessor", reinterpret_cast(&FakeCreateProcessor)}, + {"OgaDestroyMultiModalProcessor", reinterpret_cast(&FakeDestroyProcessor)}, + {"OgaProcessorProcessImages", reinterpret_cast(&FakeProcessImages)}, + {"OgaNamedTensorsGet", reinterpret_cast(&FakeNamedTensorsGet)}, }; } @@ -1180,6 +1243,34 @@ void TestOpenModelPutsTheDecoderOnTheRegisteredEpAlone() { RequireContains(g_fake.overlays[0], R"("decoder":{"session_options":{"session.disable_cpu_ep_fallback":"1"}})"); } +void TestAStagedImagePromptIsTheNextPrefill() { + auto engine = MakeEngine(); + const std::vector images{"PNG_FAKE", "JPEGFAKE"}; + const auto ids = engine->StagePromptImages("see and ", images); + TEST_REQUIRE(ids == (std::vector{7, 8, 9})); + TEST_REQUIRE(g_fake.loaded_images == (std::vector{2})); + TEST_REQUIRE(g_fake.processed_prompts.size() == 1); + TEST_REQUIRE(g_fake.processed_prompts[0] == "see and "); + // Clearing the text history must keep the staged processor inputs. + engine->clear_context(); + std::vector prompt = ids; + engine->prefill(prompt); + TEST_REQUIRE(g_fake.set_inputs.size() == 1); + TEST_REQUIRE(g_fake.set_inputs[0].ids == (std::vector{7, 8, 9})); + TEST_REQUIRE(g_fake.appends.empty()); + TEST_REQUIRE(g_fake.generators_created == 1); +} + +void TestStagingImagesNamesAMissingProcessorEntry() { + auto symbols = FakeSymbols(); + symbols.erase("OgaProcessorProcessImages"); + auto engine = MakeEngineOn({0, 17, 0, 0}, std::move(symbols)); + RequireContains(RequireThrows([&] { + engine->StagePromptImages("see ", {"PNG_FAKE"}); + }), + "missing OgaProcessorProcessImages"); +} + void TestOpenModelRejectsAModelOnAnotherDevice() { g_fake = FakeOga{}; g_fake.device = "CPU"; @@ -2122,6 +2213,54 @@ void TestTheBackendRunsThePackagesImages() { TEST_REQUIRE(backend->replayed_tokens() == 0); } +void TestGemma4UnifiedSharesThePresentBufferAndSkipsGraphCapture() { + g_fake = FakeOga{}; + const auto api = FakeApi(); + PackageDir package(kVocab, "<|im_end|>", + R"(,"image_token_id":40,"vision":{"filename":"vision_encoder/model.onnx"})", + "gemma4_unified"); + TEST_REQUIRE(flm::oga::OpenModel(*api, package.dir.path) != nullptr); + TEST_REQUIRE(g_fake.overlays.size() == 2); + RequireContains(g_fake.overlays[1], R"("past_present_share_buffer":true)"); + RequireContains(g_fake.overlays[1], R"("enable_graph_capture":"0")"); + RequireContains(g_fake.overlays[1], R"("hip_graph_enable":"0")"); + TEST_REQUIRE(g_fake.provider_options.size() == 6); + for (const char* key : {"profile", "enable_graph_capture", "hip_graph_enable"}) { + const auto matches = std::count_if(g_fake.provider_options.begin(), g_fake.provider_options.end(), + [&](const FakeOga::ProviderOption& option) { + return option.key == key; + }); + TEST_REQUIRE(matches == 2); + } +} + +void TestAnotherModelTypeDoesNotShareThePresentBuffer() { + g_fake = FakeOga{}; + const auto api = FakeApi(); + PackageDir package; + TEST_REQUIRE(flm::oga::OpenModel(*api, package.dir.path) != nullptr); + TEST_REQUIRE(g_fake.overlays.size() == 1); + TEST_REQUIRE(g_fake.provider_options.empty()); +} + +void TestTheGemma4UnifiedPackageStillRunsTextWithoutAVisionStartToken() { + g_fake = FakeOga{}; + auto api = flm::oga::OgaApi::GetOrCreateWith([] { return FakeApi(); }); + PackageDir package(kVocab, "<|im_end|>", + R"(,"image_token_id":40,"vision":{"filename":"vision_encoder/model.onnx"})", + "gemma4_unified"); + auto backend = flm::oga::hipep_factory()(package.Context(64)); + RequireContains(g_fake.overlays[1], R"("past_present_share_buffer":true)"); + std::vector text{1, 2}; + backend->engine().prefill(text); + TEST_REQUIRE(g_fake.appends.back() == (Tokens{1, 2})); + const auto image = Image(2, 2); + flm::oga::OgaVisionInput vision{{image}}; + std::vector prompt = WithImage({1}, image, {2}); + RequireContains(RequireThrows([&] { backend->engine().prefill(prompt, &vision); }), + "does not run a vision payload"); +} + void TestTheBackendRunsTheTextOfAnotherTypesVisionPackage() { g_fake = FakeOga{}; auto api = flm::oga::OgaApi::GetOrCreateWith([] { return FakeApi(); }); @@ -2262,6 +2401,11 @@ int main(int argc, char** argv) { RUN(TestARefusedRegistrationNamesTheLibrary); RUN(TestANonAsciiLibraryPathIsRefusedBeforeOgaSeesIt); RUN(TestOpenModelPutsTheDecoderOnTheRegisteredEpAlone); + RUN(TestGemma4UnifiedSharesThePresentBufferAndSkipsGraphCapture); + RUN(TestAnotherModelTypeDoesNotShareThePresentBuffer); + RUN(TestAStagedImagePromptIsTheNextPrefill); + RUN(TestStagingImagesNamesAMissingProcessorEntry); + RUN(TestTheGemma4UnifiedPackageStillRunsTextWithoutAVisionStartToken); RUN(TestOpenModelRejectsAModelOnAnotherDevice); RUN(TestDiscoveryReportsWhyEachRootIsRejected); RUN(TestDiscoveryAcceptsAVersionedPackageWithoutABinDirectory); From f6e238394170c1cecb621b481533e39370e628b1 Mon Sep 17 00:00:00 2001 From: zaneni6 Date: Mon, 16 Nov 2026 13:31:10 -0800 Subject: [PATCH 2/2] fix(oga): pull gemma4-12b-oga-hipep through the shared record; drop a stale staged prompt gemma4-12b-oga-hipep:12b ships the same package as gemma4-it-oga-hipep:12b, so it names that model_info.json record with model_info_key; without it flm pull found no file records for the tag. An image request stages the processor inputs for the next prefill. A request that stopped before its prefill (cancelled, or refused for its length) left the stage behind, and the next request's prefill was checked against it and threw. The image branch now drops the stage after its insert, prefill() takes the stage before refusing it, and StagePromptImages starts from no stage. The image branch also clears a held conversation first, so the staged sequence is prefilled from token 0 and never as a suffix after a cached prefix. Co-Authored-By: Claude --- src/common/AutoModel/modeling_gemma4_12b.cpp | 9 +++++++- src/common/oga/oga_engine.cpp | 6 +++++- src/include/oga/oga_engine.hpp | 6 +++++- src/model_list.json | 1 + src/test/oga_hipep/test_oga_hipep.cpp | 22 ++++++++++++++++++++ 5 files changed, 41 insertions(+), 3 deletions(-) diff --git a/src/common/AutoModel/modeling_gemma4_12b.cpp b/src/common/AutoModel/modeling_gemma4_12b.cpp index fcb6d4e16..9b712d273 100644 --- a/src/common/AutoModel/modeling_gemma4_12b.cpp +++ b/src/common/AutoModel/modeling_gemma4_12b.cpp @@ -394,7 +394,10 @@ bool Gemma4_12B::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, throw std::runtime_error("oga-hipep engine is not an OgaEngine"); std::vector image_tokens = engine->StagePromptImages(templated_text, oga_image_bytes); this->profiler_list[TKOEN_ENCODE_TIME].stop(image_tokens.size()); - // The processor already ran the whole prompt, so this prefill must not be chunked. + // The processor already ran the whole prompt, so this prefill must not be chunked, + // nor start after a cached prefix: the staged inputs are prefilled from token 0. + if (!this->token_history.empty()) + this->clear_context(); const int saved_prefill = meta_info.max_prefill_len; meta_info.max_prefill_len = 0; bool success = false; @@ -402,9 +405,13 @@ bool Gemma4_12B::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, success = this->_shared_insert(meta_info, image_tokens, is_cancelled, nullptr); } catch (...) { meta_info.max_prefill_len = saved_prefill; + engine->DropStagedPrompt(); throw; } meta_info.max_prefill_len = saved_prefill; + // A request that stopped before its prefill (cancelled, or refused for its + // length) must not leave its stage to the next request. + engine->DropStagedPrompt(); this->_checkpoint_history(); this->lm_engine->checkpoint(); return success; diff --git a/src/common/oga/oga_engine.cpp b/src/common/oga/oga_engine.cpp index e2da3beaf..45951d627 100644 --- a/src/common/oga/oga_engine.cpp +++ b/src/common/oga/oga_engine.cpp @@ -145,9 +145,10 @@ buffer OgaEngine::forward(int id) { return Run({id}); } buffer OgaEngine::prefill(std::vector& ids, void* payload) { if (staged_prompt_) { + // Taken first, so a refused prefill does not leave the stage to the next one. + std::unique_ptr staged = std::move(staged_prompt_); if (payload) throw std::invalid_argument(display_name_ + " staged image prompt takes no vision payload"); - std::unique_ptr staged = std::move(staged_prompt_); if (ids != staged->ids) throw std::runtime_error(display_name_ + " image prompt must be prefilled as the processor's whole sequence"); @@ -534,8 +535,11 @@ buffer OgaEngine::Logits(OgaGenerator* generator) const { return out; } +void OgaEngine::DropStagedPrompt() noexcept { staged_prompt_.reset(); } + std::vector OgaEngine::StagePromptImages(const std::string& prompt, const std::vector& encoded_images) { + staged_prompt_.reset(); if (encoded_images.empty()) throw std::invalid_argument(display_name_ + " image prompt has no images"); if (prompt.empty()) diff --git a/src/include/oga/oga_engine.hpp b/src/include/oga/oga_engine.hpp index ea3eb774b..f71d4b75b 100644 --- a/src/include/oga/oga_engine.hpp +++ b/src/include/oga/oga_engine.hpp @@ -72,6 +72,10 @@ class OgaEngine final : public causal_lm { /// \return the expanded input ids, including each image's soft tokens std::vector StagePromptImages(const std::string& prompt, const std::vector& encoded_images); + /// \brief Forget a staged image prompt that no prefill() took. + /// \note A request that stops before its prefill (cancelled, refused) calls + /// this, or the next request's prefill() would be checked against it. + void DropStagedPrompt() noexcept; /// \param payload an OgaVisionInput with the images of the runs of image /// tokens in `ids`, or null when it has none. A staged image prompt /// ignores this and consumes the processor inputs instead. @@ -175,7 +179,7 @@ class OgaEngine final : public causal_lm { std::optional saved_; // The generator is destroyed before the model it reads. OgaApi::Owned model_; - /// \brief set by StagePromptImages; prefill() takes it. Not cleared with the conversation. + /// \brief set by StagePromptImages; prefill() or DropStagedPrompt() takes it. Not cleared with the conversation. /// \note Declared after the model so processor inputs are released first. std::unique_ptr staged_prompt_; std::optional live_; diff --git a/src/model_list.json b/src/model_list.json index c0bf95d0e..6614b98e4 100644 --- a/src/model_list.json +++ b/src/model_list.json @@ -2614,6 +2614,7 @@ "revision": "43c9d014b231474fa7212dbb48a762566c2b255d" } }, + "model_info_key": "gemma4-it-oga-hipep:12b", "label": [ "text", "image", diff --git a/src/test/oga_hipep/test_oga_hipep.cpp b/src/test/oga_hipep/test_oga_hipep.cpp index 64d174d4b..8c0bbea9d 100644 --- a/src/test/oga_hipep/test_oga_hipep.cpp +++ b/src/test/oga_hipep/test_oga_hipep.cpp @@ -1261,6 +1261,27 @@ void TestAStagedImagePromptIsTheNextPrefill() { TEST_REQUIRE(g_fake.generators_created == 1); } +void TestADroppedOrRefusedStageDoesNotReachTheNextPrefill() { + auto engine = MakeEngine(); + // A request that stopped before its prefill drops its stage: the next text prompt runs as text. + engine->StagePromptImages("see ", {"PNG_FAKE"}); + engine->DropStagedPrompt(); + std::vector text{1, 2}; + engine->prefill(text); + TEST_REQUIRE(g_fake.set_inputs.empty()); + TEST_REQUIRE(g_fake.appends == (std::vector{{1, 2}})); + + // A prefill that does not match the stage is refused, and takes the stage with it. + engine->clear_context(); + engine->StagePromptImages("see ", {"PNG_FAKE"}); + std::vector suffix{8, 9}; + RequireContains(RequireThrows([&] { engine->prefill(suffix); }), + "whole sequence"); + std::vector again{1, 2}; + engine->prefill(again); + TEST_REQUIRE(g_fake.set_inputs.empty()); +} + void TestStagingImagesNamesAMissingProcessorEntry() { auto symbols = FakeSymbols(); symbols.erase("OgaProcessorProcessImages"); @@ -2404,6 +2425,7 @@ int main(int argc, char** argv) { RUN(TestGemma4UnifiedSharesThePresentBufferAndSkipsGraphCapture); RUN(TestAnotherModelTypeDoesNotShareThePresentBuffer); RUN(TestAStagedImagePromptIsTheNextPrefill); + RUN(TestADroppedOrRefusedStageDoesNotReachTheNextPrefill); RUN(TestStagingImagesNamesAMissingProcessorEntry); RUN(TestTheGemma4UnifiedPackageStillRunsTextWithoutAVisionStartToken); RUN(TestOpenModelRejectsAModelOnAnotherDevice);