diff --git a/src/common/AutoModel/builtin_backends.cpp b/src/common/AutoModel/builtin_backends.cpp index 183824689..ff8184946 100644 --- a/src/common/AutoModel/builtin_backends.cpp +++ b/src/common/AutoModel/builtin_backends.cpp @@ -135,6 +135,11 @@ void register_builtin_backends(BackendRegistry& registry) { registry.register_backend("gemma4e", flm::backend::kOgaHipepBackendId, flm::oga::hipep_factory(), flm::oga::hipep_traits()); + // The Gemma 4 12B GPU package uses the gemma4-12b frontend. The tag + // gemma4-12b-oga-hipep selects this backend; details.family stays gemma4-12b. + registry.register_backend("gemma4-12b", flm::backend::kOgaHipepBackendId, + flm::oga::hipep_factory(), + flm::oga::hipep_traits()); registry.register_backend("gpt-oss", flm::backend::kOgaHipepBackendId, flm::oga::hipep_factory(), flm::oga::hipep_traits()); diff --git a/src/common/AutoModel/modeling_gemma4_12b.cpp b/src/common/AutoModel/modeling_gemma4_12b.cpp index 498803d4c..32bbad91e 100644 --- a/src/common/AutoModel/modeling_gemma4_12b.cpp +++ b/src/common/AutoModel/modeling_gemma4_12b.cpp @@ -11,8 +11,37 @@ #include "metrices.hpp" #include "error_measure.hpp" #include "models/gemma4_12b/flm/aie2p/gemma4_12b_npu.hpp" +#if defined(FLM_ENABLE_OGA_HIPEP) +#include "base64.hpp" +#include "oga/oga_engine.hpp" +#include +#include +#endif #include +#if defined(FLM_ENABLE_OGA_HIPEP) +namespace { +std::string ImageBytesFromBase64(std::string payload) { + const auto marker = payload.find("base64,"); + if (marker != std::string::npos) payload.erase(0, marker + 7); + std::string bytes = base64::from_base64(payload); + if (bytes.size() < 8) + throw ModelRequestError(400, false, "image payload is not a valid encoded image", "invalid_request_error"); + return bytes; +} + +std::string ImageBytesFromFile(const std::string& path) { + std::ifstream in(path, std::ios::binary); + if (!in) + throw ModelRequestError(400, false, "cannot read image " + path, "invalid_request_error"); + std::string bytes((std::istreambuf_iterator(in)), std::istreambuf_iterator()); + if (bytes.size() < 8) + throw ModelRequestError(400, false, "image file is empty: " + path, "invalid_request_error"); + return bytes; +} +} // namespace +#endif + /************ Gemma4_12B family **************/ Gemma4_12B::Gemma4_12B(flm_rt::device* npu_device_inst) : AutoModel(npu_device_inst, "Gemma4_12B") {} @@ -45,8 +74,11 @@ void Gemma4_12B::load_model(std::string model_path, json model_info, int default for (size_t i = 0; i < PROFILER_TYPE_NUM; i++) { this->profiler_list[i].reset(); } - // Text-only prompts only (see insert), so no image placeholder id is given. - _prefix_attach(lm_engine, gemma4_12b_npu_state_api(lm_engine)); + // The OGA engine has no NPU prefix-slot table. NPU prompts keep theirs. + const flm_state_api* state = this->backend_id() == flm::backend::kOgaHipepBackendId + ? nullptr + : gemma4_12b_npu_state_api(lm_engine); + _prefix_attach(lm_engine, state); } void Gemma4_12B::setup_tokenizer(std::string model_path) { @@ -106,6 +138,16 @@ bool Gemma4_12B::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, // before a later return. The trim below still runs only when the prompt ends with the channel. this->feed_empty_thought_ = flm::gemma4_feed_empty_thought(this->enable_think, meta_info.continue_final_message); +#if defined(FLM_ENABLE_OGA_HIPEP) + const bool oga = this->backend_id() == flm::backend::kOgaHipepBackendId; + std::vector oga_image_bytes; + if (oga && !input.audios.empty()) { + throw ModelRequestError(400, false, + "gemma4-12b on oga-hipep does not accept audio input", + "invalid_request_error"); + } +#endif + gemma4_12b_image_payload_t image_payload; gemma4_12b_audio_payload_t audio_payload; audio_payload.num_audios = 0; @@ -129,7 +171,14 @@ bool Gemma4_12B::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, if (message.contains("images")) { for (auto& img : message["images"]) { std::string img_str = img.get(); - +#if defined(FLM_ENABLE_OGA_HIPEP) + if (oga) { + oga_image_bytes.push_back(ImageBytesFromBase64(std::move(img_str))); + newContent.push_back({{"type", "image"}, {"image", img}}); + total_images++; + continue; + } +#endif gemma4_12b_image_t image = this->load_image_base64(img_str); if (image.width <= 0 || image.height <= 0) { header_print("ERROR", "Skipping invalid base64 image; prefilling language only for this item"); @@ -157,6 +206,13 @@ bool Gemma4_12B::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, newContent.push_back({{"type", "text"}, {"text", message.value("content", "")}}); // Process Audios if (message.contains("audios")) { +#if defined(FLM_ENABLE_OGA_HIPEP) + if (oga && !message["audios"].empty()) { + throw ModelRequestError(400, false, + "gemma4-12b on oga-hipep does not accept audio input", + "invalid_request_error"); + } +#endif gemma4_12b_npu *gemma4e_engine = dynamic_cast(this->lm_engine); if (!gemma4e_engine) { throw std::runtime_error( @@ -267,6 +323,12 @@ bool Gemma4_12B::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, } if(input.images.size() > 0){ +#if defined(FLM_ENABLE_OGA_HIPEP) + if (oga) { + for (const auto& img_str : input.images) + oga_image_bytes.push_back(ImageBytesFromFile(img_str)); + } else +#endif for (const auto& img_str : input.images) { gemma4_12b_image_t image = this->load_image(img_str); @@ -317,6 +379,45 @@ bool Gemma4_12B::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, templated_text = this->apply_chat_template(messages); } +#if defined(FLM_ENABLE_OGA_HIPEP) + if (oga && !oga_image_bytes.empty()) { + // generate() feeds this empty thought channel itself when thinking is off. + // The processor must not run it, or the four tokens would be in the KV cache twice. + const std::string thought = "<|channel>thought\n"; + if (!this->enable_think && !meta_info.continue_final_message && + templated_text.size() >= thought.size() && + templated_text.compare(templated_text.size() - thought.size(), thought.size(), thought) == 0) { + templated_text.resize(templated_text.size() - thought.size()); + } + auto* engine = dynamic_cast(this->lm_engine); + if (!engine) + throw std::runtime_error("oga-hipep engine is not an OgaEngine"); + std::vector image_tokens = engine->StagePromptImages(templated_text, oga_image_bytes); + this->profiler_list[TKOEN_ENCODE_TIME].stop(image_tokens.size()); + // The processor already ran the whole prompt, so this prefill must not be chunked, + // nor start after a cached prefix: the staged inputs are prefilled from token 0. + if (!this->token_history.empty()) + this->clear_context(); + const int saved_prefill = meta_info.max_prefill_len; + meta_info.max_prefill_len = 0; + bool success = false; + try { + success = this->_shared_insert(meta_info, image_tokens, is_cancelled, nullptr); + } catch (...) { + meta_info.max_prefill_len = saved_prefill; + engine->DropStagedPrompt(); + throw; + } + meta_info.max_prefill_len = saved_prefill; + // A request that stopped before its prefill (cancelled, or refused for its + // length) must not leave its stage to the next request. + engine->DropStagedPrompt(); + this->_checkpoint_history(); + this->lm_engine->checkpoint(); + return success; + } +#endif + // update the tokens to include the image tokens std::vector tokens; std::vector tokens_init = this->tokenizer->encode(templated_text); diff --git a/src/common/oga/oga_engine.cpp b/src/common/oga/oga_engine.cpp index 7fda8e7ca..e4501229a 100644 --- a/src/common/oga/oga_engine.cpp +++ b/src/common/oga/oga_engine.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -19,6 +20,43 @@ namespace flm::oga { namespace { static_assert(sizeof(int) == sizeof(std::int32_t), "token ids are passed to OGA as int32_t"); +void Require(const void* function, const char* symbol) { + if (!function) + throw std::runtime_error(std::string("the hip-ep onnxruntime-genai is missing ") + symbol); +} + +std::vector ReadInputIds(const OgaApi& api, OgaTensor* tensor) { + const OgaFunctions& f = api.functions(); + OgaElementType type = OgaElementType_undefined; + api.Check(f.tensor_get_type(tensor, &type), "OgaTensorGetType(input_ids)"); + std::size_t rank = 0; + api.Check(f.tensor_get_shape_rank(tensor, &rank), "OgaTensorGetShapeRank(input_ids)"); + std::vector shape(rank); + if (rank) api.Check(f.tensor_get_shape(tensor, shape.data(), rank), "OgaTensorGetShape(input_ids)"); + if (rank == 2 && shape[0] == 1) shape.erase(shape.begin()); + if (shape.size() != 1 || shape[0] <= 0) + throw std::runtime_error("processor input_ids are not a single token row"); + const auto count = static_cast(shape[0]); + void* data = nullptr; + api.Check(f.tensor_get_data(tensor, &data), "OgaTensorGetData(input_ids)"); + if (!data) throw std::runtime_error("processor input_ids have no data"); + std::vector ids(count); + if (type == OgaElementType_int32) { + const auto* from = static_cast(data); + std::copy(from, from + count, ids.begin()); + } else if (type == OgaElementType_int64) { + const auto* from = static_cast(data); + for (std::size_t i = 0; i < count; ++i) { + if (from[i] < 0 || from[i] > static_cast(std::numeric_limits::max())) + throw std::runtime_error("processor input id does not fit in int"); + ids[i] = static_cast(from[i]); + } + } else { + throw std::runtime_error("processor input_ids are not int32 or int64"); + } + return ids; +} + float HalfToFloat(std::uint16_t half) { const std::uint32_t sign = static_cast(half & 0x8000u) << 16; std::uint32_t exponent = (half >> 10) & 0x1fu; @@ -115,6 +153,16 @@ OgaEngine::~OgaEngine() = default; buffer OgaEngine::forward(int id) { return Run({id}); } buffer OgaEngine::prefill(std::vector& ids, void* payload) { + if (staged_prompt_) { + // Taken first, so a refused prefill does not leave the stage to the next one. + std::unique_ptr staged = std::move(staged_prompt_); + if (payload) + throw std::invalid_argument(display_name_ + " staged image prompt takes no vision payload"); + if (ids != staged->ids) + throw std::runtime_error(display_name_ + + " image prompt must be prefilled as the processor's whole sequence"); + return PrefillStaged(*staged); + } if (payload && !vision_) throw std::invalid_argument(display_name_ + " on OGA does not run a vision payload"); if (ids.empty()) throw std::invalid_argument(display_name_ + " request contains no token IDs"); @@ -158,6 +206,8 @@ void OgaEngine::clear_context() { images_.clear(); expected_.clear(); saved_.reset(); + // staged_prompt_ stays. The caller clears a divergent history and then + // prefills the image sequence that was just staged. } void OgaEngine::expect_prefill(const std::vector& ids) { @@ -601,6 +651,86 @@ buffer OgaEngine::Logits(OgaGenerator* generator) const { return out; } +buffer OgaEngine::PrefillStaged(const StagedPrompt& staged) { + // The processor inputs hold the whole sequence and its images, which OGA takes only in a + // generator's first call: a new generator, in a free slot or the least recently used one. + active_.reset(); // so that a failure before Rebuild drops no other conversation's slot + try { + const char* why = nullptr; + Rebuild(std::nullopt, &why); + Live& run = Active(); + api_->Check(api_->functions().generator_set_inputs(run.generator.get(), staged.inputs.get()), + "OgaGenerator_SetInputs"); + buffer logits = Logits(run.generator.get()); + // Only now: a failed prefill leaves `logical` without image tokens that a later + // rebuild would replay without their images. + run.ran = staged.ids; + logical_ = staged.ids; + images_.clear(); + expected_.clear(); + saved_.reset(); + header_print("FLM", "OGA generator: image prompt of " << logical_.size() << " tokens" + << SlotText(why)); + return logits; + } catch (...) { + if (active_) Drop(*active_); + throw; + } +} + +void OgaEngine::DropStagedPrompt() noexcept { staged_prompt_.reset(); } + +std::vector OgaEngine::StagePromptImages(const std::string& prompt, + const std::vector& encoded_images) { + staged_prompt_.reset(); + if (encoded_images.empty()) + throw std::invalid_argument(display_name_ + " image prompt has no images"); + if (prompt.empty()) + throw std::invalid_argument(display_name_ + " image prompt is empty"); + const OgaFunctions& f = api_->functions(); + Require(reinterpret_cast(f.load_images_from_buffers), "OgaLoadImagesFromBuffers"); + Require(reinterpret_cast(f.destroy_images), "OgaDestroyImages"); + Require(reinterpret_cast(f.create_multimodal_processor), "OgaCreateMultiModalProcessor"); + Require(reinterpret_cast(f.destroy_multimodal_processor), "OgaDestroyMultiModalProcessor"); + Require(reinterpret_cast(f.processor_process_images), "OgaProcessorProcessImages"); + Require(reinterpret_cast(f.named_tensors_get), "OgaNamedTensorsGet"); + + std::vector buffers; + std::vector sizes; + buffers.reserve(encoded_images.size()); + sizes.reserve(encoded_images.size()); + for (const std::string& image : encoded_images) { + if (image.size() < 8) + throw std::invalid_argument(display_name_ + " image is not an encoded file"); + buffers.push_back(image.data()); + sizes.push_back(image.size()); + } + OgaImages* raw_images = nullptr; + api_->Check(f.load_images_from_buffers(buffers.data(), sizes.data(), buffers.size(), &raw_images), + "OgaLoadImagesFromBuffers"); + OgaApi::Owned images(raw_images, {f.destroy_images}); + OgaMultiModalProcessor* raw_processor = nullptr; + api_->Check(f.create_multimodal_processor(model_.get(), &raw_processor), + "OgaCreateMultiModalProcessor"); + OgaApi::Owned processor(raw_processor, {f.destroy_multimodal_processor}); + OgaNamedTensors* raw_inputs = nullptr; + api_->Check(f.processor_process_images(processor.get(), prompt.c_str(), images.get(), &raw_inputs), + "OgaProcessorProcessImages"); + OgaApi::Owned inputs(raw_inputs, {f.destroy_named_tensors}); + OgaTensor* raw_ids = nullptr; + api_->Check(f.named_tensors_get(inputs.get(), "input_ids", &raw_ids), + "OgaNamedTensorsGet(input_ids)"); + std::vector ids = ReadInputIds(*api_, raw_ids); + CheckIds(ids); + if (ids.size() > max_length_) + throw std::out_of_range(display_name_ + " image prompt has " + std::to_string(ids.size()) + + " tokens, past the context length of " + std::to_string(max_length_)); + staged_prompt_ = std::make_unique(StagedPrompt{std::move(inputs), ids}); + header_print("FLM", "OGA image processor: " << encoded_images.size() << " image(s), " + << ids.size() << " tokens"); + return ids; +} + void OgaEngine::CheckIds(const std::vector& ids) const { for (const int id : ids) if (id < 0 || static_cast(id) >= vocab_size_) diff --git a/src/common/oga/oga_hipep_backend.cpp b/src/common/oga/oga_hipep_backend.cpp index 4bfad1b99..5eaf0f604 100644 --- a/src/common/oga/oga_hipep_backend.cpp +++ b/src/common/oga/oga_hipep_backend.cpp @@ -39,6 +39,12 @@ std::string Text(const std::filesystem::path& path) { constexpr const char* kSessionOverlay = R"({"model":{"decoder":{"session_options":{"session.disable_cpu_ep_fallback":"1"}}}})"; +/// \brief gemma4_unified keeps the preallocated present buffer and does not capture a graph. +/// \note With past_present_share_buffer the present OrtValue is allocated to max_length. +/// A captured graph is baked for one sequence length, so capture stays off. +constexpr const char* kGemmaSearchOverlay = + R"({"model":{"decoder":{"session_options":{"provider_options":[{"AMDGPU":{"profile":"hip","enable_graph_capture":"0","hip_graph_enable":"0"}}]}}},"search":{"past_present_share_buffer":true}})"; + nlohmann::json ReadJson(const std::filesystem::path& path) { flm::file_access::ObserveOpen(path); std::ifstream input(path, std::ios::binary); @@ -98,7 +104,8 @@ struct GenaiConfig { /// \brief whether the engine runs the vision model of a package of this model type /// \note The engine feeds OGA the inputs of its Qwen3.5 vision pipeline (pixel_values per /// patch, image_grid_thw, num_image_tokens). A package of another type that ships a -/// vision model, such as gemma3 or gemma4, runs text only. +/// vision model, such as gemma3 or gemma4, runs text on that pipeline. gemma4_unified +/// images go through the package processor (OgaEngine::StagePromptImages) instead. bool RunsVisionOf(const std::string& type) { return type == "qwen3_5" || type == "qwen3_5_moe"; } @@ -151,6 +158,20 @@ std::int64_t PatchValues(const nlohmann::json& vision_config) { cfg_get(vision_config, "temporal_patch_size", 1) * patch * patch; } +/// \brief whether genai_config.json names a gemma4_unified package +/// \note A missing or unreadable file is not that package. OpenModel is also called +/// on paths that have no package yet. +bool IsGemma4Unified(const std::filesystem::path& root) { + try { + const auto genai = ReadJson(root / "genai_config.json"); + const auto model = genai.find("model"); + return model != genai.end() && model->is_object() && + model->value("type", std::string()) == "gemma4_unified"; + } catch (const std::exception&) { + return false; + } +} + /// \brief generators kept by default (--prefix-slots -1), and the most it takes constexpr std::size_t kDefaultSlots = 4; constexpr std::size_t kMostSlots = 32; @@ -202,6 +223,19 @@ OgaApi::Owned OpenModel(const OgaApi& api, const std::filesystem::path api.Check(f.config_clear_providers(config.get()), "OgaConfigClearProviders"); api.Check(f.config_append_provider(config.get(), kAmdGpuProviderName), "OgaConfigAppendProvider"); api.Check(f.config_overlay(config.get(), kSessionOverlay), "OgaConfigOverlay"); + if (IsGemma4Unified(root)) { + // ClearProviders drops the package's AMDGPU options. This export's KV + // grows by concatenation, which graph capture rejects, so capture stays off. + for (const char* provider : {"AMDGPU", kAmdGpuProviderName}) { + api.Check(f.config_set_provider_option(config.get(), provider, "profile", "hip"), + "OgaConfigSetProviderOption"); + api.Check(f.config_set_provider_option(config.get(), provider, "enable_graph_capture", "0"), + "OgaConfigSetProviderOption"); + api.Check(f.config_set_provider_option(config.get(), provider, "hip_graph_enable", "0"), + "OgaConfigSetProviderOption"); + } + api.Check(f.config_overlay(config.get(), kGemmaSearchOverlay), "OgaConfigOverlay"); + } OgaModel* raw_model = nullptr; api.Check(f.create_model_from_config(config.get(), &raw_model), "OgaCreateModelFromConfig"); OgaApi::Owned model(raw_model, {f.destroy_model}); diff --git a/src/include/oga/oga_api.hpp b/src/include/oga/oga_api.hpp index ef22bbf1c..19e567f99 100644 --- a/src/include/oga/oga_api.hpp +++ b/src/include/oga/oga_api.hpp @@ -28,6 +28,7 @@ X(create_config, OgaCreateConfig) \ X(config_clear_providers, OgaConfigClearProviders) \ X(config_append_provider, OgaConfigAppendProvider) \ + X(config_set_provider_option, OgaConfigSetProviderOption) \ X(config_overlay, OgaConfigOverlay) \ X(destroy_config, OgaDestroyConfig) \ X(create_model_from_config, OgaCreateModelFromConfig) \ @@ -52,10 +53,18 @@ X(generator_set_inputs, OgaGenerator_SetInputs) /// \brief entry points an OGA may lack; OgaFunctions holds null for each one it does not export -/// \note OGA 0.17 exports no OgaGenerator_SnapshotState. +/// \note OGA 0.17 exports no OgaGenerator_SnapshotState. The image-processor +/// entries are optional so a text-only API still resolves; a prompt that +/// stages images checks them and names the one that is missing. #define FLM_OGA_OPTIONAL_FUNCTIONS(X) \ X(generator_snapshot_state, OgaGenerator_SnapshotState) \ - X(generator_rewind_to, OgaGenerator_RewindTo) + X(generator_rewind_to, OgaGenerator_RewindTo) \ + X(load_images_from_buffers, OgaLoadImagesFromBuffers) \ + X(destroy_images, OgaDestroyImages) \ + X(create_multimodal_processor, OgaCreateMultiModalProcessor) \ + X(destroy_multimodal_processor, OgaDestroyMultiModalProcessor) \ + X(processor_process_images, OgaProcessorProcessImages) \ + X(named_tensors_get, OgaNamedTensorsGet) namespace flm::oga { diff --git a/src/include/oga/oga_engine.hpp b/src/include/oga/oga_engine.hpp index f8f83ceac..eb9e1cfb5 100644 --- a/src/include/oga/oga_engine.hpp +++ b/src/include/oga/oga_engine.hpp @@ -94,8 +94,22 @@ class OgaEngine final : public causal_lm { void expect_prefill(const std::vector& ids); buffer forward(int id) override; + /// \brief Run the package processor on `prompt` and the encoded image bytes, + /// and hold the inputs until the next prefill(). + /// \note Gemma 4 has no vision-start token, so the patch-grid path does not + /// apply. `prompt` has one image placeholder per image. The next + /// prefill() must be given exactly the token ids this returns, as one + /// sequence: the processor already ran all of them. + /// \return the expanded input ids, including each image's soft tokens + std::vector StagePromptImages(const std::string& prompt, + const std::vector& encoded_images); + /// \brief Forget a staged image prompt that no prefill() took. + /// \note A request that stops before its prefill (cancelled, refused) calls + /// this, or the next request's prefill() would be checked against it. + void DropStagedPrompt() noexcept; /// \param payload an OgaVisionInput with the images of the runs of image - /// tokens in `ids`, or null when it has none + /// tokens in `ids`, or null when it has none. A staged image prompt + /// ignores this and consumes the processor inputs instead. buffer prefill(std::vector& ids, void* payload = nullptr) override; void set_context_length(int length) override; void load_weights(Q4NX&) override; @@ -136,6 +150,14 @@ class OgaEngine final : public causal_lm { std::uint64_t hash = 0; }; + /// \brief processor inputs waiting for the next prefill of `ids` + /// \note clear_context() does not drop this. A new image prompt diverges + /// from the text history, and that clear happens before prefill. + struct StagedPrompt { + OgaApi::Owned inputs; + std::vector ids; + }; + /// \brief `logical` and its images, as checkpoint() left them struct Saved { std::vector tokens; @@ -181,6 +203,8 @@ class OgaEngine final : public causal_lm { void Trim(); /// \brief " (slot i of n)" when there is more than one slot, for the log std::string SlotText(const char* why = nullptr) const; + /// \brief give a new slot the staged processor inputs; `logical` becomes their ids + buffer PrefillStaged(const StagedPrompt& staged); /// \brief the active slot Live& Active() { return slots_[*active_]; } OgaApi::Owned NewGenerator(std::uint32_t max_length) const; @@ -223,6 +247,9 @@ class OgaEngine final : public causal_lm { std::optional saved_; // The generators are destroyed before the model they read. OgaApi::Owned model_; + /// \brief set by StagePromptImages; prefill() or DropStagedPrompt() takes it. Not cleared with the conversation. + /// \note Declared after the model so processor inputs are released first. + std::unique_ptr staged_prompt_; std::vector slots_; /// \brief the slot the last pass ran in, which checkpoint() snapshots std::optional active_; diff --git a/src/model_list.json b/src/model_list.json index f7ed0ed34..6614b98e4 100644 --- a/src/model_list.json +++ b/src/model_list.json @@ -2574,6 +2574,55 @@ } } }, + "gemma4-12b-oga-hipep": { + "12b": { + "name": "Gemma-4-12B-it_gpuep_1.0_gpu_128", + "url": "https://huggingface.co/amd/Gemma-4-12B-it_gpuep_1.0_gpu_128/resolve/adc217f3ae0f54bd55aa049845f132c6706ba9d9", + "file_url": "https://huggingface.co/api/models/amd/Gemma-4-12B-it_gpuep_1.0_gpu_128/tree/adc217f3ae0f54bd55aa049845f132c6706ba9d9", + "size": 9446605606, + "default_context_length": 32768, + "max_prefill_len": 4096, + "details": { + "family": "gemma4-12b", + "think": true, + "think_toggleable": true, + "parameter_size": "12B", + "quantization_level": "INT4 K-quant group-32, fp16 activations" + }, + "flm_min_version": "1.0.14", + "vlm": true, + "files": [ + "audio_encoder/model.onnx", + "audio_encoder/model.onnx.data", + "audio_feature_extraction.json", + "chat_template.jinja", + "config.json", + "decoder/model.onnx", + "decoder/model.onnx.data", + "embedding/model.onnx", + "embedding/model.onnx.data", + "genai_config.json", + "image_processor.json", + "tokenizer.json", + "tokenizer_config.json", + "vision_encoder/model.onnx", + "vision_encoder/model.onnx.data" + ], + "file_sources": { + "config.json": { + "url": "https://huggingface.co/google/gemma-4-12B-it", + "revision": "43c9d014b231474fa7212dbb48a762566c2b255d" + } + }, + "model_info_key": "gemma4-it-oga-hipep:12b", + "label": [ + "text", + "image", + "reasoning", + "tool-calling" + ] + } + }, "gemma4-it-oga-hipep": { "12b": { "name": "Gemma-4-12B-it_gpuep_1.0_gpu_128", diff --git a/src/test/oga_hipep/test_oga_hipep.cpp b/src/test/oga_hipep/test_oga_hipep.cpp index af0f5a7de..f19d3bb27 100644 --- a/src/test/oga_hipep/test_oga_hipep.cpp +++ b/src/test/oga_hipep/test_oga_hipep.cpp @@ -45,6 +45,10 @@ struct OgaGenerator { /// \brief the length of the last OgaGenerator_SnapshotState std::optional snapshot; }; +struct OgaImages { + std::size_t count = 0; +}; +struct OgaMultiModalProcessor {}; struct OgaTensor { OgaElementType type = OgaElementType_float32; std::vector shape; @@ -99,6 +103,17 @@ struct FakeOga { std::string device = "AMDGPU"; /// \brief each OgaRegisterExecutionProviderLibrary call, as name and path std::vector> registrations; + /// \brief each OgaConfigSetProviderOption call + struct ProviderOption { + std::string provider; + std::string key; + std::string value; + }; + std::vector provider_options; + /// \brief prompts handed to OgaProcessorProcessImages + std::vector processed_prompts; + /// \brief image counts handed to OgaLoadImagesFromBuffers + std::vector loaded_images; std::optional fail_registration; }; @@ -141,6 +156,11 @@ OgaResult* OGA_API_CALL FakeConfigAppendProvider(OgaConfig*, const char* provide g_fake.providers.emplace_back(provider); return nullptr; } +OgaResult* OGA_API_CALL FakeConfigSetProviderOption(OgaConfig*, const char* provider, const char* key, + const char* value) { + g_fake.provider_options.push_back({provider, key, value}); + return nullptr; +} OgaResult* OGA_API_CALL FakeConfigOverlay(OgaConfig*, const char* json) { g_fake.overlays.emplace_back(json); return nullptr; @@ -340,6 +360,42 @@ OgaResult* OGA_API_CALL FakeRewindTo(OgaGenerator* generator, size_t length) { return nullptr; } +OgaResult* OGA_API_CALL FakeLoadImages(const void**, const size_t*, size_t count, OgaImages** out) { + g_fake.loaded_images.push_back(count); + *out = new OgaImages{count}; + return nullptr; +} +void OGA_API_CALL FakeDestroyImages(OgaImages* images) { delete images; } +OgaResult* OGA_API_CALL FakeCreateProcessor(const OgaModel*, OgaMultiModalProcessor** out) { + *out = new OgaMultiModalProcessor; + return nullptr; +} +void OGA_API_CALL FakeDestroyProcessor(OgaMultiModalProcessor* processor) { delete processor; } +OgaResult* OGA_API_CALL FakeProcessImages(const OgaMultiModalProcessor*, const char* prompt, const OgaImages*, + OgaNamedTensors** out) { + g_fake.processed_prompts.emplace_back(prompt); + auto* named = new OgaNamedTensors; + OgaTensor ids; + ids.type = OgaElementType_int32; + ids.shape = {1, 3}; + const std::int32_t tokens[] = {7, 8, 9}; + ids.data.resize(sizeof(tokens)); + std::memcpy(ids.data.data(), tokens, sizeof(tokens)); + named->tensors.emplace("input_ids", std::move(ids)); + OgaTensor pixels; + pixels.type = OgaElementType_float16; + pixels.shape = {1, 0, 0}; + named->tensors.emplace("pixel_values", std::move(pixels)); + *out = named; + return nullptr; +} +OgaResult* OGA_API_CALL FakeNamedTensorsGet(OgaNamedTensors* named, const char* name, OgaTensor** out) { + const auto it = named->tensors.find(name); + if (it == named->tensors.end()) return Error(std::string("no ") + name); + *out = &it->second; + return nullptr; +} + std::map FakeSymbols() { return { {"OgaResultGetError", reinterpret_cast(&FakeResultGetError)}, @@ -349,6 +405,7 @@ std::map FakeSymbols() { {"OgaCreateConfig", reinterpret_cast(&FakeCreateConfig)}, {"OgaConfigClearProviders", reinterpret_cast(&FakeConfigClearProviders)}, {"OgaConfigAppendProvider", reinterpret_cast(&FakeConfigAppendProvider)}, + {"OgaConfigSetProviderOption", reinterpret_cast(&FakeConfigSetProviderOption)}, {"OgaConfigOverlay", reinterpret_cast(&FakeConfigOverlay)}, {"OgaDestroyConfig", reinterpret_cast(&FakeDestroyConfig)}, {"OgaCreateModelFromConfig", reinterpret_cast(&FakeCreateModelFromConfig)}, @@ -373,6 +430,12 @@ std::map FakeSymbols() { {"OgaGenerator_SetInputs", reinterpret_cast(&FakeSetInputs)}, {"OgaGenerator_SnapshotState", reinterpret_cast(&FakeSnapshotState)}, {"OgaGenerator_RewindTo", reinterpret_cast(&FakeRewindTo)}, + {"OgaLoadImagesFromBuffers", reinterpret_cast(&FakeLoadImages)}, + {"OgaDestroyImages", reinterpret_cast(&FakeDestroyImages)}, + {"OgaCreateMultiModalProcessor", reinterpret_cast(&FakeCreateProcessor)}, + {"OgaDestroyMultiModalProcessor", reinterpret_cast(&FakeDestroyProcessor)}, + {"OgaProcessorProcessImages", reinterpret_cast(&FakeProcessImages)}, + {"OgaNamedTensorsGet", reinterpret_cast(&FakeNamedTensorsGet)}, }; } @@ -1372,6 +1435,126 @@ void TestOpenModelPutsTheDecoderOnTheRegisteredEpAlone() { RequireContains(g_fake.overlays[0], R"("decoder":{"session_options":{"session.disable_cpu_ep_fallback":"1"}})"); } +void TestAStagedImagePromptIsTheNextPrefill() { + auto engine = MakeEngine(); + const std::vector images{"PNG_FAKE", "JPEGFAKE"}; + const auto ids = engine->StagePromptImages("see and ", images); + TEST_REQUIRE(ids == (std::vector{7, 8, 9})); + TEST_REQUIRE(g_fake.loaded_images == (std::vector{2})); + TEST_REQUIRE(g_fake.processed_prompts.size() == 1); + TEST_REQUIRE(g_fake.processed_prompts[0] == "see and "); + // Clearing the text history must keep the staged processor inputs. + engine->clear_context(); + std::vector prompt = ids; + engine->prefill(prompt); + TEST_REQUIRE(g_fake.set_inputs.size() == 1); + TEST_REQUIRE(g_fake.set_inputs[0].ids == (std::vector{7, 8, 9})); + TEST_REQUIRE(g_fake.appends.empty()); + TEST_REQUIRE(g_fake.generators_created == 1); +} + +void TestADroppedOrRefusedStageDoesNotReachTheNextPrefill() { + auto engine = MakeEngine(); + // A request that stopped before its prefill drops its stage: the next text prompt runs as text. + engine->StagePromptImages("see ", {"PNG_FAKE"}); + engine->DropStagedPrompt(); + std::vector text{1, 2}; + engine->prefill(text); + TEST_REQUIRE(g_fake.set_inputs.empty()); + TEST_REQUIRE(g_fake.appends == (std::vector{{1, 2}})); + + // A prefill that does not match the stage is refused, and takes the stage with it. + engine->clear_context(); + engine->StagePromptImages("see ", {"PNG_FAKE"}); + std::vector suffix{8, 9}; + RequireContains(RequireThrows([&] { engine->prefill(suffix); }), + "whole sequence"); + std::vector again{1, 2}; + engine->prefill(again); + TEST_REQUIRE(g_fake.set_inputs.empty()); +} + +/// \brief what the gemma4-12b frontend does for an image prompt: stage it, clear the +/// history, prefill the processor's whole sequence and checkpoint it +buffer RunStagedPrompt(flm::oga::OgaEngine& engine) { + std::vector ids = engine.StagePromptImages("see ", {"PNG_FAKE"}); + engine.clear_context(); + auto logits = engine.prefill(ids); + engine.DropStagedPrompt(); + engine.checkpoint(); + return logits; +} + +void TestAStagedImagePromptTakesTheLeastRecentlyUsedSlot() { + auto engine = MakeSlotsEngine(2); + NewPrompt(*engine, {1, 2, 3}); // A + engine->forward(4); + NewPrompt(*engine, {4, 5}); // B: A is now the least recently used + // the image prompt [7, 8, 9] gets a new generator in A's slot, and B keeps its own + RequireState(RunStagedPrompt(*engine), 3, 9, 24); + TEST_REQUIRE(g_fake.generators_created == 3); + TEST_REQUIRE(g_fake.generators_live == 2); + TEST_REQUIRE(g_fake.set_inputs.size() == 1); + TEST_REQUIRE(g_fake.set_inputs.back().ids == (std::vector{7, 8, 9})); + TEST_REQUIRE(engine->get_current_context_length() == 3); + // decoding continues the image slot + RequireState(engine->forward(10), 4, 10, 34); + TEST_REQUIRE(g_fake.appends.back() == (Tokens{10})); + + // B continues its generator + RequireState(NewPrompt(*engine, {4, 5, 6}), 3, 6, 15); + TEST_REQUIRE(g_fake.appends.back() == (Tokens{6})); + TEST_REQUIRE(g_fake.generators_created == 3); + // so does the image conversation's next turn, without the processor + RequireState(NewPrompt(*engine, {7, 8, 9, 10, 11}), 5, 11, 45); + TEST_REQUIRE(g_fake.appends.back() == (Tokens{11})); + TEST_REQUIRE(g_fake.generators_created == 3); + TEST_REQUIRE(g_fake.set_inputs.size() == 1); + // A was replaced, so it replays, in B's slot, now the least recently used + RequireState(NewPrompt(*engine, {1, 2, 3, 4}), 4, 4, 10); + TEST_REQUIRE(g_fake.generators_created == 4); + TEST_REQUIRE(g_fake.generators_live == 2); + TEST_REQUIRE(g_fake.appends.back() == (Tokens{1, 2, 3, 4})); +} + +void TestAStagedImagePromptAlwaysBuildsANewGenerator() { + auto engine = MakeSlotsEngine(2); + RunStagedPrompt(*engine); + // The same ids again may carry other images: the processor's inputs run in a new generator. + RunStagedPrompt(*engine); + TEST_REQUIRE(g_fake.generators_created == 2); + TEST_REQUIRE(g_fake.set_inputs.size() == 2); + TEST_REQUIRE(g_fake.appends.empty()); +} + +void TestAFailedStagedPrefillDropsOnlyItsSlot() { + auto engine = MakeSlotsEngine(2); + NewPrompt(*engine, {1, 2, 3}); + NewPrompt(*engine, {4, 5}); + std::vector ids = engine->StagePromptImages("see ", {"PNG_FAKE"}); + engine->clear_context(); + g_fake.logits_width = kVocab + 1; + RequireThrows([&] { engine->prefill(ids); }); + g_fake.logits_width = kVocab; + TEST_REQUIRE(g_fake.generators_live == 1); + // the failed prefill left no image tokens behind, and no stage + TEST_REQUIRE(engine->get_current_context_length() == 0); + RequireState(NewPrompt(*engine, {4, 5, 6}), 3, 6, 15); + TEST_REQUIRE(g_fake.appends.back() == (Tokens{6})); + TEST_REQUIRE(g_fake.generators_created == 3); + TEST_REQUIRE(g_fake.set_inputs.size() == 1); +} + +void TestStagingImagesNamesAMissingProcessorEntry() { + auto symbols = FakeSymbols(); + symbols.erase("OgaProcessorProcessImages"); + auto engine = MakeEngineOn({0, 17, 0, 0}, std::move(symbols)); + RequireContains(RequireThrows([&] { + engine->StagePromptImages("see ", {"PNG_FAKE"}); + }), + "missing OgaProcessorProcessImages"); +} + void TestOpenModelRejectsAModelOnAnotherDevice() { g_fake = FakeOga{}; g_fake.device = "CPU"; @@ -2314,6 +2497,54 @@ void TestTheBackendRunsThePackagesImages() { TEST_REQUIRE(backend->replayed_tokens() == 0); } +void TestGemma4UnifiedSharesThePresentBufferAndSkipsGraphCapture() { + g_fake = FakeOga{}; + const auto api = FakeApi(); + PackageDir package(kVocab, "<|im_end|>", + R"(,"image_token_id":40,"vision":{"filename":"vision_encoder/model.onnx"})", + "gemma4_unified"); + TEST_REQUIRE(flm::oga::OpenModel(*api, package.dir.path) != nullptr); + TEST_REQUIRE(g_fake.overlays.size() == 2); + RequireContains(g_fake.overlays[1], R"("past_present_share_buffer":true)"); + RequireContains(g_fake.overlays[1], R"("enable_graph_capture":"0")"); + RequireContains(g_fake.overlays[1], R"("hip_graph_enable":"0")"); + TEST_REQUIRE(g_fake.provider_options.size() == 6); + for (const char* key : {"profile", "enable_graph_capture", "hip_graph_enable"}) { + const auto matches = std::count_if(g_fake.provider_options.begin(), g_fake.provider_options.end(), + [&](const FakeOga::ProviderOption& option) { + return option.key == key; + }); + TEST_REQUIRE(matches == 2); + } +} + +void TestAnotherModelTypeDoesNotShareThePresentBuffer() { + g_fake = FakeOga{}; + const auto api = FakeApi(); + PackageDir package; + TEST_REQUIRE(flm::oga::OpenModel(*api, package.dir.path) != nullptr); + TEST_REQUIRE(g_fake.overlays.size() == 1); + TEST_REQUIRE(g_fake.provider_options.empty()); +} + +void TestTheGemma4UnifiedPackageStillRunsTextWithoutAVisionStartToken() { + g_fake = FakeOga{}; + auto api = flm::oga::OgaApi::GetOrCreateWith([] { return FakeApi(); }); + PackageDir package(kVocab, "<|im_end|>", + R"(,"image_token_id":40,"vision":{"filename":"vision_encoder/model.onnx"})", + "gemma4_unified"); + auto backend = flm::oga::hipep_factory()(package.Context(64)); + RequireContains(g_fake.overlays[1], R"("past_present_share_buffer":true)"); + std::vector text{1, 2}; + backend->engine().prefill(text); + TEST_REQUIRE(g_fake.appends.back() == (Tokens{1, 2})); + const auto image = Image(2, 2); + flm::oga::OgaVisionInput vision{{image}}; + std::vector prompt = WithImage({1}, image, {2}); + RequireContains(RequireThrows([&] { backend->engine().prefill(prompt, &vision); }), + "does not run a vision payload"); +} + void TestTheBackendRunsTheTextOfAnotherTypesVisionPackage() { g_fake = FakeOga{}; auto api = flm::oga::OgaApi::GetOrCreateWith([] { return FakeApi(); }); @@ -2461,6 +2692,15 @@ int main(int argc, char** argv) { RUN(TestARefusedRegistrationNamesTheLibrary); RUN(TestANonAsciiLibraryPathIsRefusedBeforeOgaSeesIt); RUN(TestOpenModelPutsTheDecoderOnTheRegisteredEpAlone); + RUN(TestGemma4UnifiedSharesThePresentBufferAndSkipsGraphCapture); + RUN(TestAnotherModelTypeDoesNotShareThePresentBuffer); + RUN(TestAStagedImagePromptIsTheNextPrefill); + RUN(TestADroppedOrRefusedStageDoesNotReachTheNextPrefill); + RUN(TestAStagedImagePromptTakesTheLeastRecentlyUsedSlot); + RUN(TestAStagedImagePromptAlwaysBuildsANewGenerator); + RUN(TestAFailedStagedPrefillDropsOnlyItsSlot); + RUN(TestStagingImagesNamesAMissingProcessorEntry); + RUN(TestTheGemma4UnifiedPackageStillRunsTextWithoutAVisionStartToken); RUN(TestOpenModelRejectsAModelOnAnotherDevice); RUN(TestDiscoveryReportsWhyEachRootIsRejected); RUN(TestDiscoveryAcceptsAVersionedPackageWithoutABinDirectory);