Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions src/common/AutoModel/builtin_backends.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,11 @@ void register_builtin_backends(BackendRegistry& registry) {
registry.register_backend("gemma4e", flm::backend::kOgaHipepBackendId,
flm::oga::hipep_factory(),
flm::oga::hipep_traits());
// The Gemma 4 12B GPU package uses the gemma4-12b frontend. The tag
// gemma4-12b-oga-hipep selects this backend; details.family stays gemma4-12b.
registry.register_backend("gemma4-12b", flm::backend::kOgaHipepBackendId,
flm::oga::hipep_factory(),
flm::oga::hipep_traits());
registry.register_backend("gpt-oss", flm::backend::kOgaHipepBackendId,
flm::oga::hipep_factory(),
flm::oga::hipep_traits());
Expand Down
107 changes: 104 additions & 3 deletions src/common/AutoModel/modeling_gemma4_12b.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,37 @@
#include "metrices.hpp"
#include "error_measure.hpp"
#include "models/gemma4_12b/flm/aie2p/gemma4_12b_npu.hpp"
#if defined(FLM_ENABLE_OGA_HIPEP)
#include "base64.hpp"
#include "oga/oga_engine.hpp"
#include <fstream>
#include <iterator>
#endif
#include <cassert>

#if defined(FLM_ENABLE_OGA_HIPEP)
namespace {
std::string ImageBytesFromBase64(std::string payload) {
const auto marker = payload.find("base64,");
if (marker != std::string::npos) payload.erase(0, marker + 7);
std::string bytes = base64::from_base64(payload);
if (bytes.size() < 8)
throw ModelRequestError(400, false, "image payload is not a valid encoded image", "invalid_request_error");
return bytes;
}

std::string ImageBytesFromFile(const std::string& path) {
std::ifstream in(path, std::ios::binary);
if (!in)
throw ModelRequestError(400, false, "cannot read image " + path, "invalid_request_error");
std::string bytes((std::istreambuf_iterator<char>(in)), std::istreambuf_iterator<char>());
if (bytes.size() < 8)
throw ModelRequestError(400, false, "image file is empty: " + path, "invalid_request_error");
return bytes;
}
} // namespace
#endif

/************ Gemma4_12B family **************/
Gemma4_12B::Gemma4_12B(flm_rt::device* npu_device_inst) : AutoModel(npu_device_inst, "Gemma4_12B") {}

Expand Down Expand Up @@ -45,8 +74,11 @@ void Gemma4_12B::load_model(std::string model_path, json model_info, int default
for (size_t i = 0; i < PROFILER_TYPE_NUM; i++) {
this->profiler_list[i].reset();
}
// Text-only prompts only (see insert), so no image placeholder id is given.
_prefix_attach(lm_engine, gemma4_12b_npu_state_api(lm_engine));
// The OGA engine has no NPU prefix-slot table. NPU prompts keep theirs.
const flm_state_api* state = this->backend_id() == flm::backend::kOgaHipepBackendId
? nullptr
: gemma4_12b_npu_state_api(lm_engine);
_prefix_attach(lm_engine, state);
}

void Gemma4_12B::setup_tokenizer(std::string model_path) {
Expand Down Expand Up @@ -106,6 +138,16 @@ bool Gemma4_12B::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input,
// before a later return. The trim below still runs only when the prompt ends with the channel.
this->feed_empty_thought_ = flm::gemma4_feed_empty_thought(this->enable_think, meta_info.continue_final_message);

#if defined(FLM_ENABLE_OGA_HIPEP)
const bool oga = this->backend_id() == flm::backend::kOgaHipepBackendId;
std::vector<std::string> oga_image_bytes;
if (oga && !input.audios.empty()) {
throw ModelRequestError(400, false,
"gemma4-12b on oga-hipep does not accept audio input",
"invalid_request_error");
}
#endif

gemma4_12b_image_payload_t image_payload;
gemma4_12b_audio_payload_t audio_payload;
audio_payload.num_audios = 0;
Expand All @@ -129,7 +171,14 @@ bool Gemma4_12B::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input,
if (message.contains("images")) {
for (auto& img : message["images"]) {
std::string img_str = img.get<std::string>();

#if defined(FLM_ENABLE_OGA_HIPEP)
if (oga) {
oga_image_bytes.push_back(ImageBytesFromBase64(std::move(img_str)));
newContent.push_back({{"type", "image"}, {"image", img}});
total_images++;
continue;
}
#endif
gemma4_12b_image_t image = this->load_image_base64(img_str);
if (image.width <= 0 || image.height <= 0) {
header_print("ERROR", "Skipping invalid base64 image; prefilling language only for this item");
Expand Down Expand Up @@ -157,6 +206,13 @@ bool Gemma4_12B::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input,
newContent.push_back({{"type", "text"}, {"text", message.value("content", "")}});
// Process Audios
if (message.contains("audios")) {
#if defined(FLM_ENABLE_OGA_HIPEP)
if (oga && !message["audios"].empty()) {
throw ModelRequestError(400, false,
"gemma4-12b on oga-hipep does not accept audio input",
"invalid_request_error");
}
#endif
gemma4_12b_npu *gemma4e_engine = dynamic_cast<gemma4_12b_npu*>(this->lm_engine);
if (!gemma4e_engine) {
throw std::runtime_error(
Expand Down Expand Up @@ -267,6 +323,12 @@ bool Gemma4_12B::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input,
}

if(input.images.size() > 0){
#if defined(FLM_ENABLE_OGA_HIPEP)
if (oga) {
for (const auto& img_str : input.images)
oga_image_bytes.push_back(ImageBytesFromFile(img_str));
} else
#endif
for (const auto& img_str : input.images) {
gemma4_12b_image_t image = this->load_image(img_str);

Expand Down Expand Up @@ -317,6 +379,45 @@ bool Gemma4_12B::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input,
templated_text = this->apply_chat_template(messages);
}

#if defined(FLM_ENABLE_OGA_HIPEP)
if (oga && !oga_image_bytes.empty()) {
// generate() feeds this empty thought channel itself when thinking is off.
// The processor must not run it, or the four tokens would be in the KV cache twice.
const std::string thought = "<|channel>thought\n<channel|>";
if (!this->enable_think && !meta_info.continue_final_message &&
templated_text.size() >= thought.size() &&
templated_text.compare(templated_text.size() - thought.size(), thought.size(), thought) == 0) {
templated_text.resize(templated_text.size() - thought.size());
}
auto* engine = dynamic_cast<flm::oga::OgaEngine*>(this->lm_engine);
if (!engine)
throw std::runtime_error("oga-hipep engine is not an OgaEngine");
std::vector<int> image_tokens = engine->StagePromptImages(templated_text, oga_image_bytes);
this->profiler_list[TKOEN_ENCODE_TIME].stop(image_tokens.size());
// The processor already ran the whole prompt, so this prefill must not be chunked,
// nor start after a cached prefix: the staged inputs are prefilled from token 0.
if (!this->token_history.empty())
this->clear_context();
const int saved_prefill = meta_info.max_prefill_len;
meta_info.max_prefill_len = 0;
bool success = false;
try {
success = this->_shared_insert(meta_info, image_tokens, is_cancelled, nullptr);
} catch (...) {
meta_info.max_prefill_len = saved_prefill;
engine->DropStagedPrompt();
throw;
}
meta_info.max_prefill_len = saved_prefill;
// A request that stopped before its prefill (cancelled, or refused for its
// length) must not leave its stage to the next request.
engine->DropStagedPrompt();
this->_checkpoint_history();
this->lm_engine->checkpoint();
return success;
}
#endif

// update the tokens to include the image tokens
std::vector<int> tokens;
std::vector<int> tokens_init = this->tokenizer->encode(templated_text);
Expand Down
130 changes: 130 additions & 0 deletions src/common/oga/oga_engine.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,51 @@
#include <cmath>
#include <cstring>
#include <iterator>
#include <limits>
#include <stdexcept>
#include <utility>

namespace flm::oga {
namespace {
static_assert(sizeof(int) == sizeof(std::int32_t), "token ids are passed to OGA as int32_t");

void Require(const void* function, const char* symbol) {
if (!function)
throw std::runtime_error(std::string("the hip-ep onnxruntime-genai is missing ") + symbol);
}

std::vector<int> ReadInputIds(const OgaApi& api, OgaTensor* tensor) {
const OgaFunctions& f = api.functions();
OgaElementType type = OgaElementType_undefined;
api.Check(f.tensor_get_type(tensor, &type), "OgaTensorGetType(input_ids)");
std::size_t rank = 0;
api.Check(f.tensor_get_shape_rank(tensor, &rank), "OgaTensorGetShapeRank(input_ids)");
std::vector<std::int64_t> shape(rank);
if (rank) api.Check(f.tensor_get_shape(tensor, shape.data(), rank), "OgaTensorGetShape(input_ids)");
if (rank == 2 && shape[0] == 1) shape.erase(shape.begin());
if (shape.size() != 1 || shape[0] <= 0)
throw std::runtime_error("processor input_ids are not a single token row");
const auto count = static_cast<std::size_t>(shape[0]);
void* data = nullptr;
api.Check(f.tensor_get_data(tensor, &data), "OgaTensorGetData(input_ids)");
if (!data) throw std::runtime_error("processor input_ids have no data");
std::vector<int> ids(count);
if (type == OgaElementType_int32) {
const auto* from = static_cast<const std::int32_t*>(data);
std::copy(from, from + count, ids.begin());
} else if (type == OgaElementType_int64) {
const auto* from = static_cast<const std::int64_t*>(data);
for (std::size_t i = 0; i < count; ++i) {
if (from[i] < 0 || from[i] > static_cast<std::int64_t>(std::numeric_limits<int>::max()))
throw std::runtime_error("processor input id does not fit in int");
ids[i] = static_cast<int>(from[i]);
}
} else {
throw std::runtime_error("processor input_ids are not int32 or int64");
}
return ids;
}

float HalfToFloat(std::uint16_t half) {
const std::uint32_t sign = static_cast<std::uint32_t>(half & 0x8000u) << 16;
std::uint32_t exponent = (half >> 10) & 0x1fu;
Expand Down Expand Up @@ -115,6 +153,16 @@ OgaEngine::~OgaEngine() = default;
buffer<bf16> OgaEngine::forward(int id) { return Run({id}); }

buffer<bf16> OgaEngine::prefill(std::vector<int>& ids, void* payload) {
if (staged_prompt_) {
// Taken first, so a refused prefill does not leave the stage to the next one.
std::unique_ptr<StagedPrompt> staged = std::move(staged_prompt_);
if (payload)
throw std::invalid_argument(display_name_ + " staged image prompt takes no vision payload");
if (ids != staged->ids)
throw std::runtime_error(display_name_ +
" image prompt must be prefilled as the processor's whole sequence");
return PrefillStaged(*staged);
}
if (payload && !vision_)
throw std::invalid_argument(display_name_ + " on OGA does not run a vision payload");
if (ids.empty()) throw std::invalid_argument(display_name_ + " request contains no token IDs");
Expand Down Expand Up @@ -158,6 +206,8 @@ void OgaEngine::clear_context() {
images_.clear();
expected_.clear();
saved_.reset();
// staged_prompt_ stays. The caller clears a divergent history and then
// prefills the image sequence that was just staged.
}

void OgaEngine::expect_prefill(const std::vector<int>& ids) {
Expand Down Expand Up @@ -601,6 +651,86 @@ buffer<bf16> OgaEngine::Logits(OgaGenerator* generator) const {
return out;
}

buffer<bf16> OgaEngine::PrefillStaged(const StagedPrompt& staged) {
// The processor inputs hold the whole sequence and its images, which OGA takes only in a
// generator's first call: a new generator, in a free slot or the least recently used one.
active_.reset(); // so that a failure before Rebuild drops no other conversation's slot
try {
const char* why = nullptr;
Rebuild(std::nullopt, &why);
Live& run = Active();
api_->Check(api_->functions().generator_set_inputs(run.generator.get(), staged.inputs.get()),
"OgaGenerator_SetInputs");
buffer<bf16> logits = Logits(run.generator.get());
// Only now: a failed prefill leaves `logical` without image tokens that a later
// rebuild would replay without their images.
run.ran = staged.ids;
logical_ = staged.ids;
images_.clear();
expected_.clear();
saved_.reset();
header_print("FLM", "OGA generator: image prompt of " << logical_.size() << " tokens"
<< SlotText(why));
return logits;
} catch (...) {
if (active_) Drop(*active_);
throw;
}
}

void OgaEngine::DropStagedPrompt() noexcept { staged_prompt_.reset(); }

std::vector<int> OgaEngine::StagePromptImages(const std::string& prompt,
const std::vector<std::string>& encoded_images) {
staged_prompt_.reset();
if (encoded_images.empty())
throw std::invalid_argument(display_name_ + " image prompt has no images");
if (prompt.empty())
throw std::invalid_argument(display_name_ + " image prompt is empty");
const OgaFunctions& f = api_->functions();
Require(reinterpret_cast<const void*>(f.load_images_from_buffers), "OgaLoadImagesFromBuffers");
Require(reinterpret_cast<const void*>(f.destroy_images), "OgaDestroyImages");
Require(reinterpret_cast<const void*>(f.create_multimodal_processor), "OgaCreateMultiModalProcessor");
Require(reinterpret_cast<const void*>(f.destroy_multimodal_processor), "OgaDestroyMultiModalProcessor");
Require(reinterpret_cast<const void*>(f.processor_process_images), "OgaProcessorProcessImages");
Require(reinterpret_cast<const void*>(f.named_tensors_get), "OgaNamedTensorsGet");

std::vector<const void*> buffers;
std::vector<std::size_t> sizes;
buffers.reserve(encoded_images.size());
sizes.reserve(encoded_images.size());
for (const std::string& image : encoded_images) {
if (image.size() < 8)
throw std::invalid_argument(display_name_ + " image is not an encoded file");
buffers.push_back(image.data());
sizes.push_back(image.size());
}
OgaImages* raw_images = nullptr;
api_->Check(f.load_images_from_buffers(buffers.data(), sizes.data(), buffers.size(), &raw_images),
"OgaLoadImagesFromBuffers");
OgaApi::Owned<OgaImages> images(raw_images, {f.destroy_images});
OgaMultiModalProcessor* raw_processor = nullptr;
api_->Check(f.create_multimodal_processor(model_.get(), &raw_processor),
"OgaCreateMultiModalProcessor");
OgaApi::Owned<OgaMultiModalProcessor> processor(raw_processor, {f.destroy_multimodal_processor});
OgaNamedTensors* raw_inputs = nullptr;
api_->Check(f.processor_process_images(processor.get(), prompt.c_str(), images.get(), &raw_inputs),
"OgaProcessorProcessImages");
OgaApi::Owned<OgaNamedTensors> inputs(raw_inputs, {f.destroy_named_tensors});
OgaTensor* raw_ids = nullptr;
api_->Check(f.named_tensors_get(inputs.get(), "input_ids", &raw_ids),
"OgaNamedTensorsGet(input_ids)");
std::vector<int> ids = ReadInputIds(*api_, raw_ids);
CheckIds(ids);
if (ids.size() > max_length_)
throw std::out_of_range(display_name_ + " image prompt has " + std::to_string(ids.size()) +
" tokens, past the context length of " + std::to_string(max_length_));
staged_prompt_ = std::make_unique<StagedPrompt>(StagedPrompt{std::move(inputs), ids});
header_print("FLM", "OGA image processor: " << encoded_images.size() << " image(s), "
<< ids.size() << " tokens");
return ids;
}

void OgaEngine::CheckIds(const std::vector<int>& ids) const {
for (const int id : ids)
if (id < 0 || static_cast<std::uint32_t>(id) >= vocab_size_)
Expand Down
Loading
Loading