From ad1363f8c76a00382fb58a896710524d4240d58b Mon Sep 17 00:00:00 2001 From: George Evmenov Date: Sat, 24 Oct 2026 22:49:52 +0300 Subject: [PATCH 1/2] Avoid copying Cereal input buffers before deserialization --- benchmarks/all/input_buffer.cpp | 31 +++++++++++ include/rfl/boost_serialization/read.hpp | 17 ++---- include/rfl/cereal/read.hpp | 12 ++-- include/rfl/internal/MemoryInputBuffer.hpp | 41 ++++++++++++++ .../boost_serialization/test_memory_input.cpp | 12 ++++ tests/cereal/test_memory_input.cpp | 11 ++++ tests/check_memory_input.hpp | 50 +++++++++++++++++ tests/generic/test_memory_input_buffer.cpp | 55 +++++++++++++++++++ 8 files changed, 213 insertions(+), 16 deletions(-) create mode 100644 benchmarks/all/input_buffer.cpp create mode 100644 include/rfl/internal/MemoryInputBuffer.hpp create mode 100644 tests/boost_serialization/test_memory_input.cpp create mode 100644 tests/cereal/test_memory_input.cpp create mode 100644 tests/check_memory_input.hpp create mode 100644 tests/generic/test_memory_input_buffer.cpp diff --git a/benchmarks/all/input_buffer.cpp b/benchmarks/all/input_buffer.cpp new file mode 100644 index 000000000..7f4eb25d0 --- /dev/null +++ b/benchmarks/all/input_buffer.cpp @@ -0,0 +1,31 @@ +#include + +#include +#include +#include + +namespace input_buffer { + +void BM_cereal_input_buffer(benchmark::State& state) { + std::vector input(state.range(0)); + for (std::size_t i = 0; i < input.size(); ++i) input[i] = i * 17; + const auto bytes = rfl::cereal::write(input); + { + const auto result = rfl::cereal::read(bytes); + if (!result || *result != input) { + state.SkipWithError("Round trip failed"); + return; + } + } + for (auto _ : state) { + auto result = rfl::cereal::read(bytes); + benchmark::DoNotOptimize(result); + } + state.SetBytesProcessed(state.iterations() * bytes.size()); +} + +BENCHMARK(BM_cereal_input_buffer) + ->Arg(1024) + ->Arg(128 * 1024) + ->Arg(4 * 1024 * 1024); +} // namespace input_buffer diff --git a/include/rfl/boost_serialization/read.hpp b/include/rfl/boost_serialization/read.hpp index a89a3540d..d336daaac 100644 --- a/include/rfl/boost_serialization/read.hpp +++ b/include/rfl/boost_serialization/read.hpp @@ -5,11 +5,11 @@ #include #include #include -#include #include #include "../Processors.hpp" #include "../concepts.hpp" +#include "../internal/MemoryInputBuffer.hpp" #include "../internal/wrap_in_rfl_array_t.hpp" #include "Parser.hpp" #include "Reader.hpp" @@ -26,14 +26,7 @@ auto read_from_archive(IArchive& _ar) { return Parser>::read(r, var); } -/// A read-only streambuf that wraps an existing memory buffer without copying. -class MemBuf : public std::streambuf { - public: - MemBuf(const char* _data, size_t _size) { - auto* p = const_cast(_data); - setg(p, p, p + _size); - } -}; +using MemBuf = internal::MemoryInputBuffer; } // namespace detail @@ -51,7 +44,8 @@ auto read_from_archive(IArchive& _ar) { } /// Parses an object from raw bytes using Boost binary archive. -/// Creates a memory stream from the bytes and deserializes using Boost.Serialization. +/// Creates a memory stream from the bytes and deserializes using +/// Boost.Serialization. /// @tparam T The type to deserialize into /// @tparam Ps Optional processors to apply during deserialization /// @param _bytes Pointer to the binary data @@ -76,7 +70,8 @@ Result> read( /// Parses an object from a byte container using Boost binary archive. /// @tparam T The type to deserialize into /// @tparam Ps Optional processors to apply during deserialization -/// @param _bytes A contiguous container (e.g., std::vector, std::string) containing the data +/// @param _bytes A contiguous container (e.g., std::vector, std::string) +/// containing the data /// @return A Result containing the deserialized object or an error template auto read(const concepts::ContiguousByteContainer auto& _bytes) { diff --git a/include/rfl/cereal/read.hpp b/include/rfl/cereal/read.hpp index 42ed07121..9c1a2a0f5 100644 --- a/include/rfl/cereal/read.hpp +++ b/include/rfl/cereal/read.hpp @@ -4,10 +4,11 @@ #include #include #include -#include +#include #include "../Processors.hpp" #include "../concepts.hpp" +#include "../internal/MemoryInputBuffer.hpp" #include "../internal/wrap_in_rfl_array_t.hpp" #include "Parser.hpp" #include "Reader.hpp" @@ -30,7 +31,7 @@ auto read(Reader::CerealArchive& _archive) { } /// Parses an object from raw bytes using Cereal portable binary format. -/// Creates a memory stream and deserializes using Cereal. +/// Reads from the supplied bytes without copying them into an owning stream. /// @tparam T The type to deserialize into /// @tparam Ps Optional processors to apply during deserialization /// @param _bytes Pointer to the binary data @@ -40,9 +41,10 @@ template Result> read( const concepts::ByteLike auto* _bytes, const size_t _size) { try { - std::stringstream ss( - std::string(reinterpret_cast(_bytes), _size)); - ::cereal::PortableBinaryInputArchive archive(ss); + internal::MemoryInputBuffer buffer(reinterpret_cast(_bytes), + _size); + std::istream stream(&buffer); + ::cereal::PortableBinaryInputArchive archive(stream); return read(archive); } catch (std::exception& e) { return error(std::string("Cereal read error: ") + e.what()); diff --git a/include/rfl/internal/MemoryInputBuffer.hpp b/include/rfl/internal/MemoryInputBuffer.hpp new file mode 100644 index 000000000..2ea24b478 --- /dev/null +++ b/include/rfl/internal/MemoryInputBuffer.hpp @@ -0,0 +1,41 @@ +#ifndef RFL_INTERNAL_MEMORYINPUTBUFFER_HPP_ +#define RFL_INTERNAL_MEMORYINPUTBUFFER_HPP_ + +#include +#include +#include +#include + +namespace rfl::internal { + +/// A non-owning input buffer. The referenced bytes must outlive the buffer. +class MemoryInputBuffer final : public std::streambuf { + public: + MemoryInputBuffer(const char* _data, const std::size_t _size) { + // streambuf requires mutable pointers for its get area, but no put area + // is installed and the default pbackfail rejects replacement characters. + auto* begin = _size == 0 ? &empty_ : const_cast(_data); + setg(begin, begin, begin + _size); + } + + MemoryInputBuffer(const MemoryInputBuffer&) = delete; + MemoryInputBuffer& operator=(const MemoryInputBuffer&) = delete; + + private: + std::streamsize xsgetn(char* _dest, std::streamsize _count) override { + if (_count <= 0) return 0; + const auto count = + std::min(_count, static_cast(egptr() - gptr())); + if (count != 0) { + std::memcpy(_dest, gptr(), static_cast(count)); + setg(eback(), gptr() + count, egptr()); + } + return count; + } + + char empty_ = '\0'; +}; + +} // namespace rfl::internal + +#endif diff --git a/tests/boost_serialization/test_memory_input.cpp b/tests/boost_serialization/test_memory_input.cpp new file mode 100644 index 000000000..938972e22 --- /dev/null +++ b/tests/boost_serialization/test_memory_input.cpp @@ -0,0 +1,12 @@ +#include + +#include "../check_memory_input.hpp" + +TEST(boost_serialization, memory_input) { + check_memory_input::check( + [](const auto& value) { return rfl::boost_serialization::write(value); }, + [](const auto* data, std::size_t size) { + return rfl::boost_serialization::read(data, + size); + }); +} diff --git a/tests/cereal/test_memory_input.cpp b/tests/cereal/test_memory_input.cpp new file mode 100644 index 000000000..ce85bfb90 --- /dev/null +++ b/tests/cereal/test_memory_input.cpp @@ -0,0 +1,11 @@ +#include + +#include "../check_memory_input.hpp" + +TEST(cereal, memory_input) { + check_memory_input::check( + [](const auto& value) { return rfl::cereal::write(value); }, + [](const auto* data, std::size_t size) { + return rfl::cereal::read(data, size); + }); +} diff --git a/tests/check_memory_input.hpp b/tests/check_memory_input.hpp new file mode 100644 index 000000000..1397395b4 --- /dev/null +++ b/tests/check_memory_input.hpp @@ -0,0 +1,50 @@ +#ifndef CHECK_MEMORY_INPUT_HPP_ +#define CHECK_MEMORY_INPUT_HPP_ + +#include + +#include +#include +#include +#include + +namespace check_memory_input { + +struct Record { + std::uint32_t id; + double value; + std::string name; + bool operator==(const Record&) const = default; +}; + +template +void check(Write write, Read read) { + const Record original{42, 1.25, std::string("a\0b", 3)}; + const auto bytes = write(original); + const auto saved = bytes; + for (int i = 0; i < 2; ++i) { + const auto result = read(bytes.data(), bytes.size()); + ASSERT_TRUE(result); + EXPECT_EQ(*result, original); + } + EXPECT_EQ(bytes, saved); + EXPECT_FALSE(read(static_cast(nullptr), 0)); + for (std::size_t size = 0; size < bytes.size(); ++size) { + EXPECT_FALSE(read(bytes.data(), size)) << "prefix size " << size; + } + std::vector unsigned_bytes(bytes.begin(), bytes.end()); + const auto result = read(unsigned_bytes.data(), unsigned_bytes.size()); + ASSERT_TRUE(result); + EXPECT_EQ(*result, original); + // A view must respect its extent, rather than reading into adjacent bytes. + std::vector surrounded{'x'}; + surrounded.insert(surrounded.end(), bytes.begin(), bytes.end()); + surrounded.push_back('y'); + const auto bounded = read(surrounded.data() + 1, bytes.size()); + ASSERT_TRUE(bounded); + EXPECT_EQ(*bounded, original); +} + +} // namespace check_memory_input + +#endif diff --git a/tests/generic/test_memory_input_buffer.cpp b/tests/generic/test_memory_input_buffer.cpp new file mode 100644 index 000000000..522ee594b --- /dev/null +++ b/tests/generic/test_memory_input_buffer.cpp @@ -0,0 +1,55 @@ +#include + +#include +#include +#include +#include +#include + +namespace test_memory_input_buffer { + +TEST(memory_input_buffer, empty) { + rfl::internal::MemoryInputBuffer buffer(nullptr, 0); + std::istream input(&buffer); + EXPECT_EQ(input.peek(), std::char_traits::eof()); + char byte = 'x'; + input.read(&byte, 1); + EXPECT_EQ(input.gcount(), 0); + EXPECT_EQ(byte, 'x'); +} + +TEST(memory_input_buffer, bounded_reads) { + const std::array bytes{'a', '\0', 'b', 'x', 'y'}; + rfl::internal::MemoryInputBuffer buffer(bytes.data(), 3); + std::istream input(&buffer); + std::array output{}; + input.read(output.data(), output.size()); + EXPECT_EQ(input.gcount(), 3); + EXPECT_TRUE(input.eof()); + EXPECT_EQ(std::string(output.data(), 3), std::string("a\0b", 3)); + EXPECT_EQ(output[3], '\0'); +} + +TEST(memory_input_buffer, putback_does_not_modify_input) { + const std::string bytes = "abc"; + rfl::internal::MemoryInputBuffer buffer(bytes.data(), bytes.size()); + std::istream input(&buffer); + EXPECT_EQ(input.get(), 'a'); + input.putback('a'); + EXPECT_TRUE(input); + EXPECT_EQ(input.get(), 'a'); + input.putback('z'); + EXPECT_FALSE(input); + EXPECT_EQ(bytes, "abc"); +} + +TEST(memory_input_buffer, output_is_rejected) { + const std::string bytes = "abc"; + rfl::internal::MemoryInputBuffer buffer(bytes.data(), bytes.size()); + std::ostream output(&buffer); + output.write("xyz", 3); + EXPECT_FALSE(output); + EXPECT_EQ(bytes, "abc"); +} + +} // namespace test_memory_input_buffer From a806b12c55c63b819f01451ac2a613faed563ddf Mon Sep 17 00:00:00 2001 From: George Evmenov Date: Sun, 25 Oct 2026 00:15:53 +0300 Subject: [PATCH 2/2] Fix GCC overflow warning in memory input buffer --- include/rfl/internal/MemoryInputBuffer.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/rfl/internal/MemoryInputBuffer.hpp b/include/rfl/internal/MemoryInputBuffer.hpp index 2ea24b478..4da23a300 100644 --- a/include/rfl/internal/MemoryInputBuffer.hpp +++ b/include/rfl/internal/MemoryInputBuffer.hpp @@ -26,7 +26,7 @@ class MemoryInputBuffer final : public std::streambuf { if (_count <= 0) return 0; const auto count = std::min(_count, static_cast(egptr() - gptr())); - if (count != 0) { + if (count > 0) { std::memcpy(_dest, gptr(), static_cast(count)); setg(eback(), gptr() + count, egptr()); }