From 67352f40eca1ffbdeffbeeab570e092115412c33 Mon Sep 17 00:00:00 2001 From: George Evmenov Date: Sat, 24 Oct 2026 21:16:38 +0300 Subject: [PATCH] Preserve typed map keys in MessagePack and binary archives Add an optional backend capability for parsing and writing map keys as values. Enable it for MessagePack, cereal, YAS, and Boost.Serialization without a legacy string-key compatibility path. Cover precision, validation, nested maps, wire formats, and truncation; document the change and the existing CBOR DOM limitation. --- docs/content/standard_containers.md | 24 +++- .../supporting_your_own_format.md | 50 +++++++++ include/rfl/boost_serialization/Reader.hpp | 24 ++-- include/rfl/boost_serialization/Writer.hpp | 35 ++++++ include/rfl/cereal/Reader.hpp | 22 ++-- include/rfl/cereal/Writer.hpp | 35 ++++++ include/rfl/msgpack/Reader.hpp | 12 ++ include/rfl/msgpack/Writer.hpp | 25 +++++ include/rfl/parsing/MapParser.hpp | 64 ++++++++--- include/rfl/parsing/Parent.hpp | 34 ++++++ .../rfl/parsing/SupportsTypedMapReader.hpp | 28 +++++ .../rfl/parsing/SupportsTypedMapWriter.hpp | 47 ++++++++ include/rfl/parsing/TypedMapReader.hpp | 48 ++++++++ .../parsing/schemaful/IsSchemafulReader.hpp | 5 +- include/rfl/yas/Reader.hpp | 25 ++--- include/rfl/yas/Writer.hpp | 34 ++++++ tests/boost_serialization/test_typed_maps.cpp | 33 ++++++ tests/cereal/test_typed_maps.cpp | 34 ++++++ tests/generic/test_typed_map_capabilities.cpp | 50 +++++++++ tests/msgpack/test_typed_maps.cpp | 25 +++++ tests/test_typed_maps.hpp | 103 ++++++++++++++++++ tests/yas/test_typed_maps.cpp | 26 +++++ 22 files changed, 727 insertions(+), 56 deletions(-) create mode 100644 include/rfl/parsing/SupportsTypedMapReader.hpp create mode 100644 include/rfl/parsing/SupportsTypedMapWriter.hpp create mode 100644 include/rfl/parsing/TypedMapReader.hpp create mode 100644 tests/boost_serialization/test_typed_maps.cpp create mode 100644 tests/cereal/test_typed_maps.cpp create mode 100644 tests/generic/test_typed_map_capabilities.cpp create mode 100644 tests/msgpack/test_typed_maps.cpp create mode 100644 tests/test_typed_maps.hpp create mode 100644 tests/yas/test_typed_maps.cpp diff --git a/docs/content/standard_containers.md b/docs/content/standard_containers.md index 32b51589a..2f954a0fb 100644 --- a/docs/content/standard_containers.md +++ b/docs/content/standard_containers.md @@ -21,7 +21,7 @@ This will be represented as follows: {"firstName":"Homer","lastName":"Simpson"} ``` -The same is true for numeric types: +For JSON and other formats that use string map keys, the same is true for numeric types: ```cpp std::map homer; @@ -56,3 +56,25 @@ will be represented as arrays. Containers for which the `value_type` is a key-value-pair will be represented as arrays of pairs. `std::expected` is an exception to this: it is serialized as its value type, or as an object with a single `error` field. Refer to the [std::expected](../expected) section for details. + +## Typed map keys in binary formats + +MessagePack, Cereal, YAS, and Boost.Serialization serialize the keys of the +maps described above as values of their original type. This applies to both +`std::map` and `std::unordered_map`, including numeric `ReflectionType` wrappers +such as `rfl::Validator`. Validation is applied when reading keys. + +For example, `std::map{{1, "Homer"}}` contains an integer key +in MessagePack, rather than the string `"1"`. The archive backends write the +entry count followed by each typed key and its value. No intermediate container +or conversion to/from decimal strings is needed. Floating-point keys retain +their precision instead of being rounded by `std::to_string`. + +This changes the wire format for numeric keys in these backends. There is no +automatic detection or compatibility mode for previously serialized string +keys; readers and writers must use the same representation. + +Other backends retain their existing representation. In particular, the CBOR +adapter currently reads through a JSON DOM with string object keys, so native +typed CBOR map keys are not supported by this change. Maps with compound keys +and multimaps continue to use arrays of pairs. diff --git a/docs/content/supported_formats/supporting_your_own_format.md b/docs/content/supported_formats/supporting_your_own_format.md index 964e91762..e745b3ce5 100644 --- a/docs/content/supported_formats/supporting_your_own_format.md +++ b/docs/content/supported_formats/supporting_your_own_format.md @@ -443,3 +443,53 @@ struct Reader { }; ``` + +### Optional typed map keys + +Backends that preserve map key types provide explicit map-entry APIs for +writing and reading. They are detected independently by +`SupportsTypedMapWriter` and `SupportsTypedMapReader`. + +A writer defines an `OutputMapEntryType` context and the following methods: + +```cpp +OutputMapEntryType map_entry(OutputMapType* map) const; +OutputArrayType add_array_to_map_entry(size_t size, OutputMapEntryType*) const; +OutputObjectType add_object_to_map_entry(size_t size, OutputMapEntryType*) const; +OutputVarType add_null_to_map_entry(OutputMapEntryType*) const; + +template +OutputVarType add_value_to_map_entry(const T& value, OutputMapEntryType*) const; + +// Additionally required for schemaful writers: +OutputMapType add_map_to_map_entry(size_t size, OutputMapEntryType*) const; +OutputUnionType add_union_to_map_entry(OutputMapEntryType*) const; +``` + +Schemaless writers use `OutputObjectType*` instead of `OutputMapType*` for +`map_entry`. The map is already open: `MapParser` uses the normal map header +for schemaful backends, or the object header for schemaless backends, and closes +it after all entries. For each entry, it creates a context and passes it through +`Parent::MapEntry` to the ordinary key and value parsers, in that order. +The context does not introduce an array or a pair-length prefix. Streaming +backends can use an empty context; other backends can retain their entry state +in it. The `add_*_to_map_entry` methods emit only the requested key or value. + +The corresponding reader provides: + +```cpp +template +std::optional read_typed_map(const MapReader& reader, + const InputMapType& map) const noexcept; +``` + +Schemaless readers use `InputObjectType` instead of `InputMapType`. For each +entry, call `reader.read(key, value)` with two `InputVarType` values and return +immediately if it returns an error. The shared `TypedMapReader` parses the key +before the value, so sequential archives may pass two handles to the same +archive. Backend traversal handles the map entry count and boundaries. + +Schemaful readers can provide either `read_map` for string keys or +`read_typed_map` for typed keys. They do not need to implement both. Backends +without the typed-map interfaces retain string-key serialization. Objects +representing struct fields continue to use their existing field-name behavior. diff --git a/include/rfl/boost_serialization/Reader.hpp b/include/rfl/boost_serialization/Reader.hpp index 13626f69f..1ce764cfe 100644 --- a/include/rfl/boost_serialization/Reader.hpp +++ b/include/rfl/boost_serialization/Reader.hpp @@ -155,24 +155,20 @@ struct Reader { return std::nullopt; } - /// Reads all key-value pairs from a map using the provided map reader. - /// Keys are read as strings from the archive. - /// @tparam MapReader The type of reader that processes individual map entries - /// @param _map_reader The reader object that processes each key-value pair - /// @param _map The map structure containing size information - /// @return std::nullopt on success, or an Error if reading fails + /// Reads keys and values directly from the archive in sequence. template - std::optional read_map(const MapReader& _map_reader, - const InputMapType& _map) const noexcept { + std::optional read_typed_map(const MapReader& _reader, + const InputMapType& _map) const noexcept { try { - for (size_t i = 0; i < _map.size; ++i) { - std::string key; - *_map.ar >> key; - auto var = InputVarType{_map.ar}; - _map_reader.read(std::string_view(key), var); + const auto size = _map.size; + for (size_t i = 0; i < size; ++i) { + const auto var = InputVarType{_map.ar}; + if (const auto err = _reader.read(var, var)) { + return err; + } } return std::nullopt; - } catch (std::exception& e) { + } catch (const std::exception& e) { return Error(e.what()); } } diff --git a/include/rfl/boost_serialization/Writer.hpp b/include/rfl/boost_serialization/Writer.hpp index 8c18842d5..d2530aad7 100644 --- a/include/rfl/boost_serialization/Writer.hpp +++ b/include/rfl/boost_serialization/Writer.hpp @@ -31,6 +31,41 @@ class Writer { using OutputUnionType = BoostOutputUnion; using OutputVarType = BoostOutputVar; + /// A context for writing a typed map entry's key, then its value. + struct OutputMapEntryType {}; + + OutputMapEntryType map_entry(OutputMapType*) const { return {}; } + + OutputArrayType add_array_to_map_entry(const size_t _size, + OutputMapEntryType*) const { + return array_as_root(_size); + } + + OutputObjectType add_object_to_map_entry(const size_t _size, + OutputMapEntryType*) const { + return object_as_root(_size); + } + + template + OutputVarType add_value_to_map_entry(const T& _var, + OutputMapEntryType*) const { + new_value(_var); + return OutputVarType{}; + } + + OutputVarType add_null_to_map_entry(OutputMapEntryType*) const { + return null_as_root(); + } + + OutputMapType add_map_to_map_entry(const size_t _size, + OutputMapEntryType*) const { + return map_as_root(_size); + } + + OutputUnionType add_union_to_map_entry(OutputMapEntryType*) const { + return union_as_root(); + } + /// Constructs a Writer with a Boost output archive. /// @param _ar Pointer to the Boost output archive to write to Writer(OArchive* _ar) : ar_(_ar) {} diff --git a/include/rfl/cereal/Reader.hpp b/include/rfl/cereal/Reader.hpp index 930be1410..12ad6aeb1 100644 --- a/include/rfl/cereal/Reader.hpp +++ b/include/rfl/cereal/Reader.hpp @@ -146,26 +146,22 @@ struct Reader { } } - /// Reads all key-value pairs from a Cereal map using the provided map reader. - /// Keys are stored as strings in the archive. - /// @tparam MapReader The type of reader that processes individual map entries - /// @param _map_reader The reader object that processes each key-value pair - /// @param _map The Cereal map structure - /// @return std::nullopt on success, or an Error if reading fails + /// Reads keys and values directly from the archive in sequence. template - std::optional read_map(const MapReader& _map_reader, - const InputMapType& _map) const noexcept { + std::optional read_typed_map(const MapReader& _reader, + const InputMapType& _map) const noexcept { try { size_t size; (*_map.archive_)(::cereal::make_size_tag(size)); for (size_t i = 0; i < size; ++i) { - std::string key; - (*_map.archive_)(key); - _map_reader.read(std::string_view(key), InputVarType{_map.archive_}); + const auto var = InputVarType{_map.archive_}; + if (const auto err = _reader.read(var, var)) { + return err; + } } return std::nullopt; - } catch (std::exception& e) { - return Error(std::string("Cereal map read error: ") + e.what()); + } catch (const std::exception& e) { + return Error(e.what()); } } diff --git a/include/rfl/cereal/Writer.hpp b/include/rfl/cereal/Writer.hpp index 15a2993fe..366c6b970 100644 --- a/include/rfl/cereal/Writer.hpp +++ b/include/rfl/cereal/Writer.hpp @@ -42,6 +42,41 @@ class Writer { using OutputUnionType = CerealOutputUnion; using OutputVarType = CerealOutputVar; + /// A context for writing a typed map entry's key, then its value. + struct OutputMapEntryType {}; + + OutputMapEntryType map_entry(OutputMapType*) const { return {}; } + + OutputArrayType add_array_to_map_entry(const size_t _size, + OutputMapEntryType*) const { + return array_as_root(_size); + } + + OutputObjectType add_object_to_map_entry(const size_t _size, + OutputMapEntryType*) const { + return object_as_root(_size); + } + + template + OutputVarType add_value_to_map_entry(const T& _var, + OutputMapEntryType*) const { + add_value(_var); + return OutputVarType{}; + } + + OutputVarType add_null_to_map_entry(OutputMapEntryType*) const { + return null_as_root(); + } + + OutputMapType add_map_to_map_entry(const size_t _size, + OutputMapEntryType*) const { + return map_as_root(_size); + } + + OutputUnionType add_union_to_map_entry(OutputMapEntryType*) const { + return union_as_root(); + } + /// Constructs a Writer with a Cereal output archive. /// @param _archive Pointer to the Cereal output archive to write to Writer(CerealArchive* _archive); diff --git a/include/rfl/msgpack/Reader.hpp b/include/rfl/msgpack/Reader.hpp index e797ce09b..5ca3afd6b 100644 --- a/include/rfl/msgpack/Reader.hpp +++ b/include/rfl/msgpack/Reader.hpp @@ -184,6 +184,18 @@ struct Reader { return std::nullopt; } + /// Reads native map entries without converting keys to field names. + template + std::optional read_typed_map( + const MapReader& _reader, const InputObjectType& _map) const noexcept { + for (uint32_t i = 0; i < _map.size; ++i) { + if (const auto err = _reader.read(_map.ptr[i].key, _map.ptr[i].val)) { + return err; + } + } + return std::nullopt; + } + /** * @brief Reads all fields of a msgpack object using a provided reader. * @tparam ObjectReader The type of the object reader. diff --git a/include/rfl/msgpack/Writer.hpp b/include/rfl/msgpack/Writer.hpp index 7334ee253..9953dcff9 100644 --- a/include/rfl/msgpack/Writer.hpp +++ b/include/rfl/msgpack/Writer.hpp @@ -26,6 +26,31 @@ class RFL_API Writer { using OutputObjectType = MsgpackOutputObject; using OutputVarType = MsgpackOutputVar; + /// A context for writing a typed map entry's key, then its value. + struct OutputMapEntryType {}; + + OutputMapEntryType map_entry(OutputObjectType*) const { return {}; } + + OutputArrayType add_array_to_map_entry(const size_t _size, + OutputMapEntryType*) const { + return array_as_root(_size); + } + + OutputObjectType add_object_to_map_entry(const size_t _size, + OutputMapEntryType*) const { + return object_as_root(_size); + } + + template + OutputVarType add_value_to_map_entry(const T& _var, + OutputMapEntryType*) const { + return new_value(_var); + } + + OutputVarType add_null_to_map_entry(OutputMapEntryType*) const { + return null_as_root(); + } + Writer(msgpack_packer* _pk); ~Writer(); diff --git a/include/rfl/parsing/MapParser.hpp b/include/rfl/parsing/MapParser.hpp index 748d6d5c3..eda6ddd00 100644 --- a/include/rfl/parsing/MapParser.hpp +++ b/include/rfl/parsing/MapParser.hpp @@ -11,6 +11,9 @@ #include "MapReader.hpp" #include "Parent.hpp" #include "Parser_base.hpp" +#include "SupportsTypedMapReader.hpp" +#include "SupportsTypedMapWriter.hpp" +#include "TypedMapReader.hpp" #include "schema/Type.hpp" #include "schemaful/IsSchemafulReader.hpp" #include "schemaful/IsSchemafulWriter.hpp" @@ -60,7 +63,9 @@ struct MapParser { */ template static void write(const W& _w, const MapType& _m, const P& _parent) { - if constexpr (schemaful::IsSchemafulWriter) { + if constexpr (SupportsTypedMapWriter) { + write_typed_map(_w, _m, _parent); + } else if constexpr (schemaful::IsSchemafulWriter) { write_map(_w, _m, _parent); } else { write_object(_w, _m, _parent); @@ -82,24 +87,57 @@ struct MapParser { private: static Result make_map(const R& _r, const auto& _obj_or_map) { MapType map; - std::vector errors; - const auto map_reader = - MapReader(&_r, &map, &errors); - if constexpr (schemaful::IsSchemafulReader) { - const auto err = _r.read_map(map_reader, _obj_or_map); - if (err) { + if constexpr (SupportsTypedMapReader< + R, std::remove_cvref_t>) { + const auto reader = + TypedMapReader{&_r, &map}; + if (const auto err = _r.read_typed_map(reader, _obj_or_map)) { return error(*err); } + return map; } else { - const auto err = _r.read_object(map_reader, _obj_or_map); - if (err) { - return error(*err); + std::vector errors; + const auto map_reader = + MapReader(&_r, &map, &errors); + if constexpr (schemaful::IsSchemafulReader) { + const auto err = _r.read_map(map_reader, _obj_or_map); + if (err) { + return error(*err); + } + } else { + const auto err = _r.read_object(map_reader, _obj_or_map); + if (err) { + return error(*err); + } + } + if (errors.size() != 0) { + return error(to_single_error_message(errors)); } + return map; } - if (errors.size() != 0) { - return error(to_single_error_message(errors)); + } + + template + static void write_typed_map(const W& _w, const MapType& _m, + const P& _parent) { + const auto write_entries = [&](auto* map) { + for (const auto& [key, value] : _m) { + auto entry = _w.map_entry(map); + const auto parent = typename ParentType::template MapEntry< + typename W::OutputMapEntryType>{&entry}; + Parser::write(_w, key, parent); + Parser::write(_w, value, parent); + } + }; + if constexpr (schemaful::IsSchemafulWriter) { + auto map = ParentType::add_map(_w, _m.size(), _parent); + write_entries(&map); + _w.end_map(&map); + } else { + auto map = ParentType::add_object(_w, _m.size(), _parent); + write_entries(&map); + _w.end_object(&map); } - return map; } template diff --git a/include/rfl/parsing/Parent.hpp b/include/rfl/parsing/Parent.hpp index 5d868d4cb..e59d6a8ac 100644 --- a/include/rfl/parsing/Parent.hpp +++ b/include/rfl/parsing/Parent.hpp @@ -21,6 +21,12 @@ struct Parent { OutputArrayType* arr_; }; + // The key and value of a typed map entry are serialized in that order. + template + struct MapEntry { + T* entry_; + }; + // For schemaful formats only. template struct Map { @@ -60,6 +66,9 @@ struct Parent { if constexpr (std::is_same()) { return _w.add_array_to_array(_size, _parent.arr_); + } else if constexpr (is_map_entry()) { + return _w.add_array_to_map_entry(_size, _parent.entry_); + } else if constexpr (std::is_same()) { return _w.add_array_to_object(_parent.name_, _size, _parent.obj_); @@ -119,6 +128,9 @@ struct Parent { if constexpr (std::is_same()) { return _w.add_map_to_array(_size, _parent.arr_); + } else if constexpr (is_map_entry()) { + return _w.add_map_to_map_entry(_size, _parent.entry_); + } else if constexpr (std::is_same>()) { return _w.add_map_to_map(_parent.name_, _size, _parent.map_); @@ -153,6 +165,9 @@ struct Parent { if constexpr (std::is_same()) { return _w.add_object_to_array(_size, _parent.arr_); + } else if constexpr (is_map_entry()) { + return _w.add_object_to_map_entry(_size, _parent.entry_); + } else if constexpr (std::is_same()) { return _w.add_object_to_object(_parent.name_, _size, _parent.obj_); @@ -188,6 +203,9 @@ struct Parent { if constexpr (std::is_same()) { return _w.add_null_to_array(_parent.arr_); + } else if constexpr (is_map_entry()) { + return _w.add_null_to_map_entry(_parent.entry_); + } else if constexpr (std::is_same()) { if constexpr (supports_attributes>) { return _w.add_null_to_object(_parent.name_, _parent.obj_, @@ -228,6 +246,9 @@ struct Parent { if constexpr (std::is_same()) { return _w.add_union_to_array(_parent.arr_); + } else if constexpr (is_map_entry()) { + return _w.add_union_to_map_entry(_parent.entry_); + } else if constexpr (std::is_same>()) { return _w.add_union_to_map(_parent.name_, _parent.map_); @@ -263,6 +284,9 @@ struct Parent { if constexpr (std::is_same()) { return _w.add_value_to_array(_var, _parent.arr_); + } else if constexpr (is_map_entry()) { + return _w.add_value_to_map_entry(_var, _parent.entry_); + } else if constexpr (std::is_same()) { if constexpr (supports_attributes>) { return _w.add_value_to_object(_parent.name_, _var, _parent.obj_, @@ -288,6 +312,16 @@ struct Parent { static_assert(always_false_v, "Unsupported option."); } } + + private: + template + static constexpr bool is_map_entry() { + if constexpr (requires { typename W::OutputMapEntryType; }) { + return std::is_same_v>; + } else { + return false; + } + } }; } // namespace rfl::parsing diff --git a/include/rfl/parsing/SupportsTypedMapReader.hpp b/include/rfl/parsing/SupportsTypedMapReader.hpp new file mode 100644 index 000000000..4a53cdc8a --- /dev/null +++ b/include/rfl/parsing/SupportsTypedMapReader.hpp @@ -0,0 +1,28 @@ +#ifndef RFL_PARSING_SUPPORTSTYPEDMAPREADER_HPP_ +#define RFL_PARSING_SUPPORTSTYPEDMAPREADER_HPP_ + +#include +#include + +#include "../Result.hpp" + +namespace rfl::parsing { + +template +struct MockTypedMapReader { + std::optional read(const typename R::InputVarType&, + const typename R::InputVarType&) const { + return std::nullopt; + } +}; + +/// A map reader that exposes keys and values without converting keys to names. +template +concept SupportsTypedMapReader = + requires(const R& r, const MapType& map, MockTypedMapReader reader) { + { r.read_typed_map(reader, map) } -> std::same_as>; + }; + +} // namespace rfl::parsing + +#endif diff --git a/include/rfl/parsing/SupportsTypedMapWriter.hpp b/include/rfl/parsing/SupportsTypedMapWriter.hpp new file mode 100644 index 000000000..45bf7c123 --- /dev/null +++ b/include/rfl/parsing/SupportsTypedMapWriter.hpp @@ -0,0 +1,47 @@ +#ifndef RFL_PARSING_SUPPORTSTYPEDMAPWRITER_HPP_ +#define RFL_PARSING_SUPPORTSTYPEDMAPWRITER_HPP_ + +#include +#include + +#include "schemaful/IsSchemafulWriter.hpp" + +namespace rfl::parsing { + +/// A writer with an explicit context for the key and value of a map entry. +template +concept SupportsTypedMapWriter = + requires(const W& w, typename W::OutputMapEntryType* entry, + std::size_t size) { + { + w.add_array_to_map_entry(size, entry) + } -> std::same_as; + { + w.add_object_to_map_entry(size, entry) + } -> std::same_as; + { + w.add_value_to_map_entry(0, entry) + } -> std::same_as; + { + w.add_null_to_map_entry(entry) + } -> std::same_as; + } && + ((schemaful::IsSchemafulWriter && + requires(const W& w, typename W::OutputMapType* map, + typename W::OutputMapEntryType* entry, std::size_t size) { + { w.map_entry(map) } -> std::same_as; + { + w.add_map_to_map_entry(size, entry) + } -> std::same_as; + { + w.add_union_to_map_entry(entry) + } -> std::same_as; + }) || + (!schemaful::IsSchemafulWriter && + requires(const W& w, typename W::OutputObjectType* map) { + { w.map_entry(map) } -> std::same_as; + })); + +} // namespace rfl::parsing + +#endif diff --git a/include/rfl/parsing/TypedMapReader.hpp b/include/rfl/parsing/TypedMapReader.hpp new file mode 100644 index 000000000..869c57d2a --- /dev/null +++ b/include/rfl/parsing/TypedMapReader.hpp @@ -0,0 +1,48 @@ +#ifndef RFL_PARSING_TYPEDMAPREADER_HPP_ +#define RFL_PARSING_TYPEDMAPREADER_HPP_ + +#include +#include +#include +#include +#include + +#include "../Result.hpp" +#include "Parser_base.hpp" + +namespace rfl::parsing { + +/// Parses map keys as values, preserving their serialized types. +template +struct TypedMapReader { + using InputVarType = typename R::InputVarType; + using Key = std::remove_cvref_t; + using Value = std::remove_cvref_t; + + const R* reader; + MapType* map; + + std::optional read(const InputVarType& _key, + const InputVarType& _value) const noexcept { + // Archive readers can pass the same stream for both arguments. Always + // finish reading the key before starting the value, and stop on failure. + auto key = Parser::read(*reader, _key); + if (!key) { + return Error("Failed to parse map key: " + key.error().what()); + } + auto value = Parser::read(*reader, _value); + if (!value) { + return Error("Failed to parse map value: " + value.error().what()); + } + try { + map->emplace(std::move(*key), std::move(*value)); + return std::nullopt; + } catch (const std::exception& e) { + return Error(e.what()); + } + } +}; + +} // namespace rfl::parsing + +#endif diff --git a/include/rfl/parsing/schemaful/IsSchemafulReader.hpp b/include/rfl/parsing/schemaful/IsSchemafulReader.hpp index bada85f37..e52cad39e 100644 --- a/include/rfl/parsing/schemaful/IsSchemafulReader.hpp +++ b/include/rfl/parsing/schemaful/IsSchemafulReader.hpp @@ -7,6 +7,7 @@ #include #include "../../Result.hpp" +#include "../SupportsTypedMapReader.hpp" namespace rfl::parsing::schemaful { @@ -58,7 +59,9 @@ concept IsSchemafulReader = /// A schemaful reader needs to differentiate between objects, for which /// the field names are known at compile time and maps, for which the /// field names are not known at compile time. - { r.read_map(map_reader, m) } -> std::same_as>; + requires SupportsTypedMapReader || requires { + { r.read_map(map_reader, m) } -> std::same_as>; + }; /// A schemaful reader can read fields by order and does not have to /// compare strings - the correct order of the fields is guaranteed by the diff --git a/include/rfl/yas/Reader.hpp b/include/rfl/yas/Reader.hpp index c06dffc3e..b09547dcb 100644 --- a/include/rfl/yas/Reader.hpp +++ b/include/rfl/yas/Reader.hpp @@ -138,25 +138,22 @@ struct Reader { } } - /// @brief Reads a map from the archive using the provided MapReader. - /// @tparam MapReader The type responsible for reading map entries. - /// @param _map_reader The reader instance for map entries. - /// @param _map The input map type containing the archive pointer. - /// @return std::nullopt on success, or an Error if reading fails. + /// Reads keys and values directly from the archive in sequence. template - std::optional read_map(const MapReader& _map_reader, - const InputMapType& _map) const noexcept { + std::optional read_typed_map(const MapReader& _reader, + const InputMapType& _map) const noexcept { try { - std::size_t size; + size_t size; (*_map.ar) & size; - for (std::size_t i = 0; i < size; ++i) { - std::string key; - (*_map.ar) & key; - _map_reader.read(std::string_view(key), InputVarType{_map.ar}); + for (size_t i = 0; i < size; ++i) { + const auto var = InputVarType{_map.ar}; + if (const auto err = _reader.read(var, var)) { + return err; + } } return std::nullopt; - } catch (std::exception& e) { - return Error(std::string("yas map read error: ") + e.what()); + } catch (const std::exception& e) { + return Error(e.what()); } } diff --git a/include/rfl/yas/Writer.hpp b/include/rfl/yas/Writer.hpp index 4d2352587..51cf37d6f 100644 --- a/include/rfl/yas/Writer.hpp +++ b/include/rfl/yas/Writer.hpp @@ -43,6 +43,40 @@ struct Writer { OArchive* ar; }; + /// A context for writing a typed map entry's key, then its value. + struct OutputMapEntryType {}; + + OutputMapEntryType map_entry(OutputMapType*) const { return {}; } + + OutputArrayType add_array_to_map_entry(const size_t _size, + OutputMapEntryType*) const { + return array_as_root(_size); + } + + OutputObjectType add_object_to_map_entry(const size_t _size, + OutputMapEntryType*) const { + return object_as_root(_size); + } + + template + OutputVarType add_value_to_map_entry(const T& _var, + OutputMapEntryType*) const { + add_value(_var); + return OutputVarType{ar_}; + } + + OutputVarType add_null_to_map_entry(OutputMapEntryType*) const { + return null_as_root(); + } + + OutputMapType add_map_to_map_entry(const size_t _size, + OutputMapEntryType*) const { + return map_as_root(_size); + } + + OutputUnionType add_union_to_map_entry(OutputMapEntryType*) const { + return union_as_root(); + } /// /// @brief Constructs a Writer with the given output archive pointer. /// @param _ar Pointer to the output archive used for serialization. diff --git a/tests/boost_serialization/test_typed_maps.cpp b/tests/boost_serialization/test_typed_maps.cpp new file mode 100644 index 000000000..ba7bc041b --- /dev/null +++ b/tests/boost_serialization/test_typed_maps.cpp @@ -0,0 +1,33 @@ +#include "../test_typed_maps.hpp" + +#include +#include + +TEST(boost_serialization, typed_map_keys) { + test_typed_maps::check( + [](const auto& value) { return rfl::boost_serialization::write(value); }, + [](const auto& bytes) { + return rfl::boost_serialization::read(bytes); + }); +} + +TEST(boost_serialization, typed_map_wire_format) { + const std::map values{{-1, 3}, {42, 4}}; + std::ostringstream stream; + { + boost::archive::binary_oarchive archive( + stream, boost::archive::no_header | boost::archive::no_tracking); + const std::uint64_t size = values.size(); + archive << size; + for (const auto& [key, value] : values) { + archive << key << value; + } + } + const auto native = stream.str(); + const std::vector expected(native.begin(), native.end()); + EXPECT_EQ(rfl::boost_serialization::write(values), expected); + const auto decoded = + rfl::boost_serialization::read>(expected); + ASSERT_TRUE(decoded) << decoded.error().what(); + EXPECT_EQ(*decoded, values); +} diff --git a/tests/cereal/test_typed_maps.cpp b/tests/cereal/test_typed_maps.cpp new file mode 100644 index 000000000..c0eebc1ef --- /dev/null +++ b/tests/cereal/test_typed_maps.cpp @@ -0,0 +1,34 @@ +#include "../test_typed_maps.hpp" + +#include +#include +#include + +TEST(cereal, typed_map_keys) { + test_typed_maps::check( + [](const auto& value) { return rfl::cereal::write(value); }, + [](const auto& bytes) { return rfl::cereal::read(bytes); }); +} + +TEST(cereal, typed_map_wire_format) { + const std::map values{{1.0, 3}, {std::nextafter(1.0, 2.0), 4}}; + std::ostringstream stream; + { + cereal::PortableBinaryOutputArchive archive(stream); + archive(values); + } + const auto native = stream.str(); + const std::vector expected(native.begin(), native.end()); + EXPECT_EQ(rfl::cereal::write(values), expected); + const auto decoded = + rfl::cereal::read>(expected); + ASSERT_TRUE(decoded) << decoded.error().what(); + EXPECT_EQ(*decoded, values); +} + +TEST(cereal, typed_map_bytestring_values) { + test_typed_maps::roundtrip( + [](const auto& value) { return rfl::cereal::write(value); }, + [](const auto& bytes) { return rfl::cereal::read(bytes); }, + std::map{{1, {std::byte{0}, std::byte{255}}}}); +} diff --git a/tests/generic/test_typed_map_capabilities.cpp b/tests/generic/test_typed_map_capabilities.cpp new file mode 100644 index 000000000..ae93f5ad0 --- /dev/null +++ b/tests/generic/test_typed_map_capabilities.cpp @@ -0,0 +1,50 @@ +#include +#include +#include + +namespace test_typed_map_capabilities { + +// A schemaful backend must be able to provide either map API independently. +struct ReaderBase { + struct InputVarType {}; + struct InputObjectType {}; + struct InputMapType {}; + struct InputUnionType {}; + + template + std::optional read_object(const ObjectReader&, + const InputObjectType&) const; + template + rfl::Result read_union(const InputUnionType&) const; + rfl::Result to_map(const InputVarType&) const; + rfl::Result to_union(const InputVarType&) const; +}; + +struct StringKeyReader : ReaderBase { + template + std::optional read_map(const MapReader&, + const InputMapType&) const; +}; + +struct TypedKeyReader : ReaderBase { + template + std::optional read_typed_map(const MapReader&, + const InputMapType&) const; +}; + +struct InvalidTypedKeyReader : ReaderBase { + template + bool read_typed_map(const MapReader&, const InputMapType&) const; +}; + +static_assert(rfl::parsing::schemaful::IsSchemafulReader); +static_assert(rfl::parsing::schemaful::IsSchemafulReader); +static_assert(!rfl::parsing::schemaful::IsSchemafulReader); +static_assert( + !rfl::parsing::schemaful::IsSchemafulReader); +static_assert(rfl::parsing::SupportsTypedMapReader); +static_assert(!rfl::parsing::SupportsTypedMapReader); + +} // namespace test_typed_map_capabilities diff --git a/tests/msgpack/test_typed_maps.cpp b/tests/msgpack/test_typed_maps.cpp new file mode 100644 index 000000000..33900c90e --- /dev/null +++ b/tests/msgpack/test_typed_maps.cpp @@ -0,0 +1,25 @@ +#include "../test_typed_maps.hpp" + +#include + +TEST(msgpack, typed_map_keys) { + test_typed_maps::check( + [](const auto& value) { return rfl::msgpack::write(value); }, + [](const auto& bytes) { return rfl::msgpack::read(bytes); }); +} + +TEST(msgpack, typed_map_wire_format) { + const std::map values{{-1, 3}, {1, 2}}; + const std::vector expected{char(0x82), char(0xff), 3, 1, 2}; + EXPECT_EQ(rfl::msgpack::write(values), expected); + const auto decoded = + rfl::msgpack::read>(expected); + ASSERT_TRUE(decoded) << decoded.error().what(); + EXPECT_EQ(*decoded, values); + + // Numeric map keys must be numbers on the wire, not numeric strings. + const auto strings = + rfl::msgpack::write(std::map{{"1", 2}}); + EXPECT_FALSE((rfl::msgpack::read>(strings))); + EXPECT_FALSE((rfl::msgpack::read>(expected))); +} diff --git a/tests/test_typed_maps.hpp b/tests/test_typed_maps.hpp new file mode 100644 index 000000000..7c3a8cded --- /dev/null +++ b/tests/test_typed_maps.hpp @@ -0,0 +1,103 @@ +#ifndef RFL_TEST_TYPED_MAPS_HPP_ +#define RFL_TEST_TYPED_MAPS_HPP_ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace test_typed_maps { + +template +void roundtrip(const auto& write, const auto& read, const T& value) { + const auto bytes = write(value); + const auto result = read.template operator()(bytes); + ASSERT_TRUE(result) << result.error().what(); + EXPECT_EQ(*result, value); +} + +struct Nested { + std::map>> values; + std::map empty; + int after; +}; + +inline void check(const auto& write, const auto& read) { + roundtrip(write, read, std::map{}); + roundtrip(write, read, + std::map{ + {std::numeric_limits::min(), 1}, + {-1, 2}, + {0, 3}, + {std::numeric_limits::max(), 4}}); + roundtrip(write, read, + std::map{ + {0, 1}, {std::numeric_limits::max(), 2}}); + roundtrip(write, read, + std::unordered_map{{-7, "a"}, {42, "b"}}); + roundtrip(write, read, std::map{{false, 1}, {true, 2}}); + roundtrip(write, read, std::map{{"", 1}, {"42", 2}}); + + roundtrip(write, read, std::map>{{1, 2}, {3, 4}}); + roundtrip(write, read, + std::vector>{{{1, 2}}, {}, {{3, 4}}}); + roundtrip(write, read, + std::variant>(std::map{{3, 4}})); + + roundtrip( + write, read, + std::map>{{1, 42}, {2, "value"}}); + roundtrip(write, read, + std::map>{{1, std::nullopt}, {2, 42}}); + + // These distinct keys collapse to the same string with std::to_string. + const double adjacent = std::nextafter(1.0, 2.0); + roundtrip(write, read, std::map{{1.0, 1}, {adjacent, 2}}); + roundtrip(write, read, + std::map{{1.0f, 1}, {std::nextafter(1.0f, 2.0f), 2}}); + + const Nested nested{{{7, {{adjacent, {1, 2, 3}}}}}, {}, 12345}; + const auto result = read.template operator()(write(nested)); + ASSERT_TRUE(result) << result.error().what(); + EXPECT_EQ(result->values, nested.values); + EXPECT_TRUE(result->empty.empty()); + EXPECT_EQ(result->after, nested.after); + + using Key = rfl::Validator>; + using ValidatedMap = std::map; + const auto valid = read.template operator()( + write(std::map{{7, 12}})); + ASSERT_TRUE(valid) << valid.error().what(); + EXPECT_EQ(valid->begin()->first.reflection(), 7); + EXPECT_EQ(write(*valid), write(std::map{{7, 12}})); + const auto invalid = read.template operator()( + write(std::map{{-1, 12}})); + EXPECT_FALSE(invalid); + + // Writing must not require copying mapped values into an intermediate vector. + std::map> owning; + owning.emplace(7, std::make_unique(42)); + const auto owned = read.template operator()(write(owning)); + ASSERT_TRUE(owned) << owned.error().what(); + ASSERT_EQ(owned->size(), 1); + ASSERT_TRUE(owned->at(7)); + EXPECT_EQ(*owned->at(7), 42); + + auto truncated = write(std::map{{7, 42}}); + truncated.pop_back(); + EXPECT_FALSE((read.template operator()>(truncated))); +} + +} // namespace test_typed_maps + +#endif diff --git a/tests/yas/test_typed_maps.cpp b/tests/yas/test_typed_maps.cpp new file mode 100644 index 000000000..5a838c07f --- /dev/null +++ b/tests/yas/test_typed_maps.cpp @@ -0,0 +1,26 @@ +#include "../test_typed_maps.hpp" + +#include + +TEST(yas, typed_map_keys) { + test_typed_maps::check( + [](const auto& value) { return rfl::yas::write(value); }, + [](const auto& bytes) { return rfl::yas::read(bytes); }); +} + +TEST(yas, typed_map_wire_format) { + const std::map values{{-1, 3}, {42, 4}}; + yas::mem_ostream stream; + yas::binary_oarchive archive(stream); + const std::size_t size = values.size(); + archive & size; + for (const auto& [key, value] : values) { + archive & key & value; + } + const auto buffer = stream.get_intrusive_buffer(); + const std::vector expected(buffer.data, buffer.data + buffer.size); + EXPECT_EQ(rfl::yas::write(values), expected); + const auto decoded = rfl::yas::read>(expected); + ASSERT_TRUE(decoded) << decoded.error().what(); + EXPECT_EQ(*decoded, values); +}