diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8d9f5bad..c02d8d16 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -81,6 +81,7 @@ cargo bench --package benchmarks --bench benchmarks -- cpc::serde - End summary sentences with punctuation, and format Rust identifiers, literals, and numeric ranges as inline code. - Put contract sections and compatibility notes before examples. When applicable, order sections as `# Errors`, `# Panics`, and `# Examples`. Include only sections that describe an actual contract. - Document return values and error conditions on the relevant methods rather than repeating them in crate or module overviews. +- Describe shared capabilities in language-neutral terms. Name another implementation when it clarifies interoperability requirements, a specific source, a format variant, or a behavior difference. Retain useful implementation references in internal comments. - Keep imports visible in doctests. A trailing hidden `Ok(...)` may be used to support `?`. ## Query semantics diff --git a/README.md b/README.md index a4a03736..7ec87caf 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,7 @@ The minimum supported Rust version is 1.86.0. The crate currently supports littl Supported serialization formats are tested with fixtures produced by Apache DataSketches Java, C++, and Go through the [DataSketches TCK](https://github.com/apache/datasketches-tck). -Serialization compatibility does not imply that an ordinary Rust `Hash` implementation produces the same update bytes as another language. When sketches must represent the same inputs across implementations, use `hash::value::{raw_bytes, canonical_float, sign_extend, natural_extend}` (and the constructors within those modules) to match the other language implementations’ hashing rules. Other DataSketches implementations skip empty strings, so skip them before updating when that behavior matters. +Serialization compatibility does not imply that an ordinary Rust `Hash` implementation produces the same update bytes across languages. When sketches must represent the same inputs across implementations, use `hash::value::{raw_bytes, canonical_float, sign_extend, natural_extend}` (and the constructors within those modules) to select matching input encodings. When interoperating with an implementation that skips empty strings, skip them before updating here as well. See the [changelog](CHANGELOG.md) for release notes and migration guidance. diff --git a/datasketches/src/bloom/sketch.rs b/datasketches/src/bloom/sketch.rs index fb311a8d..0d399d58 100644 --- a/datasketches/src/bloom/sketch.rs +++ b/datasketches/src/bloom/sketch.rs @@ -261,7 +261,7 @@ impl BloomFilter { /// * Items never inserted into this filter may return `true` (false positives), at a rate no /// higher than this filter's false positive rate before the operation. /// - /// This is the transformation other DataSketches libraries expose as A NOT B. + /// This operation is also known as A NOT B. /// /// # Errors /// @@ -379,7 +379,7 @@ impl BloomFilter { /// Serializes the filter to a byte vector. /// - /// The format is compatible with other Apache DataSketches implementations. + /// The serialized data can be read by other DataSketches implementations. /// /// # Examples /// @@ -441,6 +441,8 @@ impl BloomFilter { /// Deserializes a filter from bytes. /// + /// Also accepts data serialized by other DataSketches implementations. + /// /// # Errors /// /// Returns an error if: diff --git a/datasketches/src/common/inv_pow2.rs b/datasketches/src/common/inv_pow2.rs index a61f1ca2..64fa808d 100644 --- a/datasketches/src/common/inv_pow2.rs +++ b/datasketches/src/common/inv_pow2.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -/// Compute 1 / 2^exp using the same bit construction as DataSketches Java. +/// Returns `2^-exp` by constructing the floating-point exponent bits, as in datasketches-java. #[inline] pub fn inv_pow2(exp: u8) -> f64 { f64::from_bits((1023 - exp as u64) << 52) diff --git a/datasketches/src/countmin/sketch.rs b/datasketches/src/countmin/sketch.rs index 6c02dc00..24a60e44 100644 --- a/datasketches/src/countmin/sketch.rs +++ b/datasketches/src/countmin/sketch.rs @@ -306,7 +306,10 @@ impl CountMinSketch { Ok(()) } - /// Serializes this sketch into the DataSketches CountMin format. + /// Serializes this sketch into the Count-Min binary format. + /// + /// DataSketches C++ can read the serialized data, but item queries and updates there are + /// unreliable because the two implementations map items to counters differently. /// /// # Examples /// @@ -357,6 +360,10 @@ impl CountMinSketch { /// Deserializes a sketch from bytes using the default seed. /// + /// Can read sketches serialized by DataSketches C++. + /// Item queries and updates on those sketches are unreliable because the two + /// implementations map items to counters differently. + /// /// # Errors /// /// Returns `InvalidData` if the image is malformed or its seed hash does not match the default @@ -379,6 +386,10 @@ impl CountMinSketch { /// Deserializes a sketch from bytes using the provided seed. /// + /// Can read sketches serialized by DataSketches C++. + /// Item queries and updates on those sketches are unreliable because the two + /// implementations map items to counters differently. + /// /// # Errors /// /// Returns `InvalidData` if the image is malformed, its seed hash does not match `seed`, or diff --git a/datasketches/src/cpc/sketch.rs b/datasketches/src/cpc/sketch.rs index d946b1b2..c42b3672 100644 --- a/datasketches/src/cpc/sketch.rs +++ b/datasketches/src/cpc/sketch.rs @@ -179,8 +179,8 @@ impl CpcSketch { /// Updates the sketch with a hashable value. /// - /// You may use [`hash::value`](crate::hash::value) wrappers when another DataSketches - /// implementation requires a specific value hashing strategy. + /// Use [`hash::value`](crate::hash::value) wrappers to select an input encoding for + /// cross-language hashing. /// /// # Examples /// @@ -490,6 +490,8 @@ impl fmt::Debug for CpcSketch { impl CpcSketch { /// Serializes this `CpcSketch` to bytes. + /// + /// The serialized data can be read by other DataSketches implementations. pub fn serialize(&self) -> Vec { let flavor = self.flavor(); let has_hip = !self.merge_flag; @@ -623,6 +625,8 @@ impl CpcSketch { /// Deserializes a `CpcSketch` from bytes. /// + /// Also accepts data serialized by other DataSketches implementations. + /// /// # Errors /// /// Returns `InvalidData` if the image is malformed or its seed hash does not match the default @@ -633,6 +637,8 @@ impl CpcSketch { /// Deserializes a `CpcSketch` from bytes with the provided seed. /// + /// Also accepts data serialized by other DataSketches implementations. + /// /// # Errors /// /// Returns `InvalidData` if the image is malformed, its seed hash does not match `seed`, or diff --git a/datasketches/src/cpc/wrapper.rs b/datasketches/src/cpc/wrapper.rs index b32df277..8263e0b8 100644 --- a/datasketches/src/cpc/wrapper.rs +++ b/datasketches/src/cpc/wrapper.rs @@ -47,6 +47,8 @@ impl CpcWrapper { /// Reads the cardinality metadata without copying or fully deserializing the compressed /// payload. /// + /// Also accepts data serialized by other DataSketches implementations. + /// /// # Errors /// /// Returns `InvalidData` if the preamble is malformed or does not describe a compressed CPC diff --git a/datasketches/src/frequencies/sketch.rs b/datasketches/src/frequencies/sketch.rs index 1eed1a7a..4808ef58 100644 --- a/datasketches/src/frequencies/sketch.rs +++ b/datasketches/src/frequencies/sketch.rs @@ -40,9 +40,8 @@ type DeserializeItems = fn(SketchSlice<'_>, usize) -> Result, Error>; const LG_MIN_MAP_SIZE: u8 = 3; const MIN_MAP_SIZE: usize = 1 << LG_MIN_MAP_SIZE; -// Java represents map sizes as positive `int` powers of two, while the C++ -// implementation uses 32-bit table indices. Keep Rust configurations within -// the same cross-language range. +// datasketches-java represents map sizes as positive `int` powers of two, while datasketches-cpp +// uses 32-bit table indices. Keep Rust configurations within the same cross-language range. const LG_MAX_MAP_SIZE: u8 = 30; const MAX_MAP_SIZE: usize = 1 << LG_MAX_MAP_SIZE; const SAMPLE_SIZE: usize = 1024; @@ -680,8 +679,8 @@ impl FrequentItemsSketch { return Err(Error::deserial("empty flag does not match preamble longs")); } if is_empty { - // Java also restores empty images at the minimum size. `lg_cur` does - // not carry item state here and must not control an eager allocation. + // datasketches-java also restores empty images at the minimum size. `lg_cur` carries + // no item state here and must not control an eager allocation. return Ok(Self::with_lg_map_sizes(lg_max, LG_MIN_MAP_SIZE)); } @@ -755,6 +754,9 @@ impl FrequentItemsSketch { impl FrequentItemsSketch { /// Serializes this sketch into a byte vector. /// + /// The serialized data can be read by other DataSketches implementations when both + /// sides use the same [`FrequentItemValue`] item encoding. + /// /// # Examples /// /// Built-in support for `i64`: @@ -787,6 +789,9 @@ impl FrequentItemsSketch { /// Deserializes a sketch from bytes. /// + /// Also accepts data serialized by other DataSketches implementations when both + /// sides use the same [`FrequentItemValue`] item encoding. + /// /// # Errors /// /// Returns `InvalidData` if the image is truncated, its metadata is inconsistent, or an item diff --git a/datasketches/src/hash/value/canonical_float.rs b/datasketches/src/hash/value/canonical_float.rs index 06edd985..8b31232e 100644 --- a/datasketches/src/hash/value/canonical_float.rs +++ b/datasketches/src/hash/value/canonical_float.rs @@ -21,7 +21,7 @@ //! hashing. Signed zero values hash the same, all NaN values use one canonical NaN bit pattern, //! and equal `f32`/`f64` values hash the same. //! -//! This strategy is compatible with how other datasketches hashes floating-point numbers. +//! Use this encoding to match floating-point hashing in DataSketches Java and DataSketches C++. use std::hash::Hash; use std::hash::Hasher; @@ -109,7 +109,7 @@ impl HashStrategy for CanonicalFloatStrategy { #[inline(always)] fn hash(value: &f64, state: &mut H) { let canonical = if value.is_nan() { - // Java's Double.doubleToLongBits() NaN value. + // Canonical NaN used by datasketches-java via Double.doubleToLongBits(). 0x7ff8000000000000u64 } else { // -0.0 + 0.0 == +0.0 under IEEE754 roundTiesToEven rounding mode, diff --git a/datasketches/src/hash/value/mod.rs b/datasketches/src/hash/value/mod.rs index 88d87e7e..a4867a3d 100644 --- a/datasketches/src/hash/value/mod.rs +++ b/datasketches/src/hash/value/mod.rs @@ -27,7 +27,7 @@ //! pattern before hashing. Signed zero values hash the same, all NaN values use one canonical NaN //! bit pattern, and equal `f32`/`f64` values hash the same. //! -//! This strategy is the same as how other datasketches implementations hash floating-point numbers. +//! Use this encoding to match floating-point hashing in DataSketches Java and DataSketches C++. //! //! The concrete wrapper documentation provides more details and examples. //! @@ -37,8 +37,8 @@ //! ## Integers //! //! [`sign_extend::SignExtend`] first sign-extends values to 64-bits, and then hashes the resulting -//! integers. This strategy is the same as how datasketches-cpp hashes short integers for -//! `HllSketch` and `CpcSketch`. +//! integers. Use this encoding to match short-integer hashing in DataSketches C++'s HLL and CPC +//! sketches. //! //! The concrete wrapper documentation provides more details and examples. //! @@ -47,8 +47,8 @@ //! * [`sign_extend::from_i32`], [`sign_extend::from_u32`] //! //! [`natural_extend::NaturalExtend`] first widens signed values to `i64` and unsigned values to -//! `u64`, and then hashes the resulting integers. This strategy is the same as how datasketches-cpp -//! hashes short integers for `BloomFilter`. +//! `u64`, and then hashes the resulting integers. Use this encoding to match short-integer hashing +//! in DataSketches C++'s Bloom filters. //! //! The concrete wrapper documentation provides more details and examples. //! @@ -61,9 +61,9 @@ //! [`raw_bytes::RawBytes`] hashes byte and string inputs as raw bytes without Rust's slice or //! string length prefix. //! -//! Empty byte and string inputs have zero bytes to hash. Other datasketches implementations skip -//! empty strings before hashing, so check `is_empty` before updating a sketch when that behavior -//! matters. +//! Use this encoding to match byte and UTF-8 string hashing in DataSketches Java and +//! DataSketches C++. Their string update APIs skip empty strings; check `is_empty` before updating +//! here as well. //! //! The concrete wrapper documentation provides more details and examples. //! diff --git a/datasketches/src/hash/value/natural_extend.rs b/datasketches/src/hash/value/natural_extend.rs index 981c8b1a..29d7fcff 100644 --- a/datasketches/src/hash/value/natural_extend.rs +++ b/datasketches/src/hash/value/natural_extend.rs @@ -20,7 +20,7 @@ //! [`NaturalExtend`] first widens signed values to `i64` and unsigned values to `u64`, and then //! hashes the resulting integers. //! -//! This strategy is the same as how datasketches-cpp hashes short integers for `BloomFilter`. +//! Use this encoding to match short-integer hashing in DataSketches C++'s Bloom filters. use std::hash::Hash; use std::hash::Hasher; diff --git a/datasketches/src/hash/value/raw_bytes.rs b/datasketches/src/hash/value/raw_bytes.rs index 0c48139f..a31c3978 100644 --- a/datasketches/src/hash/value/raw_bytes.rs +++ b/datasketches/src/hash/value/raw_bytes.rs @@ -20,9 +20,9 @@ //! [`RawBytes`] hashes byte and string inputs as raw bytes without Rust's slice or string length //! prefix. //! -//! Empty byte and string inputs have zero bytes to hash. Other datasketches implementations skip -//! empty strings before hashing, so check `is_empty` before updating a sketch when that behavior -//! matters. +//! Use this encoding to match byte and UTF-8 string hashing in DataSketches Java and +//! DataSketches C++. Their string update APIs skip empty strings; check `is_empty` before updating +//! here as well. use std::hash::Hasher; diff --git a/datasketches/src/hash/value/sign_extend.rs b/datasketches/src/hash/value/sign_extend.rs index cefcafea..eaaec9d5 100644 --- a/datasketches/src/hash/value/sign_extend.rs +++ b/datasketches/src/hash/value/sign_extend.rs @@ -19,8 +19,7 @@ //! //! [`SignExtend`] first sign-extends values to 64-bits and then hashes the resulting integers. //! -//! This strategy is the same as how datasketches-cpp hashes short integers for `HllSketch` and -//! `CpcSketch`. +//! Use this encoding to match short-integer hashing in DataSketches C++'s HLL and CPC sketches. use std::hash::Hash; use std::hash::Hasher; diff --git a/datasketches/src/hll/aux_map.rs b/datasketches/src/hll/aux_map.rs index 5286fe9c..e14ef31f 100644 --- a/datasketches/src/hll/aux_map.rs +++ b/datasketches/src/hll/aux_map.rs @@ -55,7 +55,7 @@ impl PartialEq for AuxMap { } } -/// Initial table sizes from DataSketches C++: +/// Initial table sizes from datasketches-cpp: /// fn lg_aux_arr_ints(lg_config_k: u8) -> u8 { static LG_AUX_ARR_INTS: &[u8] = &[ diff --git a/datasketches/src/hll/composite_interpolation.rs b/datasketches/src/hll/composite_interpolation.rs index cf419bfe..a028a80e 100644 --- a/datasketches/src/hll/composite_interpolation.rs +++ b/datasketches/src/hll/composite_interpolation.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -//! Bias-correction tables for `lg_k` 4 through 21 from DataSketches C++: +//! Bias-correction tables for `lg_k` 4 through 21 from datasketches-cpp: //! const NUM_X_VALUES: usize = 257; @@ -4769,7 +4769,7 @@ mod tests { let x_arr = get_x_arr(4); assert_eq!(x_arr.len(), 257); - // Check first few values match the C++ data + // Check first few values match the datasketches-cpp data. assert_that!(x_arr[0], near(10.767999803534, 1e-6)); assert_that!(x_arr[1], near(11.237701481774, 1e-6)); } diff --git a/datasketches/src/hll/coupon_mapping.rs b/datasketches/src/hll/coupon_mapping.rs index 0ee8d433..f702d8e7 100644 --- a/datasketches/src/hll/coupon_mapping.rs +++ b/datasketches/src/hll/coupon_mapping.rs @@ -16,7 +16,7 @@ // under the License. //! Coupon-count correction for the 26-bit slot encoding. These X/Y pairs are used with -//! cubic interpolation and come from DataSketches C++: +//! cubic interpolation and come from datasketches-cpp: //! /// X values (coupon counts) for interpolation table diff --git a/datasketches/src/hll/estimator.rs b/datasketches/src/hll/estimator.rs index 457dff8e..a72fcca0 100644 --- a/datasketches/src/hll/estimator.rs +++ b/datasketches/src/hll/estimator.rs @@ -20,7 +20,7 @@ //! Sequential register updates use HIP (Historical Inverse Probability). Bulk merges use the //! composite estimator because register values do not retain their update order. //! -//! Estimator formulas and crossover constants follow DataSketches C++: +//! Estimator formulas and crossover constants follow datasketches-cpp: //! use crate::common::NumStdDev; @@ -274,7 +274,7 @@ impl Estimator { } /// Relative error used as `estimate / (1 + error)`: negative for upper bounds. -/// Follows DataSketches C++ `HllUtil::getRelErr`: +/// Follows datasketches-cpp's `HllUtil::getRelErr`: /// fn relative_error( lg_config_k: u8, @@ -283,7 +283,7 @@ fn relative_error( num_std_dev: NumStdDev, ) -> f64 { if lg_config_k > 12 { - // RSE factors from Apache DataSketches C++ implementation + // RSE factors from datasketches-cpp. // HLL_HIP_RSE_FACTOR = sqrt(ln(2)) ≈ 0.8325546 // HLL_NON_HIP_RSE_FACTOR = sqrt((3 * ln(2)) - 1) ≈ 1.03896 let rse_factor = if composite { diff --git a/datasketches/src/hll/harmonic_numbers.rs b/datasketches/src/hll/harmonic_numbers.rs index cef22c78..8e6c9836 100644 --- a/datasketches/src/hll/harmonic_numbers.rs +++ b/datasketches/src/hll/harmonic_numbers.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -//! Harmonic-number approximation used by the bitmap estimator, following DataSketches C++: +//! Harmonic-number approximation used by the bitmap estimator, following datasketches-cpp: //! const NUM_EXACT: usize = 25; diff --git a/datasketches/src/hll/mod.rs b/datasketches/src/hll/mod.rs index fdec9773..06db8d35 100644 --- a/datasketches/src/hll/mod.rs +++ b/datasketches/src/hll/mod.rs @@ -20,7 +20,7 @@ //! Use [`HllSketch`] to collect values and [`HllUnion`] to combine sketches. //! [`HllType`] controls register storage; `lg_k` controls the register count and accuracy. //! -//! Serialized sketches are compatible with Apache DataSketches in Java and C++. +//! The binary serialization format supports cross-language interchange. //! Cross-language updates must also use compatible hashing; see //! [`hash::value`](crate::hash::value). //! @@ -149,8 +149,8 @@ impl Coupon { /// Computes the HLL coupon for a hashable value. /// - /// Use [`hash::value`](crate::hash::value) wrappers when another DataSketches - /// implementation requires a specific value hashing strategy. + /// Use [`hash::value`](crate::hash::value) wrappers to select an input encoding for + /// cross-language hashing. /// /// # Examples /// diff --git a/datasketches/src/hll/sketch.rs b/datasketches/src/hll/sketch.rs index 06dd823a..92d4d5b3 100644 --- a/datasketches/src/hll/sketch.rs +++ b/datasketches/src/hll/sketch.rs @@ -137,8 +137,8 @@ impl HllSketch { /// Updates the sketch with a value. /// - /// You may use [`hash::value`](crate::hash::value) wrappers when another DataSketches - /// implementation requires a specific value hashing strategy. + /// Use [`hash::value`](crate::hash::value) wrappers to select an input encoding for + /// cross-language hashing. /// /// To reuse the hash across sketches, see [`Coupon`] and /// [`update_with_coupon`](Self::update_with_coupon). @@ -314,6 +314,8 @@ impl HllSketch { /// Deserializes an HLL sketch from bytes. /// + /// Also accepts data serialized by other DataSketches implementations. + /// /// # Errors /// /// Returns `InvalidData` if the image is truncated or contains an invalid preamble, @@ -443,6 +445,8 @@ impl HllSketch { /// Serializes the HLL sketch to bytes. /// + /// The serialized data can be read by other DataSketches implementations. + /// /// # Examples /// /// ``` diff --git a/datasketches/src/kll/mod.rs b/datasketches/src/kll/mod.rs index 59e0fd62..e6cf6cfd 100644 --- a/datasketches/src/kll/mod.rs +++ b/datasketches/src/kll/mod.rs @@ -21,8 +21,7 @@ //! near-optimal accuracy per retained item. It supports one-pass updates, //! approximate quantiles, ranks, PMF, and CDF queries. //! -//! This implementation follows Apache DataSketches semantics and uses the compact binary -//! serialization format shared by the Java, C++, and Go implementations. +//! The compact binary serialization format supports cross-language interchange. //! //! Items must implement [`Ord`]. Wrap `f32` or `f64` values in [`KllFloat`], which rejects NaN and //! provides their ordinary numerical order. Custom ordering should be expressed with a newtype diff --git a/datasketches/src/kll/serialization.rs b/datasketches/src/kll/serialization.rs index 41e585c3..e9450df4 100644 --- a/datasketches/src/kll/serialization.rs +++ b/datasketches/src/kll/serialization.rs @@ -17,9 +17,9 @@ //! Binary serialization format constants for KLL sketches. //! -//! Naming and layout follow the Apache DataSketches Java implementation -//! (`KllPreambleUtil`) and the C++ `kll_sketch` compact serialization format. -//! Java's updatable-memory layout is not a cross-language wire format and is +//! Naming and layout follow datasketches-java's `KllPreambleUtil` and datasketches-cpp's +//! `kll_sketch` compact serialization format. +//! datasketches-java's updatable-memory layout is not a cross-language wire format and is //! intentionally outside this module's scope. /// Serialization version for empty or full sketches (KllPreambleUtil.SERIAL_VERSION_EMPTY_FULL). diff --git a/datasketches/src/kll/sketch.rs b/datasketches/src/kll/sketch.rs index 891c4bbf..f90a07d1 100644 --- a/datasketches/src/kll/sketch.rs +++ b/datasketches/src/kll/sketch.rs @@ -636,12 +636,18 @@ fn deserialize_with_serde(bytes: &[u8]) -> Result KllSketch { - /// Serializes the sketch to bytes. + /// Serializes the sketch to bytes in compact form. + /// + /// The serialized data can be read by other DataSketches implementations when both + /// sides use the same [`KllValue`] item encoding. pub fn serialize(&self) -> Vec { serialize_with_serde(self) } - /// Deserializes a sketch from bytes. + /// Deserializes a sketch from bytes in compact form. + /// + /// Also accepts data serialized by other DataSketches implementations when both + /// sides use the same [`KllValue`] item encoding. /// /// # Errors /// diff --git a/datasketches/src/lib.rs b/datasketches/src/lib.rs index b4ff0164..813a841b 100644 --- a/datasketches/src/lib.rs +++ b/datasketches/src/lib.rs @@ -18,8 +18,6 @@ //! # Apache® DataSketches™ Core Rust Library Component //! //! This crate provides compact, mergeable summaries for answering queries over large data streams. -//! It implements a subset of the algorithms available in the other Apache DataSketches language -//! components. //! //! ## Enabling sketches //! @@ -49,7 +47,7 @@ //! ## Cross-language hashing //! //! Compatible serialization does not by itself make ordinary Rust [`Hash`](std::hash::Hash) -//! input compatible with Java, C++, or Go. Rust strings and slices include type-specific framing, +//! input compatible across languages. Rust strings and slices include type-specific framing, //! and short integers require different widening rules for different sketch families. When //! sketches must represent the same updates across languages, use the wrappers in [`hash::value`]: //! @@ -58,8 +56,8 @@ //! * `sign_extend` for short integers used with HLL and CPC; //! * `natural_extend` for short integers used with Bloom filters. //! -//! Other DataSketches implementations skip empty strings rather than hashing them. Check for empty -//! input before updating when that cross-language behavior is required. +//! When interoperating with an implementation that skips empty strings, check for empty input +//! before updating here as well. #![cfg_attr(docsrs, feature(doc_cfg))] #![deny(missing_docs)] diff --git a/datasketches/src/req/compactor.rs b/datasketches/src/req/compactor.rs index 1a4aa640..c6f64615 100644 --- a/datasketches/src/req/compactor.rs +++ b/datasketches/src/req/compactor.rs @@ -114,7 +114,7 @@ where } } // OR-ing the schedule counters can advance state past several doubling - // thresholds at once. Loop until no more doublings are needed (C++: + // thresholds at once. Loop until no more doublings are needed (datasketches-cpp: // req_compactor_impl.hpp:250 — `while (ensure_enough_sections()) {}`). while self.ensure_enough_sections() {} } @@ -364,8 +364,8 @@ where /// /// The wire format omits the compactor preamble for tiny sketches (n ≤ 4); this /// helper synthesises a fresh compactor and seeds it with the deserialized items. - /// A false wire flag stays false for byte-stable C++/Java round trips; a true flag - /// is cleared if the items are not actually sorted. + /// A false wire flag stays false for byte-stable datasketches-cpp and datasketches-java round + /// trips; a true flag is cleared if the items are not actually sorted. pub fn raw_items_compactor( k: u16, rank_accuracy: RankAccuracy, @@ -513,7 +513,7 @@ mod tests { // ensure_enough_sections() once instead of looping. Without the loop, // num_sections doubles at most once per merge — but OR-ing a high state // can advance past several doubling thresholds at once and require - // multiple doublings (matching the C++ reference at + // multiple doublings (matching datasketches-cpp at // req_compactor_impl.hpp:250 — `while (ensure_enough_sections()) {}`). // // Setup: a fresh compactor (state=0, num_sections=3) merged with another diff --git a/datasketches/src/req/mod.rs b/datasketches/src/req/mod.rs index 40b787c0..91bc2ba9 100644 --- a/datasketches/src/req/mod.rs +++ b/datasketches/src/req/mod.rs @@ -19,8 +19,7 @@ //! //! [`ReqSketch`] provides bounded-memory rank, quantile, PMF, and CDF estimates with //! configurable relative accuracy at either end of the rank domain. It is based on -//! [Relative Error Streaming Quantiles](https://arxiv.org/abs/2004.01668) and the Apache -//! DataSketches C++ implementation. +//! [Relative Error Streaming Quantiles](https://arxiv.org/abs/2004.01668). //! //! # Item ordering //! diff --git a/datasketches/src/req/serialization.rs b/datasketches/src/req/serialization.rs index 90c949a4..c4d1f235 100644 --- a/datasketches/src/req/serialization.rs +++ b/datasketches/src/req/serialization.rs @@ -29,8 +29,8 @@ pub const PREAMBLE_INTS_EXACT: u8 = 2; pub const PREAMBLE_INTS_ESTIMATION: u8 = 4; pub const RAW_ITEMS_THRESHOLD: u64 = 4; -/// Flag bits — match the C++ enum order: RESERVED1, RESERVED2, IS_EMPTY, IS_HIGH_RANK, RAW_ITEMS, -/// IS_LEVEL_ZERO_SORTED. +// Match the datasketches-cpp enum order: RESERVED1, RESERVED2, IS_EMPTY, IS_HIGH_RANK, RAW_ITEMS, +// IS_LEVEL_ZERO_SORTED. pub const FLAG_IS_EMPTY: u8 = 1 << 2; pub const FLAG_IS_HIGH_RANK: u8 = 1 << 3; pub const FLAG_RAW_ITEMS: u8 = 1 << 4; diff --git a/datasketches/src/req/sketch.rs b/datasketches/src/req/sketch.rs index 446906f1..fb0245ef 100644 --- a/datasketches/src/req/sketch.rs +++ b/datasketches/src/req/sketch.rs @@ -535,7 +535,10 @@ where size } - /// Serializes the sketch using the C++/Java-compatible REQ wire format. + /// Serializes the sketch using the REQ wire format. + /// + /// The serialized data can be read by other DataSketches implementations when both + /// sides use the same [`ReqValue`] item encoding. pub fn serialize(&self) -> Vec where T: ReqValue, @@ -583,8 +586,10 @@ where out.into_bytes() } - /// Deserialize a sketch from bytes produced by [`Self::serialize`] or by the - /// C++/Java reference implementations. + /// Deserializes a sketch from bytes in the REQ wire format. + /// + /// Also accepts data serialized by other DataSketches implementations when both + /// sides use the same [`ReqValue`] item encoding. /// /// # Errors /// @@ -803,9 +808,8 @@ where min_item: None, max_item: None, }; - // C++ parity: an empty sketch has a level-0 compactor present from the start. - // This makes is_raw_items() and flags_byte() byte-compatible with the C++/Java - // wire format for the empty case. + // Keep a level-0 compactor, as in datasketches-cpp, so is_raw_items() and flags_byte() + // produce the same empty encoding as datasketches-cpp and datasketches-java. sketch.grow(); sketch } diff --git a/datasketches/src/tdigest/sketch.rs b/datasketches/src/tdigest/sketch.rs index d7599af3..1fb4b16f 100644 --- a/datasketches/src/tdigest/sketch.rs +++ b/datasketches/src/tdigest/sketch.rs @@ -554,7 +554,9 @@ impl TDigestMut { Ok(self.view().quantiles(ranks)) } - /// Serializes this mutable t-digest to bytes. + /// Serializes this mutable t-digest to bytes in double precision. + /// + /// The serialized data can be read by other DataSketches implementations. /// /// # Examples /// @@ -581,6 +583,8 @@ impl TDigestMut { /// Deserializes a mutable t-digest from the standard double-precision format. /// + /// Also accepts data serialized by other DataSketches implementations. + /// /// The format of the [reference implementation](https://github.com/tdunning/t-digest) is /// auto-detected. Use [`deserialize_f32()`](Self::deserialize_f32) for the compact /// DataSketches C++ `tdigest` format. @@ -608,8 +612,8 @@ impl TDigestMut { /// Deserializes a mutable t-digest from the compact single-precision DataSketches format. /// - /// This format stores centroid means and weights as `(f32, u32)` and is emitted by the C++ - /// `tdigest` implementation. Its header does not identify the scalar width, so callers + /// Accepts data serialized by `tdigest` in DataSketches C++. This format stores centroid + /// means and weights as `(f32, u32)`. Its header does not identify the scalar width, so callers /// must select this entry point explicitly. /// /// # Errors @@ -1368,7 +1372,9 @@ impl TDigest { self.centroids_weight } - /// Serializes this immutable t-digest to bytes. + /// Serializes this immutable t-digest to bytes in double precision. + /// + /// The serialized data can be read by other DataSketches implementations. /// /// # Examples /// @@ -1396,6 +1402,8 @@ impl TDigest { /// Deserializes an immutable t-digest from the standard double-precision format. /// + /// Also accepts data serialized by other DataSketches implementations. + /// /// The format of the [reference implementation](https://github.com/tdunning/t-digest) is /// auto-detected. Use [`deserialize_f32()`](Self::deserialize_f32) for the compact /// DataSketches C++ `tdigest` format. @@ -1409,6 +1417,9 @@ impl TDigest { /// Deserializes an immutable t-digest from the compact single-precision DataSketches format. /// + /// Accepts data serialized by `tdigest` in DataSketches C++; see + /// [`TDigestMut::deserialize_f32`] for the encoding details. + /// /// # Errors /// /// Returns `InvalidData` under the same conditions as [`TDigestMut::deserialize_f32`]. diff --git a/datasketches/src/thetafamily/common/constants.rs b/datasketches/src/thetafamily/common/constants.rs index 8f0ee6e6..8b86f78c 100644 --- a/datasketches/src/thetafamily/common/constants.rs +++ b/datasketches/src/thetafamily/common/constants.rs @@ -17,7 +17,7 @@ /// Maximum theta value. /// -/// The value is `i64::MAX` to be compatible with datasketches-java. +/// The shared format uses a 63-bit hash domain, bounded by `i64::MAX`, as in datasketches-java. pub const MAX_THETA: u64 = i64::MAX as u64; /// Minimum log2 of K. diff --git a/datasketches/src/thetafamily/common/hash_table.rs b/datasketches/src/thetafamily/common/hash_table.rs index 5525f748..d2138c93 100644 --- a/datasketches/src/thetafamily/common/hash_table.rs +++ b/datasketches/src/thetafamily/common/hash_table.rs @@ -173,7 +173,7 @@ where let mut hasher = MurmurHash3X64128::with_seed(self.seed); value.hash(&mut hasher); let (h1, _) = hasher.finish128(); - h1 >> 1 // To make it compatible with Java version + h1 >> 1 // Match the 63-bit hash domain used by datasketches-java's Theta and Tuple sketches. } /// Inserts or updates the entry slot for a pre-hashed key. diff --git a/datasketches/src/thetafamily/theta/sketch.rs b/datasketches/src/thetafamily/theta/sketch.rs index 2c873832..b75d1aff 100644 --- a/datasketches/src/thetafamily/theta/sketch.rs +++ b/datasketches/src/thetafamily/theta/sketch.rs @@ -218,8 +218,8 @@ impl ThetaSketch { /// Updates the sketch with a hashable value. /// - /// You may use [`hash::value`](crate::hash::value) wrappers when another DataSketches - /// implementation requires a specific value hashing strategy. + /// Use [`hash::value`](crate::hash::value) wrappers to select an input encoding for + /// cross-language hashing. /// /// # Examples /// @@ -268,7 +268,7 @@ impl ThetaSketch { /// Returns theta as a `u64`. /// /// An empty sketch reports `MAX_THETA` even when it was built with a sampling probability - /// below `1.0`, matching the other DataSketches implementations. + /// below `1.0`. pub fn theta64(&self) -> u64 { if self.is_empty { MAX_THETA @@ -546,6 +546,8 @@ impl CompactThetaSketch { /// Serializes this sketch in compressed form if applicable. /// + /// The serialized data can be read by other DataSketches implementations. + /// /// Falls back to [`serialize`](Self::serialize) when compression is not applicable. pub fn serialize_compressed(&self) -> Vec { if self.is_suitable_for_compression() { @@ -563,6 +565,8 @@ impl CompactThetaSketch { /// Serializes this sketch into the uncompressed compact Theta format. /// + /// The serialized data can be read by other DataSketches implementations. + /// /// Empty sketches serialize with a zero seed hash. pub fn serialize(&self) -> Vec { if self.is_empty() { @@ -590,7 +594,8 @@ impl CompactThetaSketch { if pre_longs > 1 { bytes.write_u32_le(retained_hashes.len() as u32); - bytes.write_u32_be(0); // not used by compact sketches; match Java/C++ + // Unused by compact sketches; matches datasketches-java and datasketches-cpp. + bytes.write_u32_be(0); } if self.is_estimation_mode() { bytes.write_u64_le(self.theta64()); @@ -689,6 +694,8 @@ impl CompactThetaSketch { /// Deserializes a compact Theta sketch using the default seed. /// + /// Also accepts data serialized by other DataSketches implementations. + /// /// # Errors /// /// Returns `InvalidData` if the image is malformed or a non-empty image's seed hash does not @@ -699,6 +706,8 @@ impl CompactThetaSketch { /// Deserializes a compact Theta sketch using `seed`. /// + /// Also accepts data serialized by other DataSketches implementations. + /// /// Empty sketches use the hash of `seed` regardless of the stored seed hash. /// /// # Errors diff --git a/datasketches/src/thetafamily/tuple/serialization.rs b/datasketches/src/thetafamily/tuple/serialization.rs index d44a1049..cc208e0a 100644 --- a/datasketches/src/thetafamily/tuple/serialization.rs +++ b/datasketches/src/thetafamily/tuple/serialization.rs @@ -19,7 +19,7 @@ //! //! The Tuple compact format reuses the uncompressed Theta layout (preamble, flags, theta) but uses //! the Tuple family id, carries a sketch-type byte, and stores the user summary bytes after each -//! retained hash. See the C++/Java reference implementations for the on-disk format. +//! retained hash. See datasketches-cpp and datasketches-java for the on-disk format. //! //! A Tuple sketch keeps a user-defined summary next to every retained key. Because the summary //! type is opaque to the sketch, (de)serialization of summaries is delegated to the summary type diff --git a/datasketches/src/thetafamily/tuple/sketch.rs b/datasketches/src/thetafamily/tuple/sketch.rs index 509ef16e..e3a690fe 100644 --- a/datasketches/src/thetafamily/tuple/sketch.rs +++ b/datasketches/src/thetafamily/tuple/sketch.rs @@ -331,7 +331,7 @@ where /// Returns theta as a `u64`. /// /// An empty sketch reports `MAX_THETA` even when it was built with a sampling probability - /// below `1.0`, matching the other DataSketches implementations. + /// below `1.0`. pub fn theta64(&self) -> u64 { if self.is_empty { MAX_THETA @@ -559,8 +559,8 @@ impl CompactTupleSketch { /// Serializes this sketch into the compact Tuple binary format. /// - /// Uses [`TupleSummaryValue`] to encode summaries. Reading the output in Java or C++ requires - /// a compatible summary encoding. + /// The serialized data can be read by other DataSketches implementations when both + /// sides use the same [`TupleSummaryValue`] summary encoding. /// /// Empty sketches serialize with a zero seed hash. /// @@ -622,6 +622,9 @@ impl CompactTupleSketch { /// Deserializes a compact Tuple sketch using the default seed. /// + /// Also accepts data serialized by other DataSketches implementations when both + /// sides use the same [`TupleSummaryValue`] summary encoding. + /// /// # Errors /// /// Returns `InvalidData` if the image is malformed or a non-empty image's seed hash does not @@ -636,6 +639,9 @@ impl CompactTupleSketch { /// Deserializes a compact Tuple sketch using `seed`. /// + /// Also accepts data serialized by other DataSketches implementations when both + /// sides use the same [`TupleSummaryValue`] summary encoding. + /// /// Empty sketches use the hash of `seed` regardless of the stored seed hash. /// /// # Errors diff --git a/tests-integration/tests/serde_tests/bloom.rs b/tests-integration/tests/serde_tests/bloom.rs index 4c3e7127..14166453 100644 --- a/tests-integration/tests/serde_tests/bloom.rs +++ b/tests-integration/tests/serde_tests/bloom.rs @@ -55,7 +55,7 @@ fn test_bloom_filter_file(path: PathBuf, expected_num_items: u64, expected_num_h } // Verify the items that were inserted (integers 0 to n/10-1) - // C++ code: for (uint64_t i = 0; i < n / 10; ++i) bf.update(i); + // datasketches-cpp code: for (uint64_t i = 0; i < n / 10; ++i) bf.update(i); let num_inserted = expected_num_items / 10; if num_inserted > 0 { diff --git a/tests-integration/tests/serde_tests/countmin.rs b/tests-integration/tests/serde_tests/countmin.rs index 07786b42..4a62faaf 100644 --- a/tests-integration/tests/serde_tests/countmin.rs +++ b/tests-integration/tests/serde_tests/countmin.rs @@ -44,8 +44,8 @@ fn rejects_weights_that_cannot_bound_counter_arithmetic() { } // This test validates binary format compatibility (deserialize + byte round-trip) for -// C++ CountMin snapshots. It intentionally does not assert estimate equivalence against -// original input keys because per-row hash seed derivation differs across implementations. +// datasketches-cpp CountMin snapshots. It intentionally does not assert estimate equivalence +// against original input keys because per-row hash seed derivation differs across implementations. fn assert_cpp_snapshot( filename: &str, seed: u64, diff --git a/tests-integration/tests/serde_tests/frequencies.rs b/tests-integration/tests/serde_tests/frequencies.rs index 31c25cb1..9c1df36b 100644 --- a/tests-integration/tests/serde_tests/frequencies.rs +++ b/tests-integration/tests/serde_tests/frequencies.rs @@ -109,7 +109,7 @@ fn test_non_clone_item_round_trip() { fn test_empty_round_trip() { let sketch = FrequentItemsSketch::::new(32).unwrap(); let bytes = sketch.serialize(); - // One preamble long, matching the Java and C++ empty encoding. + // One preamble long, matching the datasketches-java and datasketches-cpp empty encoding. assert_eq!(bytes.len(), 8); let restored = FrequentItemsSketch::::deserialize(&bytes).unwrap(); assert!(restored.is_empty()); diff --git a/tests-integration/tests/serde_tests/kll.rs b/tests-integration/tests/serde_tests/kll.rs index 44ba22b1..c6ab416d 100644 --- a/tests-integration/tests/serde_tests/kll.rs +++ b/tests-integration/tests/serde_tests/kll.rs @@ -15,16 +15,6 @@ // specific language governing permissions and limitations // under the License. -//! KLL Sketch Serialization Compatibility Tests -//! -//! These tests verify binary compatibility with Apache DataSketches implementations: -//! - Java (datasketches-java) -//! - C++ (datasketches-cpp) -//! - Go (datasketches-go) -//! -//! Test data is generated by the reference implementations and stored in: -//! `tests/serde_tests/{java_generated_files,cpp_generated_files}/`. - use std::cmp::Ordering; use std::fs; use std::path::PathBuf; diff --git a/tests-integration/tests/serde_tests/req.rs b/tests-integration/tests/serde_tests/req.rs index a3a920f6..7c14763a 100644 --- a/tests-integration/tests/serde_tests/req.rs +++ b/tests-integration/tests/serde_tests/req.rs @@ -15,8 +15,6 @@ // specific language governing permissions and limitations // under the License. -//! Serialization round-trip and cross-language compatibility tests for ReqSketch. - use std::fs; use std::path::PathBuf; diff --git a/tests-integration/tests/serde_tests/tdigest.rs b/tests-integration/tests/serde_tests/tdigest.rs index 8bcd9890..a426e782 100644 --- a/tests-integration/tests/serde_tests/tdigest.rs +++ b/tests-integration/tests/serde_tests/tdigest.rs @@ -311,8 +311,9 @@ fn test_updates_normalize_overfull_deserialized_buffer_without_centroids() { assert_eq!(&bytes[8..12], &0_u32.to_le_bytes()); // num centroids assert_eq!(&bytes[12..16], &10_u32.to_le_bytes()); // num buffered - // k=100 normally compresses at 840 buffered values. Extend a real C++ image without centroids - // just past that producer threshold while keeping every value within the recorded min/max. + // k=100 normally compresses at 840 buffered values. Extend a datasketches-cpp image without + // centroids just past that producer threshold while keeping every value within the recorded + // min/max. bytes[12..16].copy_from_slice(&841_u32.to_le_bytes()); for _ in 0..831 { bytes.extend_from_slice(&10_f64.to_le_bytes()); @@ -345,7 +346,8 @@ fn test_updates_normalize_overfull_deserialized_mixed_buffer() { assert_eq!(&bytes[8..12], &89_u32.to_le_bytes()); // num centroids assert_eq!(&bytes[12..16], &160_u32.to_le_bytes()); // num buffered - // Extend the buffered tail of a real mixed C++ image just past the k=100 producer threshold. + // Extend the buffered tail of a mixed datasketches-cpp image just past the k=100 producer + // threshold. bytes[12..16].copy_from_slice(&841_u32.to_le_bytes()); for _ in 0..681 { bytes.extend_from_slice(&1_000_f64.to_le_bytes()); diff --git a/tests-integration/tests/serde_tests/theta.rs b/tests-integration/tests/serde_tests/theta.rs index b51fc2d5..4f6dd0f9 100644 --- a/tests-integration/tests/serde_tests/theta.rs +++ b/tests-integration/tests/serde_tests/theta.rs @@ -299,12 +299,12 @@ fn test_v2_java_empty_layouts() { .seed_hash(); let other_seed_hash = ThetaSketchBuilder::default().build().unwrap().seed_hash(); - // Port of Java 8.0.0 ForwardCompatibilityTest's checkSerVer2_{1,2,3}PreLong[s]_Empty: - // https://github.com/apache/datasketches-java/blob/8.0.0/src/test/java/org/apache/datasketches/theta/ForwardCompatibilityTest.java#L90-L137 + // Port of datasketches-java 8.0.0 ForwardCompatibilityTest's + // checkSerVer2_{1,2,3}PreLong[s]_Empty: https://github.com/apache/datasketches-java/blob/8.0.0/src/test/java/org/apache/datasketches/theta/ForwardCompatibilityTest.java#L90-L137 for pre_longs in [1, 2, 3] { let mut bytes = serialize_v2_exact(&[]); bytes[0] = pre_longs; - bytes[5] = 0x0e; // Java's v2 empty flags. + bytes[5] = 0x0e; // datasketches-java's v2 empty flags. bytes.resize(pre_longs as usize * 8, 0); if pre_longs == 3 { bytes[16..24].copy_from_slice(&(i64::MAX as u64).to_le_bytes());