From 0ebe894806c9dc90e7ce85ec188be41089e90217 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Thu, 22 Oct 2026 15:16:58 -0700 Subject: [PATCH 1/3] Theta: clamp the union size in JaccardSimilarity.exactlyEqual exactlyEqual() sized its union with ceilingPowerOf2(countA + countB) and no bounds, so ThetaSetOperationBuilder.setNominalEntries() threw SketchesArgumentException whenever the two retained counts summed to 8 or less, for example two sketches holding one item each, or to more than 2^26. Clamp the union size to [2^4, 2^26], as jaccard() has done since #153 and as the C++ compute_union() does for both jaccard() and exactly_equal(). Equal sketches that retain more than 2^26 entries, which needs 2^26 nominal entries, now compare as not equal instead of throwing; the javadoc notes this. Co-Authored-By: Claude Opus 5.5 --- .../apache/datasketches/theta/JaccardSimilarity.java | 9 ++++++++- .../datasketches/theta/JaccardSimilarityTest.java | 11 +++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/apache/datasketches/theta/JaccardSimilarity.java b/src/main/java/org/apache/datasketches/theta/JaccardSimilarity.java index 2fc9d4fe5..7f967198d 100644 --- a/src/main/java/org/apache/datasketches/theta/JaccardSimilarity.java +++ b/src/main/java/org/apache/datasketches/theta/JaccardSimilarity.java @@ -100,6 +100,10 @@ public static double[] jaccard(final ThetaSketch sketchA, final ThetaSketch sket /** * Returns true if the two given sketches have exactly the same hash values and the same * theta values. Thus, they are equivalent. + * + *

Note: For very large pairs of sketches, where the configured nominal entries of the sketches + * are 2^26, this method may return false even if the two sketches are equal. + * * @param sketchA the given sketch A * @param sketchB the given sketch B * @return true if the two given sketches have exactly the same hash values and the same @@ -116,8 +120,11 @@ public static boolean exactlyEqual(final ThetaSketch sketchA, final ThetaSketch final int countB = sketchB.getRetainedEntries(true); //Create the ThetaUnion + final int minK = 1 << ThetaUtil.MIN_LG_NOM_LONGS; + final int maxK = 1 << ThetaUtil.MAX_LG_NOM_LONGS; + final int newK = max(min(ceilingPowerOf2(countA + countB), maxK), minK); final ThetaUnion union = - ThetaSetOperation.builder().setNominalEntries(ceilingPowerOf2(countA + countB)).buildUnion(); + ThetaSetOperation.builder().setNominalEntries(newK).buildUnion(); union.union(sketchA); union.union(sketchB); final ThetaSketch unionAB = union.getResult(); diff --git a/src/test/java/org/apache/datasketches/theta/JaccardSimilarityTest.java b/src/test/java/org/apache/datasketches/theta/JaccardSimilarityTest.java index 4f84c5382..17cadcc92 100644 --- a/src/test/java/org/apache/datasketches/theta/JaccardSimilarityTest.java +++ b/src/test/java/org/apache/datasketches/theta/JaccardSimilarityTest.java @@ -233,6 +233,17 @@ public void checkMinK() { println(result[0] + ", " + result[1] + ", " + result[2]); } + @Test + public void checkMinKExactlyEqual() { + UpdatableThetaSketch skA = UpdatableThetaSketch.builder().build(); //4096 + UpdatableThetaSketch skB = UpdatableThetaSketch.builder().build(); //4096 + skA.update(1); + skB.update(1); + assertTrue(exactlyEqual(skA, skB)); + skB.update(2); + assertFalse(exactlyEqual(skA, skB)); + } + @Test public void printlnTest() { println("PRINTING: "+this.getClass().getName()); From ce2393c1defd11f4c39072b53aebffe6a2939e66 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 26 Oct 2026 16:40:22 -0700 Subject: [PATCH 2/3] Theta: don't order the union result in JaccardSimilarity.exactlyEqual exactlyEqual() only reads the retained count and theta of the union, so sorting the result with getResult() is wasted work. Use getResult(false, null), as jaccard() does and as the C++ compute_union() does with get_result(false). Co-Authored-By: Claude Opus 5.5 --- .../java/org/apache/datasketches/theta/JaccardSimilarity.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/apache/datasketches/theta/JaccardSimilarity.java b/src/main/java/org/apache/datasketches/theta/JaccardSimilarity.java index 7f967198d..dc1779511 100644 --- a/src/main/java/org/apache/datasketches/theta/JaccardSimilarity.java +++ b/src/main/java/org/apache/datasketches/theta/JaccardSimilarity.java @@ -127,7 +127,7 @@ public static boolean exactlyEqual(final ThetaSketch sketchA, final ThetaSketch ThetaSetOperation.builder().setNominalEntries(newK).buildUnion(); union.union(sketchA); union.union(sketchB); - final ThetaSketch unionAB = union.getResult(); + final ThetaSketch unionAB = union.getResult(false, null); final long thetaLongUAB = unionAB.getThetaLong(); final long thetaLongA = sketchA.getThetaLong(); final long thetaLongB = sketchB.getThetaLong(); From fca13888c66d6a03e891e9327ff0401e2e875031 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 26 Oct 2026 16:40:22 -0700 Subject: [PATCH 3/3] Theta: share the union and identical-sets code in JaccardSimilarity jaccard() and exactlyEqual() each sized and built the same union and ran the same identical-sets check. That duplication is how the #153 clamp reached jaccard() but not exactlyEqual(). Move both into private helpers, computeUnion() and identicalSets(), following the C++ jaccard_similarity_base, and call them from both methods. The behaviour of jaccard() is unchanged. Co-Authored-By: Claude Opus 5.5 --- .../datasketches/theta/JaccardSimilarity.java | 88 +++++++++---------- 1 file changed, 43 insertions(+), 45 deletions(-) diff --git a/src/main/java/org/apache/datasketches/theta/JaccardSimilarity.java b/src/main/java/org/apache/datasketches/theta/JaccardSimilarity.java index dc1779511..f35b246fb 100644 --- a/src/main/java/org/apache/datasketches/theta/JaccardSimilarity.java +++ b/src/main/java/org/apache/datasketches/theta/JaccardSimilarity.java @@ -61,28 +61,8 @@ public static double[] jaccard(final ThetaSketch sketchA, final ThetaSketch sket if (sketchA.isEmpty() && sketchB.isEmpty()) { return ONES.clone(); } if (sketchA.isEmpty() || sketchB.isEmpty()) { return ZEROS.clone(); } - final int countA = sketchA.getRetainedEntries(true); - final int countB = sketchB.getRetainedEntries(true); - - //Create the ThetaUnion - final int minK = 1 << ThetaUtil.MIN_LG_NOM_LONGS; - final int maxK = 1 << ThetaUtil.MAX_LG_NOM_LONGS; - final int newK = max(min(ceilingPowerOf2(countA + countB), maxK), minK); - final ThetaUnion union = - ThetaSetOperation.builder().setNominalEntries(newK).buildUnion(); - union.union(sketchA); - union.union(sketchB); - final ThetaSketch unionAB = union.getResult(false, null); - final long thetaLongUAB = unionAB.getThetaLong(); - final long thetaLongA = sketchA.getThetaLong(); - final long thetaLongB = sketchB.getThetaLong(); - final int countUAB = unionAB.getRetainedEntries(true); - - //Check for identical data - if (countUAB == countA && countUAB == countB - && thetaLongUAB == thetaLongA && thetaLongUAB == thetaLongB) { - return ONES.clone(); - } + final ThetaSketch unionAB = computeUnion(sketchA, sketchB); + if (identicalSets(sketchA, sketchB, unionAB)) { return ONES.clone(); } //Create the ThetaIntersection final ThetaIntersection inter = ThetaSetOperation.builder().buildIntersection(); @@ -116,29 +96,8 @@ public static boolean exactlyEqual(final ThetaSketch sketchA, final ThetaSketch if (sketchA.isEmpty() && sketchB.isEmpty()) { return true; } if (sketchA.isEmpty() || sketchB.isEmpty()) { return false; } - final int countA = sketchA.getRetainedEntries(true); - final int countB = sketchB.getRetainedEntries(true); - - //Create the ThetaUnion - final int minK = 1 << ThetaUtil.MIN_LG_NOM_LONGS; - final int maxK = 1 << ThetaUtil.MAX_LG_NOM_LONGS; - final int newK = max(min(ceilingPowerOf2(countA + countB), maxK), minK); - final ThetaUnion union = - ThetaSetOperation.builder().setNominalEntries(newK).buildUnion(); - union.union(sketchA); - union.union(sketchB); - final ThetaSketch unionAB = union.getResult(false, null); - final long thetaLongUAB = unionAB.getThetaLong(); - final long thetaLongA = sketchA.getThetaLong(); - final long thetaLongB = sketchB.getThetaLong(); - final int countUAB = unionAB.getRetainedEntries(true); - - //Check for identical counts and thetas - if (countUAB == countA && countUAB == countB - && thetaLongUAB == thetaLongA && thetaLongUAB == thetaLongB) { - return true; - } - return false; + final ThetaSketch unionAB = computeUnion(sketchA, sketchB); + return identicalSets(sketchA, sketchB, unionAB); } /** @@ -185,4 +144,43 @@ public static boolean dissimilarityTest(final ThetaSketch measured, final ThetaS return jRatioUB <= threshold; } + /** + * Computes the union of the two given sketches. The union is sized to hold the retained entries + * of both sketches, clamped to the legal range of nominal entries. The result is unordered, + * because only its retained entries and theta are used. + * @param sketchA the given sketch A + * @param sketchB the given sketch B + * @return the unordered union of sketch A and sketch B + */ + private static ThetaSketch computeUnion(final ThetaSketch sketchA, final ThetaSketch sketchB) { + final int countA = sketchA.getRetainedEntries(true); + final int countB = sketchB.getRetainedEntries(true); + final int minK = 1 << ThetaUtil.MIN_LG_NOM_LONGS; + final int maxK = 1 << ThetaUtil.MAX_LG_NOM_LONGS; + final int newK = max(min(ceilingPowerOf2(countA + countB), maxK), minK); + final ThetaUnion union = + ThetaSetOperation.builder().setNominalEntries(newK).buildUnion(); + union.union(sketchA); + union.union(sketchB); + return union.getResult(false, null); + } + + /** + * Returns true if the two given sketches and their union all have the same number of retained + * entries and the same theta, which means the two sketches hold identical sets. + * @param sketchA the given sketch A + * @param sketchB the given sketch B + * @param unionAB the union of sketch A and sketch B + * @return true if the two given sketches hold identical sets + */ + private static boolean identicalSets(final ThetaSketch sketchA, final ThetaSketch sketchB, + final ThetaSketch unionAB) { + final int countUAB = unionAB.getRetainedEntries(true); + final long thetaLongUAB = unionAB.getThetaLong(); + return countUAB == sketchA.getRetainedEntries(true) + && countUAB == sketchB.getRetainedEntries(true) + && thetaLongUAB == sketchA.getThetaLong() + && thetaLongUAB == sketchB.getThetaLong(); + } + }