From 348a27c366e2bbd20f53f84592752f2694f1d70f Mon Sep 17 00:00:00 2001 From: Henning Ulfarsson Date: Sun, 18 Oct 2026 16:43:20 +0000 Subject: [PATCH 01/38] Add LabelledPerms: auto_bisc input with labels kept as bitmaps With a property as input, auto_bisc keeps every good and every bad permutation up to its current length as lists of Perm objects. At length 10 the bad ones are nearly all of S_10, about 0.9 GB as a list, and runs on sets that need length 10 grew past 8 GB. LabelledPerms keeps the labels as one bitmap per length, in the order of Perm.of_length (S_10 is 454 KB), made from a property, given as bitmaps, or both. Its views good and bad stand in for auto_bisc's dictionaries of lists: auto_bisc only iterates over a length, counts it, and asks whether it is there, and the views do that from the bitmaps, generating a length's permutations again when it is read, in the same order. - auto_bisc(LabelledPerms(...)) and bisc(LabelledPerms(...)) are new inputs; with given bitmaps and no property, auto_bisc stops when it needs a length it was not given, as for a tuple of dictionaries. - auto_bisc(prop, labels="bitmaps") wraps a property in LabelledPerms; labels="lists", the default, is the previous behaviour, unchanged. The existing inputs are untouched. On stack-sortable and West-2-stack- sortable permutations every route gives the same basis as before (tests/bisc/test_labelled_perms.py); auto_bisc's printed output is also identical on Av(231), Av(321), Av(2-31) and Av(132, 4321). Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 8 ++ permuta/bisc/bisc.py | 132 +++++++++++++++++++++++++++++- tests/bisc/test_labelled_perms.py | 81 ++++++++++++++++++ 3 files changed, 219 insertions(+), 2 deletions(-) create mode 100644 tests/bisc/test_labelled_perms.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 50ecb90..e2cd282 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,14 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## Unreleased +### Added +- `LabelledPerms` in `permuta.bisc.bisc`: labels of permutations kept as one bitmap + per length (lexicographic order), made from a property, given as bitmaps, or both. + `auto_bisc` and `bisc` accept it as input. +- `auto_bisc(prop, labels="bitmaps")`: keeps a property's labels as bitmaps instead + of lists of permutations, for the same result in far less memory at longer + lengths. `labels="lists"`, the default, is the previous behaviour. + ### Fixed - Fixed bug in `Perm.rtlmax_ltrmin_decomposition` and fixed associated tests. diff --git a/permuta/bisc/bisc.py b/permuta/bisc/bisc.py index e2b6549..28fe58d 100644 --- a/permuta/bisc/bisc.py +++ b/permuta/bisc/bisc.py @@ -4,6 +4,7 @@ import os import types from collections import defaultdict +from math import factorial from permuta.bisc.bisc_subfunctions import ( forb, @@ -36,12 +37,19 @@ def bisc(A, m, n=None, report=False): elif isinstance(A, dict): D = A + elif isinstance(A, LabelledPerms): + D = A.good + + elif isinstance(A, _LengthViews): + D = A + else: print( """BiSC can accept inputs: 1. A list of permutations 2. A property of permutations - 3. A dictionary of lengths pointing to permutations""" + 3. A dictionary of lengths pointing to permutations + 4. A LabelledPerms (labels kept as bitmaps)""" ) assert False @@ -54,7 +62,20 @@ def bisc(A, m, n=None, report=False): return SG -def auto_bisc(prop): +def auto_bisc(prop, labels="lists"): + """ + labels says how the labels of a property (a function) are kept: + "lists" (the default, as before): the good and the bad permutations of + each length as lists of Perm objects; + "bitmaps": one bitmap per length (LabelledPerms), for the same result in + far less memory, at the cost of generating a length's permutations again + each time it is read. + A LabelledPerms can also be given as prop directly. + """ + if labels not in ("lists", "bitmaps"): + raise ValueError('labels must be "lists" or "bitmaps"') + if labels == "bitmaps" and isinstance(prop, types.FunctionType): + prop = LabelledPerms(prop) L = 8 # Want to sanity check on at least S8, and one above n n = 4 m = 2 @@ -88,6 +109,20 @@ def auto_bisc(prop): else: B[i].append(perm) + elif isinstance(prop, LabelledPerms): + # Labels kept as one bitmap per length: A and B are views of the good + # and the bad permutations, generated length by length when they are + # read, instead of lists. Lengths not given are labelled with the + # LabelledPerms's property, if it has one + labelled = prop + for i in range(L + 1): + if i not in labelled.bits: + if labelled.prop is None: + print("You should have labels up to length at least 8") + return + labelled.add_length(i) + A, B = labelled.good, labelled.bad + elif ( isinstance(prop, tuple) and isinstance(prop[0], dict) @@ -229,6 +264,15 @@ def auto_bisc(prop): else: B[i].append(perm) + elif isinstance(prop, LabelledPerms): + for i in range(oldL + 1, L + 1): + if i not in labelled.bits: + if labelled.prop is None: + print("You need to add labels of longer permutations") + return + print("Adding perms of length {}".format(i)) + labelled.add_length(i) + elif isinstance(prop, tuple) and L > min(max(A.keys()), max(B.keys())): print("You need to add longer permutations to the dictionaries") return @@ -269,6 +313,90 @@ def auto_bisc(prop): return +class LabelledPerms: + """Labels of permutations, one bitmap per length, as input to auto_bisc + and bisc: which permutations of each length satisfy a property (good) and + which do not (bad). + + Bit i of length n is the i-th permutation of Perm.of_length(n) + (lexicographic order), in byte i // 8, from the low bit. The bitmaps can + be given (bitmaps={n: bytes}), made from a property (add_length), or both: + auto_bisc labels the lengths it needs and is not given with prop, and + stops, as for a tuple of dictionaries, when there is no prop. + + auto_bisc reads its good and bad permutations only length by length: + iterating over a length, counting it, asking whether a length is there. + The views good and bad do that from the bitmaps, generating a length's + permutations again each time it is iterated, in the same order as the + lists a property input makes. A bitmap of S_10 is 454 KB; S_10 as a list + of Perm objects is about 0.9 GB. + """ + + def __init__(self, prop=None, bitmaps=None): + self.prop = prop + self.bits = {} + self.counts = {} + for n, data in (bitmaps or {}).items(): + data = bytes(data) + if len(data) != (factorial(n) + 7) // 8: + raise ValueError("A bitmap of length {} has the wrong size".format(n)) + self.bits[n] = data + self.counts[n] = sum(bin(b).count("1") for b in data) + self.good = _LengthViews(self, True) + self.bad = _LengthViews(self, False) + + def add_length(self, n): + bits = bytearray((factorial(n) + 7) // 8) + count = 0 + for i, perm in enumerate(Perm.of_length(n)): + if self.prop(perm): + bits[i >> 3] |= 1 << (i & 7) + count += 1 + self.bits[n] = bits + self.counts[n] = count + + +class _LengthViews: + """length -> the good (or bad) permutations of that length, read-only, as + the dictionaries of lists auto_bisc builds from the other inputs.""" + + def __init__(self, labelled, good): + self.labelled = labelled + self.is_good = good + + def keys(self): + return self.labelled.bits.keys() + + def __contains__(self, n): + return n in self.labelled.bits + + def __getitem__(self, n): + if n not in self.labelled.bits: + raise KeyError(n) + return _PermsOfLength(self.labelled, n, self.is_good) + + +class _PermsOfLength: + def __init__(self, labelled, n, good): + self.labelled = labelled + self.n = n + self.want = 1 if good else 0 + + def __iter__(self): + bits = self.labelled.bits[self.n] + want = self.want + for i, perm in enumerate(Perm.of_length(self.n)): + if (bits[i >> 3] >> (i & 7)) & 1 == want: + yield perm + + def __len__(self): + good = self.labelled.counts[self.n] + return good if self.want else factorial(self.n) - good + + def __bool__(self): + return len(self) > 0 + + def create_bisc_input(N, prop): """ Create a dictionary, D, containing keys 1, 2, 3, ..., N. Each key points to diff --git a/tests/bisc/test_labelled_perms.py b/tests/bisc/test_labelled_perms.py new file mode 100644 index 0000000..0ab7a6a --- /dev/null +++ b/tests/bisc/test_labelled_perms.py @@ -0,0 +1,81 @@ +import pytest + +from permuta.bisc.bisc import LabelledPerms, auto_bisc, bisc +from permuta.patterns.perm import Perm + + +def stack_sortable(perm): + return perm.avoids(Perm((1, 2, 0))) + + +def labelled(prop, top): + labels = LabelledPerms(prop) + for n in range(top + 1): + labels.add_length(n) + return labels + + +def test_views_match_lists(): + labels = labelled(stack_sortable, 6) + for n in range(7): + good = [perm for perm in Perm.of_length(n) if stack_sortable(perm)] + bad = [perm for perm in Perm.of_length(n) if not stack_sortable(perm)] + assert list(labels.good[n]) == good + assert list(labels.bad[n]) == bad + assert len(labels.good[n]) == len(good) + assert len(labels.bad[n]) == len(bad) + assert bool(labels.bad[n]) == bool(bad) + assert sorted(labels.good.keys()) == list(range(7)) + assert 6 in labels.bad and 7 not in labels.bad + with pytest.raises(KeyError): + _ = labels.good[7] + + +def test_bitmaps_round_trip(): + labels = labelled(stack_sortable, 6) + again = LabelledPerms(bitmaps=labels.bits) + for n in range(7): + assert list(again.good[n]) == list(labels.good[n]) + assert list(again.bad[n]) == list(labels.bad[n]) + with pytest.raises(ValueError): + LabelledPerms(bitmaps={3: b"\x00\x00"}) # 3! bits fit in one byte + + +def test_bisc_accepts_labelled_perms(): + expected = {3: {Perm((1, 2, 0)): [set()]}} + assert bisc(stack_sortable, 3, 5) == expected + assert bisc(labelled(stack_sortable, 5), 3, 5) == expected + + +def test_auto_bisc_labels_option(): + expected = {3: {Perm((1, 2, 0)): [set()]}} + assert auto_bisc(stack_sortable) == expected + assert auto_bisc(stack_sortable, labels="lists") == expected + assert auto_bisc(stack_sortable, labels="bitmaps") == expected + with pytest.raises(ValueError): + auto_bisc(stack_sortable, labels="sets") + + +def test_auto_bisc_labelled_perms_input(): + expected = {3: {Perm((1, 2, 0)): [set()]}} + assert auto_bisc(labelled(stack_sortable, 8)) == expected + # Given bitmaps and no property: the lengths auto_bisc needs must be there + bits = labelled(stack_sortable, 8).bits + assert auto_bisc(LabelledPerms(bitmaps=bits)) == expected + short = {n: bits[n] for n in range(6)} + assert auto_bisc(LabelledPerms(bitmaps=short)) is None + + +def test_west_2_stack_sortable_same_as_lists(): + def west_2(perm): + def stack_sort(word): + out, stack = [], [] + for x in word: + while stack and stack[-1] < x: + out.append(stack.pop()) + stack.append(x) + return out + stack[::-1] + + return stack_sort(stack_sort(list(perm))) == sorted(perm) + + assert auto_bisc(west_2, labels="bitmaps") == auto_bisc(west_2) From de4fa241f9952d9653ca11cd2f3fd760bf3d3ced Mon Sep 17 00:00:00 2001 From: Henning Ulfarsson Date: Sun, 18 Oct 2026 18:48:36 +0000 Subject: [PATCH 02/38] Test auto_bisc's growth past the labelled lengths with LabelledPerms When auto_bisc needs a length it was not given, a LabelledPerms with a property labels it, and one with bitmaps alone stops. Derangements in Av(213, 231, 312), with one member at each even length, reach that step in a fraction of a second: with bitmaps up to length 8 and no property auto_bisc stops and says so; with a property, stopped at the first permutation of length 9, the lists and the bitmaps have printed the same. Co-Authored-By: Claude Opus 5.5 --- tests/bisc/test_labelled_perms.py | 43 +++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/tests/bisc/test_labelled_perms.py b/tests/bisc/test_labelled_perms.py index 0ab7a6a..6eb8347 100644 --- a/tests/bisc/test_labelled_perms.py +++ b/tests/bisc/test_labelled_perms.py @@ -1,3 +1,6 @@ +import contextlib +import io + import pytest from permuta.bisc.bisc import LabelledPerms, auto_bisc, bisc @@ -79,3 +82,43 @@ def stack_sort(word): return stack_sort(stack_sort(list(perm))) == sorted(perm) assert auto_bisc(west_2, labels="bitmaps") == auto_bisc(west_2) + + +def derangement_in_av_213_231_312(perm): + """One member at each even length, none at odd: auto_bisc keeps learning longer + patterns while each step stays cheap, so it soon needs more labels.""" + return perm.avoids(Perm((1, 0, 2)), Perm((1, 2, 0)), Perm((2, 0, 1))) and all( + value != index for index, value in enumerate(perm) + ) + + +def test_auto_bisc_stops_without_labels_it_needs(): + # Bitmaps up to length 8 and no property: auto_bisc needs length 9 and stops + bits = labelled(derangement_in_av_213_231_312, 8).bits + output = io.StringIO() + with contextlib.redirect_stdout(output): + assert auto_bisc(LabelledPerms(bitmaps=bits)) is None + assert "You need to add labels of longer permutations" in output.getvalue() + + +class ReachedLength9(Exception): + pass + + +def test_auto_bisc_labels_new_lengths_from_the_property(): + # When auto_bisc grows past the labelled lengths, it labels the new ones with + # the property; stopped at the first permutation of length 9, both approaches + # have printed the same + def prop(perm): + if len(perm) == 9: + raise ReachedLength9 + return derangement_in_av_213_231_312(perm) + + outputs = [] + for labels in ("lists", "bitmaps"): + output = io.StringIO() + with contextlib.redirect_stdout(output), pytest.raises(ReachedLength9): + auto_bisc(prop, labels=labels) + outputs.append(output.getvalue()) + assert outputs[0] == outputs[1] + assert "Adding perms of length 9" in outputs[1] From b2077689815f7eb6c1f26effcaae03eea8999738 Mon Sep 17 00:00:00 2001 From: Henning Ulfarsson Date: Sun, 18 Oct 2026 19:05:12 +0000 Subject: [PATCH 03/38] Export LabelledPerms from permuta.bisc and document it in the README The README's BiSC section starts with `from permuta.bisc import *`, which did not bring in LabelledPerms. It is now exported, and the section says that bisc and auto_bisc also accept a LabelledPerms, and how auto_bisc's labels="bitmaps" keeps a property's labels in less memory, with an example that runs as a doctest. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 +- README.rst | 19 ++++++++++++++++++- permuta/bisc/__init__.py | 10 +++++++++- 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2cd282..5ae259f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## Unreleased ### Added -- `LabelledPerms` in `permuta.bisc.bisc`: labels of permutations kept as one bitmap +- `LabelledPerms` in `permuta.bisc`: labels of permutations kept as one bitmap per length (lexicographic order), made from a property, given as bitmaps, or both. `auto_bisc` and `bisc` accept it as input. - `auto_bisc(prop, labels="bitmaps")`: keeps a property's labels as bitmaps instead diff --git a/README.rst b/README.rst index 1277f39..2988a30 100644 --- a/README.rst +++ b/README.rst @@ -516,7 +516,24 @@ permutations that can be imported from ``permuta.bisc.perm_properties``, such as ``smooth``, ``forest-like``, ``baxter``, ``simsun``, ``quick_sortable``, etc. Both ``bisc`` and ``auto_bisc`` can accept input in the form of a property, -or a list of permutations (satisfying some property). +a list of permutations (satisfying some property), or a ``LabelledPerms``. + +``auto_bisc`` keeps the permutations satisfying a property, and those that do +not, as lists. For long runs it can keep them as one bitmap per length instead, +with the same result in far less memory: +``auto_bisc(Perm.stack_sortable, labels="bitmaps")``. Such labels can also be +given directly as a ``LabelledPerms``, made from a property, from bitmaps, or +both, and ``bisc`` accepts it too. + +.. code-block:: python + + >>> labels = LabelledPerms(Perm.stack_sortable) + >>> for n in range(6): + ... labels.add_length(n) + >>> len(labels.good[5]), len(labels.bad[5]) + (42, 78) + >>> bisc(labels, 3, 5) + {3: {Perm((1, 2, 0)): [set()]}} License ####### diff --git a/permuta/bisc/__init__.py b/permuta/bisc/__init__.py index 34bcdf8..acb5b85 100644 --- a/permuta/bisc/__init__.py +++ b/permuta/bisc/__init__.py @@ -1,6 +1,13 @@ # type: ignore -from .bisc import auto_bisc, bisc, create_bisc_input, read_bisc_file, write_bisc_files +from .bisc import ( + LabelledPerms, + auto_bisc, + bisc, + create_bisc_input, + read_bisc_file, + write_bisc_files, +) from .bisc_subfunctions import ( patterns_suffice_for_bad, patterns_suffice_for_good, @@ -10,6 +17,7 @@ ) __all__ = [ + "LabelledPerms", "auto_bisc", "bisc", "create_bisc_input", From f449beb2268fc67133ec0f76cb65aded2cc9b06e Mon Sep 17 00:00:00 2001 From: Henning Ulfarsson Date: Sun, 18 Oct 2026 19:48:22 +0000 Subject: [PATCH 04/38] Define LabelledPerms' bit order by Perm.unrank Bit i of length n is the label of Perm.unrank(i, n), so a permutation of length n is at its rank() less the number of shorter permutations. This is the order Perm.of_length(n) yields, which the views still use to iterate: it is the same order and about three times faster than unranking each index. A test pins the order to Perm.unrank and Perm.rank. Co-Authored-By: Claude Opus 5.5 --- permuta/bisc/bisc.py | 13 ++++++++----- tests/bisc/test_labelled_perms.py | 15 +++++++++++++++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/permuta/bisc/bisc.py b/permuta/bisc/bisc.py index 28fe58d..088266e 100644 --- a/permuta/bisc/bisc.py +++ b/permuta/bisc/bisc.py @@ -318,11 +318,14 @@ class LabelledPerms: and bisc: which permutations of each length satisfy a property (good) and which do not (bad). - Bit i of length n is the i-th permutation of Perm.of_length(n) - (lexicographic order), in byte i // 8, from the low bit. The bitmaps can - be given (bitmaps={n: bytes}), made from a property (add_length), or both: - auto_bisc labels the lengths it needs and is not given with prop, and - stops, as for a tuple of dictionaries, when there is no prop. + Bit i of length n is the label of Perm.unrank(i, n), in byte i // 8, from + the low bit; so a permutation perm of length n is at bit + perm.rank() - sum(factorial(k) for k in range(n)). This is the order in + which Perm.of_length(n) yields the permutations (lexicographic), which the + views use to iterate. The bitmaps can be given (bitmaps={n: bytes}), made + from a property (add_length), or both: auto_bisc labels the lengths it + needs and is not given with prop, and stops, as for a tuple of + dictionaries, when there is no prop. auto_bisc reads its good and bad permutations only length by length: iterating over a length, counting it, asking whether a length is there. diff --git a/tests/bisc/test_labelled_perms.py b/tests/bisc/test_labelled_perms.py index 6eb8347..0a7a2f9 100644 --- a/tests/bisc/test_labelled_perms.py +++ b/tests/bisc/test_labelled_perms.py @@ -1,5 +1,6 @@ import contextlib import io +from math import factorial import pytest @@ -34,6 +35,20 @@ def test_views_match_lists(): _ = labels.good[7] +def test_bit_order_is_perm_unrank(): + # Bit i of length n is the label of Perm.unrank(i, n), and the permutation + # perm is at bit perm.rank() minus the number of shorter permutations + labels = labelled(stack_sortable, 6) + shorter = 0 + for n in range(7): + bits = labels.bits[n] + for i in range(factorial(n)): + perm = Perm.unrank(i, n) + assert perm.rank() - shorter == i + assert bool(bits[i >> 3] >> (i & 7) & 1) == stack_sortable(perm) + shorter += factorial(n) + + def test_bitmaps_round_trip(): labels = labelled(stack_sortable, 6) again = LabelledPerms(bitmaps=labels.bits) From 9332c71d6051d8930807cbbb0da5c93b137d5cd5 Mon Sep 17 00:00:00 2001 From: Henning Ulfarsson Date: Wed, 21 Oct 2026 13:35:14 +0000 Subject: [PATCH 05/38] Name LabelledPerms' variables by what they hold Reviewers of PermutaTriangle's pull requests ask for variable names that say what they hold ("for rule_key in ..." rather than "for rk in ..."). In the code this pull request adds: - LabelledPerms and its views: a length is `length`, as in Perm.of_length, where it was `n`; a bitmap is `bitmap` where it was `data` or `bits`; a byte is `byte` where it was `b`; the bit a view wants is `wanted_bit` where it was `want`; the number of good permutations is `good_count` where it was `good`. The attribute `bits` is now `bitmaps`, as the constructor's keyword says. - The tests: `length`, `bitmap`, `bitmaps`, `max_length`, `output`, `entry`, `round_trip`, `too_short` and `shorter_count`, where they had `n`, `bits`, `top`, `out`, `x`, `again`, `short` and `shorter`. - The README's example: `length` where it had `n`. The names bisc and auto_bisc already had (A, B, D, L, i, m and prop, the BiSC paper's notation) are kept, and `i` stays the index of a bit. Nothing else changes: the bisc tests and the README's examples pass as before. Co-Authored-By: Claude Opus 5.5 --- README.rst | 4 +- permuta/bisc/bisc.py | 64 ++++++++++++++-------------- tests/bisc/test_labelled_perms.py | 70 +++++++++++++++---------------- 3 files changed, 70 insertions(+), 68 deletions(-) diff --git a/README.rst b/README.rst index 2988a30..023a798 100644 --- a/README.rst +++ b/README.rst @@ -528,8 +528,8 @@ both, and ``bisc`` accepts it too. .. code-block:: python >>> labels = LabelledPerms(Perm.stack_sortable) - >>> for n in range(6): - ... labels.add_length(n) + >>> for length in range(6): + ... labels.add_length(length) >>> len(labels.good[5]), len(labels.bad[5]) (42, 78) >>> bisc(labels, 3, 5) diff --git a/permuta/bisc/bisc.py b/permuta/bisc/bisc.py index 088266e..4e4cc69 100644 --- a/permuta/bisc/bisc.py +++ b/permuta/bisc/bisc.py @@ -116,7 +116,7 @@ def auto_bisc(prop, labels="lists"): # LabelledPerms's property, if it has one labelled = prop for i in range(L + 1): - if i not in labelled.bits: + if i not in labelled.bitmaps: if labelled.prop is None: print("You should have labels up to length at least 8") return @@ -266,7 +266,7 @@ def auto_bisc(prop, labels="lists"): elif isinstance(prop, LabelledPerms): for i in range(oldL + 1, L + 1): - if i not in labelled.bits: + if i not in labelled.bitmaps: if labelled.prop is None: print("You need to add labels of longer permutations") return @@ -337,26 +337,28 @@ class LabelledPerms: def __init__(self, prop=None, bitmaps=None): self.prop = prop - self.bits = {} + self.bitmaps = {} self.counts = {} - for n, data in (bitmaps or {}).items(): - data = bytes(data) - if len(data) != (factorial(n) + 7) // 8: - raise ValueError("A bitmap of length {} has the wrong size".format(n)) - self.bits[n] = data - self.counts[n] = sum(bin(b).count("1") for b in data) + for length, bitmap in (bitmaps or {}).items(): + bitmap = bytes(bitmap) + if len(bitmap) != (factorial(length) + 7) // 8: + raise ValueError( + "A bitmap of length {} has the wrong size".format(length) + ) + self.bitmaps[length] = bitmap + self.counts[length] = sum(bin(byte).count("1") for byte in bitmap) self.good = _LengthViews(self, True) self.bad = _LengthViews(self, False) - def add_length(self, n): - bits = bytearray((factorial(n) + 7) // 8) + def add_length(self, length): + bitmap = bytearray((factorial(length) + 7) // 8) count = 0 - for i, perm in enumerate(Perm.of_length(n)): + for i, perm in enumerate(Perm.of_length(length)): if self.prop(perm): - bits[i >> 3] |= 1 << (i & 7) + bitmap[i >> 3] |= 1 << (i & 7) count += 1 - self.bits[n] = bits - self.counts[n] = count + self.bitmaps[length] = bitmap + self.counts[length] = count class _LengthViews: @@ -368,33 +370,33 @@ def __init__(self, labelled, good): self.is_good = good def keys(self): - return self.labelled.bits.keys() + return self.labelled.bitmaps.keys() - def __contains__(self, n): - return n in self.labelled.bits + def __contains__(self, length): + return length in self.labelled.bitmaps - def __getitem__(self, n): - if n not in self.labelled.bits: - raise KeyError(n) - return _PermsOfLength(self.labelled, n, self.is_good) + def __getitem__(self, length): + if length not in self.labelled.bitmaps: + raise KeyError(length) + return _PermsOfLength(self.labelled, length, self.is_good) class _PermsOfLength: - def __init__(self, labelled, n, good): + def __init__(self, labelled, length, good): self.labelled = labelled - self.n = n - self.want = 1 if good else 0 + self.length = length + self.wanted_bit = 1 if good else 0 def __iter__(self): - bits = self.labelled.bits[self.n] - want = self.want - for i, perm in enumerate(Perm.of_length(self.n)): - if (bits[i >> 3] >> (i & 7)) & 1 == want: + bitmap = self.labelled.bitmaps[self.length] + wanted_bit = self.wanted_bit + for i, perm in enumerate(Perm.of_length(self.length)): + if (bitmap[i >> 3] >> (i & 7)) & 1 == wanted_bit: yield perm def __len__(self): - good = self.labelled.counts[self.n] - return good if self.want else factorial(self.n) - good + good_count = self.labelled.counts[self.length] + return good_count if self.wanted_bit else factorial(self.length) - good_count def __bool__(self): return len(self) > 0 diff --git a/tests/bisc/test_labelled_perms.py b/tests/bisc/test_labelled_perms.py index 0a7a2f9..4e03f06 100644 --- a/tests/bisc/test_labelled_perms.py +++ b/tests/bisc/test_labelled_perms.py @@ -12,23 +12,23 @@ def stack_sortable(perm): return perm.avoids(Perm((1, 2, 0))) -def labelled(prop, top): +def labelled(prop, max_length): labels = LabelledPerms(prop) - for n in range(top + 1): - labels.add_length(n) + for length in range(max_length + 1): + labels.add_length(length) return labels def test_views_match_lists(): labels = labelled(stack_sortable, 6) - for n in range(7): - good = [perm for perm in Perm.of_length(n) if stack_sortable(perm)] - bad = [perm for perm in Perm.of_length(n) if not stack_sortable(perm)] - assert list(labels.good[n]) == good - assert list(labels.bad[n]) == bad - assert len(labels.good[n]) == len(good) - assert len(labels.bad[n]) == len(bad) - assert bool(labels.bad[n]) == bool(bad) + for length in range(7): + good = [perm for perm in Perm.of_length(length) if stack_sortable(perm)] + bad = [perm for perm in Perm.of_length(length) if not stack_sortable(perm)] + assert list(labels.good[length]) == good + assert list(labels.bad[length]) == bad + assert len(labels.good[length]) == len(good) + assert len(labels.bad[length]) == len(bad) + assert bool(labels.bad[length]) == bool(bad) assert sorted(labels.good.keys()) == list(range(7)) assert 6 in labels.bad and 7 not in labels.bad with pytest.raises(KeyError): @@ -39,22 +39,22 @@ def test_bit_order_is_perm_unrank(): # Bit i of length n is the label of Perm.unrank(i, n), and the permutation # perm is at bit perm.rank() minus the number of shorter permutations labels = labelled(stack_sortable, 6) - shorter = 0 - for n in range(7): - bits = labels.bits[n] - for i in range(factorial(n)): - perm = Perm.unrank(i, n) - assert perm.rank() - shorter == i - assert bool(bits[i >> 3] >> (i & 7) & 1) == stack_sortable(perm) - shorter += factorial(n) + shorter_count = 0 + for length in range(7): + bitmap = labels.bitmaps[length] + for i in range(factorial(length)): + perm = Perm.unrank(i, length) + assert perm.rank() - shorter_count == i + assert bool(bitmap[i >> 3] >> (i & 7) & 1) == stack_sortable(perm) + shorter_count += factorial(length) def test_bitmaps_round_trip(): labels = labelled(stack_sortable, 6) - again = LabelledPerms(bitmaps=labels.bits) - for n in range(7): - assert list(again.good[n]) == list(labels.good[n]) - assert list(again.bad[n]) == list(labels.bad[n]) + round_trip = LabelledPerms(bitmaps=labels.bitmaps) + for length in range(7): + assert list(round_trip.good[length]) == list(labels.good[length]) + assert list(round_trip.bad[length]) == list(labels.bad[length]) with pytest.raises(ValueError): LabelledPerms(bitmaps={3: b"\x00\x00"}) # 3! bits fit in one byte @@ -78,21 +78,21 @@ def test_auto_bisc_labelled_perms_input(): expected = {3: {Perm((1, 2, 0)): [set()]}} assert auto_bisc(labelled(stack_sortable, 8)) == expected # Given bitmaps and no property: the lengths auto_bisc needs must be there - bits = labelled(stack_sortable, 8).bits - assert auto_bisc(LabelledPerms(bitmaps=bits)) == expected - short = {n: bits[n] for n in range(6)} - assert auto_bisc(LabelledPerms(bitmaps=short)) is None + bitmaps = labelled(stack_sortable, 8).bitmaps + assert auto_bisc(LabelledPerms(bitmaps=bitmaps)) == expected + too_short = {length: bitmaps[length] for length in range(6)} + assert auto_bisc(LabelledPerms(bitmaps=too_short)) is None def test_west_2_stack_sortable_same_as_lists(): def west_2(perm): def stack_sort(word): - out, stack = [], [] - for x in word: - while stack and stack[-1] < x: - out.append(stack.pop()) - stack.append(x) - return out + stack[::-1] + output, stack = [], [] + for entry in word: + while stack and stack[-1] < entry: + output.append(stack.pop()) + stack.append(entry) + return output + stack[::-1] return stack_sort(stack_sort(list(perm))) == sorted(perm) @@ -109,10 +109,10 @@ def derangement_in_av_213_231_312(perm): def test_auto_bisc_stops_without_labels_it_needs(): # Bitmaps up to length 8 and no property: auto_bisc needs length 9 and stops - bits = labelled(derangement_in_av_213_231_312, 8).bits + bitmaps = labelled(derangement_in_av_213_231_312, 8).bitmaps output = io.StringIO() with contextlib.redirect_stdout(output): - assert auto_bisc(LabelledPerms(bitmaps=bits)) is None + assert auto_bisc(LabelledPerms(bitmaps=bitmaps)) is None assert "You need to add labels of longer permutations" in output.getvalue() From 0099214e8538b6b9b6a9ce15139df51dfe16cd77 Mon Sep 17 00:00:00 2001 From: Henning Ulfarsson Date: Thu, 22 Oct 2026 21:16:16 +0000 Subject: [PATCH 06/38] Make LabelledPerms and its tests easier to read From the review of #209, round 1, its readability and duplication suggestions: - auto_bisc's docstring opens with what it does and returns, names the inputs it takes, says when to use labels="bitmaps", and says that the lengths it labels are added to a LabelledPerms it is given. - LabelledPerms: its docstring says length where it said n; counts is now good_counts; one helper, _bitmap_size, gives a bitmap's size; add_length has a docstring and keeps bytes, as the constructor does; _PermsOfLength drops __bool__, which __len__ already gives. - auto_bisc uses prop where it had the alias labelled. - The tests use Perm.stack_sortable and Perm.west_2_stack_sortable instead of copies of their own, BASIS_231 for the basis they expect, and labelled_perms for a LabelledPerms, so that labels is only the option's name. The README's example is renamed the same way. - The README and CHANGELOG say when to use bitmaps. Nothing else changes: the bisc tests and the README's examples pass. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 +- README.rst | 16 +++--- permuta/bisc/bisc.py | 81 ++++++++++++++++----------- tests/bisc/test_labelled_perms.py | 91 +++++++++++++------------------ 4 files changed, 100 insertions(+), 91 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ae259f..db41f0b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. `auto_bisc` and `bisc` accept it as input. - `auto_bisc(prop, labels="bitmaps")`: keeps a property's labels as bitmaps instead of lists of permutations, for the same result in far less memory at longer - lengths. `labels="lists"`, the default, is the previous behaviour. + lengths (the permutations of length 10 take about 0.9 GB as lists, 454 KB as a + bitmap). `labels="lists"`, the default, is the previous behaviour. ### Fixed - Fixed bug in `Perm.rtlmax_ltrmin_decomposition` and fixed associated tests. diff --git a/README.rst b/README.rst index 023a798..7b934c2 100644 --- a/README.rst +++ b/README.rst @@ -521,18 +521,20 @@ a list of permutations (satisfying some property), or a ``LabelledPerms``. ``auto_bisc`` keeps the permutations satisfying a property, and those that do not, as lists. For long runs it can keep them as one bitmap per length instead, with the same result in far less memory: -``auto_bisc(Perm.stack_sortable, labels="bitmaps")``. Such labels can also be -given directly as a ``LabelledPerms``, made from a property, from bitmaps, or -both, and ``bisc`` accepts it too. +``auto_bisc(Perm.stack_sortable, labels="bitmaps")``. Use it when ``auto_bisc`` +has to check long permutations: the permutations of length 10 take about 0.9 GB +as lists, and 454 KB as a bitmap. Such labels can also be given directly as a +``LabelledPerms``, made from a property, from bitmaps, or both, and ``bisc`` +accepts it too. .. code-block:: python - >>> labels = LabelledPerms(Perm.stack_sortable) + >>> labelled_perms = LabelledPerms(Perm.stack_sortable) >>> for length in range(6): - ... labels.add_length(length) - >>> len(labels.good[5]), len(labels.bad[5]) + ... labelled_perms.add_length(length) + >>> len(labelled_perms.good[5]), len(labelled_perms.bad[5]) (42, 78) - >>> bisc(labels, 3, 5) + >>> bisc(labelled_perms, 3, 5) {3: {Perm((1, 2, 0)): [set()]}} License diff --git a/permuta/bisc/bisc.py b/permuta/bisc/bisc.py index 4e4cc69..9008293 100644 --- a/permuta/bisc/bisc.py +++ b/permuta/bisc/bisc.py @@ -64,13 +64,27 @@ def bisc(A, m, n=None, report=False): def auto_bisc(prop, labels="lists"): """ - labels says how the labels of a property (a function) are kept: + Learn a basis of forbidden patterns for a set of permutations, growing + the length of the patterns and of the permutations they are learned from + until the patterns found describe the set on every permutation up to + length L, at least 8. Returns the basis, in the form bisc returns its + patterns, or None when the input is too short to check a basis. + + prop is the set: a property of permutations (a function), a list of + permutations, a tuple of two dictionaries (length -> the permutations in + the set, and those not in it), the name of a set in + permuta/resources/bisc, or a LabelledPerms. + + labels says how the labels of a property are kept: "lists" (the default, as before): the good and the bad permutations of each length as lists of Perm objects; "bitmaps": one bitmap per length (LabelledPerms), for the same result in far less memory, at the cost of generating a length's permutations again - each time it is read. - A LabelledPerms can also be given as prop directly. + each time it is read. Use it when the permutations checked are long: from + length 10 on the lists take gigabytes, the bitmaps under a megabyte. + + A LabelledPerms can also be given as prop directly. The lengths auto_bisc + labels with its property are added to it, so a later run reuses them. """ if labels not in ("lists", "bitmaps"): raise ValueError('labels must be "lists" or "bitmaps"') @@ -114,14 +128,13 @@ def auto_bisc(prop, labels="lists"): # and the bad permutations, generated length by length when they are # read, instead of lists. Lengths not given are labelled with the # LabelledPerms's property, if it has one - labelled = prop for i in range(L + 1): - if i not in labelled.bitmaps: - if labelled.prop is None: + if i not in prop.bitmaps: + if prop.prop is None: print("You should have labels up to length at least 8") return - labelled.add_length(i) - A, B = labelled.good, labelled.bad + prop.add_length(i) + A, B = prop.good, prop.bad elif ( isinstance(prop, tuple) @@ -266,12 +279,12 @@ def auto_bisc(prop, labels="lists"): elif isinstance(prop, LabelledPerms): for i in range(oldL + 1, L + 1): - if i not in labelled.bitmaps: - if labelled.prop is None: + if i not in prop.bitmaps: + if prop.prop is None: print("You need to add labels of longer permutations") return print("Adding perms of length {}".format(i)) - labelled.add_length(i) + prop.add_length(i) elif isinstance(prop, tuple) and L > min(max(A.keys()), max(B.keys())): print("You need to add longer permutations to the dictionaries") @@ -313,19 +326,26 @@ def auto_bisc(prop, labels="lists"): return +def _bitmap_size(length): + """The number of bytes in the bitmap of a length: a bit for each of its + permutations.""" + return (factorial(length) + 7) // 8 + + class LabelledPerms: """Labels of permutations, one bitmap per length, as input to auto_bisc and bisc: which permutations of each length satisfy a property (good) and which do not (bad). - Bit i of length n is the label of Perm.unrank(i, n), in byte i // 8, from - the low bit; so a permutation perm of length n is at bit - perm.rank() - sum(factorial(k) for k in range(n)). This is the order in - which Perm.of_length(n) yields the permutations (lexicographic), which the - views use to iterate. The bitmaps can be given (bitmaps={n: bytes}), made - from a property (add_length), or both: auto_bisc labels the lengths it - needs and is not given with prop, and stops, as for a tuple of - dictionaries, when there is no prop. + Bit i of a length's bitmap is the label of Perm.unrank(i, length), in + byte i // 8, from the low bit; so a permutation perm is at bit + perm.rank() - sum(factorial(k) for k in range(len(perm))). This is the + order in which Perm.of_length(length) yields the permutations + (lexicographic), which the views use to iterate. The bitmaps can be given + (bitmaps={length: bytes}), made from a property (add_length), or both: + auto_bisc labels the lengths it needs and is not given with prop, adding + them to this object, and stops, as for a tuple of dictionaries, when + there is no prop. auto_bisc reads its good and bad permutations only length by length: iterating over a length, counting it, asking whether a length is there. @@ -338,27 +358,29 @@ class LabelledPerms: def __init__(self, prop=None, bitmaps=None): self.prop = prop self.bitmaps = {} - self.counts = {} + self.good_counts = {} for length, bitmap in (bitmaps or {}).items(): bitmap = bytes(bitmap) - if len(bitmap) != (factorial(length) + 7) // 8: + if len(bitmap) != _bitmap_size(length): raise ValueError( "A bitmap of length {} has the wrong size".format(length) ) self.bitmaps[length] = bitmap - self.counts[length] = sum(bin(byte).count("1") for byte in bitmap) + self.good_counts[length] = sum(bin(byte).count("1") for byte in bitmap) self.good = _LengthViews(self, True) self.bad = _LengthViews(self, False) def add_length(self, length): - bitmap = bytearray((factorial(length) + 7) // 8) - count = 0 + """Label the permutations of a length with the property, and keep + their labels as the length's bitmap.""" + bitmap = bytearray(_bitmap_size(length)) + good_count = 0 for i, perm in enumerate(Perm.of_length(length)): if self.prop(perm): bitmap[i >> 3] |= 1 << (i & 7) - count += 1 - self.bitmaps[length] = bitmap - self.counts[length] = count + good_count += 1 + self.bitmaps[length] = bytes(bitmap) + self.good_counts[length] = good_count class _LengthViews: @@ -395,12 +417,9 @@ def __iter__(self): yield perm def __len__(self): - good_count = self.labelled.counts[self.length] + good_count = self.labelled.good_counts[self.length] return good_count if self.wanted_bit else factorial(self.length) - good_count - def __bool__(self): - return len(self) > 0 - def create_bisc_input(N, prop): """ diff --git a/tests/bisc/test_labelled_perms.py b/tests/bisc/test_labelled_perms.py index 4e03f06..235e69f 100644 --- a/tests/bisc/test_labelled_perms.py +++ b/tests/bisc/test_labelled_perms.py @@ -7,96 +7,83 @@ from permuta.bisc.bisc import LabelledPerms, auto_bisc, bisc from permuta.patterns.perm import Perm +BASIS_231 = {3: {Perm((1, 2, 0)): [set()]}} -def stack_sortable(perm): - return perm.avoids(Perm((1, 2, 0))) - -def labelled(prop, max_length): - labels = LabelledPerms(prop) +def labelled_up_to(prop, max_length): + labelled_perms = LabelledPerms(prop) for length in range(max_length + 1): - labels.add_length(length) - return labels + labelled_perms.add_length(length) + return labelled_perms def test_views_match_lists(): - labels = labelled(stack_sortable, 6) + labelled_perms = labelled_up_to(Perm.stack_sortable, 6) for length in range(7): - good = [perm for perm in Perm.of_length(length) if stack_sortable(perm)] - bad = [perm for perm in Perm.of_length(length) if not stack_sortable(perm)] - assert list(labels.good[length]) == good - assert list(labels.bad[length]) == bad - assert len(labels.good[length]) == len(good) - assert len(labels.bad[length]) == len(bad) - assert bool(labels.bad[length]) == bool(bad) - assert sorted(labels.good.keys()) == list(range(7)) - assert 6 in labels.bad and 7 not in labels.bad + good = [perm for perm in Perm.of_length(length) if perm.stack_sortable()] + bad = [perm for perm in Perm.of_length(length) if not perm.stack_sortable()] + assert list(labelled_perms.good[length]) == good + assert list(labelled_perms.bad[length]) == bad + assert len(labelled_perms.good[length]) == len(good) + assert len(labelled_perms.bad[length]) == len(bad) + assert bool(labelled_perms.bad[length]) == bool(bad) + assert sorted(labelled_perms.good.keys()) == list(range(7)) + assert 6 in labelled_perms.bad and 7 not in labelled_perms.bad with pytest.raises(KeyError): - _ = labels.good[7] + _ = labelled_perms.good[7] def test_bit_order_is_perm_unrank(): - # Bit i of length n is the label of Perm.unrank(i, n), and the permutation - # perm is at bit perm.rank() minus the number of shorter permutations - labels = labelled(stack_sortable, 6) + # Bit i of a length's bitmap is the label of Perm.unrank(i, length), and the + # permutation perm is at bit perm.rank() minus the number of shorter + # permutations + labelled_perms = labelled_up_to(Perm.stack_sortable, 6) shorter_count = 0 for length in range(7): - bitmap = labels.bitmaps[length] + bitmap = labelled_perms.bitmaps[length] for i in range(factorial(length)): perm = Perm.unrank(i, length) assert perm.rank() - shorter_count == i - assert bool(bitmap[i >> 3] >> (i & 7) & 1) == stack_sortable(perm) + assert bool(bitmap[i >> 3] >> (i & 7) & 1) == perm.stack_sortable() shorter_count += factorial(length) def test_bitmaps_round_trip(): - labels = labelled(stack_sortable, 6) - round_trip = LabelledPerms(bitmaps=labels.bitmaps) + labelled_perms = labelled_up_to(Perm.stack_sortable, 6) + round_trip = LabelledPerms(bitmaps=labelled_perms.bitmaps) for length in range(7): - assert list(round_trip.good[length]) == list(labels.good[length]) - assert list(round_trip.bad[length]) == list(labels.bad[length]) + assert list(round_trip.good[length]) == list(labelled_perms.good[length]) + assert list(round_trip.bad[length]) == list(labelled_perms.bad[length]) with pytest.raises(ValueError): LabelledPerms(bitmaps={3: b"\x00\x00"}) # 3! bits fit in one byte def test_bisc_accepts_labelled_perms(): - expected = {3: {Perm((1, 2, 0)): [set()]}} - assert bisc(stack_sortable, 3, 5) == expected - assert bisc(labelled(stack_sortable, 5), 3, 5) == expected + assert bisc(Perm.stack_sortable, 3, 5) == BASIS_231 + assert bisc(labelled_up_to(Perm.stack_sortable, 5), 3, 5) == BASIS_231 def test_auto_bisc_labels_option(): - expected = {3: {Perm((1, 2, 0)): [set()]}} - assert auto_bisc(stack_sortable) == expected - assert auto_bisc(stack_sortable, labels="lists") == expected - assert auto_bisc(stack_sortable, labels="bitmaps") == expected + assert auto_bisc(Perm.stack_sortable) == BASIS_231 + assert auto_bisc(Perm.stack_sortable, labels="lists") == BASIS_231 + assert auto_bisc(Perm.stack_sortable, labels="bitmaps") == BASIS_231 with pytest.raises(ValueError): - auto_bisc(stack_sortable, labels="sets") + auto_bisc(Perm.stack_sortable, labels="sets") def test_auto_bisc_labelled_perms_input(): - expected = {3: {Perm((1, 2, 0)): [set()]}} - assert auto_bisc(labelled(stack_sortable, 8)) == expected + assert auto_bisc(labelled_up_to(Perm.stack_sortable, 8)) == BASIS_231 # Given bitmaps and no property: the lengths auto_bisc needs must be there - bitmaps = labelled(stack_sortable, 8).bitmaps - assert auto_bisc(LabelledPerms(bitmaps=bitmaps)) == expected + bitmaps = labelled_up_to(Perm.stack_sortable, 8).bitmaps + assert auto_bisc(LabelledPerms(bitmaps=bitmaps)) == BASIS_231 too_short = {length: bitmaps[length] for length in range(6)} assert auto_bisc(LabelledPerms(bitmaps=too_short)) is None def test_west_2_stack_sortable_same_as_lists(): - def west_2(perm): - def stack_sort(word): - output, stack = [], [] - for entry in word: - while stack and stack[-1] < entry: - output.append(stack.pop()) - stack.append(entry) - return output + stack[::-1] - - return stack_sort(stack_sort(list(perm))) == sorted(perm) - - assert auto_bisc(west_2, labels="bitmaps") == auto_bisc(west_2) + assert auto_bisc(Perm.west_2_stack_sortable, labels="bitmaps") == auto_bisc( + Perm.west_2_stack_sortable + ) def derangement_in_av_213_231_312(perm): @@ -109,7 +96,7 @@ def derangement_in_av_213_231_312(perm): def test_auto_bisc_stops_without_labels_it_needs(): # Bitmaps up to length 8 and no property: auto_bisc needs length 9 and stops - bitmaps = labelled(derangement_in_av_213_231_312, 8).bitmaps + bitmaps = labelled_up_to(derangement_in_av_213_231_312, 8).bitmaps output = io.StringIO() with contextlib.redirect_stdout(output): assert auto_bisc(LabelledPerms(bitmaps=bitmaps)) is None From 38c9dfab7b5a4579c551cd4cc76bc2fda78ccf4a Mon Sep 17 00:00:00 2001 From: Henning Ulfarsson Date: Thu, 22 Oct 2026 21:17:24 +0000 Subject: [PATCH 07/38] Reject malformed bitmaps in LabelledPerms From the review of #209, round 1, blocking finding 2. LabelledPerms took any bitmap of the right size: - set padding bits, past a length's permutations, were counted: bitmaps={3: bytes([0xFF])} gave good[3] 6 permutations but a length of 8, and len(bad[3]) raised; - an int passed, since bytes(15) is 15 zero bytes: bitmaps={5: 15} said that no permutation of length 5 is good. A bitmap must now be bytes, a bytearray or a memoryview (TypeError otherwise), and a bit set past the length's permutations is a ValueError. The good permutations are counted with int.bit_count. The tests from the review fail on the previous commit and pass now, and a new one checks that any bytes-like bitmap is accepted and kept as bytes. Co-Authored-By: Claude Opus 5.5 --- permuta/bisc/bisc.py | 17 +++++++++++++++-- tests/bisc/test_labelled_perms.py | 24 ++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/permuta/bisc/bisc.py b/permuta/bisc/bisc.py index 9008293..6e3fae5 100644 --- a/permuta/bisc/bisc.py +++ b/permuta/bisc/bisc.py @@ -345,7 +345,9 @@ class LabelledPerms: (bitmaps={length: bytes}), made from a property (add_length), or both: auto_bisc labels the lengths it needs and is not given with prop, adding them to this object, and stops, as for a tuple of dictionaries, when - there is no prop. + there is no prop. A bitmap given must be bytes-like (bytes, a bytearray + or a memoryview), of its length's size, with no bit set past the length's + permutations. auto_bisc reads its good and bad permutations only length by length: iterating over a length, counting it, asking whether a length is there. @@ -360,13 +362,24 @@ def __init__(self, prop=None, bitmaps=None): self.bitmaps = {} self.good_counts = {} for length, bitmap in (bitmaps or {}).items(): + if not isinstance(bitmap, (bytes, bytearray, memoryview)): + raise TypeError( + "A bitmap must be bytes, a bytearray or a memoryview, " + "not {}".format(type(bitmap).__name__) + ) bitmap = bytes(bitmap) if len(bitmap) != _bitmap_size(length): raise ValueError( "A bitmap of length {} has the wrong size".format(length) ) + label_bits = int.from_bytes(bitmap, "little") + if label_bits >> factorial(length): + raise ValueError( + "A bitmap of length {} has bits set past its {} " + "permutations".format(length, factorial(length)) + ) self.bitmaps[length] = bitmap - self.good_counts[length] = sum(bin(byte).count("1") for byte in bitmap) + self.good_counts[length] = label_bits.bit_count() self.good = _LengthViews(self, True) self.bad = _LengthViews(self, False) diff --git a/tests/bisc/test_labelled_perms.py b/tests/bisc/test_labelled_perms.py index 235e69f..56761ad 100644 --- a/tests/bisc/test_labelled_perms.py +++ b/tests/bisc/test_labelled_perms.py @@ -58,6 +58,30 @@ def test_bitmaps_round_trip(): LabelledPerms(bitmaps={3: b"\x00\x00"}) # 3! bits fit in one byte +@pytest.mark.parametrize( + "length, bitmap", + [(0, b"\x03"), (1, b"\x81"), (2, b"\x04"), (3, b"\xff"), (3, b"\x40")], +) +def test_set_padding_bits_are_rejected(length, bitmap): + with pytest.raises(ValueError): + LabelledPerms(bitmaps={length: bitmap}) + + +@pytest.mark.parametrize("length, bitmap", [(0, 1), (3, 1), (4, 3), (5, 15)]) +def test_an_int_is_not_a_bitmap(length, bitmap): + with pytest.raises(TypeError): + LabelledPerms(bitmaps={length: bitmap}) + + +def test_bitmaps_may_be_any_bytes_like(): + # All six permutations of length 3 are good, and the first of length 2 + for bitmap in (b"\x3f", bytearray(b"\x3f"), memoryview(b"\x3f")): + labelled_perms = LabelledPerms(bitmaps={3: bitmap, 2: b"\x01"}) + assert len(labelled_perms.good[3]) == 6 and not labelled_perms.bad[3] + assert list(labelled_perms.good[2]) == [Perm((0, 1))] + assert isinstance(labelled_perms.bitmaps[3], bytes) + + def test_bisc_accepts_labelled_perms(): assert bisc(Perm.stack_sortable, 3, 5) == BASIS_231 assert bisc(labelled_up_to(Perm.stack_sortable, 5), 3, 5) == BASIS_231 From 58e57db92f9d504ca8601ea00b8827878f557f91 Mon Sep 17 00:00:00 2001 From: Henning Ulfarsson Date: Thu, 22 Oct 2026 21:18:09 +0000 Subject: [PATCH 08/38] Test the labels option and LabelledPerms given a property and bitmaps From the review of #209, round 1, blocking findings 1 and 3: - No test checked that labels="bitmaps" keeps bitmaps: the results are the same in both modes, so deleting the line that wraps the property in LabelledPerms passed every test. test_labels_option_selects_the_ representation records the lengths added as bitmaps: none for lists, 0 to 8 for bitmaps. It fails when that line is deleted. - The docstring, README and CHANGELOG offer labels made from a property, given as bitmaps, or both, and no test built both. The new test gives bitmaps of lengths 0, 1, 2, 3, 5 and 8, made by an encoder of its own, and checks that auto_bisc labels only lengths 4, 6 and 7 with the property. Co-Authored-By: Claude Opus 5.5 --- tests/bisc/test_labelled_perms.py | 46 +++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/tests/bisc/test_labelled_perms.py b/tests/bisc/test_labelled_perms.py index 56761ad..3ae8086 100644 --- a/tests/bisc/test_labelled_perms.py +++ b/tests/bisc/test_labelled_perms.py @@ -95,6 +95,24 @@ def test_auto_bisc_labels_option(): auto_bisc(Perm.stack_sortable, labels="sets") +def test_labels_option_selects_the_representation(monkeypatch): + # The results are the same either way, so record which lengths are kept as + # bitmaps: none for lists, every length checked for bitmaps + added_lengths = [] + real_add_length = LabelledPerms.add_length + + def recording_add_length(self, length): + added_lengths.append(length) + return real_add_length(self, length) + + monkeypatch.setattr(LabelledPerms, "add_length", recording_add_length) + assert auto_bisc(Perm.stack_sortable) == BASIS_231 + assert auto_bisc(Perm.stack_sortable, labels="lists") == BASIS_231 + assert added_lengths == [] + assert auto_bisc(Perm.stack_sortable, labels="bitmaps") == BASIS_231 + assert added_lengths == list(range(9)) + + def test_auto_bisc_labelled_perms_input(): assert auto_bisc(labelled_up_to(Perm.stack_sortable, 8)) == BASIS_231 # Given bitmaps and no property: the lengths auto_bisc needs must be there @@ -104,6 +122,34 @@ def test_auto_bisc_labelled_perms_input(): assert auto_bisc(LabelledPerms(bitmaps=too_short)) is None +def encode(prop, length): + """A length's bitmap made without LabelledPerms: bit i of the little-endian + integer is the label of Perm.unrank(i, length).""" + label_bits = sum( + 1 << i for i in range(factorial(length)) if prop(Perm.unrank(i, length)) + ) + return label_bits.to_bytes((factorial(length) + 7) // 8, "little") + + +def test_auto_bisc_labels_only_missing_lengths_with_the_property(): + labelled_lengths = [] + + def prop(perm): + labelled_lengths.append(len(perm)) + return perm.stack_sortable() + + given_lengths = (0, 1, 2, 3, 5, 8) + labelled_perms = LabelledPerms( + prop, + bitmaps={ + length: encode(Perm.stack_sortable, length) for length in given_lengths + }, + ) + assert auto_bisc(labelled_perms) == BASIS_231 + assert set(labelled_lengths) == {4, 6, 7} + assert sorted(labelled_perms.bitmaps) == list(range(9)) + + def test_west_2_stack_sortable_same_as_lists(): assert auto_bisc(Perm.west_2_stack_sortable, labels="bitmaps") == auto_bisc( Perm.west_2_stack_sortable From a5268a9d00fb5a856849ef4c47a71fc87a9ef1b8 Mon Sep 17 00:00:00 2001 From: Henning Ulfarsson Date: Thu, 22 Oct 2026 21:19:39 +0000 Subject: [PATCH 09/38] bisc labels what a LabelledPerms lacks, and labels="bitmaps" needs a property From the review of #209, round 1, suggestions 1, 3 and 4: - bisc given a LabelledPerms did not use its property: with fewer lengths labelled than n it raised a bare KeyError from mine(), and with n left out on a new LabelledPerms a ValueError from max(). It now labels the lengths up to n that are missing with the property, adding them to the LabelledPerms as auto_bisc does. Without n it uses the longest length labelled, or 7, as for a property. It raises a ValueError that says what is missing when there is no property to label with. bisc has a docstring. - auto_bisc(prop, labels="bitmaps") silently ignored the option for a list, a tuple of dictionaries or a file name. It now raises a ValueError; a property or a LabelledPerms is accepted as before. - auto_bisc's message for an input it does not take lists LabelledPerms, as bisc's does. The three new tests fail on the previous commit and pass now. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 ++- permuta/bisc/bisc.py | 36 ++++++++++++++++++++++++++++--- tests/bisc/test_labelled_perms.py | 27 +++++++++++++++++++++++ 3 files changed, 62 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index db41f0b..a7ceeb3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added - `LabelledPerms` in `permuta.bisc`: labels of permutations kept as one bitmap per length (lexicographic order), made from a property, given as bitmaps, or both. - `auto_bisc` and `bisc` accept it as input. + `auto_bisc` and `bisc` accept it as input, and label the lengths it lacks with + its property. - `auto_bisc(prop, labels="bitmaps")`: keeps a property's labels as bitmaps instead of lists of permutations, for the same result in far less memory at longer lengths (the permutations of length 10 take about 0.9 GB as lists, 454 KB as a diff --git a/permuta/bisc/bisc.py b/permuta/bisc/bisc.py index 6e3fae5..e00914e 100644 --- a/permuta/bisc/bisc.py +++ b/permuta/bisc/bisc.py @@ -20,6 +20,14 @@ def bisc(A, m, n=None, report=False): + """ + Mine the forbidden patterns of length up to m from the permutations up to + length n of a set A: a list of permutations, a property (a function), a + dictionary (length -> the permutations of that length in the set), or a + LabelledPerms. Without n, a property uses length 7, and the others their + longest length. A LabelledPerms's lengths up to n that are not labelled + are labelled with its property, and added to it. + """ if isinstance(A, list): D = defaultdict(list) for perm in A: @@ -38,6 +46,22 @@ def bisc(A, m, n=None, report=False): D = A elif isinstance(A, LabelledPerms): + if n is None: + if A.bitmaps: + n = max(A.bitmaps) + elif A.prop is not None: + print("I will use permutations up to length 7") + n = 7 + else: + raise ValueError("The LabelledPerms has no labels and no property") + for length in range(n + 1): + if length not in A.bitmaps: + if A.prop is None: + raise ValueError( + "bisc needs labels up to length {}, and the LabelledPerms " + "has none of length {} and no property".format(n, length) + ) + A.add_length(length) D = A.good elif isinstance(A, _LengthViews): @@ -88,8 +112,13 @@ def auto_bisc(prop, labels="lists"): """ if labels not in ("lists", "bitmaps"): raise ValueError('labels must be "lists" or "bitmaps"') - if labels == "bitmaps" and isinstance(prop, types.FunctionType): - prop = LabelledPerms(prop) + if labels == "bitmaps": + if isinstance(prop, types.FunctionType): + prop = LabelledPerms(prop) + elif not isinstance(prop, LabelledPerms): + raise ValueError( + 'labels="bitmaps" needs a property (a function) or a LabelledPerms' + ) L = 8 # Want to sanity check on at least S8, and one above n n = 4 m = 2 @@ -188,7 +217,8 @@ def auto_bisc(prop, labels="lists"): 1. A list of permutations 2. A property of permutations 3. A dictionary of lengths pointing to permutations - 4. A string """ + 4. A string + 5. A LabelledPerms (labels kept as bitmaps)""" ) assert False diff --git a/tests/bisc/test_labelled_perms.py b/tests/bisc/test_labelled_perms.py index 3ae8086..bb537cf 100644 --- a/tests/bisc/test_labelled_perms.py +++ b/tests/bisc/test_labelled_perms.py @@ -150,6 +150,33 @@ def prop(perm): assert sorted(labelled_perms.bitmaps) == list(range(9)) +def test_bisc_labels_missing_lengths_with_the_property(): + # Without n, bisc uses the longest length labelled, or 7 when there is none + assert bisc(labelled_up_to(Perm.stack_sortable, 5), 3) == BASIS_231 + labelled_perms = LabelledPerms(Perm.stack_sortable) + assert bisc(labelled_perms, 3, 5) == BASIS_231 + assert sorted(labelled_perms.bitmaps) == list(range(6)) + labelled_perms = LabelledPerms(Perm.stack_sortable) + assert bisc(labelled_perms, 3) == BASIS_231 + assert sorted(labelled_perms.bitmaps) == list(range(8)) + + +def test_bisc_needs_labels_or_a_property(): + bitmaps = {length: encode(Perm.stack_sortable, length) for length in range(4)} + with pytest.raises(ValueError): + bisc(LabelledPerms(bitmaps=bitmaps), 3, 5) + with pytest.raises(ValueError): + bisc(LabelledPerms(), 3) + + +def test_bitmaps_need_a_property(): + for prop in ([Perm((0,))], ({}, {}), "yt_perm_avoids_22"): + with pytest.raises(ValueError): + auto_bisc(prop, labels="bitmaps") + labelled_perms = labelled_up_to(Perm.stack_sortable, 8) + assert auto_bisc(labelled_perms, labels="bitmaps") == BASIS_231 + + def test_west_2_stack_sortable_same_as_lists(): assert auto_bisc(Perm.west_2_stack_sortable, labels="bitmaps") == auto_bisc( Perm.west_2_stack_sortable From a8b80bf290a3da1f9e4dd394d5f54c3cac7347e0 Mon Sep 17 00:00:00 2001 From: Henning Ulfarsson Date: Thu, 22 Oct 2026 21:46:46 +0000 Subject: [PATCH 10/38] Test how bisc treats a LabelledPerms, and label up to max(m, n) From the review of #209, round 2: - Blocking: no test checked that bisc chooses n from the bitmaps, or that it labels only the missing lengths; two faults made by hand passed every test. The review's two tests are added. They pass now and fail on both faults. - Recommended point 1: mine() reads the lengths up to m, so when m > n bisc raised a bare KeyError for a LabelledPerms, such as bisc(LabelledPerms(Perm.stack_sortable), 3, 0). bisc now labels up to max(m, n). Without a property it raises the ValueError that says which length is missing. A new test fails on the previous commit and passes now. Co-Authored-By: Claude Opus 5.5 --- permuta/bisc/bisc.py | 9 +++++--- tests/bisc/test_labelled_perms.py | 38 +++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/permuta/bisc/bisc.py b/permuta/bisc/bisc.py index e00914e..e03dfca 100644 --- a/permuta/bisc/bisc.py +++ b/permuta/bisc/bisc.py @@ -25,7 +25,8 @@ def bisc(A, m, n=None, report=False): length n of a set A: a list of permutations, a property (a function), a dictionary (length -> the permutations of that length in the set), or a LabelledPerms. Without n, a property uses length 7, and the others their - longest length. A LabelledPerms's lengths up to n that are not labelled + longest length. mine() reads the lengths up to m and the permutations up + to n, so a LabelledPerms's lengths up to max(m, n) that are not labelled are labelled with its property, and added to it. """ if isinstance(A, list): @@ -54,12 +55,14 @@ def bisc(A, m, n=None, report=False): n = 7 else: raise ValueError("The LabelledPerms has no labels and no property") - for length in range(n + 1): + for length in range(max(m, n) + 1): if length not in A.bitmaps: if A.prop is None: raise ValueError( "bisc needs labels up to length {}, and the LabelledPerms " - "has none of length {} and no property".format(n, length) + "has none of length {} and no property".format( + max(m, n), length + ) ) A.add_length(length) D = A.good diff --git a/tests/bisc/test_labelled_perms.py b/tests/bisc/test_labelled_perms.py index bb537cf..664c84e 100644 --- a/tests/bisc/test_labelled_perms.py +++ b/tests/bisc/test_labelled_perms.py @@ -169,6 +169,44 @@ def test_bisc_needs_labels_or_a_property(): bisc(LabelledPerms(), 3) +def test_bisc_without_n_uses_the_longest_labelled_length_with_no_property(): + bitmaps = {length: encode(Perm.stack_sortable, length) for length in range(6)} + assert bisc(LabelledPerms(bitmaps=bitmaps), 3) == BASIS_231 + + +def test_bisc_labels_only_the_missing_lengths(): + labelled_lengths = [] + + def prop(perm): + labelled_lengths.append(len(perm)) + return perm.stack_sortable() + + given_lengths = (0, 1, 2, 3, 5) + labelled_perms = LabelledPerms( + prop, + bitmaps={ + length: encode(Perm.stack_sortable, length) for length in given_lengths + }, + ) + assert bisc(labelled_perms, 3) == BASIS_231 + assert labelled_lengths == [4] * factorial(4) + assert sorted(labelled_perms.bitmaps) == list(range(6)) + assert bisc(labelled_perms, 3, 5) == BASIS_231 + assert labelled_lengths == [4] * factorial(4) + + +def test_bisc_labels_up_to_the_pattern_length(): + # mine() reads the lengths up to m, so bisc labels up to m when n < m + labelled_perms = LabelledPerms(Perm.stack_sortable) + bisc(labelled_perms, 3, 0) + assert sorted(labelled_perms.bitmaps) == [0, 1, 2, 3] + labelled_perms = LabelledPerms(Perm.stack_sortable, bitmaps={0: b"\x01"}) + bisc(labelled_perms, 3) + assert sorted(labelled_perms.bitmaps) == [0, 1, 2, 3] + with pytest.raises(ValueError): + bisc(LabelledPerms(bitmaps={0: b"\x01"}), 3) + + def test_bitmaps_need_a_property(): for prop in ([Perm((0,))], ({}, {}), "yt_perm_avoids_22"): with pytest.raises(ValueError): From 69057c8fc4b7e94f9439e726a4598c90cfeeffe1 Mon Sep 17 00:00:00 2001 From: Henning Ulfarsson Date: Thu, 22 Oct 2026 21:47:23 +0000 Subject: [PATCH 11/38] Correct the memory figures in the docs From the review of #209, round 2, recommended points 2 and 3: - auto_bisc's docstring said that from length 10 on "the bitmaps [take] under a megabyte". A bitmap is 454 KB at length 10, but 5 MB at length 11 and 60 MB at length 12; the docstring now gives these. - The README, the CHANGELOG and LabelledPerms' docstring said the permutations of length 10 take about 0.9 GB as lists. Building list(Perm.of_length(10)) grows a process by 0.88 GB on PyPy 3.11, 1.26 GB on CPython 3.11 (measured here) and 1.8 GB on CPython 3.14 (the review), so they now say 0.9 to 1.8 GB, depending on the Python. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 5 +++-- README.rst | 8 ++++---- permuta/bisc/bisc.py | 8 +++++--- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7ceeb3..530c085 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,8 +13,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. its property. - `auto_bisc(prop, labels="bitmaps")`: keeps a property's labels as bitmaps instead of lists of permutations, for the same result in far less memory at longer - lengths (the permutations of length 10 take about 0.9 GB as lists, 454 KB as a - bitmap). `labels="lists"`, the default, is the previous behaviour. + lengths (the permutations of length 10 take 0.9 to 1.8 GB as lists of `Perm` + objects, depending on the Python, and 454 KB as a bitmap). `labels="lists"`, + the default, is the previous behaviour. ### Fixed - Fixed bug in `Perm.rtlmax_ltrmin_decomposition` and fixed associated tests. diff --git a/README.rst b/README.rst index 7b934c2..bc3ab2b 100644 --- a/README.rst +++ b/README.rst @@ -522,10 +522,10 @@ a list of permutations (satisfying some property), or a ``LabelledPerms``. not, as lists. For long runs it can keep them as one bitmap per length instead, with the same result in far less memory: ``auto_bisc(Perm.stack_sortable, labels="bitmaps")``. Use it when ``auto_bisc`` -has to check long permutations: the permutations of length 10 take about 0.9 GB -as lists, and 454 KB as a bitmap. Such labels can also be given directly as a -``LabelledPerms``, made from a property, from bitmaps, or both, and ``bisc`` -accepts it too. +has to check long permutations: the permutations of length 10 take 0.9 to 1.8 GB +as lists of ``Perm`` objects, depending on the Python, and 454 KB as a bitmap. +Such labels can also be given directly as a ``LabelledPerms``, made from a +property, from bitmaps, or both, and ``bisc`` accepts it too. .. code-block:: python diff --git a/permuta/bisc/bisc.py b/permuta/bisc/bisc.py index e03dfca..7bb93de 100644 --- a/permuta/bisc/bisc.py +++ b/permuta/bisc/bisc.py @@ -107,8 +107,10 @@ def auto_bisc(prop, labels="lists"): each length as lists of Perm objects; "bitmaps": one bitmap per length (LabelledPerms), for the same result in far less memory, at the cost of generating a length's permutations again - each time it is read. Use it when the permutations checked are long: from - length 10 on the lists take gigabytes, the bitmaps under a megabyte. + each time it is read. Use it when the permutations checked are long: those + of length 10 take 0.9 to 1.8 GB as lists of Perm objects, depending on the + Python, and 454 KB as a bitmap, a bit for each permutation (5 MB at length + 11, 60 MB at length 12). A LabelledPerms can also be given as prop directly. The lengths auto_bisc labels with its property are added to it, so a later run reuses them. @@ -387,7 +389,7 @@ class LabelledPerms: The views good and bad do that from the bitmaps, generating a length's permutations again each time it is iterated, in the same order as the lists a property input makes. A bitmap of S_10 is 454 KB; S_10 as a list - of Perm objects is about 0.9 GB. + of Perm objects takes 0.9 GB on PyPy 3.11 and 1.3 to 1.8 GB on CPython. """ def __init__(self, prop=None, bitmaps=None): From c54c54200bf2b386c4894287a8f3527cc9538c56 Mon Sep 17 00:00:00 2001 From: Henning Ulfarsson Date: Thu, 22 Oct 2026 21:50:15 +0000 Subject: [PATCH 12/38] Docstrings, messages and edge-case tests for LabelledPerms From the review of #209, round 2, its optional points, but one: - auto_bisc's message for an input it does not take says "a tuple of two dictionaries", which is what it takes, where it said "a dictionary". - Docstrings: LabelledPerms' says that bisc labels missing lengths too, and that bisc raises where auto_bisc stops; bisc's says what it returns and what report does; add_length's says that it needs a property and replaces a length's bitmap. - Every pytest.raises names its message with match=, so a changed message fails a test. - New tests: lengths 0 and 1 labelled bad, an empty bitmap (too short), and bisc given a LabelledPerms's view. Not taken: one method for the three loops that label missing lengths, which, like making _LengthViews a Mapping, is left for later. Co-Authored-By: Claude Opus 5.5 --- permuta/bisc/bisc.py | 22 +++++++++++++-------- tests/bisc/test_labelled_perms.py | 33 ++++++++++++++++++++++--------- 2 files changed, 38 insertions(+), 17 deletions(-) diff --git a/permuta/bisc/bisc.py b/permuta/bisc/bisc.py index 7bb93de..0b0495c 100644 --- a/permuta/bisc/bisc.py +++ b/permuta/bisc/bisc.py @@ -27,7 +27,12 @@ def bisc(A, m, n=None, report=False): LabelledPerms. Without n, a property uses length 7, and the others their longest length. mine() reads the lengths up to m and the permutations up to n, so a LabelledPerms's lengths up to max(m, n) that are not labelled - are labelled with its property, and added to it. + are labelled with its property, and added to it. report prints the + progress of the search. + + Returns the forbidden patterns found: a dictionary from length to + {pattern: [shadings]}, where each shading is a set of boxes, and an empty + one makes the pattern classical. """ if isinstance(A, list): D = defaultdict(list) @@ -221,7 +226,7 @@ def auto_bisc(prop, labels="lists"): """BiSC can accept inputs: 1. A list of permutations 2. A property of permutations - 3. A dictionary of lengths pointing to permutations + 3. A tuple of two dictionaries of lengths pointing to permutations 4. A string 5. A LabelledPerms (labels kept as bitmaps)""" ) @@ -378,11 +383,11 @@ class LabelledPerms: order in which Perm.of_length(length) yields the permutations (lexicographic), which the views use to iterate. The bitmaps can be given (bitmaps={length: bytes}), made from a property (add_length), or both: - auto_bisc labels the lengths it needs and is not given with prop, adding - them to this object, and stops, as for a tuple of dictionaries, when - there is no prop. A bitmap given must be bytes-like (bytes, a bytearray - or a memoryview), of its length's size, with no bit set past the length's - permutations. + auto_bisc and bisc label the lengths they need and are not given with + prop, adding them to this object. Without a prop, auto_bisc stops, as for + a tuple of dictionaries, and bisc raises a ValueError. A bitmap given must + be bytes-like (bytes, a bytearray or a memoryview), of its length's size, + with no bit set past the length's permutations. auto_bisc reads its good and bad permutations only length by length: iterating over a length, counting it, asking whether a length is there. @@ -420,7 +425,8 @@ def __init__(self, prop=None, bitmaps=None): def add_length(self, length): """Label the permutations of a length with the property, and keep - their labels as the length's bitmap.""" + their labels as the length's bitmap. It needs a property, and replaces + the length's bitmap if there is one.""" bitmap = bytearray(_bitmap_size(length)) good_count = 0 for i, perm in enumerate(Perm.of_length(length)): diff --git a/tests/bisc/test_labelled_perms.py b/tests/bisc/test_labelled_perms.py index 664c84e..b3a7380 100644 --- a/tests/bisc/test_labelled_perms.py +++ b/tests/bisc/test_labelled_perms.py @@ -29,7 +29,7 @@ def test_views_match_lists(): assert bool(labelled_perms.bad[length]) == bool(bad) assert sorted(labelled_perms.good.keys()) == list(range(7)) assert 6 in labelled_perms.bad and 7 not in labelled_perms.bad - with pytest.raises(KeyError): + with pytest.raises(KeyError, match="7"): _ = labelled_perms.good[7] @@ -54,8 +54,19 @@ def test_bitmaps_round_trip(): for length in range(7): assert list(round_trip.good[length]) == list(labelled_perms.good[length]) assert list(round_trip.bad[length]) == list(labelled_perms.bad[length]) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="wrong size"): LabelledPerms(bitmaps={3: b"\x00\x00"}) # 3! bits fit in one byte + with pytest.raises(ValueError, match="wrong size"): + LabelledPerms(bitmaps={3: b""}) + + +def test_lengths_0_and_1_labelled_bad(): + labelled_perms = LabelledPerms(bitmaps={0: b"\x00", 1: b"\x00"}) + for length in (0, 1): + assert list(labelled_perms.good[length]) == [] + assert list(labelled_perms.bad[length]) == [Perm(range(length))] + assert len(labelled_perms.good[length]) == 0 + assert len(labelled_perms.bad[length]) == 1 @pytest.mark.parametrize( @@ -63,13 +74,13 @@ def test_bitmaps_round_trip(): [(0, b"\x03"), (1, b"\x81"), (2, b"\x04"), (3, b"\xff"), (3, b"\x40")], ) def test_set_padding_bits_are_rejected(length, bitmap): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="bits set past"): LabelledPerms(bitmaps={length: bitmap}) @pytest.mark.parametrize("length, bitmap", [(0, 1), (3, 1), (4, 3), (5, 15)]) def test_an_int_is_not_a_bitmap(length, bitmap): - with pytest.raises(TypeError): + with pytest.raises(TypeError, match="must be bytes"): LabelledPerms(bitmaps={length: bitmap}) @@ -91,7 +102,7 @@ def test_auto_bisc_labels_option(): assert auto_bisc(Perm.stack_sortable) == BASIS_231 assert auto_bisc(Perm.stack_sortable, labels="lists") == BASIS_231 assert auto_bisc(Perm.stack_sortable, labels="bitmaps") == BASIS_231 - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="labels must be"): auto_bisc(Perm.stack_sortable, labels="sets") @@ -163,9 +174,9 @@ def test_bisc_labels_missing_lengths_with_the_property(): def test_bisc_needs_labels_or_a_property(): bitmaps = {length: encode(Perm.stack_sortable, length) for length in range(4)} - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="none of length 4 and no property"): bisc(LabelledPerms(bitmaps=bitmaps), 3, 5) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="no labels and no property"): bisc(LabelledPerms(), 3) @@ -203,13 +214,17 @@ def test_bisc_labels_up_to_the_pattern_length(): labelled_perms = LabelledPerms(Perm.stack_sortable, bitmaps={0: b"\x01"}) bisc(labelled_perms, 3) assert sorted(labelled_perms.bitmaps) == [0, 1, 2, 3] - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="none of length 1 and no property"): bisc(LabelledPerms(bitmaps={0: b"\x01"}), 3) +def test_bisc_accepts_the_views(): + assert bisc(labelled_up_to(Perm.stack_sortable, 5).good, 3) == BASIS_231 + + def test_bitmaps_need_a_property(): for prop in ([Perm((0,))], ({}, {}), "yt_perm_avoids_22"): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="needs a property"): auto_bisc(prop, labels="bitmaps") labelled_perms = labelled_up_to(Perm.stack_sortable, 8) assert auto_bisc(labelled_perms, labels="bitmaps") == BASIS_231 From 7ade65afd7135acd160f0fdf7c0c92ae1a34b50b Mon Sep 17 00:00:00 2001 From: Henning Ulfarsson Date: Fri, 23 Oct 2026 15:01:04 +0000 Subject: [PATCH 13/38] Rename LabelledPerms to PropertyBitmaps From the review of #209, round 3, Henning's point 2: "Labelled permutations doesn't feel like the right name for the bitmap." The class holds, for each length, one bit per permutation: whether it has the property. PropertyBitmaps, the review's suggestion, is Henning's choice. Renamed throughout, with no change in behavior: bisc.py, permuta.bisc's imports and __all__, the README, the CHANGELOG, and the tests, whose file is now tests/bisc/test_property_bitmaps.py. A variable holding one is property_bitmaps, as is the views' reference to it, and the tests' helper is property_bitmaps_up_to. Words about labels and labelling stay, as does labels="bitmaps". Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 +- README.rst | 12 +- permuta/bisc/__init__.py | 4 +- permuta/bisc/bisc.py | 60 ++++---- ...lled_perms.py => test_property_bitmaps.py} | 144 +++++++++--------- 5 files changed, 111 insertions(+), 111 deletions(-) rename tests/bisc/{test_labelled_perms.py => test_property_bitmaps.py} (60%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 530c085..a550e55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## Unreleased ### Added -- `LabelledPerms` in `permuta.bisc`: labels of permutations kept as one bitmap +- `PropertyBitmaps` in `permuta.bisc`: labels of permutations kept as one bitmap per length (lexicographic order), made from a property, given as bitmaps, or both. `auto_bisc` and `bisc` accept it as input, and label the lengths it lacks with its property. diff --git a/README.rst b/README.rst index bc3ab2b..128ad0d 100644 --- a/README.rst +++ b/README.rst @@ -516,7 +516,7 @@ permutations that can be imported from ``permuta.bisc.perm_properties``, such as ``smooth``, ``forest-like``, ``baxter``, ``simsun``, ``quick_sortable``, etc. Both ``bisc`` and ``auto_bisc`` can accept input in the form of a property, -a list of permutations (satisfying some property), or a ``LabelledPerms``. +a list of permutations (satisfying some property), or a ``PropertyBitmaps``. ``auto_bisc`` keeps the permutations satisfying a property, and those that do not, as lists. For long runs it can keep them as one bitmap per length instead, @@ -524,17 +524,17 @@ with the same result in far less memory: ``auto_bisc(Perm.stack_sortable, labels="bitmaps")``. Use it when ``auto_bisc`` has to check long permutations: the permutations of length 10 take 0.9 to 1.8 GB as lists of ``Perm`` objects, depending on the Python, and 454 KB as a bitmap. -Such labels can also be given directly as a ``LabelledPerms``, made from a +Such labels can also be given directly as a ``PropertyBitmaps``, made from a property, from bitmaps, or both, and ``bisc`` accepts it too. .. code-block:: python - >>> labelled_perms = LabelledPerms(Perm.stack_sortable) + >>> property_bitmaps = PropertyBitmaps(Perm.stack_sortable) >>> for length in range(6): - ... labelled_perms.add_length(length) - >>> len(labelled_perms.good[5]), len(labelled_perms.bad[5]) + ... property_bitmaps.add_length(length) + >>> len(property_bitmaps.good[5]), len(property_bitmaps.bad[5]) (42, 78) - >>> bisc(labelled_perms, 3, 5) + >>> bisc(property_bitmaps, 3, 5) {3: {Perm((1, 2, 0)): [set()]}} License diff --git a/permuta/bisc/__init__.py b/permuta/bisc/__init__.py index acb5b85..6653d8f 100644 --- a/permuta/bisc/__init__.py +++ b/permuta/bisc/__init__.py @@ -1,7 +1,7 @@ # type: ignore from .bisc import ( - LabelledPerms, + PropertyBitmaps, auto_bisc, bisc, create_bisc_input, @@ -17,7 +17,7 @@ ) __all__ = [ - "LabelledPerms", + "PropertyBitmaps", "auto_bisc", "bisc", "create_bisc_input", diff --git a/permuta/bisc/bisc.py b/permuta/bisc/bisc.py index 0b0495c..6c01467 100644 --- a/permuta/bisc/bisc.py +++ b/permuta/bisc/bisc.py @@ -24,11 +24,11 @@ def bisc(A, m, n=None, report=False): Mine the forbidden patterns of length up to m from the permutations up to length n of a set A: a list of permutations, a property (a function), a dictionary (length -> the permutations of that length in the set), or a - LabelledPerms. Without n, a property uses length 7, and the others their + PropertyBitmaps. Without n, a property uses length 7, and the others their longest length. mine() reads the lengths up to m and the permutations up - to n, so a LabelledPerms's lengths up to max(m, n) that are not labelled - are labelled with its property, and added to it. report prints the - progress of the search. + to n, so the lengths up to max(m, n) that a PropertyBitmaps lacks are + labelled with its property, and added to it. report prints the progress + of the search. Returns the forbidden patterns found: a dictionary from length to {pattern: [shadings]}, where each shading is a set of boxes, and an empty @@ -51,7 +51,7 @@ def bisc(A, m, n=None, report=False): elif isinstance(A, dict): D = A - elif isinstance(A, LabelledPerms): + elif isinstance(A, PropertyBitmaps): if n is None: if A.bitmaps: n = max(A.bitmaps) @@ -59,12 +59,12 @@ def bisc(A, m, n=None, report=False): print("I will use permutations up to length 7") n = 7 else: - raise ValueError("The LabelledPerms has no labels and no property") + raise ValueError("The PropertyBitmaps has no labels and no property") for length in range(max(m, n) + 1): if length not in A.bitmaps: if A.prop is None: raise ValueError( - "bisc needs labels up to length {}, and the LabelledPerms " + "bisc needs labels up to length {}, and the PropertyBitmaps " "has none of length {} and no property".format( max(m, n), length ) @@ -81,7 +81,7 @@ def bisc(A, m, n=None, report=False): 1. A list of permutations 2. A property of permutations 3. A dictionary of lengths pointing to permutations - 4. A LabelledPerms (labels kept as bitmaps)""" + 4. A PropertyBitmaps (labels kept as bitmaps)""" ) assert False @@ -105,29 +105,29 @@ def auto_bisc(prop, labels="lists"): prop is the set: a property of permutations (a function), a list of permutations, a tuple of two dictionaries (length -> the permutations in the set, and those not in it), the name of a set in - permuta/resources/bisc, or a LabelledPerms. + permuta/resources/bisc, or a PropertyBitmaps. labels says how the labels of a property are kept: "lists" (the default, as before): the good and the bad permutations of each length as lists of Perm objects; - "bitmaps": one bitmap per length (LabelledPerms), for the same result in + "bitmaps": one bitmap per length (PropertyBitmaps), for the same result in far less memory, at the cost of generating a length's permutations again each time it is read. Use it when the permutations checked are long: those of length 10 take 0.9 to 1.8 GB as lists of Perm objects, depending on the Python, and 454 KB as a bitmap, a bit for each permutation (5 MB at length 11, 60 MB at length 12). - A LabelledPerms can also be given as prop directly. The lengths auto_bisc + A PropertyBitmaps can also be given as prop directly. The lengths auto_bisc labels with its property are added to it, so a later run reuses them. """ if labels not in ("lists", "bitmaps"): raise ValueError('labels must be "lists" or "bitmaps"') if labels == "bitmaps": if isinstance(prop, types.FunctionType): - prop = LabelledPerms(prop) - elif not isinstance(prop, LabelledPerms): + prop = PropertyBitmaps(prop) + elif not isinstance(prop, PropertyBitmaps): raise ValueError( - 'labels="bitmaps" needs a property (a function) or a LabelledPerms' + 'labels="bitmaps" needs a property (a function) or a PropertyBitmaps' ) L = 8 # Want to sanity check on at least S8, and one above n n = 4 @@ -162,11 +162,11 @@ def auto_bisc(prop, labels="lists"): else: B[i].append(perm) - elif isinstance(prop, LabelledPerms): + elif isinstance(prop, PropertyBitmaps): # Labels kept as one bitmap per length: A and B are views of the good # and the bad permutations, generated length by length when they are - # read, instead of lists. Lengths not given are labelled with the - # LabelledPerms's property, if it has one + # read, instead of lists. The lengths the PropertyBitmaps lacks are + # labelled with its property, if it has one for i in range(L + 1): if i not in prop.bitmaps: if prop.prop is None: @@ -228,7 +228,7 @@ def auto_bisc(prop, labels="lists"): 2. A property of permutations 3. A tuple of two dictionaries of lengths pointing to permutations 4. A string - 5. A LabelledPerms (labels kept as bitmaps)""" + 5. A PropertyBitmaps (labels kept as bitmaps)""" ) assert False @@ -317,7 +317,7 @@ def auto_bisc(prop, labels="lists"): else: B[i].append(perm) - elif isinstance(prop, LabelledPerms): + elif isinstance(prop, PropertyBitmaps): for i in range(oldL + 1, L + 1): if i not in prop.bitmaps: if prop.prop is None: @@ -372,7 +372,7 @@ def _bitmap_size(length): return (factorial(length) + 7) // 8 -class LabelledPerms: +class PropertyBitmaps: """Labels of permutations, one bitmap per length, as input to auto_bisc and bisc: which permutations of each length satisfy a property (good) and which do not (bad). @@ -441,37 +441,37 @@ class _LengthViews: """length -> the good (or bad) permutations of that length, read-only, as the dictionaries of lists auto_bisc builds from the other inputs.""" - def __init__(self, labelled, good): - self.labelled = labelled + def __init__(self, property_bitmaps, good): + self.property_bitmaps = property_bitmaps self.is_good = good def keys(self): - return self.labelled.bitmaps.keys() + return self.property_bitmaps.bitmaps.keys() def __contains__(self, length): - return length in self.labelled.bitmaps + return length in self.property_bitmaps.bitmaps def __getitem__(self, length): - if length not in self.labelled.bitmaps: + if length not in self.property_bitmaps.bitmaps: raise KeyError(length) - return _PermsOfLength(self.labelled, length, self.is_good) + return _PermsOfLength(self.property_bitmaps, length, self.is_good) class _PermsOfLength: - def __init__(self, labelled, length, good): - self.labelled = labelled + def __init__(self, property_bitmaps, length, good): + self.property_bitmaps = property_bitmaps self.length = length self.wanted_bit = 1 if good else 0 def __iter__(self): - bitmap = self.labelled.bitmaps[self.length] + bitmap = self.property_bitmaps.bitmaps[self.length] wanted_bit = self.wanted_bit for i, perm in enumerate(Perm.of_length(self.length)): if (bitmap[i >> 3] >> (i & 7)) & 1 == wanted_bit: yield perm def __len__(self): - good_count = self.labelled.good_counts[self.length] + good_count = self.property_bitmaps.good_counts[self.length] return good_count if self.wanted_bit else factorial(self.length) - good_count diff --git a/tests/bisc/test_labelled_perms.py b/tests/bisc/test_property_bitmaps.py similarity index 60% rename from tests/bisc/test_labelled_perms.py rename to tests/bisc/test_property_bitmaps.py index b3a7380..b9d6335 100644 --- a/tests/bisc/test_labelled_perms.py +++ b/tests/bisc/test_property_bitmaps.py @@ -4,43 +4,43 @@ import pytest -from permuta.bisc.bisc import LabelledPerms, auto_bisc, bisc +from permuta.bisc.bisc import PropertyBitmaps, auto_bisc, bisc from permuta.patterns.perm import Perm BASIS_231 = {3: {Perm((1, 2, 0)): [set()]}} -def labelled_up_to(prop, max_length): - labelled_perms = LabelledPerms(prop) +def property_bitmaps_up_to(prop, max_length): + property_bitmaps = PropertyBitmaps(prop) for length in range(max_length + 1): - labelled_perms.add_length(length) - return labelled_perms + property_bitmaps.add_length(length) + return property_bitmaps def test_views_match_lists(): - labelled_perms = labelled_up_to(Perm.stack_sortable, 6) + property_bitmaps = property_bitmaps_up_to(Perm.stack_sortable, 6) for length in range(7): good = [perm for perm in Perm.of_length(length) if perm.stack_sortable()] bad = [perm for perm in Perm.of_length(length) if not perm.stack_sortable()] - assert list(labelled_perms.good[length]) == good - assert list(labelled_perms.bad[length]) == bad - assert len(labelled_perms.good[length]) == len(good) - assert len(labelled_perms.bad[length]) == len(bad) - assert bool(labelled_perms.bad[length]) == bool(bad) - assert sorted(labelled_perms.good.keys()) == list(range(7)) - assert 6 in labelled_perms.bad and 7 not in labelled_perms.bad + assert list(property_bitmaps.good[length]) == good + assert list(property_bitmaps.bad[length]) == bad + assert len(property_bitmaps.good[length]) == len(good) + assert len(property_bitmaps.bad[length]) == len(bad) + assert bool(property_bitmaps.bad[length]) == bool(bad) + assert sorted(property_bitmaps.good.keys()) == list(range(7)) + assert 6 in property_bitmaps.bad and 7 not in property_bitmaps.bad with pytest.raises(KeyError, match="7"): - _ = labelled_perms.good[7] + _ = property_bitmaps.good[7] def test_bit_order_is_perm_unrank(): # Bit i of a length's bitmap is the label of Perm.unrank(i, length), and the # permutation perm is at bit perm.rank() minus the number of shorter # permutations - labelled_perms = labelled_up_to(Perm.stack_sortable, 6) + property_bitmaps = property_bitmaps_up_to(Perm.stack_sortable, 6) shorter_count = 0 for length in range(7): - bitmap = labelled_perms.bitmaps[length] + bitmap = property_bitmaps.bitmaps[length] for i in range(factorial(length)): perm = Perm.unrank(i, length) assert perm.rank() - shorter_count == i @@ -49,24 +49,24 @@ def test_bit_order_is_perm_unrank(): def test_bitmaps_round_trip(): - labelled_perms = labelled_up_to(Perm.stack_sortable, 6) - round_trip = LabelledPerms(bitmaps=labelled_perms.bitmaps) + property_bitmaps = property_bitmaps_up_to(Perm.stack_sortable, 6) + round_trip = PropertyBitmaps(bitmaps=property_bitmaps.bitmaps) for length in range(7): - assert list(round_trip.good[length]) == list(labelled_perms.good[length]) - assert list(round_trip.bad[length]) == list(labelled_perms.bad[length]) + assert list(round_trip.good[length]) == list(property_bitmaps.good[length]) + assert list(round_trip.bad[length]) == list(property_bitmaps.bad[length]) with pytest.raises(ValueError, match="wrong size"): - LabelledPerms(bitmaps={3: b"\x00\x00"}) # 3! bits fit in one byte + PropertyBitmaps(bitmaps={3: b"\x00\x00"}) # 3! bits fit in one byte with pytest.raises(ValueError, match="wrong size"): - LabelledPerms(bitmaps={3: b""}) + PropertyBitmaps(bitmaps={3: b""}) def test_lengths_0_and_1_labelled_bad(): - labelled_perms = LabelledPerms(bitmaps={0: b"\x00", 1: b"\x00"}) + property_bitmaps = PropertyBitmaps(bitmaps={0: b"\x00", 1: b"\x00"}) for length in (0, 1): - assert list(labelled_perms.good[length]) == [] - assert list(labelled_perms.bad[length]) == [Perm(range(length))] - assert len(labelled_perms.good[length]) == 0 - assert len(labelled_perms.bad[length]) == 1 + assert list(property_bitmaps.good[length]) == [] + assert list(property_bitmaps.bad[length]) == [Perm(range(length))] + assert len(property_bitmaps.good[length]) == 0 + assert len(property_bitmaps.bad[length]) == 1 @pytest.mark.parametrize( @@ -75,27 +75,27 @@ def test_lengths_0_and_1_labelled_bad(): ) def test_set_padding_bits_are_rejected(length, bitmap): with pytest.raises(ValueError, match="bits set past"): - LabelledPerms(bitmaps={length: bitmap}) + PropertyBitmaps(bitmaps={length: bitmap}) @pytest.mark.parametrize("length, bitmap", [(0, 1), (3, 1), (4, 3), (5, 15)]) def test_an_int_is_not_a_bitmap(length, bitmap): with pytest.raises(TypeError, match="must be bytes"): - LabelledPerms(bitmaps={length: bitmap}) + PropertyBitmaps(bitmaps={length: bitmap}) def test_bitmaps_may_be_any_bytes_like(): # All six permutations of length 3 are good, and the first of length 2 for bitmap in (b"\x3f", bytearray(b"\x3f"), memoryview(b"\x3f")): - labelled_perms = LabelledPerms(bitmaps={3: bitmap, 2: b"\x01"}) - assert len(labelled_perms.good[3]) == 6 and not labelled_perms.bad[3] - assert list(labelled_perms.good[2]) == [Perm((0, 1))] - assert isinstance(labelled_perms.bitmaps[3], bytes) + property_bitmaps = PropertyBitmaps(bitmaps={3: bitmap, 2: b"\x01"}) + assert len(property_bitmaps.good[3]) == 6 and not property_bitmaps.bad[3] + assert list(property_bitmaps.good[2]) == [Perm((0, 1))] + assert isinstance(property_bitmaps.bitmaps[3], bytes) -def test_bisc_accepts_labelled_perms(): +def test_bisc_accepts_property_bitmaps(): assert bisc(Perm.stack_sortable, 3, 5) == BASIS_231 - assert bisc(labelled_up_to(Perm.stack_sortable, 5), 3, 5) == BASIS_231 + assert bisc(property_bitmaps_up_to(Perm.stack_sortable, 5), 3, 5) == BASIS_231 def test_auto_bisc_labels_option(): @@ -110,13 +110,13 @@ def test_labels_option_selects_the_representation(monkeypatch): # The results are the same either way, so record which lengths are kept as # bitmaps: none for lists, every length checked for bitmaps added_lengths = [] - real_add_length = LabelledPerms.add_length + real_add_length = PropertyBitmaps.add_length def recording_add_length(self, length): added_lengths.append(length) return real_add_length(self, length) - monkeypatch.setattr(LabelledPerms, "add_length", recording_add_length) + monkeypatch.setattr(PropertyBitmaps, "add_length", recording_add_length) assert auto_bisc(Perm.stack_sortable) == BASIS_231 assert auto_bisc(Perm.stack_sortable, labels="lists") == BASIS_231 assert added_lengths == [] @@ -124,17 +124,17 @@ def recording_add_length(self, length): assert added_lengths == list(range(9)) -def test_auto_bisc_labelled_perms_input(): - assert auto_bisc(labelled_up_to(Perm.stack_sortable, 8)) == BASIS_231 +def test_auto_bisc_property_bitmaps_input(): + assert auto_bisc(property_bitmaps_up_to(Perm.stack_sortable, 8)) == BASIS_231 # Given bitmaps and no property: the lengths auto_bisc needs must be there - bitmaps = labelled_up_to(Perm.stack_sortable, 8).bitmaps - assert auto_bisc(LabelledPerms(bitmaps=bitmaps)) == BASIS_231 + bitmaps = property_bitmaps_up_to(Perm.stack_sortable, 8).bitmaps + assert auto_bisc(PropertyBitmaps(bitmaps=bitmaps)) == BASIS_231 too_short = {length: bitmaps[length] for length in range(6)} - assert auto_bisc(LabelledPerms(bitmaps=too_short)) is None + assert auto_bisc(PropertyBitmaps(bitmaps=too_short)) is None def encode(prop, length): - """A length's bitmap made without LabelledPerms: bit i of the little-endian + """A length's bitmap made without PropertyBitmaps: bit i of the little-endian integer is the label of Perm.unrank(i, length).""" label_bits = sum( 1 << i for i in range(factorial(length)) if prop(Perm.unrank(i, length)) @@ -150,39 +150,39 @@ def prop(perm): return perm.stack_sortable() given_lengths = (0, 1, 2, 3, 5, 8) - labelled_perms = LabelledPerms( + property_bitmaps = PropertyBitmaps( prop, bitmaps={ length: encode(Perm.stack_sortable, length) for length in given_lengths }, ) - assert auto_bisc(labelled_perms) == BASIS_231 + assert auto_bisc(property_bitmaps) == BASIS_231 assert set(labelled_lengths) == {4, 6, 7} - assert sorted(labelled_perms.bitmaps) == list(range(9)) + assert sorted(property_bitmaps.bitmaps) == list(range(9)) def test_bisc_labels_missing_lengths_with_the_property(): # Without n, bisc uses the longest length labelled, or 7 when there is none - assert bisc(labelled_up_to(Perm.stack_sortable, 5), 3) == BASIS_231 - labelled_perms = LabelledPerms(Perm.stack_sortable) - assert bisc(labelled_perms, 3, 5) == BASIS_231 - assert sorted(labelled_perms.bitmaps) == list(range(6)) - labelled_perms = LabelledPerms(Perm.stack_sortable) - assert bisc(labelled_perms, 3) == BASIS_231 - assert sorted(labelled_perms.bitmaps) == list(range(8)) + assert bisc(property_bitmaps_up_to(Perm.stack_sortable, 5), 3) == BASIS_231 + property_bitmaps = PropertyBitmaps(Perm.stack_sortable) + assert bisc(property_bitmaps, 3, 5) == BASIS_231 + assert sorted(property_bitmaps.bitmaps) == list(range(6)) + property_bitmaps = PropertyBitmaps(Perm.stack_sortable) + assert bisc(property_bitmaps, 3) == BASIS_231 + assert sorted(property_bitmaps.bitmaps) == list(range(8)) def test_bisc_needs_labels_or_a_property(): bitmaps = {length: encode(Perm.stack_sortable, length) for length in range(4)} with pytest.raises(ValueError, match="none of length 4 and no property"): - bisc(LabelledPerms(bitmaps=bitmaps), 3, 5) + bisc(PropertyBitmaps(bitmaps=bitmaps), 3, 5) with pytest.raises(ValueError, match="no labels and no property"): - bisc(LabelledPerms(), 3) + bisc(PropertyBitmaps(), 3) def test_bisc_without_n_uses_the_longest_labelled_length_with_no_property(): bitmaps = {length: encode(Perm.stack_sortable, length) for length in range(6)} - assert bisc(LabelledPerms(bitmaps=bitmaps), 3) == BASIS_231 + assert bisc(PropertyBitmaps(bitmaps=bitmaps), 3) == BASIS_231 def test_bisc_labels_only_the_missing_lengths(): @@ -193,41 +193,41 @@ def prop(perm): return perm.stack_sortable() given_lengths = (0, 1, 2, 3, 5) - labelled_perms = LabelledPerms( + property_bitmaps = PropertyBitmaps( prop, bitmaps={ length: encode(Perm.stack_sortable, length) for length in given_lengths }, ) - assert bisc(labelled_perms, 3) == BASIS_231 + assert bisc(property_bitmaps, 3) == BASIS_231 assert labelled_lengths == [4] * factorial(4) - assert sorted(labelled_perms.bitmaps) == list(range(6)) - assert bisc(labelled_perms, 3, 5) == BASIS_231 + assert sorted(property_bitmaps.bitmaps) == list(range(6)) + assert bisc(property_bitmaps, 3, 5) == BASIS_231 assert labelled_lengths == [4] * factorial(4) def test_bisc_labels_up_to_the_pattern_length(): # mine() reads the lengths up to m, so bisc labels up to m when n < m - labelled_perms = LabelledPerms(Perm.stack_sortable) - bisc(labelled_perms, 3, 0) - assert sorted(labelled_perms.bitmaps) == [0, 1, 2, 3] - labelled_perms = LabelledPerms(Perm.stack_sortable, bitmaps={0: b"\x01"}) - bisc(labelled_perms, 3) - assert sorted(labelled_perms.bitmaps) == [0, 1, 2, 3] + property_bitmaps = PropertyBitmaps(Perm.stack_sortable) + bisc(property_bitmaps, 3, 0) + assert sorted(property_bitmaps.bitmaps) == [0, 1, 2, 3] + property_bitmaps = PropertyBitmaps(Perm.stack_sortable, bitmaps={0: b"\x01"}) + bisc(property_bitmaps, 3) + assert sorted(property_bitmaps.bitmaps) == [0, 1, 2, 3] with pytest.raises(ValueError, match="none of length 1 and no property"): - bisc(LabelledPerms(bitmaps={0: b"\x01"}), 3) + bisc(PropertyBitmaps(bitmaps={0: b"\x01"}), 3) def test_bisc_accepts_the_views(): - assert bisc(labelled_up_to(Perm.stack_sortable, 5).good, 3) == BASIS_231 + assert bisc(property_bitmaps_up_to(Perm.stack_sortable, 5).good, 3) == BASIS_231 def test_bitmaps_need_a_property(): for prop in ([Perm((0,))], ({}, {}), "yt_perm_avoids_22"): with pytest.raises(ValueError, match="needs a property"): auto_bisc(prop, labels="bitmaps") - labelled_perms = labelled_up_to(Perm.stack_sortable, 8) - assert auto_bisc(labelled_perms, labels="bitmaps") == BASIS_231 + property_bitmaps = property_bitmaps_up_to(Perm.stack_sortable, 8) + assert auto_bisc(property_bitmaps, labels="bitmaps") == BASIS_231 def test_west_2_stack_sortable_same_as_lists(): @@ -246,10 +246,10 @@ def derangement_in_av_213_231_312(perm): def test_auto_bisc_stops_without_labels_it_needs(): # Bitmaps up to length 8 and no property: auto_bisc needs length 9 and stops - bitmaps = labelled_up_to(derangement_in_av_213_231_312, 8).bitmaps + bitmaps = property_bitmaps_up_to(derangement_in_av_213_231_312, 8).bitmaps output = io.StringIO() with contextlib.redirect_stdout(output): - assert auto_bisc(LabelledPerms(bitmaps=bitmaps)) is None + assert auto_bisc(PropertyBitmaps(bitmaps=bitmaps)) is None assert "You need to add labels of longer permutations" in output.getvalue() From 1769c1b65aa105c6307b36e87fc330d530ccd612 Mon Sep 17 00:00:00 2001 From: Henning Ulfarsson Date: Fri, 23 Oct 2026 15:02:06 +0000 Subject: [PATCH 14/38] Plainer code where Henning stopped in PropertyBitmaps.__init__ From the review of #209, round 3, Henning's point 3: - The loop over given bitmaps read "(bitmaps or {}).items()", turning a missing bitmaps (None) into an empty dictionary so that the loop did nothing. It now runs under "if bitmaps is not None:". - The padding-bit check is needed, and a comment now says why: the size check leaves spare bits in the last byte only at lengths 0 to 3 (7, 7, 6 and 2 of them), since n! is a multiple of 8 from n = 4 on, and a spare bit set would be counted as a good permutation, so that len() of the good view would disagree with its iteration and len() of the bad view could go negative. No change in behavior. Co-Authored-By: Claude Opus 5.5 --- permuta/bisc/bisc.py | 45 +++++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/permuta/bisc/bisc.py b/permuta/bisc/bisc.py index 6c01467..547109a 100644 --- a/permuta/bisc/bisc.py +++ b/permuta/bisc/bisc.py @@ -401,25 +401,32 @@ def __init__(self, prop=None, bitmaps=None): self.prop = prop self.bitmaps = {} self.good_counts = {} - for length, bitmap in (bitmaps or {}).items(): - if not isinstance(bitmap, (bytes, bytearray, memoryview)): - raise TypeError( - "A bitmap must be bytes, a bytearray or a memoryview, " - "not {}".format(type(bitmap).__name__) - ) - bitmap = bytes(bitmap) - if len(bitmap) != _bitmap_size(length): - raise ValueError( - "A bitmap of length {} has the wrong size".format(length) - ) - label_bits = int.from_bytes(bitmap, "little") - if label_bits >> factorial(length): - raise ValueError( - "A bitmap of length {} has bits set past its {} " - "permutations".format(length, factorial(length)) - ) - self.bitmaps[length] = bitmap - self.good_counts[length] = label_bits.bit_count() + if bitmaps is not None: + for length, bitmap in bitmaps.items(): + if not isinstance(bitmap, (bytes, bytearray, memoryview)): + raise TypeError( + "A bitmap must be bytes, a bytearray or a memoryview, " + "not {}".format(type(bitmap).__name__) + ) + bitmap = bytes(bitmap) + if len(bitmap) != _bitmap_size(length): + raise ValueError( + "A bitmap of length {} has the wrong size".format(length) + ) + label_bits = int.from_bytes(bitmap, "little") + # The size check leaves spare bits in the last byte only at + # lengths 0 to 3 (7, 7, 6 and 2 of them), since n! is a + # multiple of 8 from n = 4 on. A spare bit set would be counted + # as a good permutation: len() of the good view would disagree + # with its iteration, and len() of the bad view could go + # negative. + if label_bits >> factorial(length): + raise ValueError( + "A bitmap of length {} has bits set past its {} " + "permutations".format(length, factorial(length)) + ) + self.bitmaps[length] = bitmap + self.good_counts[length] = label_bits.bit_count() self.good = _LengthViews(self, True) self.bad = _LengthViews(self, False) From 658433964094390ae4801745d42148f90a0ff6f3 Mon Sep 17 00:00:00 2001 From: Henning Ulfarsson Date: Fri, 23 Oct 2026 15:04:50 +0000 Subject: [PATCH 15/38] Make the views a Mapping of Collections From the review of #209, round 3, Henning's points 1 and 3 (what _LengthViews is doing): the views stood in for the dictionaries of lists that auto_bisc builds from its other inputs by duck typing, with only keys(), in and [length]. Now they are the standard abstractions, which a reader and mypy can follow: - _LengthViews is a collections.abc.Mapping[int, _PermsOfLength], with __getitem__, __iter__ and __len__; keys(), in and get() come from Mapping. Its keys are the lengths with a bitmap. - _PermsOfLength is a collections.abc.Collection[Perm], with a docstring. Its new __contains__ reads a permutation's bit, at perm.rank() less the number of shorter permutations, as the class docstring says; a permutation of another length is not in it. - bisc's branches for a dict and for a _LengthViews are one branch for a Mapping, so bisc takes any mapping of lengths to permutations. A new test checks the views are a Mapping of Collections and that in agrees with the property for every permutation up to length 6. It fails on three faulty versions of __contains__: without the offset of the shorter permutations, without the length check, and with the good bit wanted in both views. Co-Authored-By: Claude Opus 5.5 --- permuta/bisc/bisc.py | 54 +++++++++++++++++++---------- tests/bisc/test_property_bitmaps.py | 18 ++++++++++ 2 files changed, 53 insertions(+), 19 deletions(-) diff --git a/permuta/bisc/bisc.py b/permuta/bisc/bisc.py index 547109a..d5cc3c2 100644 --- a/permuta/bisc/bisc.py +++ b/permuta/bisc/bisc.py @@ -4,6 +4,7 @@ import os import types from collections import defaultdict +from collections.abc import Collection, Mapping from math import factorial from permuta.bisc.bisc_subfunctions import ( @@ -23,12 +24,12 @@ def bisc(A, m, n=None, report=False): """ Mine the forbidden patterns of length up to m from the permutations up to length n of a set A: a list of permutations, a property (a function), a - dictionary (length -> the permutations of that length in the set), or a - PropertyBitmaps. Without n, a property uses length 7, and the others their - longest length. mine() reads the lengths up to m and the permutations up - to n, so the lengths up to max(m, n) that a PropertyBitmaps lacks are - labelled with its property, and added to it. report prints the progress - of the search. + dictionary or another Mapping (length -> the permutations of that length + in the set), or a PropertyBitmaps. Without n, a property uses length 7, + and the others their longest length. mine() reads the lengths up to m and + the permutations up to n, so the lengths up to max(m, n) that a + PropertyBitmaps lacks are labelled with its property, and added to it. + report prints the progress of the search. Returns the forbidden patterns found: a dictionary from length to {pattern: [shadings]}, where each shading is a set of boxes, and an empty @@ -48,7 +49,7 @@ def bisc(A, m, n=None, report=False): if A(perm): D[i].append(perm) - elif isinstance(A, dict): + elif isinstance(A, Mapping): D = A elif isinstance(A, PropertyBitmaps): @@ -72,9 +73,6 @@ def bisc(A, m, n=None, report=False): A.add_length(length) D = A.good - elif isinstance(A, _LengthViews): - D = A - else: print( """BiSC can accept inputs: @@ -444,32 +442,50 @@ def add_length(self, length): self.good_counts[length] = good_count -class _LengthViews: +class _LengthViews(Mapping[int, "_PermsOfLength"]): """length -> the good (or bad) permutations of that length, read-only, as - the dictionaries of lists auto_bisc builds from the other inputs.""" + the dictionaries of lists auto_bisc builds from the other inputs: a + Mapping whose keys are the lengths with a bitmap.""" def __init__(self, property_bitmaps, good): self.property_bitmaps = property_bitmaps self.is_good = good - def keys(self): - return self.property_bitmaps.bitmaps.keys() - - def __contains__(self, length): - return length in self.property_bitmaps.bitmaps - def __getitem__(self, length): if length not in self.property_bitmaps.bitmaps: raise KeyError(length) return _PermsOfLength(self.property_bitmaps, length, self.is_good) + def __iter__(self): + return iter(self.property_bitmaps.bitmaps) + + def __len__(self): + return len(self.property_bitmaps.bitmaps) + + +class _PermsOfLength(Collection[Perm]): + """The good (or bad) permutations of one length, read from its bitmap, as + the lists auto_bisc builds from the other inputs: a Collection. Iterating + generates the length's permutations in lexicographic order, the order of + the lists, and yields those whose bit is the one wanted; len() is the + count kept with the bitmap; and a permutation is in it when its bit is + the one wanted.""" -class _PermsOfLength: def __init__(self, property_bitmaps, length, good): self.property_bitmaps = property_bitmaps self.length = length self.wanted_bit = 1 if good else 0 + def __contains__(self, perm): + if not isinstance(perm, Perm) or len(perm) != self.length: + return False + shorter_count = sum( + factorial(shorter_length) for shorter_length in range(self.length) + ) + i = perm.rank() - shorter_count + bitmap = self.property_bitmaps.bitmaps[self.length] + return (bitmap[i >> 3] >> (i & 7)) & 1 == self.wanted_bit + def __iter__(self): bitmap = self.property_bitmaps.bitmaps[self.length] wanted_bit = self.wanted_bit diff --git a/tests/bisc/test_property_bitmaps.py b/tests/bisc/test_property_bitmaps.py index b9d6335..696d383 100644 --- a/tests/bisc/test_property_bitmaps.py +++ b/tests/bisc/test_property_bitmaps.py @@ -1,5 +1,6 @@ import contextlib import io +from collections.abc import Collection, Mapping from math import factorial import pytest @@ -33,6 +34,23 @@ def test_views_match_lists(): _ = property_bitmaps.good[7] +def test_views_are_a_mapping_of_collections(): + property_bitmaps = property_bitmaps_up_to(Perm.stack_sortable, 6) + for view in (property_bitmaps.good, property_bitmaps.bad): + assert isinstance(view, Mapping) + assert list(view) == list(range(7)) and len(view) == 7 + assert view.get(7) is None + assert all(isinstance(view[length], Collection) for length in view) + for length in range(7): + good, bad = property_bitmaps.good[length], property_bitmaps.bad[length] + for perm in Perm.of_length(length): + assert (perm in good) == perm.stack_sortable() + assert (perm in bad) == (not perm.stack_sortable()) + # A permutation of another length is in neither view of a length + assert Perm((0, 1)) not in property_bitmaps.good[3] + assert Perm((0, 1, 2, 3)) not in property_bitmaps.bad[3] + + def test_bit_order_is_perm_unrank(): # Bit i of a length's bitmap is the label of Perm.unrank(i, length), and the # permutation perm is at bit perm.rank() minus the number of shorter From 1c8f7acf6eb6504dfed9ac744f49ecfddf4f7466 Mon Sep 17 00:00:00 2001 From: Henning Ulfarsson Date: Fri, 23 Oct 2026 15:13:30 +0000 Subject: [PATCH 16/38] Type bisc.py and let mypy check it From the review of #209, round 3, Henning's point 1: "The pull request would be better with more Typing and mypy." bisc.py began with a file-level "# type: ignore", so mypy had never read the code #209 adds. - The ignore is gone, and mypy 2.4 passes on bisc.py, for Python 3.14 as in CI and for 3.10. - bisc_subfunctions.py keeps its errors ignored until a pull request of its own types it (Henning's choice), but as "# mypy: ignore-errors". A file-level "# type: ignore" makes mypy ignore the module's contents, so it reported that four of the functions bisc.py imports (run_clean_up, show_me, show_me_basis, to_sg_format) do not exist; with the new line mypy reads the module and reports none of its errors. - Type hints on every signature #209 adds or changes: the parameters of bisc and auto_bisc (labels: str too), PropertyBitmaps' __init__ and add_length, _bitmap_size, _LengthViews and _PermsOfLength. A given bitmap is typed bytes, bytearray or memoryview, as the check accepts. bisc and auto_bisc have no return types: what they return comes from bisc_subfunctions.py, untyped until its pull request. - The variables holding the permutations by length, bisc's D and auto_bisc's A and B, are typed Mapping[int, Collection[Perm]], which clears the six errors the review found (seven before the views became a Mapping). A list or a property is sorted into dictionaries of lists of their own, perm_lists in bisc, and good_perm_lists and bad_perm_lists in auto_bisc, which a property's new lengths are added to as L grows. - With the property typed Optional, add_length without one raises a ValueError that says so, where it failed with "'NoneType' object is not callable" (a follow-up from the round 2 review). bisc and auto_bisc check for a property before they call it, so they are unchanged. A new test covers it. What auto_bisc and bisc print and return is unchanged: compared, byte for byte, with develop (ad35732) and round 3's c54c542, for auto_bisc given a property (with and without growing past length 8), a list and a tuple of dictionaries, and bisc given a list, a property and a dictionary. Co-Authored-By: Claude Opus 5.5 --- permuta/bisc/bisc.py | 117 +++++++++++++++++++--------- permuta/bisc/bisc_subfunctions.py | 2 +- tests/bisc/test_property_bitmaps.py | 7 ++ 3 files changed, 90 insertions(+), 36 deletions(-) diff --git a/permuta/bisc/bisc.py b/permuta/bisc/bisc.py index d5cc3c2..2b062cc 100644 --- a/permuta/bisc/bisc.py +++ b/permuta/bisc/bisc.py @@ -1,11 +1,10 @@ -# type: ignore - import json import os import types from collections import defaultdict from collections.abc import Collection, Mapping from math import factorial +from typing import Callable, DefaultDict, Dict, Iterator, List, Optional, Tuple, Union from permuta.bisc.bisc_subfunctions import ( forb, @@ -20,7 +19,17 @@ from permuta.patterns.perm import Perm -def bisc(A, m, n=None, report=False): +def bisc( + A: Union[ + List[Perm], + Callable[[Perm], bool], + Mapping[int, Collection[Perm]], + "PropertyBitmaps", + ], + m: int, + n: Optional[int] = None, + report: bool = False, +): """ Mine the forbidden patterns of length up to m from the permutations up to length n of a set A: a list of permutations, a property (a function), a @@ -35,19 +44,25 @@ def bisc(A, m, n=None, report=False): {pattern: [shadings]}, where each shading is a set of boxes, and an empty one makes the pattern classical. """ + # length -> the permutations of that length in the set: for a list or a + # property, the dictionary of lists perm_lists + D: Mapping[int, Collection[Perm]] + perm_lists: DefaultDict[int, List[Perm]] if isinstance(A, list): - D = defaultdict(list) + perm_lists = defaultdict(list) for perm in A: - D[len(perm)].append(perm) + perm_lists[len(perm)].append(perm) + D = perm_lists elif isinstance(A, types.FunctionType): if n is None: print("I will use permutations up to length 7") n = 7 - D = defaultdict(list) + perm_lists = defaultdict(list) for i in range(n + 1): for perm in Perm.of_length(i): if A(perm): - D[i].append(perm) + perm_lists[i].append(perm) + D = perm_lists elif isinstance(A, Mapping): D = A @@ -92,7 +107,16 @@ def bisc(A, m, n=None, report=False): return SG -def auto_bisc(prop, labels="lists"): +def auto_bisc( + prop: Union[ + Callable[[Perm], bool], + List[Perm], + Tuple[Dict[int, List[Perm]], Dict[int, List[Perm]]], + str, + "PropertyBitmaps", + ], + labels: str = "lists", +): """ Learn a basis of forbidden patterns for a set of permutations, growing the length of the patterns and of the permutations they are learned from @@ -131,34 +155,46 @@ def auto_bisc(prop, labels="lists"): n = 4 m = 2 + # length -> the good (A) and the bad (B) permutations of that length: for + # a list or a property, the dictionaries of lists good_perm_lists and + # bad_perm_lists, to which a property's new lengths are added as L grows + A: Mapping[int, Collection[Perm]] + B: Mapping[int, Collection[Perm]] + good_perm_lists: DefaultDict[int, List[Perm]] + bad_perm_lists: DefaultDict[int, List[Perm]] + if isinstance(prop, list): # If a list is passed in then we put it in a dictionary at the start - A = defaultdict(list) + good_perm_lists = defaultdict(list) # We will put the complement up to S8 into another dictionary - B = defaultdict(list) + bad_perm_lists = defaultdict(list) for perm in prop: - A[len(perm)].append(perm) - if L not in A.keys(): + good_perm_lists[len(perm)].append(perm) + if L not in good_perm_lists.keys(): print("You should have permutations up to length at least 8") return - for i in range(max(L + 1, max(A.keys()) + 1)): + for i in range(max(L + 1, max(good_perm_lists.keys()) + 1)): if i > 7: print("Populating the dictionary of bad perms of length ", i) - B[i] = [perm for perm in Perm.of_length(i) if perm not in A[i]] + bad_perm_lists[i] = [ + perm for perm in Perm.of_length(i) if perm not in good_perm_lists[i] + ] + A, B = good_perm_lists, bad_perm_lists elif isinstance(prop, types.FunctionType): # If a property is passed in then we use it to populate both # both dictionaries up to S8 - A = defaultdict(list) - B = defaultdict(list) + good_perm_lists = defaultdict(list) + bad_perm_lists = defaultdict(list) for i in range(L + 1): - A[i] = [] - B[i] = [] + good_perm_lists[i] = [] + bad_perm_lists[i] = [] for perm in Perm.of_length(i): if prop(perm): - A[i].append(perm) + good_perm_lists[i].append(perm) else: - B[i].append(perm) + bad_perm_lists[i].append(perm) + A, B = good_perm_lists, bad_perm_lists elif isinstance(prop, PropertyBitmaps): # Labels kept as one bitmap per length: A and B are views of the good @@ -311,9 +347,9 @@ def auto_bisc(prop, labels="lists"): print("Adding perms of length {}".format(i)) for perm in Perm.of_length(i): if prop(perm): - A[i].append(perm) + good_perm_lists[i].append(perm) else: - B[i].append(perm) + bad_perm_lists[i].append(perm) elif isinstance(prop, PropertyBitmaps): for i in range(oldL + 1, L + 1): @@ -364,7 +400,7 @@ def auto_bisc(prop, labels="lists"): return -def _bitmap_size(length): +def _bitmap_size(length: int) -> int: """The number of bytes in the bitmap of a length: a bit for each of its permutations.""" return (factorial(length) + 7) // 8 @@ -395,10 +431,14 @@ class PropertyBitmaps: of Perm objects takes 0.9 GB on PyPy 3.11 and 1.3 to 1.8 GB on CPython. """ - def __init__(self, prop=None, bitmaps=None): + def __init__( + self, + prop: Optional[Callable[[Perm], bool]] = None, + bitmaps: Optional[Mapping[int, Union[bytes, bytearray, memoryview]]] = None, + ) -> None: self.prop = prop - self.bitmaps = {} - self.good_counts = {} + self.bitmaps: Dict[int, bytes] = {} + self.good_counts: Dict[int, int] = {} if bitmaps is not None: for length, bitmap in bitmaps.items(): if not isinstance(bitmap, (bytes, bytearray, memoryview)): @@ -428,10 +468,15 @@ def __init__(self, prop=None, bitmaps=None): self.good = _LengthViews(self, True) self.bad = _LengthViews(self, False) - def add_length(self, length): + def add_length(self, length: int) -> None: """Label the permutations of a length with the property, and keep their labels as the length's bitmap. It needs a property, and replaces the length's bitmap if there is one.""" + if self.prop is None: + raise ValueError( + "The PropertyBitmaps has no property to label length {} " + "with".format(length) + ) bitmap = bytearray(_bitmap_size(length)) good_count = 0 for i, perm in enumerate(Perm.of_length(length)): @@ -447,19 +492,19 @@ class _LengthViews(Mapping[int, "_PermsOfLength"]): the dictionaries of lists auto_bisc builds from the other inputs: a Mapping whose keys are the lengths with a bitmap.""" - def __init__(self, property_bitmaps, good): + def __init__(self, property_bitmaps: PropertyBitmaps, good: bool) -> None: self.property_bitmaps = property_bitmaps self.is_good = good - def __getitem__(self, length): + def __getitem__(self, length: int) -> "_PermsOfLength": if length not in self.property_bitmaps.bitmaps: raise KeyError(length) return _PermsOfLength(self.property_bitmaps, length, self.is_good) - def __iter__(self): + def __iter__(self) -> Iterator[int]: return iter(self.property_bitmaps.bitmaps) - def __len__(self): + def __len__(self) -> int: return len(self.property_bitmaps.bitmaps) @@ -471,12 +516,14 @@ class _PermsOfLength(Collection[Perm]): count kept with the bitmap; and a permutation is in it when its bit is the one wanted.""" - def __init__(self, property_bitmaps, length, good): + def __init__( + self, property_bitmaps: PropertyBitmaps, length: int, good: bool + ) -> None: self.property_bitmaps = property_bitmaps self.length = length self.wanted_bit = 1 if good else 0 - def __contains__(self, perm): + def __contains__(self, perm: object) -> bool: if not isinstance(perm, Perm) or len(perm) != self.length: return False shorter_count = sum( @@ -486,14 +533,14 @@ def __contains__(self, perm): bitmap = self.property_bitmaps.bitmaps[self.length] return (bitmap[i >> 3] >> (i & 7)) & 1 == self.wanted_bit - def __iter__(self): + def __iter__(self) -> Iterator[Perm]: bitmap = self.property_bitmaps.bitmaps[self.length] wanted_bit = self.wanted_bit for i, perm in enumerate(Perm.of_length(self.length)): if (bitmap[i >> 3] >> (i & 7)) & 1 == wanted_bit: yield perm - def __len__(self): + def __len__(self) -> int: good_count = self.property_bitmaps.good_counts[self.length] return good_count if self.wanted_bit else factorial(self.length) - good_count diff --git a/permuta/bisc/bisc_subfunctions.py b/permuta/bisc/bisc_subfunctions.py index 3de86bc..cb59688 100644 --- a/permuta/bisc/bisc_subfunctions.py +++ b/permuta/bisc/bisc_subfunctions.py @@ -1,4 +1,4 @@ -# type: ignore +# mypy: ignore-errors from itertools import chain from math import factorial diff --git a/tests/bisc/test_property_bitmaps.py b/tests/bisc/test_property_bitmaps.py index 696d383..77b07e7 100644 --- a/tests/bisc/test_property_bitmaps.py +++ b/tests/bisc/test_property_bitmaps.py @@ -198,6 +198,13 @@ def test_bisc_needs_labels_or_a_property(): bisc(PropertyBitmaps(), 3) +def test_add_length_needs_a_property(): + property_bitmaps = PropertyBitmaps(bitmaps={0: b"\x01"}) + with pytest.raises(ValueError, match="no property to label length 3 with"): + property_bitmaps.add_length(3) + assert sorted(property_bitmaps.bitmaps) == [0] + + def test_bisc_without_n_uses_the_longest_labelled_length_with_no_property(): bitmaps = {length: encode(Perm.stack_sortable, length) for length in range(6)} assert bisc(PropertyBitmaps(bitmaps=bitmaps), 3) == BASIS_231 From ea42763cb5c892ea0b858d7980c067bcb3a5ede1 Mon Sep 17 00:00:00 2001 From: Henning Ulfarsson Date: Fri, 23 Oct 2026 15:34:24 +0000 Subject: [PATCH 17/38] Name the bit operations From the review of #209, round 4, Henning's question about what "bitmap[i >> 3] |= 1 << (i & 7)" and "(bitmap[i >> 3] >> (i & 7)) & 1" do: they set and read the label of the permutation at index i, bit i % 8 of byte i // 8. Two helpers now say so, _set_label and _label, written with // and %, which give the same for an index that is not negative. add_length, _PermsOfLength.__iter__ and __contains__ use them. The call costs little, timed on S_10 (best of 3, the bit work alone with a property as cheap as perm[0] < perm[-1]): - setting, as add_length: CPython 3.11 1.17 s inline, 1.20 s with the helper; PyPy 3.11 0.107 s either way; - reading, as __iter__: CPython 1.14 s and 1.22 s; PyPy 0.127 s and 0.133 s. With a real property, stack-sortability, setting S_10 took 17.2 s and 17.3 s on CPython, and 22.6 s and 20.1 s on PyPy (noise), so the property's cost hides the call. Co-Authored-By: Claude Opus 5.5 --- permuta/bisc/bisc.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/permuta/bisc/bisc.py b/permuta/bisc/bisc.py index 2b062cc..80fe526 100644 --- a/permuta/bisc/bisc.py +++ b/permuta/bisc/bisc.py @@ -406,6 +406,17 @@ def _bitmap_size(length: int) -> int: return (factorial(length) + 7) // 8 +def _set_label(bitmap: bytearray, index: int) -> None: + """Set the label of the permutation at index: bit index % 8 of byte + index // 8.""" + bitmap[index // 8] |= 1 << (index % 8) + + +def _label(bitmap: bytes, index: int) -> int: + """The label (0 or 1) of the permutation at index.""" + return (bitmap[index // 8] >> (index % 8)) & 1 + + class PropertyBitmaps: """Labels of permutations, one bitmap per length, as input to auto_bisc and bisc: which permutations of each length satisfy a property (good) and @@ -479,9 +490,9 @@ def add_length(self, length: int) -> None: ) bitmap = bytearray(_bitmap_size(length)) good_count = 0 - for i, perm in enumerate(Perm.of_length(length)): + for index, perm in enumerate(Perm.of_length(length)): if self.prop(perm): - bitmap[i >> 3] |= 1 << (i & 7) + _set_label(bitmap, index) good_count += 1 self.bitmaps[length] = bytes(bitmap) self.good_counts[length] = good_count @@ -529,15 +540,14 @@ def __contains__(self, perm: object) -> bool: shorter_count = sum( factorial(shorter_length) for shorter_length in range(self.length) ) - i = perm.rank() - shorter_count bitmap = self.property_bitmaps.bitmaps[self.length] - return (bitmap[i >> 3] >> (i & 7)) & 1 == self.wanted_bit + return _label(bitmap, perm.rank() - shorter_count) == self.wanted_bit def __iter__(self) -> Iterator[Perm]: bitmap = self.property_bitmaps.bitmaps[self.length] wanted_bit = self.wanted_bit - for i, perm in enumerate(Perm.of_length(self.length)): - if (bitmap[i >> 3] >> (i & 7)) & 1 == wanted_bit: + for index, perm in enumerate(Perm.of_length(self.length)): + if _label(bitmap, index) == wanted_bit: yield perm def __len__(self) -> int: From 80b9f04bc831f4861d075fd4a74738ebb2f228e3 Mon Sep 17 00:00:00 2001 From: Henning Ulfarsson Date: Fri, 23 Oct 2026 15:45:47 +0000 Subject: [PATCH 18/38] Turn every input of bisc and auto_bisc into a PropertyBitmaps From the review of #209, round 4, Henning: "Lets keep accepting these different inputs in, but everything should be turned into a bitmap." He confirmed it in the BiSC agent's session. bisc and auto_bisc take what they took before. Each input is turned into a PropertyBitmaps at the start, and from there one code path reads the good and the bad permutations through its views. - labels= is gone, with its lists path. It was added by #209 and never released. - A list sets the bits of the permutations it lists. Every length up to the longest listed, and up to max(m, n) in bisc, gets a bitmap, all zeros where nothing is listed, as the defaultdict gave an empty list. - A property labels the lengths up to n in bisc, and the lengths past n, when m > n, are empty, as they were in the lists. That keeps develop's answer for that case. - A Mapping given to bisc is read for each length up to max(m, n). A length it lacks is a KeyError, as mine() raised before. - A tuple of two dictionaries, or a stored set, must give each permutation of each length, up to the shorter dictionary's longest, exactly once, as good or as bad. Otherwise it is a ValueError, where the complement used to be taken in silence. All 15 stored sets pass (4.2 s on CPython 3.11, 1.4 s on PyPy 3.11, all checked by a new test). - A permutation listed twice sets one bit. With the lists, (0, 1) listed twice among the increasing permutations made mine() take length 2 as complete, and bisc found nothing; now it finds 10, as it does without the duplicate. The CHANGELOG says so. - auto_bisc gives bisc its PropertyBitmaps. The reading of a stored set, written twice in auto_bisc, is one function, _read_stored_set, with the same logic. - The docstrings, messages, README and CHANGELOG describe the class in its own terms ("no bitmaps and no property", "You need to add bitmaps of longer permutations"), and the CHANGELOG records only the final behavior, under Added and Changed. Tests: - one for each input kind of bisc and of auto_bisc, which spies on mine() for the bitmaps it is given; - a list with a length missing, and with a duplicate; - a property with m > n; - the ValueError for a tuple missing a permutation and for one given twice; - every stored set. The tests of labels= are gone, and West-2 is compared with its basis. What auto_bisc and bisc print and return is byte for byte develop's (ad35732) in 15 runs, on CPython 3.11: - auto_bisc given stack-sortability, West-2-stack-sortability, a property that grows past length 9 (stopped at 10), a list, a shuffled list, a tuple, a tuple with its lists reversed, "smooth" and "SimSun"; - bisc given a list, a shuffled list, a property (with m <= n and m > n), a dictionary, and one with its lists reversed. The permutations are read in lexicographic order, the order of the lists that a property, a list's complement and the stored sets gave. So with a dictionary in another order, a message may name a different first permutation; no run here changed its result. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 25 +- README.rst | 15 +- permuta/bisc/bisc.py | 440 +++++++++++++++------------- tests/bisc/test_property_bitmaps.py | 171 ++++++++--- 4 files changed, 386 insertions(+), 265 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a550e55..3331000 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,15 +7,22 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## Unreleased ### Added -- `PropertyBitmaps` in `permuta.bisc`: labels of permutations kept as one bitmap - per length (lexicographic order), made from a property, given as bitmaps, or both. - `auto_bisc` and `bisc` accept it as input, and label the lengths it lacks with - its property. -- `auto_bisc(prop, labels="bitmaps")`: keeps a property's labels as bitmaps instead - of lists of permutations, for the same result in far less memory at longer - lengths (the permutations of length 10 take 0.9 to 1.8 GB as lists of `Perm` - objects, depending on the Python, and 454 KB as a bitmap). `labels="lists"`, - the default, is the previous behaviour. +- `PropertyBitmaps` in `permuta.bisc`: a property's values on the permutations, + kept as one bitmap per length (lexicographic order), made from a property, given + as bitmaps, or both. `auto_bisc` and `bisc` accept it as input, and label the + lengths it lacks with its property. + +### Changed +- `bisc` and `auto_bisc` turn every input into a `PropertyBitmaps` and read the + good and the bad permutations through its views, in lexicographic order, for + the same results in far less memory at longer lengths: the permutations of + length 10 take 454 KB as a bitmap, and 0.9 to 1.8 GB as lists of `Perm` objects, + depending on the Python. +- A permutation listed twice in the input of `bisc` or `auto_bisc` counts once. + Before, duplicates could make `bisc` take a length as complete when it was not. +- `auto_bisc` raises a `ValueError` when a tuple of dictionaries, or a stored set, + does not give each permutation of a length exactly once, as good or as bad. +- `bisc` accepts any mapping of lengths to permutations, not only a `dict`. ### Fixed - Fixed bug in `Perm.rtlmax_ltrmin_decomposition` and fixed associated tests. diff --git a/README.rst b/README.rst index 128ad0d..b9f47bc 100644 --- a/README.rst +++ b/README.rst @@ -518,14 +518,13 @@ as ``smooth``, ``forest-like``, ``baxter``, ``simsun``, ``quick_sortable``, etc. Both ``bisc`` and ``auto_bisc`` can accept input in the form of a property, a list of permutations (satisfying some property), or a ``PropertyBitmaps``. -``auto_bisc`` keeps the permutations satisfying a property, and those that do -not, as lists. For long runs it can keep them as one bitmap per length instead, -with the same result in far less memory: -``auto_bisc(Perm.stack_sortable, labels="bitmaps")``. Use it when ``auto_bisc`` -has to check long permutations: the permutations of length 10 take 0.9 to 1.8 GB -as lists of ``Perm`` objects, depending on the Python, and 454 KB as a bitmap. -Such labels can also be given directly as a ``PropertyBitmaps``, made from a -property, from bitmaps, or both, and ``bisc`` accepts it too. +Whatever the input, ``bisc`` and ``auto_bisc`` keep the permutations satisfying +the property, and those that do not, as one bitmap per length, in a +``PropertyBitmaps``: the permutations of length 10 take 454 KB as a bitmap, where +they take 0.9 to 1.8 GB as lists of ``Perm`` objects, depending on the Python. +A ``PropertyBitmaps`` can also be given directly, made from a property, from +bitmaps, or both. The lengths ``bisc`` and ``auto_bisc`` label with its property +are added to it, for a later run to reuse. .. code-block:: python diff --git a/permuta/bisc/bisc.py b/permuta/bisc/bisc.py index 80fe526..d91fbaa 100644 --- a/permuta/bisc/bisc.py +++ b/permuta/bisc/bisc.py @@ -1,10 +1,9 @@ import json import os import types -from collections import defaultdict from collections.abc import Collection, Mapping from math import factorial -from typing import Callable, DefaultDict, Dict, Iterator, List, Optional, Tuple, Union +from typing import Callable, Dict, Iterable, Iterator, List, Optional, Tuple, Union from permuta.bisc.bisc_subfunctions import ( forb, @@ -34,40 +33,59 @@ def bisc( Mine the forbidden patterns of length up to m from the permutations up to length n of a set A: a list of permutations, a property (a function), a dictionary or another Mapping (length -> the permutations of that length - in the set), or a PropertyBitmaps. Without n, a property uses length 7, - and the others their longest length. mine() reads the lengths up to m and - the permutations up to n, so the lengths up to max(m, n) that a - PropertyBitmaps lacks are labelled with its property, and added to it. + in the set), or a PropertyBitmaps. Each is turned into a PropertyBitmaps, + whose good permutations of each length up to max(m, n) mine() reads. + Without n, a property uses length 7, and the others their longest length. + - A list sets the bits of the permutations it lists, and a length with + none listed is empty. + - A property labels the lengths up to n; a length past n, when m > n, is + empty, as it was in the lists before. + - A Mapping is read for each length up to max(m, n), and one it lacks is + its KeyError. + - A PropertyBitmaps labels the lengths up to max(m, n) it lacks with its + property, and they are added to it; without a property, a ValueError + names the first. report prints the progress of the search. Returns the forbidden patterns found: a dictionary from length to {pattern: [shadings]}, where each shading is a set of boxes, and an empty one makes the pattern classical. """ - # length -> the permutations of that length in the set: for a list or a - # property, the dictionary of lists perm_lists - D: Mapping[int, Collection[Perm]] - perm_lists: DefaultDict[int, List[Perm]] if isinstance(A, list): - perm_lists = defaultdict(list) + if n is None: + n = max(len(perm) for perm in A) + perms_by_length: Dict[int, List[Perm]] = { + length: [] for length in range(max(m, n) + 1) + } for perm in A: - perm_lists[len(perm)].append(perm) - D = perm_lists + if len(perm) in perms_by_length: + perms_by_length[len(perm)].append(perm) + property_bitmaps = _from_perms_by_length(perms_by_length) + elif isinstance(A, types.FunctionType): if n is None: print("I will use permutations up to length 7") n = 7 - perm_lists = defaultdict(list) - for i in range(n + 1): - for perm in Perm.of_length(i): - if A(perm): - perm_lists[i].append(perm) - D = perm_lists + # The property labels the lengths up to n. mine() reads those up to m + # too, and a length past n is empty, as it was in the lists before + property_bitmaps = PropertyBitmaps( + A, + bitmaps={ + length: bytes(_bitmap_size(length)) for length in range(n + 1, m + 1) + }, + ) + for length in range(n + 1): + property_bitmaps.add_length(length) elif isinstance(A, Mapping): - D = A + if n is None: + n = max(A.keys()) + property_bitmaps = _from_perms_by_length( + {length: A[length] for length in range(max(m, n) + 1)} + ) elif isinstance(A, PropertyBitmaps): + property_bitmaps = A if n is None: if A.bitmaps: n = max(A.bitmaps) @@ -75,18 +93,17 @@ def bisc( print("I will use permutations up to length 7") n = 7 else: - raise ValueError("The PropertyBitmaps has no labels and no property") + raise ValueError("The PropertyBitmaps has no bitmaps and no property") for length in range(max(m, n) + 1): if length not in A.bitmaps: if A.prop is None: raise ValueError( - "bisc needs labels up to length {}, and the PropertyBitmaps " + "bisc needs bitmaps up to length {}, and the PropertyBitmaps " "has none of length {} and no property".format( max(m, n), length ) ) A.add_length(length) - D = A.good else: print( @@ -94,14 +111,11 @@ def bisc( 1. A list of permutations 2. A property of permutations 3. A dictionary of lengths pointing to permutations - 4. A PropertyBitmaps (labels kept as bitmaps)""" + 4. A PropertyBitmaps""" ) assert False - if n is None: - n = max(D.keys()) - - ci, goodpatts = mine(D, m, n, report=report) + ci, goodpatts = mine(property_bitmaps.good, m, n, report=report) SG = forb(ci, goodpatts, m, report=report) return SG @@ -115,7 +129,6 @@ def auto_bisc( str, "PropertyBitmaps", ], - labels: str = "lists", ): """ Learn a basis of forbidden patterns for a set of permutations, growing @@ -127,87 +140,59 @@ def auto_bisc( prop is the set: a property of permutations (a function), a list of permutations, a tuple of two dictionaries (length -> the permutations in the set, and those not in it), the name of a set in - permuta/resources/bisc, or a PropertyBitmaps. - - labels says how the labels of a property are kept: - "lists" (the default, as before): the good and the bad permutations of - each length as lists of Perm objects; - "bitmaps": one bitmap per length (PropertyBitmaps), for the same result in - far less memory, at the cost of generating a length's permutations again - each time it is read. Use it when the permutations checked are long: those - of length 10 take 0.9 to 1.8 GB as lists of Perm objects, depending on the - Python, and 454 KB as a bitmap, a bit for each permutation (5 MB at length - 11, 60 MB at length 12). - - A PropertyBitmaps can also be given as prop directly. The lengths auto_bisc - labels with its property are added to it, so a later run reuses them. + permuta/resources/bisc, or a PropertyBitmaps. Each is turned into a + PropertyBitmaps at the start, whose views give the good and the bad + permutations of each length. + - A property labels each length when auto_bisc first needs it. + - A list sets the bits of the permutations it lists, and the others are + bad. + - A tuple of dictionaries, or a stored set, must give each permutation of + each length up to the shorter dictionary's longest exactly once, as good + or as bad, or a ValueError is raised. A length one of them lacks is + empty in it. + - A PropertyBitmaps labels the lengths it lacks with its property, and + they are added to it, so that a later run reuses them. Without a + property, auto_bisc stops when it needs a length it lacks, as it does + for a tuple of dictionaries. """ - if labels not in ("lists", "bitmaps"): - raise ValueError('labels must be "lists" or "bitmaps"') - if labels == "bitmaps": - if isinstance(prop, types.FunctionType): - prop = PropertyBitmaps(prop) - elif not isinstance(prop, PropertyBitmaps): - raise ValueError( - 'labels="bitmaps" needs a property (a function) or a PropertyBitmaps' - ) L = 8 # Want to sanity check on at least S8, and one above n n = 4 m = 2 - # length -> the good (A) and the bad (B) permutations of that length: for - # a list or a property, the dictionaries of lists good_perm_lists and - # bad_perm_lists, to which a property's new lengths are added as L grows - A: Mapping[int, Collection[Perm]] - B: Mapping[int, Collection[Perm]] - good_perm_lists: DefaultDict[int, List[Perm]] - bad_perm_lists: DefaultDict[int, List[Perm]] - if isinstance(prop, list): - # If a list is passed in then we put it in a dictionary at the start - good_perm_lists = defaultdict(list) - # We will put the complement up to S8 into another dictionary - bad_perm_lists = defaultdict(list) - for perm in prop: - good_perm_lists[len(perm)].append(perm) - if L not in good_perm_lists.keys(): + # If a list is passed in then its permutations are the good ones, and + # the others up to its longest length are the bad ones + listed_lengths = {len(perm) for perm in prop} + if L not in listed_lengths: print("You should have permutations up to length at least 8") return - for i in range(max(L + 1, max(good_perm_lists.keys()) + 1)): - if i > 7: - print("Populating the dictionary of bad perms of length ", i) - bad_perm_lists[i] = [ - perm for perm in Perm.of_length(i) if perm not in good_perm_lists[i] - ] - A, B = good_perm_lists, bad_perm_lists + longest_length = max(listed_lengths) + for i in range(L, longest_length + 1): + print("Populating the dictionary of bad perms of length ", i) + perms_by_length: Dict[int, List[Perm]] = { + length: [] for length in range(longest_length + 1) + } + for perm in prop: + perms_by_length[len(perm)].append(perm) + property_bitmaps = _from_perms_by_length(perms_by_length) elif isinstance(prop, types.FunctionType): - # If a property is passed in then we use it to populate both - # both dictionaries up to S8 - good_perm_lists = defaultdict(list) - bad_perm_lists = defaultdict(list) + # If a property is passed in then it labels the lengths up to S8 now, + # and the longer ones as they are needed + property_bitmaps = PropertyBitmaps(prop) for i in range(L + 1): - good_perm_lists[i] = [] - bad_perm_lists[i] = [] - for perm in Perm.of_length(i): - if prop(perm): - good_perm_lists[i].append(perm) - else: - bad_perm_lists[i].append(perm) - A, B = good_perm_lists, bad_perm_lists + property_bitmaps.add_length(i) elif isinstance(prop, PropertyBitmaps): - # Labels kept as one bitmap per length: A and B are views of the good - # and the bad permutations, generated length by length when they are - # read, instead of lists. The lengths the PropertyBitmaps lacks are - # labelled with its property, if it has one + # The lengths the PropertyBitmaps lacks are labelled with its + # property, if it has one + property_bitmaps = prop for i in range(L + 1): - if i not in prop.bitmaps: - if prop.prop is None: - print("You should have labels up to length at least 8") + if i not in property_bitmaps.bitmaps: + if property_bitmaps.prop is None: + print("You should have bitmaps up to length at least 8") return - prop.add_length(i) - A, B = prop.good, prop.bad + property_bitmaps.add_length(i) elif ( isinstance(prop, tuple) @@ -215,45 +200,19 @@ def auto_bisc( and isinstance(prop[1], dict) ): # If you already have the dictionaries you can pass them in as a tuple - A = prop[0] - B = prop[1] - if L not in A.keys(): + if L not in prop[0].keys(): print("You should have permutations up to length at least 8") return + property_bitmaps = _from_good_and_bad(prop[0], prop[1]) elif isinstance(prop, str): # If you pass in a string we assume it is pointing to a file in permsets print("Attempting to read perms from permsets") - good_entry = None - bad_entry = None - bisc_dir = os.path.join(os.path.dirname(__file__), "..", "resources", "bisc") - bisc_dir = os.path.abspath(bisc_dir) - - with os.scandir(bisc_dir) as entries: - for i, entry in enumerate(entries): - en = entry.name[:-5] - spl = en.split("_") - if spl[0] == prop: - if ( - good_entry is None - and spl[1] == "good" - and int(spl[2].split("len")[1]) >= L - ): - good_entry = en - if ( - bad_entry is None - and spl[1] == "bad" - and int(spl[2].split("len")[1]) >= L - ): - bad_entry = en - if good_entry is not None and bad_entry is not None: - break - if good_entry is not None and bad_entry is not None: - A = read_bisc_file(bisc_dir + "/" + good_entry) - B = read_bisc_file(bisc_dir + "/" + bad_entry) - else: + stored_set = _read_stored_set(prop, L) + if stored_set is None: print("The required files do not exist") return + property_bitmaps = _from_good_and_bad(*stored_set) else: print( @@ -262,13 +221,16 @@ def auto_bisc( 2. A property of permutations 3. A tuple of two dictionaries of lengths pointing to permutations 4. A string - 5. A PropertyBitmaps (labels kept as bitmaps)""" + 5. A PropertyBitmaps""" ) assert False + # length -> the good (A) and the bad (B) permutations of that length + A, B = property_bitmaps.good, property_bitmaps.bad + while True: print("Learning patterns of length {} using perms of length {}".format(m, n)) - SG = bisc(A, m, n) + SG = bisc(property_bitmaps, m, n) show_me(SG, more=True) if SG != {}: @@ -341,63 +303,29 @@ def auto_bisc( print("You need to input a longer list of permutations") return - elif isinstance(prop, types.FunctionType): - # Adding perms to the dictionaries + elif isinstance(prop, (types.FunctionType, PropertyBitmaps)): + # The property labels the new lengths for i in range(oldL + 1, L + 1): - print("Adding perms of length {}".format(i)) - for perm in Perm.of_length(i): - if prop(perm): - good_perm_lists[i].append(perm) - else: - bad_perm_lists[i].append(perm) - - elif isinstance(prop, PropertyBitmaps): - for i in range(oldL + 1, L + 1): - if i not in prop.bitmaps: - if prop.prop is None: - print("You need to add labels of longer permutations") + if i not in property_bitmaps.bitmaps: + if property_bitmaps.prop is None: + print("You need to add bitmaps of longer permutations") return print("Adding perms of length {}".format(i)) - prop.add_length(i) + property_bitmaps.add_length(i) - elif isinstance(prop, tuple) and L > min(max(A.keys()), max(B.keys())): + elif isinstance(prop, tuple) and L > max(A.keys()): print("You need to add longer permutations to the dictionaries") return - elif isinstance(prop, str) and L > min(max(A.keys()), max(B.keys())): + elif isinstance(prop, str) and L > max(A.keys()): # If you pass in a string we assume it is pointing to a file in permsets print("Attempting to read perms from permsets") - good_entry = None - bad_entry = None - bisc_dir = os.path.join( - os.path.dirname(__file__), "..", "resources", "bisc" - ) - bisc_dir = os.path.abspath(bisc_dir) - with os.scandir(bisc_dir) as entries: - for i, entry in enumerate(entries): - en = entry.name[:-5] - spl = en.split("_") - if spl[0] == prop: - if ( - good_entry is None - and spl[1] == "good" - and int(spl[2].split("len")[1]) >= L - ): - good_entry = en - if ( - bad_entry is None - and spl[1] == "bad" - and int(spl[2].split("len")[1]) >= L - ): - bad_entry = en - if good_entry is not None and bad_entry is not None: - break - if good_entry is not None and bad_entry is not None: - A = read_bisc_file(bisc_dir + "/" + good_entry) - B = read_bisc_file(bisc_dir + "/" + bad_entry) - else: + stored_set = _read_stored_set(prop, L) + if stored_set is None: print("The required files do not exist") return + property_bitmaps = _from_good_and_bad(*stored_set) + A, B = property_bitmaps.good, property_bitmaps.bad def _bitmap_size(length: int) -> int: @@ -418,28 +346,27 @@ def _label(bitmap: bytes, index: int) -> int: class PropertyBitmaps: - """Labels of permutations, one bitmap per length, as input to auto_bisc - and bisc: which permutations of each length satisfy a property (good) and - which do not (bad). - - Bit i of a length's bitmap is the label of Perm.unrank(i, length), in - byte i // 8, from the low bit; so a permutation perm is at bit - perm.rank() - sum(factorial(k) for k in range(len(perm))). This is the - order in which Perm.of_length(length) yields the permutations - (lexicographic), which the views use to iterate. The bitmaps can be given - (bitmaps={length: bytes}), made from a property (add_length), or both: - auto_bisc and bisc label the lengths they need and are not given with - prop, adding them to this object. Without a prop, auto_bisc stops, as for - a tuple of dictionaries, and bisc raises a ValueError. A bitmap given must - be bytes-like (bytes, a bytearray or a memoryview), of its length's size, - with no bit set past the length's permutations. - - auto_bisc reads its good and bad permutations only length by length: - iterating over a length, counting it, asking whether a length is there. - The views good and bad do that from the bitmaps, generating a length's - permutations again each time it is iterated, in the same order as the - lists a property input makes. A bitmap of S_10 is 454 KB; S_10 as a list - of Perm objects takes 0.9 GB on PyPy 3.11 and 1.3 to 1.8 GB on CPython. + """A property's values on the permutations, one bitmap per length: which + permutations of each length satisfy it (good) and which do not (bad). + bisc and auto_bisc turn each of their inputs into one, and read its good + and bad permutations through its views, good and bad. + + Bit i of a length's bitmap is the label of Perm.unrank(i, length): bit + i % 8 of byte i // 8, from the low bit. So a permutation perm is at bit + perm.rank() - sum(factorial(k) for k in range(len(perm))), and the bits + are in the order in which Perm.of_length(length) yields the permutations + (lexicographic). The bitmaps can be given (bitmaps={length: bytes}), made + from a property (add_length), or both: auto_bisc and bisc label the + lengths they need and are not given with prop, adding them to this + object. Without a prop, auto_bisc stops, as for a tuple of dictionaries, + and bisc raises a ValueError. A bitmap given must be bytes-like (bytes, a + bytearray or a memoryview), of its length's size, with no bit set past + the length's permutations. + + The views read the bitmaps length by length: iterating over a length, + counting it, asking whether a length is there, or whether a permutation + is in a length. A bitmap of S_10 is 454 KB; S_10 as a list of Perm + objects takes 0.9 GB on PyPy 3.11 and 1.3 to 1.8 GB on CPython. """ def __init__( @@ -499,9 +426,9 @@ def add_length(self, length: int) -> None: class _LengthViews(Mapping[int, "_PermsOfLength"]): - """length -> the good (or bad) permutations of that length, read-only, as - the dictionaries of lists auto_bisc builds from the other inputs: a - Mapping whose keys are the lengths with a bitmap.""" + """length -> the good (or bad) permutations of that length, read-only: a + Mapping whose keys are the lengths with a bitmap, through which bisc and + auto_bisc read the permutations.""" def __init__(self, property_bitmaps: PropertyBitmaps, good: bool) -> None: self.property_bitmaps = property_bitmaps @@ -520,12 +447,11 @@ def __len__(self) -> int: class _PermsOfLength(Collection[Perm]): - """The good (or bad) permutations of one length, read from its bitmap, as - the lists auto_bisc builds from the other inputs: a Collection. Iterating - generates the length's permutations in lexicographic order, the order of - the lists, and yields those whose bit is the one wanted; len() is the - count kept with the bitmap; and a permutation is in it when its bit is - the one wanted.""" + """The good (or bad) permutations of one length, read from its bitmap: a + Collection. Iterating generates the length's permutations in + lexicographic order and yields those whose bit is the one wanted; len() + is the count kept with the bitmap; and a permutation is in it when its + bit is the one wanted.""" def __init__( self, property_bitmaps: PropertyBitmaps, length: int, good: bool @@ -537,11 +463,9 @@ def __init__( def __contains__(self, perm: object) -> bool: if not isinstance(perm, Perm) or len(perm) != self.length: return False - shorter_count = sum( - factorial(shorter_length) for shorter_length in range(self.length) - ) bitmap = self.property_bitmaps.bitmaps[self.length] - return _label(bitmap, perm.rank() - shorter_count) == self.wanted_bit + index = perm.rank() - _shorter_count(self.length) + return _label(bitmap, index) == self.wanted_bit def __iter__(self) -> Iterator[Perm]: bitmap = self.property_bitmaps.bitmaps[self.length] @@ -555,6 +479,104 @@ def __len__(self) -> int: return good_count if self.wanted_bit else factorial(self.length) - good_count +def _shorter_count(length: int) -> int: + """The number of permutations shorter than length: a permutation's bit is + its rank() less this.""" + return sum(factorial(shorter_length) for shorter_length in range(length)) + + +def _bitmap_of(length: int, perms: Iterable[Perm]) -> bytearray: + """The bitmap of a length with the bits of the given permutations set. A + permutation given twice sets its bit once.""" + bitmap = bytearray(_bitmap_size(length)) + shorter_count = _shorter_count(length) + for perm in perms: + if len(perm) != length: + raise ValueError( + "A permutation of length {} is given as one of length {}".format( + len(perm), length + ) + ) + if not isinstance(perm, Perm): + perm = Perm(perm) + _set_label(bitmap, perm.rank() - shorter_count) + return bitmap + + +def _from_perms_by_length( + perms_by_length: Mapping[int, Iterable[Perm]], +) -> PropertyBitmaps: + """A PropertyBitmaps, with no property, whose good permutations of each + length are the ones given for it.""" + return PropertyBitmaps( + bitmaps={ + length: _bitmap_of(length, perms) + for length, perms in perms_by_length.items() + } + ) + + +def _from_good_and_bad( + good: Mapping[int, Collection[Perm]], bad: Mapping[int, Collection[Perm]] +) -> PropertyBitmaps: + """A PropertyBitmaps, with no property, from the good and the bad + permutations of each length up to the shorter mapping's longest. A length + one of them lacks is empty in it. Each permutation of each length must be + given exactly once, as good or as bad, or a ValueError is raised.""" + bitmaps = {} + for length in range(min(max(good), max(bad)) + 1): + good_bitmap = _bitmap_of(length, good.get(length, ())) + bad_bitmap = _bitmap_of(length, bad.get(length, ())) + good_bits = int.from_bytes(good_bitmap, "little") + bad_bits = int.from_bytes(bad_bitmap, "little") + every_bit = (1 << factorial(length)) - 1 + if good_bits & bad_bits or good_bits | bad_bits != every_bit: + raise ValueError( + "The good and the bad permutations of length {} must be all of " + "its {} permutations, each once".format(length, factorial(length)) + ) + bitmaps[length] = good_bitmap + return PropertyBitmaps(bitmaps=bitmaps) + + +def _read_stored_set( + name: str, min_length: int +) -> Optional[Tuple[Dict[int, List[Perm]], Dict[int, List[Perm]]]]: + """The good and the bad permutations of the set stored under name in + permuta/resources/bisc, from the first of its files of each kind that go + up to min_length or further, or None if there are none.""" + good_entry = None + bad_entry = None + bisc_dir = os.path.join(os.path.dirname(__file__), "..", "resources", "bisc") + bisc_dir = os.path.abspath(bisc_dir) + + with os.scandir(bisc_dir) as entries: + for entry in entries: + en = entry.name[:-5] + spl = en.split("_") + if spl[0] == name: + if ( + good_entry is None + and spl[1] == "good" + and int(spl[2].split("len")[1]) >= min_length + ): + good_entry = en + if ( + bad_entry is None + and spl[1] == "bad" + and int(spl[2].split("len")[1]) >= min_length + ): + bad_entry = en + if good_entry is not None and bad_entry is not None: + break + if good_entry is not None and bad_entry is not None: + return ( + read_bisc_file(bisc_dir + "/" + good_entry), + read_bisc_file(bisc_dir + "/" + bad_entry), + ) + return None + + def create_bisc_input(N, prop): """ Create a dictionary, D, containing keys 1, 2, 3, ..., N. Each key points to diff --git a/tests/bisc/test_property_bitmaps.py b/tests/bisc/test_property_bitmaps.py index 77b07e7..61b967b 100644 --- a/tests/bisc/test_property_bitmaps.py +++ b/tests/bisc/test_property_bitmaps.py @@ -1,14 +1,27 @@ import contextlib +import importlib import io +import os from collections.abc import Collection, Mapping from math import factorial import pytest -from permuta.bisc.bisc import PropertyBitmaps, auto_bisc, bisc +from permuta.bisc.bisc import ( + PropertyBitmaps, + auto_bisc, + bisc, + create_bisc_input, + read_bisc_file, +) +from permuta.bisc.perm_properties import smooth from permuta.patterns.perm import Perm BASIS_231 = {3: {Perm((1, 2, 0)): [set()]}} +BASIS_WEST_2 = {4: {Perm((1, 2, 3, 0)): [set()], Perm((2, 1, 3, 0)): [{(1, 4)}]}} +# The module, which permuta.bisc's function bisc hides, and its mine() +bisc_module = importlib.import_module("permuta.bisc.bisc") +REAL_MINE = bisc_module.mine def property_bitmaps_up_to(prop, max_length): @@ -116,17 +129,93 @@ def test_bisc_accepts_property_bitmaps(): assert bisc(property_bitmaps_up_to(Perm.stack_sortable, 5), 3, 5) == BASIS_231 -def test_auto_bisc_labels_option(): - assert auto_bisc(Perm.stack_sortable) == BASIS_231 - assert auto_bisc(Perm.stack_sortable, labels="lists") == BASIS_231 - assert auto_bisc(Perm.stack_sortable, labels="bitmaps") == BASIS_231 - with pytest.raises(ValueError, match="labels must be"): - auto_bisc(Perm.stack_sortable, labels="sets") +def mined_bitmaps(monkeypatch, A, m, n=None): + """bisc's answer for A, and the bitmaps of the good permutations it gave + mine().""" + mined = {} + + def recording_mine(goodperms, M, N=None, report=False): + mined.update(goodperms.property_bitmaps.bitmaps) + return REAL_MINE(goodperms, M, N, report) + + monkeypatch.setattr(bisc_module, "mine", recording_mine) + return bisc(A, m, n), mined -def test_labels_option_selects_the_representation(monkeypatch): - # The results are the same either way, so record which lengths are kept as - # bitmaps: none for lists, every length checked for bitmaps +def test_bisc_turns_each_input_into_bitmaps(monkeypatch): + bitmaps = {length: encode(Perm.stack_sortable, length) for length in range(6)} + good_lists = { + length: [perm for perm in Perm.of_length(length) if perm.stack_sortable()] + for length in range(6) + } + listed = [perm for length in range(6) for perm in good_lists[length]] + for A in ( + Perm.stack_sortable, + listed, + good_lists, + property_bitmaps_up_to(Perm.stack_sortable, 5), + ): + assert mined_bitmaps(monkeypatch, A, 3, 5) == (BASIS_231, bitmaps) + # A Mapping must have each length up to max(m, n), as before + with pytest.raises(KeyError): + bisc({length: good_lists[length] for length in (0, 1, 3)}, 3) + + +def test_a_list_sets_the_bits_of_the_permutations_it_lists(monkeypatch): + # A length with none listed is empty + _, mined = mined_bitmaps(monkeypatch, [Perm((0,)), Perm((0, 1, 2))], 2, 3) + assert mined == {0: b"\x00", 1: b"\x01", 2: b"\x00", 3: b"\x01"} + # A permutation listed twice sets one bit. The lists counted (0, 1) twice, + # as if both permutations of length 2 were in the set, and found nothing; + # the increasing permutations avoid 10 + increasing = [Perm(()), Perm((0,)), Perm((0, 1)), Perm((0, 1)), Perm((0, 1, 2))] + answer, mined = mined_bitmaps(monkeypatch, increasing, 2, 3) + assert mined == {0: b"\x01", 1: b"\x01", 2: b"\x01", 3: b"\x01"} + assert answer == {2: {Perm((1, 0)): [set()]}} + + +def test_a_property_leaves_the_lengths_past_n_empty(monkeypatch): + # mine() reads the lengths up to m, and a property labels those up to n; the + # others are empty, as they were in the lists before + answer, mined = mined_bitmaps(monkeypatch, Perm.stack_sortable, 4, 2) + assert mined == { + **{length: encode(Perm.stack_sortable, length) for length in range(3)}, + 3: bytes(1), + 4: bytes(3), + } + assert answer == {3: {perm: [set()] for perm in Perm.of_length(3)}, 4: {}} + + +def first_mined_bitmaps(monkeypatch, prop): + """auto_bisc's answer for prop, and the bitmaps of the good permutations it + first gave mine().""" + mined = [] + + def recording_mine(goodperms, M, N=None, report=False): + mined.append(dict(goodperms.property_bitmaps.bitmaps)) + return REAL_MINE(goodperms, M, N, report) + + monkeypatch.setattr(bisc_module, "mine", recording_mine) + return auto_bisc(prop), mined[0] + + +def test_auto_bisc_turns_each_input_into_bitmaps(monkeypatch): + bitmaps = {length: encode(Perm.stack_sortable, length) for length in range(9)} + good, bad = create_bisc_input(8, Perm.stack_sortable) + listed = [perm for length in range(9) for perm in good[length]] + for prop in ( + Perm.stack_sortable, + listed, + (good, bad), + PropertyBitmaps(bitmaps=bitmaps), + ): + assert first_mined_bitmaps(monkeypatch, prop) == (BASIS_231, bitmaps) + answer, mined = first_mined_bitmaps(monkeypatch, "smooth") + assert answer == {4: {Perm((0, 2, 1, 3)): [set()], Perm((1, 0, 3, 2)): [set()]}} + assert mined == {length: encode(smooth, length) for length in range(9)} + + +def test_auto_bisc_labels_each_length_of_a_property(monkeypatch): added_lengths = [] real_add_length = PropertyBitmaps.add_length @@ -136,12 +225,31 @@ def recording_add_length(self, length): monkeypatch.setattr(PropertyBitmaps, "add_length", recording_add_length) assert auto_bisc(Perm.stack_sortable) == BASIS_231 - assert auto_bisc(Perm.stack_sortable, labels="lists") == BASIS_231 - assert added_lengths == [] - assert auto_bisc(Perm.stack_sortable, labels="bitmaps") == BASIS_231 assert added_lengths == list(range(9)) +def test_a_tuple_must_give_each_permutation_once(): + good, bad = create_bisc_input(8, Perm.stack_sortable) + missing_one = {**bad, 5: bad[5][1:]} + with pytest.raises(ValueError, match="permutations of length 5 must be all"): + auto_bisc((good, missing_one)) + one_twice = {**bad, 4: bad[4] + good[4][:1]} + with pytest.raises(ValueError, match="permutations of length 4 must be all"): + auto_bisc((good, one_twice)) + + +def test_the_stored_sets_give_each_permutation_once(): + folder = "permuta/resources/bisc/" + for file_name in sorted(os.listdir(folder)): + if "_good_len" in file_name: + good_name = file_name[: -len(".json")] + good = read_bisc_file(folder + good_name) + bad = read_bisc_file(folder + good_name.replace("_good_", "_bad_")) + property_bitmaps = bisc_module._from_good_and_bad(good, bad) + longest_length = int(good_name.rsplit("len", 1)[1]) + assert sorted(property_bitmaps.bitmaps) == list(range(longest_length + 1)) + + def test_auto_bisc_property_bitmaps_input(): assert auto_bisc(property_bitmaps_up_to(Perm.stack_sortable, 8)) == BASIS_231 # Given bitmaps and no property: the lengths auto_bisc needs must be there @@ -190,11 +298,11 @@ def test_bisc_labels_missing_lengths_with_the_property(): assert sorted(property_bitmaps.bitmaps) == list(range(8)) -def test_bisc_needs_labels_or_a_property(): +def test_bisc_needs_bitmaps_or_a_property(): bitmaps = {length: encode(Perm.stack_sortable, length) for length in range(4)} with pytest.raises(ValueError, match="none of length 4 and no property"): bisc(PropertyBitmaps(bitmaps=bitmaps), 3, 5) - with pytest.raises(ValueError, match="no labels and no property"): + with pytest.raises(ValueError, match="no bitmaps and no property"): bisc(PropertyBitmaps(), 3) @@ -247,18 +355,8 @@ def test_bisc_accepts_the_views(): assert bisc(property_bitmaps_up_to(Perm.stack_sortable, 5).good, 3) == BASIS_231 -def test_bitmaps_need_a_property(): - for prop in ([Perm((0,))], ({}, {}), "yt_perm_avoids_22"): - with pytest.raises(ValueError, match="needs a property"): - auto_bisc(prop, labels="bitmaps") - property_bitmaps = property_bitmaps_up_to(Perm.stack_sortable, 8) - assert auto_bisc(property_bitmaps, labels="bitmaps") == BASIS_231 - - -def test_west_2_stack_sortable_same_as_lists(): - assert auto_bisc(Perm.west_2_stack_sortable, labels="bitmaps") == auto_bisc( - Perm.west_2_stack_sortable - ) +def test_west_2_stack_sortable(): + assert auto_bisc(Perm.west_2_stack_sortable) == BASIS_WEST_2 def derangement_in_av_213_231_312(perm): @@ -269,13 +367,13 @@ def derangement_in_av_213_231_312(perm): ) -def test_auto_bisc_stops_without_labels_it_needs(): +def test_auto_bisc_stops_without_bitmaps_it_needs(): # Bitmaps up to length 8 and no property: auto_bisc needs length 9 and stops bitmaps = property_bitmaps_up_to(derangement_in_av_213_231_312, 8).bitmaps output = io.StringIO() with contextlib.redirect_stdout(output): assert auto_bisc(PropertyBitmaps(bitmaps=bitmaps)) is None - assert "You need to add labels of longer permutations" in output.getvalue() + assert "You need to add bitmaps of longer permutations" in output.getvalue() class ReachedLength9(Exception): @@ -284,18 +382,13 @@ class ReachedLength9(Exception): def test_auto_bisc_labels_new_lengths_from_the_property(): # When auto_bisc grows past the labelled lengths, it labels the new ones with - # the property; stopped at the first permutation of length 9, both approaches - # have printed the same + # the property: here it is stopped at the first permutation of length 9 def prop(perm): if len(perm) == 9: raise ReachedLength9 return derangement_in_av_213_231_312(perm) - outputs = [] - for labels in ("lists", "bitmaps"): - output = io.StringIO() - with contextlib.redirect_stdout(output), pytest.raises(ReachedLength9): - auto_bisc(prop, labels=labels) - outputs.append(output.getvalue()) - assert outputs[0] == outputs[1] - assert "Adding perms of length 9" in outputs[1] + output = io.StringIO() + with contextlib.redirect_stdout(output), pytest.raises(ReachedLength9): + auto_bisc(prop) + assert "Adding perms of length 9" in output.getvalue() From 84df8045a4f05908455b1ee7568707c1ce1db2f9 Mon Sep 17 00:00:00 2001 From: Henning Ulfarsson Date: Fri, 23 Oct 2026 15:57:18 +0000 Subject: [PATCH 19/38] Read a length with few wanted permutations by unranking them From the review of #209, round 4: with every input turned into bitmaps, reading a length through a view generated all of its permutations, even when few were good. That matters for a short list of long permutations given to bisc: every permutation of length 11 would be generated, about 14 s per read on CPython (40 million, at 0.35 us each), where the lists took microseconds. Henning chose to keep the speed (in the BiSC agent's session, 2026-10-23). A view now unranks the permutations it wants, in increasing index, which is the lexicographic order of Perm.of_length, when at most one in 16 is wanted (_SPARSE_FACTOR). Otherwise it generates the length as before. _wanted_indices skips each byte with nothing wanted, and stops at the last byte's spare bits. A length with nothing wanted is not read at all. The factor of 16: unranking a permutation costs 1.5 to 2.0 us on CPython 3.11 and 0.34 to 0.43 us on PyPy 3.11, at lengths 8 to 12, against 0.35 us and 0.03 us to generate one and test its bit. That is 5 and 13 times as much. With it, bisc given the identities up to length 3 and 20 random permutations of length 10, or of length 11, takes: - CPython 3.11: develop 0.71 s and 1.24 s, now 0.72 s and 1.30 s; - PyPy 3.11: develop 0.53 s and 0.83 s, now 0.58 s and 0.77 s. The bitmap of length 11 adds 5 MB to the peak. A new test reads, by unranking, the stack-sortable permutations of length 8 (1430 of 40320). By generating, it reads those of length 7 (429 of 5040). It also reads the bad views of a set of all but the identity, of length 6 with its first eight permutations bad (a byte of zeros), and of length 3 with none bad. It fails on three faulty versions of _wanted_indices: one that skips zero bytes in a bad view, one that yields in another order, and one that reads the spare bits. Co-Authored-By: Claude Opus 5.5 --- permuta/bisc/bisc.py | 43 ++++++++++++++++++++++++----- tests/bisc/test_property_bitmaps.py | 20 ++++++++++++++ 2 files changed, 56 insertions(+), 7 deletions(-) diff --git a/permuta/bisc/bisc.py b/permuta/bisc/bisc.py index d91fbaa..a19b375 100644 --- a/permuta/bisc/bisc.py +++ b/permuta/bisc/bisc.py @@ -345,6 +345,27 @@ def _label(bitmap: bytes, index: int) -> int: return (bitmap[index // 8] >> (index % 8)) & 1 +# A view unranks the permutations it wants, rather than generating all of the +# length's, when at most one in this many is wanted: unranking one costs about 5 +# (CPython 3.11) to 13 (PyPy 3.11) times as much as generating and testing one, +# measured at lengths 8 to 12 +_SPARSE_FACTOR = 16 + + +def _wanted_indices(bitmap: bytes, length: int, wanted_bit: int) -> Iterator[int]: + """The indices, in increasing order, of the permutations of a length whose + label is wanted_bit. A byte with no such label is skipped whole.""" + skipped_byte = 0xFF if wanted_bit == 0 else 0 + permutation_count = factorial(length) + for byte_index, byte in enumerate(bitmap): + if byte != skipped_byte: + for index in range( + 8 * byte_index, min(8 * byte_index + 8, permutation_count) + ): + if _label(bitmap, index) == wanted_bit: + yield index + + class PropertyBitmaps: """A property's values on the permutations, one bitmap per length: which permutations of each length satisfy it (good) and which do not (bad). @@ -448,10 +469,11 @@ def __len__(self) -> int: class _PermsOfLength(Collection[Perm]): """The good (or bad) permutations of one length, read from its bitmap: a - Collection. Iterating generates the length's permutations in - lexicographic order and yields those whose bit is the one wanted; len() - is the count kept with the bitmap; and a permutation is in it when its - bit is the one wanted.""" + Collection. Iterating yields those whose bit is the one wanted, in + lexicographic order: when few are wanted it unranks them, and otherwise + it generates the length's permutations and tests each bit. len() is the + count kept with the bitmap, and a permutation is in it when its bit is + the one wanted.""" def __init__( self, property_bitmaps: PropertyBitmaps, length: int, good: bool @@ -470,9 +492,16 @@ def __contains__(self, perm: object) -> bool: def __iter__(self) -> Iterator[Perm]: bitmap = self.property_bitmaps.bitmaps[self.length] wanted_bit = self.wanted_bit - for index, perm in enumerate(Perm.of_length(self.length)): - if _label(bitmap, index) == wanted_bit: - yield perm + wanted_count = len(self) + if wanted_count == 0: + return + if wanted_count * _SPARSE_FACTOR <= factorial(self.length): + for index in _wanted_indices(bitmap, self.length, wanted_bit): + yield Perm.unrank(index, self.length) + else: + for index, perm in enumerate(Perm.of_length(self.length)): + if _label(bitmap, index) == wanted_bit: + yield perm def __len__(self) -> int: good_count = self.property_bitmaps.good_counts[self.length] diff --git a/tests/bisc/test_property_bitmaps.py b/tests/bisc/test_property_bitmaps.py index 61b967b..37d111e 100644 --- a/tests/bisc/test_property_bitmaps.py +++ b/tests/bisc/test_property_bitmaps.py @@ -64,6 +64,26 @@ def test_views_are_a_mapping_of_collections(): assert Perm((0, 1, 2, 3)) not in property_bitmaps.bad[3] +def test_views_unrank_a_length_with_few_wanted(): + # A length where at most one permutation in 16 is wanted is read by + # unranking those, in the order of Perm.of_length: the good permutations at + # length 8 here (1430 of 40320), and every bad one of a set of all but the + # identity; length 7 (429 good of 5040) is read by generating the length + property_bitmaps = property_bitmaps_up_to(Perm.stack_sortable, 8) + for length in (7, 8): + good = [perm for perm in Perm.of_length(length) if perm.stack_sortable()] + assert list(property_bitmaps.good[length]) == good + not_increasing = property_bitmaps_up_to(lambda perm: not perm.is_increasing(), 6) + for length in range(7): + assert list(not_increasing.bad[length]) == [Perm(range(length))] + assert list(not_increasing.good[length]) == list(Perm.of_length(length))[1:] + # The first eight permutations of length 6 bad, a byte of zeros + first_eight_bad = PropertyBitmaps(bitmaps={6: b"\x00" + b"\xff" * 89}) + assert list(first_eight_bad.bad[6]) == list(Perm.of_length(6))[:8] + # No bad permutation of length 3: the spare bits of its byte are not read + assert list(PropertyBitmaps(bitmaps={3: b"\x3f"}).bad[3]) == [] + + def test_bit_order_is_perm_unrank(): # Bit i of a length's bitmap is the label of Perm.unrank(i, length), and the # permutation perm is at bit perm.rank() minus the number of shorter From 67f16bb5b09197bf4496aa5a442f16ed19075342 Mon Sep 17 00:00:00 2001 From: Henning Ulfarsson Date: Fri, 23 Oct 2026 20:18:55 +0000 Subject: [PATCH 20/38] Test how bisc and auto_bisc take each kind of input From the review of #209, round 4: three faults that change what bisc returns passed all 60 tests in tests/bisc. The review wrote the tests that catch them, and these are its tests: - test_views_unrank_a_byte_of_good_permutations, beside its mirror for a byte of bad ones: the sparse read must not skip a byte whose eight permutations are all good (blocking finding 1). - tests/bisc/test_bisc_inputs.py, a module of its own, 24 tests of how bisc and auto_bisc take each kind of input (blocking findings 2 and 3, and older gaps): - the default n of a list, a mapping, a PropertyBitmaps and a property; - m > n, and m or n zero, for a list and a mapping; - the empty set; - report=True; - what auto_bisc prints for each input kind, and where it stops; - growth that would label a length it was given; - a duplicate on either side of a tuple; - the views as lengths are added; - where a view starts to unrank. - Three more in tests/bisc/test_property_bitmaps.py: - a list of tuples; - a permutation under another length's key; - a tuple that lacks a length in one dictionary. The review's notes for itself are reworded for Permuta (comments that named develop, the review or the faults). Its helper bitmaps_of looks permutations up in a set rather than a list. Checked: all 88 tests in tests/bisc pass. Each of the review's faults, planted in a copy, fails a new test: the sparse read skipping a byte of ones; a list, or a mapping, read only up to n; a list's longer permutations kept; and a PropertyBitmaps's default n one short. Co-Authored-By: Claude Opus 5.5 --- tests/bisc/test_bisc_inputs.py | 388 ++++++++++++++++++++++++++++ tests/bisc/test_property_bitmaps.py | 54 ++++ 2 files changed, 442 insertions(+) create mode 100644 tests/bisc/test_bisc_inputs.py diff --git a/tests/bisc/test_bisc_inputs.py b/tests/bisc/test_bisc_inputs.py new file mode 100644 index 0000000..d245003 --- /dev/null +++ b/tests/bisc/test_bisc_inputs.py @@ -0,0 +1,388 @@ +"""Tests of how bisc and auto_bisc take each kind of input: the default n of a +list, a mapping, a PropertyBitmaps and a property; m > n for a list and a +mapping; a list with permutations longer than max(m, n); where auto_bisc stops +on a short list or tuple and on a stored set it cannot find; report=True; the +printed output of each input kind; growth that would label a length it was +given; and where a view starts to unrank. The expected values are the answers +bisc and auto_bisc gave before they kept their input as bitmaps, or follow from +the mathematics.""" + +import contextlib +import importlib +import io + +import pytest + +from permuta.bisc.bisc import PropertyBitmaps, auto_bisc, bisc, create_bisc_input +from permuta.bisc.perm_properties import smooth +from permuta.patterns.perm import Perm + +bisc_module = importlib.import_module("permuta.bisc.bisc") + +BASIS_231 = {3: {Perm((1, 2, 0)): [set()]}} +WEST_2_AT_5 = {4: {Perm((1, 2, 3, 0)): [set()], Perm((2, 1, 3, 0)): [{(1, 4)}]}} + + +def good_by_length(prop, longest): + return { + length: [perm for perm in Perm.of_length(length) if prop(perm)] + for length in range(longest + 1) + } + + +def listed(good): + return [perm for perms in good.values() for perm in perms] + + +def bitmaps_of(good): + bitmaps = {} + for length, perms in good.items(): + members = set(perms) + label_bits = 0 + for index, perm in enumerate(Perm.of_length(length)): + if perm in members: + label_bits |= 1 << index + bitmaps[length] = label_bits.to_bytes( + (len(list(Perm.of_length(length))) + 7) // 8, "little" + ) + return bitmaps + + +def run(function, *args): + output = io.StringIO() + with contextlib.redirect_stdout(output): + result = function(*args) + return result, output.getvalue() + + +# The default n of each input kind +@pytest.mark.parametrize("kind", ["list", "mapping", "bitmaps"]) +def test_bisc_without_n_reads_up_to_the_longest_length(kind): + # West-2-stack-sortable needs perms of length 5: n = 4 gives another answer + good = good_by_length(Perm.west_2_stack_sortable, 5) + make = { + "list": lambda: listed(good), + "mapping": lambda: good, + "bitmaps": lambda: PropertyBitmaps(bitmaps=bitmaps_of(good)), + }[kind] + assert bisc(make(), 4) == WEST_2_AT_5 + assert bisc(make(), 4, 4) != WEST_2_AT_5 + + +def test_bisc_of_a_property_without_n_labels_up_to_7_and_says_so(): + seen = set() + + def prop(perm): + seen.add(len(perm)) + return perm.stack_sortable() + + answer, printed = run(bisc, prop, 3) + assert answer == BASIS_231 + assert seen == set(range(8)) + assert printed == "I will use permutations up to length 7\n" + + +# m > n for a list and a mapping +def test_bisc_reads_the_lengths_up_to_m_of_a_list_or_a_mapping_past_n(): + good = good_by_length(Perm.stack_sortable, 6) + avoiding_231 = { + perm: [set()] for perm in Perm.of_length(4) if perm.avoids(Perm((1, 2, 0))) + } + # no good permutation of length 4 in a list that ends at length 3: all of + # the length-4 permutations that avoid 231 are minimal forbidden patterns + assert bisc(listed(good_by_length(Perm.stack_sortable, 3)), 4) == { + **BASIS_231, + 4: avoiding_231, + } + # the lengths up to m = 5 of a longer list or mapping are read, though only + # perms up to n = 3 are mined + expected = {**BASIS_231, 4: {}, 5: {}} + assert bisc(listed(good), 5, 3) == expected + assert bisc(good, 5, 3) == expected + + +# Lengths 0 and 1, m = 0, n = 0, and the empty input +def test_bisc_of_the_empty_set_forbids_the_empty_permutation(): + assert bisc([], 3, 5) == {0: {Perm(()): [set()]}, 1: {}, 2: {}, 3: {}} + assert bisc(lambda perm: False, 3, 5) == { + 0: {Perm(()): [set()]}, + 1: {}, + 2: {}, + 3: {}, + } + with pytest.raises(ValueError): + bisc([], 3) # no n, and no longest length + with pytest.raises(ValueError): + bisc({}, 3) + + +def test_bisc_with_m_or_n_zero(): + # the empty permutation alone: the class {empty}, whose basis is 1 + assert bisc(Perm.stack_sortable, 3, 0) == {1: {Perm((0,)): [set()]}, 2: {}, 3: {}} + assert bisc([Perm(())], 3, 0) == {1: {Perm((0,)): [set()]}, 2: {}, 3: {}} + # patterns of length 0 only: every length-0 permutation is good, "search for + # longer patterns", {} + for A in (Perm.stack_sortable, [Perm(())], {0: [Perm(())]}): + assert run(bisc, A, 0, 0) == ({}, "You need to search for longer patterns\n") + + +# report=True, byte for byte +def test_bisc_report_prints_the_progress_and_the_same_for_each_input_kind(): + good = good_by_length(Perm.stack_sortable, 5) + answer, printed = run(bisc, Perm.stack_sortable, 3, 5, True) + assert answer == BASIS_231 + assert "Starting search for allowed patterns of lengths 0, ..., 3" in printed + assert "Starting search for minimal forbidden patterns" in printed + assert run(bisc, Perm.stack_sortable, 3, 5, False)[1] == "" + for A in (listed(good), good, PropertyBitmaps(bitmaps=bitmaps_of(good))): + assert run(bisc, A, 3, 5, True) == (answer, printed) + + +# What auto_bisc prints, for each input kind +def test_auto_bisc_prints_the_same_for_each_input_kind(): + good, bad = create_bisc_input(8, Perm.stack_sortable) + answer, printed = run(auto_bisc, Perm.stack_sortable) + assert answer == BASIS_231 + # the bitmaps and the pair of dictionaries print what the property prints + assert run(auto_bisc, (good, bad)) == (answer, printed) + assert run(auto_bisc, PropertyBitmaps(bitmaps=bitmaps_of(good))) == ( + answer, + printed, + ) + # a list prints a line for the bad permutations of length 8 it populates + populating = "Populating the dictionary of bad perms of length 8\n" + assert run(auto_bisc, listed(good)) == (answer, populating + printed) + # a stored set prints the line that says so first + smooth_answer, smooth_printed = run(auto_bisc, smooth) + assert run(auto_bisc, "smooth") == ( + smooth_answer, + "Attempting to read perms from permsets\n" + smooth_printed, + ) + + +# Where auto_bisc stops: a list, a tuple, a stored set +def derangement_in_av_213_231_312(perm): + return perm.avoids(Perm((1, 0, 2)), Perm((1, 2, 0)), Perm((2, 0, 1))) and all( + value != index for index, value in enumerate(perm) + ) + + +def test_auto_bisc_stops_when_a_list_or_a_tuple_is_too_short(): + good, bad = create_bisc_input(8, derangement_in_av_213_231_312) + result, printed = run(auto_bisc, listed(good)) + assert result is None + assert printed.endswith("You need to input a longer list of permutations\n") + result, printed = run(auto_bisc, (good, bad)) + assert result is None + assert printed.endswith("You need to add longer permutations to the dictionaries\n") + # a bad dictionary that stops at length 7 stops it, though the good one reaches 8 + short_bad = {length: perms for length, perms in bad.items() if length < 8} + result, printed = run(auto_bisc, (good, short_bad)) + assert result is None + assert printed.endswith("You need to add longer permutations to the dictionaries\n") + + +@pytest.mark.parametrize("short", ["list", "tuple"]) +def test_auto_bisc_wants_permutations_up_to_length_8(short): + good, bad = create_bisc_input(7, Perm.stack_sortable) + given = listed(good) if short == "list" else (good, bad) + assert run(auto_bisc, given) == ( + None, + "You should have permutations up to length at least 8\n", + ) + assert run(auto_bisc, []) == ( + None, + "You should have permutations up to length at least 8\n", + ) + assert run(auto_bisc, ({}, {})) == ( + None, + "You should have permutations up to length at least 8\n", + ) + + +def test_auto_bisc_of_a_stored_set_that_does_not_exist_or_runs_out(): + assert run(auto_bisc, "nosuchset") == ( + None, + "Attempting to read perms from permsets\nThe required files do not exist\n", + ) + # meander is stored to length 8: auto_bisc learns from perms of length 7, and + # asks for the files again, for length 9 + result, printed = run(auto_bisc, "meander") + assert result is None + assert printed.endswith( + "Attempting to read perms from permsets\nThe required files do not exist\n" + ) + assert printed.count("Attempting to read perms from permsets") == 2 + + +def test_auto_bisc_reads_a_stored_set_again_for_a_longer_length(monkeypatch): + # No stored set that auto_bisc can name is longer than 8 and learned from perms + # of length 8 or more, so the files are faked: the increasing permutations of + # even length are no class (the bad (0) avoids their basis, 21), so auto_bisc + # keeps growing n and reading + def cycle(perm): + return len(perm) % 2 == 0 and perm.is_increasing() + + reads = [] + + def fake_read_stored_set(name, min_length): + reads.append(min_length) + return ( + create_bisc_input(8, cycle) + if min_length <= 8 + else (create_bisc_input(9, cycle) if min_length == 9 else None) + ) + + monkeypatch.setattr(bisc_module, "_read_stored_set", fake_read_stored_set) + # the sanity check of the learned patterns must see the longer set it was read + # again for: the bad permutations of length L, where L is the length asked + real_check = bisc_module.patterns_suffice_for_bad + checks = [] + + def recording_check(patterns, length, bad, stop_on_failure=False): + checks.append((length, max(bad.keys()))) + return real_check(patterns, length, bad, stop_on_failure=stop_on_failure) + + monkeypatch.setattr(bisc_module, "patterns_suffice_for_bad", recording_check) + result, printed = run(auto_bisc, "cycles") + assert result is None + assert reads == [8, 9, 10] + assert (9, 9) in checks and all(longest >= length for length, longest in checks) + assert printed.endswith( + "Attempting to read perms from permsets\nThe required files do not exist\n" + ) + + +# Growth does not replace the bitmaps auto_bisc was given +class ReachedLength10(Exception): + pass + + +def test_auto_bisc_growth_does_not_label_a_length_it_was_given(): + labeled_lengths = [] + + def prop(perm): + if len(perm) == 10: + raise ReachedLength10 + labeled_lengths.append(len(perm)) + return derangement_in_av_213_231_312(perm) + + bitmaps = bitmaps_of(good_by_length(derangement_in_av_213_231_312, 8)) + # length 9: deliberately given as all bad, and not asked of the property + bitmaps[9] = bytes(45360) + property_bitmaps = PropertyBitmaps(prop, bitmaps=bitmaps) + try: + run(auto_bisc, property_bitmaps) + except ReachedLength10: + pass + assert 9 not in labeled_lengths + assert property_bitmaps.bitmaps[9] == bytes(45360) + + +# A permutation given twice on one side counts once, for auto_bisc too +def test_a_permutation_given_twice_counts_once_in_auto_bisc(): + good, bad = create_bisc_input(8, Perm.stack_sortable) + answer, printed = run(auto_bisc, Perm.stack_sortable) + twice = {**good, 5: good[5] + good[5][:3]} + assert run(auto_bisc, (twice, bad)) == (answer, printed) + assert run(auto_bisc, listed(twice)) == ( + answer, + "Populating the dictionary of bad perms of length 8\n" + printed, + ) + also_bad_twice = {**bad, 6: bad[6] + bad[6][:2]} + assert run(auto_bisc, (good, also_bad_twice)) == (answer, printed) + + +# A dictionary of the pair may lack a length it has no permutation of +def test_a_tuple_may_lack_a_length_with_no_bad_permutations(): + good, bad = create_bisc_input(8, Perm.stack_sortable) + # every permutation of length 0, 1 and 2 is stack-sortable: no bad ones + bad = {length: perms for length, perms in bad.items() if perms} + assert sorted(bad) == [3, 4, 5, 6, 7, 8] + assert auto_bisc((good, bad)) == BASIS_231 + + +# A bytearray given as a bitmap is copied +def test_a_bytearray_bitmap_is_copied(): + raw = bytearray(b"\x3f") + property_bitmaps = PropertyBitmaps(bitmaps={3: raw}) + raw[0] = 0 + assert len(property_bitmaps.good[3]) == 6 + assert len(list(property_bitmaps.good[3])) == 6 + + +# The views follow the PropertyBitmaps as lengths are added +def test_the_views_follow_added_lengths(): + property_bitmaps = PropertyBitmaps(Perm.stack_sortable) + good, bad = property_bitmaps.good, property_bitmaps.bad + assert 4 not in good and len(good) == 0 + property_bitmaps.add_length(4) + assert 4 in good and 4 in bad and list(good) == [4] + assert len(good[4]) == 14 and len(bad[4]) == 10 + + +# add_length replaces a bitmap it was given +def test_add_length_replaces_the_bitmap_of_a_length(): + property_bitmaps = PropertyBitmaps(Perm.stack_sortable, bitmaps={3: bytes(1)}) + assert len(property_bitmaps.good[3]) == 0 + property_bitmaps.add_length(3) + assert len(property_bitmaps.good[3]) == 5 and len(property_bitmaps.bad[3]) == 1 + assert list(property_bitmaps.bad[3]) == [Perm((1, 2, 0))] + + +# A PropertyBitmaps with only a property is labeled up to length 8 at the start +class ReachedLength9(Exception): + pass + + +def test_auto_bisc_of_a_property_only_input_labels_the_lengths_up_to_8(): + def prop(perm): + if len(perm) > 8: + raise ReachedLength9 + return perm.stack_sortable() + + property_bitmaps = PropertyBitmaps(prop) + assert auto_bisc(property_bitmaps) == BASIS_231 + assert sorted(property_bitmaps.bitmaps) == list(range(9)) + + +# The tuple, the list and the bitmaps without a property stop at the same point +def test_auto_bisc_stops_at_the_same_point_for_each_input_kind(): + good, bad = create_bisc_input(8, derangement_in_av_213_231_312) + bitmaps_message = "You need to add bitmaps of longer permutations\n" + bitmaps_printed = run(auto_bisc, PropertyBitmaps(bitmaps=bitmaps_of(good)))[1] + assert bitmaps_printed.endswith(bitmaps_message) + body = bitmaps_printed[: -len(bitmaps_message)] + result, printed = run(auto_bisc, (good, bad)) + assert result is None + assert printed == body + "You need to add longer permutations to the dictionaries\n" + result, printed = run(auto_bisc, listed(good)) + assert result is None + assert printed == ( + "Populating the dictionary of bad perms of length 8\n" + + body + + "You need to input a longer list of permutations\n" + ) + + +# A view unranks when at most one in 16 is wanted, on both sides of the edge +@pytest.mark.parametrize("good", [True, False]) +def test_a_view_unranks_exactly_when_at_most_one_in_16_is_wanted(monkeypatch, good): + unranked = [] + real_unrank = Perm.unrank + + def counting_unrank(index, length): + unranked.append(index) + return real_unrank(index, length) + + monkeypatch.setattr(Perm, "unrank", staticmethod(counting_unrank)) + # length 6 has 720 permutations: 45 is one in 16, 46 is more + for wanted, expect_unranking in ((45, True), (46, False)): + label_bits = (1 << wanted) - 1 # the first `wanted` permutations + if not good: + label_bits = ((1 << 720) - 1) ^ label_bits + view = PropertyBitmaps(bitmaps={6: label_bits.to_bytes(90, "little")}) + view = view.good[6] if good else view.bad[6] + unranked.clear() + assert len(list(view)) == wanted + assert bool(unranked) == expect_unranking diff --git a/tests/bisc/test_property_bitmaps.py b/tests/bisc/test_property_bitmaps.py index 37d111e..73d2bd2 100644 --- a/tests/bisc/test_property_bitmaps.py +++ b/tests/bisc/test_property_bitmaps.py @@ -84,6 +84,21 @@ def test_views_unrank_a_length_with_few_wanted(): assert list(PropertyBitmaps(bitmaps={3: b"\x3f"}).bad[3]) == [] +def test_views_unrank_a_byte_of_good_permutations(): + # The first eight permutations of length 6 good, a byte of ones: 8 of 720, + # read by unranking, and the byte must not be skipped + first_eight_good = PropertyBitmaps(bitmaps={6: b"\xff" + b"\x00" * 89}) + assert list(first_eight_good.good[6]) == list(Perm.of_length(6))[:8] + # A property that holds on a lexicographic block: the 120 permutations of + # length 7 that start with 0, 1 fill fifteen bytes + starts_0_1 = property_bitmaps_up_to( + lambda perm: len(perm) < 2 or (perm[0] == 0 and perm[1] == 1), 7 + ) + assert list(starts_0_1.good[7]) == [ + perm for perm in Perm.of_length(7) if perm[0] == 0 and perm[1] == 1 + ] + + def test_bit_order_is_perm_unrank(): # Bit i of a length's bitmap is the label of Perm.unrank(i, length), and the # permutation perm is at bit perm.rank() minus the number of shorter @@ -412,3 +427,42 @@ def prop(perm): with contextlib.redirect_stdout(output), pytest.raises(ReachedLength9): auto_bisc(prop) assert "Adding perms of length 9" in output.getvalue() + + +def test_a_list_of_tuples_is_read_as_permutations(): + # Tuples are read as the permutations they are, as they always were + perms = [ + perm + for length in range(6) + for perm in Perm.of_length(length) + if perm.stack_sortable() + ] + assert bisc([tuple(perm) for perm in perms], 3, 5) == bisc(perms, 3, 5) + + +def test_a_permutation_under_the_wrong_length_is_refused(): + with pytest.raises(ValueError, match="length 3 is given as one of length 2"): + bisc({0: [Perm(())], 1: [Perm((0,))], 2: [Perm((0, 1, 2))]}, 2) + + +def test_a_tuple_may_lack_a_length_in_one_dictionary(monkeypatch): + # A length one dictionary lacks is empty in it: here no permutation of + # length 0 is good + good, bad = create_bisc_input(8, Perm.stack_sortable) + good = {length: perms for length, perms in good.items() if perms} + bad = dict(bad) + bad[0] = bad.get(0, []) + good.pop(0, []) + given = [] + + class MineCalled(Exception): + pass + + def recording_mine(goodperms, M, N=None, report=False): + given.append({length: list(goodperms[length]) for length in goodperms}) + raise MineCalled + + monkeypatch.setattr(bisc_module, "mine", recording_mine) + with pytest.raises(MineCalled): + auto_bisc((good, bad)) + assert given[0][0] == [] + assert given[0][3] == [perm for perm in Perm.of_length(3) if perm.stack_sortable()] From c6dcbdfbdb2cc7744b01ca1eddf99a6cb119e069 Mon Sep 17 00:00:00 2001 From: Henning Ulfarsson Date: Fri, 23 Oct 2026 20:21:23 +0000 Subject: [PATCH 21/38] Correct what the texts say a tuple and a property do From the review of #209, round 4, its statements to correct: 1. "Exactly once" was not what the code does: a permutation given twice on one side of a tuple, or of a stored set, is accepted and counted once. auto_bisc's and _from_good_and_bad's docstrings and the CHANGELOG now say that the good and the bad permutations of each length must not overlap and must together be all of that length's permutations. The ValueError says so, with how many permutations are in both and how many in neither. 2. auto_bisc's docstring said that a property labels each length when auto_bisc first needs it; lengths 0 to 8 are labeled at the start, and only the longer ones when needed. 3. The line "Populating the dictionary of bad perms of length" keeps what auto_bisc printed for a list, though nothing is populated now. A comment says so. 4. "As it was in the lists before", in bisc's docstring, a comment and the tests, is reworded for a reader who never saw the lists. The CHANGELOG also has a line for the ValueError given for a permutation under another length's key (new, from the review's untested behaviors). Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 7 ++++-- permuta/bisc/bisc.py | 38 ++++++++++++++++++----------- tests/bisc/test_property_bitmaps.py | 19 +++++++-------- 3 files changed, 38 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3331000..7dbcdaf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,8 +20,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. depending on the Python. - A permutation listed twice in the input of `bisc` or `auto_bisc` counts once. Before, duplicates could make `bisc` take a length as complete when it was not. -- `auto_bisc` raises a `ValueError` when a tuple of dictionaries, or a stored set, - does not give each permutation of a length exactly once, as good or as bad. +- `auto_bisc` raises a `ValueError` when the good and the bad permutations of a + length, from a tuple of dictionaries or a stored set, overlap or are not + together all of that length's permutations. +- `bisc` and `auto_bisc` raise a `ValueError` for a permutation given under the + key of another length. - `bisc` accepts any mapping of lengths to permutations, not only a `dict`. ### Fixed diff --git a/permuta/bisc/bisc.py b/permuta/bisc/bisc.py index a19b375..b7a8a3c 100644 --- a/permuta/bisc/bisc.py +++ b/permuta/bisc/bisc.py @@ -38,8 +38,8 @@ def bisc( Without n, a property uses length 7, and the others their longest length. - A list sets the bits of the permutations it lists, and a length with none listed is empty. - - A property labels the lengths up to n; a length past n, when m > n, is - empty, as it was in the lists before. + - A property labels the lengths up to n, and a length past n, when m > n, + is read as empty. - A Mapping is read for each length up to max(m, n), and one it lacks is its KeyError. - A PropertyBitmaps labels the lengths up to max(m, n) it lacks with its @@ -67,7 +67,7 @@ def bisc( print("I will use permutations up to length 7") n = 7 # The property labels the lengths up to n. mine() reads those up to m - # too, and a length past n is empty, as it was in the lists before + # too, and a length past n is read as empty property_bitmaps = PropertyBitmaps( A, bitmaps={ @@ -143,13 +143,15 @@ def auto_bisc( permuta/resources/bisc, or a PropertyBitmaps. Each is turned into a PropertyBitmaps at the start, whose views give the good and the bad permutations of each length. - - A property labels each length when auto_bisc first needs it. + - A property labels the lengths up to 8 at the start, and each longer one + when auto_bisc first needs it. - A list sets the bits of the permutations it lists, and the others are bad. - - A tuple of dictionaries, or a stored set, must give each permutation of - each length up to the shorter dictionary's longest exactly once, as good - or as bad, or a ValueError is raised. A length one of them lacks is - empty in it. + - For a tuple of dictionaries, or a stored set, the good and the bad + permutations of each length up to the shorter dictionary's longest must + not overlap, and must together be all of that length's permutations, or + a ValueError is raised. A length one of them lacks is empty in it, and a + permutation given twice on one side counts once. - A PropertyBitmaps labels the lengths it lacks with its property, and they are added to it, so that a later run reuses them. Without a property, auto_bisc stops when it needs a length it lacks, as it does @@ -167,6 +169,8 @@ def auto_bisc( print("You should have permutations up to length at least 8") return longest_length = max(listed_lengths) + # Printed as when the bad permutations were made into lists, so that + # what auto_bisc prints does not change; now they are the bitmaps' zeros for i in range(L, longest_length + 1): print("Populating the dictionary of bad perms of length ", i) perms_by_length: Dict[int, List[Perm]] = { @@ -550,19 +554,25 @@ def _from_good_and_bad( ) -> PropertyBitmaps: """A PropertyBitmaps, with no property, from the good and the bad permutations of each length up to the shorter mapping's longest. A length - one of them lacks is empty in it. Each permutation of each length must be - given exactly once, as good or as bad, or a ValueError is raised.""" + one of them lacks is empty in it, and a permutation given twice on one + side counts once. The good and the bad permutations of each length must + not overlap, and must together be all of its permutations, or a + ValueError is raised.""" bitmaps = {} for length in range(min(max(good), max(bad)) + 1): good_bitmap = _bitmap_of(length, good.get(length, ())) bad_bitmap = _bitmap_of(length, bad.get(length, ())) good_bits = int.from_bytes(good_bitmap, "little") bad_bits = int.from_bytes(bad_bitmap, "little") - every_bit = (1 << factorial(length)) - 1 - if good_bits & bad_bits or good_bits | bad_bits != every_bit: + in_both_count = (good_bits & bad_bits).bit_count() + in_neither_count = factorial(length) - (good_bits | bad_bits).bit_count() + if in_both_count or in_neither_count: raise ValueError( - "The good and the bad permutations of length {} must be all of " - "its {} permutations, each once".format(length, factorial(length)) + "The good and the bad permutations of length {} must not overlap " + "and must together be all of its {} permutations (in both: {}, " + "in neither: {})".format( + length, factorial(length), in_both_count, in_neither_count + ) ) bitmaps[length] = good_bitmap return PropertyBitmaps(bitmaps=bitmaps) diff --git a/tests/bisc/test_property_bitmaps.py b/tests/bisc/test_property_bitmaps.py index 73d2bd2..84f061c 100644 --- a/tests/bisc/test_property_bitmaps.py +++ b/tests/bisc/test_property_bitmaps.py @@ -191,7 +191,7 @@ def test_bisc_turns_each_input_into_bitmaps(monkeypatch): property_bitmaps_up_to(Perm.stack_sortable, 5), ): assert mined_bitmaps(monkeypatch, A, 3, 5) == (BASIS_231, bitmaps) - # A Mapping must have each length up to max(m, n), as before + # A Mapping must have each length up to max(m, n) with pytest.raises(KeyError): bisc({length: good_lists[length] for length in (0, 1, 3)}, 3) @@ -200,9 +200,8 @@ def test_a_list_sets_the_bits_of_the_permutations_it_lists(monkeypatch): # A length with none listed is empty _, mined = mined_bitmaps(monkeypatch, [Perm((0,)), Perm((0, 1, 2))], 2, 3) assert mined == {0: b"\x00", 1: b"\x01", 2: b"\x00", 3: b"\x01"} - # A permutation listed twice sets one bit. The lists counted (0, 1) twice, - # as if both permutations of length 2 were in the set, and found nothing; - # the increasing permutations avoid 10 + # A permutation listed twice sets one bit, so (0, 1) is not taken for both + # permutations of length 2, and the increasing permutations avoid 10 increasing = [Perm(()), Perm((0,)), Perm((0, 1)), Perm((0, 1)), Perm((0, 1, 2))] answer, mined = mined_bitmaps(monkeypatch, increasing, 2, 3) assert mined == {0: b"\x01", 1: b"\x01", 2: b"\x01", 3: b"\x01"} @@ -211,7 +210,7 @@ def test_a_list_sets_the_bits_of_the_permutations_it_lists(monkeypatch): def test_a_property_leaves_the_lengths_past_n_empty(monkeypatch): # mine() reads the lengths up to m, and a property labels those up to n; the - # others are empty, as they were in the lists before + # others are read as empty answer, mined = mined_bitmaps(monkeypatch, Perm.stack_sortable, 4, 2) assert mined == { **{length: encode(Perm.stack_sortable, length) for length in range(3)}, @@ -263,14 +262,14 @@ def recording_add_length(self, length): assert added_lengths == list(range(9)) -def test_a_tuple_must_give_each_permutation_once(): +def test_the_good_and_bad_of_a_tuple_must_split_each_length(): good, bad = create_bisc_input(8, Perm.stack_sortable) missing_one = {**bad, 5: bad[5][1:]} - with pytest.raises(ValueError, match="permutations of length 5 must be all"): + with pytest.raises(ValueError, match=r"length 5 .* \(in both: 0, in neither: 1\)"): auto_bisc((good, missing_one)) - one_twice = {**bad, 4: bad[4] + good[4][:1]} - with pytest.raises(ValueError, match="permutations of length 4 must be all"): - auto_bisc((good, one_twice)) + in_both = {**bad, 4: bad[4] + good[4][:1]} + with pytest.raises(ValueError, match=r"length 4 .* \(in both: 1, in neither: 0\)"): + auto_bisc((good, in_both)) def test_the_stored_sets_give_each_permutation_once(): From 925c4a94c4651f269d95318f155441a0fd9d14b7 Mon Sep 17 00:00:00 2001 From: Henning Ulfarsson Date: Fri, 23 Oct 2026 20:23:00 +0000 Subject: [PATCH 22/38] Stop auto_bisc when a tuple's bad permutations stop short From the review of #209, round 4, its edge case 5: a tuple whose bad dictionary stops before the good one's length 8 was checked only on the good one. When the bad permutations stopped before length 7, the bitmaps, which hold only the lengths both dictionaries give, lacked a length that auto_bisc learns from before its sanity checks reach length 8. So bisc raised "bisc needs bitmaps up to length 7, and the PropertyBitmaps has none of length 7 and no property", naming an object the user never gave. develop printed "You need to add longer permutations to the dictionaries" and returned None. auto_bisc now checks, after turning the tuple into bitmaps, that they reach length L - 1, and otherwise prints that message and returns None. A bad dictionary that stops at length 7 still goes the way develop went, as the review's test checks. A new case in the same test, with the bad permutations stopping at length 6, raised the ValueError before this commit. Co-Authored-By: Claude Opus 5.5 --- permuta/bisc/bisc.py | 5 +++++ tests/bisc/test_bisc_inputs.py | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/permuta/bisc/bisc.py b/permuta/bisc/bisc.py index b7a8a3c..3df6231 100644 --- a/permuta/bisc/bisc.py +++ b/permuta/bisc/bisc.py @@ -208,6 +208,11 @@ def auto_bisc( print("You should have permutations up to length at least 8") return property_bitmaps = _from_good_and_bad(prop[0], prop[1]) + if L - 1 not in property_bitmaps.bitmaps: + # The bad permutations stop short of the lengths up to L - 1, which + # auto_bisc learns from before its sanity checks reach L + print("You need to add longer permutations to the dictionaries") + return elif isinstance(prop, str): # If you pass in a string we assume it is pointing to a file in permsets diff --git a/tests/bisc/test_bisc_inputs.py b/tests/bisc/test_bisc_inputs.py index d245003..09dcade 100644 --- a/tests/bisc/test_bisc_inputs.py +++ b/tests/bisc/test_bisc_inputs.py @@ -180,6 +180,13 @@ def test_auto_bisc_stops_when_a_list_or_a_tuple_is_too_short(): result, printed = run(auto_bisc, (good, short_bad)) assert result is None assert printed.endswith("You need to add longer permutations to the dictionaries\n") + # one that stops at length 6 stops it at once: auto_bisc learns from the + # permutations up to length 7 before its sanity checks reach length 8 + shorter_bad = {length: perms for length, perms in bad.items() if length < 7} + assert run(auto_bisc, (good, shorter_bad)) == ( + None, + "You need to add longer permutations to the dictionaries\n", + ) @pytest.mark.parametrize("short", ["list", "tuple"]) From 0d437fa695990e6bfda73307d23a1338cc23e27f Mon Sep 17 00:00:00 2001 From: Henning Ulfarsson Date: Fri, 23 Oct 2026 20:25:08 +0000 Subject: [PATCH 23/38] Let in take any sequence that is a permutation of the length From the review of #209, round 4, its edge case 7: in on a view took only a Perm of the length and read the bit at its rank without checking that it is a permutation. - (0, 1, 2) in good[3] was False, where the lists found it, since a tuple equals the Perm. - Perm((1, 2, 3)) in good[3] was True, its rank landing on another permutation's bit. - Perm((5, 5, 5)) in good[3] raised IndexError. Now in takes a Perm or any other sequence of the numbers 0 to length - 1, each once, and reads the bit of that permutation; anything else is in neither view. bisc and auto_bisc do not use in. _shorter_count, which in called for every permutation, is cached, as the review suggested. The views test checks a tuple and a list of the length, and a Perm that is no permutation of it, one with an entry repeated, a string and an int. It fails on the commit before. Co-Authored-By: Claude Opus 5.5 --- permuta/bisc/bisc.py | 15 ++++++++++++--- tests/bisc/test_property_bitmaps.py | 8 ++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/permuta/bisc/bisc.py b/permuta/bisc/bisc.py index 3df6231..af0d9da 100644 --- a/permuta/bisc/bisc.py +++ b/permuta/bisc/bisc.py @@ -1,7 +1,8 @@ +import functools import json import os import types -from collections.abc import Collection, Mapping +from collections.abc import Collection, Mapping, Sequence from math import factorial from typing import Callable, Dict, Iterable, Iterator, List, Optional, Tuple, Union @@ -492,10 +493,17 @@ def __init__( self.wanted_bit = 1 if good else 0 def __contains__(self, perm: object) -> bool: - if not isinstance(perm, Perm) or len(perm) != self.length: + # A permutation of this length: a Perm, or any other sequence of the + # numbers 0 to length - 1, each once + if not ( + isinstance(perm, Sequence) + and len(perm) == self.length + and all(isinstance(value, int) for value in perm) + and set(perm) == set(range(self.length)) + ): return False bitmap = self.property_bitmaps.bitmaps[self.length] - index = perm.rank() - _shorter_count(self.length) + index = Perm(perm).rank() - _shorter_count(self.length) return _label(bitmap, index) == self.wanted_bit def __iter__(self) -> Iterator[Perm]: @@ -517,6 +525,7 @@ def __len__(self) -> int: return good_count if self.wanted_bit else factorial(self.length) - good_count +@functools.lru_cache(maxsize=None) def _shorter_count(length: int) -> int: """The number of permutations shorter than length: a permutation's bit is its rank() less this.""" diff --git a/tests/bisc/test_property_bitmaps.py b/tests/bisc/test_property_bitmaps.py index 84f061c..9e2aff3 100644 --- a/tests/bisc/test_property_bitmaps.py +++ b/tests/bisc/test_property_bitmaps.py @@ -62,6 +62,14 @@ def test_views_are_a_mapping_of_collections(): # A permutation of another length is in neither view of a length assert Perm((0, 1)) not in property_bitmaps.good[3] assert Perm((0, 1, 2, 3)) not in property_bitmaps.bad[3] + # Any sequence that is a permutation of the length is read as one, and + # nothing else is in a view + assert (0, 1, 2) in property_bitmaps.good[3] and [1, 2, 0] in property_bitmaps.bad[ + 3 + ] + for not_a_perm_of_3 in (Perm((1, 2, 3)), Perm((5, 5, 5)), (0, 0, 1), "012", 3): + assert not_a_perm_of_3 not in property_bitmaps.good[3] + assert not_a_perm_of_3 not in property_bitmaps.bad[3] def test_views_unrank_a_length_with_few_wanted(): From b4b27e365031ed4851d760c53a2e793d9e8c1a0c Mon Sep 17 00:00:00 2001 From: Henning Ulfarsson Date: Fri, 23 Oct 2026 20:28:55 +0000 Subject: [PATCH 24/38] Readability points from the review of round 4 From the review of #209, round 4, its readability points and two of its smaller suggestions: - 9: the views' flag had three names (good, is_good, wanted_bit). Now it is wanted_bit throughout, passed by keyword: _LengthViews(self, wanted_bit=1) for good and wanted_bit=0 for bad. - 10: _SPARSE_FACTOR is now _UNRANK_WHEN_ONE_IN, and its comment says why 16: above the measured 5 (CPython) and 13 (PyPy). The early return for a length with nothing wanted has a comment. - 12: bisc returns _Patterns and auto_bisc Optional[_Patterns]. _Patterns is the forbidden patterns' type, length -> {pattern: [shadings]}. auto_bisc's bare returns say None, as mypy wants with the return type. permuta/bisc/__init__.py no longer starts with "# type: ignore". bisc_subfunctions.py's "# mypy: ignore-errors" has a comment saying why and that a pull request of its own will type the module. - 13: American English, "labeled", in a comment and in the tests' names and locals. - 14: a bitmap of the wrong size says how many bytes it has and must have, and a permutation under another length's key is named. _read_stored_set's en and spl are entry_name and name_parts. - 8 and 14: the class docstring gives a bitmap's size in bytes and the errors a bad bitmap raises, and says that bitmaps and good_counts are for reading, kept together by the constructor and add_length. Not taken: - the rename of prop, which is auto_bisc's and create_bisc_input's name for the same thing ("property" would shadow the builtin); - one method for the five loops that label missing lengths, which Henning left for later in rounds 2 and 3; - a guard for two bad bases in a row (6), which the review says cannot happen naturally and needs no action. Checked: mypy 2.4.0 finds nothing for Python 3.14 and 3.10, now reading permuta/bisc/__init__.py too. The 88 tests in tests/bisc pass on CPython 3.11, and with the README's examples on PyPy 3.11. Co-Authored-By: Claude Opus 5.5 --- permuta/bisc/__init__.py | 2 - permuta/bisc/bisc.py | 109 ++++++++++++++++------------ permuta/bisc/bisc_subfunctions.py | 3 + tests/bisc/test_property_bitmaps.py | 29 ++++---- 4 files changed, 80 insertions(+), 63 deletions(-) diff --git a/permuta/bisc/__init__.py b/permuta/bisc/__init__.py index 6653d8f..d940155 100644 --- a/permuta/bisc/__init__.py +++ b/permuta/bisc/__init__.py @@ -1,5 +1,3 @@ -# type: ignore - from .bisc import ( PropertyBitmaps, auto_bisc, diff --git a/permuta/bisc/bisc.py b/permuta/bisc/bisc.py index af0d9da..2a143ee 100644 --- a/permuta/bisc/bisc.py +++ b/permuta/bisc/bisc.py @@ -4,7 +4,7 @@ import types from collections.abc import Collection, Mapping, Sequence from math import factorial -from typing import Callable, Dict, Iterable, Iterator, List, Optional, Tuple, Union +from typing import Callable, Dict, Iterable, Iterator, List, Optional, Set, Tuple, Union from permuta.bisc.bisc_subfunctions import ( forb, @@ -18,6 +18,11 @@ ) from permuta.patterns.perm import Perm +# The forbidden patterns bisc and auto_bisc return: length -> {pattern: +# [shadings]}, each shading a set of boxes, and an empty one making the pattern +# classical +_Patterns = Dict[int, Dict[Perm, List[Set[Tuple[int, int]]]]] + def bisc( A: Union[ @@ -29,7 +34,7 @@ def bisc( m: int, n: Optional[int] = None, report: bool = False, -): +) -> _Patterns: """ Mine the forbidden patterns of length up to m from the permutations up to length n of a set A: a list of permutations, a property (a function), a @@ -117,7 +122,7 @@ def bisc( assert False ci, goodpatts = mine(property_bitmaps.good, m, n, report=report) - SG = forb(ci, goodpatts, m, report=report) + SG: _Patterns = forb(ci, goodpatts, m, report=report) return SG @@ -130,7 +135,7 @@ def auto_bisc( str, "PropertyBitmaps", ], -): +) -> Optional[_Patterns]: """ Learn a basis of forbidden patterns for a set of permutations, growing the length of the patterns and of the permutations they are learned from @@ -168,7 +173,7 @@ def auto_bisc( listed_lengths = {len(perm) for perm in prop} if L not in listed_lengths: print("You should have permutations up to length at least 8") - return + return None longest_length = max(listed_lengths) # Printed as when the bad permutations were made into lists, so that # what auto_bisc prints does not change; now they are the bitmaps' zeros @@ -189,14 +194,14 @@ def auto_bisc( property_bitmaps.add_length(i) elif isinstance(prop, PropertyBitmaps): - # The lengths the PropertyBitmaps lacks are labelled with its + # The lengths the PropertyBitmaps lacks are labeled with its # property, if it has one property_bitmaps = prop for i in range(L + 1): if i not in property_bitmaps.bitmaps: if property_bitmaps.prop is None: print("You should have bitmaps up to length at least 8") - return + return None property_bitmaps.add_length(i) elif ( @@ -207,13 +212,13 @@ def auto_bisc( # If you already have the dictionaries you can pass them in as a tuple if L not in prop[0].keys(): print("You should have permutations up to length at least 8") - return + return None property_bitmaps = _from_good_and_bad(prop[0], prop[1]) if L - 1 not in property_bitmaps.bitmaps: # The bad permutations stop short of the lengths up to L - 1, which # auto_bisc learns from before its sanity checks reach L print("You need to add longer permutations to the dictionaries") - return + return None elif isinstance(prop, str): # If you pass in a string we assume it is pointing to a file in permsets @@ -221,7 +226,7 @@ def auto_bisc( stored_set = _read_stored_set(prop, L) if stored_set is None: print("The required files do not exist") - return + return None property_bitmaps = _from_good_and_bad(*stored_set) else: @@ -266,7 +271,7 @@ def auto_bisc( basis = bases[0] show_me_basis(basis, dict_numbs_to_patts) - sg = to_sg_format(basis, dict_numbs_to_patts) + sg: _Patterns = to_sg_format(basis, dict_numbs_to_patts) val, avoiding_perms = patterns_suffice_for_bad( sg, L, B, stop_on_failure=True @@ -311,7 +316,7 @@ def auto_bisc( L = n + 1 if isinstance(prop, list) and L > max(A.keys()): print("You need to input a longer list of permutations") - return + return None elif isinstance(prop, (types.FunctionType, PropertyBitmaps)): # The property labels the new lengths @@ -319,13 +324,13 @@ def auto_bisc( if i not in property_bitmaps.bitmaps: if property_bitmaps.prop is None: print("You need to add bitmaps of longer permutations") - return + return None print("Adding perms of length {}".format(i)) property_bitmaps.add_length(i) elif isinstance(prop, tuple) and L > max(A.keys()): print("You need to add longer permutations to the dictionaries") - return + return None elif isinstance(prop, str) and L > max(A.keys()): # If you pass in a string we assume it is pointing to a file in permsets @@ -333,7 +338,7 @@ def auto_bisc( stored_set = _read_stored_set(prop, L) if stored_set is None: print("The required files do not exist") - return + return None property_bitmaps = _from_good_and_bad(*stored_set) A, B = property_bitmaps.good, property_bitmaps.bad @@ -356,10 +361,10 @@ def _label(bitmap: bytes, index: int) -> int: # A view unranks the permutations it wants, rather than generating all of the -# length's, when at most one in this many is wanted: unranking one costs about 5 +# length's, when at most one in this many is wanted. Unranking one costs about 5 # (CPython 3.11) to 13 (PyPy 3.11) times as much as generating and testing one, -# measured at lengths 8 to 12 -_SPARSE_FACTOR = 16 +# measured at lengths 8 to 12, and 16 is above both +_UNRANK_WHEN_ONE_IN = 16 def _wanted_indices(bitmap: bytes, length: int, wanted_bit: int) -> Iterator[int]: @@ -386,13 +391,20 @@ class PropertyBitmaps: i % 8 of byte i // 8, from the low bit. So a permutation perm is at bit perm.rank() - sum(factorial(k) for k in range(len(perm))), and the bits are in the order in which Perm.of_length(length) yields the permutations - (lexicographic). The bitmaps can be given (bitmaps={length: bytes}), made - from a property (add_length), or both: auto_bisc and bisc label the - lengths they need and are not given with prop, adding them to this - object. Without a prop, auto_bisc stops, as for a tuple of dictionaries, - and bisc raises a ValueError. A bitmap given must be bytes-like (bytes, a - bytearray or a memoryview), of its length's size, with no bit set past - the length's permutations. + (lexicographic). The bitmap of length n is (n! + 7) // 8 bytes. + + The bitmaps can be given (bitmaps={length: bytes}), made from a property + (add_length), or both: auto_bisc and bisc label the lengths they need and + are not given with prop, adding them to this object. Without a prop, + auto_bisc stops, as for a tuple of dictionaries, and bisc raises a + ValueError. A bitmap given must be bytes-like (bytes, a bytearray or a + memoryview), or a TypeError is raised. It must be of its length's size, + with no bit set past the length's permutations, or a ValueError is + raised. + + The attributes bitmaps (length -> bytes) and good_counts (length -> the + number of good permutations) are for reading. The constructor and + add_length keep the two together, and the views read them. The views read the bitmaps length by length: iterating over a length, counting it, asking whether a length is there, or whether a permutation @@ -418,7 +430,9 @@ def __init__( bitmap = bytes(bitmap) if len(bitmap) != _bitmap_size(length): raise ValueError( - "A bitmap of length {} has the wrong size".format(length) + "A bitmap of length {} has {} bytes, and must have {}".format( + length, len(bitmap), _bitmap_size(length) + ) ) label_bits = int.from_bytes(bitmap, "little") # The size check leaves spare bits in the last byte only at @@ -434,8 +448,8 @@ def __init__( ) self.bitmaps[length] = bitmap self.good_counts[length] = label_bits.bit_count() - self.good = _LengthViews(self, True) - self.bad = _LengthViews(self, False) + self.good = _LengthViews(self, wanted_bit=1) + self.bad = _LengthViews(self, wanted_bit=0) def add_length(self, length: int) -> None: """Label the permutations of a length with the property, and keep @@ -459,16 +473,17 @@ def add_length(self, length: int) -> None: class _LengthViews(Mapping[int, "_PermsOfLength"]): """length -> the good (or bad) permutations of that length, read-only: a Mapping whose keys are the lengths with a bitmap, through which bisc and - auto_bisc read the permutations.""" + auto_bisc read the permutations. wanted_bit is the label of the + permutations it gives: 1 for the good ones, 0 for the bad ones.""" - def __init__(self, property_bitmaps: PropertyBitmaps, good: bool) -> None: + def __init__(self, property_bitmaps: PropertyBitmaps, wanted_bit: int) -> None: self.property_bitmaps = property_bitmaps - self.is_good = good + self.wanted_bit = wanted_bit def __getitem__(self, length: int) -> "_PermsOfLength": if length not in self.property_bitmaps.bitmaps: raise KeyError(length) - return _PermsOfLength(self.property_bitmaps, length, self.is_good) + return _PermsOfLength(self.property_bitmaps, length, wanted_bit=self.wanted_bit) def __iter__(self) -> Iterator[int]: return iter(self.property_bitmaps.bitmaps) @@ -486,11 +501,11 @@ class _PermsOfLength(Collection[Perm]): the one wanted.""" def __init__( - self, property_bitmaps: PropertyBitmaps, length: int, good: bool + self, property_bitmaps: PropertyBitmaps, length: int, wanted_bit: int ) -> None: self.property_bitmaps = property_bitmaps self.length = length - self.wanted_bit = 1 if good else 0 + self.wanted_bit = wanted_bit def __contains__(self, perm: object) -> bool: # A permutation of this length: a Perm, or any other sequence of the @@ -511,8 +526,9 @@ def __iter__(self) -> Iterator[Perm]: wanted_bit = self.wanted_bit wanted_count = len(self) if wanted_count == 0: + # Nothing to yield, and no need to read the bitmap return - if wanted_count * _SPARSE_FACTOR <= factorial(self.length): + if wanted_count * _UNRANK_WHEN_ONE_IN <= factorial(self.length): for index in _wanted_indices(bitmap, self.length, wanted_bit): yield Perm.unrank(index, self.length) else: @@ -540,9 +556,8 @@ def _bitmap_of(length: int, perms: Iterable[Perm]) -> bytearray: for perm in perms: if len(perm) != length: raise ValueError( - "A permutation of length {} is given as one of length {}".format( - len(perm), length - ) + "The permutation {!r} of length {} is given as one of length " + "{}".format(perm, len(perm), length) ) if not isinstance(perm, Perm): perm = Perm(perm) @@ -605,21 +620,21 @@ def _read_stored_set( with os.scandir(bisc_dir) as entries: for entry in entries: - en = entry.name[:-5] - spl = en.split("_") - if spl[0] == name: + entry_name = entry.name[:-5] + name_parts = entry_name.split("_") + if name_parts[0] == name: if ( good_entry is None - and spl[1] == "good" - and int(spl[2].split("len")[1]) >= min_length + and name_parts[1] == "good" + and int(name_parts[2].split("len")[1]) >= min_length ): - good_entry = en + good_entry = entry_name if ( bad_entry is None - and spl[1] == "bad" - and int(spl[2].split("len")[1]) >= min_length + and name_parts[1] == "bad" + and int(name_parts[2].split("len")[1]) >= min_length ): - bad_entry = en + bad_entry = entry_name if good_entry is not None and bad_entry is not None: break if good_entry is not None and bad_entry is not None: diff --git a/permuta/bisc/bisc_subfunctions.py b/permuta/bisc/bisc_subfunctions.py index cb59688..a410af5 100644 --- a/permuta/bisc/bisc_subfunctions.py +++ b/permuta/bisc/bisc_subfunctions.py @@ -1,4 +1,7 @@ # mypy: ignore-errors +# Its errors are left for a pull request of its own, which will type this +# module. Unlike a file-level type: ignore, this lets mypy read the functions +# that bisc.py imports from it. from itertools import chain from math import factorial diff --git a/tests/bisc/test_property_bitmaps.py b/tests/bisc/test_property_bitmaps.py index 9e2aff3..b27cbed 100644 --- a/tests/bisc/test_property_bitmaps.py +++ b/tests/bisc/test_property_bitmaps.py @@ -128,13 +128,14 @@ def test_bitmaps_round_trip(): for length in range(7): assert list(round_trip.good[length]) == list(property_bitmaps.good[length]) assert list(round_trip.bad[length]) == list(property_bitmaps.bad[length]) - with pytest.raises(ValueError, match="wrong size"): - PropertyBitmaps(bitmaps={3: b"\x00\x00"}) # 3! bits fit in one byte - with pytest.raises(ValueError, match="wrong size"): + # 3! bits fit in one byte + with pytest.raises(ValueError, match="length 3 has 2 bytes, and must have 1"): + PropertyBitmaps(bitmaps={3: b"\x00\x00"}) + with pytest.raises(ValueError, match="length 3 has 0 bytes, and must have 1"): PropertyBitmaps(bitmaps={3: b""}) -def test_lengths_0_and_1_labelled_bad(): +def test_lengths_0_and_1_labeled_bad(): property_bitmaps = PropertyBitmaps(bitmaps={0: b"\x00", 1: b"\x00"}) for length in (0, 1): assert list(property_bitmaps.good[length]) == [] @@ -311,10 +312,10 @@ def encode(prop, length): def test_auto_bisc_labels_only_missing_lengths_with_the_property(): - labelled_lengths = [] + labeled_lengths = [] def prop(perm): - labelled_lengths.append(len(perm)) + labeled_lengths.append(len(perm)) return perm.stack_sortable() given_lengths = (0, 1, 2, 3, 5, 8) @@ -325,12 +326,12 @@ def prop(perm): }, ) assert auto_bisc(property_bitmaps) == BASIS_231 - assert set(labelled_lengths) == {4, 6, 7} + assert set(labeled_lengths) == {4, 6, 7} assert sorted(property_bitmaps.bitmaps) == list(range(9)) def test_bisc_labels_missing_lengths_with_the_property(): - # Without n, bisc uses the longest length labelled, or 7 when there is none + # Without n, bisc uses the longest length labeled, or 7 when there is none assert bisc(property_bitmaps_up_to(Perm.stack_sortable, 5), 3) == BASIS_231 property_bitmaps = PropertyBitmaps(Perm.stack_sortable) assert bisc(property_bitmaps, 3, 5) == BASIS_231 @@ -355,16 +356,16 @@ def test_add_length_needs_a_property(): assert sorted(property_bitmaps.bitmaps) == [0] -def test_bisc_without_n_uses_the_longest_labelled_length_with_no_property(): +def test_bisc_without_n_uses_the_longest_labeled_length_with_no_property(): bitmaps = {length: encode(Perm.stack_sortable, length) for length in range(6)} assert bisc(PropertyBitmaps(bitmaps=bitmaps), 3) == BASIS_231 def test_bisc_labels_only_the_missing_lengths(): - labelled_lengths = [] + labeled_lengths = [] def prop(perm): - labelled_lengths.append(len(perm)) + labeled_lengths.append(len(perm)) return perm.stack_sortable() given_lengths = (0, 1, 2, 3, 5) @@ -375,10 +376,10 @@ def prop(perm): }, ) assert bisc(property_bitmaps, 3) == BASIS_231 - assert labelled_lengths == [4] * factorial(4) + assert labeled_lengths == [4] * factorial(4) assert sorted(property_bitmaps.bitmaps) == list(range(6)) assert bisc(property_bitmaps, 3, 5) == BASIS_231 - assert labelled_lengths == [4] * factorial(4) + assert labeled_lengths == [4] * factorial(4) def test_bisc_labels_up_to_the_pattern_length(): @@ -423,7 +424,7 @@ class ReachedLength9(Exception): def test_auto_bisc_labels_new_lengths_from_the_property(): - # When auto_bisc grows past the labelled lengths, it labels the new ones with + # When auto_bisc grows past the labeled lengths, it labels the new ones with # the property: here it is stopped at the first permutation of length 9 def prop(perm): if len(perm) == 9: From 41e9bc0a89633f6a0b322f017737c431eddcef5d Mon Sep 17 00:00:00 2001 From: Henning Ulfarsson Date: Sat, 24 Oct 2026 11:46:13 +0000 Subject: [PATCH 25/38] Keep a length with few good permutations as the indices of those From the review of #209, round 6: sparse storage, which Henning asked for, by the review's design. A short list of long permutations cost a bitmap for every length up to its longest: three permutations of length 12 peaked at 211 MB on CPython 3.14, against develop's 45 MB, and the bitmap of length 14 alone would be 10.9 GB. Now each length keeps the smaller of two forms: its bitmap, (n! + 7) // 8 bytes, or the sorted indices of its good permutations in an array("Q"), 8 bytes each, which are the smaller when fewer than one permutation in 64 (_SPARSE_WHEN_ONE_IN) is good. The form is chosen length by length, by its count, since a property's share of good permutations can rise with the length, though a class's cannot. - Every way in chooses the form. A list or a mapping that gives few permutations of a length keeps their sorted, distinct indices; one that gives more sets a bitmap, and keeps the indices instead only when permutations given twice made them look many. add_length keeps the indices while they are few, and once they would take as much space as the bitmap, sets them in one and fills it from there, so that its peak is two bitmaps. A tuple's good side chooses as a list does, and the check of the good and the bad counts the overlap in whichever forms they have. A bitmap given is turned into indices when they are the smaller. - No int is made of a whole bitmap: bits are counted, and the overlap of two bitmaps found, a chunk of 64 KB at a time. The spare bits of a bitmap given are checked on its last byte only. - The views read both forms. The good permutations of a length kept as indices are unranked from them, and its bad ones are generated, with the good indices skipped in step; in is a binary search, and len() reads the count. The path of a bitmap is unchanged. - bitmaps is now a read-only Mapping, which builds the bitmap of a length kept as indices when asked for it, but not to answer in. good_counts is unchanged. bisc and auto_bisc no longer copy a bytearray they have just built; a bytearray given is still copied, so that changing it does not change the labels. The README and the changelog say how a length is kept. Checked: every existing test passes unchanged on CPython 3.11, and with the README's examples on PyPy 3.11. What bisc and auto_bisc print and return is develop's, on both Pythons, for every input kind of round 5's comparisons and for three permutations of length 12, and of length 14. At length 13, whose three are not in lexicographic order, the shadings come in the order develop gives for the three sorted, as on b4b27e3. mypy 2.4.0 finds nothing for Python 3.14. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 10 +- README.rst | 15 +- permuta/bisc/bisc.py | 316 +++++++++++++++++++++++++++++++------------ 3 files changed, 248 insertions(+), 93 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7dbcdaf..f61de0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,16 +8,18 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added - `PropertyBitmaps` in `permuta.bisc`: a property's values on the permutations, - kept as one bitmap per length (lexicographic order), made from a property, given - as bitmaps, or both. `auto_bisc` and `bisc` accept it as input, and label the - lengths it lacks with its property. + made from a property, given as bitmaps, or both. Each length is kept as a bitmap + (lexicographic order) or, when fewer than one permutation in 64 is good, as the + sorted indices of the good ones, whichever is smaller. `auto_bisc` and `bisc` + accept it as input, and label the lengths it lacks with its property. ### Changed - `bisc` and `auto_bisc` turn every input into a `PropertyBitmaps` and read the good and the bad permutations through its views, in lexicographic order, for the same results in far less memory at longer lengths: the permutations of length 10 take 454 KB as a bitmap, and 0.9 to 1.8 GB as lists of `Perm` objects, - depending on the Python. + depending on the Python, and a length with few good permutations takes 8 bytes + for each of them. - A permutation listed twice in the input of `bisc` or `auto_bisc` counts once. Before, duplicates could make `bisc` take a length as complete when it was not. - `auto_bisc` raises a `ValueError` when the good and the bad permutations of a diff --git a/README.rst b/README.rst index b9f47bc..62ba289 100644 --- a/README.rst +++ b/README.rst @@ -519,12 +519,15 @@ Both ``bisc`` and ``auto_bisc`` can accept input in the form of a property, a list of permutations (satisfying some property), or a ``PropertyBitmaps``. Whatever the input, ``bisc`` and ``auto_bisc`` keep the permutations satisfying -the property, and those that do not, as one bitmap per length, in a -``PropertyBitmaps``: the permutations of length 10 take 454 KB as a bitmap, where -they take 0.9 to 1.8 GB as lists of ``Perm`` objects, depending on the Python. -A ``PropertyBitmaps`` can also be given directly, made from a property, from -bitmaps, or both. The lengths ``bisc`` and ``auto_bisc`` label with its property -are added to it, for a later run to reuse. +the property, and those that do not, length by length, in a +``PropertyBitmaps``. A length is kept as a bitmap, a bit for each permutation: +the permutations of length 10 take 454 KB as a bitmap, where they take 0.9 to +1.8 GB as lists of ``Perm`` objects, depending on the Python. A length with fewer +than one permutation in 64 satisfying the property keeps instead the indices of +those, 8 bytes each, which then take less space: a short list of long +permutations stays small. A ``PropertyBitmaps`` can also be given directly, made +from a property, from bitmaps, or both. The lengths ``bisc`` and ``auto_bisc`` +label with its property are added to it, for a later run to reuse. .. code-block:: python diff --git a/permuta/bisc/bisc.py b/permuta/bisc/bisc.py index 2a143ee..d7812b1 100644 --- a/permuta/bisc/bisc.py +++ b/permuta/bisc/bisc.py @@ -1,7 +1,9 @@ +import bisect import functools import json import os import types +from array import array from collections.abc import Collection, Mapping, Sequence from math import factorial from typing import Callable, Dict, Iterable, Iterator, List, Optional, Set, Tuple, Union @@ -42,8 +44,8 @@ def bisc( in the set), or a PropertyBitmaps. Each is turned into a PropertyBitmaps, whose good permutations of each length up to max(m, n) mine() reads. Without n, a property uses length 7, and the others their longest length. - - A list sets the bits of the permutations it lists, and a length with - none listed is empty. + - A list's permutations are the good ones, and a length with none listed + is empty. - A property labels the lengths up to n, and a length past n, when m > n, is read as empty. - A Mapping is read for each length up to max(m, n), and one it lacks is @@ -72,16 +74,13 @@ def bisc( if n is None: print("I will use permutations up to length 7") n = 7 - # The property labels the lengths up to n. mine() reads those up to m - # too, and a length past n is read as empty - property_bitmaps = PropertyBitmaps( - A, - bitmaps={ - length: bytes(_bitmap_size(length)) for length in range(n + 1, m + 1) - }, - ) + property_bitmaps = PropertyBitmaps(A) for length in range(n + 1): property_bitmaps.add_length(length) + # mine() reads the lengths up to m too, and a length past n is read as + # empty: none of its permutations is good + for length in range(n + 1, m + 1): + property_bitmaps._keep(length, array("Q"), 0) elif isinstance(A, Mapping): if n is None: @@ -151,8 +150,7 @@ def auto_bisc( permutations of each length. - A property labels the lengths up to 8 at the start, and each longer one when auto_bisc first needs it. - - A list sets the bits of the permutations it lists, and the others are - bad. + - A list's permutations are the good ones, and the others are bad. - For a tuple of dictionaries, or a stored set, the good and the bad permutations of each length up to the shorter dictionary's longest must not overlap, and must together be all of that length's permutations, or @@ -176,7 +174,8 @@ def auto_bisc( return None longest_length = max(listed_lengths) # Printed as when the bad permutations were made into lists, so that - # what auto_bisc prints does not change; now they are the bitmaps' zeros + # what auto_bisc prints does not change; now they are read from the + # labels of the good ones for i in range(L, longest_length + 1): print("Populating the dictionary of bad perms of length ", i) perms_by_length: Dict[int, List[Perm]] = { @@ -355,7 +354,7 @@ def _set_label(bitmap: bytearray, index: int) -> None: bitmap[index // 8] |= 1 << (index % 8) -def _label(bitmap: bytes, index: int) -> int: +def _label(bitmap: Union[bytes, bytearray], index: int) -> int: """The label (0 or 1) of the permutation at index.""" return (bitmap[index // 8] >> (index % 8)) & 1 @@ -367,7 +366,9 @@ def _label(bitmap: bytes, index: int) -> int: _UNRANK_WHEN_ONE_IN = 16 -def _wanted_indices(bitmap: bytes, length: int, wanted_bit: int) -> Iterator[int]: +def _wanted_indices( + bitmap: Union[bytes, bytearray], length: int, wanted_bit: int +) -> Iterator[int]: """The indices, in increasing order, of the permutations of a length whose label is wanted_bit. A byte with no such label is skipped whole.""" skipped_byte = 0xFF if wanted_bit == 0 else 0 @@ -381,32 +382,86 @@ def _wanted_indices(bitmap: bytes, length: int, wanted_bit: int) -> Iterator[int yield index +# A length's labels, as a PropertyBitmaps keeps them: its bitmap, or the sorted +# indices of its good permutations +_Labels = Union[bytes, bytearray, "array[int]"] + +# A length keeps the sorted indices of its good permutations, 8 bytes each, +# rather than its bitmap, a bit for each of its permutations, when fewer than +# one in this many of them are good: the indices are then the smaller +_SPARSE_WHEN_ONE_IN = 64 + + +def _is_sparse(good_count: int, length: int) -> bool: + """Whether a length with good_count good permutations keeps their indices + rather than its bitmap.""" + return good_count * _SPARSE_WHEN_ONE_IN < factorial(length) + + +def _label_in(labels: _Labels, index: int) -> int: + """The label (0 or 1) of the permutation at index, from a length's labels + in either form.""" + if isinstance(labels, array): + position = bisect.bisect_left(labels, index) + return int(position < len(labels) and labels[position] == index) + return _label(labels, index) + + +def _bitmap_from_indices(length: int, indices: Iterable[int]) -> bytearray: + """The bitmap of a length with the bits at the given indices set.""" + bitmap = bytearray(_bitmap_size(length)) + for index in indices: + _set_label(bitmap, index) + return bitmap + + +# Bitmaps are read this many bytes at a time, each chunk as one int, to count +# their bits or the bits two of them share, so that no int is made of a whole +# bitmap +_CHUNK_BYTES = 1 << 16 + + +def _count_labels(bitmap: Union[bytes, bytearray]) -> int: + """The number of bits set in a bitmap: its good permutations.""" + view = memoryview(bitmap) + return sum( + int.from_bytes(view[start : start + _CHUNK_BYTES], "little").bit_count() + for start in range(0, len(view), _CHUNK_BYTES) + ) + + class PropertyBitmaps: - """A property's values on the permutations, one bitmap per length: which + """A property's values on the permutations, length by length: which permutations of each length satisfy it (good) and which do not (bad). bisc and auto_bisc turn each of their inputs into one, and read its good and bad permutations through its views, good and bad. - Bit i of a length's bitmap is the label of Perm.unrank(i, length): bit - i % 8 of byte i // 8, from the low bit. So a permutation perm is at bit - perm.rank() - sum(factorial(k) for k in range(len(perm))), and the bits - are in the order in which Perm.of_length(length) yields the permutations - (lexicographic). The bitmap of length n is (n! + 7) // 8 bytes. - - The bitmaps can be given (bitmaps={length: bytes}), made from a property - (add_length), or both: auto_bisc and bisc label the lengths they need and - are not given with prop, adding them to this object. Without a prop, - auto_bisc stops, as for a tuple of dictionaries, and bisc raises a + A length's labels are its bitmap: bit i is the label of Perm.unrank(i, + length), bit i % 8 of byte i // 8, from the low bit. So a permutation perm + is at bit perm.rank() - sum(factorial(k) for k in range(len(perm))), and + the bits are in the order in which Perm.of_length(length) yields the + permutations (lexicographic). The bitmap of length n is (n! + 7) // 8 + bytes. A length with fewer than one good permutation in 64 keeps instead + the sorted indices of its good permutations (their bits), 8 bytes each, + which then take less space. So no length takes more than its bitmap: + three permutations of length 14 take 24 bytes, where its bitmap takes + 10.9 GB. + + The labels can be given as bitmaps (bitmaps={length: bytes}), made from a + property (add_length), or both: auto_bisc and bisc label the lengths they + need and are not given with prop, adding them to this object. Without a + prop, auto_bisc stops, as for a tuple of dictionaries, and bisc raises a ValueError. A bitmap given must be bytes-like (bytes, a bytearray or a memoryview), or a TypeError is raised. It must be of its length's size, with no bit set past the length's permutations, or a ValueError is - raised. + raised. Changing a bytearray after giving it does not change the labels. - The attributes bitmaps (length -> bytes) and good_counts (length -> the - number of good permutations) are for reading. The constructor and - add_length keep the two together, and the views read them. + The attributes bitmaps (length -> its bitmap, a read-only Mapping) and + good_counts (length -> the number of good permutations) are for reading. + The bitmap of a length kept as indices is built when bitmaps is asked + for it. - The views read the bitmaps length by length: iterating over a length, + The views read the labels length by length: iterating over a length, counting it, asking whether a length is there, or whether a permutation is in a length. A bitmap of S_10 is 454 KB; S_10 as a list of Perm objects takes 0.9 GB on PyPy 3.11 and 1.3 to 1.8 GB on CPython. @@ -418,7 +473,8 @@ def __init__( bitmaps: Optional[Mapping[int, Union[bytes, bytearray, memoryview]]] = None, ) -> None: self.prop = prop - self.bitmaps: Dict[int, bytes] = {} + # length -> its labels, in the form their number chooses (_is_sparse) + self._labels: Dict[int, _Labels] = {} self.good_counts: Dict[int, int] = {} if bitmaps is not None: for length, bitmap in bitmaps.items(): @@ -427,52 +483,105 @@ def __init__( "A bitmap must be bytes, a bytearray or a memoryview, " "not {}".format(type(bitmap).__name__) ) - bitmap = bytes(bitmap) + if isinstance(bitmap, memoryview): + # Its bytes, whatever the view's format and shape + bitmap = bytes(bitmap) if len(bitmap) != _bitmap_size(length): raise ValueError( "A bitmap of length {} has {} bytes, and must have {}".format( length, len(bitmap), _bitmap_size(length) ) ) - label_bits = int.from_bytes(bitmap, "little") # The size check leaves spare bits in the last byte only at # lengths 0 to 3 (7, 7, 6 and 2 of them), since n! is a # multiple of 8 from n = 4 on. A spare bit set would be counted # as a good permutation: len() of the good view would disagree # with its iteration, and len() of the bad view could go # negative. - if label_bits >> factorial(length): + spare_bit_count = 8 * len(bitmap) - factorial(length) + if spare_bit_count and bitmap[-1] >> (8 - spare_bit_count): raise ValueError( "A bitmap of length {} has bits set past its {} " "permutations".format(length, factorial(length)) ) - self.bitmaps[length] = bitmap - self.good_counts[length] = label_bits.bit_count() + good_count = _count_labels(bitmap) + if _is_sparse(good_count, length): + good_indices = array("Q", _wanted_indices(bitmap, length, 1)) + self._keep(length, good_indices, good_count) + elif isinstance(bitmap, bytes): + self._keep(length, bitmap, good_count) + else: + # A copy, so that a change to the bytearray does not change + # the labels + self._keep(length, bytes(bitmap), good_count) + self.bitmaps: Mapping[int, bytes] = _Bitmaps(self) self.good = _LengthViews(self, wanted_bit=1) self.bad = _LengthViews(self, wanted_bit=0) + def _keep(self, length: int, labels: _Labels, good_count: int) -> None: + """Keep a length's labels, in the form their number chooses, with that + number of good permutations. The views read them from here.""" + self._labels[length] = labels + self.good_counts[length] = good_count + def add_length(self, length: int) -> None: - """Label the permutations of a length with the property, and keep - their labels as the length's bitmap. It needs a property, and replaces - the length's bitmap if there is one.""" + """Label the permutations of a length with the property, and keep the + length's labels, replacing any it has. The indices of the good + permutations are kept while they are few, and once they would take + as much space as the length's bitmap, they are set in it, which is + filled from then on. It needs a property.""" if self.prop is None: raise ValueError( "The PropertyBitmaps has no property to label length {} " "with".format(length) ) - bitmap = bytearray(_bitmap_size(length)) + good_indices = array("Q") + bitmap: Optional[bytearray] = None good_count = 0 for index, perm in enumerate(Perm.of_length(length)): if self.prop(perm): - _set_label(bitmap, index) good_count += 1 - self.bitmaps[length] = bytes(bitmap) - self.good_counts[length] = good_count + if bitmap is not None: + _set_label(bitmap, index) + else: + good_indices.append(index) + if not _is_sparse(good_count, length): + # The indices take as much space as the bitmap: set + # them in it, and go on in it + bitmap = _bitmap_from_indices(length, good_indices) + good_indices = array("Q") + self._keep(length, good_indices if bitmap is None else bitmap, good_count) + + +class _Bitmaps(Mapping[int, bytes]): + """length -> its bitmap, read-only: the bitmap kept, or, for a length kept + as the indices of its good permutations, one built from them when + asked.""" + + def __init__(self, property_bitmaps: PropertyBitmaps) -> None: + self.property_bitmaps = property_bitmaps + + def __getitem__(self, length: int) -> bytes: + labels = self.property_bitmaps._labels[length] + if isinstance(labels, array): + return bytes(_bitmap_from_indices(length, labels)) + return bytes(labels) + + def __contains__(self, length: object) -> bool: + # Mapping's would build the bitmap of a length kept as indices, to + # answer whether the length is there + return length in self.property_bitmaps._labels + + def __iter__(self) -> Iterator[int]: + return iter(self.property_bitmaps._labels) + + def __len__(self) -> int: + return len(self.property_bitmaps._labels) class _LengthViews(Mapping[int, "_PermsOfLength"]): """length -> the good (or bad) permutations of that length, read-only: a - Mapping whose keys are the lengths with a bitmap, through which bisc and + Mapping whose keys are the lengths with labels, through which bisc and auto_bisc read the permutations. wanted_bit is the label of the permutations it gives: 1 for the good ones, 0 for the bad ones.""" @@ -481,24 +590,26 @@ def __init__(self, property_bitmaps: PropertyBitmaps, wanted_bit: int) -> None: self.wanted_bit = wanted_bit def __getitem__(self, length: int) -> "_PermsOfLength": - if length not in self.property_bitmaps.bitmaps: + if length not in self.property_bitmaps._labels: raise KeyError(length) return _PermsOfLength(self.property_bitmaps, length, wanted_bit=self.wanted_bit) def __iter__(self) -> Iterator[int]: - return iter(self.property_bitmaps.bitmaps) + return iter(self.property_bitmaps._labels) def __len__(self) -> int: - return len(self.property_bitmaps.bitmaps) + return len(self.property_bitmaps._labels) class _PermsOfLength(Collection[Perm]): - """The good (or bad) permutations of one length, read from its bitmap: a - Collection. Iterating yields those whose bit is the one wanted, in - lexicographic order: when few are wanted it unranks them, and otherwise - it generates the length's permutations and tests each bit. len() is the - count kept with the bitmap, and a permutation is in it when its bit is - the one wanted.""" + """The good (or bad) permutations of one length, read from its labels: a + Collection. Iterating yields those whose label is the one wanted, in + lexicographic order. For a length kept as the indices of its good + permutations, the good ones are unranked from them, and the bad ones are + the others, generated in order. From a bitmap, when few are wanted they + are unranked, and otherwise the length's permutations are generated and + each bit is tested. len() is the count kept with the labels, and a + permutation is in it when its label is the one wanted.""" def __init__( self, property_bitmaps: PropertyBitmaps, length: int, wanted_bit: int @@ -517,23 +628,37 @@ def __contains__(self, perm: object) -> bool: and set(perm) == set(range(self.length)) ): return False - bitmap = self.property_bitmaps.bitmaps[self.length] + labels = self.property_bitmaps._labels[self.length] index = Perm(perm).rank() - _shorter_count(self.length) - return _label(bitmap, index) == self.wanted_bit + return _label_in(labels, index) == self.wanted_bit def __iter__(self) -> Iterator[Perm]: - bitmap = self.property_bitmaps.bitmaps[self.length] + labels = self.property_bitmaps._labels[self.length] wanted_bit = self.wanted_bit wanted_count = len(self) if wanted_count == 0: - # Nothing to yield, and no need to read the bitmap + # Nothing to yield, and no need to read the labels return - if wanted_count * _UNRANK_WHEN_ONE_IN <= factorial(self.length): - for index in _wanted_indices(bitmap, self.length, wanted_bit): + if isinstance(labels, array): + if wanted_bit: + for index in labels: + yield Perm.unrank(index, self.length) + else: + # The good indices are in increasing order, as the + # permutations are generated + good_indices = iter(labels) + next_good_index = next(good_indices, None) + for index, perm in enumerate(Perm.of_length(self.length)): + if index == next_good_index: + next_good_index = next(good_indices, None) + else: + yield perm + elif wanted_count * _UNRANK_WHEN_ONE_IN <= factorial(self.length): + for index in _wanted_indices(labels, self.length, wanted_bit): yield Perm.unrank(index, self.length) else: for index, perm in enumerate(Perm.of_length(self.length)): - if _label(bitmap, index) == wanted_bit: + if _label(labels, index) == wanted_bit: yield perm def __len__(self) -> int: @@ -548,10 +673,9 @@ def _shorter_count(length: int) -> int: return sum(factorial(shorter_length) for shorter_length in range(length)) -def _bitmap_of(length: int, perms: Iterable[Perm]) -> bytearray: - """The bitmap of a length with the bits of the given permutations set. A - permutation given twice sets its bit once.""" - bitmap = bytearray(_bitmap_size(length)) +def _indices_of(length: int, perms: Iterable[Perm]) -> Iterator[int]: + """The indices (bits) of permutations given as ones of a length. One of + another length raises a ValueError.""" shorter_count = _shorter_count(length) for perm in perms: if len(perm) != length: @@ -561,21 +685,49 @@ def _bitmap_of(length: int, perms: Iterable[Perm]) -> bytearray: ) if not isinstance(perm, Perm): perm = Perm(perm) - _set_label(bitmap, perm.rank() - shorter_count) - return bitmap + yield perm.rank() - shorter_count + + +def _labels_of(length: int, perms: Collection[Perm]) -> Tuple[_Labels, int]: + """The labels of a length whose good permutations are the ones given, in + the form their number chooses, and that number. A permutation given + twice counts once.""" + if _is_sparse(len(perms), length): + good_indices = array("Q", sorted(set(_indices_of(length, perms)))) + return good_indices, len(good_indices) + bitmap = _bitmap_from_indices(length, _indices_of(length, perms)) + good_count = _count_labels(bitmap) + if _is_sparse(good_count, length): + # Permutations given more than once made them look many + return array("Q", _wanted_indices(bitmap, length, 1)), good_count + return bitmap, good_count + + +def _overlap_count(labels: _Labels, other_labels: _Labels) -> int: + """The number of permutations good in both of two labels of one length.""" + if isinstance(labels, array): + return sum(_label_in(other_labels, index) for index in labels) + if isinstance(other_labels, array): + return sum(_label_in(labels, index) for index in other_labels) + view, other_view = memoryview(labels), memoryview(other_labels) + return sum( + ( + int.from_bytes(view[start : start + _CHUNK_BYTES], "little") + & int.from_bytes(other_view[start : start + _CHUNK_BYTES], "little") + ).bit_count() + for start in range(0, len(view), _CHUNK_BYTES) + ) def _from_perms_by_length( - perms_by_length: Mapping[int, Iterable[Perm]], + perms_by_length: Mapping[int, Collection[Perm]], ) -> PropertyBitmaps: """A PropertyBitmaps, with no property, whose good permutations of each length are the ones given for it.""" - return PropertyBitmaps( - bitmaps={ - length: _bitmap_of(length, perms) - for length, perms in perms_by_length.items() - } - ) + property_bitmaps = PropertyBitmaps() + for length, perms in perms_by_length.items(): + property_bitmaps._keep(length, *_labels_of(length, perms)) + return property_bitmaps def _from_good_and_bad( @@ -587,14 +739,12 @@ def _from_good_and_bad( side counts once. The good and the bad permutations of each length must not overlap, and must together be all of its permutations, or a ValueError is raised.""" - bitmaps = {} + property_bitmaps = PropertyBitmaps() for length in range(min(max(good), max(bad)) + 1): - good_bitmap = _bitmap_of(length, good.get(length, ())) - bad_bitmap = _bitmap_of(length, bad.get(length, ())) - good_bits = int.from_bytes(good_bitmap, "little") - bad_bits = int.from_bytes(bad_bitmap, "little") - in_both_count = (good_bits & bad_bits).bit_count() - in_neither_count = factorial(length) - (good_bits | bad_bits).bit_count() + good_labels, good_count = _labels_of(length, good.get(length, ())) + bad_labels, bad_count = _labels_of(length, bad.get(length, ())) + in_both_count = _overlap_count(good_labels, bad_labels) + in_neither_count = factorial(length) - (good_count + bad_count - in_both_count) if in_both_count or in_neither_count: raise ValueError( "The good and the bad permutations of length {} must not overlap " @@ -603,8 +753,8 @@ def _from_good_and_bad( length, factorial(length), in_both_count, in_neither_count ) ) - bitmaps[length] = good_bitmap - return PropertyBitmaps(bitmaps=bitmaps) + property_bitmaps._keep(length, good_labels, good_count) + return property_bitmaps def _read_stored_set( From 841c8dc7ca69bd2084101ed481852d215786d734 Mon Sep 17 00:00:00 2001 From: Henning Ulfarsson Date: Sat, 24 Oct 2026 11:46:13 +0000 Subject: [PATCH 26/38] Test both forms in which a length's labels are kept From the review of #209, round 6, its checks, in tests/bisc/test_sparse_storage.py: - Both forms against brute force, for every view (iteration and its order, len(), in, the bad view), the count and the bitmap, and for every way in: a list or a mapping (shuffled, some permutations given twice), a tuple, bitmaps and a property. At lengths 0 to 8, with the threshold set to one in 2, so that both forms occur from length 2 on, and to one in 64, the default. The good sets have no permutation, all, as many as the threshold, one fewer, and a random number on each side of it. The bitmaps are read seven bytes at a time, so that counting them takes several chunks. - add_length switching to a bitmap in the middle of a length: early, late, at the last permutation, with the good spread out, and staying with indices one short of the threshold. - A permutation listed five times kept as one index. - A tuple's overlap counted in every pair of forms of its good and bad. - A length kept as indices builds no bitmap to answer in, to give its lengths or to read its good permutations, only when its bitmap is asked for. - bisc on three permutations of length 14 gives develop's answer, with tracemalloc's peak under 1 MB. tracemalloc is CPython's, so PyPy skips this one. Checked: the 645 tests pass on CPython 3.11, and the 114 in tests/bisc, with the README's 70 examples, on PyPy 3.11. Each of 23 faults planted in the new and changed functions of bisc.py fails at least one test. Co-Authored-By: Claude Opus 5.5 --- tests/bisc/test_sparse_storage.py | 291 ++++++++++++++++++++++++++++++ 1 file changed, 291 insertions(+) create mode 100644 tests/bisc/test_sparse_storage.py diff --git a/tests/bisc/test_sparse_storage.py b/tests/bisc/test_sparse_storage.py new file mode 100644 index 0000000..bb0c2b1 --- /dev/null +++ b/tests/bisc/test_sparse_storage.py @@ -0,0 +1,291 @@ +"""The two forms in which a PropertyBitmaps keeps a length's labels: its bitmap, +or the sorted indices of its good permutations when fewer than one in +_SPARSE_WHEN_ONE_IN is good. Each way in chooses the form by the number of good +permutations, and every view reads both forms.""" + +import importlib +import random +from array import array +from math import factorial + +import pytest + +from permuta.bisc.bisc import PropertyBitmaps, bisc +from permuta.patterns.perm import Perm + +# The module, which permuta.bisc's function bisc hides +bisc_module = importlib.import_module("permuta.bisc.bisc") + +MAX_LENGTH = 8 + + +def expected_bitmap(length, good_set): + """A length's bitmap with the bits of the permutations in good_set set, made + without PropertyBitmaps.""" + label_bits = sum( + 1 << index + for index, perm in enumerate(Perm.of_length(length)) + if perm in good_set + ) + return label_bits.to_bytes((factorial(length) + 7) // 8, "little") + + +def is_kept_as_indices(property_bitmaps, length): + return isinstance(property_bitmaps._labels[length], array) + + +def assert_views_match(property_bitmaps, length, good_set, bitmap=None): + """Every view of a length against brute force: iteration and its order, + len(), in, the bad view, the count and the bitmap.""" + perms = list(Perm.of_length(length)) + good = [perm for perm in perms if perm in good_set] + bad = [perm for perm in perms if perm not in good_set] + good_view, bad_view = property_bitmaps.good[length], property_bitmaps.bad[length] + assert list(good_view) == good + assert list(bad_view) == bad + assert len(good_view) == len(good) and len(bad_view) == len(bad) + # in, for every permutation up to length 6, and for every seventh good and + # bad one at lengths 7 and 8 + step = 1 if length < 7 else 7 + for perm in good[::step] + bad[::step]: + assert (perm in good_view) == (perm in good_set) + assert (perm in bad_view) == (perm not in good_set) + assert property_bitmaps.good_counts[length] == len(good) + if bitmap is None: + bitmap = expected_bitmap(length, good_set) + assert property_bitmaps.bitmaps[length] == bitmap + + +def each_way_in(good_sets, bitmaps, randomness): + """A PropertyBitmaps made from good_sets in each way that bisc and auto_bisc + make one, with its name: a list or a mapping (whose good permutations come + shuffled, with some twice), a tuple of the good and the bad, bitmaps, and a + property.""" + lengths = range(MAX_LENGTH + 1) + good_lists, bad_lists = {}, {} + for length in lengths: + good_lists[length] = sorted(good_sets[length]) + randomness.shuffle(good_lists[length]) + bad_lists[length] = [ + perm for perm in Perm.of_length(length) if perm not in good_sets[length] + ] + listed_twice = { + length: good_lists[length] + good_lists[length][:3] for length in lengths + } + yield "list or mapping", bisc_module._from_perms_by_length(listed_twice) + yield "tuple", bisc_module._from_good_and_bad(good_lists, bad_lists) + yield "bitmaps", PropertyBitmaps(bitmaps=bitmaps) + by_property = PropertyBitmaps(lambda perm: perm in good_sets[len(perm)]) + for length in lengths: + by_property.add_length(length) + yield "property", by_property + + +def first_dense_count(length, sparse_when_one_in): + """The fewest good permutations of a length that are kept as a bitmap.""" + return -(-factorial(length) // sparse_when_one_in) + + +SIZE_KINDS = ( + "none", + "just under the threshold", + "at the threshold", + "all", + "random under", + "random over", +) + + +def good_count_of(size_kind, length, sparse_when_one_in, randomness): + first_dense = min(first_dense_count(length, sparse_when_one_in), factorial(length)) + return { + "none": 0, + "just under the threshold": max(first_dense - 1, 0), + "at the threshold": first_dense, + "all": factorial(length), + "random under": randomness.randint(0, max(first_dense - 1, 0)), + "random over": randomness.randint(first_dense, factorial(length)), + }[size_kind] + + +# Both forms against brute force, for every view and every way in, at lengths 0 +# to 8. With one in 2, both forms occur at every length from 2 on; with one in +# 64, the default, from 5 on. The bitmaps are read seven bytes at a time, so that +# counting them takes several chunks +@pytest.mark.parametrize("sparse_when_one_in", [2, 64]) +@pytest.mark.parametrize("size_kind", SIZE_KINDS) +def test_both_forms_read_as_brute_force(monkeypatch, sparse_when_one_in, size_kind): + monkeypatch.setattr(bisc_module, "_SPARSE_WHEN_ONE_IN", sparse_when_one_in) + monkeypatch.setattr(bisc_module, "_CHUNK_BYTES", 7) + randomness = random.Random(f"{sparse_when_one_in} {size_kind}") + good_sets, bitmaps = {}, {} + for length in range(MAX_LENGTH + 1): + good_count = good_count_of(size_kind, length, sparse_when_one_in, randomness) + perms = list(Perm.of_length(length)) + good_sets[length] = set(randomness.sample(perms, good_count)) + bitmaps[length] = expected_bitmap(length, good_sets[length]) + for way, property_bitmaps in each_way_in(good_sets, bitmaps, randomness): + for length in range(MAX_LENGTH + 1): + good_count = len(good_sets[length]) + kept_as_indices = good_count * sparse_when_one_in < factorial(length) + assert is_kept_as_indices(property_bitmaps, length) == kept_as_indices, ( + way, + length, + ) + assert_views_match( + property_bitmaps, length, good_sets[length], bitmaps[length] + ) + + +# add_length keeps the indices while they are few, and switches to a bitmap in +# the middle of a length once they would take as much space. Length 7 has 5040 +# permutations, so its 79th good one makes them as many as one in 64 +@pytest.mark.parametrize( + "good_indices", + [ + range(5040), # all, the switch at the 79th + range(4800, 5040), # good from a point on, the switch late in the length + range(4961, 5040), # 79 good, the switch at the last permutation + range(4962, 5040), # 78 good, kept as indices + range(0, 5040, 50), # 101 spread out, the switch at index 3900 + range(5039, 5040), # one good + range(0), # none + ], +) +def test_add_length_switches_to_a_bitmap_when_the_good_become_many(good_indices): + length = 7 + perms = list(Perm.of_length(length)) + good_set = {perms[index] for index in good_indices} + property_bitmaps = PropertyBitmaps(lambda perm: perm in good_set) + property_bitmaps.add_length(length) + kept_as_indices = len(good_set) * 64 < factorial(length) + assert is_kept_as_indices(property_bitmaps, length) == kept_as_indices + assert_views_match(property_bitmaps, length, good_set) + + +# A permutation listed many times counts once, for the form too +def test_a_permutation_listed_many_times_is_kept_as_one_index(): + # Five copies of one permutation of length 5 look like 5 of its 120, which a + # bitmap would keep at one in 64, but they are one good permutation + perm = Perm((1, 0, 2, 3, 4)) + property_bitmaps = bisc_module._from_perms_by_length({5: [perm] * 5}) + assert is_kept_as_indices(property_bitmaps, 5) + assert_views_match(property_bitmaps, 5, {perm}) + + +def tuple_of(good_8, bad_8): + """A tuple of the good and the bad permutations: every permutation shorter + than 8 good, and those given at length 8.""" + good = {length: list(Perm.of_length(length)) for length in range(8)} + bad = {length: [] for length in range(8)} + return {**good, 8: good_8}, {**bad, 8: bad_8} + + +# The good and the bad permutations of a tuple are checked in every pair of +# forms: the good kept as indices or as a bitmap, and the bad as well +@pytest.mark.parametrize( + "good_8, bad_8, in_both, in_neither", + [ + (slice(0, 5), slice(3, None), 2, 0), # indices, bitmap + (slice(0, 40000), slice(39990, 40010), 10, 310), # bitmap, indices + (slice(0, 20000), slice(19990, None), 10, 0), # bitmap, bitmap + (slice(0, 5), slice(3, 8), 2, 40312), # indices, indices + ], +) +def test_a_tuple_is_checked_in_every_pair_of_forms( + monkeypatch, good_8, bad_8, in_both, in_neither +): + monkeypatch.setattr(bisc_module, "_CHUNK_BYTES", 7) + perms = list(Perm.of_length(8)) + good, bad = tuple_of(perms[good_8], perms[bad_8]) + with pytest.raises( + ValueError, + match=r"length 8 .* \(in both: {}, in neither: {}\)".format( + in_both, in_neither + ), + ): + bisc_module._from_good_and_bad(good, bad) + + +# Asking whether a length is there, or reading its good permutations, does not +# build the bitmap of a length kept as indices +def test_a_length_kept_as_indices_builds_no_bitmap_unless_asked(monkeypatch): + identity = Perm(range(10)) + property_bitmaps = bisc_module._from_perms_by_length({10: [identity]}) + built = [] + real_bitmap_from_indices = bisc_module._bitmap_from_indices + + def recording_bitmap_from_indices(length, indices): + built.append(length) + return real_bitmap_from_indices(length, indices) + + monkeypatch.setattr( + bisc_module, "_bitmap_from_indices", recording_bitmap_from_indices + ) + assert 10 in property_bitmaps.bitmaps and 9 not in property_bitmaps.bitmaps + assert list(property_bitmaps.bitmaps) == [10] and len(property_bitmaps.bitmaps) == 1 + assert list(property_bitmaps.good[10]) == [identity] + assert identity in property_bitmaps.good[10] + assert Perm(range(9, -1, -1)) in property_bitmaps.bad[10] + assert len(property_bitmaps.bad[10]) == factorial(10) - 1 + assert built == [] + assert property_bitmaps.bitmaps[10] == b"\x01" + bytes(453599) + assert built == [10] + + +THREE_OF_LENGTH_14 = [ + Perm((1, 9, 11, 10, 8, 3, 4, 5, 2, 6, 0, 7, 13, 12)), + Perm((7, 10, 6, 11, 1, 4, 3, 12, 2, 5, 8, 9, 0, 13)), + Perm((8, 10, 12, 4, 2, 0, 1, 13, 11, 9, 6, 3, 7, 5)), +] +# bisc(THREE_OF_LENGTH_14, 2, 14) on develop, before PropertyBitmaps +DEVELOP_ANSWER_14 = { + 0: {Perm(()): [{(0, 0)}]}, + 1: {Perm((0,)): [{(0, 1), (1, 0), (0, 0)}, {(1, 1), (0, 0)}]}, + 2: { + Perm((0, 1)): [ + {(0, 1), (0, 2), (1, 2), (2, 1), (0, 0), (1, 1)}, + {(0, 1), (0, 2), (1, 2), (2, 1), (1, 0)}, + {(0, 1), (1, 2), (2, 2), (0, 0), (1, 1)}, + {(0, 1), (0, 2), (0, 0), (1, 1), (2, 0)}, + {(0, 1), (1, 0), (0, 2), (2, 0)}, + {(0, 1), (1, 0), (2, 2)}, + ], + Perm((1, 0)): [ + {(0, 1), (1, 2), (2, 1), (0, 0), (1, 0)}, + {(0, 2), (1, 2), (0, 0), (1, 1), (2, 0)}, + {(0, 2), (1, 2), (2, 1), (0, 0), (1, 1)}, + {(0, 1), (0, 2), (1, 2), (0, 0)}, + {(2, 0), (1, 1), (2, 2), (0, 0)}, + {(1, 0), (2, 2), (0, 0)}, + {(1, 2), (2, 2), (0, 0)}, + ], + }, +} + + +def shadings_as_sets(answer): + """bisc's answer with each pattern's shadings as a set, whose order the + Python may change.""" + return { + length: { + pattern: {frozenset(shading) for shading in shadings} + for pattern, shadings in patterns.items() + } + for length, patterns in answer.items() + } + + +# Three permutations of length 14 run in megabytes: each length up to 14 is kept +# as indices, where the bitmap of length 14 alone would take 10.9 GB +def test_three_permutations_of_length_14_take_little_memory(): + # tracemalloc is CPython's + tracemalloc = pytest.importorskip("tracemalloc") + tracemalloc.start() + try: + answer = bisc(THREE_OF_LENGTH_14, 2, 14) + peak_bytes = tracemalloc.get_traced_memory()[1] + finally: + tracemalloc.stop() + assert peak_bytes < 2**20 + assert shadings_as_sets(answer) == shadings_as_sets(DEVELOP_ANSWER_14) From dd7b00f9a3f64522224484b5b6539ce912321b26 Mon Sep 17 00:00:00 2001 From: Henning Ulfarsson Date: Sat, 24 Oct 2026 17:54:22 +0000 Subject: [PATCH 27/38] Keep the indices of a length past 20 in a list From the review of #209, round 6, blocking 1. The indices of a length's good permutations were kept in an array("Q"), which holds integers to 2**64 - 1, but 21! is above 2**64: listing a permutation of length 21 or more raised OverflowError, where develop answers at once. An index of a length to 20 fits in 8 bytes, and those lengths keep their indices in the array, as before. Past length 20 they are kept in a list (_new_indices), and every place that read the array reads either. Such a length is always kept as indices, since its bitmap could not be made. The array's type code is named once (_INDEX_TYPECODE), and the threshold of one in 64 is derived from its size: an index takes 64 bits, where the bitmap takes one bit per permutation. The class docstring, the README and the changelog say so. Tests, the review's: the decreasing permutation of lengths 20, 21 and 25 listed, an index of length 21 from 2**63 to 2**64, and bisc on one permutation of length 21 or 25, against develop's answer. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 +- README.rst | 8 ++-- permuta/bisc/bisc.py | 67 ++++++++++++++++++++++--------- tests/bisc/test_sparse_storage.py | 44 +++++++++++++++++++- 4 files changed, 99 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f61de0a..27776f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - `PropertyBitmaps` in `permuta.bisc`: a property's values on the permutations, made from a property, given as bitmaps, or both. Each length is kept as a bitmap (lexicographic order) or, when fewer than one permutation in 64 is good, as the - sorted indices of the good ones, whichever is smaller. `auto_bisc` and `bisc` + sorted indices of the good ones, whichever is smaller (in a list past length + 20, where an index can need more than 8 bytes). `auto_bisc` and `bisc` accept it as input, and label the lengths it lacks with its property. ### Changed diff --git a/README.rst b/README.rst index 62ba289..a6affe4 100644 --- a/README.rst +++ b/README.rst @@ -525,9 +525,11 @@ the permutations of length 10 take 454 KB as a bitmap, where they take 0.9 to 1.8 GB as lists of ``Perm`` objects, depending on the Python. A length with fewer than one permutation in 64 satisfying the property keeps instead the indices of those, 8 bytes each, which then take less space: a short list of long -permutations stays small. A ``PropertyBitmaps`` can also be given directly, made -from a property, from bitmaps, or both. The lengths ``bisc`` and ``auto_bisc`` -label with its property are added to it, for a later run to reuse. +permutations stays small. Past length 20, where an index can need more than 8 +bytes, they are kept in a list. A ``PropertyBitmaps`` can also be given +directly, made from a property, from bitmaps, or both. The lengths ``bisc`` and +``auto_bisc`` label with its property are added to it, for a later run to +reuse. .. code-block:: python diff --git a/permuta/bisc/bisc.py b/permuta/bisc/bisc.py index d7812b1..03227d0 100644 --- a/permuta/bisc/bisc.py +++ b/permuta/bisc/bisc.py @@ -6,7 +6,18 @@ from array import array from collections.abc import Collection, Mapping, Sequence from math import factorial -from typing import Callable, Dict, Iterable, Iterator, List, Optional, Set, Tuple, Union +from typing import ( + Callable, + Dict, + Final, + Iterable, + Iterator, + List, + Optional, + Set, + Tuple, + Union, +) from permuta.bisc.bisc_subfunctions import ( forb, @@ -80,7 +91,7 @@ def bisc( # mine() reads the lengths up to m too, and a length past n is read as # empty: none of its permutations is good for length in range(n + 1, m + 1): - property_bitmaps._keep(length, array("Q"), 0) + property_bitmaps._keep(length, _new_indices(length), 0) elif isinstance(A, Mapping): if n is None: @@ -382,14 +393,31 @@ def _wanted_indices( yield index +# The sorted indices of a length's good permutations are kept in an array of +# this type code, unsigned integers of 8 bytes, while every index of the length +# fits in one: to length 20, since 21! is above 2**64. Past that they are kept in +# a list +_INDEX_TYPECODE: Final = "Q" +_INDEX_BITS = 8 * array(_INDEX_TYPECODE).itemsize +_Indices = Union["array[int]", List[int]] + # A length's labels, as a PropertyBitmaps keeps them: its bitmap, or the sorted # indices of its good permutations -_Labels = Union[bytes, bytearray, "array[int]"] +_Labels = Union[bytes, bytearray, _Indices] + +# A length keeps the sorted indices of its good permutations, _INDEX_BITS bits +# each, rather than its bitmap, a bit for each of its permutations, when fewer +# than one in this many of them are good: the indices are then the smaller +_SPARSE_WHEN_ONE_IN = _INDEX_BITS -# A length keeps the sorted indices of its good permutations, 8 bytes each, -# rather than its bitmap, a bit for each of its permutations, when fewer than -# one in this many of them are good: the indices are then the smaller -_SPARSE_WHEN_ONE_IN = 64 + +def _new_indices(length: int, indices: Iterable[int] = ()) -> _Indices: + """The sorted indices of a length's good permutations, given in increasing + order: an array while every index of the length fits in its items, and a + list past that.""" + if factorial(length) <= 1 << _INDEX_BITS: + return array(_INDEX_TYPECODE, indices) + return list(indices) def _is_sparse(good_count: int, length: int) -> bool: @@ -401,7 +429,7 @@ def _is_sparse(good_count: int, length: int) -> bool: def _label_in(labels: _Labels, index: int) -> int: """The label (0 or 1) of the permutation at index, from a length's labels in either form.""" - if isinstance(labels, array): + if isinstance(labels, (array, list)): position = bisect.bisect_left(labels, index) return int(position < len(labels) and labels[position] == index) return _label(labels, index) @@ -445,7 +473,8 @@ class PropertyBitmaps: the sorted indices of its good permutations (their bits), 8 bytes each, which then take less space. So no length takes more than its bitmap: three permutations of length 14 take 24 bytes, where its bitmap takes - 10.9 GB. + 10.9 GB. Past length 20, whose indices can exceed 8 bytes, they are kept + in a list, and there is no longest length. The labels can be given as bitmaps (bitmaps={length: bytes}), made from a property (add_length), or both: auto_bisc and bisc label the lengths they @@ -506,7 +535,9 @@ def __init__( ) good_count = _count_labels(bitmap) if _is_sparse(good_count, length): - good_indices = array("Q", _wanted_indices(bitmap, length, 1)) + good_indices = _new_indices( + length, _wanted_indices(bitmap, length, 1) + ) self._keep(length, good_indices, good_count) elif isinstance(bitmap, bytes): self._keep(length, bitmap, good_count) @@ -535,7 +566,7 @@ def add_length(self, length: int) -> None: "The PropertyBitmaps has no property to label length {} " "with".format(length) ) - good_indices = array("Q") + good_indices = _new_indices(length) bitmap: Optional[bytearray] = None good_count = 0 for index, perm in enumerate(Perm.of_length(length)): @@ -549,7 +580,7 @@ def add_length(self, length: int) -> None: # The indices take as much space as the bitmap: set # them in it, and go on in it bitmap = _bitmap_from_indices(length, good_indices) - good_indices = array("Q") + good_indices = _new_indices(length) self._keep(length, good_indices if bitmap is None else bitmap, good_count) @@ -563,7 +594,7 @@ def __init__(self, property_bitmaps: PropertyBitmaps) -> None: def __getitem__(self, length: int) -> bytes: labels = self.property_bitmaps._labels[length] - if isinstance(labels, array): + if isinstance(labels, (array, list)): return bytes(_bitmap_from_indices(length, labels)) return bytes(labels) @@ -639,7 +670,7 @@ def __iter__(self) -> Iterator[Perm]: if wanted_count == 0: # Nothing to yield, and no need to read the labels return - if isinstance(labels, array): + if isinstance(labels, (array, list)): if wanted_bit: for index in labels: yield Perm.unrank(index, self.length) @@ -693,21 +724,21 @@ def _labels_of(length: int, perms: Collection[Perm]) -> Tuple[_Labels, int]: the form their number chooses, and that number. A permutation given twice counts once.""" if _is_sparse(len(perms), length): - good_indices = array("Q", sorted(set(_indices_of(length, perms)))) + good_indices = _new_indices(length, sorted(set(_indices_of(length, perms)))) return good_indices, len(good_indices) bitmap = _bitmap_from_indices(length, _indices_of(length, perms)) good_count = _count_labels(bitmap) if _is_sparse(good_count, length): # Permutations given more than once made them look many - return array("Q", _wanted_indices(bitmap, length, 1)), good_count + return _new_indices(length, _wanted_indices(bitmap, length, 1)), good_count return bitmap, good_count def _overlap_count(labels: _Labels, other_labels: _Labels) -> int: """The number of permutations good in both of two labels of one length.""" - if isinstance(labels, array): + if isinstance(labels, (array, list)): return sum(_label_in(other_labels, index) for index in labels) - if isinstance(other_labels, array): + if isinstance(other_labels, (array, list)): return sum(_label_in(labels, index) for index in other_labels) view, other_view = memoryview(labels), memoryview(other_labels) return sum( diff --git a/tests/bisc/test_sparse_storage.py b/tests/bisc/test_sparse_storage.py index bb0c2b1..b89dbaf 100644 --- a/tests/bisc/test_sparse_storage.py +++ b/tests/bisc/test_sparse_storage.py @@ -31,7 +31,7 @@ def expected_bitmap(length, good_set): def is_kept_as_indices(property_bitmaps, length): - return isinstance(property_bitmaps._labels[length], array) + return isinstance(property_bitmaps._labels[length], (array, list)) def assert_views_match(property_bitmaps, length, good_set, bitmap=None): @@ -289,3 +289,45 @@ def test_three_permutations_of_length_14_take_little_memory(): tracemalloc.stop() assert peak_bytes < 2**20 assert shadings_as_sets(answer) == shadings_as_sets(DEVELOP_ANSWER_14) + + +@pytest.mark.parametrize("length", [20, 21, 25]) +def test_the_last_permutation_of_a_long_length_can_be_listed(length): + """The decreasing permutation of a length, whose index is its largest. 21! is + above 2**64, so past length 20 the indices are kept in a list.""" + last = Perm(range(length - 1, -1, -1)) + property_bitmaps = bisc_module._from_perms_by_length({length: [last]}) + assert list(property_bitmaps.good[length]) == [last] + assert last in property_bitmaps.good[length] + assert last not in property_bitmaps.bad[length] + assert property_bitmaps.good_counts[length] == 1 + kept_in = array if length <= 20 else list + assert isinstance(property_bitmaps._labels[length], kept_in) + + +def test_a_permutation_of_length_21_whose_index_needs_the_top_bit(): + """An index from 2**63 to 2**64, which an unsigned 64-bit integer holds and + a signed one does not.""" + perm = Perm.unrank(2**63 + 5, 21) + assert perm.rank() - bisc_module._shorter_count(21) == 2**63 + 5 + property_bitmaps = bisc_module._from_perms_by_length({21: [perm]}) + assert list(property_bitmaps.good[21]) == [perm] + assert perm in property_bitmaps.good[21] + + +# bisc([the decreasing permutation of length 21], 2, 21) on develop, before +# PropertyBitmaps, and the same at length 25 +DEVELOP_ANSWER_LONG = { + 0: {Perm(()): [{(0, 0)}]}, + 1: {Perm((0,)): [{(0, 1), (1, 0)}]}, + 2: {Perm((0, 1)): [set()], Perm((1, 0)): [{(0, 2), (1, 1), (2, 0)}]}, +} + + +@pytest.mark.parametrize("length", [21, 25]) +def test_bisc_of_one_long_permutation_is_as_before(length): + """bisc mines one listed permutation of length 21 or more, as it did before + PropertyBitmaps.""" + last = Perm(range(length - 1, -1, -1)) + answer = bisc([last], 2, length) + assert shadings_as_sets(answer) == shadings_as_sets(DEVELOP_ANSWER_LONG) From de9c0d34f3a256706f2f2d51203d06e8f2f00ba6 Mon Sep 17 00:00:00 2001 From: Henning Ulfarsson Date: Sat, 24 Oct 2026 17:57:52 +0000 Subject: [PATCH 28/38] Label a length in add_length with a bit per good permutation once its indices are many From the review of #209, round 6, blocking 2. On PyPy, auto_bisc(smooth) took 2.3 times develop's time: add_length's one loop called factorial (through _is_sparse) for every good permutation, and appended each to the array until the switch, then tested at each good one which form it was in. Now the limit is computed once (_most_indices, which _is_sparse uses too), and the loop is two. The first appends the indices of the good permutations while they are within it. Once one more is good, they are set in the bitmap, and the second loop labels the rest of the length, setting a bit for each good permutation and doing nothing more, as before the sparse storage. A length that stays within the limit keeps its indices, as before. PyPy 3.11, auto_bisc(smooth), eight runs of each, interleaved, each in its own process, with the Mac idle: median 1.02 s (1.01 to 1.46), develop 1.35 s, b4b27e3 2.50 s, 841c8dc 2.69 s; peak resident memory 106, 132, 168 and 172 MB. Co-Authored-By: Claude Opus 5.5 --- permuta/bisc/bisc.py | 46 ++++++++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/permuta/bisc/bisc.py b/permuta/bisc/bisc.py index 03227d0..e1499b3 100644 --- a/permuta/bisc/bisc.py +++ b/permuta/bisc/bisc.py @@ -420,10 +420,16 @@ def _new_indices(length: int, indices: Iterable[int] = ()) -> _Indices: return list(indices) +def _most_indices(length: int) -> int: + """The most good permutations of a length that are kept as indices: fewer + than one in _SPARSE_WHEN_ONE_IN of its permutations.""" + return (factorial(length) - 1) // _SPARSE_WHEN_ONE_IN + + def _is_sparse(good_count: int, length: int) -> bool: """Whether a length with good_count good permutations keeps their indices rather than its bitmap.""" - return good_count * _SPARSE_WHEN_ONE_IN < factorial(length) + return good_count <= _most_indices(length) def _label_in(labels: _Labels, index: int) -> int: @@ -558,30 +564,36 @@ def _keep(self, length: int, labels: _Labels, good_count: int) -> None: def add_length(self, length: int) -> None: """Label the permutations of a length with the property, and keep the length's labels, replacing any it has. The indices of the good - permutations are kept while they are few, and once they would take - as much space as the length's bitmap, they are set in it, which is - filled from then on. It needs a property.""" + permutations are kept while they are few; once there are more than a + length keeps as indices, they are set in the length's bitmap, and the + rest of the length is labeled in it. It needs a property.""" if self.prop is None: raise ValueError( "The PropertyBitmaps has no property to label length {} " "with".format(length) ) + most_indices = _most_indices(length) good_indices = _new_indices(length) - bitmap: Optional[bytearray] = None - good_count = 0 - for index, perm in enumerate(Perm.of_length(length)): + perms = enumerate(Perm.of_length(length)) + for index, perm in perms: + if self.prop(perm): + good_indices.append(index) + if len(good_indices) > most_indices: + break + else: + self._keep(length, good_indices, len(good_indices)) + return + # Too many for indices: they are set in the bitmap, and the loop goes + # on in it from the next permutation, with a bit to set for each good + # one and nothing more + bitmap = _bitmap_from_indices(length, good_indices) + good_count = len(good_indices) + del good_indices + for index, perm in perms: if self.prop(perm): + _set_label(bitmap, index) good_count += 1 - if bitmap is not None: - _set_label(bitmap, index) - else: - good_indices.append(index) - if not _is_sparse(good_count, length): - # The indices take as much space as the bitmap: set - # them in it, and go on in it - bitmap = _bitmap_from_indices(length, good_indices) - good_indices = _new_indices(length) - self._keep(length, good_indices if bitmap is None else bitmap, good_count) + self._keep(length, bitmap, good_count) class _Bitmaps(Mapping[int, bytes]): From 847e1c6f43d2d99f02174fc45a9a7a69b658aeb9 Mon Sep 17 00:00:00 2001 From: Henning Ulfarsson Date: Sat, 24 Oct 2026 17:59:46 +0000 Subject: [PATCH 29/38] Keep a dense length's bitmap as bytes, so that reading it copies nothing From the review of #209, round 6, its suggestions 2 and 5. add_length and a list or a mapping kept a dense length's bitmap as a bytearray, so that bitmaps copied it to bytes on every read: a millisecond a read at length 12. Now a bitmap is kept as bytes once built, and bitmaps gives it as it is. add_length converts its bytearray once at the end: its peak stays two bitmaps, as at the switch from indices. The constructor and _labels_of both counted a bitmap and then chose its form; _labels_from_bitmap does that for both, copying a bytearray to bytes as the constructor did, so that changing a bytearray given still does not change the labels. _Labels no longer includes bytearray. Test: a dense length from a property, from a list and from a bytearray given is kept as bytes, and bitmaps gives the same object on every read. Co-Authored-By: Claude Opus 5.5 --- permuta/bisc/bisc.py | 39 +++++++++++++++---------------- tests/bisc/test_sparse_storage.py | 19 +++++++++++++++ 2 files changed, 38 insertions(+), 20 deletions(-) diff --git a/permuta/bisc/bisc.py b/permuta/bisc/bisc.py index e1499b3..4a8f594 100644 --- a/permuta/bisc/bisc.py +++ b/permuta/bisc/bisc.py @@ -403,7 +403,7 @@ def _wanted_indices( # A length's labels, as a PropertyBitmaps keeps them: its bitmap, or the sorted # indices of its good permutations -_Labels = Union[bytes, bytearray, _Indices] +_Labels = Union[bytes, _Indices] # A length keeps the sorted indices of its good permutations, _INDEX_BITS bits # each, rather than its bitmap, a bit for each of its permutations, when fewer @@ -464,6 +464,18 @@ def _count_labels(bitmap: Union[bytes, bytearray]) -> int: ) +def _labels_from_bitmap( + length: int, bitmap: Union[bytes, bytearray] +) -> Tuple[_Labels, int]: + """A length's labels from its bitmap, in the form chosen by its number of + good permutations, and that number. A bitmap kept is bytes: a bytearray + is copied, so that changing it does not change the labels.""" + good_count = _count_labels(bitmap) + if _is_sparse(good_count, length): + return _new_indices(length, _wanted_indices(bitmap, length, 1)), good_count + return bytes(bitmap), good_count + + class PropertyBitmaps: """A property's values on the permutations, length by length: which permutations of each length satisfy it (good) and which do not (bad). @@ -539,18 +551,7 @@ def __init__( "A bitmap of length {} has bits set past its {} " "permutations".format(length, factorial(length)) ) - good_count = _count_labels(bitmap) - if _is_sparse(good_count, length): - good_indices = _new_indices( - length, _wanted_indices(bitmap, length, 1) - ) - self._keep(length, good_indices, good_count) - elif isinstance(bitmap, bytes): - self._keep(length, bitmap, good_count) - else: - # A copy, so that a change to the bytearray does not change - # the labels - self._keep(length, bytes(bitmap), good_count) + self._keep(length, *_labels_from_bitmap(length, bitmap)) self.bitmaps: Mapping[int, bytes] = _Bitmaps(self) self.good = _LengthViews(self, wanted_bit=1) self.bad = _LengthViews(self, wanted_bit=0) @@ -593,7 +594,7 @@ def add_length(self, length: int) -> None: if self.prop(perm): _set_label(bitmap, index) good_count += 1 - self._keep(length, bitmap, good_count) + self._keep(length, bytes(bitmap), good_count) class _Bitmaps(Mapping[int, bytes]): @@ -608,7 +609,7 @@ def __getitem__(self, length: int) -> bytes: labels = self.property_bitmaps._labels[length] if isinstance(labels, (array, list)): return bytes(_bitmap_from_indices(length, labels)) - return bytes(labels) + return labels def __contains__(self, length: object) -> bool: # Mapping's would build the bitmap of a length kept as indices, to @@ -738,12 +739,10 @@ def _labels_of(length: int, perms: Collection[Perm]) -> Tuple[_Labels, int]: if _is_sparse(len(perms), length): good_indices = _new_indices(length, sorted(set(_indices_of(length, perms)))) return good_indices, len(good_indices) + # Permutations given more than once can make them look many: the count of + # the bitmap's bits chooses the form bitmap = _bitmap_from_indices(length, _indices_of(length, perms)) - good_count = _count_labels(bitmap) - if _is_sparse(good_count, length): - # Permutations given more than once made them look many - return _new_indices(length, _wanted_indices(bitmap, length, 1)), good_count - return bitmap, good_count + return _labels_from_bitmap(length, bitmap) def _overlap_count(labels: _Labels, other_labels: _Labels) -> int: diff --git a/tests/bisc/test_sparse_storage.py b/tests/bisc/test_sparse_storage.py index b89dbaf..1addfdc 100644 --- a/tests/bisc/test_sparse_storage.py +++ b/tests/bisc/test_sparse_storage.py @@ -331,3 +331,22 @@ def test_bisc_of_one_long_permutation_is_as_before(length): last = Perm(range(length - 1, -1, -1)) answer = bisc([last], 2, length) assert shadings_as_sets(answer) == shadings_as_sets(DEVELOP_ANSWER_LONG) + + +def test_a_dense_length_is_kept_as_bytes_and_read_without_a_copy(): + """A length kept as a bitmap keeps bytes, however it came in, so that + bitmaps gives the same object on every read.""" + length = 6 + good_set = { + perm for index, perm in enumerate(Perm.of_length(length)) if index % 3 == 0 + } + by_property = PropertyBitmaps(lambda perm: perm in good_set) + by_property.add_length(length) + from_list = bisc_module._from_perms_by_length({length: sorted(good_set)}) + given = PropertyBitmaps( + bitmaps={length: bytearray(expected_bitmap(length, good_set))} + ) + for property_bitmaps in (by_property, from_list, given): + assert type(property_bitmaps._labels[length]) is bytes + assert property_bitmaps.bitmaps[length] is property_bitmaps.bitmaps[length] + assert property_bitmaps.bitmaps[length] == expected_bitmap(length, good_set) From e69810dc53ff8ba09960921a6c0c4cf53e214bbd Mon Sep 17 00:00:00 2001 From: Henning Ulfarsson Date: Sat, 24 Oct 2026 18:01:30 +0000 Subject: [PATCH 30/38] Test the gaps the review of round 6 found in the two forms' tests From the review of #209, round 6, its suggestion 1: its 19 tests, as it wrote them, but in tests/bisc/test_sparse_storage.py, with that file's helpers (assert_views_match for their assert_reads_as, which it contains), docstrings for the numbered comments, and one local named byte. They cover: - a bitmap given as a bytearray or a memoryview, changed afterwards, and a memoryview of another format, shape or stride; - bitmaps giving bytes, equal on every read, and a round trip through it that keeps each length's form; - the first and the last permutation of a length kept as indices; - a property that raises in add_length, before and after the switch, and add_length replacing labels in either direction between the forms; - the lengths past n, with m > n, counted as empty and kept as no indices without a bitmap sized; a length of 14 kept as indices with no bitmap sized, which runs on PyPy as well, where tracemalloc does not; - a view iterated twice or in step with itself, bisc taking a view, and a repeated permutation in a list that looks sparse. They pass on 841c8dc too: they close gaps, which faults planted by the review in copies of it showed. Co-Authored-By: Claude Opus 5.5 --- tests/bisc/test_sparse_storage.py | 292 ++++++++++++++++++++++++++++++ 1 file changed, 292 insertions(+) diff --git a/tests/bisc/test_sparse_storage.py b/tests/bisc/test_sparse_storage.py index 1addfdc..d89dec9 100644 --- a/tests/bisc/test_sparse_storage.py +++ b/tests/bisc/test_sparse_storage.py @@ -350,3 +350,295 @@ def test_a_dense_length_is_kept_as_bytes_and_read_without_a_copy(): assert type(property_bitmaps._labels[length]) is bytes assert property_bitmaps.bitmaps[length] is property_bitmaps.bitmaps[length] assert property_bitmaps.bitmaps[length] == expected_bitmap(length, good_set) + + +LENGTH_6 = list(Perm.of_length(6)) # 720 permutations: kept as indices below 12 good +SPARSE_6 = {LENGTH_6[index] for index in (0, 5, 100, 719)} +DENSE_6 = {perm for index, perm in enumerate(LENGTH_6) if index % 3 == 0} + + +@pytest.mark.parametrize("good_set", [SPARSE_6, DENSE_6], ids=["sparse", "dense"]) +@pytest.mark.parametrize("wrap", [bytearray, lambda raw: memoryview(bytearray(raw))]) +def test_a_buffer_given_and_changed_afterwards_changes_nothing(good_set, wrap): + """A bitmap given is copied, whatever its form: changing the caller's + bytearray, or the one under a memoryview, afterwards changes nothing.""" + buffer = wrap(expected_bitmap(6, good_set)) + underlying = buffer.obj if isinstance(buffer, memoryview) else buffer + property_bitmaps = PropertyBitmaps(bitmaps={6: buffer}) + assert is_kept_as_indices(property_bitmaps, 6) == (good_set is SPARSE_6) + underlying[:] = b"\xff" * len(underlying) # every permutation good + assert_views_match(property_bitmaps, 6, good_set) + underlying[:] = bytes(len(underlying)) # none good + assert_views_match(property_bitmaps, 6, good_set) + + +@pytest.mark.parametrize("good_set", [SPARSE_6, DENSE_6], ids=["sparse", "dense"]) +def test_a_memoryview_of_another_format_is_read_as_bytes(good_set): + """A memoryview of another format, shape or stride is read as its bytes.""" + raw = expected_bitmap(6, good_set) # 90 bytes + views = { + "2-byte items": memoryview(raw).cast("H"), # 45 items of 2 bytes + "3 x 30": memoryview(raw).cast("B", shape=[3, 30]), + "every other byte": memoryview(b"".join(bytes((byte, 0)) for byte in raw))[::2], + "signed": memoryview(raw).cast("b"), + } + for name, view in views.items(): + property_bitmaps = PropertyBitmaps(bitmaps={6: view}) + assert is_kept_as_indices(property_bitmaps, 6) == (good_set is SPARSE_6), name + assert_views_match(property_bitmaps, 6, good_set) + + +def test_bitmaps_give_bytes_that_are_equal_on_every_read(): + """bitmaps gives a length's bitmap as bytes, whatever form the length is + kept in, and the same on every read.""" + by_property = PropertyBitmaps(lambda perm: perm in DENSE_6 or perm in SPARSE_6) + by_property.add_length(6) + sparse_by_property = PropertyBitmaps(lambda perm: perm in SPARSE_6) + sparse_by_property.add_length(6) + from_list = bisc_module._from_perms_by_length({6: sorted(DENSE_6)}) + from_sparse_list = bisc_module._from_perms_by_length({6: sorted(SPARSE_6)}) + for property_bitmaps in ( + by_property, + sparse_by_property, + from_list, + from_sparse_list, + ): + first = property_bitmaps.bitmaps[6] + assert type(first) is bytes + assert property_bitmaps.bitmaps[6] == first + assert len(first) == 90 + + +def extremes(length): + """The first and the last permutation of a length: one for lengths 0 and 1.""" + return {Perm(range(length)), Perm(range(length - 1, -1, -1))} + + +def test_bitmaps_round_trip_with_lengths_kept_as_indices(): + """A PropertyBitmaps made from another's bitmaps keeps each length in the + same form, with the same labels.""" + property_bitmaps = PropertyBitmaps(lambda perm: perm in extremes(len(perm))) + for length in range(9): + property_bitmaps.add_length(length) + assert [is_kept_as_indices(property_bitmaps, length) for length in range(9)] == [ + False, + False, + False, + False, + False, + False, + True, + True, + True, + ] + round_trip = PropertyBitmaps(bitmaps=property_bitmaps.bitmaps) + for length in range(9): + assert is_kept_as_indices(round_trip, length) == is_kept_as_indices( + property_bitmaps, length + ) + assert round_trip.good_counts[length] == property_bitmaps.good_counts[length] + assert round_trip.bitmaps[length] == property_bitmaps.bitmaps[length] + assert list(round_trip.good[length]) == list(property_bitmaps.good[length]) + assert list(round_trip.bad[length]) == list(property_bitmaps.bad[length]) + + +LENGTH_8 = list(Perm.of_length(8)) +SPARSE_8 = {LENGTH_8[0], LENGTH_8[-1]} # the first and the last permutation + + +def test_the_first_and_the_last_permutation_of_a_sparse_length(): + """The first and the last permutation of a length kept as indices, in + either view, good or bad; and, with the good ones in between, a bitmap.""" + first, last = LENGTH_8[0], LENGTH_8[-1] + for good_set in (SPARSE_8, {first}, {last}): + property_bitmaps = bisc_module._from_perms_by_length({8: sorted(good_set)}) + assert is_kept_as_indices(property_bitmaps, 8) + good, bad = property_bitmaps.good[8], property_bitmaps.bad[8] + for perm in (first, last): + assert (perm in good) == (perm in good_set) + assert (perm in bad) == (perm not in good_set) + assert list(good) == sorted(good_set) + bad_perms = list(bad) + assert len(bad_perms) == len(bad) == factorial(8) - len(good_set) + assert bad_perms == [perm for perm in LENGTH_8 if perm not in good_set] + in_between = set(LENGTH_8[1:-1]) + property_bitmaps = bisc_module._from_perms_by_length({8: sorted(in_between)}) + assert not is_kept_as_indices(property_bitmaps, 8) + assert list(property_bitmaps.bad[8]) == [first, last] + + +class PropertyFailed(Exception): + pass + + +@pytest.mark.parametrize("fail_after", [10, 1000, 5039]) +def test_a_property_that_raises_midway_through_add_length_keeps_nothing(fail_after): + """A property that raises in the middle of add_length, while the indices + are kept or after the switch to the bitmap, leaves nothing of that call, + and a length labeled before keeps its labels.""" + perms = list(Perm.of_length(7)) # 5040; all good makes the switch at the 79th + calls = [] + + def all_good_but_fails_once_asked(perm): + calls.append(perm) + if len(calls) == fail_after + 1: + raise PropertyFailed + return True + + property_bitmaps = PropertyBitmaps(all_good_but_fails_once_asked) + with pytest.raises(PropertyFailed): + property_bitmaps.add_length(7) + assert 7 not in property_bitmaps.bitmaps and 7 not in property_bitmaps.good_counts + assert 7 not in property_bitmaps.good and 7 not in property_bitmaps.bad + property_bitmaps.prop = lambda perm: perm == perms[3] + property_bitmaps.add_length(7) + property_bitmaps.prop = all_good_but_fails_once_asked + calls.clear() + with pytest.raises(PropertyFailed): + property_bitmaps.add_length(7) + assert_views_match(property_bitmaps, 7, {perms[3]}) + # A call that succeeds replaces the labels + property_bitmaps.prop = lambda perm: True + property_bitmaps.add_length(7) + assert_views_match(property_bitmaps, 7, set(perms)) + assert not is_kept_as_indices(property_bitmaps, 7) + + +def test_add_length_replaces_labels_between_the_forms(): + """add_length replaces a length's labels, from indices to a bitmap and + back.""" + property_bitmaps = PropertyBitmaps(lambda perm: perm in SPARSE_6) + property_bitmaps.add_length(6) + assert is_kept_as_indices(property_bitmaps, 6) + property_bitmaps.prop = lambda perm: perm in DENSE_6 + property_bitmaps.add_length(6) + assert not is_kept_as_indices(property_bitmaps, 6) + assert_views_match(property_bitmaps, 6, DENSE_6) + property_bitmaps.prop = lambda perm: perm in SPARSE_6 + property_bitmaps.add_length(6) + assert is_kept_as_indices(property_bitmaps, 6) + assert_views_match(property_bitmaps, 6, SPARSE_6) + + +def test_lengths_past_n_have_a_count_of_zero(monkeypatch): + """With m > n, the lengths past n are counted as empty, not only holding no + bits: mine() takes a length for complete when its count is the length's + factorial, and with a count of 1 it would take length 1 for complete.""" + seen = [] + real_mine = bisc_module.mine + + def recording_mine(goodperms, M, N=None, report=False): + property_bitmaps = goodperms.property_bitmaps + seen.append( + { + length: ( + property_bitmaps.good_counts[length], + len(goodperms[length]), + list(goodperms[length]), + len(property_bitmaps.bad[length]), + ) + for length in goodperms + } + ) + return real_mine(goodperms, M, N, report) + + monkeypatch.setattr(bisc_module, "mine", recording_mine) + bisc(lambda perm: True, 3, 0) + assert seen[0] == { + 0: (1, 1, [Perm(())], 0), + 1: (0, 0, [], 1), + 2: (0, 0, [], 2), + 3: (0, 0, [], 6), + } + + +def test_a_sparse_view_iterates_independently(): + """The same view of a length kept as indices, iterated twice or in step + with itself, gives the same permutations.""" + property_bitmaps = bisc_module._from_perms_by_length({6: sorted(SPARSE_6)}) + for view in (property_bitmaps.good[6], property_bitmaps.bad[6]): + once = list(view) + assert list(view) == once + assert list(zip(view, view)) == [(perm, perm) for perm in once] + + +def test_bisc_takes_the_view_of_a_sparse_length(): + """bisc given the good view of a PropertyBitmaps, a Mapping, mines what it + mines from the list it was made from.""" + perms = [Perm(range(length)) for length in range(7)] + [Perm(range(6, -1, -1))] + property_bitmaps = bisc_module._from_perms_by_length( + {length: [perm for perm in perms if len(perm) == length] for length in range(8)} + ) + assert is_kept_as_indices(property_bitmaps, 7) + assert bisc(property_bitmaps.good, 2, 7) == bisc(perms, 2, 7) + + +def test_a_sparse_length_14_never_sizes_a_bitmap(monkeypatch): + """No bitmap is sized for a length kept as indices: length 14's would take + 10.9 GB. A check of the structure, so that it runs on PyPy too, which has no + tracemalloc.""" + real_bitmap_size = bisc_module._bitmap_size + + def guarded_bitmap_size(length): + assert length <= 10, "a bitmap of length {} was sized".format(length) + return real_bitmap_size(length) + + monkeypatch.setattr(bisc_module, "_bitmap_size", guarded_bitmap_size) + by_length = {length: [] for length in range(15)} + by_length[14] = THREE_OF_LENGTH_14 + by_length[13] = [Perm(range(13))] + by_length[12] = [Perm(range(12))] + by_length[11] = [Perm(range(11))] + property_bitmaps = bisc_module._from_perms_by_length(by_length) + for length in (11, 12, 13, 14): + labels = property_bitmaps._labels[length] + assert isinstance(labels, array) and len(labels) == len(by_length[length]) + assert list(property_bitmaps.good[14]) == sorted(THREE_OF_LENGTH_14) + assert len(property_bitmaps.bad[14]) == factorial(14) - 3 + for perm in THREE_OF_LENGTH_14: + assert perm in property_bitmaps.good[14] + assert perm not in property_bitmaps.bad[14] + assert Perm(range(14)) in property_bitmaps.bad[14] + assert 14 in property_bitmaps.bitmaps and len(property_bitmaps.bitmaps) == 15 + + +class MineStopped(Exception): + pass + + +def test_lengths_past_n_are_kept_as_empty_indices_and_size_no_bitmap(monkeypatch): + """With m > n, a length past n is kept as no indices, not as a bitmap: m = 14 + would size one of 10.9 GB. The lengths up to n are labeled.""" + real_bitmap_size = bisc_module._bitmap_size + kept = {} + + def guarded_bitmap_size(length): + assert length <= 10, "a bitmap of length {} was sized".format(length) + return real_bitmap_size(length) + + def stopped_mine(goodperms, M, N=None, report=False): + property_bitmaps = goodperms.property_bitmaps + kept.update(property_bitmaps._labels) + raise MineStopped + + monkeypatch.setattr(bisc_module, "_bitmap_size", guarded_bitmap_size) + monkeypatch.setattr(bisc_module, "mine", stopped_mine) + with pytest.raises(MineStopped): + bisc(lambda perm: perm.is_increasing(), 14, 2) + assert sorted(kept) == list(range(15)) + for length in range(3, 15): + assert isinstance(kept[length], array) and len(kept[length]) == 0 + + +def test_a_sparse_list_with_repeats_counts_each_permutation_once(): + """A list that looks sparse before it is counted, with a permutation + repeated, counts it once.""" + first, second = LENGTH_8[7], LENGTH_8[100] + property_bitmaps = bisc_module._from_perms_by_length( + {8: [second, first, first, first]} + ) + assert is_kept_as_indices(property_bitmaps, 8) + assert property_bitmaps.good_counts[8] == 2 + assert len(property_bitmaps.good[8]) == 2 + assert len(property_bitmaps.bad[8]) == factorial(8) - 2 + assert list(property_bitmaps.good[8]) == [first, second] From 9ac4e3d859b9f1a2448280c42fd4822f224ae966 Mon Sep 17 00:00:00 2001 From: Henning Ulfarsson Date: Sat, 24 Oct 2026 18:04:56 +0000 Subject: [PATCH 31/38] Readability and repetition points from the review of round 6 From the review of #209, round 6, its suggestions 3 to 5. - The texts say that the number of good permutations chooses a length's form, and _keep trusts its caller to have chosen it. A comment where _labels is made says that the storage is the module's: bitmaps, the views and the functions that make a PropertyBitmaps read _labels and keep a length's labels through _keep. - The class docstring says what reading bitmaps costs: the bitmap of a length kept as indices is built on each read, so items(), values(), dict() or bitmaps given to another PropertyBitmaps build every such length's, 10.9 GB for length 14; asking whether a length is there, iterating over the lengths and len() build none. - _count_labels, which counts bits, is _count_bits. It and the overlap of two bitmaps read them through one generator, _chunks. bitmaps and the views share their lengths through a base class, _Lengths. - A Perm whose index falls outside its length's, which only one whose values are not a permutation's can have, raises a ValueError naming it, whichever form its length is kept in. Before, a length kept as indices took it silently and failed later, and a bitmap raised IndexError. The changelog says so. - The tests of the two forms have docstrings, the threshold's comment says from which length a set with a good permutation is kept as indices (3 with one in 2, not 2), and the locals randomness, first_dense, good_8 and bad_8 are seeded_random, fewest_in_a_bitmap, good_of_length_8 and bad_of_length_8. A test checks the ValueError, in both forms. Not taken: one bitmap oracle for the three test files, which would change round 4's and 5's tests for no gain in what they check. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 +- permuta/bisc/bisc.py | 92 ++++++++++++++++++------------- tests/bisc/test_sparse_storage.py | 84 ++++++++++++++++------------ 3 files changed, 104 insertions(+), 75 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 27776f2..91aebce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,7 +27,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. length, from a tuple of dictionaries or a stored set, overlap or are not together all of that length's permutations. - `bisc` and `auto_bisc` raise a `ValueError` for a permutation given under the - key of another length. + key of another length, and for a `Perm` given whose values are not those of a + permutation of its length. - `bisc` accepts any mapping of lengths to permutations, not only a `dict`. ### Fixed diff --git a/permuta/bisc/bisc.py b/permuta/bisc/bisc.py index 4a8f594..a535bd0 100644 --- a/permuta/bisc/bisc.py +++ b/permuta/bisc/bisc.py @@ -455,13 +455,16 @@ def _bitmap_from_indices(length: int, indices: Iterable[int]) -> bytearray: _CHUNK_BYTES = 1 << 16 -def _count_labels(bitmap: Union[bytes, bytearray]) -> int: - """The number of bits set in a bitmap: its good permutations.""" +def _chunks(bitmap: Union[bytes, bytearray]) -> Iterator[int]: + """A bitmap's bytes, _CHUNK_BYTES at a time, each chunk as an int.""" view = memoryview(bitmap) - return sum( - int.from_bytes(view[start : start + _CHUNK_BYTES], "little").bit_count() - for start in range(0, len(view), _CHUNK_BYTES) - ) + for start in range(0, len(view), _CHUNK_BYTES): + yield int.from_bytes(view[start : start + _CHUNK_BYTES], "little") + + +def _count_bits(bitmap: Union[bytes, bytearray]) -> int: + """The number of bits set in a bitmap: its good permutations.""" + return sum(chunk.bit_count() for chunk in _chunks(bitmap)) def _labels_from_bitmap( @@ -470,7 +473,7 @@ def _labels_from_bitmap( """A length's labels from its bitmap, in the form chosen by its number of good permutations, and that number. A bitmap kept is bytes: a bytearray is copied, so that changing it does not change the labels.""" - good_count = _count_labels(bitmap) + good_count = _count_bits(bitmap) if _is_sparse(good_count, length): return _new_indices(length, _wanted_indices(bitmap, length, 1)), good_count return bytes(bitmap), good_count @@ -505,8 +508,12 @@ class PropertyBitmaps: The attributes bitmaps (length -> its bitmap, a read-only Mapping) and good_counts (length -> the number of good permutations) are for reading. - The bitmap of a length kept as indices is built when bitmaps is asked - for it. + bitmaps gives a length kept as a bitmap as it is kept, and builds the + bitmap of a length kept as indices each time it is asked for it: so + items(), values() and dict(bitmaps), or bitmaps given to another + PropertyBitmaps, build every such length's bitmap, 10.9 GB for length 14. + Asking whether a length is there, iterating over the lengths and len() + build none. The views read the labels length by length: iterating over a length, counting it, asking whether a length is there, or whether a permutation @@ -520,7 +527,10 @@ def __init__( bitmaps: Optional[Mapping[int, Union[bytes, bytearray, memoryview]]] = None, ) -> None: self.prop = prop - # length -> its labels, in the form their number chooses (_is_sparse) + # length -> its labels, in the form chosen by its number of good + # permutations (_is_sparse). The storage is the module's: bitmaps, the + # views and the functions that make a PropertyBitmaps read _labels, + # and keep a length's labels through _keep self._labels: Dict[int, _Labels] = {} self.good_counts: Dict[int, int] = {} if bitmaps is not None: @@ -557,8 +567,9 @@ def __init__( self.bad = _LengthViews(self, wanted_bit=0) def _keep(self, length: int, labels: _Labels, good_count: int) -> None: - """Keep a length's labels, in the form their number chooses, with that - number of good permutations. The views read them from here.""" + """Keep a length's labels as given, with its number of good + permutations. The caller chooses the form (_is_sparse), and is trusted + to; the views read the labels from here.""" self._labels[length] = labels self.good_counts[length] = good_count @@ -597,14 +608,25 @@ def add_length(self, length: int) -> None: self._keep(length, bytes(bitmap), good_count) -class _Bitmaps(Mapping[int, bytes]): - """length -> its bitmap, read-only: the bitmap kept, or, for a length kept - as the indices of its good permutations, one built from them when - asked.""" +class _Lengths: + """The lengths with labels in a PropertyBitmaps, which its Mappings, + bitmaps and the views, iterate over and count.""" def __init__(self, property_bitmaps: PropertyBitmaps) -> None: self.property_bitmaps = property_bitmaps + def __iter__(self) -> Iterator[int]: + return iter(self.property_bitmaps._labels) + + def __len__(self) -> int: + return len(self.property_bitmaps._labels) + + +class _Bitmaps(_Lengths, Mapping[int, bytes]): + """length -> its bitmap, read-only: the bitmap kept, or, for a length kept + as the indices of its good permutations, one built from them when + asked.""" + def __getitem__(self, length: int) -> bytes: labels = self.property_bitmaps._labels[length] if isinstance(labels, (array, list)): @@ -616,21 +638,15 @@ def __contains__(self, length: object) -> bool: # answer whether the length is there return length in self.property_bitmaps._labels - def __iter__(self) -> Iterator[int]: - return iter(self.property_bitmaps._labels) - def __len__(self) -> int: - return len(self.property_bitmaps._labels) - - -class _LengthViews(Mapping[int, "_PermsOfLength"]): +class _LengthViews(_Lengths, Mapping[int, "_PermsOfLength"]): """length -> the good (or bad) permutations of that length, read-only: a Mapping whose keys are the lengths with labels, through which bisc and auto_bisc read the permutations. wanted_bit is the label of the permutations it gives: 1 for the good ones, 0 for the bad ones.""" def __init__(self, property_bitmaps: PropertyBitmaps, wanted_bit: int) -> None: - self.property_bitmaps = property_bitmaps + super().__init__(property_bitmaps) self.wanted_bit = wanted_bit def __getitem__(self, length: int) -> "_PermsOfLength": @@ -638,12 +654,6 @@ def __getitem__(self, length: int) -> "_PermsOfLength": raise KeyError(length) return _PermsOfLength(self.property_bitmaps, length, wanted_bit=self.wanted_bit) - def __iter__(self) -> Iterator[int]: - return iter(self.property_bitmaps._labels) - - def __len__(self) -> int: - return len(self.property_bitmaps._labels) - class _PermsOfLength(Collection[Perm]): """The good (or bad) permutations of one length, read from its labels: a @@ -719,8 +729,11 @@ def _shorter_count(length: int) -> int: def _indices_of(length: int, perms: Iterable[Perm]) -> Iterator[int]: """The indices (bits) of permutations given as ones of a length. One of - another length raises a ValueError.""" + another length raises a ValueError, and so does one whose index is not + one of the length's, which only a sequence that is not a permutation can + have.""" shorter_count = _shorter_count(length) + permutation_count = factorial(length) for perm in perms: if len(perm) != length: raise ValueError( @@ -729,12 +742,17 @@ def _indices_of(length: int, perms: Iterable[Perm]) -> Iterator[int]: ) if not isinstance(perm, Perm): perm = Perm(perm) - yield perm.rank() - shorter_count + index = perm.rank() - shorter_count + if not 0 <= index < permutation_count: + raise ValueError( + "{!r} is not a permutation of 0 to {}".format(perm, length - 1) + ) + yield index def _labels_of(length: int, perms: Collection[Perm]) -> Tuple[_Labels, int]: """The labels of a length whose good permutations are the ones given, in - the form their number chooses, and that number. A permutation given + the form chosen by their number, and that number. A permutation given twice counts once.""" if _is_sparse(len(perms), length): good_indices = _new_indices(length, sorted(set(_indices_of(length, perms)))) @@ -751,13 +769,9 @@ def _overlap_count(labels: _Labels, other_labels: _Labels) -> int: return sum(_label_in(other_labels, index) for index in labels) if isinstance(other_labels, (array, list)): return sum(_label_in(labels, index) for index in other_labels) - view, other_view = memoryview(labels), memoryview(other_labels) return sum( - ( - int.from_bytes(view[start : start + _CHUNK_BYTES], "little") - & int.from_bytes(other_view[start : start + _CHUNK_BYTES], "little") - ).bit_count() - for start in range(0, len(view), _CHUNK_BYTES) + (chunk & other_chunk).bit_count() + for chunk, other_chunk in zip(_chunks(labels), _chunks(other_labels)) ) diff --git a/tests/bisc/test_sparse_storage.py b/tests/bisc/test_sparse_storage.py index d89dec9..8d3d0bf 100644 --- a/tests/bisc/test_sparse_storage.py +++ b/tests/bisc/test_sparse_storage.py @@ -56,7 +56,7 @@ def assert_views_match(property_bitmaps, length, good_set, bitmap=None): assert property_bitmaps.bitmaps[length] == bitmap -def each_way_in(good_sets, bitmaps, randomness): +def each_way_in(good_sets, bitmaps, seeded_random): """A PropertyBitmaps made from good_sets in each way that bisc and auto_bisc make one, with its name: a list or a mapping (whose good permutations come shuffled, with some twice), a tuple of the good and the bad, bitmaps, and a @@ -65,7 +65,7 @@ def each_way_in(good_sets, bitmaps, randomness): good_lists, bad_lists = {}, {} for length in lengths: good_lists[length] = sorted(good_sets[length]) - randomness.shuffle(good_lists[length]) + seeded_random.shuffle(good_lists[length]) bad_lists[length] = [ perm for perm in Perm.of_length(length) if perm not in good_sets[length] ] @@ -81,7 +81,7 @@ def each_way_in(good_sets, bitmaps, randomness): yield "property", by_property -def first_dense_count(length, sparse_when_one_in): +def fewest_kept_as_a_bitmap(length, sparse_when_one_in): """The fewest good permutations of a length that are kept as a bitmap.""" return -(-factorial(length) // sparse_when_one_in) @@ -96,35 +96,38 @@ def first_dense_count(length, sparse_when_one_in): ) -def good_count_of(size_kind, length, sparse_when_one_in, randomness): - first_dense = min(first_dense_count(length, sparse_when_one_in), factorial(length)) +def good_count_of(size_kind, length, sparse_when_one_in, seeded_random): + fewest_in_a_bitmap = min( + fewest_kept_as_a_bitmap(length, sparse_when_one_in), factorial(length) + ) return { "none": 0, - "just under the threshold": max(first_dense - 1, 0), - "at the threshold": first_dense, + "just under the threshold": max(fewest_in_a_bitmap - 1, 0), + "at the threshold": fewest_in_a_bitmap, "all": factorial(length), - "random under": randomness.randint(0, max(first_dense - 1, 0)), - "random over": randomness.randint(first_dense, factorial(length)), + "random under": seeded_random.randint(0, max(fewest_in_a_bitmap - 1, 0)), + "random over": seeded_random.randint(fewest_in_a_bitmap, factorial(length)), }[size_kind] -# Both forms against brute force, for every view and every way in, at lengths 0 -# to 8. With one in 2, both forms occur at every length from 2 on; with one in -# 64, the default, from 5 on. The bitmaps are read seven bytes at a time, so that -# counting them takes several chunks @pytest.mark.parametrize("sparse_when_one_in", [2, 64]) @pytest.mark.parametrize("size_kind", SIZE_KINDS) def test_both_forms_read_as_brute_force(monkeypatch, sparse_when_one_in, size_kind): + """Both forms against brute force, for every view and every way in, at + lengths 0 to 8. A set with a good permutation is kept as indices from length + 3 on with one in 2, and from length 5 on with one in 64, the default. The + bitmaps are read seven bytes at a time, so that counting them takes several + chunks.""" monkeypatch.setattr(bisc_module, "_SPARSE_WHEN_ONE_IN", sparse_when_one_in) monkeypatch.setattr(bisc_module, "_CHUNK_BYTES", 7) - randomness = random.Random(f"{sparse_when_one_in} {size_kind}") + seeded_random = random.Random(f"{sparse_when_one_in} {size_kind}") good_sets, bitmaps = {}, {} for length in range(MAX_LENGTH + 1): - good_count = good_count_of(size_kind, length, sparse_when_one_in, randomness) + good_count = good_count_of(size_kind, length, sparse_when_one_in, seeded_random) perms = list(Perm.of_length(length)) - good_sets[length] = set(randomness.sample(perms, good_count)) + good_sets[length] = set(seeded_random.sample(perms, good_count)) bitmaps[length] = expected_bitmap(length, good_sets[length]) - for way, property_bitmaps in each_way_in(good_sets, bitmaps, randomness): + for way, property_bitmaps in each_way_in(good_sets, bitmaps, seeded_random): for length in range(MAX_LENGTH + 1): good_count = len(good_sets[length]) kept_as_indices = good_count * sparse_when_one_in < factorial(length) @@ -137,9 +140,6 @@ def test_both_forms_read_as_brute_force(monkeypatch, sparse_when_one_in, size_ki ) -# add_length keeps the indices while they are few, and switches to a bitmap in -# the middle of a length once they would take as much space. Length 7 has 5040 -# permutations, so its 79th good one makes them as many as one in 64 @pytest.mark.parametrize( "good_indices", [ @@ -153,6 +153,10 @@ def test_both_forms_read_as_brute_force(monkeypatch, sparse_when_one_in, size_ki ], ) def test_add_length_switches_to_a_bitmap_when_the_good_become_many(good_indices): + """add_length keeps the indices while they are few, and switches to a bitmap + in the middle of a length once there are more than a length keeps as + indices. Length 7 has 5040 permutations, so its 79th good one is one too + many.""" length = 7 perms = list(Perm.of_length(length)) good_set = {perms[index] for index in good_indices} @@ -163,28 +167,26 @@ def test_add_length_switches_to_a_bitmap_when_the_good_become_many(good_indices) assert_views_match(property_bitmaps, length, good_set) -# A permutation listed many times counts once, for the form too def test_a_permutation_listed_many_times_is_kept_as_one_index(): - # Five copies of one permutation of length 5 look like 5 of its 120, which a - # bitmap would keep at one in 64, but they are one good permutation + """A permutation listed many times counts once, for the form too: five + copies of one permutation of length 5 look like 5 of its 120, which a bitmap + would keep at one in 64, but they are one good permutation.""" perm = Perm((1, 0, 2, 3, 4)) property_bitmaps = bisc_module._from_perms_by_length({5: [perm] * 5}) assert is_kept_as_indices(property_bitmaps, 5) assert_views_match(property_bitmaps, 5, {perm}) -def tuple_of(good_8, bad_8): +def tuple_of(good_of_length_8, bad_of_length_8): """A tuple of the good and the bad permutations: every permutation shorter than 8 good, and those given at length 8.""" good = {length: list(Perm.of_length(length)) for length in range(8)} bad = {length: [] for length in range(8)} - return {**good, 8: good_8}, {**bad, 8: bad_8} + return {**good, 8: good_of_length_8}, {**bad, 8: bad_of_length_8} -# The good and the bad permutations of a tuple are checked in every pair of -# forms: the good kept as indices or as a bitmap, and the bad as well @pytest.mark.parametrize( - "good_8, bad_8, in_both, in_neither", + "good_of_length_8, bad_of_length_8, in_both, in_neither", [ (slice(0, 5), slice(3, None), 2, 0), # indices, bitmap (slice(0, 40000), slice(39990, 40010), 10, 310), # bitmap, indices @@ -193,11 +195,13 @@ def tuple_of(good_8, bad_8): ], ) def test_a_tuple_is_checked_in_every_pair_of_forms( - monkeypatch, good_8, bad_8, in_both, in_neither + monkeypatch, good_of_length_8, bad_of_length_8, in_both, in_neither ): + """The good and the bad permutations of a tuple are checked in every pair of + forms: the good kept as indices or as a bitmap, and the bad as well.""" monkeypatch.setattr(bisc_module, "_CHUNK_BYTES", 7) perms = list(Perm.of_length(8)) - good, bad = tuple_of(perms[good_8], perms[bad_8]) + good, bad = tuple_of(perms[good_of_length_8], perms[bad_of_length_8]) with pytest.raises( ValueError, match=r"length 8 .* \(in both: {}, in neither: {}\)".format( @@ -207,9 +211,9 @@ def test_a_tuple_is_checked_in_every_pair_of_forms( bisc_module._from_good_and_bad(good, bad) -# Asking whether a length is there, or reading its good permutations, does not -# build the bitmap of a length kept as indices def test_a_length_kept_as_indices_builds_no_bitmap_unless_asked(monkeypatch): + """Asking whether a length is there, or reading its good permutations, does + not build the bitmap of a length kept as indices.""" identity = Perm(range(10)) property_bitmaps = bisc_module._from_perms_by_length({10: [identity]}) built = [] @@ -276,10 +280,10 @@ def shadings_as_sets(answer): } -# Three permutations of length 14 run in megabytes: each length up to 14 is kept -# as indices, where the bitmap of length 14 alone would take 10.9 GB def test_three_permutations_of_length_14_take_little_memory(): - # tracemalloc is CPython's + """Three permutations of length 14 run in megabytes: each length up to 14 is + kept as indices, where the bitmap of length 14 alone would take 10.9 GB. + tracemalloc is CPython's.""" tracemalloc = pytest.importorskip("tracemalloc") tracemalloc.start() try: @@ -642,3 +646,13 @@ def test_a_sparse_list_with_repeats_counts_each_permutation_once(): assert len(property_bitmaps.good[8]) == 2 assert len(property_bitmaps.bad[8]) == factorial(8) - 2 assert list(property_bitmaps.good[8]) == [first, second] + + +def test_a_sequence_that_is_not_a_permutation_is_refused(): + """A Perm whose values are not 0 to n - 1 has an index outside the length's, + and is refused with a ValueError, in a list kept as indices and in one kept + as a bitmap.""" + not_a_permutation = Perm(range(10, 18)) + for perms in ([not_a_permutation], [not_a_permutation] + LENGTH_8[:1000]): + with pytest.raises(ValueError, match="not a permutation of 0 to 7"): + bisc_module._from_perms_by_length({8: perms}) From b746f8950499435c54f66e2317812b6d14c12a64 Mon Sep 17 00:00:00 2001 From: Henning Ulfarsson Date: Sat, 24 Oct 2026 19:00:52 +0000 Subject: [PATCH 32/38] Iterate a view through one small generator for each form From the review of #209, round 6, its point on PyPy's speed. A view's __iter__ was one generator holding three loops: the good of a length kept as indices, the bad of one, and a bitmap's. Round 6 put the first two in front of the bitmap's loop, which was round 5's, and on PyPy auto_bisc("smooth"), the stored set, then often ran about a third slower than on round 5. Now each loop is a generator of its own (_unranked, _skipping and _generated), and __iter__ only chooses one. In six interleaved runs of each, with the Mac idle, auto_bisc("smooth") took 1.27 to 1.33 s on PyPy, where the head before took 1.39 to 2.19 s and b4b27e3 1.44 to 1.77 s; the property smooth, West-2 and stack-sortable took the same as before. Co-Authored-By: Claude Opus 5.5 --- permuta/bisc/bisc.py | 63 ++++++++++++++++++++++++++++---------------- 1 file changed, 41 insertions(+), 22 deletions(-) diff --git a/permuta/bisc/bisc.py b/permuta/bisc/bisc.py index a535bd0..53a45d8 100644 --- a/permuta/bisc/bisc.py +++ b/permuta/bisc/bisc.py @@ -688,38 +688,57 @@ def __contains__(self, perm: object) -> bool: def __iter__(self) -> Iterator[Perm]: labels = self.property_bitmaps._labels[self.length] - wanted_bit = self.wanted_bit wanted_count = len(self) if wanted_count == 0: # Nothing to yield, and no need to read the labels - return + return iter(()) if isinstance(labels, (array, list)): - if wanted_bit: - for index in labels: - yield Perm.unrank(index, self.length) - else: - # The good indices are in increasing order, as the - # permutations are generated - good_indices = iter(labels) - next_good_index = next(good_indices, None) - for index, perm in enumerate(Perm.of_length(self.length)): - if index == next_good_index: - next_good_index = next(good_indices, None) - else: - yield perm - elif wanted_count * _UNRANK_WHEN_ONE_IN <= factorial(self.length): - for index in _wanted_indices(labels, self.length, wanted_bit): - yield Perm.unrank(index, self.length) - else: - for index, perm in enumerate(Perm.of_length(self.length)): - if _label(labels, index) == wanted_bit: - yield perm + if self.wanted_bit: + return _unranked(labels, self.length) + return _skipping(labels, self.length) + if wanted_count * _UNRANK_WHEN_ONE_IN <= factorial(self.length): + return _unranked( + _wanted_indices(labels, self.length, self.wanted_bit), self.length + ) + return _generated(labels, self.length, self.wanted_bit) def __len__(self) -> int: good_count = self.property_bitmaps.good_counts[self.length] return good_count if self.wanted_bit else factorial(self.length) - good_count +# Each way of iterating over a length is a generator of its own, with one small +# loop, which PyPy's JIT compiles more steadily than one generator holding them +# all + + +def _unranked(indices: Iterable[int], length: int) -> Iterator[Perm]: + """The permutations of a length at the given indices, unranked.""" + for index in indices: + yield Perm.unrank(index, length) + + +def _skipping(good_indices: Iterable[int], length: int) -> Iterator[Perm]: + """The permutations of a length not at the given indices, which are in + increasing order: the length is generated, and the good indices skipped in + step.""" + good_index_iterator = iter(good_indices) + next_good_index = next(good_index_iterator, None) + for index, perm in enumerate(Perm.of_length(length)): + if index == next_good_index: + next_good_index = next(good_index_iterator, None) + else: + yield perm + + +def _generated(bitmap: bytes, length: int, wanted_bit: int) -> Iterator[Perm]: + """The permutations of a length whose label in the bitmap is wanted_bit: + the length is generated, and each bit tested.""" + for index, perm in enumerate(Perm.of_length(length)): + if _label(bitmap, index) == wanted_bit: + yield perm + + @functools.lru_cache(maxsize=None) def _shorter_count(length: int) -> int: """The number of permutations shorter than length: a permutation's bit is From 04e66c0464d2b4910790d8945390f2ae64c388f8 Mon Sep 17 00:00:00 2001 From: Henning Ulfarsson Date: Sat, 24 Oct 2026 21:44:21 +0000 Subject: [PATCH 33/38] Tell the two forms apart in the tests by each length's container From the review of #209, round 7, its blocking finding. Round 7 loosened is_kept_as_indices, the test helper that tells the two forms apart, to accept an array or a list at any length. A list kept where the array is meant, which takes five times the memory, then passed every test: two faults of the review's, add_length starting with a list and _labels_from_bitmap making one, passed all 141 tests of tests/bisc. The helper now asks for the container of each length: an array to length 20, and a list past it, as round 6's asked for the array. It is the review's, and all 53 tests of the file pass with it. Co-Authored-By: Claude Opus 5.5 --- tests/bisc/test_sparse_storage.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/bisc/test_sparse_storage.py b/tests/bisc/test_sparse_storage.py index 8d3d0bf..6e9b7dd 100644 --- a/tests/bisc/test_sparse_storage.py +++ b/tests/bisc/test_sparse_storage.py @@ -31,7 +31,11 @@ def expected_bitmap(length, good_set): def is_kept_as_indices(property_bitmaps, length): - return isinstance(property_bitmaps._labels[length], (array, list)) + """Whether a length's labels are its good permutations' indices, in the + container for its length: an array to length 20, and a list past it. A list + where the array is meant takes five times the memory.""" + kept_in = array if factorial(length) <= 2**64 else list + return type(property_bitmaps._labels[length]) is kept_in def assert_views_match(property_bitmaps, length, good_set, bitmap=None): From a3c966eccbea81646d0fc801010ae24d253fcabf Mon Sep 17 00:00:00 2001 From: Henning Ulfarsson Date: Sat, 24 Oct 2026 21:46:18 +0000 Subject: [PATCH 34/38] Refuse a listed sequence that is not a permutation by its values From the review of #209, round 7, its suggestion 1. The changelog, the description and a test said that a Perm whose values are not a permutation's raises a ValueError, but _indices_of checked only that its rank fell inside its length's. Most such sequences rank inside it, as another permutation: (0, 0, 0) and (0, 1, 1) as the identity, and (1, 2, 3), written from 1, as 201. At length 7, 4,166 of the 5,040 permutations written from 1 were taken so, silently. Develop mined such sequences as they were given, which made no sense of them either: for Av(231) written from 1, bisc(..., 3, 6) forbids all six patterns of length 3. _indices_of now compares each sequence's values with 0 to length - 1, as `in` does, and the range check, which this makes redundant, is gone. The check costs a set per permutation: about a tenth of rank()'s time on CPython 3.11 and about a third of it on PyPy 3.11, at lengths 8 and 10. The test of the ValueError covers sequences that rank as another permutation, and the review's that rank before the first or one past the last, through bisc and through _from_perms_by_length, in both forms. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 +- permuta/bisc/bisc.py | 17 ++++++------ tests/bisc/test_sparse_storage.py | 45 +++++++++++++++++++++++++------ 3 files changed, 46 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 91aebce..622ff00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,7 +28,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. together all of that length's permutations. - `bisc` and `auto_bisc` raise a `ValueError` for a permutation given under the key of another length, and for a `Perm` given whose values are not those of a - permutation of its length. + permutation of its length, such as one written from 1. - `bisc` accepts any mapping of lengths to permutations, not only a `dict`. ### Fixed diff --git a/permuta/bisc/bisc.py b/permuta/bisc/bisc.py index 53a45d8..f8f7e11 100644 --- a/permuta/bisc/bisc.py +++ b/permuta/bisc/bisc.py @@ -748,25 +748,24 @@ def _shorter_count(length: int) -> int: def _indices_of(length: int, perms: Iterable[Perm]) -> Iterator[int]: """The indices (bits) of permutations given as ones of a length. One of - another length raises a ValueError, and so does one whose index is not - one of the length's, which only a sequence that is not a permutation can - have.""" + another length raises a ValueError, and so does a sequence that is not a + permutation of 0 to length - 1, such as one written from 1: its rank() + would be another permutation's, or outside the length's.""" shorter_count = _shorter_count(length) - permutation_count = factorial(length) + permutation_values = set(range(length)) for perm in perms: if len(perm) != length: raise ValueError( "The permutation {!r} of length {} is given as one of length " "{}".format(perm, len(perm), length) ) - if not isinstance(perm, Perm): - perm = Perm(perm) - index = perm.rank() - shorter_count - if not 0 <= index < permutation_count: + if set(perm) != permutation_values: raise ValueError( "{!r} is not a permutation of 0 to {}".format(perm, length - 1) ) - yield index + if not isinstance(perm, Perm): + perm = Perm(perm) + yield perm.rank() - shorter_count def _labels_of(length: int, perms: Collection[Perm]) -> Tuple[_Labels, int]: diff --git a/tests/bisc/test_sparse_storage.py b/tests/bisc/test_sparse_storage.py index 6e9b7dd..c272b90 100644 --- a/tests/bisc/test_sparse_storage.py +++ b/tests/bisc/test_sparse_storage.py @@ -652,11 +652,40 @@ def test_a_sparse_list_with_repeats_counts_each_permutation_once(): assert list(property_bitmaps.good[8]) == [first, second] -def test_a_sequence_that_is_not_a_permutation_is_refused(): - """A Perm whose values are not 0 to n - 1 has an index outside the length's, - and is refused with a ValueError, in a list kept as indices and in one kept - as a bitmap.""" - not_a_permutation = Perm(range(10, 18)) - for perms in ([not_a_permutation], [not_a_permutation] + LENGTH_8[:1000]): - with pytest.raises(ValueError, match="not a permutation of 0 to 7"): - bisc_module._from_perms_by_length({8: perms}) +@pytest.mark.parametrize( + "not_a_permutation, ranked_as", + [ + (Perm((0, 0, 0)), Perm((0, 1, 2))), + (Perm((0, 1, 1)), Perm((0, 1, 2))), + (Perm((1, 2, 3)), Perm((2, 0, 1))), # written from 1 + (Perm(range(1, 9)), Perm((1, 2, 3, 4, 5, 7, 0, 6))), # written from 1 + (Perm((2, 0)), None), # one past the last of length 2 + (Perm((3, 0, 0)), None), # one past the last of length 3 + (Perm(range(-1, 2)), None), # before the first of length 3 + (Perm(range(-1, 5)), None), # before the first of length 6 + (Perm(range(10, 18)), None), # past the last of length 8 + ], + ids=lambda value: str(value), +) +def test_a_sequence_that_is_not_a_permutation_is_refused(not_a_permutation, ranked_as): + """A Perm whose values are not 0 to n - 1 is refused with a ValueError, in a + list kept as indices and in one kept as a bitmap, given to bisc or made + into a PropertyBitmaps. Its rank() is another permutation's (ranked_as), + or outside its length's (None): in the first case it would be mined as + that permutation, and in the second its bit would be another's or past the + bitmap.""" + length = len(not_a_permutation) + index = not_a_permutation.rank() - bisc_module._shorter_count(length) + if ranked_as is None: + assert index not in range(factorial(length)) + else: + assert Perm.unrank(index, length) == ranked_as + message = "is not a permutation of 0 to {}".format(length - 1) + for perms in ( + [not_a_permutation], + [not_a_permutation] + list(Perm.of_length(length)), + ): + with pytest.raises(ValueError, match=message): + bisc_module._from_perms_by_length({length: perms}) + with pytest.raises(ValueError, match=message): + bisc(perms, 1, length) From b14d825771027c5a4d88664f9c186d58afbcaca4 Mon Sep 17 00:00:00 2001 From: Henning Ulfarsson Date: Sat, 24 Oct 2026 21:48:44 +0000 Subject: [PATCH 35/38] Count a view's permutations without len(), so that past length 20 they iterate From the review of #209, round 7, its suggestion 4. A view's __iter__ began with len(self), which Python limits to sys.maxsize, so iter() and bool() of the bad view of a length past 20, whose count is past it, raised OverflowError. The list form made these lengths reachable. __iter__ and a new __bool__ read the count from _wanted_count, an int of any size, which __len__ returns too, so only len() itself raises there. The class docstring says what stops past length 20: a length's bitmap, which cannot be built, and len() of its bad permutations, which can still be iterated over and tested with in. It also says that the indices take less than the bitmap but for the few percent an array allocates ahead. The review's test of a length kept in a list, with bool(), the first bad permutations and len()'s OverflowError added, fails on round 7's code. Co-Authored-By: Claude Opus 5.5 --- permuta/bisc/bisc.py | 22 ++++++++++++++++----- tests/bisc/test_sparse_storage.py | 33 +++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/permuta/bisc/bisc.py b/permuta/bisc/bisc.py index f8f7e11..138d69d 100644 --- a/permuta/bisc/bisc.py +++ b/permuta/bisc/bisc.py @@ -492,10 +492,13 @@ class PropertyBitmaps: permutations (lexicographic). The bitmap of length n is (n! + 7) // 8 bytes. A length with fewer than one good permutation in 64 keeps instead the sorted indices of its good permutations (their bits), 8 bytes each, - which then take less space. So no length takes more than its bitmap: - three permutations of length 14 take 24 bytes, where its bitmap takes - 10.9 GB. Past length 20, whose indices can exceed 8 bytes, they are kept - in a list, and there is no longest length. + which then take less space, but for the few percent an array allocates + ahead: three permutations of length 14 take 24 bytes, where its bitmap + takes 10.9 GB. Past length 20, whose indices can exceed 8 bytes, they are + kept in a list, and there is no longest length. There the bitmap of a + length cannot be built, and its bad permutations, more than sys.maxsize, + have no len(), though they can be iterated over and tested with in: their + number is the length's factorial less its good_counts. The labels can be given as bitmaps (bitmaps={length: bytes}), made from a property (add_length), or both: auto_bisc and bisc label the lengths they @@ -688,7 +691,7 @@ def __contains__(self, perm: object) -> bool: def __iter__(self) -> Iterator[Perm]: labels = self.property_bitmaps._labels[self.length] - wanted_count = len(self) + wanted_count = self._wanted_count() if wanted_count == 0: # Nothing to yield, and no need to read the labels return iter(()) @@ -703,6 +706,15 @@ def __iter__(self) -> Iterator[Perm]: return _generated(labels, self.length, self.wanted_bit) def __len__(self) -> int: + # len() raises OverflowError past sys.maxsize, as for the bad + # permutations of a length past 20 + return self._wanted_count() + + def __bool__(self) -> bool: + return self._wanted_count() > 0 + + def _wanted_count(self) -> int: + """The number of permutations with the label wanted, of any size.""" good_count = self.property_bitmaps.good_counts[self.length] return good_count if self.wanted_bit else factorial(self.length) - good_count diff --git a/tests/bisc/test_sparse_storage.py b/tests/bisc/test_sparse_storage.py index c272b90..78a2d91 100644 --- a/tests/bisc/test_sparse_storage.py +++ b/tests/bisc/test_sparse_storage.py @@ -4,6 +4,7 @@ permutations, and every view reads both forms.""" import importlib +import itertools import random from array import array from math import factorial @@ -313,6 +314,38 @@ def test_the_last_permutation_of_a_long_length_can_be_listed(length): assert isinstance(property_bitmaps._labels[length], kept_in) +@pytest.mark.parametrize("length", [21, 25]) +def test_a_length_kept_in_a_list_counts_contains_and_iterates(length): + """The first, a middle and the last permutation of a length past 20, good or + not: len() and bool() of the good view, in and not in for both views, and + the good permutations in order. The bad view is true, and iterates from the + first bad permutation; its count is past sys.maxsize, so len() of it + raises OverflowError, and __len__() gives it.""" + first = Perm(range(length)) + last = Perm(range(length - 1, -1, -1)) + middle = Perm.unrank(factorial(length) // 3, length) + other = Perm.unrank(factorial(length) // 2, length) + property_bitmaps = bisc_module._from_perms_by_length( + {length: [last, middle, first]} + ) + good, bad = property_bitmaps.good[length], property_bitmaps.bad[length] + assert is_kept_as_indices(property_bitmaps, length) + assert len(good) == 3 and bool(good) + assert bad.__len__() == factorial(length) - 3 and bool(bad) + with pytest.raises(OverflowError): + len(bad) + assert list(good) == [first, middle, last] + assert list(itertools.islice(bad, 2)) == [ + Perm.unrank(1, length), + Perm.unrank(2, length), + ] + for perm in (first, middle, last): + assert perm in good and perm not in bad + assert other not in good and other in bad + assert Perm(range(length - 1)) not in good + assert Perm(range(length - 1)) not in bad + + def test_a_permutation_of_length_21_whose_index_needs_the_top_bit(): """An index from 2**63 to 2**64, which an unsigned 64-bit integer holds and a signed one does not.""" From b64af6327cdb57215d8d87bdfa4759886f0a63b0 Mon Sep 17 00:00:00 2001 From: Henning Ulfarsson Date: Sat, 24 Oct 2026 21:50:31 +0000 Subject: [PATCH 36/38] Test the gaps the review of round 7 found in the two forms' tests From the review of #209, round 7, its suggestions 2, 3 and 5, with the review's tests: - a permutation of length 100 or 200, whose n! / 64 a float cannot hold past length 171, kept as indices; - the bitmap of a length past 20, which cannot be built, and the overlap of labels kept as lists, which only such a length has; - add_length dropping its indices once it has the bitmap (CPython only), and a property true only on the first or the last permutation of a length, at lengths 0 to 8; - a view with nothing to yield, which reads no labels; - bisc keeping the lengths past n as empty arrays to length 20 and empty lists past it, with m = 25; an empty length 25, and bisc([], 3, 25); - the bad view of length 20, the longest kept in an array, read lazily. The test of three permutations of length 14 under tracemalloc now fails at once on a fault that would size a bitmap past length 10, rather than allocating 10.9 GB, through guard_bitmap_sizes, which the two tests that guarded the sizes each in their own words now share. The docstring of the test with an index above 2**63 says that it is kept in a list. Co-Authored-By: Claude Opus 5.5 --- tests/bisc/test_sparse_storage.py | 202 +++++++++++++++++++++++++++--- 1 file changed, 183 insertions(+), 19 deletions(-) diff --git a/tests/bisc/test_sparse_storage.py b/tests/bisc/test_sparse_storage.py index 78a2d91..f83210d 100644 --- a/tests/bisc/test_sparse_storage.py +++ b/tests/bisc/test_sparse_storage.py @@ -6,6 +6,8 @@ import importlib import itertools import random +import sys +import weakref from array import array from math import factorial @@ -61,6 +63,19 @@ def assert_views_match(property_bitmaps, length, good_set, bitmap=None): assert property_bitmaps.bitmaps[length] == bitmap +def guard_bitmap_sizes(monkeypatch): + """Make sizing the bitmap of a length past 10 fail the test at once, rather + than allocate it: length 14's would take 10.9 GB. A check of the code's + structure, so that it runs on PyPy too, which has no tracemalloc.""" + real_bitmap_size = bisc_module._bitmap_size + + def guarded_bitmap_size(length): + assert length <= 10, "a bitmap of length {} was sized".format(length) + return real_bitmap_size(length) + + monkeypatch.setattr(bisc_module, "_bitmap_size", guarded_bitmap_size) + + def each_way_in(good_sets, bitmaps, seeded_random): """A PropertyBitmaps made from good_sets in each way that bisc and auto_bisc make one, with its name: a list or a mapping (whose good permutations come @@ -172,6 +187,57 @@ def test_add_length_switches_to_a_bitmap_when_the_good_become_many(good_indices) assert_views_match(property_bitmaps, length, good_set) +@pytest.mark.skipif( + sys.implementation.name != "cpython", reason="frees at once by reference count" +) +def test_add_length_drops_its_indices_once_it_has_the_bitmap(monkeypatch): + """After add_length switches to a bitmap its indices are not kept, so that it + holds a bitmap and not a bitmap and the indices; and the bitmap is filled in + with the rest of the length. The indices are an array, which can be weakly + referenced.""" + indices_given = [] + real_bitmap_from_indices = bisc_module._bitmap_from_indices + + def recording_bitmap_from_indices(length, indices): + indices_given.append(weakref.ref(indices)) + return real_bitmap_from_indices(length, indices) + + monkeypatch.setattr( + bisc_module, "_bitmap_from_indices", recording_bitmap_from_indices + ) + still_held_while_labeling = [] + + def prop(perm): + if indices_given: + still_held_while_labeling.append(indices_given[0]() is not None) + return True + + property_bitmaps = PropertyBitmaps(prop) + property_bitmaps.add_length(7) + assert len(indices_given) == 1 + # All 5040 good: the switch at the 79th, and the rest labeled in the bitmap + assert still_held_while_labeling == [False] * (5040 - 79) + + +@pytest.mark.parametrize("which", ["first", "last", "first and last"]) +def test_add_length_when_only_the_first_or_the_last_permutations_are_good(which): + """A property true only on the first permutation of a length, only on the + last, or on both, at lengths 0 to 8: the last permutation is where add_length + switches to a bitmap, or ends the loop, at the very end of a length.""" + for length in range(9): + perms = list(Perm.of_length(length)) + good_set = { + "first": {perms[0]}, + "last": {perms[-1]}, + "first and last": {perms[0], perms[-1]}, + }[which] + property_bitmaps = PropertyBitmaps(lambda perm: perm in good_set) + property_bitmaps.add_length(length) + kept_as_indices = len(good_set) * 64 < factorial(length) + assert is_kept_as_indices(property_bitmaps, length) == kept_as_indices, length + assert_views_match(property_bitmaps, length, good_set) + + def test_a_permutation_listed_many_times_is_kept_as_one_index(): """A permutation listed many times counts once, for the form too: five copies of one permutation of length 5 look like 5 of its 120, which a bitmap @@ -216,6 +282,16 @@ def test_a_tuple_is_checked_in_every_pair_of_forms( bisc_module._from_good_and_bad(good, bad) +def test_the_overlap_of_two_lists_of_indices(): + """The overlap of labels kept as lists, which only a length past 20 has, and + that of a list and a bitmap, either way round.""" + assert bisc_module._overlap_count([1, 5, 2**70], [5, 2**70, 2**71]) == 2 + assert bisc_module._overlap_count([], [5, 2**70]) == 0 + bitmap = bytes([0b00100010]) # the bits 1 and 5 + assert bisc_module._overlap_count([1, 5, 6], bitmap) == 2 + assert bisc_module._overlap_count(bitmap, [1, 5, 6]) == 2 + + def test_a_length_kept_as_indices_builds_no_bitmap_unless_asked(monkeypatch): """Asking whether a length is there, or reading its good permutations, does not build the bitmap of a length kept as indices.""" @@ -285,11 +361,13 @@ def shadings_as_sets(answer): } -def test_three_permutations_of_length_14_take_little_memory(): +def test_three_permutations_of_length_14_take_little_memory(monkeypatch): """Three permutations of length 14 run in megabytes: each length up to 14 is kept as indices, where the bitmap of length 14 alone would take 10.9 GB. - tracemalloc is CPython's.""" + tracemalloc is CPython's. A fault that would size a bitmap past length 10 + fails at once, rather than allocating it.""" tracemalloc = pytest.importorskip("tracemalloc") + guard_bitmap_sizes(monkeypatch) tracemalloc.start() try: answer = bisc(THREE_OF_LENGTH_14, 2, 14) @@ -346,9 +424,21 @@ def test_a_length_kept_in_a_list_counts_contains_and_iterates(length): assert Perm(range(length - 1)) not in bad +@pytest.mark.parametrize("length", [21, 25]) +def test_the_bitmap_of_a_length_kept_in_a_list_cannot_be_built(length): + """Past length 20 a bitmap would take more than 6 * 10**18 bytes: asking for + it raises, and does not give the list of indices. Asking whether the length + is there builds nothing.""" + last = Perm(range(length - 1, -1, -1)) + property_bitmaps = bisc_module._from_perms_by_length({length: [last]}) + assert length in property_bitmaps.bitmaps + with pytest.raises((MemoryError, OverflowError)): + property_bitmaps.bitmaps[length] + + def test_a_permutation_of_length_21_whose_index_needs_the_top_bit(): - """An index from 2**63 to 2**64, which an unsigned 64-bit integer holds and - a signed one does not.""" + """An index of 2**63 or more, which a signed 64-bit integer would not hold, + kept in the list of a length past 20.""" perm = Perm.unrank(2**63 + 5, 21) assert perm.rank() - bisc_module._shorter_count(21) == 2**63 + 5 property_bitmaps = bisc_module._from_perms_by_length({21: [perm]}) @@ -356,6 +446,50 @@ def test_a_permutation_of_length_21_whose_index_needs_the_top_bit(): assert perm in property_bitmaps.good[21] +@pytest.mark.parametrize("length", [100, 200]) +def test_a_permutation_of_a_very_long_length_can_be_listed(length): + """The last permutation of length 100 or 200: n! / 64 is past what a float + holds from length 172 on, so the limit on the number of indices must be an + int.""" + last = Perm(range(length - 1, -1, -1)) + property_bitmaps = bisc_module._from_perms_by_length({length: [last]}) + assert is_kept_as_indices(property_bitmaps, length) + assert list(property_bitmaps.good[length]) == [last] + assert last in property_bitmaps.good[length] + assert last not in property_bitmaps.bad[length] + assert property_bitmaps.bad[length].__len__() == factorial(length) - 1 + + +def test_an_empty_list_past_length_20_is_read_as_empty(): + """No permutation of length 25 listed: the length is kept as no indices, and + both views, in, and bisc read it as empty. n = 25 reads the length too.""" + property_bitmaps = bisc_module._from_perms_by_length({25: []}) + assert property_bitmaps.good_counts[25] == 0 + good = property_bitmaps.good[25] + assert list(good) == [] and len(good) == 0 and not good + assert Perm(range(25)) not in good + assert Perm(range(25)) in property_bitmaps.bad[25] + assert bisc([], 3, 25) == {0: {Perm(()): [set()]}, 1: {}, 2: {}, 3: {}} + + +def test_the_bad_view_of_length_20_is_read_lazily(): + """Length 20 is the longest kept in an array, and its bad view can be read + only a few permutations at a time: with the first and the third permutation + good, the first three bad ones are the second, fourth and fifth.""" + first_five = list(itertools.islice(Perm.of_length(20), 5)) + property_bitmaps = bisc_module._from_perms_by_length( + {20: [first_five[0], first_five[2]]} + ) + assert is_kept_as_indices(property_bitmaps, 20) + bad = property_bitmaps.bad[20] + assert list(itertools.islice(bad, 3)) == [ + first_five[1], + first_five[3], + first_five[4], + ] + assert len(bad) == factorial(20) - 2 + + # bisc([the decreasing permutation of length 21], 2, 21) on develop, before # PropertyBitmaps, and the same at length 25 DEVELOP_ANSWER_LONG = { @@ -603,6 +737,22 @@ def test_a_sparse_view_iterates_independently(): assert list(zip(view, view)) == [(perm, perm) for perm in once] +def test_a_view_with_nothing_to_yield_reads_no_labels(monkeypatch): + """The bad view of a length with every permutation good yields nothing + without reading the bitmap.""" + all_good = PropertyBitmaps(bitmaps={6: b"\xff" * 90}) + reads = [] + real_wanted_indices = bisc_module._wanted_indices + + def recording_wanted_indices(bitmap, length, wanted_bit): + reads.append(length) + return real_wanted_indices(bitmap, length, wanted_bit) + + monkeypatch.setattr(bisc_module, "_wanted_indices", recording_wanted_indices) + assert list(all_good.bad[6]) == [] + assert reads == [] + + def test_bisc_takes_the_view_of_a_sparse_length(): """bisc given the good view of a PropertyBitmaps, a Mapping, mines what it mines from the list it was made from.""" @@ -616,15 +766,8 @@ def test_bisc_takes_the_view_of_a_sparse_length(): def test_a_sparse_length_14_never_sizes_a_bitmap(monkeypatch): """No bitmap is sized for a length kept as indices: length 14's would take - 10.9 GB. A check of the structure, so that it runs on PyPy too, which has no - tracemalloc.""" - real_bitmap_size = bisc_module._bitmap_size - - def guarded_bitmap_size(length): - assert length <= 10, "a bitmap of length {} was sized".format(length) - return real_bitmap_size(length) - - monkeypatch.setattr(bisc_module, "_bitmap_size", guarded_bitmap_size) + 10.9 GB.""" + guard_bitmap_sizes(monkeypatch) by_length = {length: [] for length in range(15)} by_length[14] = THREE_OF_LENGTH_14 by_length[13] = [Perm(range(13))] @@ -650,19 +793,14 @@ class MineStopped(Exception): def test_lengths_past_n_are_kept_as_empty_indices_and_size_no_bitmap(monkeypatch): """With m > n, a length past n is kept as no indices, not as a bitmap: m = 14 would size one of 10.9 GB. The lengths up to n are labeled.""" - real_bitmap_size = bisc_module._bitmap_size kept = {} - def guarded_bitmap_size(length): - assert length <= 10, "a bitmap of length {} was sized".format(length) - return real_bitmap_size(length) - def stopped_mine(goodperms, M, N=None, report=False): property_bitmaps = goodperms.property_bitmaps kept.update(property_bitmaps._labels) raise MineStopped - monkeypatch.setattr(bisc_module, "_bitmap_size", guarded_bitmap_size) + guard_bitmap_sizes(monkeypatch) monkeypatch.setattr(bisc_module, "mine", stopped_mine) with pytest.raises(MineStopped): bisc(lambda perm: perm.is_increasing(), 14, 2) @@ -671,6 +809,32 @@ def stopped_mine(goodperms, M, N=None, report=False): assert isinstance(kept[length], array) and len(kept[length]) == 0 +def test_lengths_past_20_are_kept_as_empty_lists_when_m_is_past_n(monkeypatch): + """With m = 25 and n = 2, the lengths 3 to 20 past n are kept as no indices in + an array, and the lengths 21 to 25 as no indices in a list; none sizes a + bitmap, which 25! bits would not fit in memory; and the views read them as + empty.""" + kept = {} + views = {} + + def stopped_mine(goodperms, M, N=None, report=False): + kept.update(goodperms.property_bitmaps._labels) + views.update({length: goodperms[length] for length in goodperms}) + raise MineStopped + + guard_bitmap_sizes(monkeypatch) + monkeypatch.setattr(bisc_module, "mine", stopped_mine) + with pytest.raises(MineStopped): + bisc(lambda perm: perm.is_increasing(), 25, 2) + assert sorted(kept) == list(range(26)) + for length in range(3, 21): + assert isinstance(kept[length], array) and len(kept[length]) == 0 + for length in range(21, 26): + assert kept[length] == [] + assert len(views[length]) == 0 and list(views[length]) == [] + assert Perm(range(length)) not in views[length] + + def test_a_sparse_list_with_repeats_counts_each_permutation_once(): """A list that looks sparse before it is counted, with a permutation repeated, counts it once.""" From 7da301ce83f04f3153bee074de314672d153bd32 Mon Sep 17 00:00:00 2001 From: Henning Ulfarsson Date: Sat, 24 Oct 2026 21:54:06 +0000 Subject: [PATCH 37/38] Readability points from the review of round 7 From the review of #209, round 7, its suggestion 6. - One helper, _kept_as_bitmap, tells the two forms apart for every reader of the labels, where five places tested isinstance(labels, (array, list)), the kind of test round 6 blocked on. It is a TypeGuard for bytes: mypy narrows only where a TypeGuard holds, and the indices' branches read the labels as any sequence of ints, so no cast is needed. - add_length names its iterator indexed_perms, and says that the second loop goes on where the first stopped, that its else is the case with no break, and why it drops the indices: they take about as much as the bitmap, so dropping them keeps the peak at two bitmaps. - _most_indices is _largest_sparse_good_count, for what it holds, and its docstring says why it is (n! - 1) // 64 and not n! // 64, which would let exactly one in 64 through when 64 divides n!. - _keep's docstring, and the comment on _labels, say who reads the labels and who keeps them. - _Lengths, which gives two Mappings their keys, is _LengthKeys, and the view's generators are _unranked_perms, _perms_skipping and _perms_with_label. Their comment says what was measured, not why. - The comment on _INDEX_TYPECODE says why it is Final, and the changelog that 8 bytes an index holds to length 20. - In the tests, LENGTH_6, SPARSE_6, DENSE_6 and LENGTH_8 are named for what they hold, SPARSE_8 is extremes(8), and a docstring says "one good permutation". Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 +- permuta/bisc/bisc.py | 114 ++++++++++++++++++------------ tests/bisc/test_sparse_storage.py | 66 +++++++++-------- 3 files changed, 106 insertions(+), 76 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 622ff00..45bec2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. the same results in far less memory at longer lengths: the permutations of length 10 take 454 KB as a bitmap, and 0.9 to 1.8 GB as lists of `Perm` objects, depending on the Python, and a length with few good permutations takes 8 bytes - for each of them. + for each of them, to length 20. - A permutation listed twice in the input of `bisc` or `auto_bisc` counts once. Before, duplicates could make `bisc` take a length as complete when it was not. - `auto_bisc` raises a `ValueError` when the good and the bad permutations of a diff --git a/permuta/bisc/bisc.py b/permuta/bisc/bisc.py index 138d69d..817069a 100644 --- a/permuta/bisc/bisc.py +++ b/permuta/bisc/bisc.py @@ -16,6 +16,7 @@ Optional, Set, Tuple, + TypeGuard, Union, ) @@ -396,7 +397,8 @@ def _wanted_indices( # The sorted indices of a length's good permutations are kept in an array of # this type code, unsigned integers of 8 bytes, while every index of the length # fits in one: to length 20, since 21! is above 2**64. Past that they are kept in -# a list +# a list. Final, so that mypy takes it as the literal "Q", and the array as one +# of ints _INDEX_TYPECODE: Final = "Q" _INDEX_BITS = 8 * array(_INDEX_TYPECODE).itemsize _Indices = Union["array[int]", List[int]] @@ -420,25 +422,36 @@ def _new_indices(length: int, indices: Iterable[int] = ()) -> _Indices: return list(indices) -def _most_indices(length: int) -> int: - """The most good permutations of a length that are kept as indices: fewer - than one in _SPARSE_WHEN_ONE_IN of its permutations.""" +def _largest_sparse_good_count(length: int) -> int: + """The most good permutations a length can have and keep their indices: + fewer than one in _SPARSE_WHEN_ONE_IN of its n! permutations, so + good_count * _SPARSE_WHEN_ONE_IN < n!, which for ints is good_count <= (n! + - 1) // _SPARSE_WHEN_ONE_IN. n! // _SPARSE_WHEN_ONE_IN would also let + through exactly one in _SPARSE_WHEN_ONE_IN when it divides n!, as 64 + divides n! from n = 8 on.""" return (factorial(length) - 1) // _SPARSE_WHEN_ONE_IN def _is_sparse(good_count: int, length: int) -> bool: """Whether a length with good_count good permutations keeps their indices rather than its bitmap.""" - return good_count <= _most_indices(length) + return good_count <= _largest_sparse_good_count(length) + + +def _kept_as_bitmap(labels: _Labels) -> TypeGuard[bytes]: + """Whether a length's labels are kept as its bitmap, rather than as the + indices of its good permutations, in an array or a list: every reader of + the labels tells the forms apart here. The bitmap is always bytes.""" + return isinstance(labels, bytes) def _label_in(labels: _Labels, index: int) -> int: """The label (0 or 1) of the permutation at index, from a length's labels in either form.""" - if isinstance(labels, (array, list)): - position = bisect.bisect_left(labels, index) - return int(position < len(labels) and labels[position] == index) - return _label(labels, index) + if _kept_as_bitmap(labels): + return _label(labels, index) + position = bisect.bisect_left(labels, index) + return int(position < len(labels) and labels[position] == index) def _bitmap_from_indices(length: int, indices: Iterable[int]) -> bytearray: @@ -531,9 +544,10 @@ def __init__( ) -> None: self.prop = prop # length -> its labels, in the form chosen by its number of good - # permutations (_is_sparse). The storage is the module's: bitmaps, the - # views and the functions that make a PropertyBitmaps read _labels, - # and keep a length's labels through _keep + # permutations (_is_sparse). The storage is the module's: bitmaps and + # the views read _labels, and the constructor, add_length, bisc and the + # functions that make a PropertyBitmaps keep a length's labels in it + # through _keep self._labels: Dict[int, _Labels] = {} self.good_counts: Dict[int, int] = {} if bitmaps is not None: @@ -570,9 +584,9 @@ def __init__( self.bad = _LengthViews(self, wanted_bit=0) def _keep(self, length: int, labels: _Labels, good_count: int) -> None: - """Keep a length's labels as given, with its number of good - permutations. The caller chooses the form (_is_sparse), and is trusted - to; the views read the labels from here.""" + """Keep a length's labels, in the form its caller chose by _is_sparse, + with its number of good permutations. Neither is checked here: the + callers make them right.""" self._labels[length] = labels self.good_counts[length] = good_count @@ -587,33 +601,38 @@ def add_length(self, length: int) -> None: "The PropertyBitmaps has no property to label length {} " "with".format(length) ) - most_indices = _most_indices(length) + largest_sparse_good_count = _largest_sparse_good_count(length) good_indices = _new_indices(length) - perms = enumerate(Perm.of_length(length)) - for index, perm in perms: + # One iterator for both loops: the second goes on where the first + # stopped + indexed_perms = enumerate(Perm.of_length(length)) + for index, perm in indexed_perms: if self.prop(perm): good_indices.append(index) - if len(good_indices) > most_indices: + if len(good_indices) > largest_sparse_good_count: break else: + # No break: the good permutations stayed few self._keep(length, good_indices, len(good_indices)) return - # Too many for indices: they are set in the bitmap, and the loop goes - # on in it from the next permutation, with a bit to set for each good - # one and nothing more + # Too many for indices: they are set in the bitmap, and the second loop + # goes on in it from the next permutation, with a bit to set for each + # good one and nothing more bitmap = _bitmap_from_indices(length, good_indices) good_count = len(good_indices) + # The indices now take about as much as the bitmap. Dropping them keeps + # the peak at two bitmaps, with the bytes made of it at the end del good_indices - for index, perm in perms: + for index, perm in indexed_perms: if self.prop(perm): _set_label(bitmap, index) good_count += 1 self._keep(length, bytes(bitmap), good_count) -class _Lengths: - """The lengths with labels in a PropertyBitmaps, which its Mappings, - bitmaps and the views, iterate over and count.""" +class _LengthKeys: + """The keys of a PropertyBitmaps' Mappings, bitmaps and the views: the + lengths it has labels for.""" def __init__(self, property_bitmaps: PropertyBitmaps) -> None: self.property_bitmaps = property_bitmaps @@ -625,16 +644,16 @@ def __len__(self) -> int: return len(self.property_bitmaps._labels) -class _Bitmaps(_Lengths, Mapping[int, bytes]): +class _Bitmaps(_LengthKeys, Mapping[int, bytes]): """length -> its bitmap, read-only: the bitmap kept, or, for a length kept as the indices of its good permutations, one built from them when asked.""" def __getitem__(self, length: int) -> bytes: labels = self.property_bitmaps._labels[length] - if isinstance(labels, (array, list)): - return bytes(_bitmap_from_indices(length, labels)) - return labels + if _kept_as_bitmap(labels): + return labels + return bytes(_bitmap_from_indices(length, labels)) def __contains__(self, length: object) -> bool: # Mapping's would build the bitmap of a length kept as indices, to @@ -642,7 +661,7 @@ def __contains__(self, length: object) -> bool: return length in self.property_bitmaps._labels -class _LengthViews(_Lengths, Mapping[int, "_PermsOfLength"]): +class _LengthViews(_LengthKeys, Mapping[int, "_PermsOfLength"]): """length -> the good (or bad) permutations of that length, read-only: a Mapping whose keys are the lengths with labels, through which bisc and auto_bisc read the permutations. wanted_bit is the label of the @@ -695,15 +714,16 @@ def __iter__(self) -> Iterator[Perm]: if wanted_count == 0: # Nothing to yield, and no need to read the labels return iter(()) - if isinstance(labels, (array, list)): - if self.wanted_bit: - return _unranked(labels, self.length) - return _skipping(labels, self.length) - if wanted_count * _UNRANK_WHEN_ONE_IN <= factorial(self.length): - return _unranked( - _wanted_indices(labels, self.length, self.wanted_bit), self.length - ) - return _generated(labels, self.length, self.wanted_bit) + if _kept_as_bitmap(labels): + if wanted_count * _UNRANK_WHEN_ONE_IN <= factorial(self.length): + return _unranked_perms( + _wanted_indices(labels, self.length, self.wanted_bit), + self.length, + ) + return _perms_with_label(labels, self.length, self.wanted_bit) + if self.wanted_bit: + return _unranked_perms(labels, self.length) + return _perms_skipping(labels, self.length) def __len__(self) -> int: # len() raises OverflowError past sys.maxsize, as for the bad @@ -720,17 +740,17 @@ def _wanted_count(self) -> int: # Each way of iterating over a length is a generator of its own, with one small -# loop, which PyPy's JIT compiles more steadily than one generator holding them -# all +# loop. With the three loops in one generator, PyPy 3.11 often took about a third +# longer over auto_bisc("smooth") -def _unranked(indices: Iterable[int], length: int) -> Iterator[Perm]: +def _unranked_perms(indices: Iterable[int], length: int) -> Iterator[Perm]: """The permutations of a length at the given indices, unranked.""" for index in indices: yield Perm.unrank(index, length) -def _skipping(good_indices: Iterable[int], length: int) -> Iterator[Perm]: +def _perms_skipping(good_indices: Iterable[int], length: int) -> Iterator[Perm]: """The permutations of a length not at the given indices, which are in increasing order: the length is generated, and the good indices skipped in step.""" @@ -743,7 +763,7 @@ def _skipping(good_indices: Iterable[int], length: int) -> Iterator[Perm]: yield perm -def _generated(bitmap: bytes, length: int, wanted_bit: int) -> Iterator[Perm]: +def _perms_with_label(bitmap: bytes, length: int, wanted_bit: int) -> Iterator[Perm]: """The permutations of a length whose label in the bitmap is wanted_bit: the length is generated, and each bit tested.""" for index, perm in enumerate(Perm.of_length(length)): @@ -795,9 +815,9 @@ def _labels_of(length: int, perms: Collection[Perm]) -> Tuple[_Labels, int]: def _overlap_count(labels: _Labels, other_labels: _Labels) -> int: """The number of permutations good in both of two labels of one length.""" - if isinstance(labels, (array, list)): + if not _kept_as_bitmap(labels): return sum(_label_in(other_labels, index) for index in labels) - if isinstance(other_labels, (array, list)): + if not _kept_as_bitmap(other_labels): return sum(_label_in(labels, index) for index in other_labels) return sum( (chunk & other_chunk).bit_count() diff --git a/tests/bisc/test_sparse_storage.py b/tests/bisc/test_sparse_storage.py index f83210d..219a576 100644 --- a/tests/bisc/test_sparse_storage.py +++ b/tests/bisc/test_sparse_storage.py @@ -134,10 +134,10 @@ def good_count_of(size_kind, length, sparse_when_one_in, seeded_random): @pytest.mark.parametrize("size_kind", SIZE_KINDS) def test_both_forms_read_as_brute_force(monkeypatch, sparse_when_one_in, size_kind): """Both forms against brute force, for every view and every way in, at - lengths 0 to 8. A set with a good permutation is kept as indices from length - 3 on with one in 2, and from length 5 on with one in 64, the default. The - bitmaps are read seven bytes at a time, so that counting them takes several - chunks.""" + lengths 0 to 8. A set with one good permutation is kept as indices from + length 3 on with one in 2, and from length 5 on with one in 64, the + default. The bitmaps are read seven bytes at a time, so that counting them + takes several chunks.""" monkeypatch.setattr(bisc_module, "_SPARSE_WHEN_ONE_IN", sparse_when_one_in) monkeypatch.setattr(bisc_module, "_CHUNK_BYTES", 7) seeded_random = random.Random(f"{sparse_when_one_in} {size_kind}") @@ -527,12 +527,17 @@ def test_a_dense_length_is_kept_as_bytes_and_read_without_a_copy(): assert property_bitmaps.bitmaps[length] == expected_bitmap(length, good_set) -LENGTH_6 = list(Perm.of_length(6)) # 720 permutations: kept as indices below 12 good -SPARSE_6 = {LENGTH_6[index] for index in (0, 5, 100, 719)} -DENSE_6 = {perm for index, perm in enumerate(LENGTH_6) if index % 3 == 0} +# 720 permutations: a set of them is kept as indices below 12 good +PERMS_OF_LENGTH_6 = list(Perm.of_length(6)) +FOUR_OF_LENGTH_6 = {PERMS_OF_LENGTH_6[index] for index in (0, 5, 100, 719)} +EVERY_THIRD_OF_LENGTH_6 = { + perm for index, perm in enumerate(PERMS_OF_LENGTH_6) if index % 3 == 0 +} -@pytest.mark.parametrize("good_set", [SPARSE_6, DENSE_6], ids=["sparse", "dense"]) +@pytest.mark.parametrize( + "good_set", [FOUR_OF_LENGTH_6, EVERY_THIRD_OF_LENGTH_6], ids=["sparse", "dense"] +) @pytest.mark.parametrize("wrap", [bytearray, lambda raw: memoryview(bytearray(raw))]) def test_a_buffer_given_and_changed_afterwards_changes_nothing(good_set, wrap): """A bitmap given is copied, whatever its form: changing the caller's @@ -540,14 +545,16 @@ def test_a_buffer_given_and_changed_afterwards_changes_nothing(good_set, wrap): buffer = wrap(expected_bitmap(6, good_set)) underlying = buffer.obj if isinstance(buffer, memoryview) else buffer property_bitmaps = PropertyBitmaps(bitmaps={6: buffer}) - assert is_kept_as_indices(property_bitmaps, 6) == (good_set is SPARSE_6) + assert is_kept_as_indices(property_bitmaps, 6) == (good_set is FOUR_OF_LENGTH_6) underlying[:] = b"\xff" * len(underlying) # every permutation good assert_views_match(property_bitmaps, 6, good_set) underlying[:] = bytes(len(underlying)) # none good assert_views_match(property_bitmaps, 6, good_set) -@pytest.mark.parametrize("good_set", [SPARSE_6, DENSE_6], ids=["sparse", "dense"]) +@pytest.mark.parametrize( + "good_set", [FOUR_OF_LENGTH_6, EVERY_THIRD_OF_LENGTH_6], ids=["sparse", "dense"] +) def test_a_memoryview_of_another_format_is_read_as_bytes(good_set): """A memoryview of another format, shape or stride is read as its bytes.""" raw = expected_bitmap(6, good_set) # 90 bytes @@ -559,19 +566,23 @@ def test_a_memoryview_of_another_format_is_read_as_bytes(good_set): } for name, view in views.items(): property_bitmaps = PropertyBitmaps(bitmaps={6: view}) - assert is_kept_as_indices(property_bitmaps, 6) == (good_set is SPARSE_6), name + assert is_kept_as_indices(property_bitmaps, 6) == ( + good_set is FOUR_OF_LENGTH_6 + ), name assert_views_match(property_bitmaps, 6, good_set) def test_bitmaps_give_bytes_that_are_equal_on_every_read(): """bitmaps gives a length's bitmap as bytes, whatever form the length is kept in, and the same on every read.""" - by_property = PropertyBitmaps(lambda perm: perm in DENSE_6 or perm in SPARSE_6) + by_property = PropertyBitmaps( + lambda perm: perm in EVERY_THIRD_OF_LENGTH_6 or perm in FOUR_OF_LENGTH_6 + ) by_property.add_length(6) - sparse_by_property = PropertyBitmaps(lambda perm: perm in SPARSE_6) + sparse_by_property = PropertyBitmaps(lambda perm: perm in FOUR_OF_LENGTH_6) sparse_by_property.add_length(6) - from_list = bisc_module._from_perms_by_length({6: sorted(DENSE_6)}) - from_sparse_list = bisc_module._from_perms_by_length({6: sorted(SPARSE_6)}) + from_list = bisc_module._from_perms_by_length({6: sorted(EVERY_THIRD_OF_LENGTH_6)}) + from_sparse_list = bisc_module._from_perms_by_length({6: sorted(FOUR_OF_LENGTH_6)}) for property_bitmaps in ( by_property, sparse_by_property, @@ -617,15 +628,14 @@ def test_bitmaps_round_trip_with_lengths_kept_as_indices(): assert list(round_trip.bad[length]) == list(property_bitmaps.bad[length]) -LENGTH_8 = list(Perm.of_length(8)) -SPARSE_8 = {LENGTH_8[0], LENGTH_8[-1]} # the first and the last permutation +PERMS_OF_LENGTH_8 = list(Perm.of_length(8)) def test_the_first_and_the_last_permutation_of_a_sparse_length(): """The first and the last permutation of a length kept as indices, in either view, good or bad; and, with the good ones in between, a bitmap.""" - first, last = LENGTH_8[0], LENGTH_8[-1] - for good_set in (SPARSE_8, {first}, {last}): + first, last = PERMS_OF_LENGTH_8[0], PERMS_OF_LENGTH_8[-1] + for good_set in (extremes(8), {first}, {last}): property_bitmaps = bisc_module._from_perms_by_length({8: sorted(good_set)}) assert is_kept_as_indices(property_bitmaps, 8) good, bad = property_bitmaps.good[8], property_bitmaps.bad[8] @@ -635,8 +645,8 @@ def test_the_first_and_the_last_permutation_of_a_sparse_length(): assert list(good) == sorted(good_set) bad_perms = list(bad) assert len(bad_perms) == len(bad) == factorial(8) - len(good_set) - assert bad_perms == [perm for perm in LENGTH_8 if perm not in good_set] - in_between = set(LENGTH_8[1:-1]) + assert bad_perms == [perm for perm in PERMS_OF_LENGTH_8 if perm not in good_set] + in_between = set(PERMS_OF_LENGTH_8[1:-1]) property_bitmaps = bisc_module._from_perms_by_length({8: sorted(in_between)}) assert not is_kept_as_indices(property_bitmaps, 8) assert list(property_bitmaps.bad[8]) == [first, last] @@ -682,17 +692,17 @@ def all_good_but_fails_once_asked(perm): def test_add_length_replaces_labels_between_the_forms(): """add_length replaces a length's labels, from indices to a bitmap and back.""" - property_bitmaps = PropertyBitmaps(lambda perm: perm in SPARSE_6) + property_bitmaps = PropertyBitmaps(lambda perm: perm in FOUR_OF_LENGTH_6) property_bitmaps.add_length(6) assert is_kept_as_indices(property_bitmaps, 6) - property_bitmaps.prop = lambda perm: perm in DENSE_6 + property_bitmaps.prop = lambda perm: perm in EVERY_THIRD_OF_LENGTH_6 property_bitmaps.add_length(6) assert not is_kept_as_indices(property_bitmaps, 6) - assert_views_match(property_bitmaps, 6, DENSE_6) - property_bitmaps.prop = lambda perm: perm in SPARSE_6 + assert_views_match(property_bitmaps, 6, EVERY_THIRD_OF_LENGTH_6) + property_bitmaps.prop = lambda perm: perm in FOUR_OF_LENGTH_6 property_bitmaps.add_length(6) assert is_kept_as_indices(property_bitmaps, 6) - assert_views_match(property_bitmaps, 6, SPARSE_6) + assert_views_match(property_bitmaps, 6, FOUR_OF_LENGTH_6) def test_lengths_past_n_have_a_count_of_zero(monkeypatch): @@ -730,7 +740,7 @@ def recording_mine(goodperms, M, N=None, report=False): def test_a_sparse_view_iterates_independently(): """The same view of a length kept as indices, iterated twice or in step with itself, gives the same permutations.""" - property_bitmaps = bisc_module._from_perms_by_length({6: sorted(SPARSE_6)}) + property_bitmaps = bisc_module._from_perms_by_length({6: sorted(FOUR_OF_LENGTH_6)}) for view in (property_bitmaps.good[6], property_bitmaps.bad[6]): once = list(view) assert list(view) == once @@ -838,7 +848,7 @@ def stopped_mine(goodperms, M, N=None, report=False): def test_a_sparse_list_with_repeats_counts_each_permutation_once(): """A list that looks sparse before it is counted, with a permutation repeated, counts it once.""" - first, second = LENGTH_8[7], LENGTH_8[100] + first, second = PERMS_OF_LENGTH_8[7], PERMS_OF_LENGTH_8[100] property_bitmaps = bisc_module._from_perms_by_length( {8: [second, first, first, first]} ) From 496d5364871e120b35fe3898937ad8a45f0bccc6 Mon Sep 17 00:00:00 2001 From: Henning Ulfarsson Date: Sun, 25 Oct 2026 09:19:47 +0000 Subject: [PATCH 38/38] Keep the README's BiSC paragraph short Henning, through the review of #209 (2026-10-25): "We do not want to bloat the readme with technical details". The BiSC section's paragraph on PropertyBitmaps now ends after the memory comparison: the details of the sparse form, the lists past length 20, and how a PropertyBitmaps is made and reused are left to the class's docstring and the changelog. The example after the paragraph, a doctest, is unchanged, and the README's 70 examples pass on CPython 3.11 and PyPy 3.11. Co-Authored-By: Claude Opus 5.5 --- README.rst | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/README.rst b/README.rst index a6affe4..36e224f 100644 --- a/README.rst +++ b/README.rst @@ -522,14 +522,7 @@ Whatever the input, ``bisc`` and ``auto_bisc`` keep the permutations satisfying the property, and those that do not, length by length, in a ``PropertyBitmaps``. A length is kept as a bitmap, a bit for each permutation: the permutations of length 10 take 454 KB as a bitmap, where they take 0.9 to -1.8 GB as lists of ``Perm`` objects, depending on the Python. A length with fewer -than one permutation in 64 satisfying the property keeps instead the indices of -those, 8 bytes each, which then take less space: a short list of long -permutations stays small. Past length 20, where an index can need more than 8 -bytes, they are kept in a list. A ``PropertyBitmaps`` can also be given -directly, made from a property, from bitmaps, or both. The lengths ``bisc`` and -``auto_bisc`` label with its property are added to it, for a later run to -reuse. +1.8 GB as lists of ``Perm`` objects. .. code-block:: python