diff --git a/CHANGELOG.md b/CHANGELOG.md index 50ecb90..45bec2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,31 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## Unreleased +### Added +- `PropertyBitmaps` in `permuta.bisc`: a property's values on the permutations, + made from a property, given as bitmaps, or both. Each length is kept as a bitmap + (lexicographic order) or, when fewer than one permutation in 64 is good, as the + sorted indices of the good ones, whichever is smaller (in a list past length + 20, where an index can need more than 8 bytes). `auto_bisc` and `bisc` + accept it as input, and label the lengths it lacks with its property. + +### Changed +- `bisc` and `auto_bisc` turn every input into a `PropertyBitmaps` and read the + good and the bad permutations through its views, in lexicographic order, for + the same results in far less memory at longer lengths: the permutations of + length 10 take 454 KB as a bitmap, and 0.9 to 1.8 GB as lists of `Perm` objects, + depending on the Python, and a length with few good permutations takes 8 bytes + for each of them, to length 20. +- A permutation listed twice in the input of `bisc` or `auto_bisc` counts once. + Before, duplicates could make `bisc` take a length as complete when it was not. +- `auto_bisc` raises a `ValueError` when the good and the bad permutations of a + length, from a tuple of dictionaries or a stored set, overlap or are not + together all of that length's permutations. +- `bisc` and `auto_bisc` raise a `ValueError` for a permutation given under the + key of another length, and for a `Perm` given whose values are not those of a + permutation of its length, such as one written from 1. +- `bisc` accepts any mapping of lengths to permutations, not only a `dict`. + ### Fixed - Fixed bug in `Perm.rtlmax_ltrmin_decomposition` and fixed associated tests. diff --git a/README.rst b/README.rst index 1277f39..36e224f 100644 --- a/README.rst +++ b/README.rst @@ -516,7 +516,23 @@ permutations that can be imported from ``permuta.bisc.perm_properties``, such as ``smooth``, ``forest-like``, ``baxter``, ``simsun``, ``quick_sortable``, etc. Both ``bisc`` and ``auto_bisc`` can accept input in the form of a property, -or a list of permutations (satisfying some property). +a list of permutations (satisfying some property), or a ``PropertyBitmaps``. + +Whatever the input, ``bisc`` and ``auto_bisc`` keep the permutations satisfying +the property, and those that do not, length by length, in a +``PropertyBitmaps``. A length is kept as a bitmap, a bit for each permutation: +the permutations of length 10 take 454 KB as a bitmap, where they take 0.9 to +1.8 GB as lists of ``Perm`` objects. + +.. code-block:: python + + >>> property_bitmaps = PropertyBitmaps(Perm.stack_sortable) + >>> for length in range(6): + ... property_bitmaps.add_length(length) + >>> len(property_bitmaps.good[5]), len(property_bitmaps.bad[5]) + (42, 78) + >>> bisc(property_bitmaps, 3, 5) + {3: {Perm((1, 2, 0)): [set()]}} License ####### diff --git a/permuta/bisc/__init__.py b/permuta/bisc/__init__.py index 34bcdf8..d940155 100644 --- a/permuta/bisc/__init__.py +++ b/permuta/bisc/__init__.py @@ -1,6 +1,11 @@ -# type: ignore - -from .bisc import auto_bisc, bisc, create_bisc_input, read_bisc_file, write_bisc_files +from .bisc import ( + PropertyBitmaps, + auto_bisc, + bisc, + create_bisc_input, + read_bisc_file, + write_bisc_files, +) from .bisc_subfunctions import ( patterns_suffice_for_bad, patterns_suffice_for_good, @@ -10,6 +15,7 @@ ) __all__ = [ + "PropertyBitmaps", "auto_bisc", "bisc", "create_bisc_input", diff --git a/permuta/bisc/bisc.py b/permuta/bisc/bisc.py index e2b6549..817069a 100644 --- a/permuta/bisc/bisc.py +++ b/permuta/bisc/bisc.py @@ -1,9 +1,24 @@ -# type: ignore - +import bisect +import functools import json import os import types -from collections import defaultdict +from array import array +from collections.abc import Collection, Mapping, Sequence +from math import factorial +from typing import ( + Callable, + Dict, + Final, + Iterable, + Iterator, + List, + Optional, + Set, + Tuple, + TypeGuard, + Union, +) from permuta.bisc.bisc_subfunctions import ( forb, @@ -17,76 +32,188 @@ ) from permuta.patterns.perm import Perm - -def bisc(A, m, n=None, report=False): +# The forbidden patterns bisc and auto_bisc return: length -> {pattern: +# [shadings]}, each shading a set of boxes, and an empty one making the pattern +# classical +_Patterns = Dict[int, Dict[Perm, List[Set[Tuple[int, int]]]]] + + +def bisc( + A: Union[ + List[Perm], + Callable[[Perm], bool], + Mapping[int, Collection[Perm]], + "PropertyBitmaps", + ], + m: int, + n: Optional[int] = None, + report: bool = False, +) -> _Patterns: + """ + Mine the forbidden patterns of length up to m from the permutations up to + length n of a set A: a list of permutations, a property (a function), a + dictionary or another Mapping (length -> the permutations of that length + in the set), or a PropertyBitmaps. Each is turned into a PropertyBitmaps, + whose good permutations of each length up to max(m, n) mine() reads. + Without n, a property uses length 7, and the others their longest length. + - A list's permutations are the good ones, and a length with none listed + is empty. + - A property labels the lengths up to n, and a length past n, when m > n, + is read as empty. + - A Mapping is read for each length up to max(m, n), and one it lacks is + its KeyError. + - A PropertyBitmaps labels the lengths up to max(m, n) it lacks with its + property, and they are added to it; without a property, a ValueError + names the first. + report prints the progress of the search. + + Returns the forbidden patterns found: a dictionary from length to + {pattern: [shadings]}, where each shading is a set of boxes, and an empty + one makes the pattern classical. + """ if isinstance(A, list): - D = defaultdict(list) + if n is None: + n = max(len(perm) for perm in A) + perms_by_length: Dict[int, List[Perm]] = { + length: [] for length in range(max(m, n) + 1) + } for perm in A: - D[len(perm)].append(perm) + if len(perm) in perms_by_length: + perms_by_length[len(perm)].append(perm) + property_bitmaps = _from_perms_by_length(perms_by_length) + elif isinstance(A, types.FunctionType): if n is None: print("I will use permutations up to length 7") n = 7 - D = defaultdict(list) - for i in range(n + 1): - for perm in Perm.of_length(i): - if A(perm): - D[i].append(perm) + property_bitmaps = PropertyBitmaps(A) + for length in range(n + 1): + property_bitmaps.add_length(length) + # mine() reads the lengths up to m too, and a length past n is read as + # empty: none of its permutations is good + for length in range(n + 1, m + 1): + property_bitmaps._keep(length, _new_indices(length), 0) + + elif isinstance(A, Mapping): + if n is None: + n = max(A.keys()) + property_bitmaps = _from_perms_by_length( + {length: A[length] for length in range(max(m, n) + 1)} + ) - elif isinstance(A, dict): - D = A + elif isinstance(A, PropertyBitmaps): + property_bitmaps = A + if n is None: + if A.bitmaps: + n = max(A.bitmaps) + elif A.prop is not None: + print("I will use permutations up to length 7") + n = 7 + else: + raise ValueError("The PropertyBitmaps has no bitmaps and no property") + for length in range(max(m, n) + 1): + if length not in A.bitmaps: + if A.prop is None: + raise ValueError( + "bisc needs bitmaps up to length {}, and the PropertyBitmaps " + "has none of length {} and no property".format( + max(m, n), length + ) + ) + A.add_length(length) else: print( """BiSC can accept inputs: 1. A list of permutations 2. A property of permutations - 3. A dictionary of lengths pointing to permutations""" + 3. A dictionary of lengths pointing to permutations + 4. A PropertyBitmaps""" ) assert False - if n is None: - n = max(D.keys()) - - ci, goodpatts = mine(D, m, n, report=report) - SG = forb(ci, goodpatts, m, report=report) + ci, goodpatts = mine(property_bitmaps.good, m, n, report=report) + SG: _Patterns = forb(ci, goodpatts, m, report=report) return SG -def auto_bisc(prop): +def auto_bisc( + prop: Union[ + Callable[[Perm], bool], + List[Perm], + Tuple[Dict[int, List[Perm]], Dict[int, List[Perm]]], + str, + "PropertyBitmaps", + ], +) -> Optional[_Patterns]: + """ + Learn a basis of forbidden patterns for a set of permutations, growing + the length of the patterns and of the permutations they are learned from + until the patterns found describe the set on every permutation up to + length L, at least 8. Returns the basis, in the form bisc returns its + patterns, or None when the input is too short to check a basis. + + prop is the set: a property of permutations (a function), a list of + permutations, a tuple of two dictionaries (length -> the permutations in + the set, and those not in it), the name of a set in + permuta/resources/bisc, or a PropertyBitmaps. Each is turned into a + PropertyBitmaps at the start, whose views give the good and the bad + permutations of each length. + - A property labels the lengths up to 8 at the start, and each longer one + when auto_bisc first needs it. + - A list's permutations are the good ones, and the others are bad. + - For a tuple of dictionaries, or a stored set, the good and the bad + permutations of each length up to the shorter dictionary's longest must + not overlap, and must together be all of that length's permutations, or + a ValueError is raised. A length one of them lacks is empty in it, and a + permutation given twice on one side counts once. + - A PropertyBitmaps labels the lengths it lacks with its property, and + they are added to it, so that a later run reuses them. Without a + property, auto_bisc stops when it needs a length it lacks, as it does + for a tuple of dictionaries. + """ L = 8 # Want to sanity check on at least S8, and one above n n = 4 m = 2 if isinstance(prop, list): - # If a list is passed in then we put it in a dictionary at the start - A = defaultdict(list) - # We will put the complement up to S8 into another dictionary - B = defaultdict(list) - for perm in prop: - A[len(perm)].append(perm) - if L not in A.keys(): + # If a list is passed in then its permutations are the good ones, and + # the others up to its longest length are the bad ones + listed_lengths = {len(perm) for perm in prop} + if L not in listed_lengths: print("You should have permutations up to length at least 8") - return - for i in range(max(L + 1, max(A.keys()) + 1)): - if i > 7: - print("Populating the dictionary of bad perms of length ", i) - B[i] = [perm for perm in Perm.of_length(i) if perm not in A[i]] + return None + longest_length = max(listed_lengths) + # Printed as when the bad permutations were made into lists, so that + # what auto_bisc prints does not change; now they are read from the + # labels of the good ones + for i in range(L, longest_length + 1): + print("Populating the dictionary of bad perms of length ", i) + perms_by_length: Dict[int, List[Perm]] = { + length: [] for length in range(longest_length + 1) + } + for perm in prop: + perms_by_length[len(perm)].append(perm) + property_bitmaps = _from_perms_by_length(perms_by_length) elif isinstance(prop, types.FunctionType): - # If a property is passed in then we use it to populate both - # both dictionaries up to S8 - A = defaultdict(list) - B = defaultdict(list) + # If a property is passed in then it labels the lengths up to S8 now, + # and the longer ones as they are needed + property_bitmaps = PropertyBitmaps(prop) for i in range(L + 1): - A[i] = [] - B[i] = [] - for perm in Perm.of_length(i): - if prop(perm): - A[i].append(perm) - else: - B[i].append(perm) + property_bitmaps.add_length(i) + + elif isinstance(prop, PropertyBitmaps): + # The lengths the PropertyBitmaps lacks are labeled with its + # property, if it has one + property_bitmaps = prop + for i in range(L + 1): + if i not in property_bitmaps.bitmaps: + if property_bitmaps.prop is None: + print("You should have bitmaps up to length at least 8") + return None + property_bitmaps.add_length(i) elif ( isinstance(prop, tuple) @@ -94,59 +221,42 @@ def auto_bisc(prop): and isinstance(prop[1], dict) ): # If you already have the dictionaries you can pass them in as a tuple - A = prop[0] - B = prop[1] - if L not in A.keys(): + if L not in prop[0].keys(): print("You should have permutations up to length at least 8") - return + return None + property_bitmaps = _from_good_and_bad(prop[0], prop[1]) + if L - 1 not in property_bitmaps.bitmaps: + # The bad permutations stop short of the lengths up to L - 1, which + # auto_bisc learns from before its sanity checks reach L + print("You need to add longer permutations to the dictionaries") + return None elif isinstance(prop, str): # If you pass in a string we assume it is pointing to a file in permsets print("Attempting to read perms from permsets") - good_entry = None - bad_entry = None - bisc_dir = os.path.join(os.path.dirname(__file__), "..", "resources", "bisc") - bisc_dir = os.path.abspath(bisc_dir) - - with os.scandir(bisc_dir) as entries: - for i, entry in enumerate(entries): - en = entry.name[:-5] - spl = en.split("_") - if spl[0] == prop: - if ( - good_entry is None - and spl[1] == "good" - and int(spl[2].split("len")[1]) >= L - ): - good_entry = en - if ( - bad_entry is None - and spl[1] == "bad" - and int(spl[2].split("len")[1]) >= L - ): - bad_entry = en - if good_entry is not None and bad_entry is not None: - break - if good_entry is not None and bad_entry is not None: - A = read_bisc_file(bisc_dir + "/" + good_entry) - B = read_bisc_file(bisc_dir + "/" + bad_entry) - else: + stored_set = _read_stored_set(prop, L) + if stored_set is None: print("The required files do not exist") - return + return None + property_bitmaps = _from_good_and_bad(*stored_set) else: print( """BiSC can accept inputs: 1. A list of permutations 2. A property of permutations - 3. A dictionary of lengths pointing to permutations - 4. A string """ + 3. A tuple of two dictionaries of lengths pointing to permutations + 4. A string + 5. A PropertyBitmaps""" ) assert False + # length -> the good (A) and the bad (B) permutations of that length + A, B = property_bitmaps.good, property_bitmaps.bad + while True: print("Learning patterns of length {} using perms of length {}".format(m, n)) - SG = bisc(A, m, n) + SG = bisc(property_bitmaps, m, n) show_me(SG, more=True) if SG != {}: @@ -172,7 +282,7 @@ def auto_bisc(prop): basis = bases[0] show_me_basis(basis, dict_numbs_to_patts) - sg = to_sg_format(basis, dict_numbs_to_patts) + sg: _Patterns = to_sg_format(basis, dict_numbs_to_patts) val, avoiding_perms = patterns_suffice_for_bad( sg, L, B, stop_on_failure=True @@ -217,56 +327,578 @@ def auto_bisc(prop): L = n + 1 if isinstance(prop, list) and L > max(A.keys()): print("You need to input a longer list of permutations") - return + return None - elif isinstance(prop, types.FunctionType): - # Adding perms to the dictionaries + elif isinstance(prop, (types.FunctionType, PropertyBitmaps)): + # The property labels the new lengths for i in range(oldL + 1, L + 1): - print("Adding perms of length {}".format(i)) - for perm in Perm.of_length(i): - if prop(perm): - A[i].append(perm) - else: - B[i].append(perm) - - elif isinstance(prop, tuple) and L > min(max(A.keys()), max(B.keys())): + if i not in property_bitmaps.bitmaps: + if property_bitmaps.prop is None: + print("You need to add bitmaps of longer permutations") + return None + print("Adding perms of length {}".format(i)) + property_bitmaps.add_length(i) + + elif isinstance(prop, tuple) and L > max(A.keys()): print("You need to add longer permutations to the dictionaries") - return + return None - elif isinstance(prop, str) and L > min(max(A.keys()), max(B.keys())): + elif isinstance(prop, str) and L > max(A.keys()): # If you pass in a string we assume it is pointing to a file in permsets print("Attempting to read perms from permsets") - good_entry = None - bad_entry = None - bisc_dir = os.path.join( - os.path.dirname(__file__), "..", "resources", "bisc" - ) - bisc_dir = os.path.abspath(bisc_dir) - with os.scandir(bisc_dir) as entries: - for i, entry in enumerate(entries): - en = entry.name[:-5] - spl = en.split("_") - if spl[0] == prop: - if ( - good_entry is None - and spl[1] == "good" - and int(spl[2].split("len")[1]) >= L - ): - good_entry = en - if ( - bad_entry is None - and spl[1] == "bad" - and int(spl[2].split("len")[1]) >= L - ): - bad_entry = en - if good_entry is not None and bad_entry is not None: - break - if good_entry is not None and bad_entry is not None: - A = read_bisc_file(bisc_dir + "/" + good_entry) - B = read_bisc_file(bisc_dir + "/" + bad_entry) - else: + stored_set = _read_stored_set(prop, L) + if stored_set is None: print("The required files do not exist") - return + return None + property_bitmaps = _from_good_and_bad(*stored_set) + A, B = property_bitmaps.good, property_bitmaps.bad + + +def _bitmap_size(length: int) -> int: + """The number of bytes in the bitmap of a length: a bit for each of its + permutations.""" + return (factorial(length) + 7) // 8 + + +def _set_label(bitmap: bytearray, index: int) -> None: + """Set the label of the permutation at index: bit index % 8 of byte + index // 8.""" + bitmap[index // 8] |= 1 << (index % 8) + + +def _label(bitmap: Union[bytes, bytearray], index: int) -> int: + """The label (0 or 1) of the permutation at index.""" + return (bitmap[index // 8] >> (index % 8)) & 1 + + +# A view unranks the permutations it wants, rather than generating all of the +# length's, when at most one in this many is wanted. Unranking one costs about 5 +# (CPython 3.11) to 13 (PyPy 3.11) times as much as generating and testing one, +# measured at lengths 8 to 12, and 16 is above both +_UNRANK_WHEN_ONE_IN = 16 + + +def _wanted_indices( + bitmap: Union[bytes, bytearray], length: int, wanted_bit: int +) -> Iterator[int]: + """The indices, in increasing order, of the permutations of a length whose + label is wanted_bit. A byte with no such label is skipped whole.""" + skipped_byte = 0xFF if wanted_bit == 0 else 0 + permutation_count = factorial(length) + for byte_index, byte in enumerate(bitmap): + if byte != skipped_byte: + for index in range( + 8 * byte_index, min(8 * byte_index + 8, permutation_count) + ): + if _label(bitmap, index) == wanted_bit: + yield index + + +# The sorted indices of a length's good permutations are kept in an array of +# this type code, unsigned integers of 8 bytes, while every index of the length +# fits in one: to length 20, since 21! is above 2**64. Past that they are kept in +# a list. Final, so that mypy takes it as the literal "Q", and the array as one +# of ints +_INDEX_TYPECODE: Final = "Q" +_INDEX_BITS = 8 * array(_INDEX_TYPECODE).itemsize +_Indices = Union["array[int]", List[int]] + +# A length's labels, as a PropertyBitmaps keeps them: its bitmap, or the sorted +# indices of its good permutations +_Labels = Union[bytes, _Indices] + +# A length keeps the sorted indices of its good permutations, _INDEX_BITS bits +# each, rather than its bitmap, a bit for each of its permutations, when fewer +# than one in this many of them are good: the indices are then the smaller +_SPARSE_WHEN_ONE_IN = _INDEX_BITS + + +def _new_indices(length: int, indices: Iterable[int] = ()) -> _Indices: + """The sorted indices of a length's good permutations, given in increasing + order: an array while every index of the length fits in its items, and a + list past that.""" + if factorial(length) <= 1 << _INDEX_BITS: + return array(_INDEX_TYPECODE, indices) + return list(indices) + + +def _largest_sparse_good_count(length: int) -> int: + """The most good permutations a length can have and keep their indices: + fewer than one in _SPARSE_WHEN_ONE_IN of its n! permutations, so + good_count * _SPARSE_WHEN_ONE_IN < n!, which for ints is good_count <= (n! + - 1) // _SPARSE_WHEN_ONE_IN. n! // _SPARSE_WHEN_ONE_IN would also let + through exactly one in _SPARSE_WHEN_ONE_IN when it divides n!, as 64 + divides n! from n = 8 on.""" + return (factorial(length) - 1) // _SPARSE_WHEN_ONE_IN + + +def _is_sparse(good_count: int, length: int) -> bool: + """Whether a length with good_count good permutations keeps their indices + rather than its bitmap.""" + return good_count <= _largest_sparse_good_count(length) + + +def _kept_as_bitmap(labels: _Labels) -> TypeGuard[bytes]: + """Whether a length's labels are kept as its bitmap, rather than as the + indices of its good permutations, in an array or a list: every reader of + the labels tells the forms apart here. The bitmap is always bytes.""" + return isinstance(labels, bytes) + + +def _label_in(labels: _Labels, index: int) -> int: + """The label (0 or 1) of the permutation at index, from a length's labels + in either form.""" + if _kept_as_bitmap(labels): + return _label(labels, index) + position = bisect.bisect_left(labels, index) + return int(position < len(labels) and labels[position] == index) + + +def _bitmap_from_indices(length: int, indices: Iterable[int]) -> bytearray: + """The bitmap of a length with the bits at the given indices set.""" + bitmap = bytearray(_bitmap_size(length)) + for index in indices: + _set_label(bitmap, index) + return bitmap + + +# Bitmaps are read this many bytes at a time, each chunk as one int, to count +# their bits or the bits two of them share, so that no int is made of a whole +# bitmap +_CHUNK_BYTES = 1 << 16 + + +def _chunks(bitmap: Union[bytes, bytearray]) -> Iterator[int]: + """A bitmap's bytes, _CHUNK_BYTES at a time, each chunk as an int.""" + view = memoryview(bitmap) + for start in range(0, len(view), _CHUNK_BYTES): + yield int.from_bytes(view[start : start + _CHUNK_BYTES], "little") + + +def _count_bits(bitmap: Union[bytes, bytearray]) -> int: + """The number of bits set in a bitmap: its good permutations.""" + return sum(chunk.bit_count() for chunk in _chunks(bitmap)) + + +def _labels_from_bitmap( + length: int, bitmap: Union[bytes, bytearray] +) -> Tuple[_Labels, int]: + """A length's labels from its bitmap, in the form chosen by its number of + good permutations, and that number. A bitmap kept is bytes: a bytearray + is copied, so that changing it does not change the labels.""" + good_count = _count_bits(bitmap) + if _is_sparse(good_count, length): + return _new_indices(length, _wanted_indices(bitmap, length, 1)), good_count + return bytes(bitmap), good_count + + +class PropertyBitmaps: + """A property's values on the permutations, length by length: which + permutations of each length satisfy it (good) and which do not (bad). + bisc and auto_bisc turn each of their inputs into one, and read its good + and bad permutations through its views, good and bad. + + A length's labels are its bitmap: bit i is the label of Perm.unrank(i, + length), bit i % 8 of byte i // 8, from the low bit. So a permutation perm + is at bit perm.rank() - sum(factorial(k) for k in range(len(perm))), and + the bits are in the order in which Perm.of_length(length) yields the + permutations (lexicographic). The bitmap of length n is (n! + 7) // 8 + bytes. A length with fewer than one good permutation in 64 keeps instead + the sorted indices of its good permutations (their bits), 8 bytes each, + which then take less space, but for the few percent an array allocates + ahead: three permutations of length 14 take 24 bytes, where its bitmap + takes 10.9 GB. Past length 20, whose indices can exceed 8 bytes, they are + kept in a list, and there is no longest length. There the bitmap of a + length cannot be built, and its bad permutations, more than sys.maxsize, + have no len(), though they can be iterated over and tested with in: their + number is the length's factorial less its good_counts. + + The labels can be given as bitmaps (bitmaps={length: bytes}), made from a + property (add_length), or both: auto_bisc and bisc label the lengths they + need and are not given with prop, adding them to this object. Without a + prop, auto_bisc stops, as for a tuple of dictionaries, and bisc raises a + ValueError. A bitmap given must be bytes-like (bytes, a bytearray or a + memoryview), or a TypeError is raised. It must be of its length's size, + with no bit set past the length's permutations, or a ValueError is + raised. Changing a bytearray after giving it does not change the labels. + + The attributes bitmaps (length -> its bitmap, a read-only Mapping) and + good_counts (length -> the number of good permutations) are for reading. + bitmaps gives a length kept as a bitmap as it is kept, and builds the + bitmap of a length kept as indices each time it is asked for it: so + items(), values() and dict(bitmaps), or bitmaps given to another + PropertyBitmaps, build every such length's bitmap, 10.9 GB for length 14. + Asking whether a length is there, iterating over the lengths and len() + build none. + + The views read the labels length by length: iterating over a length, + counting it, asking whether a length is there, or whether a permutation + is in a length. A bitmap of S_10 is 454 KB; S_10 as a list of Perm + objects takes 0.9 GB on PyPy 3.11 and 1.3 to 1.8 GB on CPython. + """ + + def __init__( + self, + prop: Optional[Callable[[Perm], bool]] = None, + bitmaps: Optional[Mapping[int, Union[bytes, bytearray, memoryview]]] = None, + ) -> None: + self.prop = prop + # length -> its labels, in the form chosen by its number of good + # permutations (_is_sparse). The storage is the module's: bitmaps and + # the views read _labels, and the constructor, add_length, bisc and the + # functions that make a PropertyBitmaps keep a length's labels in it + # through _keep + self._labels: Dict[int, _Labels] = {} + self.good_counts: Dict[int, int] = {} + if bitmaps is not None: + for length, bitmap in bitmaps.items(): + if not isinstance(bitmap, (bytes, bytearray, memoryview)): + raise TypeError( + "A bitmap must be bytes, a bytearray or a memoryview, " + "not {}".format(type(bitmap).__name__) + ) + if isinstance(bitmap, memoryview): + # Its bytes, whatever the view's format and shape + bitmap = bytes(bitmap) + if len(bitmap) != _bitmap_size(length): + raise ValueError( + "A bitmap of length {} has {} bytes, and must have {}".format( + length, len(bitmap), _bitmap_size(length) + ) + ) + # The size check leaves spare bits in the last byte only at + # lengths 0 to 3 (7, 7, 6 and 2 of them), since n! is a + # multiple of 8 from n = 4 on. A spare bit set would be counted + # as a good permutation: len() of the good view would disagree + # with its iteration, and len() of the bad view could go + # negative. + spare_bit_count = 8 * len(bitmap) - factorial(length) + if spare_bit_count and bitmap[-1] >> (8 - spare_bit_count): + raise ValueError( + "A bitmap of length {} has bits set past its {} " + "permutations".format(length, factorial(length)) + ) + self._keep(length, *_labels_from_bitmap(length, bitmap)) + self.bitmaps: Mapping[int, bytes] = _Bitmaps(self) + self.good = _LengthViews(self, wanted_bit=1) + self.bad = _LengthViews(self, wanted_bit=0) + + def _keep(self, length: int, labels: _Labels, good_count: int) -> None: + """Keep a length's labels, in the form its caller chose by _is_sparse, + with its number of good permutations. Neither is checked here: the + callers make them right.""" + self._labels[length] = labels + self.good_counts[length] = good_count + + def add_length(self, length: int) -> None: + """Label the permutations of a length with the property, and keep the + length's labels, replacing any it has. The indices of the good + permutations are kept while they are few; once there are more than a + length keeps as indices, they are set in the length's bitmap, and the + rest of the length is labeled in it. It needs a property.""" + if self.prop is None: + raise ValueError( + "The PropertyBitmaps has no property to label length {} " + "with".format(length) + ) + largest_sparse_good_count = _largest_sparse_good_count(length) + good_indices = _new_indices(length) + # One iterator for both loops: the second goes on where the first + # stopped + indexed_perms = enumerate(Perm.of_length(length)) + for index, perm in indexed_perms: + if self.prop(perm): + good_indices.append(index) + if len(good_indices) > largest_sparse_good_count: + break + else: + # No break: the good permutations stayed few + self._keep(length, good_indices, len(good_indices)) + return + # Too many for indices: they are set in the bitmap, and the second loop + # goes on in it from the next permutation, with a bit to set for each + # good one and nothing more + bitmap = _bitmap_from_indices(length, good_indices) + good_count = len(good_indices) + # The indices now take about as much as the bitmap. Dropping them keeps + # the peak at two bitmaps, with the bytes made of it at the end + del good_indices + for index, perm in indexed_perms: + if self.prop(perm): + _set_label(bitmap, index) + good_count += 1 + self._keep(length, bytes(bitmap), good_count) + + +class _LengthKeys: + """The keys of a PropertyBitmaps' Mappings, bitmaps and the views: the + lengths it has labels for.""" + + def __init__(self, property_bitmaps: PropertyBitmaps) -> None: + self.property_bitmaps = property_bitmaps + + def __iter__(self) -> Iterator[int]: + return iter(self.property_bitmaps._labels) + + def __len__(self) -> int: + return len(self.property_bitmaps._labels) + + +class _Bitmaps(_LengthKeys, Mapping[int, bytes]): + """length -> its bitmap, read-only: the bitmap kept, or, for a length kept + as the indices of its good permutations, one built from them when + asked.""" + + def __getitem__(self, length: int) -> bytes: + labels = self.property_bitmaps._labels[length] + if _kept_as_bitmap(labels): + return labels + return bytes(_bitmap_from_indices(length, labels)) + + def __contains__(self, length: object) -> bool: + # Mapping's would build the bitmap of a length kept as indices, to + # answer whether the length is there + return length in self.property_bitmaps._labels + + +class _LengthViews(_LengthKeys, Mapping[int, "_PermsOfLength"]): + """length -> the good (or bad) permutations of that length, read-only: a + Mapping whose keys are the lengths with labels, through which bisc and + auto_bisc read the permutations. wanted_bit is the label of the + permutations it gives: 1 for the good ones, 0 for the bad ones.""" + + def __init__(self, property_bitmaps: PropertyBitmaps, wanted_bit: int) -> None: + super().__init__(property_bitmaps) + self.wanted_bit = wanted_bit + + def __getitem__(self, length: int) -> "_PermsOfLength": + if length not in self.property_bitmaps._labels: + raise KeyError(length) + return _PermsOfLength(self.property_bitmaps, length, wanted_bit=self.wanted_bit) + + +class _PermsOfLength(Collection[Perm]): + """The good (or bad) permutations of one length, read from its labels: a + Collection. Iterating yields those whose label is the one wanted, in + lexicographic order. For a length kept as the indices of its good + permutations, the good ones are unranked from them, and the bad ones are + the others, generated in order. From a bitmap, when few are wanted they + are unranked, and otherwise the length's permutations are generated and + each bit is tested. len() is the count kept with the labels, and a + permutation is in it when its label is the one wanted.""" + + def __init__( + self, property_bitmaps: PropertyBitmaps, length: int, wanted_bit: int + ) -> None: + self.property_bitmaps = property_bitmaps + self.length = length + self.wanted_bit = wanted_bit + + def __contains__(self, perm: object) -> bool: + # A permutation of this length: a Perm, or any other sequence of the + # numbers 0 to length - 1, each once + if not ( + isinstance(perm, Sequence) + and len(perm) == self.length + and all(isinstance(value, int) for value in perm) + and set(perm) == set(range(self.length)) + ): + return False + labels = self.property_bitmaps._labels[self.length] + index = Perm(perm).rank() - _shorter_count(self.length) + return _label_in(labels, index) == self.wanted_bit + + def __iter__(self) -> Iterator[Perm]: + labels = self.property_bitmaps._labels[self.length] + wanted_count = self._wanted_count() + if wanted_count == 0: + # Nothing to yield, and no need to read the labels + return iter(()) + if _kept_as_bitmap(labels): + if wanted_count * _UNRANK_WHEN_ONE_IN <= factorial(self.length): + return _unranked_perms( + _wanted_indices(labels, self.length, self.wanted_bit), + self.length, + ) + return _perms_with_label(labels, self.length, self.wanted_bit) + if self.wanted_bit: + return _unranked_perms(labels, self.length) + return _perms_skipping(labels, self.length) + + def __len__(self) -> int: + # len() raises OverflowError past sys.maxsize, as for the bad + # permutations of a length past 20 + return self._wanted_count() + + def __bool__(self) -> bool: + return self._wanted_count() > 0 + + def _wanted_count(self) -> int: + """The number of permutations with the label wanted, of any size.""" + good_count = self.property_bitmaps.good_counts[self.length] + return good_count if self.wanted_bit else factorial(self.length) - good_count + + +# Each way of iterating over a length is a generator of its own, with one small +# loop. With the three loops in one generator, PyPy 3.11 often took about a third +# longer over auto_bisc("smooth") + + +def _unranked_perms(indices: Iterable[int], length: int) -> Iterator[Perm]: + """The permutations of a length at the given indices, unranked.""" + for index in indices: + yield Perm.unrank(index, length) + + +def _perms_skipping(good_indices: Iterable[int], length: int) -> Iterator[Perm]: + """The permutations of a length not at the given indices, which are in + increasing order: the length is generated, and the good indices skipped in + step.""" + good_index_iterator = iter(good_indices) + next_good_index = next(good_index_iterator, None) + for index, perm in enumerate(Perm.of_length(length)): + if index == next_good_index: + next_good_index = next(good_index_iterator, None) + else: + yield perm + + +def _perms_with_label(bitmap: bytes, length: int, wanted_bit: int) -> Iterator[Perm]: + """The permutations of a length whose label in the bitmap is wanted_bit: + the length is generated, and each bit tested.""" + for index, perm in enumerate(Perm.of_length(length)): + if _label(bitmap, index) == wanted_bit: + yield perm + + +@functools.lru_cache(maxsize=None) +def _shorter_count(length: int) -> int: + """The number of permutations shorter than length: a permutation's bit is + its rank() less this.""" + return sum(factorial(shorter_length) for shorter_length in range(length)) + + +def _indices_of(length: int, perms: Iterable[Perm]) -> Iterator[int]: + """The indices (bits) of permutations given as ones of a length. One of + another length raises a ValueError, and so does a sequence that is not a + permutation of 0 to length - 1, such as one written from 1: its rank() + would be another permutation's, or outside the length's.""" + shorter_count = _shorter_count(length) + permutation_values = set(range(length)) + for perm in perms: + if len(perm) != length: + raise ValueError( + "The permutation {!r} of length {} is given as one of length " + "{}".format(perm, len(perm), length) + ) + if set(perm) != permutation_values: + raise ValueError( + "{!r} is not a permutation of 0 to {}".format(perm, length - 1) + ) + if not isinstance(perm, Perm): + perm = Perm(perm) + yield perm.rank() - shorter_count + + +def _labels_of(length: int, perms: Collection[Perm]) -> Tuple[_Labels, int]: + """The labels of a length whose good permutations are the ones given, in + the form chosen by their number, and that number. A permutation given + twice counts once.""" + if _is_sparse(len(perms), length): + good_indices = _new_indices(length, sorted(set(_indices_of(length, perms)))) + return good_indices, len(good_indices) + # Permutations given more than once can make them look many: the count of + # the bitmap's bits chooses the form + bitmap = _bitmap_from_indices(length, _indices_of(length, perms)) + return _labels_from_bitmap(length, bitmap) + + +def _overlap_count(labels: _Labels, other_labels: _Labels) -> int: + """The number of permutations good in both of two labels of one length.""" + if not _kept_as_bitmap(labels): + return sum(_label_in(other_labels, index) for index in labels) + if not _kept_as_bitmap(other_labels): + return sum(_label_in(labels, index) for index in other_labels) + return sum( + (chunk & other_chunk).bit_count() + for chunk, other_chunk in zip(_chunks(labels), _chunks(other_labels)) + ) + + +def _from_perms_by_length( + perms_by_length: Mapping[int, Collection[Perm]], +) -> PropertyBitmaps: + """A PropertyBitmaps, with no property, whose good permutations of each + length are the ones given for it.""" + property_bitmaps = PropertyBitmaps() + for length, perms in perms_by_length.items(): + property_bitmaps._keep(length, *_labels_of(length, perms)) + return property_bitmaps + + +def _from_good_and_bad( + good: Mapping[int, Collection[Perm]], bad: Mapping[int, Collection[Perm]] +) -> PropertyBitmaps: + """A PropertyBitmaps, with no property, from the good and the bad + permutations of each length up to the shorter mapping's longest. A length + one of them lacks is empty in it, and a permutation given twice on one + side counts once. The good and the bad permutations of each length must + not overlap, and must together be all of its permutations, or a + ValueError is raised.""" + property_bitmaps = PropertyBitmaps() + for length in range(min(max(good), max(bad)) + 1): + good_labels, good_count = _labels_of(length, good.get(length, ())) + bad_labels, bad_count = _labels_of(length, bad.get(length, ())) + in_both_count = _overlap_count(good_labels, bad_labels) + in_neither_count = factorial(length) - (good_count + bad_count - in_both_count) + if in_both_count or in_neither_count: + raise ValueError( + "The good and the bad permutations of length {} must not overlap " + "and must together be all of its {} permutations (in both: {}, " + "in neither: {})".format( + length, factorial(length), in_both_count, in_neither_count + ) + ) + property_bitmaps._keep(length, good_labels, good_count) + return property_bitmaps + + +def _read_stored_set( + name: str, min_length: int +) -> Optional[Tuple[Dict[int, List[Perm]], Dict[int, List[Perm]]]]: + """The good and the bad permutations of the set stored under name in + permuta/resources/bisc, from the first of its files of each kind that go + up to min_length or further, or None if there are none.""" + good_entry = None + bad_entry = None + bisc_dir = os.path.join(os.path.dirname(__file__), "..", "resources", "bisc") + bisc_dir = os.path.abspath(bisc_dir) + + with os.scandir(bisc_dir) as entries: + for entry in entries: + entry_name = entry.name[:-5] + name_parts = entry_name.split("_") + if name_parts[0] == name: + if ( + good_entry is None + and name_parts[1] == "good" + and int(name_parts[2].split("len")[1]) >= min_length + ): + good_entry = entry_name + if ( + bad_entry is None + and name_parts[1] == "bad" + and int(name_parts[2].split("len")[1]) >= min_length + ): + bad_entry = entry_name + if good_entry is not None and bad_entry is not None: + break + if good_entry is not None and bad_entry is not None: + return ( + read_bisc_file(bisc_dir + "/" + good_entry), + read_bisc_file(bisc_dir + "/" + bad_entry), + ) + return None def create_bisc_input(N, prop): diff --git a/permuta/bisc/bisc_subfunctions.py b/permuta/bisc/bisc_subfunctions.py index 3de86bc..a410af5 100644 --- a/permuta/bisc/bisc_subfunctions.py +++ b/permuta/bisc/bisc_subfunctions.py @@ -1,4 +1,7 @@ -# type: ignore +# mypy: ignore-errors +# Its errors are left for a pull request of its own, which will type this +# module. Unlike a file-level type: ignore, this lets mypy read the functions +# that bisc.py imports from it. from itertools import chain from math import factorial diff --git a/tests/bisc/test_bisc_inputs.py b/tests/bisc/test_bisc_inputs.py new file mode 100644 index 0000000..09dcade --- /dev/null +++ b/tests/bisc/test_bisc_inputs.py @@ -0,0 +1,395 @@ +"""Tests of how bisc and auto_bisc take each kind of input: the default n of a +list, a mapping, a PropertyBitmaps and a property; m > n for a list and a +mapping; a list with permutations longer than max(m, n); where auto_bisc stops +on a short list or tuple and on a stored set it cannot find; report=True; the +printed output of each input kind; growth that would label a length it was +given; and where a view starts to unrank. The expected values are the answers +bisc and auto_bisc gave before they kept their input as bitmaps, or follow from +the mathematics.""" + +import contextlib +import importlib +import io + +import pytest + +from permuta.bisc.bisc import PropertyBitmaps, auto_bisc, bisc, create_bisc_input +from permuta.bisc.perm_properties import smooth +from permuta.patterns.perm import Perm + +bisc_module = importlib.import_module("permuta.bisc.bisc") + +BASIS_231 = {3: {Perm((1, 2, 0)): [set()]}} +WEST_2_AT_5 = {4: {Perm((1, 2, 3, 0)): [set()], Perm((2, 1, 3, 0)): [{(1, 4)}]}} + + +def good_by_length(prop, longest): + return { + length: [perm for perm in Perm.of_length(length) if prop(perm)] + for length in range(longest + 1) + } + + +def listed(good): + return [perm for perms in good.values() for perm in perms] + + +def bitmaps_of(good): + bitmaps = {} + for length, perms in good.items(): + members = set(perms) + label_bits = 0 + for index, perm in enumerate(Perm.of_length(length)): + if perm in members: + label_bits |= 1 << index + bitmaps[length] = label_bits.to_bytes( + (len(list(Perm.of_length(length))) + 7) // 8, "little" + ) + return bitmaps + + +def run(function, *args): + output = io.StringIO() + with contextlib.redirect_stdout(output): + result = function(*args) + return result, output.getvalue() + + +# The default n of each input kind +@pytest.mark.parametrize("kind", ["list", "mapping", "bitmaps"]) +def test_bisc_without_n_reads_up_to_the_longest_length(kind): + # West-2-stack-sortable needs perms of length 5: n = 4 gives another answer + good = good_by_length(Perm.west_2_stack_sortable, 5) + make = { + "list": lambda: listed(good), + "mapping": lambda: good, + "bitmaps": lambda: PropertyBitmaps(bitmaps=bitmaps_of(good)), + }[kind] + assert bisc(make(), 4) == WEST_2_AT_5 + assert bisc(make(), 4, 4) != WEST_2_AT_5 + + +def test_bisc_of_a_property_without_n_labels_up_to_7_and_says_so(): + seen = set() + + def prop(perm): + seen.add(len(perm)) + return perm.stack_sortable() + + answer, printed = run(bisc, prop, 3) + assert answer == BASIS_231 + assert seen == set(range(8)) + assert printed == "I will use permutations up to length 7\n" + + +# m > n for a list and a mapping +def test_bisc_reads_the_lengths_up_to_m_of_a_list_or_a_mapping_past_n(): + good = good_by_length(Perm.stack_sortable, 6) + avoiding_231 = { + perm: [set()] for perm in Perm.of_length(4) if perm.avoids(Perm((1, 2, 0))) + } + # no good permutation of length 4 in a list that ends at length 3: all of + # the length-4 permutations that avoid 231 are minimal forbidden patterns + assert bisc(listed(good_by_length(Perm.stack_sortable, 3)), 4) == { + **BASIS_231, + 4: avoiding_231, + } + # the lengths up to m = 5 of a longer list or mapping are read, though only + # perms up to n = 3 are mined + expected = {**BASIS_231, 4: {}, 5: {}} + assert bisc(listed(good), 5, 3) == expected + assert bisc(good, 5, 3) == expected + + +# Lengths 0 and 1, m = 0, n = 0, and the empty input +def test_bisc_of_the_empty_set_forbids_the_empty_permutation(): + assert bisc([], 3, 5) == {0: {Perm(()): [set()]}, 1: {}, 2: {}, 3: {}} + assert bisc(lambda perm: False, 3, 5) == { + 0: {Perm(()): [set()]}, + 1: {}, + 2: {}, + 3: {}, + } + with pytest.raises(ValueError): + bisc([], 3) # no n, and no longest length + with pytest.raises(ValueError): + bisc({}, 3) + + +def test_bisc_with_m_or_n_zero(): + # the empty permutation alone: the class {empty}, whose basis is 1 + assert bisc(Perm.stack_sortable, 3, 0) == {1: {Perm((0,)): [set()]}, 2: {}, 3: {}} + assert bisc([Perm(())], 3, 0) == {1: {Perm((0,)): [set()]}, 2: {}, 3: {}} + # patterns of length 0 only: every length-0 permutation is good, "search for + # longer patterns", {} + for A in (Perm.stack_sortable, [Perm(())], {0: [Perm(())]}): + assert run(bisc, A, 0, 0) == ({}, "You need to search for longer patterns\n") + + +# report=True, byte for byte +def test_bisc_report_prints_the_progress_and_the_same_for_each_input_kind(): + good = good_by_length(Perm.stack_sortable, 5) + answer, printed = run(bisc, Perm.stack_sortable, 3, 5, True) + assert answer == BASIS_231 + assert "Starting search for allowed patterns of lengths 0, ..., 3" in printed + assert "Starting search for minimal forbidden patterns" in printed + assert run(bisc, Perm.stack_sortable, 3, 5, False)[1] == "" + for A in (listed(good), good, PropertyBitmaps(bitmaps=bitmaps_of(good))): + assert run(bisc, A, 3, 5, True) == (answer, printed) + + +# What auto_bisc prints, for each input kind +def test_auto_bisc_prints_the_same_for_each_input_kind(): + good, bad = create_bisc_input(8, Perm.stack_sortable) + answer, printed = run(auto_bisc, Perm.stack_sortable) + assert answer == BASIS_231 + # the bitmaps and the pair of dictionaries print what the property prints + assert run(auto_bisc, (good, bad)) == (answer, printed) + assert run(auto_bisc, PropertyBitmaps(bitmaps=bitmaps_of(good))) == ( + answer, + printed, + ) + # a list prints a line for the bad permutations of length 8 it populates + populating = "Populating the dictionary of bad perms of length 8\n" + assert run(auto_bisc, listed(good)) == (answer, populating + printed) + # a stored set prints the line that says so first + smooth_answer, smooth_printed = run(auto_bisc, smooth) + assert run(auto_bisc, "smooth") == ( + smooth_answer, + "Attempting to read perms from permsets\n" + smooth_printed, + ) + + +# Where auto_bisc stops: a list, a tuple, a stored set +def derangement_in_av_213_231_312(perm): + return perm.avoids(Perm((1, 0, 2)), Perm((1, 2, 0)), Perm((2, 0, 1))) and all( + value != index for index, value in enumerate(perm) + ) + + +def test_auto_bisc_stops_when_a_list_or_a_tuple_is_too_short(): + good, bad = create_bisc_input(8, derangement_in_av_213_231_312) + result, printed = run(auto_bisc, listed(good)) + assert result is None + assert printed.endswith("You need to input a longer list of permutations\n") + result, printed = run(auto_bisc, (good, bad)) + assert result is None + assert printed.endswith("You need to add longer permutations to the dictionaries\n") + # a bad dictionary that stops at length 7 stops it, though the good one reaches 8 + short_bad = {length: perms for length, perms in bad.items() if length < 8} + result, printed = run(auto_bisc, (good, short_bad)) + assert result is None + assert printed.endswith("You need to add longer permutations to the dictionaries\n") + # one that stops at length 6 stops it at once: auto_bisc learns from the + # permutations up to length 7 before its sanity checks reach length 8 + shorter_bad = {length: perms for length, perms in bad.items() if length < 7} + assert run(auto_bisc, (good, shorter_bad)) == ( + None, + "You need to add longer permutations to the dictionaries\n", + ) + + +@pytest.mark.parametrize("short", ["list", "tuple"]) +def test_auto_bisc_wants_permutations_up_to_length_8(short): + good, bad = create_bisc_input(7, Perm.stack_sortable) + given = listed(good) if short == "list" else (good, bad) + assert run(auto_bisc, given) == ( + None, + "You should have permutations up to length at least 8\n", + ) + assert run(auto_bisc, []) == ( + None, + "You should have permutations up to length at least 8\n", + ) + assert run(auto_bisc, ({}, {})) == ( + None, + "You should have permutations up to length at least 8\n", + ) + + +def test_auto_bisc_of_a_stored_set_that_does_not_exist_or_runs_out(): + assert run(auto_bisc, "nosuchset") == ( + None, + "Attempting to read perms from permsets\nThe required files do not exist\n", + ) + # meander is stored to length 8: auto_bisc learns from perms of length 7, and + # asks for the files again, for length 9 + result, printed = run(auto_bisc, "meander") + assert result is None + assert printed.endswith( + "Attempting to read perms from permsets\nThe required files do not exist\n" + ) + assert printed.count("Attempting to read perms from permsets") == 2 + + +def test_auto_bisc_reads_a_stored_set_again_for_a_longer_length(monkeypatch): + # No stored set that auto_bisc can name is longer than 8 and learned from perms + # of length 8 or more, so the files are faked: the increasing permutations of + # even length are no class (the bad (0) avoids their basis, 21), so auto_bisc + # keeps growing n and reading + def cycle(perm): + return len(perm) % 2 == 0 and perm.is_increasing() + + reads = [] + + def fake_read_stored_set(name, min_length): + reads.append(min_length) + return ( + create_bisc_input(8, cycle) + if min_length <= 8 + else (create_bisc_input(9, cycle) if min_length == 9 else None) + ) + + monkeypatch.setattr(bisc_module, "_read_stored_set", fake_read_stored_set) + # the sanity check of the learned patterns must see the longer set it was read + # again for: the bad permutations of length L, where L is the length asked + real_check = bisc_module.patterns_suffice_for_bad + checks = [] + + def recording_check(patterns, length, bad, stop_on_failure=False): + checks.append((length, max(bad.keys()))) + return real_check(patterns, length, bad, stop_on_failure=stop_on_failure) + + monkeypatch.setattr(bisc_module, "patterns_suffice_for_bad", recording_check) + result, printed = run(auto_bisc, "cycles") + assert result is None + assert reads == [8, 9, 10] + assert (9, 9) in checks and all(longest >= length for length, longest in checks) + assert printed.endswith( + "Attempting to read perms from permsets\nThe required files do not exist\n" + ) + + +# Growth does not replace the bitmaps auto_bisc was given +class ReachedLength10(Exception): + pass + + +def test_auto_bisc_growth_does_not_label_a_length_it_was_given(): + labeled_lengths = [] + + def prop(perm): + if len(perm) == 10: + raise ReachedLength10 + labeled_lengths.append(len(perm)) + return derangement_in_av_213_231_312(perm) + + bitmaps = bitmaps_of(good_by_length(derangement_in_av_213_231_312, 8)) + # length 9: deliberately given as all bad, and not asked of the property + bitmaps[9] = bytes(45360) + property_bitmaps = PropertyBitmaps(prop, bitmaps=bitmaps) + try: + run(auto_bisc, property_bitmaps) + except ReachedLength10: + pass + assert 9 not in labeled_lengths + assert property_bitmaps.bitmaps[9] == bytes(45360) + + +# A permutation given twice on one side counts once, for auto_bisc too +def test_a_permutation_given_twice_counts_once_in_auto_bisc(): + good, bad = create_bisc_input(8, Perm.stack_sortable) + answer, printed = run(auto_bisc, Perm.stack_sortable) + twice = {**good, 5: good[5] + good[5][:3]} + assert run(auto_bisc, (twice, bad)) == (answer, printed) + assert run(auto_bisc, listed(twice)) == ( + answer, + "Populating the dictionary of bad perms of length 8\n" + printed, + ) + also_bad_twice = {**bad, 6: bad[6] + bad[6][:2]} + assert run(auto_bisc, (good, also_bad_twice)) == (answer, printed) + + +# A dictionary of the pair may lack a length it has no permutation of +def test_a_tuple_may_lack_a_length_with_no_bad_permutations(): + good, bad = create_bisc_input(8, Perm.stack_sortable) + # every permutation of length 0, 1 and 2 is stack-sortable: no bad ones + bad = {length: perms for length, perms in bad.items() if perms} + assert sorted(bad) == [3, 4, 5, 6, 7, 8] + assert auto_bisc((good, bad)) == BASIS_231 + + +# A bytearray given as a bitmap is copied +def test_a_bytearray_bitmap_is_copied(): + raw = bytearray(b"\x3f") + property_bitmaps = PropertyBitmaps(bitmaps={3: raw}) + raw[0] = 0 + assert len(property_bitmaps.good[3]) == 6 + assert len(list(property_bitmaps.good[3])) == 6 + + +# The views follow the PropertyBitmaps as lengths are added +def test_the_views_follow_added_lengths(): + property_bitmaps = PropertyBitmaps(Perm.stack_sortable) + good, bad = property_bitmaps.good, property_bitmaps.bad + assert 4 not in good and len(good) == 0 + property_bitmaps.add_length(4) + assert 4 in good and 4 in bad and list(good) == [4] + assert len(good[4]) == 14 and len(bad[4]) == 10 + + +# add_length replaces a bitmap it was given +def test_add_length_replaces_the_bitmap_of_a_length(): + property_bitmaps = PropertyBitmaps(Perm.stack_sortable, bitmaps={3: bytes(1)}) + assert len(property_bitmaps.good[3]) == 0 + property_bitmaps.add_length(3) + assert len(property_bitmaps.good[3]) == 5 and len(property_bitmaps.bad[3]) == 1 + assert list(property_bitmaps.bad[3]) == [Perm((1, 2, 0))] + + +# A PropertyBitmaps with only a property is labeled up to length 8 at the start +class ReachedLength9(Exception): + pass + + +def test_auto_bisc_of_a_property_only_input_labels_the_lengths_up_to_8(): + def prop(perm): + if len(perm) > 8: + raise ReachedLength9 + return perm.stack_sortable() + + property_bitmaps = PropertyBitmaps(prop) + assert auto_bisc(property_bitmaps) == BASIS_231 + assert sorted(property_bitmaps.bitmaps) == list(range(9)) + + +# The tuple, the list and the bitmaps without a property stop at the same point +def test_auto_bisc_stops_at_the_same_point_for_each_input_kind(): + good, bad = create_bisc_input(8, derangement_in_av_213_231_312) + bitmaps_message = "You need to add bitmaps of longer permutations\n" + bitmaps_printed = run(auto_bisc, PropertyBitmaps(bitmaps=bitmaps_of(good)))[1] + assert bitmaps_printed.endswith(bitmaps_message) + body = bitmaps_printed[: -len(bitmaps_message)] + result, printed = run(auto_bisc, (good, bad)) + assert result is None + assert printed == body + "You need to add longer permutations to the dictionaries\n" + result, printed = run(auto_bisc, listed(good)) + assert result is None + assert printed == ( + "Populating the dictionary of bad perms of length 8\n" + + body + + "You need to input a longer list of permutations\n" + ) + + +# A view unranks when at most one in 16 is wanted, on both sides of the edge +@pytest.mark.parametrize("good", [True, False]) +def test_a_view_unranks_exactly_when_at_most_one_in_16_is_wanted(monkeypatch, good): + unranked = [] + real_unrank = Perm.unrank + + def counting_unrank(index, length): + unranked.append(index) + return real_unrank(index, length) + + monkeypatch.setattr(Perm, "unrank", staticmethod(counting_unrank)) + # length 6 has 720 permutations: 45 is one in 16, 46 is more + for wanted, expect_unranking in ((45, True), (46, False)): + label_bits = (1 << wanted) - 1 # the first `wanted` permutations + if not good: + label_bits = ((1 << 720) - 1) ^ label_bits + view = PropertyBitmaps(bitmaps={6: label_bits.to_bytes(90, "little")}) + view = view.good[6] if good else view.bad[6] + unranked.clear() + assert len(list(view)) == wanted + assert bool(unranked) == expect_unranking diff --git a/tests/bisc/test_property_bitmaps.py b/tests/bisc/test_property_bitmaps.py new file mode 100644 index 0000000..b27cbed --- /dev/null +++ b/tests/bisc/test_property_bitmaps.py @@ -0,0 +1,476 @@ +import contextlib +import importlib +import io +import os +from collections.abc import Collection, Mapping +from math import factorial + +import pytest + +from permuta.bisc.bisc import ( + PropertyBitmaps, + auto_bisc, + bisc, + create_bisc_input, + read_bisc_file, +) +from permuta.bisc.perm_properties import smooth +from permuta.patterns.perm import Perm + +BASIS_231 = {3: {Perm((1, 2, 0)): [set()]}} +BASIS_WEST_2 = {4: {Perm((1, 2, 3, 0)): [set()], Perm((2, 1, 3, 0)): [{(1, 4)}]}} +# The module, which permuta.bisc's function bisc hides, and its mine() +bisc_module = importlib.import_module("permuta.bisc.bisc") +REAL_MINE = bisc_module.mine + + +def property_bitmaps_up_to(prop, max_length): + property_bitmaps = PropertyBitmaps(prop) + for length in range(max_length + 1): + property_bitmaps.add_length(length) + return property_bitmaps + + +def test_views_match_lists(): + property_bitmaps = property_bitmaps_up_to(Perm.stack_sortable, 6) + for length in range(7): + good = [perm for perm in Perm.of_length(length) if perm.stack_sortable()] + bad = [perm for perm in Perm.of_length(length) if not perm.stack_sortable()] + assert list(property_bitmaps.good[length]) == good + assert list(property_bitmaps.bad[length]) == bad + assert len(property_bitmaps.good[length]) == len(good) + assert len(property_bitmaps.bad[length]) == len(bad) + assert bool(property_bitmaps.bad[length]) == bool(bad) + assert sorted(property_bitmaps.good.keys()) == list(range(7)) + assert 6 in property_bitmaps.bad and 7 not in property_bitmaps.bad + with pytest.raises(KeyError, match="7"): + _ = property_bitmaps.good[7] + + +def test_views_are_a_mapping_of_collections(): + property_bitmaps = property_bitmaps_up_to(Perm.stack_sortable, 6) + for view in (property_bitmaps.good, property_bitmaps.bad): + assert isinstance(view, Mapping) + assert list(view) == list(range(7)) and len(view) == 7 + assert view.get(7) is None + assert all(isinstance(view[length], Collection) for length in view) + for length in range(7): + good, bad = property_bitmaps.good[length], property_bitmaps.bad[length] + for perm in Perm.of_length(length): + assert (perm in good) == perm.stack_sortable() + assert (perm in bad) == (not perm.stack_sortable()) + # A permutation of another length is in neither view of a length + assert Perm((0, 1)) not in property_bitmaps.good[3] + assert Perm((0, 1, 2, 3)) not in property_bitmaps.bad[3] + # Any sequence that is a permutation of the length is read as one, and + # nothing else is in a view + assert (0, 1, 2) in property_bitmaps.good[3] and [1, 2, 0] in property_bitmaps.bad[ + 3 + ] + for not_a_perm_of_3 in (Perm((1, 2, 3)), Perm((5, 5, 5)), (0, 0, 1), "012", 3): + assert not_a_perm_of_3 not in property_bitmaps.good[3] + assert not_a_perm_of_3 not in property_bitmaps.bad[3] + + +def test_views_unrank_a_length_with_few_wanted(): + # A length where at most one permutation in 16 is wanted is read by + # unranking those, in the order of Perm.of_length: the good permutations at + # length 8 here (1430 of 40320), and every bad one of a set of all but the + # identity; length 7 (429 good of 5040) is read by generating the length + property_bitmaps = property_bitmaps_up_to(Perm.stack_sortable, 8) + for length in (7, 8): + good = [perm for perm in Perm.of_length(length) if perm.stack_sortable()] + assert list(property_bitmaps.good[length]) == good + not_increasing = property_bitmaps_up_to(lambda perm: not perm.is_increasing(), 6) + for length in range(7): + assert list(not_increasing.bad[length]) == [Perm(range(length))] + assert list(not_increasing.good[length]) == list(Perm.of_length(length))[1:] + # The first eight permutations of length 6 bad, a byte of zeros + first_eight_bad = PropertyBitmaps(bitmaps={6: b"\x00" + b"\xff" * 89}) + assert list(first_eight_bad.bad[6]) == list(Perm.of_length(6))[:8] + # No bad permutation of length 3: the spare bits of its byte are not read + assert list(PropertyBitmaps(bitmaps={3: b"\x3f"}).bad[3]) == [] + + +def test_views_unrank_a_byte_of_good_permutations(): + # The first eight permutations of length 6 good, a byte of ones: 8 of 720, + # read by unranking, and the byte must not be skipped + first_eight_good = PropertyBitmaps(bitmaps={6: b"\xff" + b"\x00" * 89}) + assert list(first_eight_good.good[6]) == list(Perm.of_length(6))[:8] + # A property that holds on a lexicographic block: the 120 permutations of + # length 7 that start with 0, 1 fill fifteen bytes + starts_0_1 = property_bitmaps_up_to( + lambda perm: len(perm) < 2 or (perm[0] == 0 and perm[1] == 1), 7 + ) + assert list(starts_0_1.good[7]) == [ + perm for perm in Perm.of_length(7) if perm[0] == 0 and perm[1] == 1 + ] + + +def test_bit_order_is_perm_unrank(): + # Bit i of a length's bitmap is the label of Perm.unrank(i, length), and the + # permutation perm is at bit perm.rank() minus the number of shorter + # permutations + property_bitmaps = property_bitmaps_up_to(Perm.stack_sortable, 6) + shorter_count = 0 + for length in range(7): + bitmap = property_bitmaps.bitmaps[length] + for i in range(factorial(length)): + perm = Perm.unrank(i, length) + assert perm.rank() - shorter_count == i + assert bool(bitmap[i >> 3] >> (i & 7) & 1) == perm.stack_sortable() + shorter_count += factorial(length) + + +def test_bitmaps_round_trip(): + property_bitmaps = property_bitmaps_up_to(Perm.stack_sortable, 6) + round_trip = PropertyBitmaps(bitmaps=property_bitmaps.bitmaps) + for length in range(7): + assert list(round_trip.good[length]) == list(property_bitmaps.good[length]) + assert list(round_trip.bad[length]) == list(property_bitmaps.bad[length]) + # 3! bits fit in one byte + with pytest.raises(ValueError, match="length 3 has 2 bytes, and must have 1"): + PropertyBitmaps(bitmaps={3: b"\x00\x00"}) + with pytest.raises(ValueError, match="length 3 has 0 bytes, and must have 1"): + PropertyBitmaps(bitmaps={3: b""}) + + +def test_lengths_0_and_1_labeled_bad(): + property_bitmaps = PropertyBitmaps(bitmaps={0: b"\x00", 1: b"\x00"}) + for length in (0, 1): + assert list(property_bitmaps.good[length]) == [] + assert list(property_bitmaps.bad[length]) == [Perm(range(length))] + assert len(property_bitmaps.good[length]) == 0 + assert len(property_bitmaps.bad[length]) == 1 + + +@pytest.mark.parametrize( + "length, bitmap", + [(0, b"\x03"), (1, b"\x81"), (2, b"\x04"), (3, b"\xff"), (3, b"\x40")], +) +def test_set_padding_bits_are_rejected(length, bitmap): + with pytest.raises(ValueError, match="bits set past"): + PropertyBitmaps(bitmaps={length: bitmap}) + + +@pytest.mark.parametrize("length, bitmap", [(0, 1), (3, 1), (4, 3), (5, 15)]) +def test_an_int_is_not_a_bitmap(length, bitmap): + with pytest.raises(TypeError, match="must be bytes"): + PropertyBitmaps(bitmaps={length: bitmap}) + + +def test_bitmaps_may_be_any_bytes_like(): + # All six permutations of length 3 are good, and the first of length 2 + for bitmap in (b"\x3f", bytearray(b"\x3f"), memoryview(b"\x3f")): + property_bitmaps = PropertyBitmaps(bitmaps={3: bitmap, 2: b"\x01"}) + assert len(property_bitmaps.good[3]) == 6 and not property_bitmaps.bad[3] + assert list(property_bitmaps.good[2]) == [Perm((0, 1))] + assert isinstance(property_bitmaps.bitmaps[3], bytes) + + +def test_bisc_accepts_property_bitmaps(): + assert bisc(Perm.stack_sortable, 3, 5) == BASIS_231 + assert bisc(property_bitmaps_up_to(Perm.stack_sortable, 5), 3, 5) == BASIS_231 + + +def mined_bitmaps(monkeypatch, A, m, n=None): + """bisc's answer for A, and the bitmaps of the good permutations it gave + mine().""" + mined = {} + + def recording_mine(goodperms, M, N=None, report=False): + mined.update(goodperms.property_bitmaps.bitmaps) + return REAL_MINE(goodperms, M, N, report) + + monkeypatch.setattr(bisc_module, "mine", recording_mine) + return bisc(A, m, n), mined + + +def test_bisc_turns_each_input_into_bitmaps(monkeypatch): + bitmaps = {length: encode(Perm.stack_sortable, length) for length in range(6)} + good_lists = { + length: [perm for perm in Perm.of_length(length) if perm.stack_sortable()] + for length in range(6) + } + listed = [perm for length in range(6) for perm in good_lists[length]] + for A in ( + Perm.stack_sortable, + listed, + good_lists, + property_bitmaps_up_to(Perm.stack_sortable, 5), + ): + assert mined_bitmaps(monkeypatch, A, 3, 5) == (BASIS_231, bitmaps) + # A Mapping must have each length up to max(m, n) + with pytest.raises(KeyError): + bisc({length: good_lists[length] for length in (0, 1, 3)}, 3) + + +def test_a_list_sets_the_bits_of_the_permutations_it_lists(monkeypatch): + # A length with none listed is empty + _, mined = mined_bitmaps(monkeypatch, [Perm((0,)), Perm((0, 1, 2))], 2, 3) + assert mined == {0: b"\x00", 1: b"\x01", 2: b"\x00", 3: b"\x01"} + # A permutation listed twice sets one bit, so (0, 1) is not taken for both + # permutations of length 2, and the increasing permutations avoid 10 + increasing = [Perm(()), Perm((0,)), Perm((0, 1)), Perm((0, 1)), Perm((0, 1, 2))] + answer, mined = mined_bitmaps(monkeypatch, increasing, 2, 3) + assert mined == {0: b"\x01", 1: b"\x01", 2: b"\x01", 3: b"\x01"} + assert answer == {2: {Perm((1, 0)): [set()]}} + + +def test_a_property_leaves_the_lengths_past_n_empty(monkeypatch): + # mine() reads the lengths up to m, and a property labels those up to n; the + # others are read as empty + answer, mined = mined_bitmaps(monkeypatch, Perm.stack_sortable, 4, 2) + assert mined == { + **{length: encode(Perm.stack_sortable, length) for length in range(3)}, + 3: bytes(1), + 4: bytes(3), + } + assert answer == {3: {perm: [set()] for perm in Perm.of_length(3)}, 4: {}} + + +def first_mined_bitmaps(monkeypatch, prop): + """auto_bisc's answer for prop, and the bitmaps of the good permutations it + first gave mine().""" + mined = [] + + def recording_mine(goodperms, M, N=None, report=False): + mined.append(dict(goodperms.property_bitmaps.bitmaps)) + return REAL_MINE(goodperms, M, N, report) + + monkeypatch.setattr(bisc_module, "mine", recording_mine) + return auto_bisc(prop), mined[0] + + +def test_auto_bisc_turns_each_input_into_bitmaps(monkeypatch): + bitmaps = {length: encode(Perm.stack_sortable, length) for length in range(9)} + good, bad = create_bisc_input(8, Perm.stack_sortable) + listed = [perm for length in range(9) for perm in good[length]] + for prop in ( + Perm.stack_sortable, + listed, + (good, bad), + PropertyBitmaps(bitmaps=bitmaps), + ): + assert first_mined_bitmaps(monkeypatch, prop) == (BASIS_231, bitmaps) + answer, mined = first_mined_bitmaps(monkeypatch, "smooth") + assert answer == {4: {Perm((0, 2, 1, 3)): [set()], Perm((1, 0, 3, 2)): [set()]}} + assert mined == {length: encode(smooth, length) for length in range(9)} + + +def test_auto_bisc_labels_each_length_of_a_property(monkeypatch): + added_lengths = [] + real_add_length = PropertyBitmaps.add_length + + def recording_add_length(self, length): + added_lengths.append(length) + return real_add_length(self, length) + + monkeypatch.setattr(PropertyBitmaps, "add_length", recording_add_length) + assert auto_bisc(Perm.stack_sortable) == BASIS_231 + assert added_lengths == list(range(9)) + + +def test_the_good_and_bad_of_a_tuple_must_split_each_length(): + good, bad = create_bisc_input(8, Perm.stack_sortable) + missing_one = {**bad, 5: bad[5][1:]} + with pytest.raises(ValueError, match=r"length 5 .* \(in both: 0, in neither: 1\)"): + auto_bisc((good, missing_one)) + in_both = {**bad, 4: bad[4] + good[4][:1]} + with pytest.raises(ValueError, match=r"length 4 .* \(in both: 1, in neither: 0\)"): + auto_bisc((good, in_both)) + + +def test_the_stored_sets_give_each_permutation_once(): + folder = "permuta/resources/bisc/" + for file_name in sorted(os.listdir(folder)): + if "_good_len" in file_name: + good_name = file_name[: -len(".json")] + good = read_bisc_file(folder + good_name) + bad = read_bisc_file(folder + good_name.replace("_good_", "_bad_")) + property_bitmaps = bisc_module._from_good_and_bad(good, bad) + longest_length = int(good_name.rsplit("len", 1)[1]) + assert sorted(property_bitmaps.bitmaps) == list(range(longest_length + 1)) + + +def test_auto_bisc_property_bitmaps_input(): + assert auto_bisc(property_bitmaps_up_to(Perm.stack_sortable, 8)) == BASIS_231 + # Given bitmaps and no property: the lengths auto_bisc needs must be there + bitmaps = property_bitmaps_up_to(Perm.stack_sortable, 8).bitmaps + assert auto_bisc(PropertyBitmaps(bitmaps=bitmaps)) == BASIS_231 + too_short = {length: bitmaps[length] for length in range(6)} + assert auto_bisc(PropertyBitmaps(bitmaps=too_short)) is None + + +def encode(prop, length): + """A length's bitmap made without PropertyBitmaps: bit i of the little-endian + integer is the label of Perm.unrank(i, length).""" + label_bits = sum( + 1 << i for i in range(factorial(length)) if prop(Perm.unrank(i, length)) + ) + return label_bits.to_bytes((factorial(length) + 7) // 8, "little") + + +def test_auto_bisc_labels_only_missing_lengths_with_the_property(): + labeled_lengths = [] + + def prop(perm): + labeled_lengths.append(len(perm)) + return perm.stack_sortable() + + given_lengths = (0, 1, 2, 3, 5, 8) + property_bitmaps = PropertyBitmaps( + prop, + bitmaps={ + length: encode(Perm.stack_sortable, length) for length in given_lengths + }, + ) + assert auto_bisc(property_bitmaps) == BASIS_231 + assert set(labeled_lengths) == {4, 6, 7} + assert sorted(property_bitmaps.bitmaps) == list(range(9)) + + +def test_bisc_labels_missing_lengths_with_the_property(): + # Without n, bisc uses the longest length labeled, or 7 when there is none + assert bisc(property_bitmaps_up_to(Perm.stack_sortable, 5), 3) == BASIS_231 + property_bitmaps = PropertyBitmaps(Perm.stack_sortable) + assert bisc(property_bitmaps, 3, 5) == BASIS_231 + assert sorted(property_bitmaps.bitmaps) == list(range(6)) + property_bitmaps = PropertyBitmaps(Perm.stack_sortable) + assert bisc(property_bitmaps, 3) == BASIS_231 + assert sorted(property_bitmaps.bitmaps) == list(range(8)) + + +def test_bisc_needs_bitmaps_or_a_property(): + bitmaps = {length: encode(Perm.stack_sortable, length) for length in range(4)} + with pytest.raises(ValueError, match="none of length 4 and no property"): + bisc(PropertyBitmaps(bitmaps=bitmaps), 3, 5) + with pytest.raises(ValueError, match="no bitmaps and no property"): + bisc(PropertyBitmaps(), 3) + + +def test_add_length_needs_a_property(): + property_bitmaps = PropertyBitmaps(bitmaps={0: b"\x01"}) + with pytest.raises(ValueError, match="no property to label length 3 with"): + property_bitmaps.add_length(3) + assert sorted(property_bitmaps.bitmaps) == [0] + + +def test_bisc_without_n_uses_the_longest_labeled_length_with_no_property(): + bitmaps = {length: encode(Perm.stack_sortable, length) for length in range(6)} + assert bisc(PropertyBitmaps(bitmaps=bitmaps), 3) == BASIS_231 + + +def test_bisc_labels_only_the_missing_lengths(): + labeled_lengths = [] + + def prop(perm): + labeled_lengths.append(len(perm)) + return perm.stack_sortable() + + given_lengths = (0, 1, 2, 3, 5) + property_bitmaps = PropertyBitmaps( + prop, + bitmaps={ + length: encode(Perm.stack_sortable, length) for length in given_lengths + }, + ) + assert bisc(property_bitmaps, 3) == BASIS_231 + assert labeled_lengths == [4] * factorial(4) + assert sorted(property_bitmaps.bitmaps) == list(range(6)) + assert bisc(property_bitmaps, 3, 5) == BASIS_231 + assert labeled_lengths == [4] * factorial(4) + + +def test_bisc_labels_up_to_the_pattern_length(): + # mine() reads the lengths up to m, so bisc labels up to m when n < m + property_bitmaps = PropertyBitmaps(Perm.stack_sortable) + bisc(property_bitmaps, 3, 0) + assert sorted(property_bitmaps.bitmaps) == [0, 1, 2, 3] + property_bitmaps = PropertyBitmaps(Perm.stack_sortable, bitmaps={0: b"\x01"}) + bisc(property_bitmaps, 3) + assert sorted(property_bitmaps.bitmaps) == [0, 1, 2, 3] + with pytest.raises(ValueError, match="none of length 1 and no property"): + bisc(PropertyBitmaps(bitmaps={0: b"\x01"}), 3) + + +def test_bisc_accepts_the_views(): + assert bisc(property_bitmaps_up_to(Perm.stack_sortable, 5).good, 3) == BASIS_231 + + +def test_west_2_stack_sortable(): + assert auto_bisc(Perm.west_2_stack_sortable) == BASIS_WEST_2 + + +def derangement_in_av_213_231_312(perm): + """One member at each even length, none at odd: auto_bisc keeps learning longer + patterns while each step stays cheap, so it soon needs more labels.""" + return perm.avoids(Perm((1, 0, 2)), Perm((1, 2, 0)), Perm((2, 0, 1))) and all( + value != index for index, value in enumerate(perm) + ) + + +def test_auto_bisc_stops_without_bitmaps_it_needs(): + # Bitmaps up to length 8 and no property: auto_bisc needs length 9 and stops + bitmaps = property_bitmaps_up_to(derangement_in_av_213_231_312, 8).bitmaps + output = io.StringIO() + with contextlib.redirect_stdout(output): + assert auto_bisc(PropertyBitmaps(bitmaps=bitmaps)) is None + assert "You need to add bitmaps of longer permutations" in output.getvalue() + + +class ReachedLength9(Exception): + pass + + +def test_auto_bisc_labels_new_lengths_from_the_property(): + # When auto_bisc grows past the labeled lengths, it labels the new ones with + # the property: here it is stopped at the first permutation of length 9 + def prop(perm): + if len(perm) == 9: + raise ReachedLength9 + return derangement_in_av_213_231_312(perm) + + output = io.StringIO() + with contextlib.redirect_stdout(output), pytest.raises(ReachedLength9): + auto_bisc(prop) + assert "Adding perms of length 9" in output.getvalue() + + +def test_a_list_of_tuples_is_read_as_permutations(): + # Tuples are read as the permutations they are, as they always were + perms = [ + perm + for length in range(6) + for perm in Perm.of_length(length) + if perm.stack_sortable() + ] + assert bisc([tuple(perm) for perm in perms], 3, 5) == bisc(perms, 3, 5) + + +def test_a_permutation_under_the_wrong_length_is_refused(): + with pytest.raises(ValueError, match="length 3 is given as one of length 2"): + bisc({0: [Perm(())], 1: [Perm((0,))], 2: [Perm((0, 1, 2))]}, 2) + + +def test_a_tuple_may_lack_a_length_in_one_dictionary(monkeypatch): + # A length one dictionary lacks is empty in it: here no permutation of + # length 0 is good + good, bad = create_bisc_input(8, Perm.stack_sortable) + good = {length: perms for length, perms in good.items() if perms} + bad = dict(bad) + bad[0] = bad.get(0, []) + good.pop(0, []) + given = [] + + class MineCalled(Exception): + pass + + def recording_mine(goodperms, M, N=None, report=False): + given.append({length: list(goodperms[length]) for length in goodperms}) + raise MineCalled + + monkeypatch.setattr(bisc_module, "mine", recording_mine) + with pytest.raises(MineCalled): + auto_bisc((good, bad)) + assert given[0][0] == [] + assert given[0][3] == [perm for perm in Perm.of_length(3) if perm.stack_sortable()] diff --git a/tests/bisc/test_sparse_storage.py b/tests/bisc/test_sparse_storage.py new file mode 100644 index 0000000..219a576 --- /dev/null +++ b/tests/bisc/test_sparse_storage.py @@ -0,0 +1,898 @@ +"""The two forms in which a PropertyBitmaps keeps a length's labels: its bitmap, +or the sorted indices of its good permutations when fewer than one in +_SPARSE_WHEN_ONE_IN is good. Each way in chooses the form by the number of good +permutations, and every view reads both forms.""" + +import importlib +import itertools +import random +import sys +import weakref +from array import array +from math import factorial + +import pytest + +from permuta.bisc.bisc import PropertyBitmaps, bisc +from permuta.patterns.perm import Perm + +# The module, which permuta.bisc's function bisc hides +bisc_module = importlib.import_module("permuta.bisc.bisc") + +MAX_LENGTH = 8 + + +def expected_bitmap(length, good_set): + """A length's bitmap with the bits of the permutations in good_set set, made + without PropertyBitmaps.""" + label_bits = sum( + 1 << index + for index, perm in enumerate(Perm.of_length(length)) + if perm in good_set + ) + return label_bits.to_bytes((factorial(length) + 7) // 8, "little") + + +def is_kept_as_indices(property_bitmaps, length): + """Whether a length's labels are its good permutations' indices, in the + container for its length: an array to length 20, and a list past it. A list + where the array is meant takes five times the memory.""" + kept_in = array if factorial(length) <= 2**64 else list + return type(property_bitmaps._labels[length]) is kept_in + + +def assert_views_match(property_bitmaps, length, good_set, bitmap=None): + """Every view of a length against brute force: iteration and its order, + len(), in, the bad view, the count and the bitmap.""" + perms = list(Perm.of_length(length)) + good = [perm for perm in perms if perm in good_set] + bad = [perm for perm in perms if perm not in good_set] + good_view, bad_view = property_bitmaps.good[length], property_bitmaps.bad[length] + assert list(good_view) == good + assert list(bad_view) == bad + assert len(good_view) == len(good) and len(bad_view) == len(bad) + # in, for every permutation up to length 6, and for every seventh good and + # bad one at lengths 7 and 8 + step = 1 if length < 7 else 7 + for perm in good[::step] + bad[::step]: + assert (perm in good_view) == (perm in good_set) + assert (perm in bad_view) == (perm not in good_set) + assert property_bitmaps.good_counts[length] == len(good) + if bitmap is None: + bitmap = expected_bitmap(length, good_set) + assert property_bitmaps.bitmaps[length] == bitmap + + +def guard_bitmap_sizes(monkeypatch): + """Make sizing the bitmap of a length past 10 fail the test at once, rather + than allocate it: length 14's would take 10.9 GB. A check of the code's + structure, so that it runs on PyPy too, which has no tracemalloc.""" + real_bitmap_size = bisc_module._bitmap_size + + def guarded_bitmap_size(length): + assert length <= 10, "a bitmap of length {} was sized".format(length) + return real_bitmap_size(length) + + monkeypatch.setattr(bisc_module, "_bitmap_size", guarded_bitmap_size) + + +def each_way_in(good_sets, bitmaps, seeded_random): + """A PropertyBitmaps made from good_sets in each way that bisc and auto_bisc + make one, with its name: a list or a mapping (whose good permutations come + shuffled, with some twice), a tuple of the good and the bad, bitmaps, and a + property.""" + lengths = range(MAX_LENGTH + 1) + good_lists, bad_lists = {}, {} + for length in lengths: + good_lists[length] = sorted(good_sets[length]) + seeded_random.shuffle(good_lists[length]) + bad_lists[length] = [ + perm for perm in Perm.of_length(length) if perm not in good_sets[length] + ] + listed_twice = { + length: good_lists[length] + good_lists[length][:3] for length in lengths + } + yield "list or mapping", bisc_module._from_perms_by_length(listed_twice) + yield "tuple", bisc_module._from_good_and_bad(good_lists, bad_lists) + yield "bitmaps", PropertyBitmaps(bitmaps=bitmaps) + by_property = PropertyBitmaps(lambda perm: perm in good_sets[len(perm)]) + for length in lengths: + by_property.add_length(length) + yield "property", by_property + + +def fewest_kept_as_a_bitmap(length, sparse_when_one_in): + """The fewest good permutations of a length that are kept as a bitmap.""" + return -(-factorial(length) // sparse_when_one_in) + + +SIZE_KINDS = ( + "none", + "just under the threshold", + "at the threshold", + "all", + "random under", + "random over", +) + + +def good_count_of(size_kind, length, sparse_when_one_in, seeded_random): + fewest_in_a_bitmap = min( + fewest_kept_as_a_bitmap(length, sparse_when_one_in), factorial(length) + ) + return { + "none": 0, + "just under the threshold": max(fewest_in_a_bitmap - 1, 0), + "at the threshold": fewest_in_a_bitmap, + "all": factorial(length), + "random under": seeded_random.randint(0, max(fewest_in_a_bitmap - 1, 0)), + "random over": seeded_random.randint(fewest_in_a_bitmap, factorial(length)), + }[size_kind] + + +@pytest.mark.parametrize("sparse_when_one_in", [2, 64]) +@pytest.mark.parametrize("size_kind", SIZE_KINDS) +def test_both_forms_read_as_brute_force(monkeypatch, sparse_when_one_in, size_kind): + """Both forms against brute force, for every view and every way in, at + lengths 0 to 8. A set with one good permutation is kept as indices from + length 3 on with one in 2, and from length 5 on with one in 64, the + default. The bitmaps are read seven bytes at a time, so that counting them + takes several chunks.""" + monkeypatch.setattr(bisc_module, "_SPARSE_WHEN_ONE_IN", sparse_when_one_in) + monkeypatch.setattr(bisc_module, "_CHUNK_BYTES", 7) + seeded_random = random.Random(f"{sparse_when_one_in} {size_kind}") + good_sets, bitmaps = {}, {} + for length in range(MAX_LENGTH + 1): + good_count = good_count_of(size_kind, length, sparse_when_one_in, seeded_random) + perms = list(Perm.of_length(length)) + good_sets[length] = set(seeded_random.sample(perms, good_count)) + bitmaps[length] = expected_bitmap(length, good_sets[length]) + for way, property_bitmaps in each_way_in(good_sets, bitmaps, seeded_random): + for length in range(MAX_LENGTH + 1): + good_count = len(good_sets[length]) + kept_as_indices = good_count * sparse_when_one_in < factorial(length) + assert is_kept_as_indices(property_bitmaps, length) == kept_as_indices, ( + way, + length, + ) + assert_views_match( + property_bitmaps, length, good_sets[length], bitmaps[length] + ) + + +@pytest.mark.parametrize( + "good_indices", + [ + range(5040), # all, the switch at the 79th + range(4800, 5040), # good from a point on, the switch late in the length + range(4961, 5040), # 79 good, the switch at the last permutation + range(4962, 5040), # 78 good, kept as indices + range(0, 5040, 50), # 101 spread out, the switch at index 3900 + range(5039, 5040), # one good + range(0), # none + ], +) +def test_add_length_switches_to_a_bitmap_when_the_good_become_many(good_indices): + """add_length keeps the indices while they are few, and switches to a bitmap + in the middle of a length once there are more than a length keeps as + indices. Length 7 has 5040 permutations, so its 79th good one is one too + many.""" + length = 7 + perms = list(Perm.of_length(length)) + good_set = {perms[index] for index in good_indices} + property_bitmaps = PropertyBitmaps(lambda perm: perm in good_set) + property_bitmaps.add_length(length) + kept_as_indices = len(good_set) * 64 < factorial(length) + assert is_kept_as_indices(property_bitmaps, length) == kept_as_indices + assert_views_match(property_bitmaps, length, good_set) + + +@pytest.mark.skipif( + sys.implementation.name != "cpython", reason="frees at once by reference count" +) +def test_add_length_drops_its_indices_once_it_has_the_bitmap(monkeypatch): + """After add_length switches to a bitmap its indices are not kept, so that it + holds a bitmap and not a bitmap and the indices; and the bitmap is filled in + with the rest of the length. The indices are an array, which can be weakly + referenced.""" + indices_given = [] + real_bitmap_from_indices = bisc_module._bitmap_from_indices + + def recording_bitmap_from_indices(length, indices): + indices_given.append(weakref.ref(indices)) + return real_bitmap_from_indices(length, indices) + + monkeypatch.setattr( + bisc_module, "_bitmap_from_indices", recording_bitmap_from_indices + ) + still_held_while_labeling = [] + + def prop(perm): + if indices_given: + still_held_while_labeling.append(indices_given[0]() is not None) + return True + + property_bitmaps = PropertyBitmaps(prop) + property_bitmaps.add_length(7) + assert len(indices_given) == 1 + # All 5040 good: the switch at the 79th, and the rest labeled in the bitmap + assert still_held_while_labeling == [False] * (5040 - 79) + + +@pytest.mark.parametrize("which", ["first", "last", "first and last"]) +def test_add_length_when_only_the_first_or_the_last_permutations_are_good(which): + """A property true only on the first permutation of a length, only on the + last, or on both, at lengths 0 to 8: the last permutation is where add_length + switches to a bitmap, or ends the loop, at the very end of a length.""" + for length in range(9): + perms = list(Perm.of_length(length)) + good_set = { + "first": {perms[0]}, + "last": {perms[-1]}, + "first and last": {perms[0], perms[-1]}, + }[which] + property_bitmaps = PropertyBitmaps(lambda perm: perm in good_set) + property_bitmaps.add_length(length) + kept_as_indices = len(good_set) * 64 < factorial(length) + assert is_kept_as_indices(property_bitmaps, length) == kept_as_indices, length + assert_views_match(property_bitmaps, length, good_set) + + +def test_a_permutation_listed_many_times_is_kept_as_one_index(): + """A permutation listed many times counts once, for the form too: five + copies of one permutation of length 5 look like 5 of its 120, which a bitmap + would keep at one in 64, but they are one good permutation.""" + perm = Perm((1, 0, 2, 3, 4)) + property_bitmaps = bisc_module._from_perms_by_length({5: [perm] * 5}) + assert is_kept_as_indices(property_bitmaps, 5) + assert_views_match(property_bitmaps, 5, {perm}) + + +def tuple_of(good_of_length_8, bad_of_length_8): + """A tuple of the good and the bad permutations: every permutation shorter + than 8 good, and those given at length 8.""" + good = {length: list(Perm.of_length(length)) for length in range(8)} + bad = {length: [] for length in range(8)} + return {**good, 8: good_of_length_8}, {**bad, 8: bad_of_length_8} + + +@pytest.mark.parametrize( + "good_of_length_8, bad_of_length_8, in_both, in_neither", + [ + (slice(0, 5), slice(3, None), 2, 0), # indices, bitmap + (slice(0, 40000), slice(39990, 40010), 10, 310), # bitmap, indices + (slice(0, 20000), slice(19990, None), 10, 0), # bitmap, bitmap + (slice(0, 5), slice(3, 8), 2, 40312), # indices, indices + ], +) +def test_a_tuple_is_checked_in_every_pair_of_forms( + monkeypatch, good_of_length_8, bad_of_length_8, in_both, in_neither +): + """The good and the bad permutations of a tuple are checked in every pair of + forms: the good kept as indices or as a bitmap, and the bad as well.""" + monkeypatch.setattr(bisc_module, "_CHUNK_BYTES", 7) + perms = list(Perm.of_length(8)) + good, bad = tuple_of(perms[good_of_length_8], perms[bad_of_length_8]) + with pytest.raises( + ValueError, + match=r"length 8 .* \(in both: {}, in neither: {}\)".format( + in_both, in_neither + ), + ): + bisc_module._from_good_and_bad(good, bad) + + +def test_the_overlap_of_two_lists_of_indices(): + """The overlap of labels kept as lists, which only a length past 20 has, and + that of a list and a bitmap, either way round.""" + assert bisc_module._overlap_count([1, 5, 2**70], [5, 2**70, 2**71]) == 2 + assert bisc_module._overlap_count([], [5, 2**70]) == 0 + bitmap = bytes([0b00100010]) # the bits 1 and 5 + assert bisc_module._overlap_count([1, 5, 6], bitmap) == 2 + assert bisc_module._overlap_count(bitmap, [1, 5, 6]) == 2 + + +def test_a_length_kept_as_indices_builds_no_bitmap_unless_asked(monkeypatch): + """Asking whether a length is there, or reading its good permutations, does + not build the bitmap of a length kept as indices.""" + identity = Perm(range(10)) + property_bitmaps = bisc_module._from_perms_by_length({10: [identity]}) + built = [] + real_bitmap_from_indices = bisc_module._bitmap_from_indices + + def recording_bitmap_from_indices(length, indices): + built.append(length) + return real_bitmap_from_indices(length, indices) + + monkeypatch.setattr( + bisc_module, "_bitmap_from_indices", recording_bitmap_from_indices + ) + assert 10 in property_bitmaps.bitmaps and 9 not in property_bitmaps.bitmaps + assert list(property_bitmaps.bitmaps) == [10] and len(property_bitmaps.bitmaps) == 1 + assert list(property_bitmaps.good[10]) == [identity] + assert identity in property_bitmaps.good[10] + assert Perm(range(9, -1, -1)) in property_bitmaps.bad[10] + assert len(property_bitmaps.bad[10]) == factorial(10) - 1 + assert built == [] + assert property_bitmaps.bitmaps[10] == b"\x01" + bytes(453599) + assert built == [10] + + +THREE_OF_LENGTH_14 = [ + Perm((1, 9, 11, 10, 8, 3, 4, 5, 2, 6, 0, 7, 13, 12)), + Perm((7, 10, 6, 11, 1, 4, 3, 12, 2, 5, 8, 9, 0, 13)), + Perm((8, 10, 12, 4, 2, 0, 1, 13, 11, 9, 6, 3, 7, 5)), +] +# bisc(THREE_OF_LENGTH_14, 2, 14) on develop, before PropertyBitmaps +DEVELOP_ANSWER_14 = { + 0: {Perm(()): [{(0, 0)}]}, + 1: {Perm((0,)): [{(0, 1), (1, 0), (0, 0)}, {(1, 1), (0, 0)}]}, + 2: { + Perm((0, 1)): [ + {(0, 1), (0, 2), (1, 2), (2, 1), (0, 0), (1, 1)}, + {(0, 1), (0, 2), (1, 2), (2, 1), (1, 0)}, + {(0, 1), (1, 2), (2, 2), (0, 0), (1, 1)}, + {(0, 1), (0, 2), (0, 0), (1, 1), (2, 0)}, + {(0, 1), (1, 0), (0, 2), (2, 0)}, + {(0, 1), (1, 0), (2, 2)}, + ], + Perm((1, 0)): [ + {(0, 1), (1, 2), (2, 1), (0, 0), (1, 0)}, + {(0, 2), (1, 2), (0, 0), (1, 1), (2, 0)}, + {(0, 2), (1, 2), (2, 1), (0, 0), (1, 1)}, + {(0, 1), (0, 2), (1, 2), (0, 0)}, + {(2, 0), (1, 1), (2, 2), (0, 0)}, + {(1, 0), (2, 2), (0, 0)}, + {(1, 2), (2, 2), (0, 0)}, + ], + }, +} + + +def shadings_as_sets(answer): + """bisc's answer with each pattern's shadings as a set, whose order the + Python may change.""" + return { + length: { + pattern: {frozenset(shading) for shading in shadings} + for pattern, shadings in patterns.items() + } + for length, patterns in answer.items() + } + + +def test_three_permutations_of_length_14_take_little_memory(monkeypatch): + """Three permutations of length 14 run in megabytes: each length up to 14 is + kept as indices, where the bitmap of length 14 alone would take 10.9 GB. + tracemalloc is CPython's. A fault that would size a bitmap past length 10 + fails at once, rather than allocating it.""" + tracemalloc = pytest.importorskip("tracemalloc") + guard_bitmap_sizes(monkeypatch) + tracemalloc.start() + try: + answer = bisc(THREE_OF_LENGTH_14, 2, 14) + peak_bytes = tracemalloc.get_traced_memory()[1] + finally: + tracemalloc.stop() + assert peak_bytes < 2**20 + assert shadings_as_sets(answer) == shadings_as_sets(DEVELOP_ANSWER_14) + + +@pytest.mark.parametrize("length", [20, 21, 25]) +def test_the_last_permutation_of_a_long_length_can_be_listed(length): + """The decreasing permutation of a length, whose index is its largest. 21! is + above 2**64, so past length 20 the indices are kept in a list.""" + last = Perm(range(length - 1, -1, -1)) + property_bitmaps = bisc_module._from_perms_by_length({length: [last]}) + assert list(property_bitmaps.good[length]) == [last] + assert last in property_bitmaps.good[length] + assert last not in property_bitmaps.bad[length] + assert property_bitmaps.good_counts[length] == 1 + kept_in = array if length <= 20 else list + assert isinstance(property_bitmaps._labels[length], kept_in) + + +@pytest.mark.parametrize("length", [21, 25]) +def test_a_length_kept_in_a_list_counts_contains_and_iterates(length): + """The first, a middle and the last permutation of a length past 20, good or + not: len() and bool() of the good view, in and not in for both views, and + the good permutations in order. The bad view is true, and iterates from the + first bad permutation; its count is past sys.maxsize, so len() of it + raises OverflowError, and __len__() gives it.""" + first = Perm(range(length)) + last = Perm(range(length - 1, -1, -1)) + middle = Perm.unrank(factorial(length) // 3, length) + other = Perm.unrank(factorial(length) // 2, length) + property_bitmaps = bisc_module._from_perms_by_length( + {length: [last, middle, first]} + ) + good, bad = property_bitmaps.good[length], property_bitmaps.bad[length] + assert is_kept_as_indices(property_bitmaps, length) + assert len(good) == 3 and bool(good) + assert bad.__len__() == factorial(length) - 3 and bool(bad) + with pytest.raises(OverflowError): + len(bad) + assert list(good) == [first, middle, last] + assert list(itertools.islice(bad, 2)) == [ + Perm.unrank(1, length), + Perm.unrank(2, length), + ] + for perm in (first, middle, last): + assert perm in good and perm not in bad + assert other not in good and other in bad + assert Perm(range(length - 1)) not in good + assert Perm(range(length - 1)) not in bad + + +@pytest.mark.parametrize("length", [21, 25]) +def test_the_bitmap_of_a_length_kept_in_a_list_cannot_be_built(length): + """Past length 20 a bitmap would take more than 6 * 10**18 bytes: asking for + it raises, and does not give the list of indices. Asking whether the length + is there builds nothing.""" + last = Perm(range(length - 1, -1, -1)) + property_bitmaps = bisc_module._from_perms_by_length({length: [last]}) + assert length in property_bitmaps.bitmaps + with pytest.raises((MemoryError, OverflowError)): + property_bitmaps.bitmaps[length] + + +def test_a_permutation_of_length_21_whose_index_needs_the_top_bit(): + """An index of 2**63 or more, which a signed 64-bit integer would not hold, + kept in the list of a length past 20.""" + perm = Perm.unrank(2**63 + 5, 21) + assert perm.rank() - bisc_module._shorter_count(21) == 2**63 + 5 + property_bitmaps = bisc_module._from_perms_by_length({21: [perm]}) + assert list(property_bitmaps.good[21]) == [perm] + assert perm in property_bitmaps.good[21] + + +@pytest.mark.parametrize("length", [100, 200]) +def test_a_permutation_of_a_very_long_length_can_be_listed(length): + """The last permutation of length 100 or 200: n! / 64 is past what a float + holds from length 172 on, so the limit on the number of indices must be an + int.""" + last = Perm(range(length - 1, -1, -1)) + property_bitmaps = bisc_module._from_perms_by_length({length: [last]}) + assert is_kept_as_indices(property_bitmaps, length) + assert list(property_bitmaps.good[length]) == [last] + assert last in property_bitmaps.good[length] + assert last not in property_bitmaps.bad[length] + assert property_bitmaps.bad[length].__len__() == factorial(length) - 1 + + +def test_an_empty_list_past_length_20_is_read_as_empty(): + """No permutation of length 25 listed: the length is kept as no indices, and + both views, in, and bisc read it as empty. n = 25 reads the length too.""" + property_bitmaps = bisc_module._from_perms_by_length({25: []}) + assert property_bitmaps.good_counts[25] == 0 + good = property_bitmaps.good[25] + assert list(good) == [] and len(good) == 0 and not good + assert Perm(range(25)) not in good + assert Perm(range(25)) in property_bitmaps.bad[25] + assert bisc([], 3, 25) == {0: {Perm(()): [set()]}, 1: {}, 2: {}, 3: {}} + + +def test_the_bad_view_of_length_20_is_read_lazily(): + """Length 20 is the longest kept in an array, and its bad view can be read + only a few permutations at a time: with the first and the third permutation + good, the first three bad ones are the second, fourth and fifth.""" + first_five = list(itertools.islice(Perm.of_length(20), 5)) + property_bitmaps = bisc_module._from_perms_by_length( + {20: [first_five[0], first_five[2]]} + ) + assert is_kept_as_indices(property_bitmaps, 20) + bad = property_bitmaps.bad[20] + assert list(itertools.islice(bad, 3)) == [ + first_five[1], + first_five[3], + first_five[4], + ] + assert len(bad) == factorial(20) - 2 + + +# bisc([the decreasing permutation of length 21], 2, 21) on develop, before +# PropertyBitmaps, and the same at length 25 +DEVELOP_ANSWER_LONG = { + 0: {Perm(()): [{(0, 0)}]}, + 1: {Perm((0,)): [{(0, 1), (1, 0)}]}, + 2: {Perm((0, 1)): [set()], Perm((1, 0)): [{(0, 2), (1, 1), (2, 0)}]}, +} + + +@pytest.mark.parametrize("length", [21, 25]) +def test_bisc_of_one_long_permutation_is_as_before(length): + """bisc mines one listed permutation of length 21 or more, as it did before + PropertyBitmaps.""" + last = Perm(range(length - 1, -1, -1)) + answer = bisc([last], 2, length) + assert shadings_as_sets(answer) == shadings_as_sets(DEVELOP_ANSWER_LONG) + + +def test_a_dense_length_is_kept_as_bytes_and_read_without_a_copy(): + """A length kept as a bitmap keeps bytes, however it came in, so that + bitmaps gives the same object on every read.""" + length = 6 + good_set = { + perm for index, perm in enumerate(Perm.of_length(length)) if index % 3 == 0 + } + by_property = PropertyBitmaps(lambda perm: perm in good_set) + by_property.add_length(length) + from_list = bisc_module._from_perms_by_length({length: sorted(good_set)}) + given = PropertyBitmaps( + bitmaps={length: bytearray(expected_bitmap(length, good_set))} + ) + for property_bitmaps in (by_property, from_list, given): + assert type(property_bitmaps._labels[length]) is bytes + assert property_bitmaps.bitmaps[length] is property_bitmaps.bitmaps[length] + assert property_bitmaps.bitmaps[length] == expected_bitmap(length, good_set) + + +# 720 permutations: a set of them is kept as indices below 12 good +PERMS_OF_LENGTH_6 = list(Perm.of_length(6)) +FOUR_OF_LENGTH_6 = {PERMS_OF_LENGTH_6[index] for index in (0, 5, 100, 719)} +EVERY_THIRD_OF_LENGTH_6 = { + perm for index, perm in enumerate(PERMS_OF_LENGTH_6) if index % 3 == 0 +} + + +@pytest.mark.parametrize( + "good_set", [FOUR_OF_LENGTH_6, EVERY_THIRD_OF_LENGTH_6], ids=["sparse", "dense"] +) +@pytest.mark.parametrize("wrap", [bytearray, lambda raw: memoryview(bytearray(raw))]) +def test_a_buffer_given_and_changed_afterwards_changes_nothing(good_set, wrap): + """A bitmap given is copied, whatever its form: changing the caller's + bytearray, or the one under a memoryview, afterwards changes nothing.""" + buffer = wrap(expected_bitmap(6, good_set)) + underlying = buffer.obj if isinstance(buffer, memoryview) else buffer + property_bitmaps = PropertyBitmaps(bitmaps={6: buffer}) + assert is_kept_as_indices(property_bitmaps, 6) == (good_set is FOUR_OF_LENGTH_6) + underlying[:] = b"\xff" * len(underlying) # every permutation good + assert_views_match(property_bitmaps, 6, good_set) + underlying[:] = bytes(len(underlying)) # none good + assert_views_match(property_bitmaps, 6, good_set) + + +@pytest.mark.parametrize( + "good_set", [FOUR_OF_LENGTH_6, EVERY_THIRD_OF_LENGTH_6], ids=["sparse", "dense"] +) +def test_a_memoryview_of_another_format_is_read_as_bytes(good_set): + """A memoryview of another format, shape or stride is read as its bytes.""" + raw = expected_bitmap(6, good_set) # 90 bytes + views = { + "2-byte items": memoryview(raw).cast("H"), # 45 items of 2 bytes + "3 x 30": memoryview(raw).cast("B", shape=[3, 30]), + "every other byte": memoryview(b"".join(bytes((byte, 0)) for byte in raw))[::2], + "signed": memoryview(raw).cast("b"), + } + for name, view in views.items(): + property_bitmaps = PropertyBitmaps(bitmaps={6: view}) + assert is_kept_as_indices(property_bitmaps, 6) == ( + good_set is FOUR_OF_LENGTH_6 + ), name + assert_views_match(property_bitmaps, 6, good_set) + + +def test_bitmaps_give_bytes_that_are_equal_on_every_read(): + """bitmaps gives a length's bitmap as bytes, whatever form the length is + kept in, and the same on every read.""" + by_property = PropertyBitmaps( + lambda perm: perm in EVERY_THIRD_OF_LENGTH_6 or perm in FOUR_OF_LENGTH_6 + ) + by_property.add_length(6) + sparse_by_property = PropertyBitmaps(lambda perm: perm in FOUR_OF_LENGTH_6) + sparse_by_property.add_length(6) + from_list = bisc_module._from_perms_by_length({6: sorted(EVERY_THIRD_OF_LENGTH_6)}) + from_sparse_list = bisc_module._from_perms_by_length({6: sorted(FOUR_OF_LENGTH_6)}) + for property_bitmaps in ( + by_property, + sparse_by_property, + from_list, + from_sparse_list, + ): + first = property_bitmaps.bitmaps[6] + assert type(first) is bytes + assert property_bitmaps.bitmaps[6] == first + assert len(first) == 90 + + +def extremes(length): + """The first and the last permutation of a length: one for lengths 0 and 1.""" + return {Perm(range(length)), Perm(range(length - 1, -1, -1))} + + +def test_bitmaps_round_trip_with_lengths_kept_as_indices(): + """A PropertyBitmaps made from another's bitmaps keeps each length in the + same form, with the same labels.""" + property_bitmaps = PropertyBitmaps(lambda perm: perm in extremes(len(perm))) + for length in range(9): + property_bitmaps.add_length(length) + assert [is_kept_as_indices(property_bitmaps, length) for length in range(9)] == [ + False, + False, + False, + False, + False, + False, + True, + True, + True, + ] + round_trip = PropertyBitmaps(bitmaps=property_bitmaps.bitmaps) + for length in range(9): + assert is_kept_as_indices(round_trip, length) == is_kept_as_indices( + property_bitmaps, length + ) + assert round_trip.good_counts[length] == property_bitmaps.good_counts[length] + assert round_trip.bitmaps[length] == property_bitmaps.bitmaps[length] + assert list(round_trip.good[length]) == list(property_bitmaps.good[length]) + assert list(round_trip.bad[length]) == list(property_bitmaps.bad[length]) + + +PERMS_OF_LENGTH_8 = list(Perm.of_length(8)) + + +def test_the_first_and_the_last_permutation_of_a_sparse_length(): + """The first and the last permutation of a length kept as indices, in + either view, good or bad; and, with the good ones in between, a bitmap.""" + first, last = PERMS_OF_LENGTH_8[0], PERMS_OF_LENGTH_8[-1] + for good_set in (extremes(8), {first}, {last}): + property_bitmaps = bisc_module._from_perms_by_length({8: sorted(good_set)}) + assert is_kept_as_indices(property_bitmaps, 8) + good, bad = property_bitmaps.good[8], property_bitmaps.bad[8] + for perm in (first, last): + assert (perm in good) == (perm in good_set) + assert (perm in bad) == (perm not in good_set) + assert list(good) == sorted(good_set) + bad_perms = list(bad) + assert len(bad_perms) == len(bad) == factorial(8) - len(good_set) + assert bad_perms == [perm for perm in PERMS_OF_LENGTH_8 if perm not in good_set] + in_between = set(PERMS_OF_LENGTH_8[1:-1]) + property_bitmaps = bisc_module._from_perms_by_length({8: sorted(in_between)}) + assert not is_kept_as_indices(property_bitmaps, 8) + assert list(property_bitmaps.bad[8]) == [first, last] + + +class PropertyFailed(Exception): + pass + + +@pytest.mark.parametrize("fail_after", [10, 1000, 5039]) +def test_a_property_that_raises_midway_through_add_length_keeps_nothing(fail_after): + """A property that raises in the middle of add_length, while the indices + are kept or after the switch to the bitmap, leaves nothing of that call, + and a length labeled before keeps its labels.""" + perms = list(Perm.of_length(7)) # 5040; all good makes the switch at the 79th + calls = [] + + def all_good_but_fails_once_asked(perm): + calls.append(perm) + if len(calls) == fail_after + 1: + raise PropertyFailed + return True + + property_bitmaps = PropertyBitmaps(all_good_but_fails_once_asked) + with pytest.raises(PropertyFailed): + property_bitmaps.add_length(7) + assert 7 not in property_bitmaps.bitmaps and 7 not in property_bitmaps.good_counts + assert 7 not in property_bitmaps.good and 7 not in property_bitmaps.bad + property_bitmaps.prop = lambda perm: perm == perms[3] + property_bitmaps.add_length(7) + property_bitmaps.prop = all_good_but_fails_once_asked + calls.clear() + with pytest.raises(PropertyFailed): + property_bitmaps.add_length(7) + assert_views_match(property_bitmaps, 7, {perms[3]}) + # A call that succeeds replaces the labels + property_bitmaps.prop = lambda perm: True + property_bitmaps.add_length(7) + assert_views_match(property_bitmaps, 7, set(perms)) + assert not is_kept_as_indices(property_bitmaps, 7) + + +def test_add_length_replaces_labels_between_the_forms(): + """add_length replaces a length's labels, from indices to a bitmap and + back.""" + property_bitmaps = PropertyBitmaps(lambda perm: perm in FOUR_OF_LENGTH_6) + property_bitmaps.add_length(6) + assert is_kept_as_indices(property_bitmaps, 6) + property_bitmaps.prop = lambda perm: perm in EVERY_THIRD_OF_LENGTH_6 + property_bitmaps.add_length(6) + assert not is_kept_as_indices(property_bitmaps, 6) + assert_views_match(property_bitmaps, 6, EVERY_THIRD_OF_LENGTH_6) + property_bitmaps.prop = lambda perm: perm in FOUR_OF_LENGTH_6 + property_bitmaps.add_length(6) + assert is_kept_as_indices(property_bitmaps, 6) + assert_views_match(property_bitmaps, 6, FOUR_OF_LENGTH_6) + + +def test_lengths_past_n_have_a_count_of_zero(monkeypatch): + """With m > n, the lengths past n are counted as empty, not only holding no + bits: mine() takes a length for complete when its count is the length's + factorial, and with a count of 1 it would take length 1 for complete.""" + seen = [] + real_mine = bisc_module.mine + + def recording_mine(goodperms, M, N=None, report=False): + property_bitmaps = goodperms.property_bitmaps + seen.append( + { + length: ( + property_bitmaps.good_counts[length], + len(goodperms[length]), + list(goodperms[length]), + len(property_bitmaps.bad[length]), + ) + for length in goodperms + } + ) + return real_mine(goodperms, M, N, report) + + monkeypatch.setattr(bisc_module, "mine", recording_mine) + bisc(lambda perm: True, 3, 0) + assert seen[0] == { + 0: (1, 1, [Perm(())], 0), + 1: (0, 0, [], 1), + 2: (0, 0, [], 2), + 3: (0, 0, [], 6), + } + + +def test_a_sparse_view_iterates_independently(): + """The same view of a length kept as indices, iterated twice or in step + with itself, gives the same permutations.""" + property_bitmaps = bisc_module._from_perms_by_length({6: sorted(FOUR_OF_LENGTH_6)}) + for view in (property_bitmaps.good[6], property_bitmaps.bad[6]): + once = list(view) + assert list(view) == once + assert list(zip(view, view)) == [(perm, perm) for perm in once] + + +def test_a_view_with_nothing_to_yield_reads_no_labels(monkeypatch): + """The bad view of a length with every permutation good yields nothing + without reading the bitmap.""" + all_good = PropertyBitmaps(bitmaps={6: b"\xff" * 90}) + reads = [] + real_wanted_indices = bisc_module._wanted_indices + + def recording_wanted_indices(bitmap, length, wanted_bit): + reads.append(length) + return real_wanted_indices(bitmap, length, wanted_bit) + + monkeypatch.setattr(bisc_module, "_wanted_indices", recording_wanted_indices) + assert list(all_good.bad[6]) == [] + assert reads == [] + + +def test_bisc_takes_the_view_of_a_sparse_length(): + """bisc given the good view of a PropertyBitmaps, a Mapping, mines what it + mines from the list it was made from.""" + perms = [Perm(range(length)) for length in range(7)] + [Perm(range(6, -1, -1))] + property_bitmaps = bisc_module._from_perms_by_length( + {length: [perm for perm in perms if len(perm) == length] for length in range(8)} + ) + assert is_kept_as_indices(property_bitmaps, 7) + assert bisc(property_bitmaps.good, 2, 7) == bisc(perms, 2, 7) + + +def test_a_sparse_length_14_never_sizes_a_bitmap(monkeypatch): + """No bitmap is sized for a length kept as indices: length 14's would take + 10.9 GB.""" + guard_bitmap_sizes(monkeypatch) + by_length = {length: [] for length in range(15)} + by_length[14] = THREE_OF_LENGTH_14 + by_length[13] = [Perm(range(13))] + by_length[12] = [Perm(range(12))] + by_length[11] = [Perm(range(11))] + property_bitmaps = bisc_module._from_perms_by_length(by_length) + for length in (11, 12, 13, 14): + labels = property_bitmaps._labels[length] + assert isinstance(labels, array) and len(labels) == len(by_length[length]) + assert list(property_bitmaps.good[14]) == sorted(THREE_OF_LENGTH_14) + assert len(property_bitmaps.bad[14]) == factorial(14) - 3 + for perm in THREE_OF_LENGTH_14: + assert perm in property_bitmaps.good[14] + assert perm not in property_bitmaps.bad[14] + assert Perm(range(14)) in property_bitmaps.bad[14] + assert 14 in property_bitmaps.bitmaps and len(property_bitmaps.bitmaps) == 15 + + +class MineStopped(Exception): + pass + + +def test_lengths_past_n_are_kept_as_empty_indices_and_size_no_bitmap(monkeypatch): + """With m > n, a length past n is kept as no indices, not as a bitmap: m = 14 + would size one of 10.9 GB. The lengths up to n are labeled.""" + kept = {} + + def stopped_mine(goodperms, M, N=None, report=False): + property_bitmaps = goodperms.property_bitmaps + kept.update(property_bitmaps._labels) + raise MineStopped + + guard_bitmap_sizes(monkeypatch) + monkeypatch.setattr(bisc_module, "mine", stopped_mine) + with pytest.raises(MineStopped): + bisc(lambda perm: perm.is_increasing(), 14, 2) + assert sorted(kept) == list(range(15)) + for length in range(3, 15): + assert isinstance(kept[length], array) and len(kept[length]) == 0 + + +def test_lengths_past_20_are_kept_as_empty_lists_when_m_is_past_n(monkeypatch): + """With m = 25 and n = 2, the lengths 3 to 20 past n are kept as no indices in + an array, and the lengths 21 to 25 as no indices in a list; none sizes a + bitmap, which 25! bits would not fit in memory; and the views read them as + empty.""" + kept = {} + views = {} + + def stopped_mine(goodperms, M, N=None, report=False): + kept.update(goodperms.property_bitmaps._labels) + views.update({length: goodperms[length] for length in goodperms}) + raise MineStopped + + guard_bitmap_sizes(monkeypatch) + monkeypatch.setattr(bisc_module, "mine", stopped_mine) + with pytest.raises(MineStopped): + bisc(lambda perm: perm.is_increasing(), 25, 2) + assert sorted(kept) == list(range(26)) + for length in range(3, 21): + assert isinstance(kept[length], array) and len(kept[length]) == 0 + for length in range(21, 26): + assert kept[length] == [] + assert len(views[length]) == 0 and list(views[length]) == [] + assert Perm(range(length)) not in views[length] + + +def test_a_sparse_list_with_repeats_counts_each_permutation_once(): + """A list that looks sparse before it is counted, with a permutation + repeated, counts it once.""" + first, second = PERMS_OF_LENGTH_8[7], PERMS_OF_LENGTH_8[100] + property_bitmaps = bisc_module._from_perms_by_length( + {8: [second, first, first, first]} + ) + assert is_kept_as_indices(property_bitmaps, 8) + assert property_bitmaps.good_counts[8] == 2 + assert len(property_bitmaps.good[8]) == 2 + assert len(property_bitmaps.bad[8]) == factorial(8) - 2 + assert list(property_bitmaps.good[8]) == [first, second] + + +@pytest.mark.parametrize( + "not_a_permutation, ranked_as", + [ + (Perm((0, 0, 0)), Perm((0, 1, 2))), + (Perm((0, 1, 1)), Perm((0, 1, 2))), + (Perm((1, 2, 3)), Perm((2, 0, 1))), # written from 1 + (Perm(range(1, 9)), Perm((1, 2, 3, 4, 5, 7, 0, 6))), # written from 1 + (Perm((2, 0)), None), # one past the last of length 2 + (Perm((3, 0, 0)), None), # one past the last of length 3 + (Perm(range(-1, 2)), None), # before the first of length 3 + (Perm(range(-1, 5)), None), # before the first of length 6 + (Perm(range(10, 18)), None), # past the last of length 8 + ], + ids=lambda value: str(value), +) +def test_a_sequence_that_is_not_a_permutation_is_refused(not_a_permutation, ranked_as): + """A Perm whose values are not 0 to n - 1 is refused with a ValueError, in a + list kept as indices and in one kept as a bitmap, given to bisc or made + into a PropertyBitmaps. Its rank() is another permutation's (ranked_as), + or outside its length's (None): in the first case it would be mined as + that permutation, and in the second its bit would be another's or past the + bitmap.""" + length = len(not_a_permutation) + index = not_a_permutation.rank() - bisc_module._shorter_count(length) + if ranked_as is None: + assert index not in range(factorial(length)) + else: + assert Perm.unrank(index, length) == ranked_as + message = "is not a permutation of 0 to {}".format(length - 1) + for perms in ( + [not_a_permutation], + [not_a_permutation] + list(Perm.of_length(length)), + ): + with pytest.raises(ValueError, match=message): + bisc_module._from_perms_by_length({length: perms}) + with pytest.raises(ValueError, match=message): + bisc(perms, 1, length)