diff --git a/README-dist.rst b/README-dist.rst index 2497721..cb0ee40 100644 --- a/README-dist.rst +++ b/README-dist.rst @@ -116,6 +116,17 @@ The "re2" backend uses the `google-re2`_ library (not to be confused with the be significantly faster than "simple", and 3 times faster than "hyperscan" at high pattern counts. +When a pattern is a precompiled ``re.Pattern``, the "re2" and "hyperscan" +backends translate its compile flags to the global inline flags ``(?ims)`` that +both engines understand. The flags that can't be translated -- ``re.VERBOSE`` +(RE2 has no "verbose" mode), ``re.LOCALE``, and ``re.ASCII`` combined with +``re.IGNORECASE`` -- are only supported by the "simple" backend: "best" uses +"simple" for those patterns, and explicitly selecting "re2" or "hyperscan" +raises ``ValueError``. ``re.UNICODE`` gets no prefix because neither engine +accepts ``(?u)``: both match ``\w``, ``\d``, ``\s``, and ``\b`` as ASCII +regardless, which is the same difference the native backends already have for +the patterns they generate themselves. + See `benchmarks_backends.md`_ for comparisons between native Python regular expressions and the optional backends. diff --git a/README.rst b/README.rst index 9824fb9..b0580a5 100644 --- a/README.rst +++ b/README.rst @@ -116,6 +116,17 @@ The "re2" backend uses the `google-re2`_ library (not to be confused with the be significantly faster than "simple", and 3 times faster than "hyperscan" at high pattern counts. +When a pattern is a precompiled ``re.Pattern``, the "re2" and "hyperscan" +backends translate its compile flags to the global inline flags ``(?ims)`` that +both engines understand. The flags that can't be translated -- ``re.VERBOSE`` +(RE2 has no "verbose" mode), ``re.LOCALE``, and ``re.ASCII`` combined with +``re.IGNORECASE`` -- are only supported by the "simple" backend: "best" uses +"simple" for those patterns, and explicitly selecting "re2" or "hyperscan" +raises ``ValueError``. ``re.UNICODE`` gets no prefix because neither engine +accepts ``(?u)``: both match ``\w``, ``\d``, ``\s``, and ``\b`` as ASCII +regardless, which is the same difference the native backends already have for +the patterns they generate themselves. + See `benchmarks_backends.md`_ for comparisons between native Python regular expressions and the optional backends. diff --git a/pathspec/_backends/_utils.py b/pathspec/_backends/_utils.py index 92390c3..25e9178 100644 --- a/pathspec/_backends/_utils.py +++ b/pathspec/_backends/_utils.py @@ -9,10 +9,12 @@ from collections.abc import ( Iterable) from typing import ( + Union, # Replaced by `X | Y` in 3.10. TypeVar) from pathspec.pattern import ( - Pattern) + Pattern, + RegexPattern) from pathspec._typing import ( AnyStr) @@ -28,6 +30,129 @@ TPattern = TypeVar("TPattern", bound=Pattern) +_TRANSLATABLE_FLAGS = ( + (int(re.IGNORECASE), 'i'), + (int(re.MULTILINE), 'm'), + (int(re.DOTALL), 's'), +) +""" +The compile flags that both RE2 and Hyperscan understand as global inline flags +at the start of an expression (`(?ims)`). + +RE2 rejects `(?x)` (it has no verbose mode), and both engines reject `(?a)`, +`(?u)`, `(?U)`, and `(?L)`. Note that RE2 reads `(?U)` as *non-greedy*, which +is not what Python's `re.UNICODE` means. +""" + +_NOOP_FLAGS = int(re.UNICODE | re.DEBUG) +r""" +The compile flags that need no translation. + +- `re.UNICODE` is implied for str patterns, and both engines match `\w`, `\d`, + `\s`, and `\b` as ASCII regardless. + +- `re.DEBUG` only affects how Python compiles the expression. +""" + +_ASCII_FLAGS = int(re.ASCII) +""" +The compile flags that both engines satisfy by default because their character +classes and word boundaries are ASCII-only. + +`re.ASCII` cannot be relied upon this way when it is combined with +`re.IGNORECASE`: the engines fold the case of Unicode characters that Python +would leave alone. +""" + + +def supports_regex_flags(flags: int) -> bool: + """ + Check whether a native backend can preserve the compile *flags*. + + *flags* (:class:`int`) is the flags of a compiled :class:`re.Pattern`. + + Returns the answer (:class:`bool`). + """ + representable = ( + _NOOP_FLAGS | _ASCII_FLAGS + | int(re.IGNORECASE | re.MULTILINE | re.DOTALL) + ) + if flags & ~representable: + return False + + if flags & re.ASCII and flags & re.IGNORECASE: + return False + + return True + + +def translate_regex_flags(regex: AnyStr, flags: int) -> AnyStr: + """ + Prepend the global inline flags that a native backend understands. + + *regex* (:class:`bytes` or :class:`str`) is the regular expression. + + *flags* (:class:`int`) is the flags of the compiled :class:`re.Pattern` + that *regex* came from. Only flags accepted by + :func:`supports_regex_flags` may be translated. + + Returns the regular expression (:class:`bytes` or :class:`str`). + """ + letters = ''.join( + letter + for flag, letter in _TRANSLATABLE_FLAGS + if flags & flag + ) + if not letters: + return regex + + prefix = f'(?{letters})' + if isinstance(regex, bytes): + return prefix.encode('ascii') + regex + + return prefix + regex + + +def get_regex_flags(regex: Union[re.Pattern, str, bytes, None]) -> int: + """ + Get the compile flags of a pattern's regular expression. + + *regex* (:class:`re.Pattern`, :class:`bytes`, :class:`str`, or + :data:`None`) is the regular expression of a :class:`.RegexPattern`. + + Returns the flags (:class:`int`) of a regular expression compiled by + Python's *re* module, or zero (:class:`int`) for anything else (a native + backend only ever sees the expression of a foreign pattern). + """ + if isinstance(regex, re.Pattern): + return int(regex.flags) + + return 0 + + +def has_unsupported_flags(patterns: Iterable[Pattern]) -> bool: + """ + Check for active patterns with compile flags a native backend can't keep. + + *patterns* (:class:`Iterable` of :class:`.Pattern`) contains the patterns. + + Returns the answer (:class:`bool`). + """ + for pattern in patterns: + if pattern.include is None or not isinstance(pattern, RegexPattern): + continue + + regex = pattern.regex + if not isinstance(regex, re.Pattern): + # The pattern was not compiled by Python's *re* module. Its flags + # are unknown, and only the simple backend can use it faithfully. + continue + + if not supports_regex_flags(regex.flags): + return True + + return False + def enumerate_patterns( patterns: Iterable[TPattern], diff --git a/pathspec/_backends/agg.py b/pathspec/_backends/agg.py index c387146..ff1803d 100644 --- a/pathspec/_backends/agg.py +++ b/pathspec/_backends/agg.py @@ -18,6 +18,9 @@ Pattern, RegexPattern) +from ._utils import ( + has_unsupported_flags) + from .hyperscan.base import ( hyperscan_error) from .hyperscan.gitignore import ( @@ -65,6 +68,10 @@ def make_gitignore_backend( """ if name == 'best': name = _BEST_BACKEND + if name != 'simple' and has_unsupported_flags(patterns): + # Only the simple backend can keep the compile flags of the + # patterns it was given. + name = 'simple' if name == 'hyperscan': return HyperscanGiBackend(cast(Sequence[RegexPattern], patterns)) @@ -93,6 +100,10 @@ def make_pathspec_backend( """ if name == 'best': name = _BEST_BACKEND + if name != 'simple' and has_unsupported_flags(patterns): + # Only the simple backend can keep the compile flags of the + # patterns it was given. + name = 'simple' if name == 'hyperscan': return HyperscanPsBackend(cast(Sequence[RegexPattern], patterns)) diff --git a/pathspec/_backends/hyperscan/gitignore.py b/pathspec/_backends/hyperscan/gitignore.py index b0ad2d7..a3efa70 100644 --- a/pathspec/_backends/hyperscan/gitignore.py +++ b/pathspec/_backends/hyperscan/gitignore.py @@ -30,7 +30,9 @@ override) # Added in 3.12. from .._utils import ( - translate_end_anchor) + get_regex_flags, + translate_end_anchor, + translate_regex_flags) from ._base import ( HS_FLAGS, @@ -156,6 +158,10 @@ def _init_db( for regex, is_dir_pattern in use_regexes: regex = translate_end_anchor(regex) + regex = translate_regex_flags( # type: ignore[type-var] + regex, + get_regex_flags(pattern.regex), + ) if isinstance(regex, bytes): regex_bytes = regex else: diff --git a/pathspec/_backends/hyperscan/pathspec.py b/pathspec/_backends/hyperscan/pathspec.py index 23821ce..dedbac8 100644 --- a/pathspec/_backends/hyperscan/pathspec.py +++ b/pathspec/_backends/hyperscan/pathspec.py @@ -27,7 +27,10 @@ from .._utils import ( enumerate_patterns, - translate_end_anchor) + get_regex_flags, + has_unsupported_flags, + translate_end_anchor, + translate_regex_flags) from .base import ( hyperscan_error) @@ -67,6 +70,12 @@ def __init__( if patterns and not isinstance(patterns[0], RegexPattern): raise TypeError(f"{patterns[0]=!r} must be a RegexPattern.") + if has_unsupported_flags(patterns): + raise ValueError( + "The hyperscan backend cannot preserve these regex compile flags. " + "Use backend='simple'." + ) + use_patterns = enumerate_patterns( patterns, filter=True, reverse=False, ) @@ -157,7 +166,10 @@ def _init_db( # Encode regex. assert isinstance(pattern, RegexPattern), pattern - regex = translate_end_anchor(pattern.regex.pattern) + regex = translate_regex_flags( + translate_end_anchor(pattern.regex.pattern), + get_regex_flags(pattern.regex), + ) if isinstance(regex, bytes): regex_bytes = regex diff --git a/pathspec/_backends/re2/gitignore.py b/pathspec/_backends/re2/gitignore.py index e49a5d2..2eaaaec 100644 --- a/pathspec/_backends/re2/gitignore.py +++ b/pathspec/_backends/re2/gitignore.py @@ -27,7 +27,9 @@ override) # Added in 3.12. from .._utils import ( - translate_end_anchor) + get_regex_flags, + translate_end_anchor, + translate_regex_flags) from ._base import ( Re2RegexDat, @@ -119,6 +121,10 @@ def _init_set( for regex, is_dir_pattern in use_regexes: regex = translate_end_anchor(regex) + regex = translate_regex_flags( # type: ignore[type-var] + regex, + get_regex_flags(pattern.regex), + ) if debug: regex_data.append(Re2RegexDebug( include=pattern_include, diff --git a/pathspec/_backends/re2/pathspec.py b/pathspec/_backends/re2/pathspec.py index d71ec89..48de18e 100644 --- a/pathspec/_backends/re2/pathspec.py +++ b/pathspec/_backends/re2/pathspec.py @@ -26,7 +26,10 @@ from .._utils import ( enumerate_patterns, - translate_end_anchor) + get_regex_flags, + has_unsupported_flags, + translate_end_anchor, + translate_regex_flags) from .base import ( re2_error) @@ -61,6 +64,12 @@ def __init__( if patterns and not isinstance(patterns[0], RegexPattern): raise TypeError(f"{patterns[0]=!r} must be a RegexPattern.") + if has_unsupported_flags(patterns): + raise ValueError( + "The re2 backend cannot preserve these regex compile flags. " + "Use backend='simple'." + ) + use_patterns = dict(enumerate_patterns( patterns, filter=True, reverse=False, )) @@ -134,7 +143,10 @@ def _init_set( assert pattern.regex is not None, pattern assert isinstance(pattern, RegexPattern), pattern - regex = translate_end_anchor(pattern.regex.pattern) + regex = translate_regex_flags( + translate_end_anchor(pattern.regex.pattern), + get_regex_flags(pattern.regex), + ) if debug: regex_data.append(Re2RegexDebug( diff --git a/tests/test_08_regex_flags.py b/tests/test_08_regex_flags.py new file mode 100644 index 0000000..9064962 --- /dev/null +++ b/tests/test_08_regex_flags.py @@ -0,0 +1,332 @@ +""" +This script tests that the native backends preserve the compile flags of an +externally compiled regular expression. +""" + +import re +import unittest + +from pathspec import ( + GitIgnoreSpec, + PathSpec, + RegexPattern) +from pathspec._backends import ( + agg) +from pathspec._backends._utils import ( + get_regex_flags, + has_unsupported_flags, + supports_regex_flags, + translate_regex_flags) +from pathspec.patterns.gitignore.basic import ( + GitIgnoreBasicPattern) +from pathspec.patterns.gitignore.spec import ( + GitIgnoreSpecPattern) + +from .util import ( + require_backend) + +BACKENDS = ('simple', 're2', 'hyperscan') +""" +The backends that can be required individually. +""" + +NATIVE_BACKEND_CLASSES = { + ('re2', PathSpec): 'Re2PsBackend', + ('re2', GitIgnoreSpec): 'Re2GiBackend', + ('hyperscan', PathSpec): 'HyperscanPsBackend', + ('hyperscan', GitIgnoreSpec): 'HyperscanGiBackend', +} +""" +The name of the backend class used by each native backend. +""" + +# Translate: expression, flags, a path that matches with the flags, and a path +# that must not match. +TRANSLATABLE_CASES = [ + ('secret', re.IGNORECASE, 'SECRET', 'public'), + ('secret', re.UNICODE, 'secret', 'SECRET'), + ('^secret$', re.MULTILINE, 'x\nsecret\ny', 'xsecret'), + ('^a.b$', re.DOTALL, 'a\nb', 'aXbY'), + ('^[a-z]+$', re.IGNORECASE, 'SECRET', '123'), + ('^secret.*end$', re.IGNORECASE | re.MULTILINE | re.DOTALL, 'x\nSECRET\nEND\ny', 'xsecret'), + ('^Secret$', re.IGNORECASE | re.MULTILINE | re.DOTALL, 'x\nSECRET\ny', 'xsecret'), + ('secret', 0, 'secret', 'SECRET'), + ('(?i)secret', 0, 'SECRET', 'public'), + ('(?m)^secret$', 0, 'x\nsecret\ny', 'xsecret'), + ('(?s)^a.b$', 0, 'a\nb', 'aXbY'), +] +""" +The cases for the flags that both native engines can express as global inline +flags (`(?ims)`), or that need no translation at all. +""" + +UNTRANSLATABLE_CASES = [ + ('s e c r e t', re.VERBOSE, 'secret', 's e c r e t'), + ('secret # note', re.VERBOSE, 'secret', 'public'), + ('secret # [', re.VERBOSE, 'secret', 'public'), + ('^[a-z]+$', re.ASCII | re.IGNORECASE, 'SECRET', '\u017f'), +] +""" +The cases for the flags that no native engine can express: RE2 has no `(?x)`, +and neither engine can express `(?a)` or `(?L)`. +""" + + +class RegexFlagsTest(unittest.TestCase): + """ + The compile flags of a pattern must not be silently discarded. + """ + + def test_translatable_flags(self): + """ + A native backend matches like the simple backend. + """ + for expression, flags, path, no_path in TRANSLATABLE_CASES: + for backend in BACKENDS: + with self.subTest(expression=expression, flags=flags, path=path, backend=backend): + require_backend(backend) + for spec_class in (PathSpec, GitIgnoreSpec): + with self.subTest(spec_class=spec_class): + spec = spec_class( + [RegexPattern(re.compile(expression, flags), include=True)], + backend=backend, + ) + self.assertTrue(spec.match_file(path)) + self.assertFalse(spec.match_file(no_path)) + + def test_translatable_flags_backend_agreement(self): + """ + Every backend agrees with the simple backend. + """ + for expression, flags, path, no_path in TRANSLATABLE_CASES: + for spec_class in (PathSpec, GitIgnoreSpec): + with self.subTest(expression=expression, flags=flags, spec_class=spec_class): + require_backend('simple') + expected = [] + for backend in BACKENDS: + require_backend(backend) + spec = spec_class( + [RegexPattern(re.compile(expression, flags), include=True)], + backend=backend, + ) + expected.append((spec.match_file(path), spec.match_file(no_path))) + + self.assertEqual(expected, [expected[0]] * len(expected)) + + def test_default_backend(self): + """ + The default backend keeps the flags of a compiled pattern. + """ + for expression, flags, path, no_path in TRANSLATABLE_CASES + UNTRANSLATABLE_CASES: + for spec_class in (PathSpec, GitIgnoreSpec): + for backend in (None, 'best'): + with self.subTest(expression=expression, flags=flags, + spec_class=spec_class, backend=backend): + spec = spec_class( + [RegexPattern(re.compile(expression, flags), include=True)], + backend=backend, + ) + self.assertTrue(spec.match_file(path)) + self.assertFalse(spec.match_file(no_path)) + + def test_untranslatable_flags_fall_back(self): + """ + Flags that no native engine can express select the simple backend. + """ + for expression, flags, path, no_path in UNTRANSLATABLE_CASES: + for spec_class in (PathSpec, GitIgnoreSpec): + for backend in (None, 'best'): + with self.subTest(expression=expression, flags=flags, + spec_class=spec_class, backend=backend): + spec = spec_class( + [RegexPattern(re.compile(expression, flags), include=True)], + backend=backend, + ) + self.assertTrue( + type(spec._backend).__name__.startswith('Simple'), + spec._backend, + ) + self.assertTrue(spec.match_file(path)) + self.assertFalse(spec.match_file(no_path)) + + def test_untranslatable_flags_explicit_backend(self): + """ + Explicitly selecting a native backend reports the unsupported flags. + """ + for backend in ('re2', 'hyperscan'): + with self.subTest(backend=backend): + require_backend(backend) + for expression, flags, _, _ in UNTRANSLATABLE_CASES: + for spec_class in (PathSpec, GitIgnoreSpec): + with self.subTest(expression=expression, flags=flags, + spec_class=spec_class): + patterns = [RegexPattern(re.compile(expression, flags), include=True)] + with self.assertRaisesRegex(ValueError, "backend='simple'"): + spec_class(patterns, backend=backend) + + def test_ordinary_patterns_stay_native(self): + """ + Patterns without external flags are not demoted to the simple backend. + """ + lines = ['*.txt', '!keep.txt', 'logs/', '!logs/KEEP'] + for backend in BACKENDS + ('best', None): + with self.subTest(backend=backend): + require_backend(backend) + specs = ( + PathSpec.from_lines(GitIgnoreBasicPattern, lines, backend=backend), + GitIgnoreSpec.from_lines(lines, backend=backend), + ) + for spec in specs: + with self.subTest(spec=type(spec)): + if backend == 'simple': + continue + elif backend is None or backend == 'best': + if agg._BEST_BACKEND != 'simple': + self.assertFalse( + type(spec._backend).__name__.startswith('Simple'), + spec._backend, + ) + else: + self.assertEqual( + type(spec._backend).__name__, + NATIVE_BACKEND_CLASSES[(backend, type(spec))], + spec._backend, + ) + + # Matching is unaffected. + self.assertTrue(specs[1].match_file('a.txt')) + self.assertFalse(specs[1].match_file('keep.txt')) + + def test_pattern_indices(self): + """ + The preserved flags do not disturb the pattern indices. + """ + patterns = [ + RegexPattern(None), + RegexPattern(re.compile('secret', re.IGNORECASE), include=True), + RegexPattern(re.compile('secret'), include=False), + ] + for backend in BACKENDS: + with self.subTest(backend=backend): + require_backend(backend) + spec = PathSpec(patterns, backend=backend) + upper = spec.check_file('SECRET') + lower = spec.check_file('secret') + self.assertEqual((upper.include, upper.index), (True, 1)) + self.assertEqual((lower.include, lower.index), (False, 2)) + + def test_gitignore_directory_precedence(self): + """ + A case-insensitive spec still applies the excluded-directory rule. + """ + patterns = [] + for line in ('logs/', '!logs/KEEP'): + pattern = GitIgnoreSpecPattern(line) + assert pattern.regex is not None, pattern + # Rebuild the pattern as if it had been compiled case-insensitively. + patterns.append(GitIgnoreSpecPattern( + re.compile(pattern.regex.pattern, re.IGNORECASE), pattern.include, + )) + for backend in BACKENDS: + with self.subTest(backend=backend): + require_backend(backend) + spec = GitIgnoreSpec(patterns, backend=backend) + result = spec.check_file('LOGS/KEEP') + self.assertEqual((result.include, result.index), (True, 0)) + plain_result = PathSpec(patterns, backend=backend).check_file('LOGS/KEEP') + self.assertEqual((plain_result.include, plain_result.index), (False, 1)) + + def test_translate_regex_flags(self): + """ + Only the flags a native engine understands are prepended. + """ + cases = [ + ('secret', 0, 'secret'), + ('secret', re.UNICODE, 'secret'), + ('secret', re.DEBUG, 'secret'), + ('secret', re.ASCII, 'secret'), + ('secret', re.IGNORECASE, '(?i)secret'), + ('secret', re.MULTILINE, '(?m)secret'), + ('secret', re.DOTALL, '(?s)secret'), + ('secret', re.IGNORECASE | re.DOTALL, '(?is)secret'), + ('secret', re.IGNORECASE | re.MULTILINE | re.DOTALL, '(?ims)secret'), + (b'secret', 0, b'secret'), + (b'secret', re.IGNORECASE, b'(?i)secret'), + (b'secret', re.IGNORECASE | re.DOTALL, b'(?is)secret'), + ] + for expression, flags, expected in cases: + with self.subTest(expression=expression, flags=flags): + self.assertEqual(translate_regex_flags(expression, flags), expected) + + def test_supports_regex_flags(self): + """ + The flags that the native engines cannot express are reported. + """ + cases = [ + (re.compile('secret').flags, True), + (re.compile(b'secret').flags, True), + (re.compile('secret', re.IGNORECASE).flags, True), + (re.compile('secret', re.MULTILINE).flags, True), + (re.compile('secret', re.DOTALL).flags, True), + (re.compile('secret', re.ASCII).flags, True), + (re.compile('secret', re.IGNORECASE | re.MULTILINE | re.DOTALL).flags, True), + (re.compile('(?i)secret').flags, True), + (re.compile('secret', re.VERBOSE).flags, False), + (re.compile('secret', re.IGNORECASE | re.VERBOSE).flags, False), + (re.compile('secret', re.ASCII | re.IGNORECASE).flags, False), + (int(re.UNICODE | re.DEBUG), True), + (int(re.UNICODE | (1 << 20)), False), + ] + for flags, expected in cases: + with self.subTest(flags=flags): + self.assertEqual(supports_regex_flags(flags), expected) + + def test_get_regex_flags(self): + """ + A pattern that Python did not compile has no flags to translate. + """ + class ForeignPattern(object): + """ + A regular expression that was not compiled by the *re* module. + """ + pattern = '(?i)secret' + + compiled = re.compile('secret', re.IGNORECASE) + cases = [ + (compiled, int(compiled.flags)), + (re.compile('secret'), int(re.compile('secret').flags)), + ('secret', 0), + (b'secret', 0), + (None, 0), + (ForeignPattern(), 0), + ] + for regex, expected in cases: + with self.subTest(regex=regex): + self.assertEqual(get_regex_flags(regex), expected) + + def test_has_unsupported_flags(self): + """ + Null patterns and foreign patterns are not reported as unsupported. + """ + cases = [ + ([], False), + ([RegexPattern(None)], False), + ([RegexPattern(re.compile('secret'), include=True)], False), + ([RegexPattern(re.compile('secret', re.IGNORECASE), include=True)], False), + ([RegexPattern(re.compile('secret', re.VERBOSE), include=True)], True), + ([ + RegexPattern(re.compile('secret', re.IGNORECASE), include=True), + RegexPattern(re.compile('secret', re.VERBOSE), include=False), + ], True), + ([ + RegexPattern(re.compile('secret', re.IGNORECASE), include=None), + RegexPattern(re.compile('secret', re.VERBOSE), include=None), + ], False), + ] + for patterns, expected in cases: + with self.subTest(patterns=patterns): + self.assertEqual(has_unsupported_flags(patterns), expected) + + +if __name__ == '__main__': + unittest.main()