Preserve externally compiled regex flags during backend selection

README-dist.rst

@@ -116,6 +116,17 @@ The "re2" backend uses the `google-re2`_ library (not to be confused with the
 be significantly faster than "simple", and 3 times faster than "hyperscan" at
 high pattern counts.
 
+When a pattern is a precompiled ``re.Pattern``, the "re2" and "hyperscan"
+backends translate its compile flags to the global inline flags ``(?ims)`` that
+both engines understand. The flags that can't be translated -- ``re.VERBOSE``
+(RE2 has no "verbose" mode), ``re.LOCALE``, and ``re.ASCII`` combined with
+``re.IGNORECASE`` -- are only supported by the "simple" backend: "best" uses
+"simple" for those patterns, and explicitly selecting "re2" or "hyperscan"
+raises ``ValueError``. ``re.UNICODE`` gets no prefix because neither engine
+accepts ``(?u)``: both match ``\w``, ``\d``, ``\s``, and ``\b`` as ASCII
+regardless, which is the same difference the native backends already have for
+the patterns they generate themselves.
+
 See `benchmarks_backends.md`_ for comparisons between native Python regular
 expressions and the optional backends.
 

README.rst

@@ -116,6 +116,17 @@ The "re2" backend uses the `google-re2`_ library (not to be confused with the
 be significantly faster than "simple", and 3 times faster than "hyperscan" at
 high pattern counts.
 
+When a pattern is a precompiled ``re.Pattern``, the "re2" and "hyperscan"
+backends translate its compile flags to the global inline flags ``(?ims)`` that
+both engines understand. The flags that can't be translated -- ``re.VERBOSE``
+(RE2 has no "verbose" mode), ``re.LOCALE``, and ``re.ASCII`` combined with
+``re.IGNORECASE`` -- are only supported by the "simple" backend: "best" uses
+"simple" for those patterns, and explicitly selecting "re2" or "hyperscan"
+raises ``ValueError``. ``re.UNICODE`` gets no prefix because neither engine
+accepts ``(?u)``: both match ``\w``, ``\d``, ``\s``, and ``\b`` as ASCII
+regardless, which is the same difference the native backends already have for
+the patterns they generate themselves.
+
 See `benchmarks_backends.md`_ for comparisons between native Python regular
 expressions and the optional backends.
 

pathspec/_backends/_utils.py

@@ -9,10 +9,12 @@
 from collections.abc import (
 	Iterable)
 from typing import (
+	Union,  # Replaced by `X | Y` in 3.10.
 	TypeVar)
 
 from pathspec.pattern import (
-	Pattern)
+	Pattern,
+	RegexPattern)
 from pathspec._typing import (
 	AnyStr)
 
@@ -28,6 +30,129 @@
 
 TPattern = TypeVar("TPattern", bound=Pattern)
 
+_TRANSLATABLE_FLAGS = (
+	(int(re.IGNORECASE), 'i'),
+	(int(re.MULTILINE), 'm'),
+	(int(re.DOTALL), 's'),
+)
+"""
+The compile flags that both RE2 and Hyperscan understand as global inline flags
+at the start of an expression (`(?ims)`).
+
+RE2 rejects `(?x)` (it has no verbose mode), and both engines reject `(?a)`,
+`(?u)`, `(?U)`, and `(?L)`. Note that RE2 reads `(?U)` as *non-greedy*, which
+is not what Python's `re.UNICODE` means.
+"""
+
+_NOOP_FLAGS = int(re.UNICODE | re.DEBUG)
+r"""
+The compile flags that need no translation.
+
+-	`re.UNICODE` is implied for str patterns, and both engines match `\w`, `\d`,
+	`\s`, and `\b` as ASCII regardless.
+
+-	`re.DEBUG` only affects how Python compiles the expression.
+"""
+
+_ASCII_FLAGS = int(re.ASCII)
+"""
+The compile flags that both engines satisfy by default because their character
+classes and word boundaries are ASCII-only.
+
+`re.ASCII` cannot be relied upon this way when it is combined with
+`re.IGNORECASE`: the engines fold the case of Unicode characters that Python
+would leave alone.
+"""
+
+
+def supports_regex_flags(flags: int) -> bool:
+	"""
+	Check whether a native backend can preserve the compile *flags*.
+
+	*flags* (:class:`int`) is the flags of a compiled :class:`re.Pattern`.
+
+	Returns the answer (:class:`bool`).
+	"""
+	representable = (
+		_NOOP_FLAGS | _ASCII_FLAGS
+		| int(re.IGNORECASE | re.MULTILINE | re.DOTALL)
+	)
+	if flags & ~representable:
+		return False
+
+	if flags & re.ASCII and flags & re.IGNORECASE:
+		return False
+
+	return True
+
+
+def translate_regex_flags(regex: AnyStr, flags: int) -> AnyStr:
+	"""
+	Prepend the global inline flags that a native backend understands.
+
+	*regex* (:class:`bytes` or :class:`str`) is the regular expression.
+
+	*flags* (:class:`int`) is the flags of the compiled :class:`re.Pattern`
+	that *regex* came from. Only flags accepted by
+	:func:`supports_regex_flags` may be translated.
+
+	Returns the regular expression (:class:`bytes` or :class:`str`).
+	"""
+	letters = ''.join(
+		letter
+		for flag, letter in _TRANSLATABLE_FLAGS
+		if flags & flag
+	)
+	if not letters:
+		return regex
+
+	prefix = f'(?{letters})'
+	if isinstance(regex, bytes):
+		return prefix.encode('ascii') + regex
+
+	return prefix + regex
+
+
+def get_regex_flags(regex: Union[re.Pattern, str, bytes, None]) -> int:
+	"""
+	Get the compile flags of a pattern's regular expression.
+
+	*regex* (:class:`re.Pattern`, :class:`bytes`, :class:`str`, or
+	:data:`None`) is the regular expression of a :class:`.RegexPattern`.
+
+	Returns the flags (:class:`int`) of a regular expression compiled by
+	Python's *re* module, or zero (:class:`int`) for anything else (a native
+	backend only ever sees the expression of a foreign pattern).
+	"""
+	if isinstance(regex, re.Pattern):
+		return int(regex.flags)
+
+	return 0
+
+
+def has_unsupported_flags(patterns: Iterable[Pattern]) -> bool:
+	"""
+	Check for active patterns with compile flags a native backend can't keep.
+
+	*patterns* (:class:`Iterable` of :class:`.Pattern`) contains the patterns.
+
+	Returns the answer (:class:`bool`).
+	"""
+	for pattern in patterns:
+		if pattern.include is None or not isinstance(pattern, RegexPattern):
+			continue
+
+		regex = pattern.regex
+		if not isinstance(regex, re.Pattern):
+			# The pattern was not compiled by Python's *re* module. Its flags
+			# are unknown, and only the simple backend can use it faithfully.
+			continue
+
+		if not supports_regex_flags(regex.flags):
+			return True
+
+	return False
+
 
 def enumerate_patterns(
 	patterns: Iterable[TPattern],

pathspec/_backends/agg.py

@@ -18,6 +18,9 @@
 	Pattern,
 	RegexPattern)
 
+from ._utils import (
+	has_unsupported_flags)
+
 from .hyperscan.base import (
 	hyperscan_error)
 from .hyperscan.gitignore import (
@@ -65,6 +68,10 @@ def make_gitignore_backend(
 	"""
 	if name == 'best':
 		name = _BEST_BACKEND
+		if name != 'simple' and has_unsupported_flags(patterns):
+			# Only the simple backend can keep the compile flags of the
+			# patterns it was given.
+			name = 'simple'
 
 	if name == 'hyperscan':
 		return HyperscanGiBackend(cast(Sequence[RegexPattern], patterns))
@@ -93,6 +100,10 @@ def make_pathspec_backend(
 	"""
 	if name == 'best':
 		name = _BEST_BACKEND
+		if name != 'simple' and has_unsupported_flags(patterns):
+			# Only the simple backend can keep the compile flags of the
+			# patterns it was given.
+			name = 'simple'
 
 	if name == 'hyperscan':
 		return HyperscanPsBackend(cast(Sequence[RegexPattern], patterns))

pathspec/_backends/hyperscan/gitignore.py

@@ -30,7 +30,9 @@
 	override)  # Added in 3.12.
 
 from .._utils import (
-	translate_end_anchor)
+	get_regex_flags,
+	translate_end_anchor,
+	translate_regex_flags)
 
 from ._base import (
 	HS_FLAGS,
@@ -156,6 +158,10 @@ def _init_db(
 
 			for regex, is_dir_pattern in use_regexes:
 				regex = translate_end_anchor(regex)
+				regex = translate_regex_flags(  # type: ignore[type-var]
+					regex,
+					get_regex_flags(pattern.regex),
+				)
 				if isinstance(regex, bytes):
 					regex_bytes = regex
 				else:

pathspec/_backends/hyperscan/pathspec.py

@@ -27,7 +27,10 @@
 
 from .._utils import (
 	enumerate_patterns,
-	translate_end_anchor)
+	get_regex_flags,
+	has_unsupported_flags,
+	translate_end_anchor,
+	translate_regex_flags)
 
 from .base import (
 	hyperscan_error)
@@ -67,6 +70,12 @@ def __init__(
 		if patterns and not isinstance(patterns[0], RegexPattern):
 			raise TypeError(f"{patterns[0]=!r} must be a RegexPattern.")
 
+		if has_unsupported_flags(patterns):
+			raise ValueError(
+				"The hyperscan backend cannot preserve these regex compile flags. "
+				"Use backend='simple'."
+			)
+
 		use_patterns = enumerate_patterns(
 			patterns, filter=True, reverse=False,
 		)
@@ -157,7 +166,10 @@ def _init_db(
 
 			# Encode regex.
 			assert isinstance(pattern, RegexPattern), pattern
-			regex = translate_end_anchor(pattern.regex.pattern)
+			regex = translate_regex_flags(
+				translate_end_anchor(pattern.regex.pattern),
+				get_regex_flags(pattern.regex),
+			)
 
 			if isinstance(regex, bytes):
 				regex_bytes = regex

pathspec/_backends/re2/gitignore.py

@@ -27,7 +27,9 @@
 	override)  # Added in 3.12.
 
 from .._utils import (
-	translate_end_anchor)
+	get_regex_flags,
+	translate_end_anchor,
+	translate_regex_flags)
 
 from ._base import (
 	Re2RegexDat,
@@ -119,6 +121,10 @@ def _init_set(
 
 			for regex, is_dir_pattern in use_regexes:
 				regex = translate_end_anchor(regex)
+				regex = translate_regex_flags(  # type: ignore[type-var]
+					regex,
+					get_regex_flags(pattern.regex),
+				)
 				if debug:
 					regex_data.append(Re2RegexDebug(
 						include=pattern_include,

pathspec/_backends/re2/pathspec.py

@@ -26,7 +26,10 @@
 
 from .._utils import (
 	enumerate_patterns,
-	translate_end_anchor)
+	get_regex_flags,
+	has_unsupported_flags,
+	translate_end_anchor,
+	translate_regex_flags)
 
 from .base import (
 	re2_error)
@@ -61,6 +64,12 @@ def __init__(
 		if patterns and not isinstance(patterns[0], RegexPattern):
 			raise TypeError(f"{patterns[0]=!r} must be a RegexPattern.")
 
+		if has_unsupported_flags(patterns):
+			raise ValueError(
+				"The re2 backend cannot preserve these regex compile flags. "
+				"Use backend='simple'."
+			)
+
 		use_patterns = dict(enumerate_patterns(
 			patterns, filter=True, reverse=False,
 		))
@@ -134,7 +143,10 @@ def _init_set(
 
 			assert pattern.regex is not None, pattern
 			assert isinstance(pattern, RegexPattern), pattern
-			regex = translate_end_anchor(pattern.regex.pattern)
+			regex = translate_regex_flags(
+				translate_end_anchor(pattern.regex.pattern),
+				get_regex_flags(pattern.regex),
+			)
 
 			if debug:
 				regex_data.append(Re2RegexDebug(