Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGES.rst
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ New features:

Bug fixes:

- Preserve overridden ``RegexPattern.match_file()`` implementations when selecting a backend.
- Hyperscan backend no longer fails on multibyte UTF-8 characters.
- `Pull #123`_: Ignore invalid gitignore bracket ranges for `GitIgnoreSpec`.
- `Pull #128`_: Support POSIX character classes (e.g. `[[:alpha:]]`) in gitignore bracket expressions.
Expand Down
1 change: 1 addition & 0 deletions CHANGES_1.in.rst
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ New features:

Bug fixes:

- Preserve overridden ``RegexPattern.match_file()`` implementations when selecting a backend.
- Hyperscan backend no longer fails on multibyte UTF-8 characters.
- `Pull #123`_: Ignore invalid gitignore bracket ranges for `GitIgnoreSpec`.
- `Pull #128`_: Support POSIX character classes (e.g. `[[:alpha:]]`) in gitignore bracket expressions.
Expand Down
4 changes: 4 additions & 0 deletions README-dist.rst
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,9 @@ the patterns they generate themselves.
Precompiled expressions from other regex libraries keep their original engine
when using "best" or "simple". Explicit native backends reject these expressions
because their compile options cannot be preserved when rebuilding the pattern.
Patterns overriding ``match_file()`` also use "simple" during automatic backend
selection and are rejected by explicit native backends, preserving their custom
matching behavior.

See `benchmarks_backends.md`_ for comparisons between native Python regular
expressions and the optional backends.
Expand Down Expand Up @@ -251,6 +254,7 @@ Major changes:

Bug fixes:

- Preserve overridden ``RegexPattern.match_file()`` implementations when selecting a backend.
- Avoid following symbolic link targets when checking file types in ``iter_tree_files(follow_links=False)``.

- `Pull #123`_: Ignore invalid gitignore bracket ranges for `GitIgnoreSpec`.
Expand Down
3 changes: 3 additions & 0 deletions README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,9 @@ ignored.
Precompiled expressions from other regex libraries keep their original engine
when using "best" or "simple". Explicit native backends reject these expressions
because their compile options cannot be preserved when rebuilding the pattern.
Patterns overriding ``match_file()`` also use "simple" during automatic backend
selection and are rejected by explicit native backends, preserving their custom
matching behavior.

See `benchmarks_backends.md`_ for comparisons between native Python regular
expressions and the optional backends.
Expand Down
8 changes: 8 additions & 0 deletions pathspec/_backends/_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,14 @@ def is_native_compatible(pattern: Pattern) -> bool:
# Custom patterns can only be matched by calling them.
return False

match_file = pattern.match_file
if (
getattr(match_file, '__func__', None) is not RegexPattern.match_file
or getattr(match_file, '__self__', None) is not pattern
):
# Recompiling the regex would bypass a custom matching implementation.
return False

regex = pattern.regex
if not isinstance(regex, re.Pattern):
# The pattern was not compiled by Python's *re* module. Its flags and
Expand Down
119 changes: 119 additions & 0 deletions tests/test_10_custom_regex_matching.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
"""Custom regex matching must not change when a native backend is installed."""

import re
import unittest
from unittest.mock import patch

from pathspec import GitIgnoreSpec, PathSpec, RegexPattern
from pathspec.patterns.gitignore.spec import GitIgnoreSpecPattern

from .util import require_backend

SPEC_CLASSES: tuple[type[PathSpec], ...] = (PathSpec, GitIgnoreSpec)


class PublicPythonPattern(RegexPattern):
def match_file(self, file):
if file.startswith('private/'):
return None
return super().match_file(file)


class InheritedPublicPythonPattern(PublicPythonPattern):
pass


class OrdinaryRegexPattern(RegexPattern):
pass


class CustomRegexMatchingTest(unittest.TestCase):
def test_automatic_backend_preserves_override(self):
for backend in ('re2', 'hyperscan'):
for spec_class in SPEC_CLASSES:
for pattern_class in (PublicPythonPattern, InheritedPublicPythonPattern):
for selection in (None, 'best'):
with self.subTest(backend=backend, spec_class=spec_class,
pattern_class=pattern_class, selection=selection):
require_backend(backend)
pattern = pattern_class(re.compile(r'.*\.py$'), include=True)
with patch('pathspec._backends.agg._BEST_BACKEND', backend):
spec = spec_class([pattern], backend=selection)
self.assertEqual(
list(spec.match_files(['public/tool.py', 'private/secret.py'])),
['public/tool.py'],
)
self.assertEqual(spec.check_file('public/tool.py').index, 0)
self.assertFalse(spec.match_file('private/secret.py'))

def test_explicit_native_backend_rejects_override(self):
for backend in ('re2', 'hyperscan'):
for spec_class in SPEC_CLASSES:
with self.subTest(backend=backend, spec_class=spec_class):
require_backend(backend)
pattern = PublicPythonPattern(re.compile(r'.*\.py$'), include=True)
with self.assertRaisesRegex(ValueError, 'Use the simple backend instead'):
spec_class([pattern], backend=backend)

def test_combination_preserves_custom_exclusion(self):
for backend in ('re2', 'hyperscan'):
for spec_class in SPEC_CLASSES:
with self.subTest(backend=backend, spec_class=spec_class):
require_backend(backend)
with patch('pathspec._backends.agg._BEST_BACKEND', backend):
original = spec_class([RegexPattern(re.compile(r'.*\.py$'), True)])
exclusion = spec_class([PublicPythonPattern(re.compile(r'.*\.py$'), False)])
combined = original + exclusion
original += exclusion
for spec in (combined, original):
self.assertFalse(spec.match_file('public/tool.py'))
self.assertTrue(spec.match_file('private/secret.py'))
self.assertEqual(spec.check_file('public/tool.py').index, 1)
self.assertEqual(spec.check_file('private/secret.py').index, 0)

def test_inactive_override_keeps_native_backend(self):
for backend in ('re2', 'hyperscan'):
for spec_class in SPEC_CLASSES:
with self.subTest(backend=backend, spec_class=spec_class):
require_backend(backend)
patterns = [PublicPythonPattern(None), RegexPattern(re.compile(r'.*\.py$'), True)]
with patch('pathspec._backends.agg._BEST_BACKEND', backend):
spec = spec_class(patterns)
self.assertFalse(type(spec._backend).__name__.startswith('Simple'))
self.assertTrue(spec.match_file('private/secret.py'))
self.assertEqual(spec.check_file('private/secret.py').index, 1)

def test_inherited_default_matching_keeps_native_backend(self):
for backend in ('re2', 'hyperscan'):
for spec_class in SPEC_CLASSES:
for pattern_class in (OrdinaryRegexPattern, GitIgnoreSpecPattern):
with self.subTest(backend=backend, spec_class=spec_class, pattern_class=pattern_class):
require_backend(backend)
pattern = pattern_class(re.compile(r'.*\.py$'), include=True)
with patch('pathspec._backends.agg._BEST_BACKEND', backend):
spec = spec_class([pattern])
self.assertFalse(type(spec._backend).__name__.startswith('Simple'))
self.assertTrue(spec.match_file('public/tool.py'))

def test_instance_override(self):
for backend in ('re2', 'hyperscan'):
with self.subTest(backend=backend):
require_backend(backend)
pattern = OrdinaryRegexPattern(re.compile(r'.*\.py$'), include=True)
pattern.match_file = lambda file: None
with patch('pathspec._backends.agg._BEST_BACKEND', backend):
spec = PathSpec([pattern])
self.assertFalse(spec.match_file('public/tool.py'))

def test_instance_delegating_to_another_pattern(self):
for backend in ('re2', 'hyperscan'):
for spec_class in SPEC_CLASSES:
with self.subTest(backend=backend, spec_class=spec_class):
require_backend(backend)
pattern = OrdinaryRegexPattern(re.compile(r'.*\.py$'), include=True)
other = RegexPattern(re.compile(r'.*\.txt$'), include=True)
pattern.match_file = other.match_file
with patch('pathspec._backends.agg._BEST_BACKEND', backend):
spec = spec_class([pattern])
self.assertTrue(spec.match_file('readme.txt'))
self.assertFalse(spec.match_file('tool.py'))
Loading