Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGES_1.in.rst
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ New features:
Bug fixes:

- Hyperscan backend no longer fails on multibyte UTF-8 characters.
- `Issue #176`_: Normalize redundant interior separators and current-directory components in matching paths, preserving original paths in matching results.
- `Pull #123`_: Ignore invalid gitignore bracket ranges for `GitIgnoreSpec`.
- `Pull #128`_: Support POSIX character classes (e.g. `[[:alpha:]]`) in gitignore bracket expressions.
- `Issue #129`_ / `Pull #132`_: Fix GitIgnoreSpec re-including files under an excluded directory
Expand Down Expand Up @@ -90,6 +91,7 @@ Bug fixes:
.. _`Pull #171`: https://github.com/cpburnz/python-pathspec/pull/171
.. _`Pull #173`: https://github.com/cpburnz/python-pathspec/pull/173
.. _`Pull #174`: https://github.com/cpburnz/python-pathspec/pull/174
.. _`Issue #176`: https://github.com/cpburnz/python-pathspec/issues/176
.. _`Pull #181`: https://github.com/cpburnz/python-pathspec/pull/181


Expand Down
10 changes: 9 additions & 1 deletion pathspec/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import os.path
import pathlib
import posixpath
import re
import stat
import warnings
from collections.abc import (
Expand Down Expand Up @@ -62,6 +63,8 @@
:data:`os.altsep`.
"""

_NORMALIZE_INTERIOR_PATH = re.compile(r'/(?:/|\./)+')

_registered_patterns: dict[str, Callable[[Union[str, bytes]], Pattern]] = {}
"""
*_registered_patterns* (:class:`dict`) maps a name (:class:`str`) to the
Expand Down Expand Up @@ -614,14 +617,16 @@ def normalize_file(
"""
Normalizes the file path to use the POSIX path separator (i.e., ``"/"``), and
make the paths relative (remove leading ``"/"``).
Redundant interior separators and current-directory components are removed.
Parent-directory components and a trailing separator are preserved.

*file* (:class:`str` or :class:`os.PathLike`) is the file path.

*separators* (:class:`~collections.abc.Collection` of :class:`str`; or
:data:`None`) optionally contains the path separators to normalize. This does
not need to include the POSIX path separator (``"/"``), but including it will
not affect the results. Default is ``None`` for :data:`.NORMALIZE_PATH_SEPS`.
To prevent normalization, pass an empty container (e.g., an empty tuple
To prevent separator replacement, pass an empty container (e.g., an empty tuple
``()``).

Returns the normalized file path (:class:`str`).
Expand All @@ -648,6 +653,9 @@ def normalize_file(
prefix_length += 2
norm_file = norm_file[prefix_length:]

if '//' in norm_file or '/./' in norm_file:
norm_file = _NORMALIZE_INTERIOR_PATH.sub('/', norm_file)

return norm_file


Expand Down
87 changes: 87 additions & 0 deletions tests/test_10_path_normalization.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
"""Regression coverage for redundant interior components in matching paths."""

from unittest import TestCase

from pathspec import GitIgnoreSpec, PathSpec
from pathspec.util import normalize_file

from .util import require_backend


class InteriorPathNormalizationTest(TestCase):

def test_normalize_components(self):
cases = {
'data//private.txt': 'data/private.txt',
'data/./private.txt': 'data/private.txt',
'data///././/private.txt': 'data/private.txt',
'data//child/./private.txt': 'data/child/private.txt',
'data//./': 'data/',
'data/././': 'data/',
'data/private.txt': 'data/private.txt',
'data/.hidden': 'data/.hidden',
'data/..hidden/file': 'data/..hidden/file',
'data/../private.txt': 'data/../private.txt',
'data/.': 'data/.',
'data/': 'data/',
'': '',
}
for path, expected in cases.items():
with self.subTest(path=path):
self.assertEqual(normalize_file(path, separators=()), expected)

def test_public_matching_apis_preserve_input_spelling(self):
paths = [
'data/private.txt',
'data//private.txt',
'data/./private.txt',
'data///././private.txt',
'data/keep.txt',
'data//./keep.txt',
'data/public.txt',
'data/./public.txt',
]
expected = [(True, 0)] * 4 + [(False, 2)] * 2 + [(None, None)] * 2
lines = ['/data/private.txt', '/data/keep.txt', '!/data/keep.txt']
for cls in (PathSpec, GitIgnoreSpec):
for backend in ('simple', 're2', 'hyperscan'):
with self.subTest(cls=cls.__name__, backend=backend):
require_backend(backend)
if cls is PathSpec:
spec = cls.from_lines('gitignore', lines, backend=backend)
else:
spec = cls.from_lines(lines, backend=backend)
self.assertEqual(list(spec.match_files(paths)), paths[:4])
self.assertEqual(list(spec.match_files(paths, negate=True)), paths[4:])
results = list(spec.check_files(paths))
self.assertEqual([result.file for result in results], paths)
self.assertEqual(
[(result.include, result.index) for result in results], expected)
for path, (include, index) in zip(paths, expected):
self.assertEqual(spec.match_file(path), bool(include))
result = spec.check_file(path)
self.assertEqual((result.file, result.include, result.index), (path, include, index))

def test_custom_separator_and_literal_backslash(self):
path = r'data\\.\private.txt'
self.assertEqual(normalize_file(path, separators=('\\',)), 'data/private.txt')
self.assertEqual(normalize_file(path, separators=()), path)
for cls in (PathSpec, GitIgnoreSpec):
with self.subTest(cls=cls.__name__):
if cls is PathSpec:
spec = cls.from_lines('gitignore', ['/data/private.txt'], backend='simple')
else:
spec = cls.from_lines(['/data/private.txt'], backend='simple')
self.assertTrue(spec.match_file(path, separators=('\\',)))
self.assertFalse(spec.match_file(path, separators=()))

def test_directory_markers_and_parent_components(self):
for cls in (PathSpec, GitIgnoreSpec):
with self.subTest(cls=cls.__name__):
if cls is PathSpec:
spec = cls.from_lines('gitignore', ['/data/'], backend='simple')
else:
spec = cls.from_lines(['/data/'], backend='simple')
self.assertTrue(spec.match_file('data//./'))
self.assertFalse(spec.match_file('data'))
self.assertEqual(normalize_file('data//../file.txt'), 'data/../file.txt')