diff --git a/src/license_expression/_pyahocorasick.py b/src/license_expression/_pyahocorasick.py index 2a1f5bb..156b7af 100644 --- a/src/license_expression/_pyahocorasick.py +++ b/src/license_expression/_pyahocorasick.py @@ -632,11 +632,11 @@ def overlap(self, other): # tokenize to separate text from parens _tokenizer = re.compile( r""" - (?P[^\s\(\)]+) + (?:[^\s\(\)]+) # text | - (?P\s+) + (?:\s+) # space | - (?P[\(\)]) + (?:[\(\)]) # parens """, re.VERBOSE | re.MULTILINE | re.UNICODE, ) @@ -646,4 +646,4 @@ def get_tokens(tokens_string): """ Return an iterable of strings splitting on spaces and parens. """ - return [match for match in _tokenizer.split(tokens_string.lower()) if match] + return _tokenizer.findall(tokens_string.lower())