From 94a84242922d69ff3414f610653415d537fdd255 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 10 Jul 2026 12:07:36 +0300 Subject: [PATCH 1/3] gh-153494: Encode imaplib search criteria to the declared CHARSET IMAP4.search(), sort(), thread() and the uid SORT/THREAD variants now encode str search criteria to the declared charset, so international search text can be passed as ordinary str. A criterion passed as bytes is sent unchanged, for use with a charset that Python has no codec for. Co-Authored-By: Claude Opus 4.8 --- Doc/library/imaplib.rst | 20 +++++ Lib/imaplib.py | 19 +++++ Lib/test/test_imaplib.py | 75 +++++++++++++++++-- ...-07-10-16-00-00.gh-issue-153494.qCh8rT.rst | 6 ++ 4 files changed, 114 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-07-10-16-00-00.gh-issue-153494.qCh8rT.rst diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 6872ba0d6cbfcb..14776f22e31929 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -584,6 +584,14 @@ An :class:`IMAP4` instance has the following methods: the ``UTF8=ACCEPT`` capability was enabled using the :meth:`enable` command. + A criterion passed as :class:`str` is encoded to *charset* (which must name + a codec known to Python); pass :class:`bytes` to send a criterion that is + already encoded, for example when *charset* is one that Python does not + support. + + .. versionchanged:: next + ``str`` search criteria are encoded to *charset*. + Example:: # M is a connected IMAP4 instance... @@ -651,8 +659,14 @@ An :class:`IMAP4` instance has the following methods: the interpretation of strings in the searching criteria. It then returns the numbers of matching messages. + As with :meth:`search`, a *search_criterion* passed as :class:`str` is + encoded to *charset*; pass :class:`bytes` to send one already encoded. + This is an ``IMAP4rev1`` extension command. + .. versionchanged:: next + ``str`` search criteria are encoded to *charset*. + .. method:: IMAP4.starttls(ssl_context=None) @@ -729,8 +743,14 @@ An :class:`IMAP4` instance has the following methods: returns the matching messages threaded according to the specified threading algorithm. + As with :meth:`search`, a *search_criterion* passed as :class:`str` is + encoded to *charset*; pass :class:`bytes` to send one already encoded. + This is an ``IMAP4rev1`` extension command. + .. versionchanged:: next + ``str`` search criteria are encoded to *charset*. + .. method:: IMAP4.uid(command, arg[, ...]) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index d8acf085e7a128..9065adf7499634 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -920,11 +920,15 @@ def search(self, charset, *criteria): 'data' is space separated list of matching message numbers. If UTF8 is enabled, charset MUST be None. + + A 'criteria' passed as str is encoded to 'charset'; pass bytes to + send criteria that are already encoded. """ name = 'SEARCH' if charset is not None: if self.utf8_enabled: raise IMAP4.error("Non-None charset not valid in UTF8 mode") + criteria = self._encode_criteria(charset, criteria) typ, dat = self._simple_command(name, 'CHARSET', self._astring(charset), *criteria) else: @@ -1000,6 +1004,7 @@ def sort(self, sort_criteria, charset, *search_criteria): #if not name in self.capabilities: # Let the server decide! # raise self.error('unimplemented extension command: %s' % name) sort_criteria = self._set_quote(sort_criteria) + search_criteria = self._encode_criteria(charset, search_criteria) if charset is not None: charset = self._astring(charset) typ, dat = self._simple_command(name, sort_criteria, charset, *search_criteria) @@ -1068,6 +1073,7 @@ def thread(self, threading_algorithm, charset, *search_criteria): (type, [data]) = .thread(threading_algorithm, charset, search_criteria, ...) """ name = 'THREAD' + search_criteria = self._encode_criteria(charset, search_criteria) if charset is not None: charset = self._astring(charset) typ, dat = self._simple_command(name, self._atom(threading_algorithm), @@ -1106,12 +1112,14 @@ def uid(self, command, *args): self._set_quote(flags)) elif command == 'SORT': sort_criteria, charset, *search_criteria = args + search_criteria = self._encode_criteria(charset, search_criteria) if charset is not None: charset = self._astring(charset) args = (self._set_quote(sort_criteria), charset, *search_criteria) elif command == 'THREAD': threading_algorithm, charset, *search_criteria = args + search_criteria = self._encode_criteria(charset, search_criteria) if charset is not None: charset = self._astring(charset) args = (self._atom(threading_algorithm), charset, @@ -1524,6 +1532,17 @@ def _fetch_parts(self, arg): return arg return self._set_quote(arg) + def _encode_criteria(self, charset, criteria): + # Encode str search criteria to the declared CHARSET so the bytes on + # the wire match it. bytes criteria are already encoded and pass + # through unchanged. charset is None when no CHARSET is sent. + if charset is None: + return criteria + if isinstance(charset, (bytes, bytearray)): + charset = str(charset, 'ascii') + return tuple(c.encode(charset) if isinstance(c, str) else c + for c in criteria) + def _quote(self, arg): if isinstance(arg, str): arg = bytes(arg, self._encoding) diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index 16a96381992b68..8e1291485e1aad 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -205,6 +205,31 @@ def test_astring(self): self.assertEqual(m._astring('Entwürfe'), '"Entwürfe"'.encode()) self.assertEqual(m._astring(b'Entw\xc3\xbcrfe'), b'"Entw\xc3\xbcrfe"') + def test_encode_criteria(self): + m = imaplib.IMAP4.__new__(imaplib.IMAP4) + enc = m._encode_criteria + # No charset: criteria are returned unchanged. + self.assertEqual(enc(None, ('TEXT', 'x')), ('TEXT', 'x')) + # str criteria are encoded to the charset; ASCII is charset-independent. + self.assertEqual(enc('UTF-8', ('TEXT', 'XXXXXX')), (b'TEXT', b'XXXXXX')) + # Non-ASCII text is encoded to the declared charset, including charsets + # other than ASCII, Latin-1 and UTF-8. + self.assertEqual(enc('UTF-8', ('"café"',)), ('"café"'.encode('utf-8'),)) + self.assertEqual(enc('ISO-8859-1', ('"café"',)), + ('"café"'.encode('latin-1'),)) + self.assertEqual(enc('KOI8-U', ('"Київ"',)), + ('"Київ"'.encode('koi8-u'),)) + self.assertEqual(enc('SHIFT_JIS', ('"日本"',)), + ('"日本"'.encode('shift_jis'),)) + # bytes criteria are already encoded and pass through unchanged. + self.assertEqual(enc('SHIFT_JIS', (b'"already"',)), (b'"already"',)) + # The charset name may itself be bytes. + self.assertEqual(enc(b'UTF-8', ('"café"',)), ('"café"'.encode('utf-8'),)) + # A charset with no Python codec cannot encode str criteria; bytes + # criteria must be used with such server-only charsets. + self.assertRaises(LookupError, enc, 'NF_Z_62-010_(1973)', ('TEXT',)) + self.assertEqual(enc('NF_Z_62-010_(1973)', (b'TEXT',)), (b'TEXT',)) + def test_astring_idempotent(self): # Quoting an already quoted argument should not change it, so that # quoting twice gives the same result as quoting once. @@ -337,7 +362,11 @@ def handle(self): except StopIteration: self.continuation = None continue - splitline = splitargs(line.decode().removesuffix('\r\n')) + self.server.line = line + # surrogateescape so a criterion encoded in a non-UTF-8 charset + # does not crash the handler; tests inspect server.line for bytes. + splitline = splitargs(line.decode('utf-8', 'surrogateescape') + .removesuffix('\r\n')) tag = splitline[0] cmd = splitline[1] args = splitline[2:] @@ -1494,7 +1523,17 @@ def test_search(self): self.assertEqual(data, [b'43']) self.assertEqual(server.args, ['CHARSET', 'UTF-8', 'TEXT', 'XXXXXX']) - typ, data = client.search('NF_Z_62-010_(1973)', 'TEXT', 'XXXXXX') + # A non-ASCII str criterion is encoded to the declared charset (KOI8-U + # here, which is not UTF-8, so check the encoded bytes on the wire). + response[:] = ['* SEARCH 43'] + typ, data = client.search('KOI8-U', 'SUBJECT', '"Київ"') + self.assertEqual(typ, 'OK') + self.assertIn(b'CHARSET KOI8-U ', server.line) + self.assertIn('"Київ"'.encode('koi8-u'), server.line) + + # bytes criteria: NF_Z_62-010 has no Python codec, so this exercises + # charset-name quoting without criteria encoding. + typ, data = client.search('NF_Z_62-010_(1973)', b'TEXT', b'XXXXXX') self.assertEqual(typ, 'OK') self.assertEqual(server.args, ['CHARSET', '"NF_Z_62-010_(1973)"', 'TEXT', 'XXXXXX']) @@ -1549,10 +1588,16 @@ def test_sort(self): self.assertEqual(data, [br'']) self.assertEqual(server.args, ['(SUBJECT)', 'US-ASCII', 'TEXT', '"not in mailbox"']) - typ, data = client.sort('SUBJECT', 'NF_Z_62-010_(1973)', 'TEXT', '"not in mailbox"') + typ, data = client.sort('SUBJECT', 'NF_Z_62-010_(1973)', b'TEXT', b'"not in mailbox"') self.assertEqual(typ, 'OK') self.assertEqual(server.args, ['(SUBJECT)', '"NF_Z_62-010_(1973)"', 'TEXT', '"not in mailbox"']) + # A non-ASCII str criterion is encoded to the declared charset. + response[:] = ['* SORT'] + typ, data = client.sort('(SUBJECT)', 'KOI8-U', 'TEXT', '"Київ"') + self.assertEqual(typ, 'OK') + self.assertIn('"Київ"'.encode('koi8-u'), server.line) + def test_uid_sort(self): response = [] client, server = self._setup(make_simple_handler('UID', response, @@ -1577,10 +1622,16 @@ def test_uid_sort(self): self.assertEqual(data, [br'']) self.assertEqual(server.args, ['SORT', '(SUBJECT)', 'US-ASCII', 'TEXT', '"not in mailbox"']) - typ, data = client.uid('sort', 'SUBJECT', 'NF_Z_62-010_(1973)', 'TEXT', '"not in mailbox"') + typ, data = client.uid('sort', 'SUBJECT', 'NF_Z_62-010_(1973)', b'TEXT', b'"not in mailbox"') self.assertEqual(typ, 'OK') self.assertEqual(server.args, ['SORT', '(SUBJECT)', '"NF_Z_62-010_(1973)"', 'TEXT', '"not in mailbox"']) + # A non-ASCII str criterion is encoded to the declared charset. + response[:] = ['* SORT'] + typ, data = client.uid('sort', '(SUBJECT)', 'KOI8-U', 'TEXT', '"Київ"') + self.assertEqual(typ, 'OK') + self.assertIn('"Київ"'.encode('koi8-u'), server.line) + def test_thread(self): response = [] client, server = self._setup(make_simple_handler('THREAD', response)) @@ -1618,10 +1669,16 @@ def test_thread(self): b'(199)(200 202)(201)(203)(204)(205 206 207)(208)']) self.assertEqual(server.args, ['ORDEREDSUBJECT', 'US-ASCII', 'TEXT', '"gewp"']) - typ, data = client.thread('ORDEREDSUBJECT', 'NF_Z_62-010_(1973)', 'TEXT', '"gewp"') + typ, data = client.thread('ORDEREDSUBJECT', 'NF_Z_62-010_(1973)', b'TEXT', b'"gewp"') self.assertEqual(typ, 'OK') self.assertEqual(server.args, ['ORDEREDSUBJECT', '"NF_Z_62-010_(1973)"', 'TEXT', '"gewp"']) + # A non-ASCII str criterion is encoded to the declared charset. + response[:] = ['* THREAD (1)'] + typ, data = client.thread('ORDEREDSUBJECT', 'KOI8-U', 'TEXT', '"Київ"') + self.assertEqual(typ, 'OK') + self.assertIn('"Київ"'.encode('koi8-u'), server.line) + def test_uid_thread(self): response = [] client, server = self._setup(make_simple_handler('UID', response, @@ -1660,10 +1717,16 @@ def test_uid_thread(self): b'(199)(200 202)(201)(203)(204)(205 206 207)(208)']) self.assertEqual(server.args, ['THREAD', 'ORDEREDSUBJECT', 'US-ASCII', 'TEXT', '"gewp"']) - typ, data = client.uid('THREAD', 'ORDEREDSUBJECT', 'NF_Z_62-010_(1973)', 'TEXT', '"gewp"') + typ, data = client.uid('THREAD', 'ORDEREDSUBJECT', 'NF_Z_62-010_(1973)', b'TEXT', b'"gewp"') self.assertEqual(typ, 'OK') self.assertEqual(server.args, ['THREAD', 'ORDEREDSUBJECT', '"NF_Z_62-010_(1973)"', 'TEXT', '"gewp"']) + # A non-ASCII str criterion is encoded to the declared charset. + response[:] = ['* THREAD (1)'] + typ, data = client.uid('THREAD', 'ORDEREDSUBJECT', 'KOI8-U', 'TEXT', '"Київ"') + self.assertEqual(typ, 'OK') + self.assertIn('"Київ"'.encode('koi8-u'), server.line) + def test_delete(self): client, server = self._setup(make_simple_handler('DELETE')) client.login('user', 'pass') diff --git a/Misc/NEWS.d/next/Library/2026-07-10-16-00-00.gh-issue-153494.qCh8rT.rst b/Misc/NEWS.d/next/Library/2026-07-10-16-00-00.gh-issue-153494.qCh8rT.rst new file mode 100644 index 00000000000000..4c795d0a5bc0c2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-10-16-00-00.gh-issue-153494.qCh8rT.rst @@ -0,0 +1,6 @@ +:meth:`imaplib.IMAP4.search`, :meth:`~imaplib.IMAP4.sort` and +:meth:`~imaplib.IMAP4.thread` (and the corresponding ``uid`` commands) now +encode :class:`str` search criteria to the declared *charset*, so +international search text can be passed as ordinary :class:`str`. A criterion +passed as :class:`bytes` is sent unchanged, for use with a charset that Python +has no codec for. From 4e46ab2f45fdf1293146b7069a72169bf741a14b Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 10 Jul 2026 14:34:54 +0300 Subject: [PATCH 2/3] gh-153494: Use a charset unknown to iconv in the LookupError test The iconv-based codec (gh-152997) now resolves NF_Z_62-010 on platforms that provide iconv(3), so it no longer triggers LookupError. Use a name that neither Python nor iconv knows. Co-Authored-By: Claude Opus 4.8 --- Lib/test/test_imaplib.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index 8e1291485e1aad..831ccbae3806b5 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -225,10 +225,11 @@ def test_encode_criteria(self): self.assertEqual(enc('SHIFT_JIS', (b'"already"',)), (b'"already"',)) # The charset name may itself be bytes. self.assertEqual(enc(b'UTF-8', ('"café"',)), ('"café"'.encode('utf-8'),)) - # A charset with no Python codec cannot encode str criteria; bytes - # criteria must be used with such server-only charsets. - self.assertRaises(LookupError, enc, 'NF_Z_62-010_(1973)', ('TEXT',)) - self.assertEqual(enc('NF_Z_62-010_(1973)', (b'TEXT',)), (b'TEXT',)) + # A charset with no codec at all (not even via the iconv codec) cannot + # encode str criteria; bytes criteria must be used with such + # server-only charsets. + self.assertRaises(LookupError, enc, 'no-such-charset', ('TEXT',)) + self.assertEqual(enc('no-such-charset', (b'TEXT',)), (b'TEXT',)) def test_astring_idempotent(self): # Quoting an already quoted argument should not change it, so that @@ -1531,8 +1532,8 @@ def test_search(self): self.assertIn(b'CHARSET KOI8-U ', server.line) self.assertIn('"Київ"'.encode('koi8-u'), server.line) - # bytes criteria: NF_Z_62-010 has no Python codec, so this exercises - # charset-name quoting without criteria encoding. + # bytes criteria keep this focused on charset-name quoting (the + # parentheses force the name to be quoted) without criteria encoding. typ, data = client.search('NF_Z_62-010_(1973)', b'TEXT', b'XXXXXX') self.assertEqual(typ, 'OK') self.assertEqual(server.args, ['CHARSET', '"NF_Z_62-010_(1973)"', 'TEXT', 'XXXXXX']) From 9ac84852513c4c1e51e13fbaff31fd8276de514d Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 10 Jul 2026 15:42:38 +0300 Subject: [PATCH 3/3] gh-153494: Add What's New entry and clarify the charset docs Document that str search criteria are encoded to the declared charset only when a charset is given; under UTF8=ACCEPT the charset is None and criteria are sent using the connection encoding. Co-Authored-By: Claude Opus 4.8 --- Doc/library/imaplib.rst | 20 +++++++++++-------- Doc/whatsnew/3.16.rst | 8 ++++++++ ...-07-10-16-00-00.gh-issue-153494.qCh8rT.rst | 14 +++++++------ 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index a7b09a8ac9e96b..7b79d81790b667 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -610,10 +610,12 @@ An :class:`IMAP4` instance has the following methods: If *uid* is true, the message numbers in the response are UIDs (``UID SEARCH``). - A criterion passed as :class:`str` is encoded to *charset* (which must name - a codec known to Python); pass :class:`bytes` to send a criterion that is - already encoded, for example when *charset* is one that Python does not - support. + A criterion passed as :class:`str` is encoded to *charset* + (which must name a codec known to Python); + pass :class:`bytes` to send a criterion that is already encoded, + for example when *charset* is one that Python does not support. + When *charset* is ``None`` (as it must be under ``UTF8=ACCEPT``), + the criterion is sent using the connection's encoding instead. Example:: @@ -690,8 +692,9 @@ An :class:`IMAP4` instance has the following methods: If *uid* is true, the message numbers in the response are UIDs (``UID SORT``). - As with :meth:`search`, a *search_criterion* passed as :class:`str` is - encoded to *charset*; pass :class:`bytes` to send one already encoded. + As with :meth:`search`, + a *search_criterion* passed as :class:`str` is encoded to *charset*; + pass :class:`bytes` to send one already encoded. This is an ``IMAP4rev1`` extension command. @@ -786,8 +789,9 @@ An :class:`IMAP4` instance has the following methods: If *uid* is true, the message numbers in the response are UIDs (``UID THREAD``). - As with :meth:`search`, a *search_criterion* passed as :class:`str` is - encoded to *charset*; pass :class:`bytes` to send one already encoded. + As with :meth:`search`, + a *search_criterion* passed as :class:`str` is encoded to *charset*; + pass :class:`bytes` to send one already encoded. This is an ``IMAP4rev1`` extension command. diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index 5e8abc7034bea0..4d2bf80336c70a 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -276,6 +276,14 @@ imaplib :meth:`~imaplib.IMAP4.uid`. (Contributed by Serhiy Storchaka in :gh:`153502`.) +* :meth:`~imaplib.IMAP4.search`, :meth:`~imaplib.IMAP4.sort` + and :meth:`~imaplib.IMAP4.thread` (and the corresponding ``uid`` commands) + now encode :class:`str` search criteria to the declared *charset*, + so international search text can be passed as an ordinary :class:`str`. + When *charset* is ``None`` (as it must be under ``UTF8=ACCEPT``), + the criteria are sent using the connection encoding instead. + (Contributed by Serhiy Storchaka in :gh:`153494`.) + ipaddress --------- diff --git a/Misc/NEWS.d/next/Library/2026-07-10-16-00-00.gh-issue-153494.qCh8rT.rst b/Misc/NEWS.d/next/Library/2026-07-10-16-00-00.gh-issue-153494.qCh8rT.rst index 4c795d0a5bc0c2..6e83940efcc180 100644 --- a/Misc/NEWS.d/next/Library/2026-07-10-16-00-00.gh-issue-153494.qCh8rT.rst +++ b/Misc/NEWS.d/next/Library/2026-07-10-16-00-00.gh-issue-153494.qCh8rT.rst @@ -1,6 +1,8 @@ -:meth:`imaplib.IMAP4.search`, :meth:`~imaplib.IMAP4.sort` and -:meth:`~imaplib.IMAP4.thread` (and the corresponding ``uid`` commands) now -encode :class:`str` search criteria to the declared *charset*, so -international search text can be passed as ordinary :class:`str`. A criterion -passed as :class:`bytes` is sent unchanged, for use with a charset that Python -has no codec for. +:meth:`imaplib.IMAP4.search`, :meth:`~imaplib.IMAP4.sort` +and :meth:`~imaplib.IMAP4.thread` (and the corresponding ``uid`` commands) +now encode :class:`str` search criteria to the declared *charset*, +so international search text can be passed as ordinary :class:`str`. +When *charset* is ``None`` (as it must be under ``UTF8=ACCEPT``), +the criteria are sent using the connection encoding instead. +A criterion passed as :class:`bytes` is sent unchanged, +for use with a charset that Python has no codec for.