gh-152905: Decode LC_TIME items in nl_langinfo() from glibc wide data

Doc/library/locale.rst

@@ -342,6 +342,10 @@ The :mod:`!locale` module defines the following exception and functions:
    .. versionchanged:: 3.14
       The function now temporarily sets the ``LC_CTYPE`` locale in some cases.
 
+   .. versionchanged:: next
+      On glibc, the ``LC_TIME`` items (except ``ERA``) are now decoded
+      independently of the ``LC_CTYPE`` encoding.
+
 
 .. function:: getdefaultlocale([envvars])
 

Lib/test/test__locale.py

@@ -1,3 +1,4 @@
+import _locale
 from _locale import (setlocale, LC_ALL, LC_CTYPE, LC_NUMERIC, LC_TIME, localeconv, Error)
 try:
     from _locale import (RADIXCHAR, THOUSEP, nl_langinfo)
@@ -6,8 +7,9 @@
 
 import locale
 import sys
+import unicodedata
 import unittest
-from platform import uname
+from platform import uname, libc_ver
 
 from test import support
 
@@ -271,6 +273,79 @@ def test_era_nl_langinfo(self):
         if not tested:
             self.skipTest('no suitable locales')
 
+    @unittest.skipUnless(nl_langinfo, "nl_langinfo is not available")
+    @unittest.skipUnless(libc_ver()[0] == 'glibc',
+                         "wide nl_langinfo variants are glibc-specific")
+    def test_nl_langinfo_encoding_independent(self):
+        # gh-152905: The LC_TIME text items are decoded independently of the
+        # LC_CTYPE encoding (on glibc via the wide nl_langinfo variants), so
+        # the same locale in different encodings yields identical strings.
+        # ERA has no wide variant and is not tested.
+        self.addCleanup(setlocale, LC_TIME, setlocale(LC_TIME))
+
+        names = [f'MON_{i}' for i in range(1, 13)]
+        names += [f'ABMON_{i}' for i in range(1, 13)]
+        names += [f'DAY_{i}' for i in range(1, 8)]
+        names += [f'ABDAY_{i}' for i in range(1, 8)]
+        names += ['AM_STR', 'PM_STR', 'D_T_FMT', 'D_FMT', 'T_FMT']
+        names += [name for name in ('T_FMT_AMPM', 'ERA_D_FMT', 'ERA_D_T_FMT',
+                                    'ERA_T_FMT', 'ALT_DIGITS', '_DATE_FMT')
+                  if hasattr(_locale, name)]
+        items = [(name, getattr(_locale, name)) for name in names]
+
+        # Legacy (non-UTF-8) locales, compared against their UTF-8 variant.
+        legacy_locales = [
+            'en_US.ISO8859-1',
+            'es_ES.ISO8859-1',
+            'fr_FR.ISO8859-1',
+            'de_DE.ISO8859-1',
+            'pl_PL.ISO8859-2',
+            'mt_MT.ISO8859-3',
+            'ar_SA.ISO8859-6',
+            'el_GR.ISO8859-7',
+            'he_IL.ISO8859-8',
+            'tr_TR.ISO8859-9',
+            'lt_LT.ISO8859-13',
+            'cy_GB.ISO8859-14',
+            'et_EE.ISO8859-15',
+            'uk_UA.KOI8-U',
+            'bg_BG.CP1251',
+            'ja_JP.EUC-JP',
+            'ko_KR.EUC-KR',
+            'zh_CN.GB2312',
+            'zh_TW.BIG5',
+            'th_TH.TIS-620',
+        ]
+
+        # An 8-bit locale substitutes an equivalent for a space it cannot
+        # encode (e.g. es_ES has U+202F in UTF-8 but U+00A0 in ISO-8859-1),
+        # so fold spaces before comparing.
+        def fold_spaces(s):
+            return ''.join(' ' if unicodedata.category(c) == 'Zs' else c
+                           for c in s)
+
+        tested = False
+        for legacy_locale in legacy_locales:
+            locs = (legacy_locale.partition('.')[0] + '.UTF-8', legacy_locale)
+            values = []
+            for loc in locs:
+                try:
+                    setlocale(LC_TIME, loc)
+                except Error:
+                    break
+                values.append({name: nl_langinfo(item) for name, item in items})
+            if len(values) < 2:
+                continue
+            tested = True
+
+            # The result must not depend on the locale encoding.
+            for name, item in items:
+                with self.subTest(locales=locs, name=name):
+                    self.assertEqual(fold_spaces(values[0][name]),
+                                     fold_spaces(values[1][name]))
+        if not tested:
+            self.skipTest('no suitable locale pairs')
+
     def test_float_parsing(self):
         # Bug #1391872: Test whether float parsing is okay on European
         # locales.

Misc/NEWS.d/next/Library/2026-07-01-23-17-27.gh-issue-152905.wLnGqR.rst

@@ -0,0 +1,3 @@
+On glibc, :func:`locale.nl_langinfo` now decodes the ``LC_TIME`` items (such
+as the month and day names) using the wide locale data, so the result no
+longer depends on the ``LC_CTYPE`` encoding.

Modules/_localemodule.c

@@ -561,11 +561,23 @@ _locale__getdefaultlocale_impl(PyObject *module)
 #endif
 
 #ifdef HAVE_LANGINFO_H
-#define LANGINFO(X, Y) {#X, X, Y}
+/* On glibc, LC_TIME text items also have a wide (_NL_W*) form, named
+   _NL_W<narrow name>, that nl_langinfo() returns as wchar_t*.  LANGINFO_NW
+   marks the items that have no wide counterpart. */
+#ifdef __GLIBC__
+#  define LANGINFO(X, Y) {#X, X, Y, _NL_W ## X}
+#  define LANGINFO_NW(X, Y) {#X, X, Y, 0}
+#else
+#  define LANGINFO(X, Y) {#X, X, Y}
+#  define LANGINFO_NW(X, Y) {#X, X, Y}
+#endif
 static struct langinfo_constant{
     const char *name;
     int value;
     int category;
+#ifdef __GLIBC__
+    nl_item wide;  /* wide _NL_W* counterpart, or 0 if none */
+#endif
 } langinfo_constants[] =
 {
     /* These constants should exist on any langinfo implementation */
@@ -613,16 +625,16 @@ static struct langinfo_constant{
 
 #ifdef RADIXCHAR
     /* The following are not available with glibc 2.0 */
-    LANGINFO(RADIXCHAR, LC_NUMERIC),
-    LANGINFO(THOUSEP, LC_NUMERIC),
+    LANGINFO_NW(RADIXCHAR, LC_NUMERIC),
+    LANGINFO_NW(THOUSEP, LC_NUMERIC),
     /* YESSTR and NOSTR are deprecated in glibc, since they are
        a special case of message translation, which should be rather
        done using gettext. So we don't expose it to Python in the
        first place.
     LANGINFO(YESSTR, LC_MESSAGES),
     LANGINFO(NOSTR, LC_MESSAGES),
     */
-    LANGINFO(CRNCYSTR, LC_MONETARY),
+    LANGINFO_NW(CRNCYSTR, LC_MONETARY),
 #endif
 
     LANGINFO(D_T_FMT, LC_TIME),
@@ -636,13 +648,13 @@ static struct langinfo_constant{
        a few of the others.
        Solution: ifdef-test them all. */
 #ifdef CODESET
-    LANGINFO(CODESET, LC_CTYPE),
+    LANGINFO_NW(CODESET, LC_CTYPE),
 #endif
 #ifdef T_FMT_AMPM
     LANGINFO(T_FMT_AMPM, LC_TIME),
 #endif
 #ifdef ERA
-    LANGINFO(ERA, LC_TIME),
+    LANGINFO_NW(ERA, LC_TIME),
 #endif
 #ifdef ERA_D_FMT
     LANGINFO(ERA_D_FMT, LC_TIME),
@@ -657,10 +669,10 @@ static struct langinfo_constant{
     LANGINFO(ALT_DIGITS, LC_TIME),
 #endif
 #ifdef YESEXPR
-    LANGINFO(YESEXPR, LC_MESSAGES),
+    LANGINFO_NW(YESEXPR, LC_MESSAGES),
 #endif
 #ifdef NOEXPR
-    LANGINFO(NOEXPR, LC_MESSAGES),
+    LANGINFO_NW(NOEXPR, LC_MESSAGES),
 #endif
 #ifdef _DATE_FMT
     /* This is not available in all glibc versions that have CODESET. */
@@ -709,15 +721,14 @@ restore_locale(char *oldloc)
 }
 
 #ifdef __GLIBC__
-#if defined(ALT_DIGITS) || defined(ERA)
 static PyObject *
-decode_strings(const char *result, size_t max_count)
+decode_strings(const char *result)
 {
     /* Convert a sequence of NUL-separated C strings to a Python string
      * containing semicolon separated items. */
     size_t i = 0;
     size_t count = 0;
-    for (; count < max_count && result[i]; count++) {
+    for (; result[i]; count++) {
         i += strlen(result + i) + 1;
     }
     char *buf = PyMem_Malloc(i);
@@ -737,7 +748,35 @@ decode_strings(const char *result, size_t max_count)
     return pyresult;
 }
 #endif
-#endif
+
+#ifdef __GLIBC__
+static PyObject *
+decode_wide_strings(const wchar_t *result, size_t max_count)
+{
+    /* Wide-character counterpart of decode_strings(): convert a sequence of
+     * NUL-separated wchar_t strings to a str with semicolon-separated items. */
+    size_t i = 0;
+    size_t count = 0;
+    for (; count < max_count && result[i]; count++) {
+        i += wcslen(result + i) + 1;
+    }
+    wchar_t *buf = PyMem_New(wchar_t, i);
+    if (buf == NULL) {
+        PyErr_NoMemory();
+        return NULL;
+    }
+    memcpy(buf, result, i * sizeof(wchar_t));
+    /* Replace all NULs with semicolons. */
+    i = 0;
+    while (--count) {
+        i += wcslen(buf + i);
+        buf[i++] = L';';
+    }
+    PyObject *pyresult = PyUnicode_FromWideChar(buf, -1);
+    PyMem_Free(buf);
+    return pyresult;
+}
+#endif  /* __GLIBC__ */
 
 /*[clinic input]
 _locale.nl_langinfo
@@ -758,6 +797,22 @@ _locale_nl_langinfo_impl(PyObject *module, int item)
        crash PyUnicode_FromString.  */
     for (i = 0; langinfo_constants[i].name; i++) {
         if (langinfo_constants[i].value == item) {
+#ifdef __GLIBC__
+            /* Prefer the wide variant: it is decoded independently of the
+               LC_CTYPE encoding. */
+            nl_item wide = langinfo_constants[i].wide;
+            if (wide) {
+                const wchar_t *wresult = (const wchar_t *)nl_langinfo(wide);
+                if (wresult == NULL) {
+                    wresult = L"";
+                }
+                /* ALT_DIGITS is a sequence of NUL-separated strings. */
+                if (item == ALT_DIGITS && *wresult) {
+                    return decode_wide_strings(wresult, 100);
+                }
+                return PyUnicode_FromWideChar(wresult, -1);
+            }
+#endif
             /* Check NULL as a workaround for GNU libc's returning NULL
                instead of an empty string for nl_langinfo(ERA).  */
             const char *result = nl_langinfo(item);
@@ -766,13 +821,8 @@ _locale_nl_langinfo_impl(PyObject *module, int item)
             if (langinfo_constants[i].category != LC_CTYPE
                 && *result && (
 #ifdef __GLIBC__
-                    // gh-133740: Always change the locale for ALT_DIGITS and ERA
-#  ifdef ALT_DIGITS
-                    item == ALT_DIGITS ||
-#  endif
-#  ifdef ERA
+                    // gh-133740: Always change the locale for ERA
                     item == ERA ||
-#  endif
 #endif
                     !is_all_ascii(result))
                 && change_locale(langinfo_constants[i].category, &oldloc) < 0)
@@ -784,18 +834,10 @@ _locale_nl_langinfo_impl(PyObject *module, int item)
             /* According to the POSIX specification the result must be
              * a sequence of semicolon-separated strings.
              * But in Glibc they are NUL-separated. */
-#ifdef ALT_DIGITS
-            if (item == ALT_DIGITS && *result) {
-                pyresult = decode_strings(result, 100);
-            }
-            else
-#endif
-#ifdef ERA
             if (item == ERA && *result) {
-                pyresult = decode_strings(result, SIZE_MAX);
+                pyresult = decode_strings(result);
             }
             else
-#endif
 #endif
             {
                 pyresult = PyUnicode_DecodeLocale(result, NULL);