Language: 48622: Set LC_ALL for native gettext on macOS and BSD

components/ILIAS/Language/README.md

@@ -178,7 +178,9 @@ Only other modules (plugins) are read from `lng_data`/`lng_modules`.
   * macOS and FreeBSD: gettext from GNU libintl
     (Homebrew, Ports) and a system locale of `NativeGettext::SYSTEM_LOCALES`;
     the libc there ignores `LOCPATH`, no locale is shipped (`C.UTF-8` does not
-    work: GNU libintl ignores `LANGUAGE` there);
+    work: GNU libintl ignores `LANGUAGE` there); locale variables of the
+    environment (e.g. `LANG=C.UTF-8`) do not matter: native gettext sets the
+    environment variable `LC_ALL` to that locale (child processes inherit it);
   * musl is not supported (ignores `LOCPATH`).
 
   Build on the platform that serves ILIAS (with Docker in the container): the

components/ILIAS/Language/src/ComponentTranslation/NativeGettext.php

@@ -30,7 +30,8 @@
  *   LOCALE_NAME; activate() loads it via LOCPATH and removes LOCPATH again right away (while it is set,
  *   setlocale() finds no system locale);
  * - macOS/BSD (see usesSystemLocale()): the C library ignores LOCPATH, gettext comes from GNU libintl, so
- *   activate() sets the first existing of SYSTEM_LOCALES (GNU libintl ignores LANGUAGE for "C.<codeset>");
+ *   activate() sets the first existing of SYSTEM_LOCALES for LC_MESSAGES and the environment variable LC_ALL
+ *   (GNU libintl ignores LANGUAGE for "C.<codeset>");
  * - LANGUAGE is CATALOG_LANGUAGE, so "<language>" is always "messages"; the language of a catalog is part
  *   of its domain ("<module>.<lang>", see MigratedLanguageFilePaths), as the C library caches per process
  *   and a later LANGUAGE change would not reliably switch catalogs;
@@ -40,7 +41,8 @@
  * setlocale() reset to "C"), so no shutdown function is needed (it would stop later shutdown functions
  * from translating). In a thread-safe (ZTS) PHP the end of another thread's request can take them back
  * mid-request: isActive() checks, reactivateIfLost() activates again (the administration shows a hint,
- * see isThreadSafe()). Child processes inherit LANGUAGE=messages, which is harmless.
+ * see isThreadSafe()). Child processes inherit LANGUAGE=messages, which is harmless, and on macOS/BSD
+ * LC_ALL=<active locale>, which overrides the locale of the environment for them.
  *
  * The C library never reloads a catalog file it has read once (also not a replaced one), so every new
  * state (a build, an overlay revision) gets a directory of its own, and binding the domain loads it.
@@ -70,10 +72,10 @@ final class NativeGettext
     private static ?string $active_locale = null;
 
     /**
-     * LANGUAGE and LC_MESSAGES before the first activate() since the last restore() (`false`: not
-     * set), see restore().
+     * LANGUAGE, LC_MESSAGES and the environment variable LC_ALL before the first activate() since the
+     * last restore() (`false`: not set), see restore().
      *
-     * @var array{language: string|false, messages: string|false}|null
+     * @var array{language: string|false, messages: string|false, all: string|false}|null
      */
     private static ?array $previous = null;
 
@@ -144,7 +146,7 @@ public static function activate(string $locale_directory): bool
             return false;
         }
 
-        self::$previous ??= ['language' => getenv('LANGUAGE'), 'messages' => setlocale(LC_MESSAGES, '0')];
+        self::$previous ??= self::processState();
         $locpath = getenv('LOCPATH');
         putenv('LOCPATH=' . $locale_directory);
         $locale = setlocale(LC_MESSAGES, self::LOCALE_NAME);
@@ -160,7 +162,7 @@ public static function activate(string $locale_directory): bool
 
     private static function activateSystemLocale(string $locale_directory): bool
     {
-        self::$previous ??= ['language' => getenv('LANGUAGE'), 'messages' => setlocale(LC_MESSAGES, '0')];
+        self::$previous ??= self::processState();
         $locale = setlocale(LC_MESSAGES, self::SYSTEM_LOCALES);
         if ($locale === false) {
             self::$problem = sprintf(
@@ -170,10 +172,20 @@ private static function activateSystemLocale(string $locale_directory): bool
             );
             return false;
         }
+        // GNU libintl on macOS reads the locale of the environment before setlocale()'s, and "C" there ignores LANGUAGE
+        putenv('LC_ALL=' . $locale);
 
         return self::activated($locale_directory, $locale);
     }
 
+    /**
+     * @return array{language: string|false, messages: string|false, all: string|false}
+     */
+    private static function processState(): array
+    {
+        return ['language' => getenv('LANGUAGE'), 'messages' => setlocale(LC_MESSAGES, '0'), 'all' => getenv('LC_ALL')];
+    }
+
     private static function activated(string $locale_directory, string $locale): bool
     {
         putenv('LANGUAGE=' . self::CATALOG_LANGUAGE);
@@ -186,13 +198,14 @@ private static function activated(string $locale_directory, string $locale): boo
 
     /**
      * Whether activate() succeeded and its settings are still in place (other code, e.g.
-     * ilInitialisation::initLocale() with LC_ALL, may have changed LC_MESSAGES since).
+     * ilInitialisation::initLocale() with setlocale(LC_ALL, …), may have changed LC_MESSAGES since).
      */
     public static function isActive(): bool
     {
         return self::$locale_directory !== null
             && setlocale(LC_MESSAGES, '0') === self::$active_locale
-            && getenv('LANGUAGE') === self::CATALOG_LANGUAGE;
+            && getenv('LANGUAGE') === self::CATALOG_LANGUAGE
+            && (!self::usesSystemLocale() || getenv('LC_ALL') === self::$active_locale);
     }
 
     /**
@@ -211,7 +224,7 @@ public static function reactivateIfLost(): bool
     }
 
     /**
-     * Takes LANGUAGE and LC_MESSAGES back to the state before the first activate(); settings changed by
+     * Takes LANGUAGE, LC_MESSAGES and LC_ALL back to the state before the first activate(); settings changed by
      * other code since stay. Only for long-running processes (e.g. Setup's build after its check), a web
      * request needs no call. No-op in a thread-safe PHP, where other threads may use them. The activated
      * directory is kept, so reactivateIfLost() can activate it again.
@@ -227,6 +240,9 @@ public static function restore(): void
         if (getenv('LANGUAGE') === self::CATALOG_LANGUAGE) {
             putenv(self::$previous['language'] === false ? 'LANGUAGE' : 'LANGUAGE=' . self::$previous['language']);
         }
+        if (self::usesSystemLocale() && getenv('LC_ALL') === self::$active_locale) {
+            putenv(self::$previous['all'] === false ? 'LC_ALL' : 'LC_ALL=' . self::$previous['all']);
+        }
         self::$previous = null;
     }
 

components/ILIAS/Language/tests/ComponentTranslation/NativeGettextTest.php

@@ -27,7 +27,7 @@
 use PHPUnit\Framework\TestCase;
 
 /**
- * NativeGettext changes process-wide state (LC_MESSAGES, LANGUAGE, LOCPATH): what activate() leaves behind and
+ * NativeGettext changes process-wide state (LC_MESSAGES, LANGUAGE, LOCPATH, LC_ALL): what activate() leaves behind and
  * restore() gives back, losing and regaining the settings to other code, a build native gettext cannot use, lookups of
  * messages that cannot be looked up.
  *
@@ -42,7 +42,7 @@ class NativeGettextTest extends TestCase
     private string $locale_directory;
 
     /**
-     * @var array{language: string|false, locpath: string|false, messages: string|false}
+     * @var array{language: string|false, locpath: string|false, all: string|false, messages: string|false}
      */
     private array $process_state;
 
@@ -52,6 +52,7 @@ protected function setUp(): void
         $this->process_state = [
             'language' => getenv('LANGUAGE'),
             'locpath' => getenv('LOCPATH'),
+            'all' => getenv('LC_ALL'),
             'messages' => setlocale(LC_MESSAGES, '0'),
         ];
         $this->root = sys_get_temp_dir() . '/ilias_native_gettext_' . bin2hex(random_bytes(6));
@@ -78,6 +79,7 @@ protected function tearDown(): void
         // whatever a test did to the process, the next one starts as this one did
         putenv($this->process_state['language'] === false ? 'LANGUAGE' : 'LANGUAGE=' . $this->process_state['language']);
         putenv($this->process_state['locpath'] === false ? 'LOCPATH' : 'LOCPATH=' . $this->process_state['locpath']);
+        putenv($this->process_state['all'] === false ? 'LC_ALL' : 'LC_ALL=' . $this->process_state['all']);
         if ($this->process_state['messages'] !== false) {
             setlocale(LC_MESSAGES, $this->process_state['messages']);
         }
@@ -205,7 +207,21 @@ public function testSystemLocaleIsOnlyActiveAfterActivateAndRestoreGivesTheProce
         $this->assertSame($language, getenv('LANGUAGE'));
     }
 
-    #[DataProvider('foreignChanges')]
+    /**
+     * @return iterable<string, array{\Closure(): void}>
+     */
+    public static function foreignChangesInTheSystemLocaleBranch(): iterable
+    {
+        yield from self::foreignChanges();
+        yield 'LC_ALL environment variable set to C' => [static function (): void {
+            putenv('LC_ALL=C');
+        }];
+        yield 'LC_ALL environment variable removed' => [static function (): void {
+            putenv('LC_ALL');
+        }];
+    }
+
+    #[DataProvider('foreignChangesInTheSystemLocaleBranch')]
     public function testReactivateIfLostRepairsWhatOtherCodeChangedInTheSystemLocaleBranch(\Closure $change): void
     {
         $this->requireSystemLocale();
@@ -220,6 +236,100 @@ public function testReactivateIfLostRepairsWhatOtherCodeChangedInTheSystemLocale
         $this->assertSame('Hallo', NativeGettext::translate(self::DOMAIN, 'greeting'));
     }
 
+    /**
+     * @return iterable<string, array{string|false}>
+     */
+    public static function lcAllBeforeSystemLocaleActivation(): iterable
+    {
+        yield 'unset' => [false];
+        yield 'C' => ['C'];
+        yield 'another locale' => ['de_DE.UTF-8'];
+    }
+
+    #[DataProvider('lcAllBeforeSystemLocaleActivation')]
+    public function testSystemLocaleBranchSetsLcAllToTheActiveLocaleAndRestoreGivesItBackExactly(string|false $lc_all): void
+    {
+        $this->requireSystemLocale();
+        NativeGettext::useSystemLocaleForTests(true);
+        putenv($lc_all === false ? 'LC_ALL' : 'LC_ALL=' . $lc_all);
+
+        $this->activateSystemLocale();
+
+        $this->assertSame(setlocale(LC_MESSAGES, '0'), getenv('LC_ALL'));
+        $this->assertTrue(NativeGettext::isActive());
+        $this->assertSame('Hallo', NativeGettext::translate(self::DOMAIN, 'greeting'));
+
+        NativeGettext::restore();
+
+        $this->assertSame($lc_all, getenv('LC_ALL'));
+        $this->assertFalse(NativeGettext::isActive());
+    }
+
+    #[DataProvider('lcAllBeforeSystemLocaleActivation')]
+    public function testRestoreAfterARepeatedActivationGivesBackLcAllFromBeforeTheFirstOne(string|false $lc_all): void
+    {
+        $this->requireSystemLocale();
+        NativeGettext::useSystemLocaleForTests(true);
+        putenv($lc_all === false ? 'LC_ALL' : 'LC_ALL=' . $lc_all);
+        $this->activateSystemLocale();
+
+        $this->assertTrue(NativeGettext::activate($this->root . '/other-no-locale'), (string) NativeGettext::getProblem());
+        putenv('LC_ALL=fr_FR.UTF-8');
+        $this->assertTrue(NativeGettext::reactivateIfLost());
+        $this->assertSame(setlocale(LC_MESSAGES, '0'), getenv('LC_ALL'));
+
+        NativeGettext::restore();
+
+        $this->assertSame($lc_all, getenv('LC_ALL'));
+    }
+
+    public function testRestoreChangesNothingInAThreadSafePhpInTheSystemLocaleBranch(): void
+    {
+        $this->requireSystemLocale();
+        NativeGettext::useSystemLocaleForTests(true);
+        putenv('LC_ALL=C');
+        $this->activateSystemLocale();
+        $active = setlocale(LC_MESSAGES, '0');
+        NativeGettext::useThreadSafeForTests(true);
+
+        NativeGettext::restore();
+
+        $this->assertSame($active, getenv('LC_ALL'));
+        $this->assertTrue(NativeGettext::isActive());
+    }
+
+    public function testRestoreLeavesALcAllOtherCodeChangedSinceAsItIsInTheSystemLocaleBranch(): void
+    {
+        $this->requireSystemLocale();
+        NativeGettext::useSystemLocaleForTests(true);
+        putenv('LC_ALL');
+        $this->activateSystemLocale();
+        putenv('LC_ALL=fr_FR.UTF-8');
+
+        NativeGettext::restore();
+
+        $this->assertSame('fr_FR.UTF-8', getenv('LC_ALL'));
+    }
+
+    public function testGlibcBranchNeverTouchesLcAll(): void
+    {
+        MigratedPoFixture::requireGlibcLocale($this);
+        putenv('LC_ALL=de_DE.UTF-8');
+
+        $this->activateBuild();
+        $this->assertTrue(NativeGettext::isActive());
+        $this->assertSame('de_DE.UTF-8', getenv('LC_ALL'));
+
+        NativeGettext::restore();
+        $this->assertSame('de_DE.UTF-8', getenv('LC_ALL'));
+
+        putenv('LC_ALL');
+        $this->activateBuild();
+        $this->assertFalse(getenv('LC_ALL'));
+        NativeGettext::restore();
+        $this->assertFalse(getenv('LC_ALL'));
+    }
+
     private function activateSystemLocale(): void
     {
         $this->assertTrue(NativeGettext::activate($this->root . '/no-locale'), (string) NativeGettext::getProblem());

components/ILIAS/Language/tests/ComponentTranslation/ShippedTranslationsBuildTest.php

@@ -38,13 +38,15 @@ class ShippedTranslationsBuildTest extends TestCase
     private string $artifacts;
     private string|false $messages_locale;
     private string|false $language;
+    private string|false $lc_all;
 
     protected function setUp(): void
     {
         // never the build of the installation, see MigratedPoFixture::resetRuntime()
         MigratedPoFixture::resetRuntime();
         $this->messages_locale = setlocale(LC_MESSAGES, '0');
         $this->language = getenv('LANGUAGE');
+        $this->lc_all = getenv('LC_ALL');
         $this->root = sys_get_temp_dir() . '/ilias_shipped_build_test_' . bin2hex(random_bytes(6));
         mkdir($this->root, 0775, true);
         $this->artifacts = MigratedLanguageFilePaths::shippedArtifactDirectory($this->root);
@@ -57,6 +59,7 @@ protected function tearDown(): void
             setlocale(LC_MESSAGES, $this->messages_locale);
         }
         putenv($this->language === false ? 'LANGUAGE' : 'LANGUAGE=' . $this->language);
+        putenv($this->lc_all === false ? 'LC_ALL' : 'LC_ALL=' . $this->lc_all);
         MigratedPoFixture::removeDirectory($this->root);
         MigratedPoFixture::resetRuntime();
     }