From ab66de5c55b61415d5a659330caafdb06be5c430 Mon Sep 17 00:00:00 2001 From: Christian Knof Date: Thu, 29 Oct 2026 12:06:02 +0100 Subject: [PATCH] Language: 48622: Set LC_ALL for native gettext on macOS and BSD GNU libintl on macOS takes the locale from LC_ALL, LC_MESSAGES or LANG before setlocale() and ignores LANGUAGE for "C" and "C.", so `setup build` aborted e.g. with LANG=C.UTF-8. With a system locale `NativeGettext` now also sets the environment variable LC_ALL to that locale and restores it like LANGUAGE. --- components/ILIAS/Language/README.md | 4 +- .../ComponentTranslation/NativeGettext.php | 36 ++++-- .../NativeGettextTest.php | 116 +++++++++++++++++- .../ShippedTranslationsBuildTest.php | 3 + 4 files changed, 145 insertions(+), 14 deletions(-) diff --git a/components/ILIAS/Language/README.md b/components/ILIAS/Language/README.md index 257ba06feb9e..fdf90ff455a7 100755 --- a/components/ILIAS/Language/README.md +++ b/components/ILIAS/Language/README.md @@ -178,7 +178,9 @@ Only other modules (plugins) are read from `lng_data`/`lng_modules`. * macOS and FreeBSD: gettext from GNU libintl (Homebrew, Ports) and a system locale of `NativeGettext::SYSTEM_LOCALES`; the libc there ignores `LOCPATH`, no locale is shipped (`C.UTF-8` does not - work: GNU libintl ignores `LANGUAGE` there); + work: GNU libintl ignores `LANGUAGE` there); locale variables of the + environment (e.g. `LANG=C.UTF-8`) do not matter: native gettext sets the + environment variable `LC_ALL` to that locale (child processes inherit it); * musl is not supported (ignores `LOCPATH`). Build on the platform that serves ILIAS (with Docker in the container): the diff --git a/components/ILIAS/Language/src/ComponentTranslation/NativeGettext.php b/components/ILIAS/Language/src/ComponentTranslation/NativeGettext.php index 16f1623e45d6..4dadb768d912 100644 --- a/components/ILIAS/Language/src/ComponentTranslation/NativeGettext.php +++ b/components/ILIAS/Language/src/ComponentTranslation/NativeGettext.php @@ -30,7 +30,8 @@ * LOCALE_NAME; activate() loads it via LOCPATH and removes LOCPATH again right away (while it is set, * setlocale() finds no system locale); * - macOS/BSD (see usesSystemLocale()): the C library ignores LOCPATH, gettext comes from GNU libintl, so - * activate() sets the first existing of SYSTEM_LOCALES (GNU libintl ignores LANGUAGE for "C."); + * activate() sets the first existing of SYSTEM_LOCALES for LC_MESSAGES and the environment variable LC_ALL + * (GNU libintl ignores LANGUAGE for "C."); * - LANGUAGE is CATALOG_LANGUAGE, so "" is always "messages"; the language of a catalog is part * of its domain (".", see MigratedLanguageFilePaths), as the C library caches per process * and a later LANGUAGE change would not reliably switch catalogs; @@ -40,7 +41,8 @@ * setlocale() reset to "C"), so no shutdown function is needed (it would stop later shutdown functions * from translating). In a thread-safe (ZTS) PHP the end of another thread's request can take them back * mid-request: isActive() checks, reactivateIfLost() activates again (the administration shows a hint, - * see isThreadSafe()). Child processes inherit LANGUAGE=messages, which is harmless. + * see isThreadSafe()). Child processes inherit LANGUAGE=messages, which is harmless, and on macOS/BSD + * LC_ALL=, which overrides the locale of the environment for them. * * The C library never reloads a catalog file it has read once (also not a replaced one), so every new * state (a build, an overlay revision) gets a directory of its own, and binding the domain loads it. @@ -70,10 +72,10 @@ final class NativeGettext private static ?string $active_locale = null; /** - * LANGUAGE and LC_MESSAGES before the first activate() since the last restore() (`false`: not - * set), see restore(). + * LANGUAGE, LC_MESSAGES and the environment variable LC_ALL before the first activate() since the + * last restore() (`false`: not set), see restore(). * - * @var array{language: string|false, messages: string|false}|null + * @var array{language: string|false, messages: string|false, all: string|false}|null */ private static ?array $previous = null; @@ -144,7 +146,7 @@ public static function activate(string $locale_directory): bool return false; } - self::$previous ??= ['language' => getenv('LANGUAGE'), 'messages' => setlocale(LC_MESSAGES, '0')]; + self::$previous ??= self::processState(); $locpath = getenv('LOCPATH'); putenv('LOCPATH=' . $locale_directory); $locale = setlocale(LC_MESSAGES, self::LOCALE_NAME); @@ -160,7 +162,7 @@ public static function activate(string $locale_directory): bool private static function activateSystemLocale(string $locale_directory): bool { - self::$previous ??= ['language' => getenv('LANGUAGE'), 'messages' => setlocale(LC_MESSAGES, '0')]; + self::$previous ??= self::processState(); $locale = setlocale(LC_MESSAGES, self::SYSTEM_LOCALES); if ($locale === false) { self::$problem = sprintf( @@ -170,10 +172,20 @@ private static function activateSystemLocale(string $locale_directory): bool ); return false; } + // GNU libintl on macOS reads the locale of the environment before setlocale()'s, and "C" there ignores LANGUAGE + putenv('LC_ALL=' . $locale); return self::activated($locale_directory, $locale); } + /** + * @return array{language: string|false, messages: string|false, all: string|false} + */ + private static function processState(): array + { + return ['language' => getenv('LANGUAGE'), 'messages' => setlocale(LC_MESSAGES, '0'), 'all' => getenv('LC_ALL')]; + } + private static function activated(string $locale_directory, string $locale): bool { putenv('LANGUAGE=' . self::CATALOG_LANGUAGE); @@ -186,13 +198,14 @@ private static function activated(string $locale_directory, string $locale): boo /** * Whether activate() succeeded and its settings are still in place (other code, e.g. - * ilInitialisation::initLocale() with LC_ALL, may have changed LC_MESSAGES since). + * ilInitialisation::initLocale() with setlocale(LC_ALL, …), may have changed LC_MESSAGES since). */ public static function isActive(): bool { return self::$locale_directory !== null && setlocale(LC_MESSAGES, '0') === self::$active_locale - && getenv('LANGUAGE') === self::CATALOG_LANGUAGE; + && getenv('LANGUAGE') === self::CATALOG_LANGUAGE + && (!self::usesSystemLocale() || getenv('LC_ALL') === self::$active_locale); } /** @@ -211,7 +224,7 @@ public static function reactivateIfLost(): bool } /** - * Takes LANGUAGE and LC_MESSAGES back to the state before the first activate(); settings changed by + * Takes LANGUAGE, LC_MESSAGES and LC_ALL back to the state before the first activate(); settings changed by * other code since stay. Only for long-running processes (e.g. Setup's build after its check), a web * request needs no call. No-op in a thread-safe PHP, where other threads may use them. The activated * directory is kept, so reactivateIfLost() can activate it again. @@ -227,6 +240,9 @@ public static function restore(): void if (getenv('LANGUAGE') === self::CATALOG_LANGUAGE) { putenv(self::$previous['language'] === false ? 'LANGUAGE' : 'LANGUAGE=' . self::$previous['language']); } + if (self::usesSystemLocale() && getenv('LC_ALL') === self::$active_locale) { + putenv(self::$previous['all'] === false ? 'LC_ALL' : 'LC_ALL=' . self::$previous['all']); + } self::$previous = null; } diff --git a/components/ILIAS/Language/tests/ComponentTranslation/NativeGettextTest.php b/components/ILIAS/Language/tests/ComponentTranslation/NativeGettextTest.php index 90636d2d4fd5..ecf5ab20a1c9 100644 --- a/components/ILIAS/Language/tests/ComponentTranslation/NativeGettextTest.php +++ b/components/ILIAS/Language/tests/ComponentTranslation/NativeGettextTest.php @@ -27,7 +27,7 @@ use PHPUnit\Framework\TestCase; /** - * NativeGettext changes process-wide state (LC_MESSAGES, LANGUAGE, LOCPATH): what activate() leaves behind and + * NativeGettext changes process-wide state (LC_MESSAGES, LANGUAGE, LOCPATH, LC_ALL): what activate() leaves behind and * restore() gives back, losing and regaining the settings to other code, a build native gettext cannot use, lookups of * messages that cannot be looked up. * @@ -42,7 +42,7 @@ class NativeGettextTest extends TestCase private string $locale_directory; /** - * @var array{language: string|false, locpath: string|false, messages: string|false} + * @var array{language: string|false, locpath: string|false, all: string|false, messages: string|false} */ private array $process_state; @@ -52,6 +52,7 @@ protected function setUp(): void $this->process_state = [ 'language' => getenv('LANGUAGE'), 'locpath' => getenv('LOCPATH'), + 'all' => getenv('LC_ALL'), 'messages' => setlocale(LC_MESSAGES, '0'), ]; $this->root = sys_get_temp_dir() . '/ilias_native_gettext_' . bin2hex(random_bytes(6)); @@ -78,6 +79,7 @@ protected function tearDown(): void // whatever a test did to the process, the next one starts as this one did putenv($this->process_state['language'] === false ? 'LANGUAGE' : 'LANGUAGE=' . $this->process_state['language']); putenv($this->process_state['locpath'] === false ? 'LOCPATH' : 'LOCPATH=' . $this->process_state['locpath']); + putenv($this->process_state['all'] === false ? 'LC_ALL' : 'LC_ALL=' . $this->process_state['all']); if ($this->process_state['messages'] !== false) { setlocale(LC_MESSAGES, $this->process_state['messages']); } @@ -205,7 +207,21 @@ public function testSystemLocaleIsOnlyActiveAfterActivateAndRestoreGivesTheProce $this->assertSame($language, getenv('LANGUAGE')); } - #[DataProvider('foreignChanges')] + /** + * @return iterable + */ + public static function foreignChangesInTheSystemLocaleBranch(): iterable + { + yield from self::foreignChanges(); + yield 'LC_ALL environment variable set to C' => [static function (): void { + putenv('LC_ALL=C'); + }]; + yield 'LC_ALL environment variable removed' => [static function (): void { + putenv('LC_ALL'); + }]; + } + + #[DataProvider('foreignChangesInTheSystemLocaleBranch')] public function testReactivateIfLostRepairsWhatOtherCodeChangedInTheSystemLocaleBranch(\Closure $change): void { $this->requireSystemLocale(); @@ -220,6 +236,100 @@ public function testReactivateIfLostRepairsWhatOtherCodeChangedInTheSystemLocale $this->assertSame('Hallo', NativeGettext::translate(self::DOMAIN, 'greeting')); } + /** + * @return iterable + */ + public static function lcAllBeforeSystemLocaleActivation(): iterable + { + yield 'unset' => [false]; + yield 'C' => ['C']; + yield 'another locale' => ['de_DE.UTF-8']; + } + + #[DataProvider('lcAllBeforeSystemLocaleActivation')] + public function testSystemLocaleBranchSetsLcAllToTheActiveLocaleAndRestoreGivesItBackExactly(string|false $lc_all): void + { + $this->requireSystemLocale(); + NativeGettext::useSystemLocaleForTests(true); + putenv($lc_all === false ? 'LC_ALL' : 'LC_ALL=' . $lc_all); + + $this->activateSystemLocale(); + + $this->assertSame(setlocale(LC_MESSAGES, '0'), getenv('LC_ALL')); + $this->assertTrue(NativeGettext::isActive()); + $this->assertSame('Hallo', NativeGettext::translate(self::DOMAIN, 'greeting')); + + NativeGettext::restore(); + + $this->assertSame($lc_all, getenv('LC_ALL')); + $this->assertFalse(NativeGettext::isActive()); + } + + #[DataProvider('lcAllBeforeSystemLocaleActivation')] + public function testRestoreAfterARepeatedActivationGivesBackLcAllFromBeforeTheFirstOne(string|false $lc_all): void + { + $this->requireSystemLocale(); + NativeGettext::useSystemLocaleForTests(true); + putenv($lc_all === false ? 'LC_ALL' : 'LC_ALL=' . $lc_all); + $this->activateSystemLocale(); + + $this->assertTrue(NativeGettext::activate($this->root . '/other-no-locale'), (string) NativeGettext::getProblem()); + putenv('LC_ALL=fr_FR.UTF-8'); + $this->assertTrue(NativeGettext::reactivateIfLost()); + $this->assertSame(setlocale(LC_MESSAGES, '0'), getenv('LC_ALL')); + + NativeGettext::restore(); + + $this->assertSame($lc_all, getenv('LC_ALL')); + } + + public function testRestoreChangesNothingInAThreadSafePhpInTheSystemLocaleBranch(): void + { + $this->requireSystemLocale(); + NativeGettext::useSystemLocaleForTests(true); + putenv('LC_ALL=C'); + $this->activateSystemLocale(); + $active = setlocale(LC_MESSAGES, '0'); + NativeGettext::useThreadSafeForTests(true); + + NativeGettext::restore(); + + $this->assertSame($active, getenv('LC_ALL')); + $this->assertTrue(NativeGettext::isActive()); + } + + public function testRestoreLeavesALcAllOtherCodeChangedSinceAsItIsInTheSystemLocaleBranch(): void + { + $this->requireSystemLocale(); + NativeGettext::useSystemLocaleForTests(true); + putenv('LC_ALL'); + $this->activateSystemLocale(); + putenv('LC_ALL=fr_FR.UTF-8'); + + NativeGettext::restore(); + + $this->assertSame('fr_FR.UTF-8', getenv('LC_ALL')); + } + + public function testGlibcBranchNeverTouchesLcAll(): void + { + MigratedPoFixture::requireGlibcLocale($this); + putenv('LC_ALL=de_DE.UTF-8'); + + $this->activateBuild(); + $this->assertTrue(NativeGettext::isActive()); + $this->assertSame('de_DE.UTF-8', getenv('LC_ALL')); + + NativeGettext::restore(); + $this->assertSame('de_DE.UTF-8', getenv('LC_ALL')); + + putenv('LC_ALL'); + $this->activateBuild(); + $this->assertFalse(getenv('LC_ALL')); + NativeGettext::restore(); + $this->assertFalse(getenv('LC_ALL')); + } + private function activateSystemLocale(): void { $this->assertTrue(NativeGettext::activate($this->root . '/no-locale'), (string) NativeGettext::getProblem()); diff --git a/components/ILIAS/Language/tests/ComponentTranslation/ShippedTranslationsBuildTest.php b/components/ILIAS/Language/tests/ComponentTranslation/ShippedTranslationsBuildTest.php index 65b3b60cdebf..356fe38d92f9 100644 --- a/components/ILIAS/Language/tests/ComponentTranslation/ShippedTranslationsBuildTest.php +++ b/components/ILIAS/Language/tests/ComponentTranslation/ShippedTranslationsBuildTest.php @@ -38,6 +38,7 @@ class ShippedTranslationsBuildTest extends TestCase private string $artifacts; private string|false $messages_locale; private string|false $language; + private string|false $lc_all; protected function setUp(): void { @@ -45,6 +46,7 @@ protected function setUp(): void MigratedPoFixture::resetRuntime(); $this->messages_locale = setlocale(LC_MESSAGES, '0'); $this->language = getenv('LANGUAGE'); + $this->lc_all = getenv('LC_ALL'); $this->root = sys_get_temp_dir() . '/ilias_shipped_build_test_' . bin2hex(random_bytes(6)); mkdir($this->root, 0775, true); $this->artifacts = MigratedLanguageFilePaths::shippedArtifactDirectory($this->root); @@ -57,6 +59,7 @@ protected function tearDown(): void setlocale(LC_MESSAGES, $this->messages_locale); } putenv($this->language === false ? 'LANGUAGE' : 'LANGUAGE=' . $this->language); + putenv($this->lc_all === false ? 'LC_ALL' : 'LC_ALL=' . $this->lc_all); MigratedPoFixture::removeDirectory($this->root); MigratedPoFixture::resetRuntime(); }