diff --git a/test/graphemetest.c b/test/graphemetest.c index 64fca0d..bd81b02 100644 --- a/test/graphemetest.c +++ b/test/graphemetest.c @@ -139,6 +139,48 @@ int main(int argc, char **argv) } }; + /* UTF8PROC_STRIPCC must not strip the 0xff markers of UTF8PROC_CHARBOUND */ + { + /* Cover tabs, C1 controls, and a control before the first printable character. */ + utf8proc_uint8_t input[][8] = { + {0x61,0x0a,0x62,0x01,0x63,0x00}, + {0x61,0x09,0x62,0x00}, + {0x61,0xc2,0x90,0x62,0x00}, + {0x01,0x61,0x00} + }; + utf8proc_uint8_t output[][9] = { + {0xff,0x61,0xff,0x20,0xff,0x62,0xff,0x63,0x00}, + {0xff,0x61,0xff,0x20,0xff,0x62,0x00}, + {0xff,0x61,0xff,0x62,0x00}, + {0xff,0x61,0x00} + }; + const char *plain[] = {"a bc", "a b", "ab", "a"}; + size_t i; + for (i = 0; i < sizeof(input) / sizeof(input[0]); i++) { + utf8proc_ssize_t glen; + utf8proc_uint8_t *g; + glen = utf8proc_map(input[i], 0, &g, UTF8PROC_NULLTERM | UTF8PROC_CHARBOUND | UTF8PROC_STRIPCC); + check(glen == (utf8proc_ssize_t)strlen((char*)output[i]), "CHARBOUND|STRIPCC returned wrong length in case %zu", i); + check(!strcmp((char*)g, (char*)output[i]), "CHARBOUND|STRIPCC dropped grapheme markers in case %zu", i); + utf8proc_free(g); + glen = utf8proc_map(input[i], 0, &g, UTF8PROC_NULLTERM | UTF8PROC_STRIPCC); + check(glen == (utf8proc_ssize_t)strlen(plain[i]), "STRIPCC returned wrong length in case %zu", i); + check(!strcmp((char*)g, plain[i]), "STRIPCC changed output in case %zu", i); + { + utf8proc_uint8_t *separate, *combined; + utf8proc_ssize_t separate_len, combined_len; + separate_len = utf8proc_map(g, 0, &separate, UTF8PROC_NULLTERM | UTF8PROC_CHARBOUND); + combined_len = utf8proc_map(input[i], 0, &combined, UTF8PROC_NULLTERM | UTF8PROC_CHARBOUND | UTF8PROC_STRIPCC); + check(separate_len >= 0 && combined_len >= 0, "two-step mapping failed in case %zu", i); + check(separate_len == combined_len, "two-step mapping returned different length in case %zu", i); + check(!memcmp(separate, combined, (size_t)combined_len + 1), "two-step mapping differs from combined flags in case %zu", i); + utf8proc_free(separate); + utf8proc_free(combined); + } + utf8proc_free(g); + } + }; + /* https://github.com/JuliaLang/julia/issues/37680 */ checkline("/ 1f1f8 1f1ea / 1f1f8 1f1ea /", true); /* Two swedish flags after each other */ checkline("/ 1f926 1f3fc 200d 2642 fe0f /", true); /* facepalm + pale skin + zwj + male sign + FE0F */ diff --git a/utf8proc.c b/utf8proc.c index 0e277e0..87091ba 100644 --- a/utf8proc.c +++ b/utf8proc.c @@ -639,8 +639,13 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_normalize_utf32(utf8proc_int32_t *b } } } else if ((options & UTF8PROC_STRIPCC) && - (uc < 0x0020 || (uc >= 0x007F && uc < 0x00A0))) { + /* uc < 0 is a grapheme-break sentinel from UTF8PROC_CHARBOUND, + not a control character, so keep it */ + ((uc >= 0 && uc < 0x0020) || (uc >= 0x007F && uc < 0x00A0))) { if (uc == 0x0009) buffer[wpos++] = 0x0020; + /* a removed control character is a grapheme of its own, so also + drop the boundary sentinel that UTF8PROC_CHARBOUND put before it */ + else if ((options & UTF8PROC_CHARBOUND) && wpos > 0 && buffer[wpos-1] == -1) wpos--; } else { buffer[wpos++] = uc; }