diff --git a/test/graphemetest.c b/test/graphemetest.c index bd81b02..6494a33 100644 --- a/test/graphemetest.c +++ b/test/graphemetest.c @@ -141,20 +141,23 @@ int main(int argc, char **argv) /* UTF8PROC_STRIPCC must not strip the 0xff markers of UTF8PROC_CHARBOUND */ { - /* Cover tabs, C1 controls, and a control before the first printable character. */ + /* Cover tabs, C1 controls, a control before the first printable character, + and a control before a combining mark. */ utf8proc_uint8_t input[][8] = { {0x61,0x0a,0x62,0x01,0x63,0x00}, {0x61,0x09,0x62,0x00}, {0x61,0xc2,0x90,0x62,0x00}, - {0x01,0x61,0x00} + {0x01,0x61,0x00}, + {0x61,0x01,0xcc,0x81,0x00} }; utf8proc_uint8_t output[][9] = { {0xff,0x61,0xff,0x20,0xff,0x62,0xff,0x63,0x00}, {0xff,0x61,0xff,0x20,0xff,0x62,0x00}, {0xff,0x61,0xff,0x62,0x00}, - {0xff,0x61,0x00} + {0xff,0x61,0x00}, + {0xff,0x61,0xcc,0x81,0x00} }; - const char *plain[] = {"a bc", "a b", "ab", "a"}; + const char *plain[] = {"a bc", "a b", "ab", "a", "a\xcc\x81"}; size_t i; for (i = 0; i < sizeof(input) / sizeof(input[0]); i++) { utf8proc_ssize_t glen; diff --git a/test/misc.c b/test/misc.c index 03834cc..1823cfb 100644 --- a/test/misc.c +++ b/test/misc.c @@ -25,6 +25,21 @@ static void issue102(void) /* #102 */ check_compare("NFKC_Casefold", input, correct, utf8proc_NFKC_Casefold(input), 1); } +static void stripcc_between_marks(void) +{ + /* STRIPCC removes the control character, so the marks on both sides + have to be put in canonical order (and composed) together */ + utf8proc_uint8_t input[] = {0x6f, 0xcc, 0x85, 0x01, 0xcc, 0xa8, 0x00}; /* "o\u0305\x01\u0328" */ + utf8proc_uint8_t nfc[] = {0xc7, 0xab, 0xcc, 0x85, 0x00}; /* "\u01eb\u0305" */ + utf8proc_uint8_t nfd[] = {0x6f, 0xcc, 0xa8, 0xcc, 0x85, 0x00}; /* "o\u0328\u0305" */ + utf8proc_uint8_t *output; + + utf8proc_map(input, 0, &output, UTF8PROC_NULLTERM | UTF8PROC_STABLE | UTF8PROC_COMPOSE | UTF8PROC_STRIPCC); + check_compare("NFC+stripcc", input, nfc, output, 1); + utf8proc_map(input, 0, &output, UTF8PROC_NULLTERM | UTF8PROC_STABLE | UTF8PROC_DECOMPOSE | UTF8PROC_STRIPCC); + check_compare("NFD+stripcc", input, nfd, output, 1); +} + static void issue317(void) /* #317 */ { utf8proc_uint8_t input[] = {0xec, 0xa3, 0xa0, 0xe1, 0x86, 0xa7, 0x00}; /* "\uc8e0\u11a7" */ @@ -91,6 +106,7 @@ int main(void) { issue128(); issue102(); + stripcc_between_marks(); issue317(); issue288(); #ifdef UNICODE_VERSION diff --git a/utf8proc.c b/utf8proc.c index 87091ba..88fe122 100644 --- a/utf8proc.c +++ b/utf8proc.c @@ -571,6 +571,13 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_custom( if (custom_func != NULL) { uc = custom_func(uc, custom_data); /* user-specified custom mapping */ } + /* drop the control characters that STRIPCC removes (all but HT, LF, VT, + FF, CR and NEL, which utf8proc_normalize_utf32 converts) here already, + so that the canonical ordering and grapheme breaks below see the + combining marks around them as adjacent */ + if ((options & UTF8PROC_STRIPCC) && uc >= 0 && + ((uc < 0x20 && (uc < 0x09 || uc > 0x0D)) || (uc >= 0x7F && uc < 0xA0 && uc != 0x85))) + continue; decomp_result = utf8proc_decompose_char( uc, buffer ? buffer+wpos : buffer, (bufsize > wpos) ? (bufsize - wpos) : 0, options, &boundclass