Skip to content

Strip control characters before canonical ordering #378

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
emincan2000 wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Strip control characters before canonical ordering #378

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 7 additions & 4 deletions test/graphemetest.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,20 +141,23 @@ int main(int argc, char **argv)

/* UTF8PROC_STRIPCC must not strip the 0xff markers of UTF8PROC_CHARBOUND */
{
/* Cover tabs, C1 controls, and a control before the first printable character. */
/* Cover tabs, C1 controls, a control before the first printable character,
and a control before a combining mark. */
utf8proc_uint8_t input[][8] = {
{0x61,0x0a,0x62,0x01,0x63,0x00},
{0x61,0x09,0x62,0x00},
{0x61,0xc2,0x90,0x62,0x00},
{0x01,0x61,0x00}
{0x01,0x61,0x00},
{0x61,0x01,0xcc,0x81,0x00}
};
utf8proc_uint8_t output[][9] = {
{0xff,0x61,0xff,0x20,0xff,0x62,0xff,0x63,0x00},
{0xff,0x61,0xff,0x20,0xff,0x62,0x00},
{0xff,0x61,0xff,0x62,0x00},
{0xff,0x61,0x00}
{0xff,0x61,0x00},
{0xff,0x61,0xcc,0x81,0x00}
};
const char *plain[] = {"a bc", "a b", "ab", "a"};
const char *plain[] = {"a bc", "a b", "ab", "a", "a\xcc\x81"};
size_t i;
for (i = 0; i < sizeof(input) / sizeof(input[0]); i++) {
utf8proc_ssize_t glen;
Expand Down
16 changes: 16 additions & 0 deletions test/misc.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,21 @@ static void issue102(void) /* #102 */
check_compare("NFKC_Casefold", input, correct, utf8proc_NFKC_Casefold(input), 1);
}

static void stripcc_between_marks(void)
{
/* STRIPCC removes the control character, so the marks on both sides
have to be put in canonical order (and composed) together */
utf8proc_uint8_t input[] = {0x6f, 0xcc, 0x85, 0x01, 0xcc, 0xa8, 0x00}; /* "o\u0305\x01\u0328" */
utf8proc_uint8_t nfc[] = {0xc7, 0xab, 0xcc, 0x85, 0x00}; /* "\u01eb\u0305" */
utf8proc_uint8_t nfd[] = {0x6f, 0xcc, 0xa8, 0xcc, 0x85, 0x00}; /* "o\u0328\u0305" */
utf8proc_uint8_t *output;

utf8proc_map(input, 0, &output, UTF8PROC_NULLTERM | UTF8PROC_STABLE | UTF8PROC_COMPOSE | UTF8PROC_STRIPCC);
check_compare("NFC+stripcc", input, nfc, output, 1);
utf8proc_map(input, 0, &output, UTF8PROC_NULLTERM | UTF8PROC_STABLE | UTF8PROC_DECOMPOSE | UTF8PROC_STRIPCC);
check_compare("NFD+stripcc", input, nfd, output, 1);
}

static void issue317(void) /* #317 */
{
utf8proc_uint8_t input[] = {0xec, 0xa3, 0xa0, 0xe1, 0x86, 0xa7, 0x00}; /* "\uc8e0\u11a7" */
Expand Down Expand Up @@ -91,6 +106,7 @@ int main(void)
{
issue128();
issue102();
stripcc_between_marks();
issue317();
issue288();
#ifdef UNICODE_VERSION
Expand Down
7 changes: 7 additions & 0 deletions utf8proc.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -571,6 +571,13 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_custom(
if (custom_func != NULL) {
uc = custom_func(uc, custom_data); /* user-specified custom mapping */
}
/* drop the control characters that STRIPCC removes (all but HT, LF, VT,
FF, CR and NEL, which utf8proc_normalize_utf32 converts) here already,
so that the canonical ordering and grapheme breaks below see the
combining marks around them as adjacent */
if ((options & UTF8PROC_STRIPCC) && uc >= 0 &&
((uc < 0x20 && (uc < 0x09 || uc > 0x0D)) || (uc >= 0x7F && uc < 0xA0 && uc != 0x85)))
continue;
decomp_result = utf8proc_decompose_char(
uc, buffer ? buffer+wpos : buffer, (bufsize > wpos) ? (bufsize - wpos) : 0, options,
&boundclass
Expand Down