From 36d81dcb5c00a886d8fef536ebd44efdb99575a3 Mon Sep 17 00:00:00 2001 From: shams Date: Mon, 19 Oct 2026 14:22:07 +0530 Subject: [PATCH] make canonical ordering linear in the length of a combining-mark run --- test/misc.c | 42 +++++++++++++++++++++++++++++++++ utf8proc.c | 68 +++++++++++++++++++++++++++++++++-------------------- 2 files changed, 85 insertions(+), 25 deletions(-) diff --git a/test/misc.c b/test/misc.c index 03834cc..a2fb7cb 100644 --- a/test/misc.c +++ b/test/misc.c @@ -87,12 +87,54 @@ static void issue288(void) /* #288 */ printf("PASSED: utf8proc_normalize_utf32 preserves uninterpretable codepoints\n"); } +/* canonical ordering of a run of n combining marks: the output must be the + stable sort by combining class for any run length (long runs take a + different code path than short ones), and must not take quadratic time */ +static void check_canonical_order(utf8proc_ssize_t n) +{ + /* three classes, three marks each, so stability within a class is checked */ + static const utf8proc_int32_t marks[3][3] = { + {0x0300, 0x0301, 0x0302}, /* combining class 230 */ + {0x0323, 0x0324, 0x0325}, /* combining class 220 */ + {0x0334, 0x0335, 0x0336} /* combining class 1 */ + }; + utf8proc_uint8_t *input = (utf8proc_uint8_t *) malloc((size_t) (2*n + 2)); + utf8proc_uint8_t *expected = (utf8proc_uint8_t *) malloc((size_t) (2*n + 2)); + utf8proc_uint8_t *output; + utf8proc_ssize_t i, cls, len, explen, result; + + check(input && expected, "canonical order: out of memory"); + /* input: 'a' followed by the three classes in descending order */ + len = utf8proc_encode_char(0x61, input); + for (i = 0; i < n; ++i) + len += utf8proc_encode_char(marks[i*3/n][i%3], input + len); + input[len] = 0; + /* expected: 'a' followed by the same marks in ascending class order, + the order within each class unchanged */ + explen = utf8proc_encode_char(0x61, expected); + for (cls = 2; cls >= 0; --cls) + for (i = 0; i < n; ++i) + if (i*3/n == cls) + explen += utf8proc_encode_char(marks[cls][i%3], expected + explen); + expected[explen] = 0; + + result = utf8proc_map(input, len, &output, UTF8PROC_STABLE | UTF8PROC_DECOMPOSE); + check(result == explen, "canonical order (n=%zd): NFD returned %zd, expected %zd", n, result, explen); + check(!memcmp(output, expected, (size_t) explen), "canonical order (n=%zd): NFD output is not the stable sort by combining class", n); + utf8proc_free(output); + free(input); + free(expected); + printf("PASSED: canonical ordering of %zd combining marks\n", n); +} + int main(void) { issue128(); issue102(); issue317(); issue288(); + check_canonical_order(9); /* sorted in place */ + check_canonical_order(1<<16); /* sorted through a scratch buffer */ #ifdef UNICODE_VERSION printf("Unicode version: Makefile has %s, has API %s\n", UNICODE_VERSION, utf8proc_unicode_version()); check(!strcmp(UNICODE_VERSION, utf8proc_unicode_version()), "utf8proc_unicode_version mismatch"); diff --git a/utf8proc.c b/utf8proc.c index 87091ba..dd59624 100644 --- a/utf8proc.c +++ b/utf8proc.c @@ -537,6 +537,40 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose( return utf8proc_decompose_custom(str, strlen, buffer, bufsize, options, NULL, NULL); } +/* Stable sort of a run of `len` non-starters by combining class (canonical + ordering, UAX #15 section 3.11). The in-place insertion sort is quadratic + in the run length, so for long runs the sort is done as a counting sort + through a scratch copy instead: a long run of marks in descending class + order then costs O(len) rather than O(len^2) swaps. If the scratch buffer + cannot be allocated, the insertion sort is used; it yields the same result. */ +#define UTF8PROC_INSERTION_SORT_MAX 32 +static void canonical_order(utf8proc_int32_t *run, utf8proc_ssize_t len) { + utf8proc_ssize_t i, j; + if (len > UTF8PROC_INSERTION_SORT_MAX) { + utf8proc_int32_t *tmp = (utf8proc_int32_t *) malloc((utf8proc_size_t)len * sizeof(utf8proc_int32_t)); + if (tmp) { + utf8proc_ssize_t count[256] = {0}; /* combining classes are 0..254 */ + for (i = 0; i < len; i++) count[unsafe_get_property(run[i])->combining_class]++; + for (i = 0, j = 0; i < 256; i++) { + utf8proc_ssize_t n = count[i]; + count[i] = j; + j += n; + } + for (i = 0; i < len; i++) tmp[count[unsafe_get_property(run[i])->combining_class]++] = run[i]; + for (i = 0; i < len; i++) run[i] = tmp[i]; + free(tmp); + return; + } + } + for (i = 1; i < len; i++) { + utf8proc_int32_t uc = run[i]; + utf8proc_propval_t cc = unsafe_get_property(uc)->combining_class; + for (j = i; j > 0 && unsafe_get_property(run[j-1])->combining_class > cc; j--) + run[j] = run[j-1]; + run[j] = uc; + } +} + UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_custom( const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_int32_t *buffer, utf8proc_ssize_t bufsize, unsigned int options, @@ -583,32 +617,16 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_custom( } } if ((options & (UTF8PROC_COMPOSE|UTF8PROC_DECOMPOSE)) && bufsize >= wpos) { + /* canonical ordering: sort each run of non-starters by combining class. + starters and grapheme-break sentinels (uc < 0) delimit the runs. */ utf8proc_ssize_t pos = 0; - while (pos < wpos-1) { - utf8proc_int32_t uc1, uc2; - const utf8proc_property_t *property1, *property2; - uc1 = buffer[pos]; - if (uc1 < 0) { - /* skip grapheme break */ - pos++; - continue; - } - uc2 = buffer[pos+1]; - if (uc2 < 0) { - /* cannot recombine; skip grapheme break */ - pos+=2; - continue; - } - property1 = unsafe_get_property(uc1); - property2 = unsafe_get_property(uc2); - if (property1->combining_class > property2->combining_class && - property2->combining_class > 0) { - buffer[pos] = uc2; - buffer[pos+1] = uc1; - if (pos > 0) pos--; else pos++; - } else { - pos++; - } + while (pos < wpos) { + utf8proc_ssize_t end = pos; + while (end < wpos && buffer[end] >= 0 && + unsafe_get_property(buffer[end])->combining_class > 0) + end++; + if (end - pos > 1) canonical_order(buffer + pos, end - pos); + pos = end + 1; } } return wpos;