make canonical ordering linear in the length of a combining-mark run

test/misc.c

@@ -87,12 +87,54 @@ static void issue288(void) /* #288 */
     printf("PASSED: utf8proc_normalize_utf32 preserves uninterpretable codepoints\n");
 }
 
+/* canonical ordering of a run of n combining marks: the output must be the
+   stable sort by combining class for any run length (long runs take a
+   different code path than short ones), and must not take quadratic time */
+static void check_canonical_order(utf8proc_ssize_t n)
+{
+    /* three classes, three marks each, so stability within a class is checked */
+    static const utf8proc_int32_t marks[3][3] = {
+        {0x0300, 0x0301, 0x0302}, /* combining class 230 */
+        {0x0323, 0x0324, 0x0325}, /* combining class 220 */
+        {0x0334, 0x0335, 0x0336}  /* combining class 1 */
+    };
+    utf8proc_uint8_t *input = (utf8proc_uint8_t *) malloc((size_t) (2*n + 2));
+    utf8proc_uint8_t *expected = (utf8proc_uint8_t *) malloc((size_t) (2*n + 2));
+    utf8proc_uint8_t *output;
+    utf8proc_ssize_t i, cls, len, explen, result;
+
+    check(input && expected, "canonical order: out of memory");
+    /* input: 'a' followed by the three classes in descending order */
+    len = utf8proc_encode_char(0x61, input);
+    for (i = 0; i < n; ++i)
+        len += utf8proc_encode_char(marks[i*3/n][i%3], input + len);
+    input[len] = 0;
+    /* expected: 'a' followed by the same marks in ascending class order,
+       the order within each class unchanged */
+    explen = utf8proc_encode_char(0x61, expected);
+    for (cls = 2; cls >= 0; --cls)
+        for (i = 0; i < n; ++i)
+            if (i*3/n == cls)
+                explen += utf8proc_encode_char(marks[cls][i%3], expected + explen);
+    expected[explen] = 0;
+
+    result = utf8proc_map(input, len, &output, UTF8PROC_STABLE | UTF8PROC_DECOMPOSE);
+    check(result == explen, "canonical order (n=%zd): NFD returned %zd, expected %zd", n, result, explen);
+    check(!memcmp(output, expected, (size_t) explen), "canonical order (n=%zd): NFD output is not the stable sort by combining class", n);
+    utf8proc_free(output);
+    free(input);
+    free(expected);
+    printf("PASSED: canonical ordering of %zd combining marks\n", n);
+}
+
 int main(void)
 {
     issue128();
     issue102();
     issue317();
     issue288();
+    check_canonical_order(9);     /* sorted in place */
+    check_canonical_order(1<<16); /* sorted through a scratch buffer */
 #ifdef UNICODE_VERSION
     printf("Unicode version: Makefile has %s, has API %s\n", UNICODE_VERSION, utf8proc_unicode_version());
     check(!strcmp(UNICODE_VERSION, utf8proc_unicode_version()), "utf8proc_unicode_version mismatch");

utf8proc.c

@@ -537,6 +537,40 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose(
     return utf8proc_decompose_custom(str, strlen, buffer, bufsize, options, NULL, NULL);
 }
 
+/* Stable sort of a run of `len` non-starters by combining class (canonical
+   ordering, UAX #15 section 3.11).  The in-place insertion sort is quadratic
+   in the run length, so for long runs the sort is done as a counting sort
+   through a scratch copy instead: a long run of marks in descending class
+   order then costs O(len) rather than O(len^2) swaps.  If the scratch buffer
+   cannot be allocated, the insertion sort is used; it yields the same result. */
+#define UTF8PROC_INSERTION_SORT_MAX 32
+static void canonical_order(utf8proc_int32_t *run, utf8proc_ssize_t len) {
+  utf8proc_ssize_t i, j;
+  if (len > UTF8PROC_INSERTION_SORT_MAX) {
+    utf8proc_int32_t *tmp = (utf8proc_int32_t *) malloc((utf8proc_size_t)len * sizeof(utf8proc_int32_t));
+    if (tmp) {
+      utf8proc_ssize_t count[256] = {0}; /* combining classes are 0..254 */
+      for (i = 0; i < len; i++) count[unsafe_get_property(run[i])->combining_class]++;
+      for (i = 0, j = 0; i < 256; i++) {
+        utf8proc_ssize_t n = count[i];
+        count[i] = j;
+        j += n;
+      }
+      for (i = 0; i < len; i++) tmp[count[unsafe_get_property(run[i])->combining_class]++] = run[i];
+      for (i = 0; i < len; i++) run[i] = tmp[i];
+      free(tmp);
+      return;
+    }
+  }
+  for (i = 1; i < len; i++) {
+    utf8proc_int32_t uc = run[i];
+    utf8proc_propval_t cc = unsafe_get_property(uc)->combining_class;
+    for (j = i; j > 0 && unsafe_get_property(run[j-1])->combining_class > cc; j--)
+      run[j] = run[j-1];
+    run[j] = uc;
+  }
+}
+
 UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_custom(
   const utf8proc_uint8_t *str, utf8proc_ssize_t strlen,
   utf8proc_int32_t *buffer, utf8proc_ssize_t bufsize, unsigned int options,
@@ -583,32 +617,16 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_custom(
     }
   }
   if ((options & (UTF8PROC_COMPOSE|UTF8PROC_DECOMPOSE)) && bufsize >= wpos) {
+    /* canonical ordering: sort each run of non-starters by combining class.
+       starters and grapheme-break sentinels (uc < 0) delimit the runs. */
     utf8proc_ssize_t pos = 0;
-    while (pos < wpos-1) {
-      utf8proc_int32_t uc1, uc2;
-      const utf8proc_property_t *property1, *property2;
-      uc1 = buffer[pos];
-      if (uc1 < 0) {
-        /* skip grapheme break */
-        pos++;
-        continue;
-      }
-      uc2 = buffer[pos+1];
-      if (uc2 < 0) {
-        /* cannot recombine; skip grapheme break */
-        pos+=2;
-        continue;
-      }
-      property1 = unsafe_get_property(uc1);
-      property2 = unsafe_get_property(uc2);
-      if (property1->combining_class > property2->combining_class &&
-          property2->combining_class > 0) {
-        buffer[pos] = uc2;
-        buffer[pos+1] = uc1;
-        if (pos > 0) pos--; else pos++;
-      } else {
-        pos++;
-      }
+    while (pos < wpos) {
+      utf8proc_ssize_t end = pos;
+      while (end < wpos && buffer[end] >= 0 &&
+             unsafe_get_property(buffer[end])->combining_class > 0)
+        end++;
+      if (end - pos > 1) canonical_order(buffer + pos, end - pos);
+      pos = end + 1;
     }
   }
   return wpos;