From d1740ea51c050722d4c64bd6aebb593f6bcd6b7f Mon Sep 17 00:00:00 2001
From: fhgffy <102001626+fhgffy@users.noreply.github.com>
Date: Fri, 9 Oct 2026 17:27:48 +0800
Subject: [PATCH 1/4] normalize_utf32: keep grapheme-break sentinels when
 stripping control characters

With UTF8PROC_STRIPCC, everything below 0x20 was treated as a control
character, including the negative sentinels that UTF8PROC_CHARBOUND
inserts before each grapheme. So CHARBOUND|STRIPCC returned the text
without any 0xff boundary markers.

Only strip non-negative codepoints, and when a control character is
removed, drop the sentinel in front of it as well so that it doesn't
leave an empty grapheme behind.
---
 test/graphemetest.c | 12 ++++++++++++
 utf8proc.c          |  7 ++++++-
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/test/graphemetest.c b/test/graphemetest.c
index 64fca0d..f6f1a91 100644
--- a/test/graphemetest.c
+++ b/test/graphemetest.c
@@ -139,6 +139,18 @@ int main(int argc, char **argv)
         }
     };
 
+    /* UTF8PROC_STRIPCC must not strip the 0xff markers of UTF8PROC_CHARBOUND */
+    {
+        utf8proc_uint8_t input[] = {0x61,0x0a,0x62,0x01,0x63,0x00}; /* "a\nb\x01c" */
+        utf8proc_uint8_t output[] = {0xff,0x61,0xff,0x20,0xff,0x62,0xff,0x63,0x00};
+        utf8proc_ssize_t glen;
+        utf8proc_uint8_t *g;
+        glen = utf8proc_map(input, 0, &g, UTF8PROC_NULLTERM | UTF8PROC_CHARBOUND | UTF8PROC_STRIPCC);
+        check(glen == 8, "CHARBOUND|STRIPCC returned length %zd, expected 8", glen);
+        check(!strcmp((char*)g, (char*)output), "CHARBOUND|STRIPCC dropped grapheme markers");
+        utf8proc_free(g);
+    };
+
     /* https://github.com/JuliaLang/julia/issues/37680 */
     checkline("/ 1f1f8 1f1ea / 1f1f8 1f1ea /", true); /* Two swedish flags after each other */
     checkline("/ 1f926 1f3fc 200d 2642 fe0f /", true); /* facepalm + pale skin + zwj + male sign + FE0F */
diff --git a/utf8proc.c b/utf8proc.c
index 0e277e0..535af11 100644
--- a/utf8proc.c
+++ b/utf8proc.c
@@ -639,8 +639,13 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_normalize_utf32(utf8proc_int32_t *b
           }
         }
       } else if ((options & UTF8PROC_STRIPCC) &&
-          (uc < 0x0020 || (uc >= 0x007F && uc < 0x00A0))) {
+          /* uc < 0 is a grapheme-break sentinel from UTF8PROC_CHARBOUND,
+             not a control character, so keep it */
+          ((uc >= 0 && uc < 0x0020) || (uc >= 0x007F && uc < 0x00A0))) {
         if (uc == 0x0009) buffer[wpos++] = 0x0020;
+        /* a removed control character is a grapheme of its own, so also
+           drop the boundary sentinel that UTF8PROC_CHARBOUND put before it */
+        else if (wpos > 0 && buffer[wpos-1] < 0) wpos--;
       } else {
         buffer[wpos++] = uc;
       }

From 23fb80db43cc7e22373586dfea6e30676ef0caa4 Mon Sep 17 00:00:00 2001
From: fhgffy <102001626+fhgffy@users.noreply.github.com>
Date: Sat, 10 Oct 2026 02:17:10 +0800
Subject: [PATCH 2/4] Tighten stripped grapheme sentinel guard and extend
 control tests

---
 test/graphemetest.c | 35 +++++++++++++++++++++++++++--------
 utf8proc.c          |  2 +-
 2 files changed, 28 insertions(+), 9 deletions(-)

diff --git a/test/graphemetest.c b/test/graphemetest.c
index f6f1a91..29ee306 100644
--- a/test/graphemetest.c
+++ b/test/graphemetest.c
@@ -141,14 +141,33 @@ int main(int argc, char **argv)
 
     /* UTF8PROC_STRIPCC must not strip the 0xff markers of UTF8PROC_CHARBOUND */
     {
-        utf8proc_uint8_t input[] = {0x61,0x0a,0x62,0x01,0x63,0x00}; /* "a\nb\x01c" */
-        utf8proc_uint8_t output[] = {0xff,0x61,0xff,0x20,0xff,0x62,0xff,0x63,0x00};
-        utf8proc_ssize_t glen;
-        utf8proc_uint8_t *g;
-        glen = utf8proc_map(input, 0, &g, UTF8PROC_NULLTERM | UTF8PROC_CHARBOUND | UTF8PROC_STRIPCC);
-        check(glen == 8, "CHARBOUND|STRIPCC returned length %zd, expected 8", glen);
-        check(!strcmp((char*)g, (char*)output), "CHARBOUND|STRIPCC dropped grapheme markers");
-        utf8proc_free(g);
+        /* 2026-10-10: Cover tabs, C1 controls, and a control before the first printable character. */
+        utf8proc_uint8_t input[][8] = {
+            {0x61,0x0a,0x62,0x01,0x63,0x00},
+            {0x61,0x09,0x62,0x00},
+            {0x61,0xc2,0x90,0x62,0x00},
+            {0x01,0x61,0x00}
+        };
+        utf8proc_uint8_t output[][9] = {
+            {0xff,0x61,0xff,0x20,0xff,0x62,0xff,0x63,0x00},
+            {0xff,0x61,0xff,0x20,0xff,0x62,0x00},
+            {0xff,0x61,0xff,0x62,0x00},
+            {0xff,0x61,0x00}
+        };
+        const char *plain[] = {"a bc", "a b", "ab", "a"};
+        size_t i;
+        for (i = 0; i < sizeof(input) / sizeof(input[0]); i++) {
+            utf8proc_ssize_t glen;
+            utf8proc_uint8_t *g;
+            glen = utf8proc_map(input[i], 0, &g, UTF8PROC_NULLTERM | UTF8PROC_CHARBOUND | UTF8PROC_STRIPCC);
+            check(glen == (utf8proc_ssize_t)strlen((char*)output[i]), "CHARBOUND|STRIPCC returned wrong length in case %zu", i);
+            check(!strcmp((char*)g, (char*)output[i]), "CHARBOUND|STRIPCC dropped grapheme markers in case %zu", i);
+            utf8proc_free(g);
+            glen = utf8proc_map(input[i], 0, &g, UTF8PROC_NULLTERM | UTF8PROC_STRIPCC);
+            check(glen == (utf8proc_ssize_t)strlen(plain[i]), "STRIPCC returned wrong length in case %zu", i);
+            check(!strcmp((char*)g, plain[i]), "STRIPCC changed output in case %zu", i);
+            utf8proc_free(g);
+        }
     };
 
     /* https://github.com/JuliaLang/julia/issues/37680 */
diff --git a/utf8proc.c b/utf8proc.c
index 535af11..3727a7a 100644
--- a/utf8proc.c
+++ b/utf8proc.c
@@ -645,7 +645,7 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_normalize_utf32(utf8proc_int32_t *b
         if (uc == 0x0009) buffer[wpos++] = 0x0020;
         /* a removed control character is a grapheme of its own, so also
            drop the boundary sentinel that UTF8PROC_CHARBOUND put before it */
-        else if (wpos > 0 && buffer[wpos-1] < 0) wpos--;
+        else if ((options & UTF8PROC_CHARBOUND) && wpos > 0 && buffer[wpos-1] == -1) wpos--; /* 2026-10-10 */
       } else {
         buffer[wpos++] = uc;
       }

From 8d8d921334873c73c9af65a832910376990136e5 Mon Sep 17 00:00:00 2001
From: fhgffy <102001626+fhgffy@users.noreply.github.com>
Date: Sun, 11 Oct 2026 00:39:42 +0800
Subject: [PATCH 3/4] Remove dates from grapheme regression comments

---
 test/graphemetest.c | 2 +-
 utf8proc.c          | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/graphemetest.c b/test/graphemetest.c
index 29ee306..a73b963 100644
--- a/test/graphemetest.c
+++ b/test/graphemetest.c
@@ -141,7 +141,7 @@ int main(int argc, char **argv)
 
     /* UTF8PROC_STRIPCC must not strip the 0xff markers of UTF8PROC_CHARBOUND */
     {
-        /* 2026-10-10: Cover tabs, C1 controls, and a control before the first printable character. */
+        /* Cover tabs, C1 controls, and a control before the first printable character. */
         utf8proc_uint8_t input[][8] = {
             {0x61,0x0a,0x62,0x01,0x63,0x00},
             {0x61,0x09,0x62,0x00},
diff --git a/utf8proc.c b/utf8proc.c
index 3727a7a..87091ba 100644
--- a/utf8proc.c
+++ b/utf8proc.c
@@ -645,7 +645,7 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_normalize_utf32(utf8proc_int32_t *b
         if (uc == 0x0009) buffer[wpos++] = 0x0020;
         /* a removed control character is a grapheme of its own, so also
            drop the boundary sentinel that UTF8PROC_CHARBOUND put before it */
-        else if ((options & UTF8PROC_CHARBOUND) && wpos > 0 && buffer[wpos-1] == -1) wpos--; /* 2026-10-10 */
+        else if ((options & UTF8PROC_CHARBOUND) && wpos > 0 && buffer[wpos-1] == -1) wpos--;
       } else {
         buffer[wpos++] = uc;
       }

From fcd4574b60c4e727fd53ab85ac362fa60bdce904 Mon Sep 17 00:00:00 2001
From: fhgffy <102001626+fhgffy@users.noreply.github.com>
Date: Sun, 11 Oct 2026 17:02:45 +0800
Subject: [PATCH 4/4] Test two-step control stripping against combined grapheme
 flags

---
 test/graphemetest.c | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/test/graphemetest.c b/test/graphemetest.c
index a73b963..bd81b02 100644
--- a/test/graphemetest.c
+++ b/test/graphemetest.c
@@ -166,6 +166,17 @@ int main(int argc, char **argv)
             glen = utf8proc_map(input[i], 0, &g, UTF8PROC_NULLTERM | UTF8PROC_STRIPCC);
             check(glen == (utf8proc_ssize_t)strlen(plain[i]), "STRIPCC returned wrong length in case %zu", i);
             check(!strcmp((char*)g, plain[i]), "STRIPCC changed output in case %zu", i);
+            {
+                utf8proc_uint8_t *separate, *combined;
+                utf8proc_ssize_t separate_len, combined_len;
+                separate_len = utf8proc_map(g, 0, &separate, UTF8PROC_NULLTERM | UTF8PROC_CHARBOUND);
+                combined_len = utf8proc_map(input[i], 0, &combined, UTF8PROC_NULLTERM | UTF8PROC_CHARBOUND | UTF8PROC_STRIPCC);
+                check(separate_len >= 0 && combined_len >= 0, "two-step mapping failed in case %zu", i);
+                check(separate_len == combined_len, "two-step mapping returned different length in case %zu", i);
+                check(!memcmp(separate, combined, (size_t)combined_len + 1), "two-step mapping differs from combined flags in case %zu", i);
+                utf8proc_free(separate);
+                utf8proc_free(combined);
+            }
             utf8proc_free(g);
         }
     };