From 55f1ce570b7209fe1c303fead8f34356d98a3760 Mon Sep 17 00:00:00 2001 From: niksedk Date: Thu, 13 Aug 2026 21:01:03 +0200 Subject: [PATCH] Add missing -ize/-ise words and fix two reversed word-list entries SubtitleEdit/subtitleedit#13583 reports "fantasize" not being converted. The word list only covered 66 of the -ize verbs, so most common ones (maximize, patronize, authorize, characterize, ...) were missed too. Add 608 pairs: 100 regular -ize/-ise verbs with their conjugations, the missing conjugations for "sanitize"/"deodorize", agent nouns (synthesizer, sterilizer, ...) and -ization/-isation nouns, including those whose verb was already listed. Only pairs where both spellings are genuine variants of the same word, so BR->US stays safe as well - -ise-in-both words (advertise, improvise, capsize, prize, surprise, ...) are left out, and -yze/-yse verbs are skipped because "catalyses"/"analyses" collide with noun plurals. Also fix two entries found while validating: - us="dustbin" br="garbage can" was reversed, so it Americanized in the Britishizer and vice versa. - "jewelery" preceded "jewelry", and for BR->US the first rule matching "jewellery" wins, so it produced the misspelling. 50% more rules would have made the conversion pass slower still, so gate the rules on a token index: a pattern starts with "\b", so its first word can only match a whole word of the input, and a line without that word can never match. Each line now runs the handful of rules whose words it contains instead of all 5541. Output is byte-identical to the old scan over a 49,560-line corpus in both directions; converting 2000 lines drops from 770 ms to 1.5 ms and construction from 96 ms to 12 ms. Co-Authored-By: Claude Opus 5 --- se5/AmericanToBritish/README.md | 2 +- se5/BritishToAmerican/README.md | 2 +- se5/Plugin-Shared/EnglishVariantConverter.cs | 85 ++- se5/Plugin-Shared/WordList.xml | 616 ++++++++++++++++++- 4 files changed, 694 insertions(+), 11 deletions(-) diff --git a/se5/AmericanToBritish/README.md b/se5/AmericanToBritish/README.md index 76be9eff..12dc4091 100644 --- a/se5/AmericanToBritish/README.md +++ b/se5/AmericanToBritish/README.md @@ -1,7 +1,7 @@ # American to British (Subtitle Edit 5 plugin) Converts American English spellings to British English in the subtitle, using a -bundled word list (~1000 pairs). Shows a checkable preview of every proposed +bundled word list (~1850 pairs). Shows a checkable preview of every proposed change so you can review and toggle individual conversions before applying. First Subtitle Edit 5 plugin built on top of the shared diff --git a/se5/BritishToAmerican/README.md b/se5/BritishToAmerican/README.md index dc3243d0..3c4aee79 100644 --- a/se5/BritishToAmerican/README.md +++ b/se5/BritishToAmerican/README.md @@ -1,7 +1,7 @@ # British to American (Subtitle Edit 5 plugin) Mirror of the [American to British](../AmericanToBritish/) plugin in the -opposite direction. Reuses the bundled word list (~1000 pairs) and the shared +opposite direction. Reuses the bundled word list (~1850 pairs) and the shared `EnglishVariantConverter` from [`Plugin-Shared`](../Plugin-Shared/) with `Direction = BrToUs`. diff --git a/se5/Plugin-Shared/EnglishVariantConverter.cs b/se5/Plugin-Shared/EnglishVariantConverter.cs index 47d1a560..f02a4688 100644 --- a/se5/Plugin-Shared/EnglishVariantConverter.cs +++ b/se5/Plugin-Shared/EnglishVariantConverter.cs @@ -13,13 +13,20 @@ public enum EnglishVariantDirection /// /// Converts between American and British English using the bundled WordList.xml -/// (~1000 pairs). Each pair becomes three case-aware regexes: lowercase, +/// (~1850 pairs). Each pair becomes three case-aware regexes: lowercase, /// UPPERCASE, and Titlecase, all matched as whole words. Picks direction via /// . /// public sealed class EnglishVariantConverter { private readonly List<(Regex Pattern, string Replacement)> _rules = new(); + + /// + /// Rule indexes keyed by the first word of the pattern, so a line only pays for the + /// handful of rules whose word it actually contains instead of all ~5500. + /// + private readonly Dictionary> _rulesByFirstWord = new(StringComparer.Ordinal); + private readonly EnglishVariantDirection _direction; public EnglishVariantConverter(EnglishVariantDirection direction) @@ -37,12 +44,11 @@ public string Convert(string text) return text; } - foreach (var (pattern, replacement) in _rules) + var candidates = CollectCandidateRules(text); + foreach (var index in candidates) { - if (pattern.IsMatch(text)) - { - text = pattern.Replace(text, replacement); - } + var (pattern, replacement) = _rules[index]; + text = pattern.Replace(text, replacement); } return RevertFontColorAttribute(text); @@ -85,9 +91,74 @@ private void LoadBuiltInWordList() private void AddRule(string from, string to) { - _rules.Add((new Regex("\\b" + Regex.Escape(from) + "\\b", RegexOptions.ExplicitCapture | RegexOptions.Compiled), to)); + var index = _rules.Count; + _rules.Add((new Regex("\\b" + Regex.Escape(from) + "\\b", RegexOptions.ExplicitCapture), to)); + + var firstWord = FirstWord(from); + if (!_rulesByFirstWord.TryGetValue(firstWord, out var indexes)) + { + indexes = new List(); + _rulesByFirstWord.Add(firstWord, indexes); + } + + indexes.Add(index); } + /// + /// Rules whose first word occurs in , in rule order. A pattern + /// starts with "\b", so its first word can only match a whole word of the input - a line + /// without that word can never match the pattern and does not need to run it. + /// + private SortedSet CollectCandidateRules(string text) + { + var candidates = new SortedSet(); + var i = 0; + while (i < text.Length) + { + if (!IsWordChar(text[i])) + { + i++; + continue; + } + + var start = i; + while (i < text.Length && IsWordChar(text[i])) + { + i++; + } + + if (_rulesByFirstWord.TryGetValue(text.Substring(start, i - start), out var indexes)) + { + foreach (var index in indexes) + { + candidates.Add(index); + } + } + } + + return candidates; + } + + private static string FirstWord(string s) + { + var start = 0; + while (start < s.Length && !IsWordChar(s[start])) + { + start++; + } + + var end = start; + while (end < s.Length && IsWordChar(s[end])) + { + end++; + } + + return s.Substring(start, end - start); + } + + /// Matches what "\w" (and therefore "\b") considers a word character. + private static bool IsWordChar(char c) => char.IsLetterOrDigit(c) || c == '_'; + /// /// "color" inside <font color="..."> is HTML attribute syntax and must not be Britishized /// by the word-list pass — that would corrupt the tag. Undo "colour" back to "color" inside diff --git a/se5/Plugin-Shared/WordList.xml b/se5/Plugin-Shared/WordList.xml index 0d7ff891..b469cd6a 100644 --- a/se5/Plugin-Shared/WordList.xml +++ b/se5/Plugin-Shared/WordList.xml @@ -281,7 +281,6 @@ - @@ -400,6 +399,8 @@ + + @@ -511,8 +512,10 @@ - + + @@ -1244,4 +1247,613 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +