From 198c55866d98b6a503e386d3ca8bb185a6e872ed Mon Sep 17 00:00:00 2001 From: tdgao Date: Thu, 17 Sep 2026 15:33:32 -0600 Subject: [PATCH 1/2] feat: tune language validation for summary on false positives with english translation --- .../labrinth/src/validate/project/language.rs | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/apps/labrinth/src/validate/project/language.rs b/apps/labrinth/src/validate/project/language.rs index 3d6daef67b..bd98905b5a 100644 --- a/apps/labrinth/src/validate/project/language.rs +++ b/apps/labrinth/src/validate/project/language.rs @@ -6,6 +6,7 @@ use unicode_normalization::UnicodeNormalization; use unicode_segmentation::UnicodeSegmentation; const MIN_ENGLISH_TO_BEST_RATIO: f64 = 0.5; +const MIN_SUMMARY_TRANSLATION_ENGLISH_TO_BEST_RATIO: f64 = 0.1; const MIN_DESCRIPTION_CONFIDENCE: f64 = 0.35; const MIN_DESCRIPTION_MARGIN: f64 = 0.15; const MIN_NON_ENGLISH_CONFIDENCE: f64 = 0.8; @@ -18,6 +19,8 @@ const MIN_SHORT_TEXT_MARGIN: f64 = 0.5; const MIN_DESCRIPTION_ENGLISH_PROPORTION: f64 = 0.2; const MIN_PASSAGE_WORDS: usize = 4; const MIN_PASSAGE_CHARS: usize = 25; +const MIN_SUMMARY_TRANSLATION_WORDS: usize = 4; +const MIN_SUMMARY_TRANSLATION_CHARS: usize = 18; static DETECTOR: LazyLock = LazyLock::new(|| LanguageDetectorBuilder::from_all_languages().build()); @@ -195,13 +198,15 @@ fn has_lowercase_prose(text: &str) -> bool { } fn summary_translation_passage(text: &str) -> Passage { - let mut passage = description_passage(text); - if passage.qualifies_as_english - && passage.english_grammatical_words < MIN_ENGLISH_GRAMMATICAL_WORDS - { - passage.qualifies_as_english = false; - } - passage + let eligible = alphabetic_word_count(text) >= MIN_SUMMARY_TRANSLATION_WORDS + && text.trim().graphemes(true).count() >= MIN_SUMMARY_TRANSLATION_CHARS; + classify_passage( + text.to_owned(), + eligible, + MIN_SUMMARY_TRANSLATION_ENGLISH_TO_BEST_RATIO, + &DETECTOR, + eligible, + ) } /// Bound detector input without overlapping or splitting words. @@ -341,11 +346,7 @@ fn summary_rescue_passages(text: &str) -> Vec { let normalized = text.split_whitespace().collect::>().join(" "); let whole = summary_translation_passage(&normalized); - if whole.qualifies_as_english - || !whole.eligible - || whole.english_grammatical_words - < MIN_ENGLISH_GRAMMATICAL_WORDS - { + if whole.qualifies_as_english || !whole.eligible { return vec![whole]; } let spans = mixed_language_passages(&normalized, &DETECTOR); From 8241efabb132ccc4cfb341630ce2e160e80414f4 Mon Sep 17 00:00:00 2001 From: tdgao Date: Thu, 17 Sep 2026 15:35:52 -0600 Subject: [PATCH 2/2] fmt --- .../labrinth/src/validate/project/language.rs | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/apps/labrinth/src/validate/project/language.rs b/apps/labrinth/src/validate/project/language.rs index bd98905b5a..24a4d31493 100644 --- a/apps/labrinth/src/validate/project/language.rs +++ b/apps/labrinth/src/validate/project/language.rs @@ -198,15 +198,15 @@ fn has_lowercase_prose(text: &str) -> bool { } fn summary_translation_passage(text: &str) -> Passage { - let eligible = alphabetic_word_count(text) >= MIN_SUMMARY_TRANSLATION_WORDS - && text.trim().graphemes(true).count() >= MIN_SUMMARY_TRANSLATION_CHARS; - classify_passage( - text.to_owned(), - eligible, - MIN_SUMMARY_TRANSLATION_ENGLISH_TO_BEST_RATIO, - &DETECTOR, - eligible, - ) + let eligible = alphabetic_word_count(text) >= MIN_SUMMARY_TRANSLATION_WORDS + && text.trim().graphemes(true).count() >= MIN_SUMMARY_TRANSLATION_CHARS; + classify_passage( + text.to_owned(), + eligible, + MIN_SUMMARY_TRANSLATION_ENGLISH_TO_BEST_RATIO, + &DETECTOR, + eligible, + ) } /// Bound detector input without overlapping or splitting words. @@ -346,7 +346,7 @@ fn summary_rescue_passages(text: &str) -> Vec { let normalized = text.split_whitespace().collect::>().join(" "); let whole = summary_translation_passage(&normalized); - if whole.qualifies_as_english || !whole.eligible { + if whole.qualifies_as_english || !whole.eligible { return vec![whole]; } let spans = mixed_language_passages(&normalized, &DETECTOR);