Skip to content

feat: tune language validation for summaries with english translation #7614

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
tdgao wants to merge 2 commits into
base: main
Choose a base branch
from
+13 −12
Open

feat: tune language validation for summaries with english translation #7614

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
198c558
feat: tune language validation for summary on false positives with en…
tdgao Sep 17, 2026
8241efa
fmt
tdgao Sep 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 13 additions & 12 deletions apps/labrinth/src/validate/project/language.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ use unicode_normalization::UnicodeNormalization;
use unicode_segmentation::UnicodeSegmentation;

const MIN_ENGLISH_TO_BEST_RATIO: f64 = 0.5;
const MIN_SUMMARY_TRANSLATION_ENGLISH_TO_BEST_RATIO: f64 = 0.1;
const MIN_DESCRIPTION_CONFIDENCE: f64 = 0.35;
const MIN_DESCRIPTION_MARGIN: f64 = 0.15;
const MIN_NON_ENGLISH_CONFIDENCE: f64 = 0.8;
Expand All @@ -18,6 +19,8 @@ const MIN_SHORT_TEXT_MARGIN: f64 = 0.5;
const MIN_DESCRIPTION_ENGLISH_PROPORTION: f64 = 0.2;
const MIN_PASSAGE_WORDS: usize = 4;
const MIN_PASSAGE_CHARS: usize = 25;
const MIN_SUMMARY_TRANSLATION_WORDS: usize = 4;
const MIN_SUMMARY_TRANSLATION_CHARS: usize = 18;

static DETECTOR: LazyLock<LanguageDetector> =
LazyLock::new(|| LanguageDetectorBuilder::from_all_languages().build());
Expand Down Expand Up @@ -195,13 +198,15 @@ fn has_lowercase_prose(text: &str) -> bool {
}

fn summary_translation_passage(text: &str) -> Passage {
let mut passage = description_passage(text);
if passage.qualifies_as_english
&& passage.english_grammatical_words < MIN_ENGLISH_GRAMMATICAL_WORDS
{
passage.qualifies_as_english = false;
}
passage
let eligible = alphabetic_word_count(text) >= MIN_SUMMARY_TRANSLATION_WORDS
&& text.trim().graphemes(true).count() >= MIN_SUMMARY_TRANSLATION_CHARS;
classify_passage(
text.to_owned(),
eligible,
MIN_SUMMARY_TRANSLATION_ENGLISH_TO_BEST_RATIO,
&DETECTOR,
eligible,
)
}

/// Bound detector input without overlapping or splitting words.
Expand Down Expand Up @@ -341,11 +346,7 @@ fn summary_rescue_passages(text: &str) -> Vec<Passage> {
let normalized =
text.split_whitespace().collect::<Vec<_>>().join(" ");
let whole = summary_translation_passage(&normalized);
if whole.qualifies_as_english
|| !whole.eligible
|| whole.english_grammatical_words
< MIN_ENGLISH_GRAMMATICAL_WORDS
{
if whole.qualifies_as_english || !whole.eligible {
return vec![whole];
}
let spans = mixed_language_passages(&normalized, &DETECTOR);
Expand Down
Loading