first pass of merging in warp (doesn't build)

This commit is contained in:
Ryan Ward
2026-07-01 16:08:58 -05:00
parent 2f64909469
commit 4770ac06b5
3662 changed files with 414574 additions and 89772 deletions
+4 -2
View File
@@ -1,4 +1,6 @@
use std::{iter::Peekable, mem, str::Chars};
use std::iter::Peekable;
use std::mem;
use std::str::Chars;
use itertools::Itertools;
@@ -33,7 +35,7 @@ fn convert_char_to_delimiter(c: char) -> Option<WordDelimiter> {
}
}
/// Parse a sentence into tokens for natural language classificiation.
/// Parse a sentence into tokens for natural language classification.
/// The main difference from the default "split_whitespace" parser is
/// for matching double quotes and single quotes, we would keep their
/// closed content as one token.