fix: correct the split behavior during collecting dict entries (rime#762

)
WhiredPlanck · Nov 25, 2023 · 4e455c9 · 4e455c9
1 parent bafe97f
commit 4e455c9
Show file tree

Hide file tree

Showing 3 changed files with 4 additions and 4 deletions.
diff --git a/src/rime/algo/strings.cc b/src/rime/algo/strings.cc
@@ -17,7 +17,7 @@ vector<string> split(const string& str,
 
   while (std::string::npos != pos || std::string::npos != lastPos) {
     strings.emplace_back(str.substr(lastPos, pos - lastPos));
-    if (behavior == SplitBehavior::SkipEmpty) {
+    if (behavior == SplitBehavior::SkipToken) {
       lastPos = str.find_first_not_of(delim, pos);
     } else {
       if (pos == std::string::npos) {
@@ -31,7 +31,7 @@ vector<string> split(const string& str,
 };
 
 vector<string> split(const string& str, const string& delim) {
-  return split(str, delim, SplitBehavior::SkipEmpty);
+  return split(str, delim, SplitBehavior::SkipToken);
 };
 
 }  // namespace strings

diff --git a/src/rime/algo/strings.h b/src/rime/algo/strings.h
@@ -7,7 +7,7 @@
 namespace rime {
 namespace strings {
 
-enum class SplitBehavior { KeepEmpty, SkipEmpty };
+enum class SplitBehavior { KeepToken, SkipToken };
 
 vector<string> split(const string& str,
                      const string& delim,

diff --git a/src/rime/dict/entry_collector.cc b/src/rime/dict/entry_collector.cc
@@ -87,7 +87,7 @@ void EntryCollector::Collect(const string& dict_file) {
       continue;
     }
     // read a dict entry
-    auto row = strings::split(line, "\t");
+    auto row = strings::split(line, "\t", strings::SplitBehavior::KeepToken);
     int num_columns = static_cast<int>(row.size());
     if (num_columns <= text_column || row[text_column].empty()) {
       LOG(WARNING) << "Missing entry text at #" << num_entries << ".";