output invalid pinyin the same as input

Author: pkufool

cppinyin/csrc/cppinyin.cc

@@ -232,13 +232,13 @@ void PinyinEncoder::Cut(const std::string &str,
       fail_bytes = 0;
       for (const auto &value : values_[std::get<2>(route[i])]) {
         auto value_t = value;
+        auto it = tone_to_normal_.find(value);
+        if (it == tone_to_normal_.end()) {
+          ostrs->push_back(value_t);
+          continue;
+        }
         if (tone == "normal") {
-          if (tone_to_normal_.find(value) != tone_to_normal_.end()) {
-            value_t = tone_to_normal_.at(value);
-          } else {
-            std::cerr << "PinyinEncoder: " << value
-                      << " is not in the NORMAL_TO_TONE map. " << std::endl;
-          }
+          value_t = it->second;
         }
         if (partial) {
           auto initial = GetInitial(value_t);
@@ -359,13 +359,9 @@ void PinyinEncoder::LoadVocab(std::istream &is) {
     std::vector<std::string> values;
     while (iss >> value) {
       // Always convert to tone in internal
-      if (!std::isdigit(value.back())) {
-        if (NORMAL_TO_TONE.find(value) == NORMAL_TO_TONE.end()) {
-          std::cerr << "PinyinEncoder: " << value
-                    << " is not in the NORMAL_TO_TONE map. " << std::endl;
-        } else {
-          value = NORMAL_TO_TONE.at(value);
-        }
+      auto it = NORMAL_TO_TONE.find(value);
+      if (it != NORMAL_TO_TONE.end()) {
+        value = it->second;
       }
       values.push_back(value);
     }
@@ -450,7 +446,8 @@ std::string PinyinEncoder::ToInitial(const std::string &s) const {
     return s;
   }
   if (tone_to_normal_.count(s) == 0 && NORMAL_TO_TONE.count(s) == 0) {
-    std::cerr << "ToInitial: " << s << " is not a valid pinyin. " << std::endl;
+    // std::cerr << "ToInitial: " << s << " is not a valid pinyin. " <<
+    // std::endl;
     return std::string();
   }
   return GetInitial(s);
@@ -477,7 +474,7 @@ std::string PinyinEncoder::ToFinal(const std::string &s,
   } else if (NORMAL_TO_TONE.find(s) != NORMAL_TO_TONE.end()) {
     value = NORMAL_TO_TONE.at(s);
   } else {
-    std::cerr << "ToFinal: " << s << " is not a valid pinyin. " << std::endl;
+    // std::cerr << "ToFinal: " << s << " is not a valid pinyin. " << std::endl;
     return std::string();
   }
   if (tone == "none") {
@@ -551,13 +548,9 @@ size_t PinyinEncoder::LoadValues(std::istream &ifile) {
     for (uint32_t j = 0; j < sub_size; ++j) {
       offset += ReadString(ifile, &value);
       // Always convert to number tone in internal
-      if (!std::isdigit(value.back())) {
-        if (NORMAL_TO_TONE.find(value) == NORMAL_TO_TONE.end()) {
-          std::cerr << "PinyinEncoder: " << value
-                    << " is not in the NORMAL_TO_TONE map. " << std::endl;
-        } else {
-          value = NORMAL_TO_TONE.at(value);
-        }
+      auto it = NORMAL_TO_TONE.find(value);
+      if (it != NORMAL_TO_TONE.end()) {
+        value = it->second;
       }
       values_[i][j] = value;
     }

cppinyin/python/cppinyin/cli.py

@@ -110,9 +110,7 @@ def build(output: Path, dict_path: Path, user_dict_path: Path):
     """
     output = Path(output)
     output.parent.mkdir(parents=True, exist_ok=True)
-
     encoder = Encoder(get_dict_path(dict_path, user_dict_path))
-
     encoder.save(str(output))
 
 
@@ -127,11 +125,10 @@ def build(output: Path, dict_path: Path, user_dict_path: Path):
     "--user-dict-path", type=Path, help="The path to user customized dict."
 )
 @click.option(
-    "--no-tone",
-    is_flag=True,
-    show_default=True,
-    default=False,
-    help="Whether to include tones in output pinyins or not.",
+    "--tone",
+    type=click.Choice(["normal", "number", "none"]),
+    default="number",
+    help="Choose the tone style of pinyin",
 )
 @click.option(
     "--partial",
@@ -144,7 +141,7 @@ def encode(
     input: str,
     dict_path: Path,
     user_dict_path: Path,
-    no_tone: bool,
+    tone: str,
     partial: bool,
 ):
     """
@@ -161,14 +158,10 @@ def encode(
         with open(input, "r") as fi:
             for line in fi:
                 pinyin = " ".join(
-                    encoder.encode(
-                        line.strip(), tone=not no_tone, partial=partial
-                    )
+                    encoder.encode(line.strip(), tone=tone, partial=partial)
                 )
                 click.echo(f"{line.strip()}\t{pinyin}")
     else:
         click.echo(
-            " ".join(
-                encoder.encode(input.strip(), tone=not no_tone, partial=partial)
-            )
+            " ".join(encoder.encode(input.strip(), tone=tone, partial=partial))
         )