Fix HMM segmentation of digit+letter tokens to align with Python jieba

include/cppjieba/HMMSegment.hpp

@@ -91,9 +91,20 @@ class HMMSegment: public SegmentBase {
         break;
       }
     }
+    // Optionally consume a decimal suffix (e.g. "abc1.2" -> keep together)
+    if (begin != end && begin->rune == '.') {
+      RuneStrArray::const_iterator after_dot = begin + 1;
+      if (after_dot != end && '0' <= after_dot->rune && after_dot->rune <= '9') {
+        begin++; // consume '.'
+        while (begin != end && '0' <= begin->rune && begin->rune <= '9') {
+          begin++;
+        }
+      }
+    }
     return begin;
   }
-  //
+  // numbers rule: digits optionally followed by letters, then an optional decimal suffix
+  // Handles the digit-starting subset of Python jieba finalseg re_skip = "[a-zA-Z0-9]+(?:\.\d+)?"
   RuneStrArray::const_iterator NumbersRule(RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end) const {
     Rune x = begin->rune;
     if ('0' <= x && x <= '9') {
@@ -103,12 +114,22 @@ class HMMSegment: public SegmentBase {
     }
     while (begin != end) {
       x = begin->rune;
-      if ( ('0' <= x && x <= '9') || x == '.') {
+      if (('0' <= x && x <= '9') || ('a' <= x && x <= 'z') || ('A' <= x && x <= 'Z')) {
         begin++;
       } else {
         break;
       }
     }
+    // Optionally consume a decimal suffix (e.g. "3.14" or "5G3.14")
+    if (begin != end && begin->rune == '.') {
+      RuneStrArray::const_iterator after_dot = begin + 1;
+      if (after_dot != end && '0' <= after_dot->rune && after_dot->rune <= '9') {
+        begin++; // consume '.'
+        while (begin != end && '0' <= begin->rune && begin->rune <= '9') {
+          begin++;
+        }
+      }
+    }
     return begin;
   }
   void InternalCut(RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end, vector<WordRange>& res) const {

test/unittest/segments_test.cpp

@@ -32,6 +32,24 @@ TEST(MixSegmentTest, Test1) {
     ASSERT_EQ(actual, expected);
   }
 
+  {
+    // Digit+letter combinations should be kept together, matching Python jieba behavior
+    sentence = "5G网络速度快";
+    expected = "5G/网络/速度/快";
+    segment.Cut(sentence, words);
+    actual = Join(words.begin(), words.end(), "/");
+    ASSERT_EQ(actual, expected);
+  }
+
+  {
+    // Decimal number followed by letters: decimal and letters remain separate
+    sentence = "3.5KG重量";
+    expected = "3.5/KG/重量";
+    segment.Cut(sentence, words);
+    actual = Join(words.begin(), words.end(), "/");
+    ASSERT_EQ(actual, expected);
+  }
+
   {
     sentence = "他来到了网易杭研大厦";
     expected = "他/来到/了/网易/杭/研/大厦";
@@ -185,6 +203,45 @@ TEST(HMMSegmentTest, Test1) {
   }
 }
 
+TEST(HMMSegmentTest, AlphanumericCombinations) {
+  // Tests for digit+letter and letter+decimal combinations to align with
+  // Python jieba finalseg behavior: re_skip = "[a-zA-Z0-9]+(?:\.\d+)?"
+  HMMSegment segment(DICT_DIR "/hmm_model.utf8");
+  vector<string> words;
+
+  {
+    // digit followed by letter (no decimal) should stay together
+    const char* str = "5G";
+    const char* res[] = {"5G"};
+    segment.Cut(str, words);
+    ASSERT_EQ(words, vector<string>(res, res + sizeof(res)/sizeof(res[0])));
+  }
+
+  {
+    // digit followed by letter then Chinese
+    const char* str = "3D打印";
+    const char* res[] = {"3D", "打印"};
+    segment.Cut(str, words);
+    ASSERT_EQ(words, vector<string>(res, res + sizeof(res)/sizeof(res[0])));
+  }
+
+  {
+    // decimal number followed by letters: decimal and letters are separate
+    const char* str = "3.5KG";
+    const char* res[] = {"3.5", "KG"};
+    segment.Cut(str, words);
+    ASSERT_EQ(words, vector<string>(res, res + sizeof(res)/sizeof(res[0])));
+  }
+
+  {
+    // letter followed by digit followed by decimal
+    const char* str = "v1.2";
+    const char* res[] = {"v1.2"};
+    segment.Cut(str, words);
+    ASSERT_EQ(words, vector<string>(res, res + sizeof(res)/sizeof(res[0])));
+  }
+}
+
 TEST(FullSegment, Test1) {
   FullSegment segment(TEST_DATA_DIR "/extra_dict/jieba.dict.small.utf8");
   vector<string> words;