diff --git a/cleantext/clean.py b/cleantext/clean.py index c3cfc98..47e297e 100644 --- a/cleantext/clean.py +++ b/cleantext/clean.py @@ -127,6 +127,14 @@ def replace_urls(text, replace_with=""): """ Replace all URLs in ``text`` str with ``replace_with`` str. """ + CJK = r'[\u4e00-\u9fff\u3040-\u30ff]' # CJK Unicode ranges: \u4e00-\u9fff (Chinese/Japanese Kanji), \u3040-\u30ff (Hiragana/Katakana) + B = r'[^\s,。?!;:、()【】「」“”‘’\)\]\}]' + text = re.sub( + r'(' + B + r'*?' + CJK + B + r'*?https?://' + B + r'*|' + + B + r'*?https?://' + B + r'*?' + CJK + B + r'*)', + lambda m: ''.join(re.findall(CJK, m.group(1))) + replace_with, + text + ) return constants.URL_REGEX.sub(replace_with, text) diff --git a/tests/test_clean.py b/tests/test_clean.py index 38def79..9232203 100644 --- a/tests/test_clean.py +++ b/tests/test_clean.py @@ -30,6 +30,30 @@ def test_replace_urls(): assert cleantext.replace_urls(text, "*URL*") == proc_text +def test_replace_CJK_urls(): + texts = [ + [ + "请点击http://www.example.com查看内容", + "请点击查看内容*URL*", + ], + [ + "http://www.tie你好ba.com", + "你好*URL*", + ], + [ + "こんにちはhttp://example.com/テスト", + "こんにちはテスト*URL*", + ], + [ + "前部http://leetcode.com後ろにある", + "前部後ろにある*URL*", + ], + ] + + for text, proc_text in texts: + assert cleantext.replace_urls(text, "*URL*") == proc_text + + email_addresses = [ "mustermann@fh-aachen.de", "mustermann(at)fh-aachen.de",