diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 54999a9..5f994cb 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -22,6 +22,9 @@ jobs: - name: Run unit tests shell: pwsh run: python -m unittest discover -s tests -v + - name: Test native UI preference persistence + shell: powershell + run: .\tests\native_ui_preference_smoke.ps1 - name: Build PIME overlay shell: powershell run: .\build_pime_overlay.ps1 @@ -43,5 +46,5 @@ jobs: uses: softprops/action-gh-release@v2 with: files: | - release/Smart-Priority-Bopomofo-Setup-0.4.2.exe + release/Smart-Priority-Bopomofo-Setup-0.6.0.exe release/SHA256SUMS.txt diff --git a/AGENTS.md b/AGENTS.md index ad673a5..4972e65 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -23,6 +23,7 @@ When PIME is installed, also run: ```powershell & 'C:\Program Files (x86)\PIME\python\python3\python.exe' ./tests/pime_adapter_smoke.py +& 'C:\Program Files (x86)\PIME\python\python3\python.exe' ./tests/pime_all_readings_audit.py ``` ## Protected interaction contracts @@ -30,24 +31,78 @@ When PIME is installed, also run: - Tapping Shift toggles persistent Chinese/English mode. - Holding Shift and pressing A-Z emits one temporary uppercase English letter without changing modes. Pending Chinese must be committed before the letter. -- Shift punctuation, the five-item candidate menu, candidate expansion, and - right-side character editing are core behaviors. Do not replace or remove - them without an explicit user request. +- Shift punctuation, ten-item two-column candidate pages, Right/Down pagination, + and right-side character editing are core behaviors. The left column must + contain 1-5 from top to bottom and the right column 6-0; never bind A-J as + candidate labels. Do not replace or + remove these behaviors without an explicit user request. - Candidate editing must offer and atomically apply common/personal phrases - spanning 2-12 syllables while keeping single-character choices available in - the compact five-row menu. + spanning 2-12 syllables while keeping high-frequency single-character and + literal-Zhuyin choices available on the first ten-item page. At the end of + the composition, phrase/sentence choices lead. After the caret moves inside + the composition, the single character to its right must lead, so candidate + 1 locks only that syllable and advances instead of confirming the sentence. +- Automatic composition and the candidate editor share + `_ranked_phrase_options()`. Conservative fuzzy-reading and exact typo + corrections must enter that function as visible whole-sentence candidates; + Enter must never perform a hidden final rewrite. Before opening Down or + committing through any path, `_apply_phrase_ranking()` must synchronize the + editable buffer. A better whole-buffer candidate must never be hidden behind + Down; arrows are for overriding the default, not obtaining a default the + engine already knew. - Ranking order is a core contract: an explicit user candidate selection is first. Bundled candidates compare exact reading-span coverage before source weight, so a shorter suffix cannot overwrite a longer complete conversion; equal spans prefer Taiwan character/word frequencies, then engine/Rime. + A stored single-character pin stays candidate zero for isolated input but + is not a context lock; a reliable word or whole-buffer conversion may + override it. Only a choice made explicitly in the current composition or a + learned personal phrase locks its covered segments. Automatic commits must not silently teach the personal stores. -- Activation, a Windows keyboard-close status, and forced focus termination - restore the profile to keyboard-open Bopomofo mode. Password fields and apps - that explicitly disable IMEs remain under Windows control. -- Numpad 0-9 and VK_DECIMAL always emit digits/dot and never Bopomofo or - candidate numbers. +- Single-character ranking preserves libchewing's reading-aware dictionary + default, then uses global Taiwan frequency for the remaining tail. Global + frequency has no pronunciation information and must never promote a common + alternate-reading character such as 員 over 運 for ㄩㄣˋ. +- Text/weight-only frequency indexes are search aids, not pronunciation + evidence. Before a corpus phrase or fuzzy-reading correction changes live + text, validate the complete span through `phrase_candidates(readings)`. + Character-column membership alone must never let an alternate reading + borrow an unrelated word (for example 貝殼 for ㄅㄟˋ ㄑㄩㄝˋ). +- Autocorrection is conservative, offline, and visible before commit. Apply + only exact, same-length high-confidence rules from `data/common_typos.json` + as whole-sentence candidates. A currently selected candidate or a learned + personal phrase is protected and must outrank every autocorrection rule; a + stored single-character pin alone is not a context lock. Do not add + context-dependent pairs such as 的/得/地 or + 在/再 as unconditional replacements, and never upload text for correction. +- Unlocked spans are re-ranked live from their retained Bopomofo readings + through `phonetic_corrector.py` and the bundled phrase indexes. Reviewed + fuzzy phonetic-slot confusions and fallback typo rules must become visible + whole-sentence candidates before Enter. Surface variants such as 音該/英該 + must not be enumerated as separate rules, and a valid exact-reading phrase + must be preserved as an alternate candidate. +- Activation and forced focus termination reset only the profile's internal + Shift toggle. Respect the TSF keyboard-open state supplied by the host; + never reopen a compartment after an app closes it. This prevents games and + secure/custom controls from entering an open/close feedback loop. +- Numpad 0-9, decimal, divide, multiply, subtract, and add always emit their + literal ASCII characters and never Bopomofo or candidate numbers. Shift+A-Z and Shift punctuation replace only an unfinished active syllable, preserve completed Chinese, and emit at the active caret. +- Literal Bopomofo is a core behavior. For any lone initial, medial, or rime, + Space first asks the dictionary for the corresponding first-tone Chinese + syllable; do not classify every initial as invalid because ㄙ and ㄓ, + and similar syllabic initials are real readings. When candidate zero is + Chinese, it is auto-selected and raw Zhuyin stays within the first four + candidates. When candidate zero is literal Zhuyin, open the literal-first + menu even if libchewing appended obscure Han characters to the tail. + Completed readings also offer their literal spelling within the + first four candidates, including sentence editing. With no active syllable, + bare DaQian tone keys emit their + tone marks (3=ˇ, 6=ˊ, 4=ˋ, 7=˙), while Shift+Space emits ˉ. Numpad digits + and operators remain literal text. When completed segments already exist, + raw Zhuyin and bare tone marks stay as protected segments in the editable + composition; they must not commit the surrounding text like Enter. - Feedback collection records only explicit conversion corrections. Never add surrounding text, application identity, automatic upload, or a network call to the IME runtime. The user must review records before opening a report. @@ -57,8 +112,24 @@ When PIME is installed, also run: - `bopomofo_core/data/taiwan_frequency.json` is generated from the Ministry of Education's official open-data character and word tables. Never hand-edit it; preserve attribution, source hashes, generator, and data notice. +- `bopomofo_core/data/common_typos.json` is a reviewed source-attributed rule + list. Keep source identifiers and URLs, require equal-length replacements, + and add regression tests for every policy change. - Add a regression assertion to `tests/pime_adapter_smoke.py` whenever a core interaction is fixed. +- Single-character ranking changes must also pass + `tests/pime_all_readings_audit.py`, which enumerates every Bopomofo slot and + tone combination accepted by libchewing. Never replace this with a handful + of reported syllable examples. +- Candidate font, grid width, labels, and navigation can be changed through + PIME's Python protocol. The authorized native style lives under `native_ui/` + and is built from the exact bundled PIME `v1.3.0-stable` source. Install it + only through the explicit `-EnableUnsignedNativeUi` opt-in, only when PIME + has no unrelated modules, preserve the original DLLs, and restore them only + when the installed hashes still match our build. Fresh installs keep the + signed DLLs for game compatibility. Once opted in, persist that preference + across normal and EXE updates; never silently restore the signed UI during + a Python-layer update. `-DisableUnsignedNativeUi` is the explicit rollback. Run `./build_release.ps1` for a distributable installer. Update all version locations together when cutting a new version. diff --git a/AI_MAINTENANCE.md b/AI_MAINTENANCE.md index 99c43a4..4b7a68b 100644 --- a/AI_MAINTENANCE.md +++ b/AI_MAINTENANCE.md @@ -19,9 +19,46 @@ - `bopomofo_core/pinned_store.py`:單一讀音的個人優先字。 - `bopomofo_core/phrase_store.py`:使用者確認過的 2–12 字詞語。 - `bopomofo_core/frequency_lexicon.py`:離線高頻詞索引查詢;資料位於 `bopomofo_core/data/`。 +- `bopomofo_core/autocorrect.py`:產生可見完整句候選的離線高可信錯字修正;規則位於 `bopomofo_core/data/common_typos.json`。 +- `bopomofo_core/phonetic_corrector.py`:以每個字保留的注音、候選欄與常用詞庫重新解碼;同一讀音或保守的注音槽位混淆不應展開成大量表面錯字規則。 - `tools/build_frequency_lexicon.py`:從固定版本 Rime Essay 重建臺灣正體高頻詞索引。生成的 JSON 不應手工修改。 - `tests/`:核心與 PIME 整合測試。 - `installer/`:正式安裝與解除安裝流程。 +- `native_ui/`:候選框的完整 LGPL 原始碼、可重現建置腳本及 x86/x64 產物。 + +候選視窗的字型、每列數量、選擇標籤與方向鍵行為由 Python 模組控制;圓角、顏色、 +邊框與選取樣式位於 `native_ui/src/CandidateWindow.cpp`。原生元件固定從隨附的 +PIME `v1.3.0-stable` 重建,但它沒有正式程式碼簽章,可能與遊戲反作弊衝突。 +全新安裝與尚未選擇介面的使用者保留 PIME 原廠簽章 DLL;只有使用者明確傳入 +`-EnableUnsignedNativeUi` 且沒有其他 PIME 模組時才首次套用自訂 DLL。這項選擇必須 +跨一般更新、EXE 與 AI 維護持久保存,除非明確傳入 `-DisableUnsignedNativeUi`,不得在 +更新 Python 層時恢復舊簽章介面。安裝器須備份與還原原始 DLL;被 TSF 鎖定時以 +`MoveFileEx` 排程至重開機,不能把這些限制拿掉。 + +候選窗每頁固定 10 個、分成兩個直欄:左欄由上而下 `1–5`,右欄由上而下 `6–0`;`→` 直接翻到下一頁,`↓` 依數字順序逐項移動並在 +本頁末端翻頁。完整讀音的原始注音須留在前四個候選。單獨注音按空白時一律先向 +字典查詢補上一聲後的候選,不可單靠 initial/medial/rime 分類判定,因為 `ㄙ`、 +`ㄓ` 等聲母本身也是完整音節。字典候選零是中文字時直接採用;候選零仍是原注音 +時必須開啟原符號第一的選單,不可讓尾端生僻字自動勝出。原注音仍須保留在前四項。 +沒有作用中的音節時,大千聲調鍵直接輸出符號(3=ˇ、6=ˊ、4=ˋ、7=˙);已有未提交 +文字時,聲調與候選中的原始注音須以受保護片段插入同一組字區,不得提交其他片段。 +右側數字鍵盤的數字、小數點與 `/ * - +` 必須直接輸出原字元,不可進入注音映射。 + +精確注音、保守模糊讀音與補充錯字規則都必須先產生可見的完整句候選,並即時更新 +未鎖定的組字內容;Enter 不得在送出瞬間暗中改字。原始精確讀音句要保留為後續候選。 +規則必須等長、精確、高可信且附來源。當次親自選字與個人詞彙所涵蓋的字元須設為保護範圍; +已儲存的單字優先只控制單一讀音排序,不得鎖死整句脈絡。`的/得/地`、`在/再`、合法異形詞等需要語境的 +項目不可加入無條件規則。執行期間不得傳送文字到網路,也不得把自動修正當成個人學習。 + +完整句預設與候選視窗必須共用 `_ranked_phrase_options()`;開啟候選及任何送出動作前 +都要先執行 `_apply_phrase_ranking()`。禁止新增只在候選視窗可見、但不會同步到組字區 +的另一套第一候選邏輯,也禁止在 Enter 路徑另外做不可見修正。方向鍵是改選功能, +不是取得系統已知正解的必要步驟。 + +Rime Essay 與台灣字頻只有文字/權重,沒有完整詞語讀音;其逐字候選交集只能用來搜尋, +不可直接更改組字。任何高頻詞或模糊音修正都必須再經 `phrase_candidates(readings)` +驗證整段讀音。游標在句中時,候選零必須是游標右側的單字候選,選擇後只鎖定涵蓋的 +音節並前進;詞語與完整句候選保留在同一頁。游標在句尾時才可把完整句/詞語排在前面。 ## 修改後必須驗證 @@ -29,6 +66,8 @@ python -m unittest discover -s tests -v .\build_pime_overlay.ps1 & 'C:\Program Files (x86)\PIME\python\python3\python.exe' .\tests\pime_adapter_smoke.py +& 'C:\Program Files (x86)\PIME\python\python3\python.exe' .\tests\pime_all_readings_audit.py +.\native_ui\build_native_ui.ps1 .\build_release.ps1 ``` @@ -63,7 +102,10 @@ Git Bash 使用: - `tools/build_taiwan_frequency.py` 從教育部字頻/詞頻 CSV 重建台灣預設排序; `bopomofo_core/data/taiwan_frequency.json` 是產物,不可手改。 -- 使用者明確選擇永遠最高;內建候選先比較詞彙涵蓋的音節數,較短後綴不可 +- 單字候選必須保留 libchewing 讀音字典的第一項,再用全域台灣字頻整理其餘候選; + 全域字頻不含讀音資訊,不得讓多音字(例如 `員`)覆蓋 `ㄩㄣˋ→運` 等讀音首選。 +- 當次使用者明確選擇永遠最高;儲存的單字優先在孤立讀音中列為候選零,但可靠的 + 整詞/整句脈絡可覆蓋它。內建候選先比較詞彙涵蓋的音節數,較短後綴不可 覆蓋較長完整轉換,涵蓋相同時再依台灣官方字詞頻、其他內建詞庫排序。 一般送出文字不可自動強化個人權重。 - `bopomofo_core/feedback_store.py` 與 `feedback-report.ps1` 只保存明確改選 diff --git a/README.md b/README.md index fb79464..3786728 100644 --- a/README.md +++ b/README.md @@ -18,25 +18,30 @@ Git Bash 使用者可把最後一行改為 `./install.sh`。 ## 主要功能 - 注音聲母、介音、韻母與聲調可用任意順序輸入;同類按鍵會取代舊值,不會誤接成下一句。 -- 完成一個注音後自動採用第一候選,不必每字按空白鍵。 -- `↓` 開啟前 5 個候選;繼續向下超過第 5 個時,展開最多 20 個實用候選,不列出整條生僻字尾。 -- 編輯選字時會優先列出游標附近的 2–12 字常用/個人詞彙,選擇後整個詞一次更新;前五項仍保留單字候選,不會犧牲逐字編輯。 -- 選過的單字會記成該讀音的第一優先,之後先出現。 -- 會參考 libchewing 詞庫、Rime Essay 轉製的 113,738 條臺灣正體高頻詞,以及使用者確認過的 2–12 字詞,改善「優化、樹葉、人工智慧」一類上下文選字。 +- 完成一個注音後自動採用第一候選,不必每字按空白鍵。單字先保留讀音字典的首選,再以台灣字頻整理後續候選,避免全域高頻的多音字誤蓋正確讀音,例如 `ㄩㄣˋ` 固定先出「運」而不是通常讀作 `ㄩㄢˊ` 的「員」。 +- `↓` 開啟候選,每頁像微軟注音一樣顯示兩個直欄:左欄由上而下為 `1–5`,右欄為 `6–0`,不以英文字母選字。按 `→` 前往下一頁,或用 `↓` 依數字順序移動並在第 10 項後翻頁,最多保留 20 個實用候選。 +- 在句尾開啟選字時,會先列完整句修正及游標附近的 2–12 字個人/常用詞彙;把游標移進句中後,則先列游標右側的單字候選,並在同一頁保留詞語/整句候選。這樣按 `1` 只會鎖定正在編輯的音節並自動前往下一字,不會意外確認整句;選擇詞語時仍會一次更新整個詞。 +- 選過的單字會記成該讀音的第一優先,單獨輸入時先出現;可靠的整詞/整句脈絡仍可調整它,避免舊的「仙/不」偏好阻止「你先開始下一步吧」。當次在組字區親自選定的字仍會鎖定。 +- `ㄉㄜ˙`、`ㄗㄨㄛˋ`、`ㄕˋ` 的台灣預設第一候選分別固定為「的、做、是」;只有使用者親自改選其他字時,個人學習才會暫時排在它前面。 +- 會參考 libchewing 詞庫、Rime Essay 轉製的 113,738 條臺灣正體高頻詞,以及使用者確認過的 2–12 字詞,改善「優化、樹葉、人工智慧」一類上下文選字。文字型高頻詞必須再通過整段注音驗證,避免把罕見破音字拼成讀音不符的詞,例如不會把 `ㄅㄟˋ ㄑㄩㄝˋ` 誤判成「貝殼」。 - 「不」在 `ㄅㄨˋ` 中優先,並內建「不要、不是」及「再見、現在、跟在」等保守的繁體中文常用規則;有歧義的「在做/再做」仍由上下文判斷。 +- 組字時會持續以精確注音、保守近音判斷與離線常用詞庫更新整段第一候選,讓「優仙→優先」「部合理→不合理」以及把 `ㄣ/ㄥ` 混淆成 `ㄧㄣ ㄍㄞ` 的「音該→應該」直接成為可見的整句首選,不必等到按 `Enter` 才暗中變字,也不必逐一列舉錯字表面形式。另有 46 條高可信用字規則處理「以經→已經」「迫不急待→迫不及待」等錯字;當次親自選定或個人詞彙鎖定的字不會被覆蓋。 +- 自動組字與候選視窗共用同一份詞彙排序;若候選系統已把完整句修正排在第一,組字區會先直接採用,不必按 `↓` 才取得正確答案,原始精確讀音句則保留在後續候選供明確改選。按 `↓`、空白、Enter、標點、Shift 或切換輸入框前都會再次同步第一候選;Enter 只提交畫面上已採用的文字,方向鍵只用來改選其他答案。 - 整段尚未送出的文字均可編輯;候選目標位於游標右邊,選完會前往下一字。 - 在段落中間按 Backspace 刪除後,下一個注音會補進原位置,不會跳到最右邊。 - 短按 `Shift` 切換中英文;按住 `Shift` 再按字母則暫時輸入大寫英文且不切換模式。兩種操作都會正確處理尚未完成或尚未送出的注音,不會讓英文跳到中文字前方。 -- 啟用輸入法、切換輸入框或 Windows 關閉中文鍵盤狀態時會恢復注音模式;密碼欄位或明確停用輸入法的應用程式仍由 Windows 決定。 -- 常用 `Shift` 標點包括 `?`、`:`、`+`、`——`、`,`、`。`、`『』` 與 `"`。 +- 啟用輸入法、切換輸入框或組字被強制終止時只重設內部的中英文切換,不會強制改寫應用程式的 TSF 開關;遊戲、密碼欄位或明確停用輸入法的控制項可保持關閉,避免反覆開關造成凍結。 +- 完整盤點標準鍵盤的 `Shift` 組合:數字列、`?`、`:`、`+`、`——`、`,`、`。`、`『』`、`「」`、`~` 與 `|` 都有明確輸出;`Shift+Tab`、`Shift+F1–F12`、`Ctrl+Shift` 與 `Alt+Shift` 仍交給應用程式或 Windows。 +- 單獨注音按空白時會先依字典補上一聲:字典第一候選是中文字就直接採用,例如 `ㄧ→一`、`ㄚ→啊/阿`,而 `ㄙ`、`ㄓ` 也會進入司/思、之/知等中文字候選;第一候選仍是原注音時則開啟候選單,不讓尾端生僻字冒充常用字。原始注音仍保留在第一頁前四項。完整的 `ㄢˊ` 等讀音同樣保留原注音候選。沒有正在組字時,直接按大千聲調鍵 `3`、`6`、`4`、`7` 會輸出 `ˇ`、`ˊ`、`ˋ`、`˙`;已有未提交文字時,原始注音與聲調會插入同一個可編輯組字區,不會產生 Enter 式提交。`Shift+Space` 輸出 `ˉ`。 - 快速重疊按鍵有實體鍵碼備援,降低漏掉前一個注音符號的機率。 +- 候選依微軟注音習慣先排滿左欄 `1–5`,再排右欄 `6–0`。一般安裝保留 PIME 原廠簽章 DLL,以降低遊戲與反作弊程式的載入風險;未簽章的日式原生候選外觀只提供開發者明確選用。 ## 安裝與移除 一般使用者只需要執行發佈資料夾中的: ```text -Smart-Priority-Bopomofo-Setup-0.4.2.exe +Smart-Priority-Bopomofo-Setup-0.6.0.exe ``` 從 GitHub clone 原始碼後,可在 PowerShell 執行一鍵安裝: @@ -60,11 +65,22 @@ PIME 1.3.0;若已經有 PIME,則保留現有版本,不做降版。完成 安裝程式會把智慧優先注音排在繁體中文鍵盤清單第一順位,並設成預設輸入法,讓 Alt+Shift 切回繁體中文時優先選到它。 +全新安裝預設保留 PIME 原廠簽章 DLL。只有使用者明確接受遊戲/反作弊相容性風險時, +才可從原始碼執行 `.\install.ps1 -EnableUnsignedNativeUi` 啟用日式原生外觀。這項選擇會 +保存在本機,後續一般更新及新版 EXE 都會沿用,不會默默恢復舊簽章介面;若要主動切回 +原廠簽章版,執行 `.\install.ps1 -DisableUnsignedNativeUi`。檔案正被 TSF 使用時,切換 +介面會安全排程至下次重新啟動 Windows。 + 若安裝程式在新電腦上首次安裝 PIME,會隱藏隨附的「新酷音」,只留下智慧優先 注音;若電腦原本已有 PIME,則不會擅自刪除使用者既有的新酷音。 解除安裝請到 Windows「設定 → 應用程式 → 已安裝的應用程式」。它只會移除 -「智慧優先注音」,不移除共用的 PIME,也不刪除個人學習資料。 +「智慧優先注音」,不移除共用的 PIME,也不刪除個人學習資料。新版候選框只會在 +PIME 沒有其他輸入法模組時套用;安裝前會備份原始 PIME DLL,解除安裝時安全還原。 +若 Windows 正在使用候選框 DLL,安裝仍會完成,並在下一次重新啟動 Windows 後 +自動套用新介面。備份與待更新狀態統一存放於 +`%ProgramData%\SmartPriorityBopomofo\native-state`,所以從 EXE 或 Git clone +安裝都能由解除安裝程式正確辨識。 ## 使用者資料與維護 @@ -93,6 +109,13 @@ python -m unittest discover -s tests -v .\build_pime_overlay.ps1 ``` +已安裝 PIME 時,再執行實際鍵盤事件與全讀音稽核: + +```powershell +& 'C:\Program Files (x86)\PIME\python\python3\python.exe' .\tests\pime_adapter_smoke.py +& 'C:\Program Files (x86)\PIME\python\python3\python.exe' .\tests\pime_all_readings_audit.py +``` + 建立正式安裝包(需要 NSIS 3.12): ```powershell @@ -116,4 +139,7 @@ python -m unittest discover -s tests -v 候選排序固定先保留使用者明確選擇;內建來源先比較完整涵蓋的音節數,避免 「對話框」被較短的「畫框」覆蓋;涵蓋相同時才依教育部台灣字詞頻資料、 內建詞庫的順序決定。 -右側數字鍵盤的 0–9 與小數點永遠輸出數字或 `.`;Shift+英文字母或符號會取代當前未完成音節。 +完整句的自動組字與候選欄必須呼叫同一個排序來源;保守近音與高可信錯字修正也必須先成為可見的完整句候選,不可等到 Enter 才暗中修改。所有送出路徑只會先同步該可見預設,再提交畫面上的文字。 +右側數字鍵盤的 `0–9`、小數點及 `/ * - +` 永遠直接輸出對應字元,不會被解讀成注音或候選鍵;Shift+英文字母或符號會取代當前未完成音節。 + +Enter 修正會先由 `bopomofo_core/phonetic_corrector.py` 使用每個字保留下來的注音、候選字與離線常用詞庫重新解碼,並保守處理 `ㄣ/ㄥ`、`ㄓ/ㄗ`、`ㄔ/ㄘ`、`ㄕ/ㄙ` 等常見發音混淆。補充規則位於 `bopomofo_core/data/common_typos.json`,只保存錯字、正字及公開來源識別,不保存使用者輸入。規則參考國家教育研究院用字分析,並以教育部國語辭典核對正詞;不把「的/得/地、在/再」或合法異形詞放進無條件自動替換。 diff --git a/THIRD_PARTY_NOTICES.txt b/THIRD_PARTY_NOTICES.txt index ffe71d7..75dfbf4 100644 --- a/THIRD_PARTY_NOTICES.txt +++ b/THIRD_PARTY_NOTICES.txt @@ -1,4 +1,4 @@ -智慧優先注音 0.4.2 — 第三方軟體聲明 +智慧優先注音 0.6.0 — 第三方軟體聲明 ======================================== 本安裝包包含或搭配下列開放原始碼元件: @@ -6,10 +6,14 @@ 1. PIME 專案:https://github.com/EasyIME/PIME 本安裝包附帶版本:1.3.0 stable - 原始碼版本(開發驗證):571759f471c93e288682305148df751a12f5415e + 隨附安裝程式來源版本:571759f471c93e288682305148df751a12f5415e + 修改候選框來源版本:26fcf6ac8874e76b8f75f6826811b03bfdfc2e89 + (PIME v1.3.0-stable;libIME2 8ad3c9b433d930ce5614c483461dfa78cedb5efd) 授權:GNU Lesser General Public License 2.0(各附帶元件依 PIME LICENSE.txt 所列授權) 官方安裝檔保留原作者有效的數位簽章;若電腦已有 PIME,安裝程式不會降版。 + `native_ui` 內附修改後的完整來源、LGPL 條款與可重現建置腳本;候選框 DLL + 僅在沒有其他 PIME 輸入法模組時套用,解除安裝時還原備份。 2. libchewing 專案:https://github.com/chewing/libchewing diff --git a/bopomofo_core/autocorrect.py b/bopomofo_core/autocorrect.py new file mode 100644 index 0000000..2ca98be --- /dev/null +++ b/bopomofo_core/autocorrect.py @@ -0,0 +1,115 @@ +"""Conservative, offline correction of high-confidence Traditional Chinese typos.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Sequence + + +DEFAULT_RULES = Path(__file__).with_name("data") / "common_typos.json" + + +@dataclass(frozen=True) +class AutocorrectRule: + wrong: str + correct: str + source: str + + +@dataclass(frozen=True) +class Autocorrection: + start: int + wrong: str + correct: str + source: str + + +class Autocorrector: + """Apply exact, same-length rules without touching protected characters.""" + + def __init__(self, path: str | Path = DEFAULT_RULES) -> None: + self.path = Path(path) + self.rules: tuple[AutocorrectRule, ...] = () + self.sources: dict[str, dict[str, str]] = {} + if not self.path.exists(): + return + try: + raw = json.loads(self.path.read_text(encoding="utf-8")) + if not isinstance(raw, dict): + return + sources = raw.get("meta", {}).get("sources", []) + if isinstance(sources, list): + self.sources = { + str(source["id"]): { + str(key): str(value) for key, value in source.items() + } + for source in sources + if isinstance(source, dict) and source.get("id") + } + + unique: dict[str, AutocorrectRule] = {} + rows = raw.get("rules", []) + if not isinstance(rows, list): + return + for row in rows: + if not isinstance(row, dict): + continue + wrong = row.get("wrong") + correct = row.get("correct") + source = row.get("source") + if not all(isinstance(value, str) for value in (wrong, correct, source)): + continue + # The first release deliberately keeps one character aligned + # with one completed syllable. This makes user-selected spans + # safe to protect and avoids surprising insertion/deletion. + if not wrong or wrong == correct or len(wrong) != len(correct): + continue + if source not in self.sources: + continue + unique.setdefault(wrong, AutocorrectRule(wrong, correct, source)) + self.rules = tuple( + sorted(unique.values(), key=lambda rule: (-len(rule.wrong), rule.wrong)) + ) + except (OSError, ValueError, TypeError, KeyError): + # Autocorrection is optional. A missing or damaged rule file must + # never stop the input method from starting. + self.rules = () + self.sources = {} + + @property + def rule_count(self) -> int: + return len(self.rules) + + def correct( + self, text: str, protected: Sequence[bool] | None = None + ) -> tuple[str, list[Autocorrection]]: + """Return corrected text and an audit trail of applied exact rules.""" + if protected is None: + protected = (False,) * len(text) + if len(protected) != len(text): + raise ValueError("protected character mask must match text length") + + characters = list(text) + changes: list[Autocorrection] = [] + index = 0 + while index < len(characters): + current = "".join(characters) + matched = False + for rule in self.rules: + end = index + len(rule.wrong) + if end > len(characters) or current[index:end] != rule.wrong: + continue + if any(protected[index:end]): + continue + characters[index:end] = rule.correct + changes.append( + Autocorrection(index, rule.wrong, rule.correct, rule.source) + ) + index = end + matched = True + break + if not matched: + index += 1 + return "".join(characters), changes diff --git a/bopomofo_core/data/common_typos.json b/bopomofo_core/data/common_typos.json new file mode 100644 index 0000000..8491132 --- /dev/null +++ b/bopomofo_core/data/common_typos.json @@ -0,0 +1,67 @@ +{ + "meta": { + "name": "智慧優先注音高可信常見錯字規則", + "version": 3, + "policy": "只收錄可作精確字串替換、字數相同且不依賴歧義語境的臺灣正體中文錯字。教育部資料僅用於正詞與用例核對,不重製辭典釋義。", + "sources": [ + { + "id": "naer_usage_analysis", + "title": "國家教育研究院《國語文非選擇題怎麼寫》附錄一:用字分析", + "url": "https://teric.naer.edu.tw/wSite/PDFReader?fileName=1421184541220&format=pdf&xmlId=1820524" + }, + { + "id": "moe_dictionary_validation", + "title": "教育部國語辭典公眾授權網(簡編本、小字典與成語典)", + "url": "https://language.moe.gov.tw/001/Upload/Files/site_content/M0001/respub/index.html" + } + ] + }, + "rules": [ + {"wrong": "以經", "correct": "已經", "source": "naer_usage_analysis"}, + {"wrong": "由其", "correct": "尤其", "source": "naer_usage_analysis"}, + {"wrong": "而以", "correct": "而已", "source": "naer_usage_analysis"}, + {"wrong": "即然", "correct": "既然", "source": "moe_dictionary_validation"}, + {"wrong": "必竟", "correct": "畢竟", "source": "moe_dictionary_validation"}, + {"wrong": "一但", "correct": "一旦", "source": "naer_usage_analysis"}, + {"wrong": "致於", "correct": "至於", "source": "naer_usage_analysis"}, + {"wrong": "難勉", "correct": "難免", "source": "naer_usage_analysis"}, + {"wrong": "份內", "correct": "分內", "source": "naer_usage_analysis"}, + {"wrong": "本份", "correct": "本分", "source": "naer_usage_analysis"}, + {"wrong": "迴饋", "correct": "回饋", "source": "naer_usage_analysis"}, + {"wrong": "分折", "correct": "分析", "source": "naer_usage_analysis"}, + {"wrong": "家庭煮婦", "correct": "家庭主婦", "source": "naer_usage_analysis"}, + {"wrong": "不同凡想", "correct": "不同凡響", "source": "naer_usage_analysis"}, + {"wrong": "迫不急待", "correct": "迫不及待", "source": "moe_dictionary_validation"}, + {"wrong": "按步就班", "correct": "按部就班", "source": "moe_dictionary_validation"}, + {"wrong": "一股作氣", "correct": "一鼓作氣", "source": "moe_dictionary_validation"}, + {"wrong": "再接再勵", "correct": "再接再厲", "source": "moe_dictionary_validation"}, + {"wrong": "不徑而走", "correct": "不脛而走", "source": "moe_dictionary_validation"}, + {"wrong": "默守成規", "correct": "墨守成規", "source": "moe_dictionary_validation"}, + {"wrong": "談笑風聲", "correct": "談笑風生", "source": "moe_dictionary_validation"}, + {"wrong": "走頭無路", "correct": "走投無路", "source": "moe_dictionary_validation"}, + {"wrong": "一愁莫展", "correct": "一籌莫展", "source": "moe_dictionary_validation"}, + {"wrong": "針貶時事", "correct": "針砭時事", "source": "moe_dictionary_validation"}, + {"wrong": "出奇不意", "correct": "出其不意", "source": "moe_dictionary_validation"}, + {"wrong": "名列前矛", "correct": "名列前茅", "source": "moe_dictionary_validation"}, + {"wrong": "一如即往", "correct": "一如既往", "source": "moe_dictionary_validation"}, + {"wrong": "天翻地復", "correct": "天翻地覆", "source": "moe_dictionary_validation"}, + {"wrong": "食不裹腹", "correct": "食不果腹", "source": "moe_dictionary_validation"}, + {"wrong": "有侍無恐", "correct": "有恃無恐", "source": "moe_dictionary_validation"}, + {"wrong": "甘敗下風", "correct": "甘拜下風", "source": "moe_dictionary_validation"}, + {"wrong": "蛛絲螞跡", "correct": "蛛絲馬跡", "source": "moe_dictionary_validation"}, + {"wrong": "聲名雀起", "correct": "聲名鵲起", "source": "moe_dictionary_validation"}, + {"wrong": "鬼鬼崇崇", "correct": "鬼鬼祟祟", "source": "moe_dictionary_validation"}, + {"wrong": "金榜提名", "correct": "金榜題名", "source": "moe_dictionary_validation"}, + {"wrong": "脈博", "correct": "脈搏", "source": "moe_dictionary_validation"}, + {"wrong": "松馳", "correct": "鬆弛", "source": "moe_dictionary_validation"}, + {"wrong": "修茸", "correct": "修葺", "source": "moe_dictionary_validation"}, + {"wrong": "編篡", "correct": "編纂", "source": "moe_dictionary_validation"}, + {"wrong": "親睞", "correct": "青睞", "source": "moe_dictionary_validation"}, + {"wrong": "冒然", "correct": "貿然", "source": "moe_dictionary_validation"}, + {"wrong": "不醒人事", "correct": "不省人事", "source": "moe_dictionary_validation"}, + {"wrong": "一泄千里", "correct": "一瀉千里", "source": "moe_dictionary_validation"}, + {"wrong": "一諾千斤", "correct": "一諾千金", "source": "moe_dictionary_validation"}, + {"wrong": "老生長談", "correct": "老生常談", "source": "moe_dictionary_validation"}, + {"wrong": "相形見拙", "correct": "相形見絀", "source": "moe_dictionary_validation"} + ] +} diff --git a/bopomofo_core/frequency_lexicon.py b/bopomofo_core/frequency_lexicon.py index 5ebbc05..e2542b6 100644 --- a/bopomofo_core/frequency_lexicon.py +++ b/bopomofo_core/frequency_lexicon.py @@ -64,3 +64,16 @@ def candidates( ranked.items(), key=lambda row: (-row[1], row[0]) )[:limit] ] + + def contains(self, phrase: str) -> bool: + """Return whether the complete phrase exists beyond the result limit.""" + if len(phrase) < 2: + return False + bucket = self._buckets.get(str(len(phrase)), {}) + if not isinstance(bucket, dict): + return False + rows = bucket.get(phrase[0], []) + return isinstance(rows, list) and any( + isinstance(row, list) and len(row) == 2 and row[0] == phrase + for row in rows + ) diff --git a/bopomofo_core/libchewing_provider.py b/bopomofo_core/libchewing_provider.py index f61cb80..4dc2903 100644 --- a/bopomofo_core/libchewing_provider.py +++ b/bopomofo_core/libchewing_provider.py @@ -31,7 +31,7 @@ (("ㄗˋ", "ㄐㄧˇ"), "自己"), (("ㄗˋ", "ㄨㄛˇ"), "自我"), (("ㄨㄛˇ", "ㄅㄨˋ", "ㄧㄠˋ"), "我不要"), - (("ㄅㄨˋ", "ㄓ", "ㄉㄠˋ"), "不知道"), + (("ㄅㄨˋ", "ㄓˉ", "ㄉㄠˋ"), "不知道"), (("ㄅㄨˋ", "ㄧㄠˋ"), "不要"), (("ㄅㄨˋ", "ㄕˋ"), "不是"), (("ㄅㄨˋ", "ㄏㄨㄟˋ"), "不會"), @@ -39,14 +39,14 @@ (("ㄅㄨˋ", "ㄩㄥˋ"), "不用"), (("ㄅㄨˋ", "ㄒㄧㄤˇ"), "不想"), (("ㄗㄞˋ", "ㄐㄧㄢˋ"), "再見"), - (("ㄗㄞˋ", "ㄧ", "ㄘˋ"), "再一次"), + (("ㄗㄞˋ", "ㄧˉ", "ㄘˋ"), "再一次"), (("ㄗㄞˋ", "ㄧㄝˇ"), "再也"), (("ㄗㄞˋ", "ㄌㄞˊ"), "再來"), (("ㄒㄧㄢˋ", "ㄗㄞˋ"), "現在"), (("ㄙㄨㄛˇ", "ㄗㄞˋ"), "所在"), - (("ㄍㄣ", "ㄗㄞˋ"), "跟在"), + (("ㄍㄣˉ", "ㄗㄞˋ"), "跟在"), (("ㄓㄥˋ", "ㄗㄞˋ"), "正在"), - (("ㄗㄞˋ", "ㄐㄧㄚ"), "在家"), + (("ㄗㄞˋ", "ㄐㄧㄚˉ"), "在家"), (("ㄗㄞˋ", "ㄓㄜˋ"), "在這"), (("ㄗㄞˋ", "ㄋㄚˋ"), "在那"), (("ㄗㄞˋ", "ㄋㄚˇ"), "在哪"), @@ -57,12 +57,47 @@ def prioritize_common_character(reading: str, candidates: list[str]) -> list[str """Put a small set of overwhelmingly common characters first.""" # These defaults only control an isolated syllable. Contextual word rules # below may still turn ㄗˋ into 自 in words such as 自己 and 自我. - preferred = {"ㄅㄨˋ": "不", "ㄗˋ": "字"}.get(reading) + preferred = { + "ㄅㄨˋ": "不", + "ㄉㄜ˙": "的", + "ㄋㄚˋ": "那", + "ㄕˋ": "是", + "ㄗˋ": "字", + "ㄗㄨㄛˋ": "做", + }.get(reading) if preferred is None or preferred not in candidates: return candidates return [preferred] + [candidate for candidate in candidates if candidate != preferred] +def add_literal_bopomofo_candidate( + reading: str, candidates: list[str] +) -> list[str]: + """Expose the typed Zhuyin itself near the front of complete readings. + + Microsoft Bopomofo offers the literal phonetic spelling as a candidate. + First tone is conventionally unmarked in displayed Zhuyin. Keeping this at + position two preserves the normal Chinese default while keeping the + phonetic spelling visible in a compact Microsoft-style five-item page. + """ + if not reading: + return candidates + literal = reading[:-1] if reading.endswith("ˉ") else reading + if literal in candidates: + current_index = candidates.index(literal) + if current_index <= 1: + return candidates + # Some dictionary rows already contain the raw spelling deep in the + # tail. Presence alone is not enough: the UI contract requires it in + # the first four, so normalize an existing literal to position two. + without_literal = [ + candidate for candidate in candidates if candidate != literal + ] + return without_literal[:1] + [literal] + without_literal[1:] + insert_at = min(1, len(candidates)) + return candidates[:insert_at] + [literal] + candidates[insert_at:] + + def apply_common_usage_overrides(readings: list[str], phrase: str) -> str: """Correct unambiguous high-frequency phrases in an engine result.""" if len(readings) != len(phrase): @@ -116,14 +151,19 @@ def candidates(self, reading: str) -> list[str]: total = self.context.cand_TotalChoice() if total <= 0: - return [] + return add_literal_bopomofo_candidate(reading, []) self.context.cand_Enumerate() results: list[str] = [] limit = min(total, MAX_CANDIDATES) while self.context.cand_hasNext() and len(results) < limit: results.append(self.context.cand_String().decode("utf-8")) - results = TAIWAN_FREQUENCY.rank_characters(results) - return prioritize_common_character(reading, results) + # Dictionary order is reading-aware; global character frequency is + # not. Preserve the exact-reading default so a common alternate + # pronunciation cannot promote 員 over 運 for ㄩㄣˋ, then use Taiwan + # frequency to organize the remaining candidates. + results = TAIWAN_FREQUENCY.rank_characters(results, preserve_first=True) + results = prioritize_common_character(reading, results) + return add_literal_bopomofo_candidate(reading, results)[:MAX_CANDIDATES] def best_phrase(self, readings: list[str]) -> str: """Return the phrase dictionary's best text for complete readings.""" @@ -193,3 +233,10 @@ def frequent_phrase_candidates( candidate_columns, limit=MAX_CANDIDATES ) return list(dict.fromkeys(taiwan + expanded))[:MAX_CANDIDATES] + + @staticmethod + def is_frequent_phrase(phrase: str) -> bool: + """Recognize valid bundled words even outside the top-20 result tail.""" + return TAIWAN_FREQUENCY.contains_phrase(phrase) or FREQUENCY_LEXICON.contains( + phrase + ) diff --git a/bopomofo_core/phonetic_corrector.py b/bopomofo_core/phonetic_corrector.py new file mode 100644 index 0000000..1d05b9c --- /dev/null +++ b/bopomofo_core/phonetic_corrector.py @@ -0,0 +1,187 @@ +"""Reading-aware phrase correction for an uncommitted Bopomofo buffer.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Callable, List, Sequence + + +CandidateLookup = Callable[[str], List[str]] +PhraseLookup = Callable[[List[List[str]]], List[str]] +KnownPhraseLookup = Callable[[List[str]], List[str]] +PhraseValidator = Callable[[str], bool] + +_INITIAL_CONFUSIONS = { + "ㄓ": "ㄗ", + "ㄗ": "ㄓ", + "ㄔ": "ㄘ", + "ㄘ": "ㄔ", + "ㄕ": "ㄙ", + "ㄙ": "ㄕ", +} +_RIME_CONFUSIONS = {"ㄣ": "ㄥ", "ㄥ": "ㄣ"} +_TONES = frozenset("ˉˊˇˋ˙") + + +def reading_variants(reading: str) -> tuple[str, ...]: + """Return conservative Taiwanese Bopomofo confusions for one syllable. + + The original reading is always first. Variants change one phonetic slot at + a time, so the language model—not a list of wrong output characters—decides + whether a common word exists for the nearby pronunciation. + """ + if not reading: + return () + tone = reading[-1] if reading[-1] in _TONES else "" + body = reading[:-1] if tone else reading + variants = [reading] + + if body[:1] in _INITIAL_CONFUSIONS: + variants.append(_INITIAL_CONFUSIONS[body[0]] + body[1:] + tone) + if body[-1:] in _RIME_CONFUSIONS: + variants.append(body[:-1] + _RIME_CONFUSIONS[body[-1]] + tone) + return tuple(dict.fromkeys(variants)) + + +@dataclass(frozen=True) +class PhoneticCorrection: + start: int + original: str + corrected: str + used_fuzzy_reading: bool + + +class PhoneticCorrector: + """Re-decode words from readings and the bundled phrase indexes.""" + + def __init__(self, max_phrase_length: int = 12) -> None: + self.max_phrase_length = max(2, max_phrase_length) + + @staticmethod + def _single_character_candidates( + current: str, readings: Sequence[str], candidate_lookup: CandidateLookup + ) -> tuple[list[str], list[str]]: + exact = [current] + for candidate in candidate_lookup(readings[0]): + if len(candidate) == 1 and candidate not in exact: + exact.append(candidate) + + expanded = list(exact) + for variant in readings[1:]: + for candidate in candidate_lookup(variant): + if len(candidate) == 1 and candidate not in expanded: + expanded.append(candidate) + return exact, expanded + + def correct( + self, + readings: Sequence[str], + text: str, + protected: Sequence[bool], + candidate_lookup: CandidateLookup, + phrase_lookup: PhraseLookup, + known_phrase_lookup: KnownPhraseLookup | None = None, + phrase_validator: PhraseValidator | None = None, + allow_fuzzy: bool = True, + replacement_phrase_lookup: KnownPhraseLookup | None = None, + ) -> tuple[str, list[PhoneticCorrection]]: + if len(readings) != len(text) or len(protected) != len(text): + raise ValueError("readings, text, and protection mask must align") + if len(text) < 2: + return text, [] + + exact_columns: list[list[str]] = [] + expanded_columns: list[list[str]] = [] + for current, reading in zip(text, readings): + variants = reading_variants(reading) + if not variants: + exact_columns.append([current]) + expanded_columns.append([current]) + continue + exact, expanded = self._single_character_candidates( + current, variants, candidate_lookup + ) + exact_columns.append(exact) + expanded_columns.append(expanded if allow_fuzzy else exact) + + characters = list(text) + changes: list[PhoneticCorrection] = [] + start = 0 + while start < len(characters) - 1: + remaining = len(characters) - start + matched = False + for width in range(min(self.max_phrase_length, remaining), 1, -1): + end = start + width + if any(protected[start:end]): + continue + current = "".join(characters[start:end]) + + if phrase_validator is not None and phrase_validator(current): + start = end + matched = True + break + if known_phrase_lookup is not None and current in known_phrase_lookup( + list(readings[start:end]) + ): + start = end + matched = True + break + + span_readings = list(readings[start:end]) + exact_phrases = phrase_lookup(exact_columns[start:end]) + evidence_lookup = ( + replacement_phrase_lookup or known_phrase_lookup + ) + if evidence_lookup is not None: + known_exact = set(evidence_lookup(span_readings)) + exact_phrases = [ + phrase for phrase in exact_phrases if phrase in known_exact + ] + # A known phrase matching the user's current text is valid, + # even if another phrase has a higher raw corpus frequency. + if current in exact_phrases: + start = end + matched = True + break + if exact_phrases: + replacement = exact_phrases[0] + fuzzy = False + elif allow_fuzzy: + fuzzy_phrases = phrase_lookup(expanded_columns[start:end]) + if evidence_lookup is not None: + # Fuzzy correction is deliberately limited to one + # changed phonetic slot. Validate every proposed word + # against the exact phrase engine for that nearby + # reading, rather than trusting text-only corpus data. + known_fuzzy: set[str] = set() + for offset, reading in enumerate(span_readings): + for variant in reading_variants(reading)[1:]: + variant_readings = list(span_readings) + variant_readings[offset] = variant + known_fuzzy.update( + evidence_lookup(variant_readings) + ) + fuzzy_phrases = [ + phrase + for phrase in fuzzy_phrases + if phrase in known_fuzzy + ] + if not fuzzy_phrases: + continue + replacement = fuzzy_phrases[0] + fuzzy = True + else: + continue + + if len(replacement) != width or replacement == current: + continue + characters[start:end] = replacement + changes.append( + PhoneticCorrection(start, current, replacement, fuzzy) + ) + start = end + matched = True + break + if not matched: + start += 1 + return "".join(characters), changes diff --git a/bopomofo_core/session.py b/bopomofo_core/session.py index 0a06cfc..01a515a 100644 --- a/bopomofo_core/session.py +++ b/bopomofo_core/session.py @@ -35,6 +35,10 @@ def _is_complete(self, reading: str) -> bool: def _engine_candidates(self, reading: str) -> list[str]: return list(dict.fromkeys(self.provider.candidates(reading))) + def candidates_for_reading(self, reading: str) -> list[str]: + """Return ranked candidates without mutating the active editor.""" + return self._prioritize(reading, self._engine_candidates(reading)) + def best_phrase(self, readings: list[str]) -> str: converter = getattr(self.provider, "best_phrase", None) return converter(readings) if converter is not None else "" @@ -63,15 +67,50 @@ def frequent_phrase_candidates( : self.max_candidates ] + def validated_frequent_phrase_candidates( + self, readings: list[str], candidate_columns: list[list[str]] + ) -> list[str]: + """Return corpus phrases proven to match the complete readings. + + The expanded frequency index stores text and weights, but not a + phrase's pronunciation. A character-by-character membership check is + therefore only a broad search and can confuse alternate readings + (for example, matching 貝殼 to ㄅㄟˋ ㄑㄩㄝˋ). Require the phrase engine's + exact-reading evidence before a corpus match may affect live text. + """ + exact_phrases = set(self.phrase_candidates(readings)) + if not exact_phrases: + return [] + return [ + phrase + for phrase in self.frequent_phrase_candidates(candidate_columns) + if phrase in exact_phrases + ] + + def is_frequent_phrase(self, phrase: str) -> bool: + validator = getattr(self.provider, "is_frequent_phrase", None) + return bool(validator(phrase)) if validator is not None else False + + def _valid_pins(self, reading: str) -> list[str]: + return [ + candidate + for candidate in self.pins.phrases_for(reading) + if len(candidate) == 1 + ] + def _prioritize(self, reading: str, engine_candidates: list[str]) -> list[str]: - pinned = self.pins.phrases_for(reading) + # A per-reading pin normally represents exactly one output character. + # Ignore malformed/manual multi-character values so one damaged + # preference cannot break the one-syllable/one-segment alignment used + # by editing and Enter correction. + pinned = self._valid_pins(reading) # Explicit user choices always outrank every bundled language model. return list(dict.fromkeys(pinned + engine_candidates))[: self.max_candidates] def _validate(self, reading: str) -> bool: if not self._is_complete(reading): return True - return bool(self._engine_candidates(reading) or self.pins.phrases_for(reading)) + return bool(self._engine_candidates(reading) or self._valid_pins(reading)) def _refresh(self) -> None: if not self._is_complete(self.preedit): diff --git a/bopomofo_core/taiwan_frequency.py b/bopomofo_core/taiwan_frequency.py index ca7f505..6bc69ab 100644 --- a/bopomofo_core/taiwan_frequency.py +++ b/bopomofo_core/taiwan_frequency.py @@ -42,16 +42,32 @@ def __init__(self, path: str | Path = DEFAULT_INDEX) -> None: self._characters = {} self._buckets = {} - def rank_characters(self, candidates: list[str]) -> list[str]: - """Rank known characters by Taiwan frequency, preserving unknown order.""" + def rank_characters( + self, candidates: list[str], *, preserve_first: bool = False + ) -> list[str]: + """Rank by Taiwan frequency without losing reading-specific evidence. + + ``candidates`` normally comes from a pronunciation-aware dictionary. + Its first entry is therefore stronger evidence than a character's + global frequency, which cannot distinguish polyphonic readings such + as 員 (usually ㄩㄢˊ, but also present under ㄩㄣˋ). Callers may keep + that dictionary default while frequency-sorting the remaining tail. + """ + unique = list(dict.fromkeys(candidates)) + if not unique: + return [] positions = {candidate: index for index, candidate in enumerate(candidates)} - return sorted( - dict.fromkeys(candidates), + ranked = sorted( + unique, key=lambda candidate: ( -self._characters.get(candidate, 0), positions[candidate], ), ) + if not preserve_first: + return ranked + first = unique[0] + return [first] + [candidate for candidate in ranked if candidate != first] def phrase_candidates( self, candidate_columns: list[list[str]], limit: int = 20 @@ -84,3 +100,16 @@ def phrase_candidates( ranked.items(), key=lambda item: (-item[1], item[0]) )[:limit] ] + + def contains_phrase(self, phrase: str) -> bool: + """Return whether the official word table contains the whole phrase.""" + if len(phrase) < 2: + return False + bucket = self._buckets.get(str(len(phrase)), {}) + if not isinstance(bucket, dict): + return False + rows = bucket.get(phrase[0], []) + return isinstance(rows, list) and any( + isinstance(row, list) and len(row) == 2 and row[0] == phrase + for row in rows + ) diff --git a/build_pime_overlay.ps1 b/build_pime_overlay.ps1 index be116ee..0892c5f 100644 --- a/build_pime_overlay.ps1 +++ b/build_pime_overlay.ps1 @@ -44,4 +44,18 @@ foreach ($name in "word.dat", "tsi.dat", "swkb.dat", "symbols.dat") { Copy-Item -LiteralPath (Join-Path $runtimeChewing "data\$name") -Destination (Join-Path $chewingDest "data") -Force } +$nativeUiSource = Join-Path $projectRoot "native_ui" +$nativeUiDest = Join-Path $distRoot "native_ui" +foreach ($required in @( + (Join-Path $nativeUiSource "bin\x86\PIMETextService.dll"), + (Join-Path $nativeUiSource "bin\x64\PIMETextService.dll"), + (Join-Path $nativeUiSource "src\CandidateWindow.cpp"), + (Join-Path $nativeUiSource "LGPL-2.0.txt") +)) { + if (-not (Test-Path -LiteralPath $required)) { + throw "The native candidate UI payload is missing: $required" + } +} +Copy-Item -LiteralPath $nativeUiSource -Destination $nativeUiDest -Recurse -Force + Write-Output $distRoot diff --git a/build_release.ps1 b/build_release.ps1 index dc6b076..5a6f797 100644 --- a/build_release.ps1 +++ b/build_release.ps1 @@ -89,7 +89,7 @@ if (-not $MakensisPath -or -not (Test-Path -LiteralPath $MakensisPath)) { & $MakensisPath "/INPUTCHARSET" "UTF8" (Join-Path $projectRoot "installer\SmartPriorityBopomofo.nsi") if ($LASTEXITCODE -ne 0) { throw "NSIS build failed with exit code $LASTEXITCODE." } -$artifact = Join-Path $releaseRoot "Smart-Priority-Bopomofo-Setup-0.4.2.exe" +$artifact = Join-Path $releaseRoot "Smart-Priority-Bopomofo-Setup-0.6.0.exe" if (-not (Test-Path -LiteralPath $artifact)) { throw "The installer artifact was not created." } diff --git a/install.ps1 b/install.ps1 index 3457888..e26e335 100644 --- a/install.ps1 +++ b/install.ps1 @@ -1,20 +1,33 @@ param( - [switch]$Elevated + [switch]$Elevated, + [switch]$EnableUnsignedNativeUi, + [switch]$DisableUnsignedNativeUi ) $ErrorActionPreference = "Stop" $projectRoot = $PSScriptRoot $scriptPath = $MyInvocation.MyCommand.Path +if ($EnableUnsignedNativeUi -and $DisableUnsignedNativeUi) { + throw "EnableUnsignedNativeUi and DisableUnsignedNativeUi cannot be used together." +} + $identity = [Security.Principal.WindowsIdentity]::GetCurrent() $principal = [Security.Principal.WindowsPrincipal]::new($identity) if (-not $principal.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)) { - $process = Start-Process -FilePath "powershell.exe" -Verb RunAs -Wait -PassThru -ArgumentList @( + $elevatedArguments = @( "-NoProfile", "-ExecutionPolicy", "Bypass", "-File", ('"' + $scriptPath + '"'), "-Elevated" ) + if ($EnableUnsignedNativeUi) { + $elevatedArguments += "-EnableUnsignedNativeUi" + } + if ($DisableUnsignedNativeUi) { + $elevatedArguments += "-DisableUnsignedNativeUi" + } + $process = Start-Process -FilePath "powershell.exe" -Verb RunAs -Wait -PassThru -ArgumentList $elevatedArguments if ($process.ExitCode -ne 0) { throw "Installation failed with exit code $($process.ExitCode)." } @@ -47,7 +60,9 @@ try { New-Item -ItemType Directory -Path $resolvedStaging -Force | Out-Null Copy-Item -LiteralPath $overlayRoot -Destination (Join-Path $resolvedStaging "overlay") -Recurse -Force Copy-Item -LiteralPath $pimeInstaller -Destination $resolvedStaging -Force - & $formalInstaller -PayloadRoot $resolvedStaging + & $formalInstaller -PayloadRoot $resolvedStaging ` + -EnableUnsignedNativeUi:$EnableUnsignedNativeUi ` + -DisableUnsignedNativeUi:$DisableUnsignedNativeUi } finally { Set-Location -LiteralPath $env:TEMP diff --git a/installer/SmartPriorityBopomofo.nsi b/installer/SmartPriorityBopomofo.nsi index 8eb2966..4784464 100644 --- a/installer/SmartPriorityBopomofo.nsi +++ b/installer/SmartPriorityBopomofo.nsi @@ -3,7 +3,7 @@ RequestExecutionLevel admin SetCompressor /SOLID lzma !define PRODUCT_NAME "智慧優先注音" -!define PRODUCT_VERSION "0.4.2" +!define PRODUCT_VERSION "0.6.0" !define PRODUCT_PUBLISHER "Smart Priority Bopomofo contributors" !define PRODUCT_KEY "Software\Microsoft\Windows\CurrentVersion\Uninstall\SmartPriorityBopomofo" @@ -14,7 +14,7 @@ InstallDirRegKey HKLM "${PRODUCT_KEY}" "InstallLocation" ShowInstDetails show ShowUninstDetails show -VIProductVersion "0.4.2.0" +VIProductVersion "0.6.0.0" VIAddVersionKey /LANG=1028 "ProductName" "${PRODUCT_NAME}" VIAddVersionKey /LANG=1028 "CompanyName" "${PRODUCT_PUBLISHER}" VIAddVersionKey /LANG=1028 "FileDescription" "${PRODUCT_NAME} 安裝程式" @@ -44,6 +44,7 @@ Section "安裝智慧優先注音" SEC_MAIN SetShellVarContext current SetOutPath "$INSTDIR" File "install.ps1" + File "native_ui_preference.ps1" File "uninstall.ps1" File "..\release-staging\THIRD_PARTY_NOTICES.txt" File "..\release-staging\PIME-LICENSE.txt" diff --git a/installer/install.ps1 b/installer/install.ps1 index 7d99bc0..7fc297f 100644 --- a/installer/install.ps1 +++ b/installer/install.ps1 @@ -1,17 +1,133 @@ param( [Parameter(Mandatory = $true)] - [string]$PayloadRoot + [string]$PayloadRoot, + [switch]$EnableUnsignedNativeUi, + [switch]$DisableUnsignedNativeUi ) $ErrorActionPreference = "Stop" +. (Join-Path $PSScriptRoot "native_ui_preference.ps1") $tip = "0404:{35F67E9D-A54D-4177-9697-8B0AB71A9E04}{26EA5CF3-D515-40BE-9535-E7E98D5EE554}" $moduleName = "pinned_bopomofo" $logRoot = Join-Path $env:ProgramData "SmartPriorityBopomofo" $logPath = Join-Path $logRoot "install.log" +$nativeStateRoot = Join-Path $logRoot "native-state" +$nativePreferencePath = Join-Path $logRoot "native-ui-preference.json" $installedPimeThisRun = $false New-Item -ItemType Directory -Path $logRoot -Force | Out-Null Start-Transcript -LiteralPath $logPath -Force | Out-Null +function Restore-OriginalPimeTextService { + param( + [Parameter(Mandatory = $true)] + [string]$PimeRoot, + [Parameter(Mandatory = $true)] + [string]$StateRoot + ) + + $nativeMarker = Join-Path $StateRoot "native-ui.json" + $backupRoot = Join-Path $StateRoot "backup" + $pendingRoot = Join-Path $StateRoot "pending" + if (-not (Test-Path -LiteralPath $nativeMarker) -or + -not (Test-Path -LiteralPath $backupRoot)) { + return + } + + $nativeHashes = Get-Content -LiteralPath $nativeMarker -Raw -Encoding UTF8 | + ConvertFrom-Json + $nativeStateCanBeRemoved = $true + $restoreScheduled = $false + if (-not ("SmartPriorityNativeMethods" -as [type])) { + Add-Type @" +using System; +using System.Runtime.InteropServices; + +public static class SmartPriorityNativeMethods { + [DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool MoveFileEx( + string existingFile, + string newFile, + int flags + ); +} +"@ + } + + foreach ($architecture in @("x86", "x64")) { + $targetDll = Join-Path $PimeRoot "$architecture\PIMETextService.dll" + $backupDll = Join-Path $backupRoot "$architecture\PIMETextService.dll" + $pendingDll = Join-Path $pendingRoot "$architecture\PIMETextService.dll" + $expectedCustomHash = $nativeHashes.$architecture + if (-not (Test-Path -LiteralPath $targetDll) -or + -not (Test-Path -LiteralPath $backupDll)) { + $nativeStateCanBeRemoved = $false + continue + } + + $backupSignature = Get-AuthenticodeSignature -LiteralPath $backupDll + if ($backupSignature.Status -ne [System.Management.Automation.SignatureStatus]::Valid) { + throw "Refusing to restore an unsigned or invalid $architecture PIME backup." + } + $targetHash = (Get-FileHash -Algorithm SHA256 -LiteralPath $targetDll).Hash + $backupHash = (Get-FileHash -Algorithm SHA256 -LiteralPath $backupDll).Hash + if ($targetHash -eq $backupHash) { + continue + } + + if ($expectedCustomHash -and $targetHash -eq $expectedCustomHash) { + try { + Copy-Item -LiteralPath $backupDll -Destination $targetDll -Force + } + catch [System.IO.IOException], [System.UnauthorizedAccessException] { + # The TSF DLL is normally loaded by desktop and game clients. + # Keep a persistent signed source and replace it at reboot. + $restoreDll = Join-Path $StateRoot "restore\$architecture\PIMETextService.dll" + New-Item -ItemType Directory -Path (Split-Path -Parent $restoreDll) -Force | Out-Null + Copy-Item -LiteralPath $backupDll -Destination $restoreDll -Force + $scheduled = [SmartPriorityNativeMethods]::MoveFileEx( + $restoreDll, + $targetDll, + 0x4 -bor 0x1 + ) + if (-not $scheduled) { + $win32Error = [Runtime.InteropServices.Marshal]::GetLastWin32Error() + throw "Unable to schedule the $architecture signed PIME DLL restore (Win32 $win32Error)." + } + $nativeStateCanBeRemoved = $false + $restoreScheduled = $true + } + continue + } + + if ($nativeHashes.($architecture + "Pending") -and + (Test-Path -LiteralPath $pendingDll)) { + # Removing the source cancels an unsigned replacement that Windows + # has not performed yet. + Remove-Item -LiteralPath $pendingDll -Force + if ($targetHash -ne $backupHash) { + $nativeStateCanBeRemoved = $false + } + continue + } + + # Another product changed the shared DLL. Preserve every backup and + # avoid overwriting an unknown installation. + $nativeStateCanBeRemoved = $false + } + + if ($nativeStateCanBeRemoved -and (Test-Path -LiteralPath $StateRoot)) { + Remove-Item -LiteralPath $StateRoot -Recurse -Force + Write-Output "Restored the signed PIME text-service DLLs." + } + elseif ($restoreScheduled) { + Write-Output "The signed PIME text-service DLL restore will finish after Windows restarts." + } + else { + Write-Output "Preserved the native UI backup because the shared PIME DLL was changed by another product." + } +} + try { $identity = [Security.Principal.WindowsIdentity]::GetCurrent() $principal = [Security.Principal.WindowsPrincipal]::new($identity) @@ -129,6 +245,125 @@ try { if ($LASTEXITCODE -ne 0) { throw "The module compile check failed." } } + # A fresh installation keeps PIME's signed DLL. Once a user explicitly + # enables the custom UI, remember that choice across source, EXE, and AI- + # assisted updates so a routine Python-layer update cannot silently revert + # the appearance. The signed UI remains explicitly recoverable. + $useUnsignedNativeUi = Resolve-SmartPriorityNativeUiPreference ` + -PreferencePath $nativePreferencePath ` + -PimeRoot $resolvedRoot ` + -StateRoot $nativeStateRoot ` + -EnableUnsignedNativeUi:$EnableUnsignedNativeUi ` + -DisableUnsignedNativeUi:$DisableUnsignedNativeUi + if (-not $useUnsignedNativeUi) { + Restore-OriginalPimeTextService -PimeRoot $resolvedRoot -StateRoot $nativeStateRoot + } + + # Apply the optional native candidate window only when explicitly enabled + # and when this PIME installation contains no unrelated input methods. + # Preserve the original DLLs so a safe reinstall/uninstall can restore them. + $pythonMethods = Join-Path $resolvedRoot "python\input_methods" + $nodeMethods = Join-Path $resolvedRoot "node\input_methods" + $otherPythonModules = @( + Get-ChildItem -LiteralPath $pythonMethods -Directory -ErrorAction SilentlyContinue | + Where-Object { $_.Name -notin @($moduleName, "__pycache__") } + ) + $otherNodeModules = @( + Get-ChildItem -LiteralPath $nodeMethods -Directory -ErrorAction SilentlyContinue + ) + $nativeUiPayload = Join-Path $PayloadRoot "overlay\native_ui\bin" + $canInstallNativeUi = ( + $useUnsignedNativeUi -and + $otherPythonModules.Count -eq 0 -and $otherNodeModules.Count -eq 0 -and + (Test-Path -LiteralPath (Join-Path $nativeUiPayload "x86\PIMETextService.dll")) -and + (Test-Path -LiteralPath (Join-Path $nativeUiPayload "x64\PIMETextService.dll")) + ) + if ($useUnsignedNativeUi -and -not $canInstallNativeUi) { + throw "The remembered custom candidate UI cannot be installed because its payload is missing or unrelated PIME modules are present." + } + if ($canInstallNativeUi) { + $backupRoot = Join-Path $nativeStateRoot "backup" + $pendingRoot = Join-Path $nativeStateRoot "pending" + New-Item -ItemType Directory -Path $backupRoot -Force | Out-Null + $nativeHashes = @{} + $nativeUpdatePending = $false + if (-not ("SmartPriorityNativeMethods" -as [type])) { + Add-Type @" +using System; +using System.Runtime.InteropServices; + +public static class SmartPriorityNativeMethods { + [DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool MoveFileEx( + string existingFile, + string newFile, + int flags + ); +} +"@ + } + foreach ($architecture in @("x86", "x64")) { + $sourceDll = Join-Path $nativeUiPayload "$architecture\PIMETextService.dll" + $targetDll = Join-Path $resolvedRoot "$architecture\PIMETextService.dll" + $backupDll = Join-Path $backupRoot "$architecture\PIMETextService.dll" + $sourceHash = (Get-FileHash -Algorithm SHA256 -LiteralPath $sourceDll).Hash + New-Item -ItemType Directory -Path (Split-Path -Parent $backupDll) -Force | Out-Null + if ((Test-Path -LiteralPath $targetDll) -and -not (Test-Path -LiteralPath $backupDll)) { + Copy-Item -LiteralPath $targetDll -Destination $backupDll -Force + } + # Reinstalling the Python layer is common during development. Do + # not touch or re-queue a native DLL that is already current. + if ((Test-Path -LiteralPath $targetDll) -and + (Get-FileHash -Algorithm SHA256 -LiteralPath $targetDll).Hash -eq $sourceHash) { + $nativeHashes[$architecture] = $sourceHash + continue + } + try { + Copy-Item -LiteralPath $sourceDll -Destination $targetDll -Force + } + catch [System.IO.IOException], [System.UnauthorizedAccessException] { + # TSF loads this DLL into every text application, so Windows + # commonly keeps it locked. Stage a persistent copy and ask + # the kernel to replace it safely at the next reboot. + $pendingDll = Join-Path $pendingRoot "$architecture\PIMETextService.dll" + New-Item -ItemType Directory -Path (Split-Path -Parent $pendingDll) -Force | Out-Null + Copy-Item -LiteralPath $sourceDll -Destination $pendingDll -Force + $sessionManager = "HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager" + $pendingMoves = (Get-ItemProperty -LiteralPath $sessionManager ` + -Name PendingFileRenameOperations -ErrorAction SilentlyContinue).PendingFileRenameOperations + $pendingText = @($pendingMoves) -join "`n" + $alreadyScheduled = ( + $pendingText.IndexOf($pendingDll, [StringComparison]::OrdinalIgnoreCase) -ge 0 -and + $pendingText.IndexOf($targetDll, [StringComparison]::OrdinalIgnoreCase) -ge 0 + ) + if (-not $alreadyScheduled) { + $moveFileDelayUntilReboot = 0x4 + $moveFileReplaceExisting = 0x1 + $scheduled = [SmartPriorityNativeMethods]::MoveFileEx( + $pendingDll, + $targetDll, + $moveFileDelayUntilReboot -bor $moveFileReplaceExisting + ) + if (-not $scheduled) { + $win32Error = [Runtime.InteropServices.Marshal]::GetLastWin32Error() + throw "Unable to schedule the $architecture native UI update (Win32 $win32Error)." + } + } + $nativeUpdatePending = $true + $nativeHashes[$architecture + "Pending"] = $true + } + $nativeHashes[$architecture] = $sourceHash + } + $nativeHashes | ConvertTo-Json | Set-Content ` + -LiteralPath (Join-Path $nativeStateRoot "native-ui.json") -Encoding UTF8 + if ($nativeUpdatePending) { + Write-Output "The modern candidate UI will finish installing after the next Windows restart." + } + } + Save-SmartPriorityNativeUiPreference ` + -PreferencePath $nativePreferencePath -Enabled $useUnsignedNativeUi + $x86Dll = Join-Path $resolvedRoot "x86\PIMETextService.dll" $x64Dll = Join-Path $resolvedRoot "x64\PIMETextService.dll" if (Test-Path -LiteralPath $x86Dll) { diff --git a/installer/native_ui_preference.ps1 b/installer/native_ui_preference.ps1 new file mode 100644 index 0000000..9956e77 --- /dev/null +++ b/installer/native_ui_preference.ps1 @@ -0,0 +1,115 @@ +$ErrorActionPreference = "Stop" + +function Read-SmartPriorityNativeUiPreference { + param( + [Parameter(Mandatory = $true)] + [string]$PreferencePath + ) + + if (-not (Test-Path -LiteralPath $PreferencePath)) { + return $null + } + try { + $value = Get-Content -LiteralPath $PreferencePath -Raw -Encoding UTF8 | + ConvertFrom-Json + if ($null -eq $value.enabled) { + return $null + } + return [bool]$value.enabled + } + catch { + # A damaged preference must never make installation fail. Fall back to + # recognizing an already active custom UI, then to the signed default. + return $null + } +} + +function Test-SmartPriorityCustomNativeUiState { + param( + [Parameter(Mandatory = $true)] + [string]$PimeRoot, + [Parameter(Mandatory = $true)] + [string]$StateRoot + ) + + $markerPath = Join-Path $StateRoot "native-ui.json" + if (-not (Test-Path -LiteralPath $markerPath)) { + return $false + } + try { + $marker = Get-Content -LiteralPath $markerPath -Raw -Encoding UTF8 | + ConvertFrom-Json + foreach ($architecture in @("x86", "x64")) { + $expectedHash = $marker.$architecture + if (-not $expectedHash) { + return $false + } + $activeDll = Join-Path $PimeRoot "$architecture\PIMETextService.dll" + if ((Test-Path -LiteralPath $activeDll) -and + (Get-FileHash -Algorithm SHA256 -LiteralPath $activeDll).Hash -eq $expectedHash) { + continue + } + $pendingDll = Join-Path $StateRoot "pending\$architecture\PIMETextService.dll" + $pendingProperty = $architecture + "Pending" + if ($marker.$pendingProperty -and + (Test-Path -LiteralPath $pendingDll) -and + (Get-FileHash -Algorithm SHA256 -LiteralPath $pendingDll).Hash -eq $expectedHash) { + continue + } + return $false + } + return $true + } + catch { + return $false + } +} + +function Resolve-SmartPriorityNativeUiPreference { + param( + [Parameter(Mandatory = $true)] + [string]$PreferencePath, + [Parameter(Mandatory = $true)] + [string]$PimeRoot, + [Parameter(Mandatory = $true)] + [string]$StateRoot, + [switch]$EnableUnsignedNativeUi, + [switch]$DisableUnsignedNativeUi + ) + + if ($EnableUnsignedNativeUi -and $DisableUnsignedNativeUi) { + throw "EnableUnsignedNativeUi and DisableUnsignedNativeUi cannot be used together." + } + if ($EnableUnsignedNativeUi) { + return $true + } + if ($DisableUnsignedNativeUi) { + return $false + } + + $stored = Read-SmartPriorityNativeUiPreference -PreferencePath $PreferencePath + if ($null -ne $stored) { + return [bool]$stored + } + + # Migrate installations created before the preference file existed. An + # active or correctly staged custom DLL proves that the user opted in. + return Test-SmartPriorityCustomNativeUiState ` + -PimeRoot $PimeRoot -StateRoot $StateRoot +} + +function Save-SmartPriorityNativeUiPreference { + param( + [Parameter(Mandatory = $true)] + [string]$PreferencePath, + [Parameter(Mandatory = $true)] + [bool]$Enabled + ) + + $parent = Split-Path -Parent $PreferencePath + New-Item -ItemType Directory -Path $parent -Force | Out-Null + @{ + enabled = $Enabled + version = 1 + } | ConvertTo-Json | Set-Content -LiteralPath $PreferencePath -Encoding UTF8 +} diff --git a/installer/uninstall.ps1 b/installer/uninstall.ps1 index 6eed3d0..e7cefd0 100644 --- a/installer/uninstall.ps1 +++ b/installer/uninstall.ps1 @@ -5,6 +5,7 @@ $textServiceGuid = "{35F67E9D-A54D-4177-9697-8B0AB71A9E04}" $moduleName = "pinned_bopomofo" $logRoot = Join-Path $env:ProgramData "SmartPriorityBopomofo" $logPath = Join-Path $logRoot "uninstall.log" +$nativeStateRoot = Join-Path $logRoot "native-state" New-Item -ItemType Directory -Path $logRoot -Force | Out-Null Start-Transcript -LiteralPath $logPath -Force | Out-Null @@ -45,6 +46,92 @@ try { if (Test-Path -LiteralPath $targetModule) { Remove-Item -LiteralPath $targetModule -Recurse -Force } + + $nativeMarker = Join-Path $nativeStateRoot "native-ui.json" + $backupRoot = Join-Path $nativeStateRoot "backup" + $pendingRoot = Join-Path $nativeStateRoot "pending" + if ((Test-Path -LiteralPath $nativeMarker) -and (Test-Path -LiteralPath $backupRoot)) { + $nativeHashes = Get-Content -LiteralPath $nativeMarker -Raw -Encoding UTF8 | + ConvertFrom-Json + $nativeStateCanBeRemoved = $true + if (-not ("SmartPriorityNativeMethods" -as [type])) { + Add-Type @" +using System; +using System.Runtime.InteropServices; + +public static class SmartPriorityNativeMethods { + [DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool MoveFileEx( + string existingFile, + string newFile, + int flags + ); +} +"@ + } + foreach ($architecture in @("x86", "x64")) { + $targetDll = Join-Path $resolvedRoot "$architecture\PIMETextService.dll" + $backupDll = Join-Path $backupRoot "$architecture\PIMETextService.dll" + $pendingDll = Join-Path $pendingRoot "$architecture\PIMETextService.dll" + $expectedHash = $nativeHashes.$architecture + if ((Test-Path -LiteralPath $targetDll) -and + (Test-Path -LiteralPath $backupDll) -and + (Get-FileHash -Algorithm SHA256 -LiteralPath $targetDll).Hash -eq + (Get-FileHash -Algorithm SHA256 -LiteralPath $backupDll).Hash) { + # A prior safe reinstall already restored PIME's signed + # DLL. There is nothing left to overwrite. + continue + } + elseif ((Test-Path -LiteralPath $targetDll) -and + (Test-Path -LiteralPath $backupDll) -and + $expectedHash -and + (Get-FileHash -Algorithm SHA256 -LiteralPath $targetDll).Hash -eq $expectedHash) { + try { + Copy-Item -LiteralPath $backupDll -Destination $targetDll -Force + } + catch [System.IO.IOException], [System.UnauthorizedAccessException] { + $restoreDll = Join-Path $nativeStateRoot "restore\$architecture\PIMETextService.dll" + New-Item -ItemType Directory -Path (Split-Path -Parent $restoreDll) -Force | Out-Null + Copy-Item -LiteralPath $backupDll -Destination $restoreDll -Force + $scheduled = [SmartPriorityNativeMethods]::MoveFileEx( + $restoreDll, + $targetDll, + 0x4 -bor 0x1 + ) + if (-not $scheduled) { + $win32Error = [Runtime.InteropServices.Marshal]::GetLastWin32Error() + throw "Unable to schedule the $architecture native UI restore (Win32 $win32Error)." + } + $nativeStateCanBeRemoved = $false + } + } + elseif ($nativeHashes.($architecture + "Pending") -and + (Test-Path -LiteralPath $pendingDll)) { + # Removing the persistent source cancels a replacement + # that Windows has not performed yet. + Remove-Item -LiteralPath $pendingDll -Force + } + else { + # A third party changed the shared DLL. Preserve our + # backup rather than overwriting or deleting evidence. + $nativeStateCanBeRemoved = $false + } + } + foreach ($architecture in @("x86", "x64")) { + $dll = Join-Path $resolvedRoot "$architecture\PIMETextService.dll" + if (-not (Test-Path -LiteralPath $dll)) { continue } + $regsvr = if ($architecture -eq "x86") { + Join-Path $env:WINDIR "SysWOW64\regsvr32.exe" + } else { + Join-Path $env:WINDIR "System32\regsvr32.exe" + } + & $regsvr /s $dll + } + if ($nativeStateCanBeRemoved -and (Test-Path -LiteralPath $nativeStateRoot)) { + Remove-Item -LiteralPath $nativeStateRoot -Recurse -Force + } + } } $languages = Get-WinUserLanguageList @@ -83,6 +170,11 @@ try { Remove-Item -LiteralPath $shortcutRoot -Force } + $nativePreferencePath = Join-Path $logRoot "native-ui-preference.json" + if (Test-Path -LiteralPath $nativePreferencePath) { + Remove-Item -LiteralPath $nativePreferencePath -Force + } + # Remove only this language profile. PIME and every other PIME input # method are shared and must remain registered. $profileKeys = @( diff --git a/native_ui/LGPL-2.0.txt b/native_ui/LGPL-2.0.txt new file mode 100644 index 0000000..f166cc5 --- /dev/null +++ b/native_ui/LGPL-2.0.txt @@ -0,0 +1,502 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Libraries + + If you develop a new library, and you want it to be of the greatest +possible use to the public, we recommend making it free software that +everyone can redistribute and change. You can do so by permitting +redistribution under these terms (or, alternatively, under the terms of the +ordinary General Public License). + + To apply these terms, attach the following notices to the library. It is +safest to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least the +"copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +Also add information on how to contact you by electronic and paper mail. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the library, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the + library `Frob' (a library for tweaking knobs) written by James Random Hacker. + + , 1 April 1990 + Ty Coon, President of Vice + +That's all there is to it! \ No newline at end of file diff --git a/native_ui/README.md b/native_ui/README.md new file mode 100644 index 0000000..4784896 --- /dev/null +++ b/native_ui/README.md @@ -0,0 +1,27 @@ +# Smart Priority Bopomofo native candidate UI + +This directory contains the source and reproducible build script for the +Japanese-inspired candidate window bundled with Smart Priority Bopomofo. + +- Upstream PIME commit: `26fcf6ac8874e76b8f75f6826811b03bfdfc2e89` + (`v1.3.0-stable`, matching the bundled PIME runtime) +- Upstream libIME2 commit: `8ad3c9b433d930ce5614c483461dfa78cedb5efd` +- Upstream: https://github.com/EasyIME/PIME +- License: LGPL-2.0-or-later; see `LGPL-2.0.txt`. + +`src/CandidateWindow.cpp` is the complete modified source. The UI uses a warm +paper background, indigo accents, rounded clipping, a thin neutral border, +DPI-scaled spacing, and double-buffered painting. `src/CMakeLists.txt` excludes +PIMELauncher because the product keeps the signed launcher's existing binary +and rebuilds only `PIMETextService.dll`. + +Run `build_native_ui.ps1` on Windows with Visual Studio 2022 C++ tools to +reproduce `bin/x86/PIMETextService.dll` and `bin/x64/PIMETextService.dll`. + +These locally built DLLs are not code-signed and are therefore not installed +by default. The source installer accepts the explicit +`-EnableUnsignedNativeUi` switch when no other PIME input-method modules are +present. It backs up the original shared DLLs and remembers the opt-in across +normal and EXE updates. `-DisableUnsignedNativeUi` or uninstall explicitly +restores the signed PIME binaries when game or anti-cheat compatibility is +more important than the custom appearance. diff --git a/native_ui/bin/x64/PIMETextService.dll b/native_ui/bin/x64/PIMETextService.dll new file mode 100644 index 0000000..b33ce81 Binary files /dev/null and b/native_ui/bin/x64/PIMETextService.dll differ diff --git a/native_ui/bin/x86/PIMETextService.dll b/native_ui/bin/x86/PIMETextService.dll new file mode 100644 index 0000000..6eb6354 Binary files /dev/null and b/native_ui/bin/x86/PIMETextService.dll differ diff --git a/native_ui/build_native_ui.ps1 b/native_ui/build_native_ui.ps1 new file mode 100644 index 0000000..03c2030 --- /dev/null +++ b/native_ui/build_native_ui.ps1 @@ -0,0 +1,54 @@ +param() + +$ErrorActionPreference = "Stop" +$nativeRoot = $PSScriptRoot +$pimeCommit = "26fcf6ac8874e76b8f75f6826811b03bfdfc2e89" +$temporaryRoot = Join-Path $env:TEMP ("SmartPriorityPimeUi-" + [Guid]::NewGuid().ToString("N")) +$cmake = "C:\Program Files\Microsoft Visual Studio\2022\Community\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin\cmake.exe" + +if (-not (Test-Path -LiteralPath $cmake)) { + throw "Visual Studio 2022 CMake was not found: $cmake" +} + +$resolvedTemp = [IO.Path]::GetFullPath($env:TEMP).TrimEnd("\") +$resolvedBuild = [IO.Path]::GetFullPath($temporaryRoot).TrimEnd("\") +if (-not $resolvedBuild.StartsWith($resolvedTemp + "\", [StringComparison]::OrdinalIgnoreCase)) { + throw "Refusing to build outside the temporary directory." +} + +try { + & git clone --quiet https://github.com/EasyIME/PIME.git $resolvedBuild + & git -C $resolvedBuild checkout --quiet $pimeCommit + & git -C $resolvedBuild submodule update --init --depth 1 libIME2 jsoncpp + if ($LASTEXITCODE -ne 0) { throw "Unable to fetch the pinned PIME source." } + + Copy-Item -LiteralPath (Join-Path $nativeRoot "src\CandidateWindow.cpp") ` + -Destination (Join-Path $resolvedBuild "libIME2\src\CandidateWindow.cpp") -Force + Copy-Item -LiteralPath (Join-Path $nativeRoot "src\CMakeLists.txt") ` + -Destination (Join-Path $resolvedBuild "CMakeLists.txt") -Force + + foreach ($architecture in @( + @{ Name = "x86"; Platform = "Win32" }, + @{ Name = "x64"; Platform = "x64" } + )) { + $buildDir = Join-Path $resolvedBuild ("build-" + $architecture.Name) + & $cmake -S $resolvedBuild -B $buildDir -G "Visual Studio 17 2022" ` + -A $architecture.Platform "-DCMAKE_POLICY_VERSION_MINIMUM=3.5" + if ($LASTEXITCODE -ne 0) { throw "CMake configure failed for $($architecture.Name)." } + & $cmake --build $buildDir --config Release --target PIMETextService + if ($LASTEXITCODE -ne 0) { throw "Native UI build failed for $($architecture.Name)." } + $dll = Join-Path $buildDir "PIMETextService\Release\PIMETextService.dll" + $destination = Join-Path $nativeRoot ("bin\" + $architecture.Name) + New-Item -ItemType Directory -Path $destination -Force | Out-Null + Copy-Item -LiteralPath $dll -Destination $destination -Force + } + + Get-FileHash -Algorithm SHA256 -LiteralPath ` + (Join-Path $nativeRoot "bin\x86\PIMETextService.dll"), ` + (Join-Path $nativeRoot "bin\x64\PIMETextService.dll") +} +finally { + if (Test-Path -LiteralPath $resolvedBuild) { + Remove-Item -LiteralPath $resolvedBuild -Recurse -Force + } +} diff --git a/native_ui/src/CMakeLists.txt b/native_ui/src/CMakeLists.txt new file mode 100644 index 0000000..137d453 --- /dev/null +++ b/native_ui/src/CMakeLists.txt @@ -0,0 +1,43 @@ +cmake_minimum_required(VERSION 2.8.11) + +# override default c/c++ flags, add /MT to link VC++ runtime statically. +# NOTE: these two lines should go before the project() command according to CMake FAQ: +# https://cmake.org/Wiki/CMake_FAQ#Make_Override_Files +set(CMAKE_USER_MAKE_RULES_OVERRIDE ${CMAKE_CURRENT_SOURCE_DIR}/cmake/c_flag_overrides.cmake) +set(CMAKE_USER_MAKE_RULES_OVERRIDE_CXX ${CMAKE_CURRENT_SOURCE_DIR}/cmake/cxx_flag_overrides.cmake) + +project(PIME) + +# load current number from file +file(READ "version.txt" PIME_VERSION) +# we use semantic versioning: ..[-] +# convert the version string into a list (separated by ;) +string(REGEX MATCHALL "[0-9]+" PIME_VERSION_PARTS "${PIME_VERSION}") +list(GET PIME_VERSION_PARTS 0 PIME_VERSION_MAJOR) +list(GET PIME_VERSION_PARTS 1 PIME_VERSION_MINOR) +list(GET PIME_VERSION_PARTS 2 PIME_VERSION_PATCH) + +# http://www.utf8everywhere.org/ +add_definitions( + /D_UNICODE=1 /DUNICODE=1 # do Unicode build + /D_CRT_SECURE_NO_WARNINGS # disable warnings about old libc functions + /GR- # turn off C++ RTTI + -DWIN32_LEAN_AND_MEAN + -D_WIN32_WINNT=0x0600 +) + +add_subdirectory(${PROJECT_SOURCE_DIR}/libIME2) + +# PIME 1.3.0 links the TSF client against jsoncpp. Keep the library while +# excluding the unrelated launcher executable from this UI-only rebuild. +set(JSONCPP_WITH_TESTS OFF CACHE BOOL "") +set(JSONCPP_WITH_POST_BUILD_UNITTEST OFF CACHE BOOL "") +set(JSONCPP_WITH_PKGCONFIG_SUPPORT OFF CACHE BOOL "") +add_subdirectory(${PROJECT_SOURCE_DIR}/jsoncpp) + +add_subdirectory(${PROJECT_SOURCE_DIR}/PIMETextService) + +# The Smart Priority UI build only needs the TSF DLL. PIMELauncher is retained +# from the signed PIME installer and is intentionally not rebuilt here. + +enable_testing() diff --git a/native_ui/src/CandidateWindow.cpp b/native_ui/src/CandidateWindow.cpp new file mode 100644 index 0000000..fca00e9 --- /dev/null +++ b/native_ui/src/CandidateWindow.cpp @@ -0,0 +1,439 @@ +// +// Copyright (C) 2013 - 2020 Hong Jen Yee (PCMan) +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Library General Public +// License as published by the Free Software Foundation; either +// version 2 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Library General Public License for more details. +// +// You should have received a copy of the GNU Library General Public +// License along with this library; if not, write to the +// Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, +// Boston, MA 02110-1301, USA. +// + +#include "CandidateWindow.h" +#include "DrawUtils.h" +#include "TextService.h" +#include "EditSession.h" + +#include +#include + +#include +#include + +using namespace std; + +namespace Ime { + +namespace { + +const COLORREF kBackground = RGB(250, 249, 247); +const COLORREF kBorder = RGB(218, 216, 211); +const COLORREF kText = RGB(38, 37, 35); +const COLORREF kLabel = RGB(111, 105, 96); +const COLORREF kSelectedBackground = RGB(231, 235, 248); +const COLORREF kSelectedLabel = RGB(72, 83, 145); + +int scaledPixel(int value) { + HDC screen = ::GetDC(NULL); + int dpi = screen ? ::GetDeviceCaps(screen, LOGPIXELSX) : 96; + if(screen) + ::ReleaseDC(NULL, screen); + return ::MulDiv(value, dpi, 96); +} + +void candidateGrid(int itemCount, int maxColumns, int& rows, int& columns) { + if(itemCount <= 0) { + rows = 0; + columns = 0; + return; + } + + // Smart Priority Bopomofo uses two columns of five. Fill the left column + // (1-5) before the right column (6-0), matching Microsoft Bopomofo instead + // of the original row-major 1/2, 3/4 layout. + if(maxColumns == 2 && itemCount <= 10) { + rows = min(5, itemCount); + columns = (itemCount + rows - 1) / rows; + return; + } + + columns = min(maxColumns, itemCount); + rows = (itemCount + columns - 1) / columns; +} + +void fillRoundRect(HDC hdc, const RECT& rect, int radius, COLORREF color) { + HBRUSH brush = ::CreateSolidBrush(color); + HPEN pen = ::CreatePen(PS_NULL, 0, color); + HGDIOBJ oldBrush = ::SelectObject(hdc, brush); + HGDIOBJ oldPen = ::SelectObject(hdc, pen); + ::RoundRect(hdc, rect.left, rect.top, rect.right, rect.bottom, radius, radius); + ::SelectObject(hdc, oldPen); + ::SelectObject(hdc, oldBrush); + ::DeleteObject(pen); + ::DeleteObject(brush); +} + +} // namespace + +CandidateWindow::CandidateWindow(TextService* service, EditSession* session): + ImeWindow(service), + shown_(false), + candPerRow_(1), + textWidth_(0), + itemHeight_(0), + currentSel_(0), + hasResult_(false), + useCursor_(true), + selKeyWidth_(0) { + + // A restrained Japanese-inspired surface: generous whitespace, a warm + // paper background, subtle indigo selection, and no legacy 3D chrome. + margin_ = scaledPixel(10); + rowSpacing_ = scaledPixel(4); + colSpacing_ = scaledPixel(8); + + HWND parent = service->compositionWindow(session); + create(parent, WS_POPUP|WS_CLIPCHILDREN, + WS_EX_TOOLWINDOW|WS_EX_TOPMOST|WS_EX_NOACTIVATE); +} + +CandidateWindow::~CandidateWindow(void) { +} + +// ITfUIElement +STDMETHODIMP CandidateWindow::GetDescription(BSTR *pbstrDescription) { + if (!pbstrDescription) + return E_INVALIDARG; + *pbstrDescription = SysAllocString(L"Candidate window~"); + return S_OK; +} + +// {BD7CCC94-57CD-41D3-A789-AF47890CEB29} +STDMETHODIMP CandidateWindow::GetGUID(GUID *pguid) { + if (!pguid) + return E_INVALIDARG; + *pguid = { 0xbd7ccc94, 0x57cd, 0x41d3, { 0xa7, 0x89, 0xaf, 0x47, 0x89, 0xc, 0xeb, 0x29 } }; + return S_OK; +} + +STDMETHODIMP CandidateWindow::Show(BOOL bShow) { + shown_ = bShow; + if (shown_) + show(); + else + hide(); + return S_OK; +} + +STDMETHODIMP CandidateWindow::IsShown(BOOL *pbShow) { + if (!pbShow) + return E_INVALIDARG; + *pbShow = shown_; + return S_OK; +} + +// ITfCandidateListUIElement +STDMETHODIMP CandidateWindow::GetUpdatedFlags(DWORD *pdwFlags) { + if (!pdwFlags) + return E_INVALIDARG; + /// XXX update all!!! + *pdwFlags = TF_CLUIE_DOCUMENTMGR | TF_CLUIE_COUNT | TF_CLUIE_SELECTION | TF_CLUIE_STRING | TF_CLUIE_PAGEINDEX | TF_CLUIE_CURRENTPAGE; + return S_OK; +} + +STDMETHODIMP CandidateWindow::GetDocumentMgr(ITfDocumentMgr **ppdim) { + if (!textService_) + return E_FAIL; + return textService_->currentContext()->GetDocumentMgr(ppdim); +} + +STDMETHODIMP CandidateWindow::GetCount(UINT *puCount) { + if (!puCount) + return E_INVALIDARG; + *puCount = std::min(10, items_.size()); + return S_OK; +} + +STDMETHODIMP CandidateWindow::GetSelection(UINT *puIndex) { + assert(currentSel_ >= 0); + if (!puIndex) + return E_INVALIDARG; + *puIndex = static_cast(currentSel_); + return S_OK; +} + +STDMETHODIMP CandidateWindow::GetString(UINT uIndex, BSTR *pbstr) { + if (!pbstr) + return E_INVALIDARG; + if (uIndex >= items_.size()) + return E_INVALIDARG; + *pbstr = SysAllocString(items_[uIndex].c_str()); + return S_OK; +} + +STDMETHODIMP CandidateWindow::GetPageIndex(UINT *puIndex, UINT uSize, UINT *puPageCnt) { + /// XXX Always return the same single page index. + if (!puPageCnt) + return E_INVALIDARG; + *puPageCnt = 1; + if (puIndex) { + if (uSize < *puPageCnt) { + return E_INVALIDARG; + } + puIndex[0] = 0; + } + return S_OK; +} + +STDMETHODIMP CandidateWindow::SetPageIndex(UINT *puIndex, UINT uPageCnt) { + /// XXX Do not let app set page indices. + if (!puIndex) + return E_INVALIDARG; + return S_OK; +} + +STDMETHODIMP CandidateWindow::GetCurrentPage(UINT *puPage) { + if (!puPage) + return E_INVALIDARG; + *puPage = 0; + return S_OK; +} + +LRESULT CandidateWindow::wndProc(UINT msg, WPARAM wp , LPARAM lp) { + switch (msg) { + case WM_PAINT: + onPaint(wp, lp); + break; + case WM_ERASEBKGND: + return TRUE; + break; + case WM_LBUTTONDOWN: + onLButtonDown(wp, lp); + break; + case WM_MOUSEMOVE: + onMouseMove(wp, lp); + break; + case WM_LBUTTONUP: + onLButtonUp(wp, lp); + break; + case WM_MOUSEACTIVATE: + return MA_NOACTIVATE; + default: + return Window::wndProc(msg, wp, lp); + } + return 0; +} + +void CandidateWindow::onPaint(WPARAM wp, LPARAM lp) { + PAINTSTRUCT ps; + BeginPaint(hwnd_, &ps); + RECT rc; + GetClientRect(hwnd_,&rc); + + // Double-buffer the small popup to keep rapid candidate changes quiet and + // flicker-free. + HDC hDC = ::CreateCompatibleDC(ps.hdc); + HBITMAP bitmap = ::CreateCompatibleBitmap( + ps.hdc, rc.right - rc.left, rc.bottom - rc.top); + HGDIOBJ oldBitmap = ::SelectObject(hDC, bitmap); + HFONT oldFont = (HFONT)::SelectObject(hDC, font_); + ::SetBkMode(hDC, TRANSPARENT); + + fillRoundRect(hDC, rc, scaledPixel(12), kBackground); + HPEN borderPen = ::CreatePen(PS_SOLID, scaledPixel(1), kBorder); + HGDIOBJ oldPen = ::SelectObject(hDC, borderPen); + HGDIOBJ oldBrush = ::SelectObject(hDC, ::GetStockObject(NULL_BRUSH)); + ::RoundRect(hDC, rc.left, rc.top, rc.right - 1, rc.bottom - 1, + scaledPixel(12), scaledPixel(12)); + ::SelectObject(hDC, oldBrush); + ::SelectObject(hDC, oldPen); + ::DeleteObject(borderPen); + + // paint items + for(int i = 0, n = items_.size(); i < n; ++i) { + RECT item; + itemRect(i, item); + paintItem(hDC, i, item.left, item.top); + } + ::BitBlt(ps.hdc, 0, 0, rc.right, rc.bottom, hDC, 0, 0, SRCCOPY); + ::SelectObject(hDC, oldFont); + ::SelectObject(hDC, oldBitmap); + ::DeleteObject(bitmap); + ::DeleteDC(hDC); + EndPaint(hwnd_, &ps); +} + +void CandidateWindow::recalculateSize() { + if(items_.empty()) { + resize(margin_ * 2, margin_ * 2); + } + + HDC hDC = ::GetWindowDC(hwnd()); + int height = 0; + int width = 0; + selKeyWidth_ = 0; + textWidth_ = 0; + itemHeight_ = 0; + + HGDIOBJ oldFont = ::SelectObject(hDC, font_); + vector::const_iterator it; + for(int i = 0, n = items_.size(); i < n; ++i) { + SIZE selKeySize; + int lineHeight = 0; + // the selection key string + wchar_t selKey[] = L"?"; + selKey[0] = selKeys_[i]; + ::GetTextExtentPoint32W(hDC, selKey, 1, &selKeySize); + if(selKeySize.cx > selKeyWidth_) + selKeyWidth_ = selKeySize.cx; + + // the candidate string + SIZE candidateSize; + wstring& item = items_.at(i); + ::GetTextExtentPoint32W(hDC, item.c_str(), item.length(), &candidateSize); + if(candidateSize.cx > textWidth_) + textWidth_ = candidateSize.cx; + int itemHeight = max(candidateSize.cy, selKeySize.cy); + if(itemHeight > itemHeight_) + itemHeight_ = itemHeight; + } + itemHeight_ += scaledPixel(12); + selKeyWidth_ += scaledPixel(16); + textWidth_ += scaledPixel(16); + ::SelectObject(hDC, oldFont); + ::ReleaseDC(hwnd(), hDC); + + int rowCount, columnCount; + candidateGrid(static_cast(items_.size()), candPerRow_, rowCount, columnCount); + width = columnCount * (selKeyWidth_ + textWidth_); + width += colSpacing_ * max(0, columnCount - 1); + width += margin_ * 2; + height = itemHeight_ * rowCount + rowSpacing_ * max(0, rowCount - 1); + height += margin_ * 2; + resize(width, height); + HRGN region = ::CreateRoundRectRgn( + 0, 0, width + 1, height + 1, scaledPixel(12), scaledPixel(12)); + if(!::SetWindowRgn(hwnd(), region, TRUE)) + ::DeleteObject(region); +} + +void CandidateWindow::setCandPerRow(int n) { + if(n != candPerRow_) { + candPerRow_ = n; + recalculateSize(); + } +} + +bool CandidateWindow::filterKeyEvent(KeyEvent& keyEvent) { + // select item with arrow keys + int oldSel = currentSel_; + int itemCount = static_cast(items_.size()); + int rowCount, columnCount; + candidateGrid(itemCount, candPerRow_, rowCount, columnCount); + int row = rowCount > 0 ? currentSel_ % rowCount : 0; + int col = rowCount > 0 ? currentSel_ / rowCount : 0; + switch(keyEvent.keyCode()) { + case VK_UP: + if(row > 0) + --currentSel_; + break; + case VK_DOWN: + if(row + 1 < rowCount && currentSel_ + 1 < itemCount) + ++currentSel_; + break; + case VK_LEFT: + if(col > 0) + currentSel_ -= rowCount; + break; + case VK_RIGHT: + if(col + 1 < columnCount && currentSel_ + rowCount < itemCount) + currentSel_ += rowCount; + break; + case VK_RETURN: + hasResult_ = true; + return true; + default: + return false; + } + // if currently selected item is changed, redraw + if(currentSel_ != oldSel) { + // repaint the old and new items + RECT rect; + itemRect(oldSel, rect); + ::InvalidateRect(hwnd_, &rect, TRUE); + itemRect(currentSel_, rect); + ::InvalidateRect(hwnd_, &rect, TRUE); + return true; + } + return false; +} + +void CandidateWindow::setCurrentSel(int sel) { + if(sel >= items_.size()) + sel = 0; + if (currentSel_ != sel) { + currentSel_ = sel; + if (isVisible()) + ::InvalidateRect(hwnd_, NULL, TRUE); + } +} + +void CandidateWindow::clear() { + items_.clear(); + selKeys_.clear(); + currentSel_ = 0; + hasResult_ = false; +} + +void CandidateWindow::setUseCursor(bool use) { + useCursor_ = use; + if(isVisible()) + ::InvalidateRect(hwnd_, NULL, TRUE); +} + +void CandidateWindow::paintItem(HDC hDC, int i, int x, int y) { + RECT cell = { + x, + y, + x + selKeyWidth_ + textWidth_, + y + itemHeight_ + }; + if(useCursor_ && i == currentSel_) { + RECT selected = cell; + ::InflateRect(&selected, -scaledPixel(2), -scaledPixel(1)); + fillRoundRect(hDC, selected, scaledPixel(8), kSelectedBackground); + } + + wchar_t selKey[] = L"?"; + selKey[0] = selKeys_[i]; + int textY = y + scaledPixel(6); + ::SetTextColor(hDC, i == currentSel_ ? kSelectedLabel : kLabel); + ::TextOutW(hDC, x + scaledPixel(8), textY, selKey, 1); + + wstring& item = items_.at(i); + ::SetTextColor(hDC, kText); + ::TextOutW(hDC, x + selKeyWidth_, textY, item.c_str(), item.length()); +} + +void CandidateWindow::itemRect(int i, RECT& rect) { + int row, col; + int rowCount, columnCount; + candidateGrid(static_cast(items_.size()), candPerRow_, rowCount, columnCount); + row = rowCount > 0 ? i % rowCount : 0; + col = rowCount > 0 ? i / rowCount : 0; + rect.left = margin_ + col * (selKeyWidth_ + textWidth_ + colSpacing_); + rect.top = margin_ + row * (itemHeight_ + rowSpacing_); + rect.right = rect.left + (selKeyWidth_ + textWidth_); + rect.bottom = rect.top + itemHeight_; +} + + +} // namespace Ime diff --git a/pime_module/ime.json b/pime_module/ime.json index 942dc2d..d604735 100644 --- a/pime_module/ime.json +++ b/pime_module/ime.json @@ -1,6 +1,6 @@ { "name": "智慧優先注音", - "version": "0.4.2", + "version": "0.6.0", "guid": "{26EA5CF3-D515-40BE-9535-E7E98D5EE554}", "locale": "zh-Hant-TW", "fallbackLocale": "zh-TW", diff --git a/pime_module/pinned_bopomofo_ime.py b/pime_module/pinned_bopomofo_ime.py index 762479d..2aca5a7 100644 --- a/pime_module/pinned_bopomofo_ime.py +++ b/pime_module/pinned_bopomofo_ime.py @@ -12,16 +12,19 @@ from textService import TextService from . import pinned_libchewing +from .bopomofo_core.autocorrect import Autocorrector from .bopomofo_core.feedback_store import FeedbackStore from .bopomofo_core.keymap import symbol_for_event from .bopomofo_core.libchewing_provider import LibChewingProvider from .bopomofo_core.phrase_store import MAX_PHRASE_LENGTH, PhraseStore +from .bopomofo_core.phonetic_corrector import PhoneticCorrector from .bopomofo_core.pinned_store import PinnedStore from .bopomofo_core.session import CandidateSession -from .bopomofo_core.state import Event, EventKind +from .bopomofo_core.state import INITIALS, MEDIALS, RIMES, TONES, Event, EventKind SHIFT_PUNCTUATION = { + 0x20: "ˉ", # Shift+Space: standalone first-tone mark 0x31: "!", 0x32: "@", 0x33: "#", @@ -38,16 +41,25 @@ 0xBD: "——", # Shift+- 0xBE: "。", # Shift+. 0xBF: "?", # Shift+/ + 0xC0: "~", # Shift+` 0xDB: "『", # Shift+[ + 0xDC: "|", # Shift+\ 0xDD: "』", # Shift+] } VK_OEM_QUOTE = 0xDE -COMPACT_CANDIDATE_COUNT = 5 +CANDIDATE_PAGE_SIZE = 10 MAX_PHRASE_CHOICES = 12 +CANDIDATES_PER_ROW = 2 +CANDIDATE_SELECTION_LABELS = "1234567890" NUMPAD_TEXT = { **{key_code: str(key_code - 0x60) for key_code in range(0x60, 0x6A)}, + 0x6A: "*", # VK_MULTIPLY + 0x6B: "+", # VK_ADD + 0x6D: "-", # VK_SUBTRACT 0x6E: ".", # VK_DECIMAL + 0x6F: "/", # VK_DIVIDE } +BOPOMOFO_TEXT = INITIALS | MEDIALS | RIMES | TONES @dataclass @@ -86,6 +98,8 @@ def __init__(self, client): self.session = CandidateSession(provider, PinnedStore(pin_path)) self.phrase_store = PhraseStore(phrase_path) self.feedback_store = FeedbackStore(feedback_path) + self.autocorrector = Autocorrector() + self.phonetic_corrector = PhoneticCorrector(MAX_PHRASE_LENGTH) self.quote_open = False self.english_mode = False self.last_key_event = None @@ -100,7 +114,7 @@ def __init__(self, client): self.focus_index: int | None = None self.replacement_index: int | None = None self.reading_open = False - self.candidate_expanded = False + self.candidate_page = 0 self.candidate_choices: list[CandidateChoice] = [] @property @@ -110,12 +124,21 @@ def provisional(self) -> bool: def onActivate(self): super().onActivate() - self._restore_chinese_keyboard() - self.setSelKeys("12345") + # Reset our field-local Shift toggle, but respect the keyboard-open + # state supplied by TSF. Games and secure/custom controls may + # deliberately close their input context; forcing it open here can + # create an open/close feedback loop with the application. + self._reset_internal_chinese_mode() + # Each page exposes exactly ten numeric selection keys. The native UI + # fills its two columns from top to bottom: 1-5 on the left and 6-0 on + # the right. Keeping the native list and label string the same length + # prevents PIME from + # reading past the labels and painting garbage characters. + self.setSelKeys(CANDIDATE_SELECTION_LABELS) self.customizeUI( candFontName="Microsoft JhengHei UI", - candFontSize=18, - candPerRow=1, + candFontSize=16, + candPerRow=CANDIDATES_PER_ROW, candUseCursor=True, ) @@ -128,14 +151,16 @@ def onDeactivate(self): def onKeyboardStatusChanged(self, opened): self.keyboardOpen = opened - if not opened: - self._restore_chinese_keyboard() + self.pending_shift_toggle = False + self.last_key_down_time = 0.0 + if opened: + self.english_mode = False - def _restore_chinese_keyboard(self) -> None: - """Make this profile start open and in its internal Chinese mode.""" + def _reset_internal_chinese_mode(self) -> None: + """Reset the field-local mode without overriding the host app.""" self.english_mode = False - self.keyboardOpen = True - self.setKeyboardOpen(True) + self.pending_shift_toggle = False + self.last_key_down_time = 0.0 def filterKeyDown(self, keyEvent): self.last_key_event = keyEvent @@ -157,10 +182,15 @@ def filterKeyDown(self, keyEvent): return True if keyEvent.keyCode == VK_SPACE: return self.isComposing() + if self.showCandidates and keyEvent.keyCode in ( + VK_UP, + VK_DOWN, + VK_LEFT, + VK_RIGHT, + ): + return True if keyEvent.keyCode == VK_DOWN: return self._has_candidate_target() - if keyEvent.keyCode == VK_UP: - return self.showCandidates if keyEvent.keyCode in (VK_LEFT, VK_RIGHT): return bool(self.segments) and not self.session.preedit if keyEvent.keyCode in (VK_RETURN, VK_BACK, VK_ESCAPE): @@ -202,11 +232,15 @@ def onKeyDown(self, keyEvent): if keyEvent.keyCode == VK_DOWN and not self.showCandidates: if not self._has_candidate_target(): return False + # The menu must never reveal a better default that the editable + # composition has not already adopted. Synchronize through the + # same phrase ranking source before presenting alternatives. + self._apply_phrase_ranking() self.candidate_choices = self._build_candidate_choices() candidates, selected = self._candidate_target() if not candidates: return False - self.candidate_expanded = False + self.candidate_page = 0 visible = self._visible_candidates(candidates) self.setCandidateList(visible) self.setCandidateCursor(min(selected, len(visible) - 1)) @@ -214,26 +248,13 @@ def onKeyDown(self, keyEvent): self._render_buffer(keep_candidates=True) return True - if self.showCandidates and keyEvent.keyCode in (VK_UP, VK_DOWN): - candidates, _ = self._candidate_target() - if candidates: - visible = self._visible_candidates(candidates) - if ( - keyEvent.keyCode == VK_DOWN - and not self.candidate_expanded - and len(candidates) > len(visible) - and self.candidateCursor == len(visible) - 1 - ): - self.candidate_expanded = True - visible = self._visible_candidates(candidates) - self.setCandidateList(visible) - self.setCandidateCursor(COMPACT_CANDIDATE_COUNT) - else: - delta = -1 if keyEvent.keyCode == VK_UP else 1 - self.setCandidateCursor( - (self.candidateCursor + delta) % len(visible) - ) - self._render_buffer(keep_candidates=True) + if self.showCandidates and keyEvent.keyCode in ( + VK_UP, + VK_DOWN, + VK_LEFT, + VK_RIGHT, + ): + self._navigate_candidate_menu(keyEvent.keyCode) return True if self.showCandidates and keyEvent.keyCode == VK_RETURN: @@ -257,9 +278,32 @@ def onKeyDown(self, keyEvent): return False if self.session.preedit: if self.reading_open and self.session.candidates: + if self._is_literal_bopomofo(self.session.candidates[0]): + self._emit_or_buffer_literal(self.session.candidates[0]) + return True self._accept_active_candidate(0) return True + # Space supplies the implicit first tone. Do not decide from + # the symbol's slot alone: several symbols classed as initials + # are also complete Mandarin syllables (ㄙ→司/思, ㄓ→之/知, + # ㄔ→吃, ㄕ→師, ㄗ→資, ...). The dictionary is the source of + # truth; only a reading with no Chinese candidate falls back + # to literal Zhuyin. + literal = self.session.preedit event = self.session.input_symbol("ˉ") + if event.kind is EventKind.BELL and self.session.preedit: + # This exact first-tone reading has no Chinese syllable, + # but Microsoft Bopomofo still lets users output the raw + # Zhuyin symbol through the candidate window. + literal = self.session.preedit + self.session.candidates = [literal] + self.reading_open = True + self.candidate_page = 0 + self.setCandidateList([literal]) + self.setCandidateCursor(0) + self.setShowCandidates(True) + self._render_buffer(keep_candidates=True) + return True self._handle_session_event(event) return True self._commit_buffer(" ") @@ -316,6 +360,9 @@ def onKeyDown(self, keyEvent): symbol = symbol_for_event(keyEvent.keyCode, keyEvent.charCode) if symbol is not None: + if symbol in TONES and not self.session.preedit: + self._emit_or_buffer_literal(symbol) + return True # Ordinary typing continues at the end unless Backspace left an # explicit insertion gap inside the composition. if not self.session.preedit and self.replacement_index is None: @@ -372,17 +419,22 @@ def onCompositionTerminated(self, forced): self._clear_all() if forced: # TSF uses forced termination when focus moves to another edit - # context. Do not carry a field-local English toggle across it. - self._restore_chinese_keyboard() + # context. Reset our local toggle, but never fight an application + # that intentionally disabled its IME context. + self._reset_internal_chinese_mode() def _candidate_number(self, keyEvent) -> int | None: # The right-hand numeric keypad is text input, never a candidate key. if keyEvent.keyCode in NUMPAD_TEXT: return None - if 0x31 <= keyEvent.keyCode <= 0x35: + if 0x31 <= keyEvent.keyCode <= 0x39: return keyEvent.keyCode - 0x31 - if ord("1") <= keyEvent.charCode <= ord("5"): + if keyEvent.keyCode == 0x30: + return 9 + if ord("1") <= keyEvent.charCode <= ord("9"): return keyEvent.charCode - ord("1") + if keyEvent.charCode == ord("0"): + return 9 return None @staticmethod @@ -425,6 +477,49 @@ def _emit_punctuation(self, punctuation: str) -> None: """Replace an unfinished reading with the requested punctuation.""" self._emit_direct_text(punctuation) + def _emit_or_buffer_literal(self, text: str) -> None: + """Keep literal Bopomofo inside an existing editable composition.""" + if self.segments: + self._buffer_literal_text(text) + else: + self._emit_direct_text(text) + + def _buffer_literal_text( + self, text: str, start: int | None = None, end: int | None = None + ) -> None: + """Insert protected literal symbols without committing other text. + + One buffer segment is kept per Unicode symbol so readings, protection + masks, and rendered text remain aligned even for a spelling such as + ㄢˊ. Literal segments are locked because phrase ranking must never + silently turn a deliberately selected phonetic spelling into Hanzi. + """ + if not text: + return + if start is None: + if self.replacement_index is not None: + start = self.replacement_index + elif self.focus_index is not None: + start = self.focus_index + 1 + else: + start = len(self.segments) + if end is None: + end = start + literal_segments = [ + BufferedSyllable( + reading=character, + candidates=[character], + locked=True, + ) + for character in text + ] + self.segments[start:end] = literal_segments + self.session.clear() + self.replacement_index = None + self.reading_open = False + self.focus_index = start + len(literal_segments) - 1 + self._render_buffer() + def _emit_direct_text(self, text: str) -> None: """Insert non-Bopomofo text at the active caret and finish composition.""" if self.replacement_index is not None: @@ -503,6 +598,75 @@ def _candidate_phrase_spans(self, target: int) -> list[tuple[int, int]]: spans.append((target, target + width)) return spans + def _ranked_phrase_options(self, start: int, end: int) -> list[str]: + """Return one shared ranking for automatic text and candidate UI. + + Personal phrases win equal spans. For the whole buffer, conservative + reading-aware and exact typo corrections are then promoted ahead of + their uncorrected source sentences. Taiwan/frequency-backed phrases + follow, then the conversion engine. Unverified engine guesses are + allowed only for the whole buffer, where they represent the engine's + sentence-level default; intermediate guesses must be known words. + """ + if start < 0 or end > len(self.segments) or end - start < 2: + return [] + readings = [segment.reading for segment in self.segments[start:end]] + personal = self.phrase_store.exact(readings) + frequent = self.session.validated_frequent_phrase_candidates( + readings, + [segment.candidates for segment in self.segments[start:end]], + ) + whole_buffer = start == 0 and end == len(self.segments) + engine = [ + phrase + for phrase in self.session.phrase_candidates(readings) + if whole_buffer or self.session.is_frequent_phrase(phrase) + ] + corrected: list[str] = [] + current_text = "".join( + segment.text for segment in self.segments[start:end] + ) + if whole_buffer: + protected = [segment.locked for segment in self.segments[start:end]] + # Re-decode both the visible text and the exact-reading sources. + # On the next render the visible text may already be corrected; + # retaining the engine sources here makes the corrected sentence + # reproducible as candidate zero instead of a one-frame mutation. + for source in dict.fromkeys([current_text] + frequent + engine): + if len(source) != end - start: + continue + suggestion, phonetic_changes = self.phonetic_corrector.correct( + readings, + source, + protected, + self.session.candidates_for_reading, + self.session.frequent_phrase_candidates, + known_phrase_lookup=( + self.session.dictionary_phrase_candidates + ), + replacement_phrase_lookup=self.session.phrase_candidates, + ) + suggestion, typo_changes = self.autocorrector.correct( + suggestion, protected + ) + if ( + suggestion != source + and (phonetic_changes or typo_changes) + and suggestion not in corrected + ): + corrected.append(suggestion) + return [ + phrase + for phrase in dict.fromkeys( + ([personal] if personal else []) + + corrected + + frequent + + engine + + ([current_text] if corrected else []) + ) + if len(phrase) == end - start + ] + def _build_candidate_choices(self) -> list[CandidateChoice]: target = self._candidate_segment_index() if target is None: @@ -511,14 +675,22 @@ def _build_candidate_choices(self) -> list[CandidateChoice]: phrase_choices: list[CandidateChoice] = [] displayed: set[str] = set() for start, end in self._candidate_phrase_spans(target): - readings = [segment.reading for segment in self.segments[start:end]] - personal = self.phrase_store.exact(readings) - frequent = self.session.frequent_phrase_candidates( - [segment.candidates for segment in self.segments[start:end]] + current_text = "".join( + segment.text for segment in self.segments[start:end] ) - engine = self.session.phrase_candidates(readings) - for phrase in ([personal] if personal else []) + frequent + engine: - if len(phrase) != end - start or phrase in displayed: + for phrase in self._ranked_phrase_options(start, end): + # Keep exactly one whole-buffer choice even when automatic + # ranking has already adopted it. This preserves sentence + # selection/confirmation in the candidate editor. Shorter + # no-op spans are still noise: without this distinction a + # long composition fills the menu with the sentence, minus + # one character, minus two characters, and so on. + whole_buffer = start == 0 and end == len(self.segments) + if ( + len(phrase) != end - start + or (phrase == current_text and not whole_buffer) + or phrase in displayed + ): continue phrase_choices.append(CandidateChoice(phrase, start, end)) displayed.add(phrase) @@ -537,31 +709,111 @@ def _build_candidate_choices(self) -> list[CandidateChoice]: if len(single_choices) >= self.session.max_candidates: break - # Keep both editing styles visible on the first five rows: up to three - # word choices followed by enough single-character choices to fill the - # compact menu. Expanded navigation exposes the remaining candidates. - front_phrases = phrase_choices[:3] - front_single_count = COMPACT_CANDIDATE_COUNT - len(front_phrases) - choices = ( - front_phrases - + single_choices[:front_single_count] - + phrase_choices[len(front_phrases) :] - + single_choices[front_single_count:] + # At the end, sentence/word candidates remain first for whole-sentence + # confirmation. After the caret is moved into the composition, the + # user is explicitly editing the character to its right: put that + # character's candidates first so pressing 1 cannot accidentally lock + # the entire remaining sentence. Keep three phrase choices on page one + # so word-level editing remains one key away. + editing_inside = ( + self.focus_index is not None + and self.focus_index < len(self.segments) - 1 ) + if editing_inside: + front_phrase_count = min(3, len(phrase_choices)) + front_single_count = CANDIDATE_PAGE_SIZE - front_phrase_count + choices = ( + single_choices[:front_single_count] + + phrase_choices[:front_phrase_count] + + single_choices[front_single_count:] + + phrase_choices[front_phrase_count:] + ) + else: + front_phrases = phrase_choices[:3] + front_single_count = CANDIDATE_PAGE_SIZE - len(front_phrases) + choices = ( + front_phrases + + single_choices[:front_single_count] + + phrase_choices[len(front_phrases) :] + + single_choices[front_single_count:] + ) + # A literal Zhuyin spelling must never disappear onto page two while + # editing an uncommitted sentence. Keep it within the first four + # positions without displacing the highest-ranked Chinese choice. + literal_index = next( + ( + index + for index, choice in enumerate(choices) + if choice.width == 1 and self._is_literal_bopomofo(choice.text) + ), + None, + ) + if literal_index is not None and literal_index > 3: + literal_choice = choices.pop(literal_index) + choices.insert(3, literal_choice) return choices[: self.session.max_candidates] def _visible_candidates(self, candidates: list[str]) -> list[str]: - if self.candidate_expanded: - return candidates - return candidates[:COMPACT_CANDIDATE_COUNT] + start = self.candidate_page * CANDIDATE_PAGE_SIZE + return candidates[start : start + CANDIDATE_PAGE_SIZE] + + def _absolute_candidate_index(self, page_index: int) -> int: + return self.candidate_page * CANDIDATE_PAGE_SIZE + page_index + + @staticmethod + def _is_literal_bopomofo(text: str) -> bool: + return bool(text) and all(character in BOPOMOFO_TEXT for character in text) + + def _navigate_candidate_menu(self, key_code: int) -> None: + """Navigate a ten-item, two-column page without letter labels.""" + candidates, _ = self._candidate_target() + if not candidates: + return + visible = self._visible_candidates(candidates) + cursor = min(self.candidateCursor, len(visible) - 1) + + last_page = max(0, (len(candidates) - 1) // CANDIDATE_PAGE_SIZE) + target = cursor + if key_code == VK_RIGHT and self.candidate_page < last_page: + self.candidate_page += 1 + visible = self._visible_candidates(candidates) + target = 0 + self.setCandidateList(visible) + elif key_code == VK_LEFT and self.candidate_page > 0: + self.candidate_page -= 1 + visible = self._visible_candidates(candidates) + target = min(cursor, len(visible) - 1) + self.setCandidateList(visible) + elif key_code == VK_DOWN: + if cursor + 1 < len(visible): + target = cursor + 1 + elif self.candidate_page < last_page: + self.candidate_page += 1 + visible = self._visible_candidates(candidates) + target = 0 + self.setCandidateList(visible) + elif key_code == VK_UP: + if cursor > 0: + target = cursor - 1 + elif self.candidate_page > 0: + self.candidate_page -= 1 + visible = self._visible_candidates(candidates) + target = len(visible) - 1 + self.setCandidateList(visible) + self.setCandidateCursor(max(0, min(len(visible) - 1, target))) + self._render_buffer(keep_candidates=True) def _choose_highlighted_candidate(self, index: int) -> None: candidates, _ = self._candidate_target() + index = self._absolute_candidate_index(index) if index < 0 or index >= len(candidates): self._bell("沒有這個候選字") return if self.reading_open and self.session.candidates: selected = candidates[index] + if self._is_literal_bopomofo(selected): + self._emit_or_buffer_literal(selected) + return self.feedback_store.record( [self.session.preedit], candidates[0], selected ) @@ -576,6 +828,17 @@ def _choose_highlighted_candidate(self, index: int) -> None: self._render_buffer() def _apply_candidate_choice(self, choice: CandidateChoice) -> None: + if self._is_literal_bopomofo(choice.text): + if choice.start == 0 and choice.end == len(self.segments): + # Preserve the established standalone behavior: choosing raw + # Zhuyin with no surrounding editable text commits it at once. + self.setCommitString(choice.text) + self._clear_all() + self.setCompositionString("") + self.setCompositionCursor(0) + else: + self._buffer_literal_text(choice.text, choice.start, choice.end) + return readings = [ segment.reading for segment in self.segments[choice.start : choice.end] ] @@ -601,7 +864,7 @@ def _apply_candidate_choice(self, choice: CandidateChoice) -> None: def _pin_highlighted_candidate(self) -> None: candidates, _ = self._candidate_target() - index = self.candidateCursor + index = self._absolute_candidate_index(self.candidateCursor) if index < 0 or index >= len(candidates): self._bell("沒有可固定的候選字") return @@ -623,12 +886,28 @@ def _pin_highlighted_candidate(self) -> None: for offset, candidate in enumerate(self.candidate_choices) if offset != index ] + self.candidate_page = 0 self.setCandidateCursor(0) self._render_buffer(keep_candidates=True) self.showMessage("已固定為這組注音的第一候選", 2) def _handle_session_event(self, event: Event) -> None: if event.kind is EventKind.UPDATED and self.session.candidates: + # libchewing can append extremely rare Han characters even for a + # symbol that is not a normal standalone syllable (for example a + # lone ㄑ). Its ranked first candidate is the reliable boundary: + # Chinese first means auto-accept it; literal Zhuyin first means + # show the menu and do not let an obscure tail candidate win. + if self._is_literal_bopomofo(self.session.candidates[0]): + self.reading_open = True + self.candidate_page = 0 + self.setCandidateList( + self._visible_candidates(self.session.candidates) + ) + self.setCandidateCursor(0) + self.setShowCandidates(True) + self._render_buffer(keep_candidates=True) + return self._accept_active_candidate(0) return self._render_event(event) @@ -637,10 +916,16 @@ def _accept_active_candidate(self, index: int, advance_focus: bool = False) -> N if not self.session.candidates or not 0 <= index < len(self.session.candidates): self._bell("沒有這個候選字") return + reading = self.session.preedit segment = BufferedSyllable( - reading=self.session.preedit, + reading=reading, candidates=list(self.session.candidates), selected=index, + # A stored single-character preference remains candidate zero for + # isolated input, but must not become a permanent context lock. + # Otherwise an old 仙/不 preference blocks the reliable phrase + # correction 你先開始下一步吧. A choice made explicitly in this + # composition still arrives with advance_focus=True and is locked. locked=advance_focus, ) insertion = ( @@ -664,9 +949,10 @@ def _apply_phrase_ranking(self) -> None: if len(self.segments) < 2: return readings = [segment.reading for segment in self.segments] - phrase = self.session.best_phrase(readings) - if phrase and len(phrase) <= len(self.segments): - self._apply_ranked_phrase(len(phrase), phrase) + whole_options = self._ranked_phrase_options(0, len(self.segments)) + whole_default = whole_options[0] if whole_options else "" + if whole_default: + self._apply_ranked_phrase(len(self.segments), whole_default) personal_length, personal_phrase = self.phrase_store.best_suffix( readings @@ -680,8 +966,9 @@ def _apply_phrase_ranking(self) -> None: frequent_match: tuple[int, str] | None = None for width in range(max_width, 1, -1): suffix = self.segments[-width:] - frequent = self.session.frequent_phrase_candidates( - [segment.candidates for segment in suffix] + frequent = self.session.validated_frequent_phrase_candidates( + [segment.reading for segment in suffix], + [segment.candidates for segment in suffix], ) if frequent: frequent_match = (width, frequent[0]) @@ -691,8 +978,8 @@ def _apply_phrase_ranking(self) -> None: # current span even when libchewing does not expose that word through # its alternate-candidate API (對話框 is one such real case). engine_match = ( - (len(phrase), phrase) - if phrase and len(phrase) == len(self.segments) + (len(self.segments), whole_default) + if whole_default else None ) lexical_match = frequent_match @@ -706,9 +993,37 @@ def _apply_phrase_ranking(self) -> None: # Explicit user selection is the strongest layer, even when it covers # a shorter suffix than a bundled phrase. if personal_length: - self._apply_ranked_phrase(personal_length, personal_phrase) + self._apply_ranked_phrase(personal_length, personal_phrase, lock=True) + self._apply_live_phonetic_ranking() + + def _apply_live_phonetic_ranking(self) -> None: + """Re-rank every unlocked word from exact readings while composing.""" + if len(self.segments) < 2: + return + readings = [segment.reading for segment in self.segments] + text = "".join(segment.text for segment in self.segments) + protected = [segment.locked for segment in self.segments] + corrected, _ = self.phonetic_corrector.correct( + readings, + text, + protected, + self.session.candidates_for_reading, + self.session.frequent_phrase_candidates, + known_phrase_lookup=self.session.dictionary_phrase_candidates, + replacement_phrase_lookup=self.session.phrase_candidates, + allow_fuzzy=False, + ) + for segment, character in zip(self.segments, corrected): + if segment.locked or segment.text == character: + continue + segment.candidates = list( + dict.fromkeys([character] + segment.candidates) + )[: self.session.max_candidates] + segment.selected = 0 - def _apply_ranked_phrase(self, width: int, phrase: str) -> None: + def _apply_ranked_phrase( + self, width: int, phrase: str, lock: bool = False + ) -> None: """Move one phrase to the front without replacing locked segments.""" if width < 1 or len(phrase) != width or width > len(self.segments): return @@ -719,6 +1034,8 @@ def _apply_ranked_phrase(self, width: int, phrase: str) -> None: dict.fromkeys([suggested] + segment.candidates) )[: self.session.max_candidates] segment.selected = 0 + if lock: + segment.locked = True def _render_event(self, event: Event) -> None: self._render_buffer() @@ -737,12 +1054,13 @@ def _render_buffer(self, keep_candidates: bool = False) -> None: if keep_candidates and self.showCandidates: candidates, _ = self._candidate_target() - if candidates and 0 <= self.candidateCursor < len(candidates): - preview = candidates[self.candidateCursor] + candidate_index = self._absolute_candidate_index(self.candidateCursor) + if candidates and 0 <= candidate_index < len(candidates): + preview = candidates[candidate_index] if self.reading_open and self.session.candidates: active_text = preview elif self.candidate_choices: - choice = self.candidate_choices[self.candidateCursor] + choice = self.candidate_choices[candidate_index] segment_texts[choice.start : choice.end] = list(choice.text) else: target_index = self._candidate_segment_index() @@ -771,13 +1089,17 @@ def _render_buffer(self, keep_candidates: bool = False) -> None: # flag as closed even though Python still thinks it is open. self.setShowCandidates(True) else: - self.candidate_expanded = False + self.candidate_page = 0 self.candidate_choices = [] self.setCandidateList([]) self.setShowCandidates(False) self.setCandidateCursor(0) def _commit_buffer(self, suffix: str = "") -> None: + # Space, Enter, punctuation, Shift, deactivation, and direct English + # insertion all commit through here. Re-apply the shared default so a + # correct candidate cannot remain hidden behind Down at send time. + self._apply_phrase_ranking() text = "".join(segment.text for segment in self.segments) if not text: self._bell("沒有可以送出的文字") @@ -795,7 +1117,7 @@ def _clear_all(self) -> None: self.focus_index = None self.replacement_index = None self.reading_open = False - self.candidate_expanded = False + self.candidate_page = 0 self.candidate_choices = [] self.setCandidateList([]) self.setShowCandidates(False) diff --git a/pime_runtime/libchewing/data/swkb.dat b/pime_runtime/libchewing/data/swkb.dat index 66f6025..34307da 100644 --- a/pime_runtime/libchewing/data/swkb.dat +++ b/pime_runtime/libchewing/data/swkb.dat @@ -1,26 +1,26 @@ -Q 〔 -W 〕 -A 【 -S 】 -Z 《 -X 》 -E { -R } -D 「 -F 」 -C 『 -V 』 -T ‘ -Y ’ -G “ -H ” -B 〝 -N 〞 -U + -I - -O × -P ÷ -J ≠ -K ≒ -L Orz -M … +Q 〔 +W 〕 +A 【 +S 】 +Z 《 +X 》 +E { +R } +D 「 +F 」 +C 『 +V 』 +T ‘ +Y ’ +G “ +H ” +B 〝 +N 〞 +U + +I - +O × +P ÷ +J ≠ +K ≒ +L Orz +M … diff --git a/pime_runtime/libchewing/data/symbols.dat b/pime_runtime/libchewing/data/symbols.dat index 0864400..2ca2541 100644 --- a/pime_runtime/libchewing/data/symbols.dat +++ b/pime_runtime/libchewing/data/symbols.dat @@ -1,22 +1,22 @@ -… -※ -常用符號=,、。.?!;:‧‥﹐﹒˙·‘’“”〝〞‵′〃~$%@&#* -左右括號=()「」〔〕{}〈〉『』《》【】﹙﹚﹝﹞﹛﹜ -上下括號=︵︶﹁﹂︹︺︷︸︿﹀﹃﹄︽︾︻︼ -希臘字母=αβγδεζηθικλμνξοπρστυφχψωΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΤΥΦΧΨΩ -數學符號=+-×÷=≠≒∞±√<>﹤﹥≦≧∩∪ˇ⊥∠∟⊿㏒㏑∫∮∵∴╳﹢ -特殊圖形=↑↓←→↖↗↙↘㊣◎○●⊕⊙△▲☆★◇◆□■▽▼§¥〒¢£※♀♂ -Unicode=♨☀☁☂☃♠♥♣♦♩♪♫♬☺☻ -單線框=├─┼┴┬┤┌┐╞═╪╡│▕└┘╭╮╰╯ -雙線框=╔╦╗╠═╬╣╓╥╖╒╤╕║╚╩╝╟╫╢╙╨╜╞╪╡╘╧╛ -填色方塊=_ˍ▁▂▃▄▅▆▇█▏▎▍▌▋▊▉◢◣◥◤ -線段=﹣﹦≡|∣∥–︱—︳╴¯ ̄﹉﹊﹍﹎﹋﹌﹏︴∕﹨╱╲/\ -表情與情緒=😀😃😄😁😆😅🤣😂🙂🙃😉😊😇🥰😍🤩😘😗😚😙🥲😋😛😜🤪😝🤑🤗🤭🤫🤔🤐🤨😐😑😶😏😒🙄😬🤥😌😔😪🤤😴😷🤒🤕🤢🤮🤧🥵🥶🥴😵🤯🤠🥳🥸😎🤓🧐😕😟🙁😮😯😲😳🥺😦😧😨😰😥😢😭😱😖😣😞😓😩😫🥱😤😡😠🤬😈👿💀💩🤡👹👺👻👽👾🤖😺😸😹😻😼😽🙀😿😾🙈🙉🙊💋💌💘💝💖💗💓💞💕💟💔🧡💛💚💙💜🤎🖤🤍💯💢💥💫💦💨💣💬💭💤 -人物與身體=👋🤚✋🖖👌🤌🤏🤞🤟🤘🤙👈👉👆🖕👇👍👎✊👊🤛🤜👏🙌👐🤲🤝🙏💅🤳💪🦾🦿🦵🦶👂🦻👃🧠🫀🫁🦷🦴👀👅👄👶🧒👦👧🧑👱👨🧔👩🧓👴👵🙍🙎🙅🙆💁🙋🧏🙇🤦🤷👮💂🥷👷🤴👸👳👲🧕🤵👰🤰🤱👼🎅🤶🦸🦹🧙🧚🧛🧜🧝🧞🧟💆💇🚶🧍🧎🏃💃🕺👯🧖🧗🤺🏇🏂🏄🚣🏊🚴🚵🤸🤼🤽🤾🤹🧘🛀🛌👭👫👬💏💑👪👤👥🫂👣 -動物、大自然=🐵🐒🦍🦧🐶🐕🦮🐩🐺🦊🦝🐱🐈🦁🐯🐅🐆🐴🐎🦄🦓🦌🦬🐮🐂🐃🐄🐷🐖🐗🐽🐏🐑🐐🐪🐫🦙🦒🐘🦣🦏🦛🐭🐁🐀🐹🐰🐇🦫🦔🦇🐻🐨🐼🦥🦦🦨🦘🦡🐾🦃🐔🐓🐣🐤🐥🐦🐧🦅🦆🦢🦉🦤🪶🦩🦚🦜🐸🐊🐢🦎🐍🐲🐉🦕🦖🐳🐋🐬🦭🐟🐠🐡🦈🐙🐚🐌🦋🐛🐜🐝🪲🐞🦗🪳🦂🦟🪰🪱🦠💐🌸💮🌹🥀🌺🌻🌼🌷🌱🪴🌲🌳🌴🌵🌾🌿🍀🍁🍂🍃 -飲食=🍇🍈🍉🍊🍋🍌🍍🥭🍎🍏🍐🍑🍒🍓🫐🥝🍅🫒🥥🥑🍆🥔🥕🌽🫑🥒🥬🥦🧄🧅🍄🥜🌰🍞🥐🥖🫓🥨🥯🥞🧇🧀🍖🍗🥩🥓🍔🍟🍕🌭🥪🌮🌯🫔🥙🧆🥚🍳🥘🍲🫕🥣🥗🍿🧈🧂🥫🍱🍘🍙🍚🍛🍜🍝🍠🍢🍣🍤🍥🥮🍡🥟🥠🥡🦀🦞🦐🦑🦪🍦🍧🍨🍩🍪🎂🍰🧁🥧🍫🍬🍭🍮🍯🍼🥛☕🫖🍵🍶🍾🍷🍸🍹🍺🍻🥂🥃🥤🧋🧃🧉🧊🥢🍴🥄🔪🏺 -時間=⌛⏳⌚⏰🕛🕧🕐🕜🕑🕝🕒🕞🕓🕟🕔🕠🕕🕡🕖🕢🕗🕣🕘🕤🕙🕥🕚🕦📅📆 -天象與天氣=🌑🌒🌓🌔🌕🌖🌗🌘🌙🌚🌛🌜🌝🌞🪐⭐🌟🌠🌌⛅🌀🌈🌂☔⚡⛄🔥💧🌊 -節慶與活動=🎃🎄🎆🎇🧨✨🎈🎉🎊🎋🎍🎎🎏🎐🎑🧧🎀🎁🎫🏆🏅🥇🥈🥉⚽⚾🥎🏀🏐🏈🏉🎾🥏🎳🏏🏑🏒🥍🏓🏸🥊🥋🥅⛳🎣🤿🎽🎿🛷🥌🎯🪀🪁🎱🔮🪄🧿🎮🎰🎲🧩🧸🪅🪆🃏🀄🎴🎭 -物品=🎨🧵🪡🧶🪢👓🥽🥼🦺👔👕👖🧣🧤🧥🧦👗👘🥻🩱🩲🩳👙👚👛👜👝🎒🩴👞👟🥾🥿👠👡🩰👢👑👒🎩🎓🧢🪖📿💄💍💎🔇🔈🔉🔊📢📣📯🔔🔕🎼🎵🎶🎤🎧📻🎷🪗🎸🎹🎺🎻🪕🥁🪘📱📲📞📟📠🔋🔌💻💽💾💿📀🧮🎥🎬📺📷📸📹📼🔍🔎💡🔦🏮🪔📔📕📖📗📘📙📚📓📒📃📜📄📰📑🔖💰🪙💴💵💶💷💸💳🧾💹📧📨📩📤📥📦📫📪📬📭📮📝💼📁📂📇📈📉📊📋📌📍📎📏📐🔒🔓🔏🔐🔑🔨🪓🔫🪃🏹🪚🔧🪛🔩🦯🔗🪝🧰🧲🪜🧪🧫🧬🔬🔭📡💉🩸💊🩹🩺🚪🛗🪞🪟🪑🚽🪠🚿🛁🪤🪒🧴🧷🧹🧺🧻🪣🧼🪥🧽🧯🛒🚬🪦🗿🪧🏧🏁🚩🎌🏴 -符號與標示=🚮🚰♿🚹🚺🚻🚼🚾🛂🛃🛄🛅🚸⛔🚫🚳🚭🚯🚱🚷📵🔞🔃🔄🔙🔚🔛🔜🔝🛐🕎🔯♈♉♊♋♌♍♎♏♐♑♒♓⛎🔀🔁🔂⏩⏪🔼⏫🔽⏬🎦🔅🔆📶📳📴➕➖➗❓❔❕❗💱💲🔱📛🔰⭕✅❌❎➰➿🔟🔠🔡🔢🔣🔤🆎🆑🆒🆓🆔🆕🆖🆗🆘🆙🆚🈁🈶🈯🉐🈹🈚🈲🉑🈸🈴🈳🈺🈵🔴🟠🟡🟢🔵🟣🟤⚫⚪🟥🟧🟨🟩🟦🟪🟫⬛⬜◾◽🔶🔷🔸🔹🔺🔻💠🔘🔳🔲 +… +※ +常用符號=,、。.?!;:‧‥﹐﹒˙·‘’“”〝〞‵′〃~$%@&#* +左右括號=()「」〔〕{}〈〉『』《》【】﹙﹚﹝﹞﹛﹜ +上下括號=︵︶﹁﹂︹︺︷︸︿﹀﹃﹄︽︾︻︼ +希臘字母=αβγδεζηθικλμνξοπρστυφχψωΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΤΥΦΧΨΩ +數學符號=+-×÷=≠≒∞±√<>﹤﹥≦≧∩∪ˇ⊥∠∟⊿㏒㏑∫∮∵∴╳﹢ +特殊圖形=↑↓←→↖↗↙↘㊣◎○●⊕⊙△▲☆★◇◆□■▽▼§¥〒¢£※♀♂ +Unicode=♨☀☁☂☃♠♥♣♦♩♪♫♬☺☻ +單線框=├─┼┴┬┤┌┐╞═╪╡│▕└┘╭╮╰╯ +雙線框=╔╦╗╠═╬╣╓╥╖╒╤╕║╚╩╝╟╫╢╙╨╜╞╪╡╘╧╛ +填色方塊=_ˍ▁▂▃▄▅▆▇█▏▎▍▌▋▊▉◢◣◥◤ +線段=﹣﹦≡|∣∥–︱—︳╴¯ ̄﹉﹊﹍﹎﹋﹌﹏︴∕﹨╱╲/\ +表情與情緒=😀😃😄😁😆😅🤣😂🙂🙃😉😊😇🥰😍🤩😘😗😚😙🥲😋😛😜🤪😝🤑🤗🤭🤫🤔🤐🤨😐😑😶😏😒🙄😬🤥😌😔😪🤤😴😷🤒🤕🤢🤮🤧🥵🥶🥴😵🤯🤠🥳🥸😎🤓🧐😕😟🙁😮😯😲😳🥺😦😧😨😰😥😢😭😱😖😣😞😓😩😫🥱😤😡😠🤬😈👿💀💩🤡👹👺👻👽👾🤖😺😸😹😻😼😽🙀😿😾🙈🙉🙊💋💌💘💝💖💗💓💞💕💟💔🧡💛💚💙💜🤎🖤🤍💯💢💥💫💦💨💣💬💭💤 +人物與身體=👋🤚✋🖖👌🤌🤏🤞🤟🤘🤙👈👉👆🖕👇👍👎✊👊🤛🤜👏🙌👐🤲🤝🙏💅🤳💪🦾🦿🦵🦶👂🦻👃🧠🫀🫁🦷🦴👀👅👄👶🧒👦👧🧑👱👨🧔👩🧓👴👵🙍🙎🙅🙆💁🙋🧏🙇🤦🤷👮💂🥷👷🤴👸👳👲🧕🤵👰🤰🤱👼🎅🤶🦸🦹🧙🧚🧛🧜🧝🧞🧟💆💇🚶🧍🧎🏃💃🕺👯🧖🧗🤺🏇🏂🏄🚣🏊🚴🚵🤸🤼🤽🤾🤹🧘🛀🛌👭👫👬💏💑👪👤👥🫂👣 +動物、大自然=🐵🐒🦍🦧🐶🐕🦮🐩🐺🦊🦝🐱🐈🦁🐯🐅🐆🐴🐎🦄🦓🦌🦬🐮🐂🐃🐄🐷🐖🐗🐽🐏🐑🐐🐪🐫🦙🦒🐘🦣🦏🦛🐭🐁🐀🐹🐰🐇🦫🦔🦇🐻🐨🐼🦥🦦🦨🦘🦡🐾🦃🐔🐓🐣🐤🐥🐦🐧🦅🦆🦢🦉🦤🪶🦩🦚🦜🐸🐊🐢🦎🐍🐲🐉🦕🦖🐳🐋🐬🦭🐟🐠🐡🦈🐙🐚🐌🦋🐛🐜🐝🪲🐞🦗🪳🦂🦟🪰🪱🦠💐🌸💮🌹🥀🌺🌻🌼🌷🌱🪴🌲🌳🌴🌵🌾🌿🍀🍁🍂🍃 +飲食=🍇🍈🍉🍊🍋🍌🍍🥭🍎🍏🍐🍑🍒🍓🫐🥝🍅🫒🥥🥑🍆🥔🥕🌽🫑🥒🥬🥦🧄🧅🍄🥜🌰🍞🥐🥖🫓🥨🥯🥞🧇🧀🍖🍗🥩🥓🍔🍟🍕🌭🥪🌮🌯🫔🥙🧆🥚🍳🥘🍲🫕🥣🥗🍿🧈🧂🥫🍱🍘🍙🍚🍛🍜🍝🍠🍢🍣🍤🍥🥮🍡🥟🥠🥡🦀🦞🦐🦑🦪🍦🍧🍨🍩🍪🎂🍰🧁🥧🍫🍬🍭🍮🍯🍼🥛☕🫖🍵🍶🍾🍷🍸🍹🍺🍻🥂🥃🥤🧋🧃🧉🧊🥢🍴🥄🔪🏺 +時間=⌛⏳⌚⏰🕛🕧🕐🕜🕑🕝🕒🕞🕓🕟🕔🕠🕕🕡🕖🕢🕗🕣🕘🕤🕙🕥🕚🕦📅📆 +天象與天氣=🌑🌒🌓🌔🌕🌖🌗🌘🌙🌚🌛🌜🌝🌞🪐⭐🌟🌠🌌⛅🌀🌈🌂☔⚡⛄🔥💧🌊 +節慶與活動=🎃🎄🎆🎇🧨✨🎈🎉🎊🎋🎍🎎🎏🎐🎑🧧🎀🎁🎫🏆🏅🥇🥈🥉⚽⚾🥎🏀🏐🏈🏉🎾🥏🎳🏏🏑🏒🥍🏓🏸🥊🥋🥅⛳🎣🤿🎽🎿🛷🥌🎯🪀🪁🎱🔮🪄🧿🎮🎰🎲🧩🧸🪅🪆🃏🀄🎴🎭 +物品=🎨🧵🪡🧶🪢👓🥽🥼🦺👔👕👖🧣🧤🧥🧦👗👘🥻🩱🩲🩳👙👚👛👜👝🎒🩴👞👟🥾🥿👠👡🩰👢👑👒🎩🎓🧢🪖📿💄💍💎🔇🔈🔉🔊📢📣📯🔔🔕🎼🎵🎶🎤🎧📻🎷🪗🎸🎹🎺🎻🪕🥁🪘📱📲📞📟📠🔋🔌💻💽💾💿📀🧮🎥🎬📺📷📸📹📼🔍🔎💡🔦🏮🪔📔📕📖📗📘📙📚📓📒📃📜📄📰📑🔖💰🪙💴💵💶💷💸💳🧾💹📧📨📩📤📥📦📫📪📬📭📮📝💼📁📂📇📈📉📊📋📌📍📎📏📐🔒🔓🔏🔐🔑🔨🪓🔫🪃🏹🪚🔧🪛🔩🦯🔗🪝🧰🧲🪜🧪🧫🧬🔬🔭📡💉🩸💊🩹🩺🚪🛗🪞🪟🪑🚽🪠🚿🛁🪤🪒🧴🧷🧹🧺🧻🪣🧼🪥🧽🧯🛒🚬🪦🗿🪧🏧🏁🚩🎌🏴 +符號與標示=🚮🚰♿🚹🚺🚻🚼🚾🛂🛃🛄🛅🚸⛔🚫🚳🚭🚯🚱🚷📵🔞🔃🔄🔙🔚🔛🔜🔝🛐🕎🔯♈♉♊♋♌♍♎♏♐♑♒♓⛎🔀🔁🔂⏩⏪🔼⏫🔽⏬🎦🔅🔆📶📳📴➕➖➗❓❔❕❗💱💲🔱📛🔰⭕✅❌❎➰➿🔟🔠🔡🔢🔣🔤🆎🆑🆒🆓🆔🆕🆖🆗🆘🆙🆚🈁🈶🈯🉐🈹🈚🈲🉑🈸🈴🈳🈺🈵🔴🟠🟡🟢🔵🟣🟤⚫⚪🟥🟧🟨🟩🟦🟪🟫⬛⬜◾◽🔶🔷🔸🔹🔺🔻💠🔘🔳🔲 diff --git a/tests/native_ui_preference_smoke.ps1 b/tests/native_ui_preference_smoke.ps1 new file mode 100644 index 0000000..3b42da3 --- /dev/null +++ b/tests/native_ui_preference_smoke.ps1 @@ -0,0 +1,69 @@ +$ErrorActionPreference = "Stop" + +. (Join-Path $PSScriptRoot "..\installer\native_ui_preference.ps1") + +$temporaryRoot = Join-Path $env:TEMP ("SmartPriorityNativePreference-" + [Guid]::NewGuid().ToString("N")) +try { + $pimeRoot = Join-Path $temporaryRoot "PIME" + $stateRoot = Join-Path $temporaryRoot "state" + $preferencePath = Join-Path $temporaryRoot "native-ui-preference.json" + foreach ($architecture in @("x86", "x64")) { + New-Item -ItemType Directory -Path (Join-Path $pimeRoot $architecture) -Force | Out-Null + Set-Content -LiteralPath (Join-Path $pimeRoot "$architecture\PIMETextService.dll") ` + -Value "signed-$architecture" -Encoding ASCII + } + + $fresh = Resolve-SmartPriorityNativeUiPreference ` + -PreferencePath $preferencePath -PimeRoot $pimeRoot -StateRoot $stateRoot + if ($fresh) { throw "A fresh install must keep the signed default." } + + $enabled = Resolve-SmartPriorityNativeUiPreference ` + -PreferencePath $preferencePath -PimeRoot $pimeRoot -StateRoot $stateRoot ` + -EnableUnsignedNativeUi + if (-not $enabled) { throw "Explicit custom UI opt-in was ignored." } + Save-SmartPriorityNativeUiPreference -PreferencePath $preferencePath -Enabled $enabled + $remembered = Resolve-SmartPriorityNativeUiPreference ` + -PreferencePath $preferencePath -PimeRoot $pimeRoot -StateRoot $stateRoot + if (-not $remembered) { throw "Custom UI preference was not remembered." } + + $disabled = Resolve-SmartPriorityNativeUiPreference ` + -PreferencePath $preferencePath -PimeRoot $pimeRoot -StateRoot $stateRoot ` + -DisableUnsignedNativeUi + if ($disabled) { throw "Explicit signed UI selection was ignored." } + Save-SmartPriorityNativeUiPreference -PreferencePath $preferencePath -Enabled $disabled + $rememberedDisabled = Resolve-SmartPriorityNativeUiPreference ` + -PreferencePath $preferencePath -PimeRoot $pimeRoot -StateRoot $stateRoot + if ($rememberedDisabled) { throw "Signed UI preference was not remembered." } + + Remove-Item -LiteralPath $preferencePath -Force + $hashes = @{} + foreach ($architecture in @("x86", "x64")) { + $customDll = Join-Path $pimeRoot "$architecture\PIMETextService.dll" + Set-Content -LiteralPath $customDll -Value "custom-$architecture" -Encoding ASCII + $hashes[$architecture] = (Get-FileHash -Algorithm SHA256 -LiteralPath $customDll).Hash + } + New-Item -ItemType Directory -Path $stateRoot -Force | Out-Null + $hashes | ConvertTo-Json | Set-Content ` + -LiteralPath (Join-Path $stateRoot "native-ui.json") -Encoding UTF8 + $migrated = Resolve-SmartPriorityNativeUiPreference ` + -PreferencePath $preferencePath -PimeRoot $pimeRoot -StateRoot $stateRoot + if (-not $migrated) { throw "Existing custom UI state was not migrated." } + + $conflictRejected = $false + try { + Resolve-SmartPriorityNativeUiPreference ` + -PreferencePath $preferencePath -PimeRoot $pimeRoot -StateRoot $stateRoot ` + -EnableUnsignedNativeUi -DisableUnsignedNativeUi | Out-Null + } + catch { + $conflictRejected = $true + } + if (-not $conflictRejected) { throw "Conflicting UI switches were accepted." } + + Write-Output "PASS: native UI preference is persistent and explicitly reversible" +} +finally { + if (Test-Path -LiteralPath $temporaryRoot) { + Remove-Item -LiteralPath $temporaryRoot -Recurse -Force + } +} diff --git a/tests/pime_adapter_smoke.py b/tests/pime_adapter_smoke.py index f951099..cd0a65f 100644 --- a/tests/pime_adapter_smoke.py +++ b/tests/pime_adapter_smoke.py @@ -20,6 +20,8 @@ from pinned_bopomofo.pinned_bopomofo_ime import PinnedBopomofoTextService from pinned_bopomofo.bopomofo_core.keymap import keys_for_reading +from pinned_bopomofo.bopomofo_core.state import INITIALS, MEDIALS, RIMES +from pime_all_readings_audit import audit_all_readings class DummyClient: @@ -69,6 +71,13 @@ def special_key(service, key_code: int, sequence: int) -> dict: return reply +def candidate_selection_key(service, index: int, sequence: int) -> dict: + if not 0 <= index <= 9: + raise AssertionError(f"candidate index outside current page: {index}") + key_code = 0x30 if index == 9 else 0x31 + index + return special_key(service, key_code, sequence) + + def shifted_key(service, key_code: int, sequence: int) -> dict: key_states = [0] * 256 key_states[0x10] = 0x80 # VK_SHIFT @@ -127,10 +136,40 @@ def numpad_decimal_key(service, sequence: int) -> dict: return reply -def filter_key(service, method: str, key_code: int, sequence: int, shift=False): +def numpad_operator_key(service, key_code: int, sequence: int) -> dict: + reply = service.handleRequest( + { + "method": "onKeyDown", + "seqNum": sequence, + "charCode": 0, + "keyCode": key_code, + "repeatCount": 1, + "scanCode": 0, + "isExtended": key_code == 0x6F, + "keyStates": [0] * 256, + } + ) + assert reply["success"] + assert reply["return"] is True + return reply + + +def filter_key( + service, + method: str, + key_code: int, + sequence: int, + shift: bool = False, + control: bool = False, + alt: bool = False, +): key_states = [0] * 256 if shift: key_states[0x10] = 0x80 + if control: + key_states[0x11] = 0x80 + if alt: + key_states[0x12] = 0x80 if method == "filterKeyDown": key_states[key_code] = 0x80 return service.handleRequest( @@ -209,7 +248,7 @@ def type_readings(service, readings: list[str], sequence: int) -> int: return sequence -def force_composition_text(service, text: str) -> None: +def force_composition_text(service, text: str, *, locked: bool = True) -> None: """Put a known wrong homophone into completed segments for edit tests.""" assert len(service.segments) == len(text) for segment, character in zip(service.segments, text): @@ -217,7 +256,7 @@ def force_composition_text(service, text: str) -> None: dict.fromkeys([character] + segment.candidates) ) segment.selected = 0 - segment.locked = True + segment.locked = locked service.focus_index = len(service.segments) - 1 service._render_buffer() @@ -227,6 +266,17 @@ def main() -> None: os.environ["APPDATA"] = appdata service = PinnedBopomofoTextService(DummyClient()) + # Match Microsoft Bopomofo's vertical-first grid: 1-5 in the left + # column, 6-0 in the right column, and no letter selectors. + activation_reply = service.handleRequest( + {"method": "onActivate", "seqNum": 0, "isKeyboardOpen": False} + ) + assert "openKeyboard" not in activation_reply + assert not service.keyboardOpen + assert activation_reply["setSelKeys"] == "1234567890" + assert activation_reply["customizeUI"]["candPerRow"] == 2 + assert activation_reply["customizeUI"]["candFontSize"] == 16 + # ㄋㄧˇ -> 你 remains inside the composition instead of being sent # to the application immediately. press(service, "s", 1) @@ -258,15 +308,20 @@ def main() -> None: special_key(service, 0x28, 9) # VK_DOWN assert service.showCandidates original_reading = service.segments[0].reading - second_candidate = service.segments[0].candidates[1] + second_candidate = next( + candidate + for candidate in service.segments[0].candidates[1:] + if not service._is_literal_bopomofo(candidate) + ) single_choice_index = next( index for index, choice in enumerate(service.candidate_choices) if choice.width == 1 and choice.text == second_candidate ) - assert single_choice_index < 5 - special_key( - service, 0x31 + single_choice_index, 10 + for page_offset in range(single_choice_index // 10): + special_key(service, 0x27, 10 + page_offset) + candidate_selection_key( + service, single_choice_index % 10, 10 + single_choice_index // 10 ) # physical number, even when charCode is absent assert not service.showCandidates assert service.segments[0].selected == 0 @@ -277,6 +332,32 @@ def main() -> None: assert service.focus_index == 0 assert service._candidate_segment_index() == 1 + # Editing the first syllable of a longer uncommitted sentence must + # put single-character choices before whole-sentence choices. Choosing + # 1 locks only that syllable and advances to the next one; it never + # commits or freezes all remaining text. + three_service = PinnedBopomofoTextService(DummyClient()) + three_sequence = type_readings( + three_service, ["ㄒㄧㄝˇ", "ㄔㄥˊ", "ㄕˋ"], 900 + ) + for offset in range(3): + special_key(three_service, 0x25, three_sequence + offset) + assert three_service.focus_index == -1 + special_key(three_service, 0x28, three_sequence + 3) + assert three_service.candidate_choices[0].width == 1 + assert three_service.candidate_choices[0].start == 0 + select_first_reply = candidate_selection_key( + three_service, 0, three_sequence + 4 + ) + assert "commitString" not in select_first_reply + assert [segment.locked for segment in three_service.segments] == [ + True, + False, + False, + ] + assert three_service.focus_index == 0 + assert three_service._candidate_segment_index() == 1 + # Backspace on a completed character deletes it directly; it must not # turn back into a Bopomofo reading. special_key(service, 0x27, 11) # VK_RIGHT, move caret to the end @@ -313,6 +394,127 @@ def main() -> None: assert service.compositionString == "" assert service.segments == [] + # High-confidence offline corrections become visible sentence + # candidates before Enter and never write the surrounding sentence to + # a store or network service. + autocorrect_service = PinnedBopomofoTextService(DummyClient()) + autocorrect_sequence = type_readings( + autocorrect_service, + ["ㄨㄛˇ", "ㄧㄣˉ", "ㄍㄞˉ", "ㄅㄨˋ", "ㄏㄨㄟˋ", "ㄑㄩˋ"], + 1100, + ) + # High-confidence fuzzy correction is a visible sentence candidate, + # not an invisible Enter-time mutation. The exact-reading sentence is + # retained as candidate two so the user can explicitly override it. + assert autocorrect_service.compositionString == "我應該不會去" + special_key(autocorrect_service, 0x28, autocorrect_sequence) + autocorrect_sequence += 1 + assert autocorrect_service.candidateList[:2] == [ + "我應該不會去", + "我音該不會去", + ] + assert all( + choice.width == len(autocorrect_service.segments) + for choice in autocorrect_service.candidate_choices[:2] + ) + special_key(autocorrect_service, 0x1B, autocorrect_sequence) + autocorrect_sequence += 1 + force_composition_text(autocorrect_service, "我音該不會去", locked=False) + autocorrect_reply = special_key( + autocorrect_service, 0x0D, autocorrect_sequence + ) + assert autocorrect_reply["commitString"] == "我應該不會去" + + visible_override = PinnedBopomofoTextService(DummyClient()) + override_sequence = type_readings( + visible_override, + ["ㄨㄛˇ", "ㄧㄣˉ", "ㄍㄞˉ", "ㄅㄨˋ", "ㄏㄨㄟˋ", "ㄑㄩˋ"], + 1200, + ) + special_key(visible_override, 0x28, override_sequence) + candidate_selection_key(visible_override, 1, override_sequence + 1) + assert visible_override.compositionString == "我音該不會去" + assert all(segment.locked for segment in visible_override.segments) + override_reply = special_key(visible_override, 0x0D, override_sequence + 2) + assert override_reply["commitString"] == "我音該不會去" + + # The same decoder also handles an exact ㄧㄥ reading without needing + # a separate 英該 surface rule. + exact_reading_service = PinnedBopomofoTextService(DummyClient()) + exact_reading_sequence = type_readings( + exact_reading_service, ["ㄧㄥˉ", "ㄍㄞˉ"], 1150 + ) + force_composition_text(exact_reading_service, "英該", locked=False) + exact_reading_reply = special_key( + exact_reading_service, 0x0D, exact_reading_sequence + ) + assert exact_reading_reply["commitString"] == "應該" + + # A real word using ㄧㄣ remains untouched; fuzzy ㄣ/ㄥ decoding must + # not turn every occurrence of 音 into 應. + valid_phrase_service = PinnedBopomofoTextService(DummyClient()) + valid_phrase_sequence = type_readings( + valid_phrase_service, ["ㄧㄣˉ", "ㄍㄢˇ"], 1160 + ) + force_composition_text(valid_phrase_service, "音感", locked=False) + valid_phrase_reply = special_key( + valid_phrase_service, 0x0D, valid_phrase_sequence + ) + assert valid_phrase_reply["commitString"] == "音感" + + # Exact-reading language-model corrections also update the live + # composition instead of waiting for the candidate menu or Enter. + for readings, wrong, expected, start_sequence in ( + (["ㄧㄡˉ", "ㄒㄧㄢˉ"], "優仙", "優先", 1170), + (["ㄅㄨˋ", "ㄏㄜˊ", "ㄌㄧˇ"], "部合理", "不合理", 1180), + ): + live_service = PinnedBopomofoTextService(DummyClient()) + type_readings(live_service, readings, start_sequence) + force_composition_text(live_service, wrong, locked=False) + live_service._apply_phrase_ranking() + live_service._render_buffer() + assert live_service.compositionString == expected, ( + readings, + wrong, + live_service.compositionString, + ) + + # A character explicitly chosen by the user is protected. This is the + # strongest priority and intentionally preserves a literal discussion + # of a misspelling such as 因該. + protected_service = PinnedBopomofoTextService(DummyClient()) + protected_sequence = type_readings( + protected_service, ["ㄧㄣˉ", "ㄍㄞˉ"], 1120 + ) + force_composition_text(protected_service, "因該") + protected_service.segments[0].locked = True + protected_reply = special_key( + protected_service, 0x0D, protected_sequence + ) + assert protected_reply["commitString"] == "因該" + + personal_service = PinnedBopomofoTextService(DummyClient()) + personal_service.phrase_store.learn(["ㄧㄣˉ", "ㄍㄞˉ"], "因該") + personal_sequence = type_readings( + personal_service, ["ㄧㄣˉ", "ㄍㄞˉ"], 1130 + ) + assert personal_service.compositionString == "因該" + assert all(segment.locked for segment in personal_service.segments) + personal_reply = special_key( + personal_service, 0x0D, personal_sequence + ) + assert personal_reply["commitString"] == "因該" + + # An explicitly selected spelling is protected for every commit path; + # ordinary Space therefore preserves it exactly as composed. + literal_service = PinnedBopomofoTextService(DummyClient()) + literal_sequence = type_readings( + literal_service, ["ㄧㄣˉ", "ㄍㄞˉ"], 1140 + ) + force_composition_text(literal_service, "因該") + literal_reply = press(literal_service, " ", literal_sequence) + assert literal_reply["commitString"] == "因該 " + # A candidate chosen with a number key is learned automatically and # becomes first for the same reading in a fresh service instance. learned_service = PinnedBopomofoTextService(DummyClient()) @@ -331,7 +533,7 @@ def main() -> None: assert learned_service.compositionString == "" assert not learned_service.provisional assert not learned_service.english_mode - assert terminated_reply["openKeyboard"] is True + assert "openKeyboard" not in terminated_reply # The full tsi.dat phrase index ranks common 2/3-character words while # the UI continues to hold independently editable character segments. @@ -341,6 +543,10 @@ def main() -> None: (["ㄐㄧㄚˇ", "ㄕㄜˋ"], "假設"), (["ㄒㄧㄝˇ", "ㄔㄥˊ", "ㄕˋ"], "寫程式"), (["ㄉㄨㄟˋ", "ㄏㄨㄚˋ", "ㄎㄨㄤˉ"], "對話框"), + (["ㄅㄨˋ", "ㄓˉ", "ㄉㄠˋ"], "不知道"), + (["ㄍㄣˉ", "ㄗㄞˋ"], "跟在"), + (["ㄗㄞˋ", "ㄐㄧㄚˉ"], "在家"), + (["ㄗㄞˋ", "ㄧˉ", "ㄘˋ"], "再一次"), ) sequence = 21 for readings, expected in phrase_examples: @@ -417,10 +623,53 @@ def main() -> None: zi_service = PinnedBopomofoTextService(DummyClient()) sequence = type_readings(zi_service, ["ㄗˋ"], sequence) assert zi_service.compositionString == "字" + na_service = PinnedBopomofoTextService(DummyClient()) + sequence = type_readings(na_service, ["ㄋㄚˋ"], sequence) + assert na_service.compositionString == "那" + zuo_service = PinnedBopomofoTextService(DummyClient()) + sequence = type_readings(zuo_service, ["ㄗㄨㄛˋ"], sequence) + assert zuo_service.compositionString == "做" + shi_service = PinnedBopomofoTextService(DummyClient()) + sequence = type_readings(shi_service, ["ㄕˋ"], sequence) + assert shi_service.compositionString == "是" self_service = PinnedBopomofoTextService(DummyClient()) sequence = type_readings(self_service, ["ㄗˋ", "ㄐㄧˇ"], sequence) assert self_service.compositionString == "自己" + # Text-only frequency data may locate a phrase, but it cannot prove + # its pronunciation. 殼 is a rare alternate character candidate for + # ㄑㄩㄝˋ; that must not let 貝殼 borrow the wrong second syllable. + wrong_shell_service = PinnedBopomofoTextService(DummyClient()) + wrong_shell_readings = ["ㄅㄟˋ", "ㄑㄩㄝˋ"] + sequence = type_readings( + wrong_shell_service, wrong_shell_readings, sequence + ) + raw_shell_matches = ( + wrong_shell_service.session.frequent_phrase_candidates( + [ + segment.candidates + for segment in wrong_shell_service.segments + ] + ) + ) + assert "貝殼" in raw_shell_matches + assert "貝殼" not in ( + wrong_shell_service.session.validated_frequent_phrase_candidates( + wrong_shell_readings, + [ + segment.candidates + for segment in wrong_shell_service.segments + ], + ) + ) + assert wrong_shell_service.compositionString != "貝殼" + + shell_service = PinnedBopomofoTextService(DummyClient()) + sequence = type_readings( + shell_service, ["ㄅㄟˋ", "ㄎㄜˊ"], sequence + ) + assert shell_service.compositionString == "貝殼" + # Once a high-confidence word is resolved, later syllables cannot # reach backwards and alter it. The protected 優先級 spelling also # beats the homophonous 優先及. @@ -447,6 +696,30 @@ def main() -> None: sequence = type_readings(personal_priority, optimize_readings, sequence) assert personal_priority.compositionString == "優話" + # A new explicit single-character correction replaces the previous + # personal priority for that reading. In particular, selecting 字 after + # an old 自 preference must persist and win in the next service. + relearn_service = PinnedBopomofoTextService(DummyClient()) + relearn_service.session.pins.pin("ㄗˋ", "自") + sequence = type_readings(relearn_service, ["ㄗˋ"], sequence) + assert relearn_service.compositionString == "自" + special_key(relearn_service, 0x28, sequence) # candidate menu + sequence += 1 + zi_choice = next( + index + for index, choice in enumerate(relearn_service.candidate_choices) + if choice.width == 1 and choice.text == "字" + ) + for _ in range(zi_choice // 10): + special_key(relearn_service, 0x27, sequence) # next page + sequence += 1 + candidate_selection_key(relearn_service, zi_choice % 10, sequence) + sequence += 1 + assert relearn_service.session.pins.phrases_for("ㄗˋ")[0] == "字" + relearned_service = PinnedBopomofoTextService(DummyClient()) + sequence = type_readings(relearned_service, ["ㄗˋ"], sequence) + assert relearned_service.compositionString == "字" + program_service = PinnedBopomofoTextService(DummyClient()) program_readings = ["ㄒㄧㄝˇ", "ㄔㄥˊ", "ㄕˋ"] sequence = type_readings(program_service, program_readings, sequence) @@ -491,6 +764,42 @@ def main() -> None: assert punctuation_reply["commitString"] == "字典?" assert punctuation_service.compositionString == "" + # Every printable Shift symbol on the standard keyboard is accounted + # for. This protects the two easy-to-miss OEM keys as well as the + # existing Chinese punctuation mappings. + expected_shift_symbols = { + 0x20: "ˉ", + 0x31: "!", + 0x32: "@", + 0x33: "#", + 0x34: "$", + 0x35: "%", + 0x36: "……", + 0x37: "&", + 0x38: "*", + 0x39: "(", + 0x30: ")", + 0xBA: ":", + 0xBB: "+", + 0xBC: ",", + 0xBD: "——", + 0xBE: "。", + 0xBF: "?", + 0xC0: "~", + 0xDB: "『", + 0xDC: "|", + 0xDD: "』", + } + for key_code, expected in expected_shift_symbols.items(): + symbol_service = PinnedBopomofoTextService(DummyClient()) + symbol_reply = shifted_key(symbol_service, key_code, sequence) + sequence += 1 + assert symbol_reply["commitString"] == expected, ( + hex(key_code), + symbol_reply["commitString"], + ) + assert not symbol_service.english_mode + # Shift+quote alternates Taiwanese corner quotes. open_quote = shifted_key(punctuation_service, 0xDE, sequence) sequence += 1 @@ -509,8 +818,8 @@ def main() -> None: } ) sequence += 1 - assert activation_reply["openKeyboard"] is True - assert mode_service.keyboardOpen + assert "openKeyboard" not in activation_reply + assert not mode_service.keyboardOpen assert not mode_service.english_mode sequence, _ = tap_shift(mode_service, sequence + 1) assert mode_service.english_mode @@ -524,8 +833,9 @@ def main() -> None: sequence, _ = tap_shift(mode_service, sequence + 2) assert not mode_service.english_mode - # Windows can close the keyboard compartment when focus changes. - # This profile reopens it and resets its field-local English toggle. + # Games and custom/secure controls can intentionally close the TSF + # keyboard compartment. Respect that state instead of repeatedly + # reopening it and racing the host application. mode_service.english_mode = True status_reply = mode_service.handleRequest( { @@ -535,10 +845,97 @@ def main() -> None: } ) sequence += 1 - assert status_reply["openKeyboard"] is True + assert "openKeyboard" not in status_reply + assert not mode_service.keyboardOpen + assert mode_service.english_mode + + # Reopening the context resets the internal Shift toggle to Chinese, + # but still does not write back to the TSF compartment. + opened_reply = mode_service.handleRequest( + { + "method": "onKeyboardStatusChanged", + "seqNum": sequence, + "opened": True, + } + ) + sequence += 1 + assert "openKeyboard" not in opened_reply assert mode_service.keyboardOpen assert not mode_service.english_mode + # Forced composition termination also stays host-controlled. + mode_service.english_mode = True + terminated_reply = mode_service.handleRequest( + { + "method": "onCompositionTerminated", + "seqNum": sequence, + "forced": True, + } + ) + sequence += 1 + assert "openKeyboard" not in terminated_reply + assert not mode_service.english_mode + + # Shift+Tab, Shift+F1-F12, Ctrl+Shift and Alt+Shift belong to the + # application/Windows. The IME must not consume them. + passthrough_service = PinnedBopomofoTextService(DummyClient()) + assert filter_key( + passthrough_service, + "filterKeyDown", + 0x09, + sequence, + shift=True, + )["return"] is False + sequence += 1 + for key_code in range(0x70, 0x7C): + assert filter_key( + passthrough_service, + "filterKeyDown", + key_code, + sequence, + shift=True, + )["return"] is False + sequence += 1 + for key_code in (0x08, 0x0D, 0x1B, 0x21, 0x22, 0x23, 0x24, 0x2D, 0x2E): + assert filter_key( + passthrough_service, + "filterKeyDown", + key_code, + sequence, + shift=True, + )["return"] is False + sequence += 1 + assert filter_key( + passthrough_service, + "filterKeyDown", + ord("A"), + sequence, + shift=True, + control=True, + )["return"] is False + sequence += 1 + assert filter_key( + passthrough_service, + "filterKeyDown", + 0x10, + sequence, + shift=True, + alt=True, + )["return"] is False + sequence += 1 + + # Every Shift+A-Z combination emits its corresponding uppercase ASCII + # letter while leaving persistent Chinese mode unchanged. + for character in "abcdefghijklmnopqrstuvwxyz": + letter_service = PinnedBopomofoTextService(DummyClient()) + sequence, letter_reply = hold_shift_letter( + letter_service, + character, + sequence, + ) + assert letter_reply["commitString"] == character.upper() + assert not letter_service.english_mode + # CORE CONTRACT: holding Shift while pressing A-Z emits a temporary # uppercase English letter and stays in Chinese mode. This behavior # is distinct from tapping Shift to toggle the persistent mode. @@ -624,6 +1021,23 @@ def main() -> None: assert decimal_reply["commitString"] == "字." assert not decimal_composition.showCandidates + # Every visible numpad operator is literal text, never a DaQian + # Bopomofo key. Check physical key codes with charCode absent. + for key_code, expected in ( + (0x6F, "/"), # VK_DIVIDE + (0x6A, "*"), # VK_MULTIPLY + (0x6D, "-"), # VK_SUBTRACT + (0x6B, "+"), # VK_ADD + ): + operator_service = PinnedBopomofoTextService(DummyClient()) + press(operator_service, "a", sequence) # incomplete ㄇ + operator_reply = numpad_operator_key( + operator_service, key_code, sequence + 1 + ) + sequence += 2 + assert operator_reply["commitString"] == expected + assert operator_service.session.preedit == "" + insertion_direct = PinnedBopomofoTextService(DummyClient()) sequence = type_readings( insertion_direct, ["ㄨㄣˊ", "ㄗˋ"], sequence @@ -635,6 +1049,278 @@ def main() -> None: ) assert direct_reply["commitString"] == "文D" + # As in Microsoft Bopomofo, a complete reading exposes the literal + # Zhuyin spelling near the front without replacing normal Chinese. + for reading, expected_literal in ( + ("ㄢˉ", "ㄢ"), + ("ㄢˊ", "ㄢˊ"), + ("ㄢˇ", "ㄢˇ"), + ("ㄢˋ", "ㄢˋ"), + ("ㄢ˙", "ㄢ˙"), + ): + tone_service = PinnedBopomofoTextService(DummyClient()) + sequence = type_readings(tone_service, [reading], sequence) + if not tone_service.showCandidates: + special_key(tone_service, 0x28, sequence) + sequence += 1 + available_literals = ( + [choice.text for choice in tone_service.candidate_choices] + if tone_service.candidate_choices + else list(tone_service.session.candidates) + ) + tone_choice = next( + index + for index, candidate in enumerate(available_literals) + if candidate == expected_literal + ) + assert tone_choice < 5 + literal_reply = candidate_selection_key( + tone_service, tone_choice, sequence + ) + sequence += 1 + assert literal_reply.get("commitString") == expected_literal, ( + reading, + expected_literal, + literal_reply, + ) + assert tone_service.compositionString == "" + + # Space supplies first tone and the dictionary decides whether a lone + # symbol can form Chinese. ㄉ cannot, while ㄜ and the syllabic initial + # ㄙ can; raw Zhuyin remains available as an alternate. + initial_literal = PinnedBopomofoTextService(DummyClient()) + press(initial_literal, "2", sequence) # ㄉ + initial_menu = press(initial_literal, " ", sequence + 1) + assert initial_menu["candidateList"] == ["ㄉ"], initial_menu + initial_reply = candidate_selection_key(initial_literal, 0, sequence + 2) + assert initial_reply["commitString"] == "ㄉ" + + rime_literal = PinnedBopomofoTextService(DummyClient()) + press(rime_literal, "k", sequence + 3) # ㄜ + press(rime_literal, " ", sequence + 4) + assert rime_literal.segments + assert rime_literal.compositionString != "ㄜ" + special_key(rime_literal, 0x28, sequence + 5) + assert "ㄜ" in [choice.text for choice in rime_literal.candidate_choices[:4]] + + syllabic_initial = PinnedBopomofoTextService(DummyClient()) + syllabic_candidates = syllabic_initial.session.provider.candidates("ㄙˉ") + press(syllabic_initial, keys_for_reading("ㄙ"), sequence + 6) + press(syllabic_initial, " ", sequence + 7) + assert syllabic_initial.compositionString == syllabic_candidates[0] + assert not syllabic_initial.showCandidates + + # The reported real-world sequence must flow without opening a raw + # Zhuyin menu: ㄧˋ + ㄙ + Space => 意思. Candidate zero and the text + # actually inserted into the composition must always agree. + meaning_service = PinnedBopomofoTextService(DummyClient()) + sequence = type_readings(meaning_service, ["ㄧˋ"], sequence + 8) + press(meaning_service, keys_for_reading("ㄙ"), sequence) + press(meaning_service, " ", sequence + 1) + assert meaning_service.compositionString == "意思" + assert not meaning_service.showCandidates + assert all( + segment.text == segment.candidates[0] + for segment in meaning_service.segments + ) + + # Stored single-character preferences rank isolated input but cannot + # freeze a bad character inside strong sentence context. This mirrors + # a profile contaminated by older builds that learned 仙 and 不. + contextual_readings = [ + "ㄋㄧˇ", + "ㄒㄧㄢˉ", + "ㄎㄞˉ", + "ㄕˇ", + "ㄒㄧㄚˋ", + "ㄧˉ", + "ㄅㄨˋ", + "ㄅㄚ˙", + ] + contextual_service = PinnedBopomofoTextService(DummyClient()) + contextual_service.session.pins.pin("ㄒㄧㄢˉ", "仙") + contextual_service.session.pins.pin("ㄅㄨˋ", "不") + sequence = type_readings( + contextual_service, contextual_readings, sequence + 2 + ) + assert contextual_service.compositionString == "你先開始下一步吧" + assert not any(segment.locked for segment in contextual_service.segments) + + # The candidate editor may keep one whole-sentence correction, but it + # must not expose unverified intermediate engine guesses such as + # 你先開始夏衣 or 你掀開 ahead of useful words and characters. + force_composition_text( + contextual_service, "你仙開始下一不吧", locked=False + ) + contextual_service.focus_index = -1 + choices = contextual_service._build_candidate_choices() + choice_texts = [choice.text for choice in choices] + assert choices[0].width == 1, choices + assert choices[0].start == 0, choices + assert "你先開始下一步吧" in choice_texts[:10], choice_texts + assert "你先開始夏衣" not in choice_texts + assert "你掀開" not in choice_texts + assert "你先" in choice_texts[:10], choice_texts + + # The actual Down-key path first applies that shared default. The + # correct sentence must already be visible in the editable buffer, + # rather than existing only as candidate number one. + special_key(contextual_service, 0x28, sequence) + sequence += 1 + assert contextual_service.compositionString == "你先開始下一步吧" + assert contextual_service.candidate_choices[0].width == 1 + assert "你先開始下一步吧" in contextual_service.candidateList + assert "你先開始夏衣" not in contextual_service.candidateList + assert "你掀開" not in contextual_service.candidateList + + # Every commit path synchronizes the same default even if an unlocked + # stale buffer somehow survived from an older build or edit action. + enter_default = PinnedBopomofoTextService(DummyClient()) + sequence = type_readings(enter_default, contextual_readings, sequence) + force_composition_text(enter_default, "你仙開始下一不吧", locked=False) + enter_reply = special_key(enter_default, 0x0D, sequence) + sequence += 1 + assert enter_reply["commitString"] == "你先開始下一步吧" + + space_default = PinnedBopomofoTextService(DummyClient()) + sequence = type_readings(space_default, contextual_readings, sequence) + force_composition_text(space_default, "你仙開始下一不吧", locked=False) + space_reply = press(space_default, " ", sequence) + sequence += 1 + assert space_reply["commitString"] == "你先開始下一步吧 " + + # Opening the editor at the end preserves one whole-sentence choice + # for explicit confirmation, but must not waste the remaining slots + # on no-op copies of progressively shorter suffix spans. + no_op_service = PinnedBopomofoTextService(DummyClient()) + no_op_readings = ["ㄨㄛˇ", "ㄧㄥˉ", "ㄍㄞˉ", "ㄅㄨˋ", "ㄏㄨㄟˋ", "ㄑㄩˋ"] + sequence = type_readings(no_op_service, no_op_readings, sequence) + special_key(no_op_service, 0x28, sequence) + whole_no_ops = [] + for choice in no_op_service.candidate_choices: + if choice.width == 1: + continue + occupied = "".join( + segment.text for segment in no_op_service.segments[choice.start : choice.end] + ) + if choice.text == occupied: + whole_no_ops.append(choice) + assert len(whole_no_ops) == 1, whole_no_ops + assert whole_no_ops[0].start == 0 + assert whole_no_ops[0].end == len(no_op_service.segments) + assert no_op_service.candidate_choices[0] == whole_no_ops[0] + + # Global character frequency must not override the dictionary's + # reading-aware default: 員 is common overall but normally pronounced + # ㄩㄢˊ, so ㄩㄣˋ must remain 運 rather than being promoted to 員. + yun_service = PinnedBopomofoTextService(DummyClient()) + sequence = type_readings(yun_service, ["ㄩㄣˋ"], sequence + 1) + assert yun_service.compositionString == "運" + assert yun_service.segments[0].candidates[0] == "運" + assert "員" in yun_service.segments[0].candidates + + # Standalone tones still commit directly when no composition exists. + standalone_tone = PinnedBopomofoTextService(DummyClient()) + tone_reply = press(standalone_tone, "3", sequence + 1) + assert tone_reply["commitString"] == "ˇ" + sequence += 2 + + # Inside an existing composition, tones are protected literal + # segments. They must not have the Enter-like effect of committing all + # previously editable text. + buffered_tone = PinnedBopomofoTextService(DummyClient()) + sequence = type_readings( + buffered_tone, ["ㄋㄧˇ", "ㄏㄠˇ"], sequence + ) + fourth_reply = press(buffered_tone, "4", sequence) + sequence += 1 + assert not fourth_reply.get("commitString") + assert buffered_tone.compositionString == "你好ˋ" + third_reply = press(buffered_tone, "3", sequence) + sequence += 1 + assert not third_reply.get("commitString") + assert buffered_tone.compositionString == "你好ˋˇ" + assert all(segment.locked for segment in buffered_tone.segments[-2:]) + buffered_tone_commit = special_key(buffered_tone, 0x0D, sequence) + sequence += 1 + assert buffered_tone_commit["commitString"] == "你好ˋˇ" + + # Choosing a raw Zhuyin candidate inside a sentence follows the same + # non-committing path; a later tone stays in that composition too. + buffered_literal = PinnedBopomofoTextService(DummyClient()) + sequence = type_readings( + buffered_literal, ["ㄋㄧˇ", "ㄢˊ"], sequence + ) + special_key(buffered_literal, 0x28, sequence) + sequence += 1 + literal_index = next( + index + for index, choice in enumerate(buffered_literal.candidate_choices[:10]) + if choice.text == "ㄢˊ" + ) + literal_reply = candidate_selection_key( + buffered_literal, literal_index, sequence + ) + sequence += 1 + assert not literal_reply.get("commitString") + assert buffered_literal.compositionString == "你ㄢˊ" + trailing_tone = press(buffered_literal, "4", sequence) + sequence += 1 + assert not trailing_tone.get("commitString") + assert buffered_literal.compositionString == "你ㄢˊˋ" + + # Apply the dictionary-driven rule to every initial, medial, and rime, + # not only examples reported by the user. Valid standalone forms must + # select candidate zero and retain raw Zhuyin on page one; invalid + # forms fall back to the literal safely. + standalone_defaults = {} + for literal in sorted(INITIALS | MEDIALS | RIMES): + literal_service = PinnedBopomofoTextService(DummyClient()) + key = keys_for_reading(literal) + assert len(key) == 1, (literal, key) + candidates = literal_service.session.provider.candidates(literal + "ˉ") + chinese = [ + candidate + for candidate in candidates + if len(candidate) == 1 and not literal_service._is_literal_bopomofo(candidate) + ] + chinese_first = bool(candidates) and not literal_service._is_literal_bopomofo( + candidates[0] + ) + press(literal_service, key, sequence) + reply = press(literal_service, " ", sequence + 1) + if chinese_first: + assert literal_service.segments, (literal, reply) + assert literal_service.compositionString == candidates[0], ( + literal, + candidates, + literal_service.compositionString, + ) + assert literal_service.segments[0].text == ( + literal_service.segments[0].candidates[0] + ) + standalone_defaults[literal] = chinese + special_key(literal_service, 0x28, sequence + 2) + choices = [choice.text for choice in literal_service.candidate_choices] + assert literal in choices[:4], (literal, choices) + else: + assert reply["candidateList"][0] == literal, (literal, reply) + sequence += 3 + assert standalone_defaults["ㄧ"][0] == "一" + assert "阿" in standalone_defaults["ㄚ"] + + # Literal Zhuyin also remains on page one when editing a whole + # uncommitted sentence containing phrase candidates. + sentence_literal = PinnedBopomofoTextService(DummyClient()) + sequence = type_readings( + sentence_literal, ["ㄉㄨㄟˋ", "ㄏㄨㄚˋ", "ㄎㄨㄤˉ"], sequence + ) + special_key(sentence_literal, 0x28, sequence) + sequence += 1 + assert "ㄎㄨㄤ" in sentence_literal.candidateList, ( + sentence_literal.candidateList + ) + # Shift also switches when an unfinished Bopomofo reading is present; # the partial reading is cancelled instead of blocking mode changes. partial_service = PinnedBopomofoTextService(DummyClient()) @@ -659,30 +1345,65 @@ def main() -> None: assert toggle_reply["compositionString"] == "" assert toggle_ordering_service.english_mode - # The candidate menu starts compact. Moving down past candidate five - # expands the practical (top-20) list instead of wrapping around. + # The first page contains ten candidates in two vertical-first columns. + # Right opens the next page, while Down walks 1-5 then 6-0 and flips. expanded_service = PinnedBopomofoTextService(DummyClient()) press(expanded_service, "g", sequence) # ㄕ press(expanded_service, "4", sequence + 1) # ˋ - assert len(expanded_service.segments[0].candidates) > 5 + assert len(expanded_service.segments[0].candidates) > 10 open_reply = special_key( expanded_service, 0x28, sequence + 2 - ) # open compact list + ) # open first page assert open_reply["showCandidates"] is True - assert len(expanded_service.candidateList) == 5 - for offset in range(3, 7): - navigation_reply = special_key( - expanded_service, 0x28, sequence + offset - ) - assert navigation_reply["showCandidates"] is True - assert expanded_service.candidateCursor == 4 - expansion_reply = special_key( - expanded_service, 0x28, sequence + 7 - ) - assert expansion_reply["showCandidates"] is True - assert expanded_service.candidate_expanded - assert len(expanded_service.candidateList) > 5 - assert expanded_service.candidateCursor == 5 + assert len(expanded_service.candidateList) == 10 + first_page = list(expanded_service.candidateList) + next_page_reply = special_key( + expanded_service, 0x27, sequence + 3 + ) + assert next_page_reply["showCandidates"] is True + assert expanded_service.candidate_page == 1 + assert expanded_service.candidateList + assert expanded_service.candidateList != first_page + assert expanded_service.candidateCursor == 0 + special_key(expanded_service, 0x25, sequence + 4) # Left = prior page + assert expanded_service.candidate_page == 0 + + down_page_service = PinnedBopomofoTextService(DummyClient()) + press(down_page_service, "g", sequence + 5) + press(down_page_service, "4", sequence + 6) + special_key(down_page_service, 0x28, sequence + 7) + for offset in range(10): + special_key(down_page_service, 0x28, sequence + 8 + offset) + assert down_page_service.candidate_page == 1 + assert down_page_service.candidateCursor == 0 + + # Key 5 chooses the bottom of the left column and key 6 chooses the top + # of the right column. Letters are not candidate selectors. + fifth_service = PinnedBopomofoTextService(DummyClient()) + press(fifth_service, "g", sequence + 13) + press(fifth_service, "4", sequence + 14) + special_key(fifth_service, 0x28, sequence + 15) + fifth_choice = fifth_service.candidate_choices[4] + candidate_selection_key(fifth_service, 4, sequence + 16) + assert fifth_service.compositionString == fifth_choice.text + + sixth_service = PinnedBopomofoTextService(DummyClient()) + press(sixth_service, "g", sequence + 17) + press(sixth_service, "4", sequence + 18) + special_key(sixth_service, 0x28, sequence + 19) + sixth_choice = sixth_service.candidate_choices[5] + candidate_selection_key(sixth_service, 5, sequence + 20) + assert sixth_service.compositionString == sixth_choice.text + sequence += 21 + + # The full reading space is part of every PIME regression run. This + # prevents a future global-frequency change from reintroducing the + # same class of polyphonic error one reported reading at a time. + reading_audit = audit_all_readings() + assert reading_audit["dictionary_readings"] >= 1_400 + assert reading_audit["distinct_characters_audited"] >= 13_000 + assert reading_audit["old_global_frequency_promotions_blocked"] >= 400 + assert reading_audit["errors"] == [], reading_audit["errors"] # Invalid phonetics make only the configured gentle sound; the old # yellow showMessage tooltip must not be present in the PIME reply. diff --git a/tests/pime_all_readings_audit.py b/tests/pime_all_readings_audit.py new file mode 100644 index 0000000..55d826a --- /dev/null +++ b/tests/pime_all_readings_audit.py @@ -0,0 +1,140 @@ +"""Exhaustively audit single-syllable Bopomofo defaults with PIME's Python. + +This is intentionally separate from normal 64-bit unit tests because the +bundled libchewing DLL is 32-bit. It enumerates every initial/medial/rime/tone +slot combination, lets libchewing decide which readings exist, and verifies +that global character frequency never displaces the reading-aware default. +""" + +from __future__ import annotations + +import itertools +import json +import os +import sys +from pathlib import Path + + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +PIME_ROOT = Path(os.environ.get("PIME_ROOT", r"C:\Program Files (x86)\PIME")) +sys.path.insert(0, str(PIME_ROOT / "python")) +sys.path.insert( + 0, + str(PROJECT_ROOT / "dist" / "PIME-overlay" / "python" / "input_methods"), +) + +from pinned_bopomofo import pinned_libchewing +from pinned_bopomofo.bopomofo_core.keymap import keys_for_reading +from pinned_bopomofo.bopomofo_core.libchewing_provider import ( + MAX_CANDIDATES, + TAIWAN_FREQUENCY, + LibChewingProvider, + prioritize_common_character, +) +from pinned_bopomofo.bopomofo_core.state import INITIALS, MEDIALS, RIMES, TONES + + +def raw_candidates(provider: LibChewingProvider, reading: str) -> list[str]: + context = provider.context + context.Reset() + for key in keys_for_reading(reading): + context.handle_Default(ord(key)) + if context.cand_TotalChoice() <= 0: + return [] + context.cand_Enumerate() + results: list[str] = [] + while context.cand_hasNext() and len(results) < MAX_CANDIDATES: + candidate = context.cand_String().decode("utf-8") + if candidate not in results: + results.append(candidate) + return results + + +def completed_readings(): + initials = ("",) + tuple(sorted(INITIALS)) + medials = ("",) + tuple(sorted(MEDIALS)) + rimes = ("",) + tuple(sorted(RIMES)) + tones = tuple(sorted(TONES)) + for initial, medial, rime, tone in itertools.product( + initials, medials, rimes, tones + ): + body = initial + medial + rime + if body: + yield body + tone + + +def audit_all_readings() -> dict[str, object]: + provider = LibChewingProvider(pinned_libchewing) + total = 0 + valid = 0 + audited_characters: set[str] = set() + old_frequency_promotions: list[dict[str, str]] = [] + errors: list[str] = [] + + for reading in completed_readings(): + total += 1 + raw = raw_candidates(provider, reading) + if not raw: + continue + valid += 1 + audited_characters.update( + candidate for candidate in raw if len(candidate) == 1 + ) + + preserved = TAIWAN_FREQUENCY.rank_characters( + raw, preserve_first=True + ) + expected = prioritize_common_character(reading, preserved) + final = provider.candidates(reading) + if not expected or not final: + errors.append(f"{reading!r}: dictionary candidates disappeared") + continue + if final[0] != expected[0]: + errors.append( + f"{reading!r}: expected {expected[0]!r}, got {final[0]!r}" + ) + if raw[0] not in final: + errors.append( + f"{reading!r}: raw default {raw[0]!r} missing from final list" + ) + if len(final) != len(dict.fromkeys(final)): + errors.append(f"{reading!r}: duplicate final candidates") + literal = reading[:-1] if reading.endswith("ˉ") else reading + if literal not in final[:4]: + errors.append(f"{reading!r}: literal spelling is not in first four") + + old_ranked = prioritize_common_character( + reading, TAIWAN_FREQUENCY.rank_characters(raw) + ) + if old_ranked and old_ranked[0] != expected[0]: + old_frequency_promotions.append( + { + "reading": reading, + "dictionary": expected[0], + "old_global_frequency": old_ranked[0], + } + ) + + summary = { + "generated_combinations": total, + "dictionary_readings": valid, + "distinct_characters_audited": len(audited_characters), + "old_global_frequency_promotions_blocked": len( + old_frequency_promotions + ), + "promotion_examples": old_frequency_promotions[:20], + "errors": errors[:50], + } + return summary + + +def main() -> None: + summary = audit_all_readings() + print(json.dumps(summary, ensure_ascii=True, indent=2)) + errors = summary["errors"] + if errors: + raise AssertionError(f"all-reading audit found {len(errors)} errors") + + +if __name__ == "__main__": + main() diff --git a/tests/test_autocorrect.py b/tests/test_autocorrect.py new file mode 100644 index 0000000..4156345 --- /dev/null +++ b/tests/test_autocorrect.py @@ -0,0 +1,84 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from bopomofo_core.autocorrect import Autocorrector + + +class AutocorrectTests(unittest.TestCase): + def test_bundled_rules_correct_high_confidence_typos(self) -> None: + autocorrector = Autocorrector() + self.assertGreaterEqual(autocorrector.rule_count, 40) + self.assertTrue( + {"因該", "音該", "英該"}.isdisjoint( + rule.wrong for rule in autocorrector.rules + ) + ) + corrected, changes = autocorrector.correct("我以經迫不急待了") + self.assertEqual("我已經迫不及待了", corrected) + self.assertEqual( + ["以經", "迫不急待"], + [c.wrong for c in changes], + ) + + def test_explicitly_selected_character_protects_the_whole_rule_span(self) -> None: + autocorrector = Autocorrector() + corrected, changes = autocorrector.correct("以經", [True, False]) + self.assertEqual("以經", corrected) + self.assertEqual([], changes) + + def test_does_not_force_context_dependent_or_valid_variants(self) -> None: + autocorrector = Autocorrector() + text = "他的作法布置得很好而且我在這裡" + self.assertEqual((text, []), autocorrector.correct(text)) + + def test_longest_rule_wins_and_rules_do_not_cascade(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) / "rules.json" + path.write_text( + json.dumps( + { + "meta": {"sources": [{"id": "test"}]}, + "rules": [ + {"wrong": "甲乙", "correct": "乙丙", "source": "test"}, + {"wrong": "甲乙丙", "correct": "丙乙甲", "source": "test"}, + {"wrong": "乙丙", "correct": "丁戊", "source": "test"}, + ], + }, + ensure_ascii=False, + ), + encoding="utf-8", + ) + corrected, changes = Autocorrector(path).correct("甲乙丙") + self.assertEqual("丙乙甲", corrected) + self.assertEqual(["甲乙丙"], [change.wrong for change in changes]) + + def test_missing_corrupt_or_invalid_rules_are_safe(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) / "rules.json" + self.assertEqual(("以經", []), Autocorrector(path).correct("以經")) + path.write_text("{broken", encoding="utf-8") + self.assertEqual(0, Autocorrector(path).rule_count) + path.write_text( + json.dumps( + { + "meta": {"sources": [{"id": "test"}]}, + "rules": [ + {"wrong": "長短", "correct": "不等長", "source": "test"}, + {"wrong": "未知", "correct": "來源", "source": "missing"}, + ], + }, + ensure_ascii=False, + ), + encoding="utf-8", + ) + self.assertEqual(0, Autocorrector(path).rule_count) + + def test_rejects_a_mismatched_protection_mask(self) -> None: + with self.assertRaises(ValueError): + Autocorrector().correct("以經", [False]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_common_usage.py b/tests/test_common_usage.py index 7c9ad6b..cbd0af9 100644 --- a/tests/test_common_usage.py +++ b/tests/test_common_usage.py @@ -1,6 +1,8 @@ import unittest from bopomofo_core.libchewing_provider import ( + COMMON_USAGE_RULES, + add_literal_bopomofo_candidate, apply_common_usage_overrides, prioritize_common_character, ) @@ -27,6 +29,52 @@ def test_zi_defaults_to_character_but_context_can_select_self(self) -> None: apply_common_usage_overrides(["ㄗˋ", "ㄨㄛˇ"], "字我"), ) + def test_na_defaults_to_demonstrative_not_inside(self) -> None: + self.assertEqual( + ["那", "內", "納"], + prioritize_common_character("ㄋㄚˋ", ["內", "那", "納"]), + ) + + def test_de_defaults_to_possessive_particle(self) -> None: + self.assertEqual( + ["的", "得", "地"], + prioritize_common_character("ㄉㄜ˙", ["得", "地", "的"]), + ) + + def test_zuo_and_shi_default_to_everyday_verbs(self) -> None: + self.assertEqual( + ["做", "作", "座"], + prioritize_common_character("ㄗㄨㄛˋ", ["作", "座", "做"]), + ) + self.assertEqual( + ["是", "市", "事"], + prioritize_common_character("ㄕˋ", ["市", "是", "事"]), + ) + + def test_complete_reading_exposes_literal_zhuyin_without_replacing_default(self) -> None: + for reading, literal in ( + ("ㄢˉ", "ㄢ"), + ("ㄢˊ", "ㄢˊ"), + ("ㄢˇ", "ㄢˇ"), + ("ㄢˋ", "ㄢˋ"), + ("ㄢ˙", "ㄢ˙"), + ): + with self.subTest(reading=reading): + ranked = add_literal_bopomofo_candidate( + reading, ["安", "鞍", "庵", "諳"] + ) + self.assertEqual("安", ranked[0]) + self.assertEqual(literal, ranked[1]) + + # A dictionary-provided literal in the long tail must be moved, not + # mistaken for an already-visible candidate. + self.assertEqual( + ["安", "ㄢ", "鞍", "庵"], + add_literal_bopomofo_candidate( + "ㄢˉ", ["安", "鞍", "庵", "ㄢ"] + ), + ) + def test_bu_phrases_are_forced_to_common_usage(self) -> None: self.assertEqual( "我不要", @@ -57,8 +105,10 @@ def test_zai_rules_distinguish_unambiguous_traditional_usage(self) -> None: examples = ( (["ㄗㄞˋ", "ㄐㄧㄢˋ"], "在見", "再見"), (["ㄒㄧㄢˋ", "ㄗㄞˋ"], "現再", "現在"), - (["ㄍㄣ", "ㄗㄞˋ"], "跟再", "跟在"), - (["ㄗㄞˋ", "ㄐㄧㄚ"], "再家", "在家"), + (["ㄍㄣˉ", "ㄗㄞˋ"], "跟再", "跟在"), + (["ㄗㄞˋ", "ㄐㄧㄚˉ"], "再家", "在家"), + (["ㄗㄞˋ", "ㄧˉ", "ㄘˋ"], "在一刺", "再一次"), + (["ㄅㄨˋ", "ㄓˉ", "ㄉㄠˋ"], "部之到", "不知道"), ) for readings, engine_result, expected in examples: with self.subTest(expected=expected): @@ -67,6 +117,12 @@ def test_zai_rules_distinguish_unambiguous_traditional_usage(self) -> None: apply_common_usage_overrides(readings, engine_result), ) + def test_every_common_usage_reading_is_a_completed_syllable(self) -> None: + tones = frozenset("ˉˊˇˋ˙") + for pattern, replacement in COMMON_USAGE_RULES: + with self.subTest(replacement=replacement): + self.assertTrue(all(reading[-1:] in tones for reading in pattern)) + def test_ambiguous_zai_phrase_is_left_to_the_engine(self) -> None: self.assertEqual( "在做", diff --git a/tests/test_frequency_lexicon.py b/tests/test_frequency_lexicon.py index bc83060..e5f755b 100644 --- a/tests/test_frequency_lexicon.py +++ b/tests/test_frequency_lexicon.py @@ -7,6 +7,11 @@ class FrequencyLexiconTests(unittest.TestCase): + def test_full_phrase_membership_is_not_limited_to_candidate_tail(self) -> None: + lexicon = FrequencyLexicon() + self.assertTrue(lexicon.contains("寫程式")) + self.assertFalse(lexicon.contains("寫成式")) + def test_bundled_index_is_large_and_contains_common_taiwan_words(self) -> None: lexicon = FrequencyLexicon() self.assertGreaterEqual(lexicon.entry_count, 100_000) diff --git a/tests/test_phonetic_corrector.py b/tests/test_phonetic_corrector.py new file mode 100644 index 0000000..c7e82a1 --- /dev/null +++ b/tests/test_phonetic_corrector.py @@ -0,0 +1,161 @@ +import unittest + +from bopomofo_core.phonetic_corrector import PhoneticCorrector, reading_variants + + +class PhoneticCorrectorTests(unittest.TestCase): + def setUp(self) -> None: + self.candidates = { + "ㄧㄣˉ": ["音", "因"], + "ㄧㄥˉ": ["應", "英"], + "ㄍㄞˉ": ["該"], + "ㄍㄢˇ": ["感", "敢"], + } + + def candidate_lookup(reading: str) -> list[str]: + return list(self.candidates.get(reading, [])) + + def phrase_lookup(columns: list[list[str]]) -> list[str]: + phrases = ["應該", "音感"] + return [ + phrase + for phrase in phrases + if len(phrase) == len(columns) + and all(character in column for character, column in zip(phrase, columns)) + ] + + def known_phrase_lookup(readings: list[str]) -> list[str]: + known = { + ("ㄧㄥˉ", "ㄍㄞˉ"): ["應該"], + ("ㄧㄣˉ", "ㄍㄢˇ"): ["音感"], + } + return known.get(tuple(readings), []) + + self.candidate_lookup = candidate_lookup + self.phrase_lookup = phrase_lookup + self.known_phrase_lookup = known_phrase_lookup + self.corrector = PhoneticCorrector() + + def correct(self, readings, text, protected=None): + return self.corrector.correct( + readings, + text, + protected or [False] * len(text), + self.candidate_lookup, + self.phrase_lookup, + self.known_phrase_lookup, + ) + + def test_uses_readings_instead_of_enumerating_wrong_characters(self) -> None: + corrected, changes = self.correct(["ㄧㄣˉ", "ㄍㄞˉ"], "音該") + self.assertEqual("應該", corrected) + self.assertTrue(changes[0].used_fuzzy_reading) + + corrected, changes = self.correct(["ㄧㄥˉ", "ㄍㄞˉ"], "英該") + self.assertEqual("應該", corrected) + self.assertFalse(changes[0].used_fuzzy_reading) + + def test_preserves_a_valid_phrase_and_explicit_selection(self) -> None: + self.assertEqual( + ("音感", []), self.correct(["ㄧㄣˉ", "ㄍㄢˇ"], "音感") + ) + self.assertEqual( + ("音該", []), + self.correct(["ㄧㄣˉ", "ㄍㄞˉ"], "音該", [True, False]), + ) + + def test_common_bopomofo_confusions_are_slot_based(self) -> None: + self.assertEqual(("ㄧㄣˉ", "ㄧㄥˉ"), reading_variants("ㄧㄣˉ")) + self.assertEqual(("ㄕˋ", "ㄙˋ"), reading_variants("ㄕˋ")) + self.assertEqual(("ㄗㄥˉ", "ㄓㄥˉ", "ㄗㄣˉ"), reading_variants("ㄗㄥˉ")) + + def test_live_ranking_can_disable_fuzzy_pronunciation_changes(self) -> None: + corrected, changes = self.corrector.correct( + ["ㄧㄣˉ", "ㄍㄞˉ"], + "音該", + [False, False], + self.candidate_lookup, + self.phrase_lookup, + allow_fuzzy=False, + ) + self.assertEqual(("音該", []), (corrected, changes)) + + def test_text_only_word_cannot_borrow_an_unrelated_pronunciation(self) -> None: + def candidate_lookup(reading: str) -> list[str]: + return { + "ㄅㄟˋ": ["被", "貝"], + "ㄑㄩㄝˋ": ["卻", "殼"], + "ㄎㄜˊ": ["殼"], + }.get(reading, []) + + def phrase_lookup(columns: list[list[str]]) -> list[str]: + return [ + phrase + for phrase in ["貝殼"] + if all(character in column for character, column in zip(phrase, columns)) + ] + + def known_phrase_lookup(readings: list[str]) -> list[str]: + return { + ("ㄅㄟˋ", "ㄑㄩㄝˋ"): ["被卻"], + ("ㄅㄟˋ", "ㄎㄜˊ"): ["貝殼"], + }.get(tuple(readings), []) + + wrong = self.corrector.correct( + ["ㄅㄟˋ", "ㄑㄩㄝˋ"], + "貝卻", + [False, False], + candidate_lookup, + phrase_lookup, + known_phrase_lookup, + allow_fuzzy=False, + ) + correct = self.corrector.correct( + ["ㄅㄟˋ", "ㄎㄜˊ"], + "被殼", + [False, False], + candidate_lookup, + phrase_lookup, + known_phrase_lookup, + allow_fuzzy=False, + ) + self.assertEqual(("貝卻", []), wrong) + self.assertEqual("貝殼", correct[0]) + self.assertFalse(correct[1][0].used_fuzzy_reading) + + def test_long_valid_word_blocks_shorter_overlapping_rewrite(self) -> None: + columns = { + "寫": ["寫"], + "程": ["程", "成"], + "式": ["式"], + } + + def phrase_lookup(candidate_columns): + if len(candidate_columns) == 2 and "寫" in candidate_columns[0]: + return ["寫成"] + return [] + + corrected, changes = self.corrector.correct( + ["寫", "程", "式"], + "寫程式", + [False, False, False], + lambda reading: columns[reading], + phrase_lookup, + phrase_validator=lambda phrase: phrase == "寫程式", + allow_fuzzy=False, + ) + self.assertEqual(("寫程式", []), (corrected, changes)) + + def test_rejects_misaligned_state(self) -> None: + with self.assertRaises(ValueError): + self.corrector.correct( + ["ㄧㄣˉ"], + "音該", + [False, False], + self.candidate_lookup, + self.phrase_lookup, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_session.py b/tests/test_session.py index 0e00b11..2cdb572 100644 --- a/tests/test_session.py +++ b/tests/test_session.py @@ -75,7 +75,7 @@ def best_phrase(self, readings): return "樹葉" if readings == ["ㄕㄨˋ", "ㄧㄝˋ"] else "" def phrase_candidates(self, readings): - return ["樹葉", "樹液", "數夜"] + return ["樹葉", "樹液"] def dictionary_phrase_candidates(self, readings): return ["樹葉", "樹液"] @@ -86,7 +86,7 @@ def frequent_phrase_candidates(self, candidate_columns): session = CandidateSession(ContextualProvider()) self.assertEqual("樹葉", session.best_phrase(["ㄕㄨˋ", "ㄧㄝˋ"])) self.assertEqual( - ["樹葉", "樹液", "數夜"], + ["樹葉", "樹液"], session.phrase_candidates(["ㄕㄨˋ", "ㄧㄝˋ"]), ) self.assertEqual( @@ -97,6 +97,12 @@ def frequent_phrase_candidates(self, candidate_columns): ["樹葉", "數夜"], session.frequent_phrase_candidates([["樹", "數"], ["葉", "夜"]]), ) + self.assertEqual( + ["樹葉"], + session.validated_frequent_phrase_candidates( + ["ㄕㄨˋ", "ㄧㄝˋ"], [["樹", "數"], ["葉", "夜"]] + ), + ) def test_phrase_ranking_is_optional(self) -> None: session = CandidateSession(FakeProvider()) @@ -108,6 +114,12 @@ def test_phrase_ranking_is_optional(self) -> None: self.assertEqual( [], session.frequent_phrase_candidates([["樹"], ["葉"]]) ) + self.assertEqual( + [], + session.validated_frequent_phrase_candidates( + ["ㄕㄨˋ", "ㄧㄝˋ"], [["樹"], ["葉"]] + ), + ) def test_user_pin_outranks_the_bundled_default(self) -> None: class ProtectedProvider(FakeProvider): @@ -125,6 +137,26 @@ def prioritize_candidates(self, reading, candidates): session.input_symbol(symbol) self.assertEqual(["自", "字"], session.candidates) + def test_malformed_multi_character_pin_cannot_break_segment_alignment(self) -> None: + class CharacterProvider(FakeProvider): + TABLE = {"ㄗˋ": ["字", "自"]} + + pins = PinnedStore() + pins.pin("ㄗˋ", "不是單字") + session = CandidateSession(CharacterProvider(), pins) + for symbol in "ㄗˋ": + session.input_symbol(symbol) + self.assertEqual(["字", "自"], session.candidates) + self.assertTrue(all(len(candidate) == 1 for candidate in session.candidates)) + + invalid_pins = PinnedStore() + invalid_pins.pin("ㄅˋ", "不是單字") + invalid = CandidateSession(CharacterProvider(), invalid_pins) + invalid.input_symbol("ㄅ") + event = invalid.input_symbol("ˋ") + self.assertEqual(EventKind.BELL, event.kind) + self.assertEqual("ㄅ", invalid.preedit) + def test_corrupt_pin_file_falls_back_without_blocking_startup(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: path = Path(temp_dir) / "pins.json" diff --git a/tests/test_taiwan_frequency.py b/tests/test_taiwan_frequency.py index 5302dbb..2855757 100644 --- a/tests/test_taiwan_frequency.py +++ b/tests/test_taiwan_frequency.py @@ -7,6 +7,11 @@ class TaiwanFrequencyTests(unittest.TestCase): + def test_official_phrase_membership(self) -> None: + frequency = TaiwanFrequency() + self.assertTrue(frequency.contains_phrase("音感")) + self.assertFalse(frequency.contains_phrase("音該")) + def test_bundled_official_index_is_present(self) -> None: frequency = TaiwanFrequency() self.assertGreaterEqual(frequency.character_count, 4_000) @@ -38,6 +43,15 @@ def test_character_ranking_uses_frequency_and_preserves_unknown_order(self) -> N frequency.rank_characters(["自", "甲", "字", "乙"]), ) + def test_reading_dictionary_default_can_be_preserved(self) -> None: + frequency = TaiwanFrequency() + self.assertEqual( + ["運", "員", "均"], + frequency.rank_characters( + ["運", "均", "員"], preserve_first=True + ), + ) + def test_missing_index_is_optional(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: frequency = TaiwanFrequency(Path(temp_dir) / "missing.json")