From 55adb26b89f016ca1221e8187e2da647b0c6b61d Mon Sep 17 00:00:00 2001 From: xr843 <137012659+xr843@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:23:12 +0800 Subject: [PATCH 01/13] feat: validate dataset compatibility --- src/data.rs | 102 ++++++++++++++++++++++++++++++++++++++++++++++++++ tests/data.rs | 100 ++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 201 insertions(+), 1 deletion(-) diff --git a/src/data.rs b/src/data.rs index be44f35..fa40df5 100644 --- a/src/data.rs +++ b/src/data.rs @@ -1,4 +1,5 @@ use anyhow::{anyhow, Context, Result}; +use rusqlite::OptionalExtension; use std::io::Read; use std::path::{Path, PathBuf}; use std::time::Duration; @@ -11,6 +12,9 @@ const CONNECT_TIMEOUT: Duration = Duration::from_secs(30); /// guaranteeing the CLI can never hang indefinitely. const READ_TIMEOUT: Duration = Duration::from_secs(900); +pub const EXPECTED_DATA_VERSION: &str = "v1"; +pub const EXPECTED_NORM_RULESET: &str = "t2s-char-1to1-v1"; + pub struct DataSource<'a> { pub url: &'a str, pub sha256: &'a str, @@ -162,6 +166,72 @@ pub fn open_db(path: &Path) -> Result { rusqlite::Connection::open(path).with_context(|| format!("打开数据失败: {}", path.display())) } +#[derive(Debug, serde::Serialize)] +pub struct DatasetCompatibility { + pub version: String, + pub norm_ruleset: String, +} + +pub fn validate_compatibility(conn: &rusqlite::Connection) -> Result { + require_schema(conn, "meta", "SELECT key, value FROM meta LIMIT 0")?; + require_schema( + conn, + "parallels", + "SELECT id, zh_text, zh_norm, foreign_lang, foreign_text, cbeta_id FROM parallels LIMIT 0", + )?; + require_schema( + conn, + "norm_map", + "SELECT from_char, to_char FROM norm_map LIMIT 0", + )?; + + Ok(DatasetCompatibility { + version: require_expected_meta(conn, "version", EXPECTED_DATA_VERSION)?, + norm_ruleset: require_expected_meta(conn, "norm_ruleset", EXPECTED_NORM_RULESET)?, + }) +} + +pub fn verify_dataset(conn: &rusqlite::Connection) -> Result { + let compatibility = validate_compatibility(conn)?; + let mut stmt = conn.prepare("PRAGMA quick_check").map_err(|e| { + anyhow!( + "dataset incompatibility: could not run PRAGMA quick_check: {e}. Run `fojin data update`." + ) + })?; + let diagnostics = stmt + .query_map([], |row| row.get::<_, String>(0)) + .map_err(|e| { + anyhow!( + "dataset incompatibility: could not read PRAGMA quick_check output: {e}. Run `fojin data update`." + ) + })? + .collect::>>() + .map_err(|e| { + anyhow!( + "dataset incompatibility: could not collect PRAGMA quick_check output: {e}. Run `fojin data update`." + ) + })?; + + if diagnostics.as_slice() == ["ok"] { + return Ok(compatibility); + } + + let summary = if diagnostics.is_empty() { + "no diagnostics returned".to_string() + } else { + diagnostics.join("; ") + }; + Err(anyhow!( + "dataset incompatibility: PRAGMA quick_check failed: {summary}. Run `fojin data update`." + )) +} + +pub fn open_compatible_db(path: &Path) -> Result { + let conn = open_db(path)?; + validate_compatibility(&conn)?; + Ok(conn) +} + #[derive(Debug, serde::Serialize)] pub struct DatasetStats { pub version: Option, @@ -205,3 +275,35 @@ pub fn dataset_stats(conn: &rusqlite::Connection) -> Result { texts, }) } + +fn require_schema(conn: &rusqlite::Connection, name: &str, sql: &str) -> Result<()> { + conn.prepare(sql).map(|_| ()).map_err(|e| { + anyhow!( + "dataset incompatibility: required schema `{name}` is missing or invalid: {e}. Run `fojin data update`." + ) + }) +} + +fn require_expected_meta(conn: &rusqlite::Connection, key: &str, expected: &str) -> Result { + let got = conn + .query_row("SELECT value FROM meta WHERE key=?1", [key], |row| row.get::<_, String>(0)) + .optional() + .map_err(|e| { + anyhow!( + "dataset incompatibility: could not read meta `{key}`: {e}. Run `fojin data update`." + ) + })? + .ok_or_else(|| { + anyhow!( + "dataset incompatibility: required meta `{key}` is missing. Run `fojin data update`." + ) + })?; + + if got != expected { + return Err(anyhow!( + "dataset incompatibility: meta `{key}` expected `{expected}` but found `{got}`. Run `fojin data update`." + )); + } + + Ok(got) +} diff --git a/tests/data.rs b/tests/data.rs index ec9dfb6..afb0634 100644 --- a/tests/data.rs +++ b/tests/data.rs @@ -1,4 +1,7 @@ -use fojin_cli::data::{ensure_data, gunzip, resolve_data_path, verify_sha256, DataSource}; +use fojin_cli::data::{ + ensure_data, gunzip, open_compatible_db, resolve_data_path, validate_compatibility, + verify_dataset, verify_sha256, DataSource, EXPECTED_DATA_VERSION, EXPECTED_NORM_RULESET, +}; use std::io::Write; use std::path::PathBuf; @@ -174,3 +177,98 @@ fn dataset_stats_reads_meta_and_counts() { vec![("bo".to_string(), 1), ("sa".to_string(), 2)] ); } + +#[test] +fn compatibility_accepts_expected_dataset_metadata() { + let conn = compatible_conn(); + let got = validate_compatibility(&conn).unwrap(); + assert_eq!(got.version, EXPECTED_DATA_VERSION); + assert_eq!(got.norm_ruleset, EXPECTED_NORM_RULESET); + assert_eq!( + serde_json::to_value(&got).unwrap(), + serde_json::json!({ + "version": "v1", + "norm_ruleset": "t2s-char-1to1-v1" + }) + ); +} + +#[test] +fn compatibility_rejects_wrong_version() { + let conn = compatible_conn(); + conn.execute("UPDATE meta SET value='v0' WHERE key='version'", []) + .unwrap(); + + let err = validate_compatibility(&conn).unwrap_err().to_string(); + assert!(err.contains("version"), "got: {err}"); + assert!(err.contains(EXPECTED_DATA_VERSION), "got: {err}"); + assert!(err.contains("fojin data update"), "got: {err}"); +} + +#[test] +fn compatibility_rejects_wrong_norm_ruleset() { + let conn = compatible_conn(); + conn.execute( + "UPDATE meta SET value='legacy-rules' WHERE key='norm_ruleset'", + [], + ) + .unwrap(); + + let err = validate_compatibility(&conn).unwrap_err().to_string(); + assert!(err.contains("norm_ruleset"), "got: {err}"); + assert!(err.contains(EXPECTED_NORM_RULESET), "got: {err}"); + assert!(err.contains("fojin data update"), "got: {err}"); +} + +#[test] +fn compatibility_rejects_missing_required_schema() { + let conn = compatible_conn(); + conn.execute("DROP TABLE norm_map", []).unwrap(); + + let err = validate_compatibility(&conn).unwrap_err().to_string(); + assert!(err.contains("norm_map"), "got: {err}"); + assert!(err.contains("fojin data update"), "got: {err}"); +} + +#[test] +fn compatibility_open_compatible_db_checks_file_before_returning_connection() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("data.sqlite"); + let conn = rusqlite::Connection::open(&path).unwrap(); + fojin_cli::schema::init_schema(&conn).unwrap(); + insert_compat_meta(&conn); + drop(conn); + + let conn = open_compatible_db(&path).unwrap(); + let got = validate_compatibility(&conn).unwrap(); + assert_eq!(got.version, EXPECTED_DATA_VERSION); + assert_eq!(got.norm_ruleset, EXPECTED_NORM_RULESET); +} + +#[test] +fn verify_dataset_runs_quick_check_on_compatible_db() { + let conn = compatible_conn(); + let got = verify_dataset(&conn).unwrap(); + assert_eq!(got.version, EXPECTED_DATA_VERSION); + assert_eq!(got.norm_ruleset, EXPECTED_NORM_RULESET); +} + +fn compatible_conn() -> rusqlite::Connection { + let conn = rusqlite::Connection::open_in_memory().unwrap(); + fojin_cli::schema::init_schema(&conn).unwrap(); + insert_compat_meta(&conn); + conn +} + +fn insert_compat_meta(conn: &rusqlite::Connection) { + for (key, value) in [ + ("version", EXPECTED_DATA_VERSION), + ("norm_ruleset", EXPECTED_NORM_RULESET), + ] { + conn.execute( + "INSERT INTO meta(key,value) VALUES (?1,?2)", + rusqlite::params![key, value], + ) + .unwrap(); + } +} From e227f0d172dd1eaf84ae72c8d41267e9cd30dba2 Mon Sep 17 00:00:00 2001 From: xr843 <137012659+xr843@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:35:27 +0800 Subject: [PATCH 02/13] fix: validate complete query schema --- src/data.rs | 8 +++++++- tests/data.rs | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/src/data.rs b/src/data.rs index fa40df5..28d2dce 100644 --- a/src/data.rs +++ b/src/data.rs @@ -177,7 +177,13 @@ pub fn validate_compatibility(conn: &rusqlite::Connection) -> Result Date: Fri, 10 Jul 2026 13:42:18 +0800 Subject: [PATCH 03/13] feat: add dataset verification command --- src/cli.rs | 48 ++++++++++++++++++++++------- tests/command.rs | 79 +++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 115 insertions(+), 12 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index 15a6e8d..fe0b57e 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -135,6 +135,15 @@ pub enum DataAction { #[arg(long)] data_dir: Option, }, + /// 校验本地数据版本 / 规范 / SQLite 完整性(不触发下载) + Verify { + /// 机器可读 JSON 输出 + #[arg(long)] + json: bool, + /// 指定数据目录(覆盖默认缓存) + #[arg(long)] + data_dir: Option, + }, } pub fn compute_output( @@ -190,16 +199,7 @@ pub fn run() -> Result { } let preflight = normalize::normalize(raw.trim(), &normalize::NormMap::new()); normalize::validate_query_length(&preflight)?; - let path = data::resolve_data_path(data_dir)?; - data::ensure_data( - &path, - offline, - &data::DataSource { - url: DATA_URL, - sha256: DATA_SHA256, - }, - )?; - let conn = data::open_db(&path)?; + let conn = open_ensured(data_dir, offline)?; let langs: Option> = lang.map(|l| { l.split(',') .map(|s| s.trim().to_string()) @@ -288,7 +288,7 @@ fn open_ensured(data_dir: Option, offline: bool) -> Result sha256: DATA_SHA256, }, )?; - data::open_db(&path) + data::open_compatible_db(&path) } fn run_data(action: DataAction) -> Result { @@ -363,8 +363,34 @@ fn run_data(action: DataAction) -> Result { sha256: DATA_SHA256, }, )?; + let _ = data::open_compatible_db(&path)?; println!("数据已更新: {}", path.display()); Ok(0) } + DataAction::Verify { json, data_dir } => { + let path = data::resolve_data_path(data_dir)?; + if !path.exists() { + anyhow::bail!( + "本地数据不存在: {}。请先运行 `fojin data update`。", + path.display() + ); + } + let conn = data::open_db(&path)?; + let compatibility = data::verify_dataset(&conn)?; + if json { + let v = serde_json::json!({ + "ok": true, + "version": compatibility.version, + "norm_ruleset": compatibility.norm_ruleset, + }); + println!("{}", serde_json::to_string_pretty(&v)?); + } else { + println!( + "数据校验通过 version={} norm_ruleset={}", + compatibility.version, compatibility.norm_ruleset + ); + } + Ok(0) + } } } diff --git a/tests/command.rs b/tests/command.rs index 86d9c03..f60ad9b 100644 --- a/tests/command.rs +++ b/tests/command.rs @@ -5,6 +5,13 @@ fn write_offline_db(dir: &std::path::Path) { let db_path = dir.join("data.sqlite"); let conn = Connection::open(db_path).unwrap(); init_schema(&conn).unwrap(); + for (k, v) in [("version", "v1"), ("norm_ruleset", "t2s-char-1to1-v1")] { + conn.execute( + "INSERT INTO meta(key,value) VALUES (?1,?2)", + rusqlite::params![k, v], + ) + .unwrap(); + } } #[test] @@ -83,7 +90,11 @@ fn zero_limit_is_rejected_during_clap_parsing_before_data_access() { fn write_fixture_db(dir: &std::path::Path) { let conn = Connection::open(dir.join("data.sqlite")).unwrap(); init_schema(&conn).unwrap(); - for (k, v) in [("version", "v1"), ("license", "CC BY-SA 4.0")] { + for (k, v) in [ + ("version", "v1"), + ("norm_ruleset", "t2s-char-1to1-v1"), + ("license", "CC BY-SA 4.0"), + ] { conn.execute( "INSERT INTO meta(key,value) VALUES (?1,?2)", rusqlite::params![k, v], @@ -159,6 +170,72 @@ fn data_status_json_reports_dataset() { assert!(stdout.contains("\"exists\": true"), "got: {stdout}"); } +#[test] +fn data_verify_json_reports_exact_compatibility_shape() { + let dir = tempfile::tempdir().unwrap(); + write_fixture_db(dir.path()); + + let out = run_fojin(&["data", "verify", "--json"], dir.path()); + + assert_eq!(out.status.code(), Some(0)); + let stdout = String::from_utf8(out.stdout).unwrap(); + let got: serde_json::Value = serde_json::from_str(&stdout).unwrap(); + assert_eq!( + got, + serde_json::json!({ + "ok": true, + "version": "v1", + "norm_ruleset": "t2s-char-1to1-v1" + }) + ); +} + +#[test] +fn data_verify_human_reports_compatibility_summary() { + let dir = tempfile::tempdir().unwrap(); + write_fixture_db(dir.path()); + + let out = run_fojin(&["data", "verify"], dir.path()); + + assert_eq!(out.status.code(), Some(0)); + let stdout = String::from_utf8(out.stdout).unwrap(); + assert!(stdout.starts_with("数据校验通过"), "got: {stdout}"); + assert!(stdout.contains("v1"), "got: {stdout}"); + assert!(stdout.contains("t2s-char-1to1-v1"), "got: {stdout}"); +} + +#[test] +fn data_verify_missing_data_exits_one_without_creating_file() { + let dir = tempfile::tempdir().unwrap(); + let data_path = dir.path().join("data.sqlite"); + + let out = run_fojin(&["data", "verify"], dir.path()); + + assert_eq!(out.status.code(), Some(1)); + assert!( + !data_path.exists(), + "data verify must not create data.sqlite" + ); +} + +#[test] +fn incompatible_version_is_rejected_before_parallel_query() { + let dir = tempfile::tempdir().unwrap(); + write_fixture_db(dir.path()); + let conn = Connection::open(dir.path().join("data.sqlite")).unwrap(); + conn.execute("UPDATE meta SET value='v0' WHERE key='version'", []) + .unwrap(); + + let out = run_fojin(&["parallel", "色即是空", "--offline"], dir.path()); + + assert_eq!(out.status.code(), Some(1)); + let stdout = String::from_utf8(out.stdout).unwrap(); + let stderr = String::from_utf8(out.stderr).unwrap(); + assert!(stdout.trim().is_empty(), "got stdout: {stdout}"); + assert!(stderr.contains("dataset incompatibility"), "got: {stderr}"); + assert!(stderr.contains("version"), "got: {stderr}"); +} + #[test] fn data_status_reports_missing_data_without_downloading() { let dir = tempfile::tempdir().unwrap(); From 92dc80c321cb3df94b152f34fcbc448742dea1d3 Mon Sep 17 00:00:00 2001 From: xr843 <137012659+xr843@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:56:01 +0800 Subject: [PATCH 04/13] fix: open verified datasets read only --- src/cli.rs | 2 +- src/data.rs | 22 +++++++++++++++++++--- tests/data.rs | 38 ++++++++++++++++++++++++++++++++++++-- 3 files changed, 56 insertions(+), 6 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index fe0b57e..0dba11b 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -375,7 +375,7 @@ fn run_data(action: DataAction) -> Result { path.display() ); } - let conn = data::open_db(&path)?; + let conn = data::open_read_only_db(&path)?; let compatibility = data::verify_dataset(&conn)?; if json { let v = serde_json::json!({ diff --git a/src/data.rs b/src/data.rs index 28d2dce..6867608 100644 --- a/src/data.rs +++ b/src/data.rs @@ -1,5 +1,5 @@ use anyhow::{anyhow, Context, Result}; -use rusqlite::OptionalExtension; +use rusqlite::{OpenFlags, OptionalExtension}; use std::io::Read; use std::path::{Path, PathBuf}; use std::time::Duration; @@ -166,6 +166,11 @@ pub fn open_db(path: &Path) -> Result { rusqlite::Connection::open(path).with_context(|| format!("打开数据失败: {}", path.display())) } +pub fn open_read_only_db(path: &Path) -> Result { + rusqlite::Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_ONLY) + .with_context(|| format!("打开数据失败: {}", path.display())) +} + #[derive(Debug, serde::Serialize)] pub struct DatasetCompatibility { pub version: String, @@ -218,7 +223,11 @@ pub fn verify_dataset(conn: &rusqlite::Connection) -> Result Result bool { + // SQLite 3.45.0 implements FTS5 xIntegrity through a special INSERT. + // SQLite 3.45.1 fixed that command failing on read-only databases. + message.starts_with("unable to validate the inverted index for FTS5 table ") + && message.ends_with(": attempt to write a readonly database") +} + pub fn open_compatible_db(path: &Path) -> Result { - let conn = open_db(path)?; + let conn = open_read_only_db(path)?; validate_compatibility(&conn)?; Ok(conn) } diff --git a/tests/data.rs b/tests/data.rs index d74f392..1b2bd4e 100644 --- a/tests/data.rs +++ b/tests/data.rs @@ -1,6 +1,7 @@ use fojin_cli::data::{ - ensure_data, gunzip, open_compatible_db, resolve_data_path, validate_compatibility, - verify_dataset, verify_sha256, DataSource, EXPECTED_DATA_VERSION, EXPECTED_NORM_RULESET, + ensure_data, gunzip, open_compatible_db, open_read_only_db, resolve_data_path, + validate_compatibility, verify_dataset, verify_sha256, DataSource, EXPECTED_DATA_VERSION, + EXPECTED_NORM_RULESET, }; use std::io::Write; use std::path::PathBuf; @@ -178,6 +179,23 @@ fn dataset_stats_reads_meta_and_counts() { ); } +#[test] +fn open_read_only_db_rejects_create_table() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("data.sqlite"); + let conn = rusqlite::Connection::open(&path).unwrap(); + conn.execute("CREATE TABLE existing (id INTEGER)", []) + .unwrap(); + drop(conn); + + let conn = open_read_only_db(&path).unwrap(); + let err = conn + .execute("CREATE TABLE forbidden (id INTEGER)", []) + .unwrap_err(); + + assert_eq!(err.sqlite_error_code(), Some(rusqlite::ErrorCode::ReadOnly)); +} + #[test] fn compatibility_accepts_expected_dataset_metadata() { let conn = compatible_conn(); @@ -294,6 +312,22 @@ fn verify_dataset_runs_quick_check_on_compatible_db() { assert_eq!(got.norm_ruleset, EXPECTED_NORM_RULESET); } +#[test] +fn verify_dataset_accepts_compatible_read_only_db() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("data.sqlite"); + let conn = rusqlite::Connection::open(&path).unwrap(); + fojin_cli::schema::init_schema(&conn).unwrap(); + insert_compat_meta(&conn); + drop(conn); + + let conn = open_read_only_db(&path).unwrap(); + let got = verify_dataset(&conn).unwrap(); + + assert_eq!(got.version, EXPECTED_DATA_VERSION); + assert_eq!(got.norm_ruleset, EXPECTED_NORM_RULESET); +} + fn compatible_conn() -> rusqlite::Connection { let conn = rusqlite::Connection::open_in_memory().unwrap(); fojin_cli::schema::init_schema(&conn).unwrap(); From 612e1439cc2fd5ef8492939e6f10c7524b8bfd3e Mon Sep 17 00:00:00 2001 From: xr843 <137012659+xr843@users.noreply.github.com> Date: Fri, 10 Jul 2026 14:03:39 +0800 Subject: [PATCH 05/13] fix: upgrade sqlite for read-only verification --- Cargo.lock | 8 ++++---- Cargo.toml | 2 +- src/data.rs | 13 +------------ 3 files changed, 6 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e0291a3..09b40d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -486,9 +486,9 @@ dependencies = [ [[package]] name = "libsqlite3-sys" -version = "0.28.0" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c10584274047cb335c23d3e61bcef8e323adae7c5c8c760540f73610177fc3f" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" dependencies = [ "cc", "pkg-config", @@ -619,9 +619,9 @@ dependencies = [ [[package]] name = "rusqlite" -version = "0.31.0" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b838eba278d213a8beaf485bd313fd580ca4505a00d5871caeb1457c55322cae" +checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" dependencies = [ "bitflags", "fallible-iterator", diff --git a/Cargo.toml b/Cargo.toml index 689d028..3bcfed0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,7 @@ path = "src/main.rs" [dependencies] clap = { version = "4", features = ["derive"] } -rusqlite = { version = "0.31", features = ["bundled"] } +rusqlite = { version = "0.32.1", features = ["bundled"] } serde = { version = "1", features = ["derive"] } serde_json = "1" directories = "5" diff --git a/src/data.rs b/src/data.rs index 6867608..9b0cc99 100644 --- a/src/data.rs +++ b/src/data.rs @@ -223,11 +223,7 @@ pub fn verify_dataset(conn: &rusqlite::Connection) -> Result Result bool { - // SQLite 3.45.0 implements FTS5 xIntegrity through a special INSERT. - // SQLite 3.45.1 fixed that command failing on read-only databases. - message.starts_with("unable to validate the inverted index for FTS5 table ") - && message.ends_with(": attempt to write a readonly database") -} - pub fn open_compatible_db(path: &Path) -> Result { let conn = open_read_only_db(path)?; validate_compatibility(&conn)?; From 9d11fd5ed904d303842837e4e9e8a9f9c7e8e25a Mon Sep 17 00:00:00 2001 From: xr843 <137012659+xr843@users.noreply.github.com> Date: Fri, 10 Jul 2026 14:26:33 +0800 Subject: [PATCH 06/13] ci: enforce rust 1.77 compatibility --- .github/workflows/ci.yml | 6 + Cargo.lock | 316 ++++++++------------------------------- Cargo.toml | 6 +- 3 files changed, 72 insertions(+), 256 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3d56def..a792dcf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,12 @@ name: ci on: [push, pull_request] jobs: + msrv: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@1.77.0 + - run: cargo test --all --locked rust: runs-on: ubuntu-latest steps: diff --git a/Cargo.lock b/Cargo.lock index 09b40d9..56c0654 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -22,9 +22,9 @@ dependencies = [ [[package]] name = "anstream" -version = "1.0.0" +version = "0.6.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" dependencies = [ "anstyle", "anstyle-parse", @@ -43,9 +43,9 @@ checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" [[package]] name = "anstyle-parse" -version = "1.0.0" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" dependencies = [ "utf8parse", ] @@ -115,9 +115,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "clap" -version = "4.6.1" +version = "4.5.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +checksum = "c9e340e012a1bf4935f5282ed1436d1489548e8f72308207ea5df0e23d2d03f8" dependencies = [ "clap_builder", "clap_derive", @@ -125,9 +125,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.0" +version = "4.5.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +checksum = "d76b5d13eaa18c901fd2f7fca939fefe3a0727a953561fefdf3b2922b8569d00" dependencies = [ "anstream", "anstyle", @@ -137,9 +137,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.1" +version = "4.5.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +checksum = "2a0b5487afeab2deb2ff4e03a807ad1a03ac532ff5a2cee5d86884440c7f7671" dependencies = [ "heck", "proc-macro2", @@ -149,9 +149,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "1.1.0" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +checksum = "c3e64b0cc0439b12df2fa678eae89a1c56a529fd067a9115f7827f1fffd22b32" [[package]] name = "colorchoice" @@ -218,17 +218,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "displaydoc" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "errno" version = "0.3.14" @@ -321,13 +310,14 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.4.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", "libc", "r-efi", + "wasip2", ] [[package]] @@ -354,107 +344,14 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "icu_collections" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" -dependencies = [ - "displaydoc", - "potential_utf", - "utf8_iter", - "yoke", - "zerofrom", - "zerovec", -] - -[[package]] -name = "icu_locale_core" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" -dependencies = [ - "displaydoc", - "litemap", - "tinystr", - "writeable", - "zerovec", -] - -[[package]] -name = "icu_normalizer" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" -dependencies = [ - "icu_collections", - "icu_normalizer_data", - "icu_properties", - "icu_provider", - "smallvec", - "zerovec", -] - -[[package]] -name = "icu_normalizer_data" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" - -[[package]] -name = "icu_properties" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" -dependencies = [ - "icu_collections", - "icu_locale_core", - "icu_properties_data", - "icu_provider", - "zerotrie", - "zerovec", -] - -[[package]] -name = "icu_properties_data" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" - -[[package]] -name = "icu_provider" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" -dependencies = [ - "displaydoc", - "icu_locale_core", - "writeable", - "yoke", - "zerofrom", - "zerotrie", - "zerovec", -] - [[package]] name = "idna" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" -dependencies = [ - "idna_adapter", - "smallvec", - "utf8_iter", -] - -[[package]] -name = "idna_adapter" -version = "1.2.2" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" dependencies = [ - "icu_normalizer", - "icu_properties", + "unicode-bidi", + "unicode-normalization", ] [[package]] @@ -501,12 +398,6 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" -[[package]] -name = "litemap" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" - [[package]] name = "log" version = "0.4.33" @@ -559,15 +450,6 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" -[[package]] -name = "potential_utf" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" -dependencies = [ - "zerovec", -] - [[package]] name = "proc-macro2" version = "1.0.106" @@ -588,9 +470,9 @@ dependencies = [ [[package]] name = "r-efi" -version = "6.0.0" +version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" [[package]] name = "redox_users" @@ -751,12 +633,6 @@ version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" -[[package]] -name = "stable_deref_trait" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" - [[package]] name = "strsim" version = "0.11.1" @@ -780,17 +656,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "synstructure" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "tempfile" version = "3.27.0" @@ -798,7 +663,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix", "windows-sys 0.61.2", @@ -825,27 +690,47 @@ dependencies = [ ] [[package]] -name = "tinystr" -version = "0.8.3" +name = "tinyvec" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" dependencies = [ - "displaydoc", - "zerovec", + "tinyvec_macros", ] +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "typenum" version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + [[package]] name = "untrusted" version = "0.9.0" @@ -870,22 +755,15 @@ dependencies = [ [[package]] name = "url" -version = "2.5.8" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" dependencies = [ "form_urlencoded", "idna", "percent-encoding", - "serde", ] -[[package]] -name = "utf8_iter" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" - [[package]] name = "utf8parse" version = "0.2.2" @@ -910,6 +788,15 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + [[package]] name = "webpki-roots" version = "0.26.11" @@ -1083,33 +970,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] -name = "writeable" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" - -[[package]] -name = "yoke" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" -dependencies = [ - "stable_deref_trait", - "yoke-derive", - "zerofrom", -] - -[[package]] -name = "yoke-derive" -version = "0.8.2" +name = "wit-bindgen" +version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "synstructure", -] +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" [[package]] name = "zerocopy" @@ -1131,65 +995,11 @@ dependencies = [ "syn", ] -[[package]] -name = "zerofrom" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" -dependencies = [ - "zerofrom-derive", -] - -[[package]] -name = "zerofrom-derive" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "synstructure", -] - [[package]] name = "zeroize" -version = "1.9.0" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" - -[[package]] -name = "zerotrie" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" -dependencies = [ - "displaydoc", - "yoke", - "zerofrom", -] - -[[package]] -name = "zerovec" -version = "0.11.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" -dependencies = [ - "yoke", - "zerofrom", - "zerovec-derive", -] - -[[package]] -name = "zerovec-derive" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" [[package]] name = "zmij" diff --git a/Cargo.toml b/Cargo.toml index 3bcfed0..2178c6b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ name = "fojin-cli" version = "0.2.1" edition = "2021" -rust-version = "1.74" +rust-version = "1.77" description = "fojin 跨藏对读 CLI —— 离线查询佛典汉文的梵/巴/藏平行 (offline, no login)" license = "MIT OR Apache-2.0" repository = "https://github.com/xr843/fojin-cli" @@ -20,8 +20,8 @@ name = "fojin" path = "src/main.rs" [dependencies] -clap = { version = "4", features = ["derive"] } -rusqlite = { version = "0.32.1", features = ["bundled"] } +clap = { version = "=4.5.53", features = ["derive"] } +rusqlite = { version = "=0.32.1", features = ["bundled"] } serde = { version = "1", features = ["derive"] } serde_json = "1" directories = "5" From 538bc8bbd262cb4bb0fb522f9331ad856a404c90 Mon Sep 17 00:00:00 2001 From: xr843 <137012659+xr843@users.noreply.github.com> Date: Fri, 10 Jul 2026 14:50:20 +0800 Subject: [PATCH 07/13] fix: harden dataset verification and updates --- .github/workflows/ci.yml | 1 + README.md | 4 +- src/cli.rs | 7 +- src/data.rs | 161 ++++++++++++++++++++++++++++++++++++++- tests/data.rs | 123 +++++++++++++++++++++++++++++- 5 files changed, 285 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a792dcf..95ce1ec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,7 @@ jobs: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@1.77.0 - run: cargo test --all --locked + - run: cargo install --path . --locked rust: runs-on: ubuntu-latest steps: diff --git a/README.md b/README.md index 4bf5fe0..3adbc0a 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ $ fojin parallel "色即是空" 通过 [crates.io](https://crates.io/crates/fojin-cli) 安装(命令为 `fojin`）： ```bash -cargo install fojin-cli +cargo install fojin-cli --locked ``` 没有 Rust 环境?一行脚本自动安装对应平台的预编译二进制(Linux x64 / macOS ARM+Intel)： @@ -39,7 +39,7 @@ curl -fsSL https://raw.githubusercontent.com/xr843/fojin-cli/master/install.sh | 也可从 [Releases](https://github.com/xr843/fojin-cli/releases/latest) 手动下载各平台二进制(含 Windows x64 zip),或从源码安装： ```bash -cargo install --git https://github.com/xr843/fojin-cli +cargo install --git https://github.com/xr843/fojin-cli --locked ``` 首次运行 `fojin parallel` 会自动下载对齐数据集(约 183 MB,带进度显示,见下方「数据集」),之后完全离线。 diff --git a/src/cli.rs b/src/cli.rs index 0dba11b..dfe1278 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -352,18 +352,13 @@ fn run_data(action: DataAction) -> Result { } DataAction::Update { data_dir } => { let path = data::resolve_data_path(data_dir)?; - if path.exists() { - std::fs::remove_file(&path)?; - } - data::ensure_data( + data::update_data( &path, - false, &data::DataSource { url: DATA_URL, sha256: DATA_SHA256, }, )?; - let _ = data::open_compatible_db(&path)?; println!("数据已更新: {}", path.display()); Ok(0) } diff --git a/src/data.rs b/src/data.rs index 9b0cc99..ddfa000 100644 --- a/src/data.rs +++ b/src/data.rs @@ -115,6 +115,44 @@ pub fn ensure_data(path: &Path, offline: bool, source: &DataSource) -> Result<() if let Some(parent) = path.parent() { std::fs::create_dir_all(parent).context("创建缓存目录失败")?; } + let raw = download_and_unpack(path, source)?; + write_atomic(path, &raw) +} + +/// Download a replacement dataset to a sibling candidate, validate it, then +/// replace the live dataset. Failures before replacement leave `path` intact. +pub fn update_data(path: &Path, source: &DataSource) -> Result<()> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).context("创建缓存目录失败")?; + } + + let candidate = sibling_path(path, ".candidate")?; + remove_candidate_artifacts(&candidate)?; + + let result = (|| { + let raw = download_and_unpack(path, source)?; + std::fs::write(&candidate, raw) + .with_context(|| format!("写入候选数据失败: {}", candidate.display()))?; + + let conn = open_read_only_db(&candidate)?; + verify_dataset(&conn)?; + drop(conn); + + replace_with_candidate(path, &candidate) + })(); + + match result { + Ok(()) => Ok(()), + Err(error) => { + if let Err(cleanup_error) = remove_candidate_artifacts(&candidate) { + return Err(error.context(format!("清理候选数据失败: {cleanup_error}"))); + } + Err(error) + } + } +} + +fn download_and_unpack(path: &Path, source: &DataSource) -> Result> { let gz = http_get(source.url).map_err(|e| { anyhow!( "{e:#}\n请手动下载:\n {}\n解压后放到: {}", @@ -129,8 +167,88 @@ pub fn ensure_data(path: &Path, offline: bool, source: &DataSource) -> Result<() path.display() )); } - let raw = gunzip(&gz)?; - write_atomic(path, &raw)?; + gunzip(&gz) +} + +fn sibling_path(path: &Path, suffix: &str) -> Result { + let file_name = path + .file_name() + .ok_or_else(|| anyhow!("数据路径没有文件名: {}", path.display()))?; + let mut sibling = file_name.to_os_string(); + sibling.push(suffix); + Ok(path.with_file_name(sibling)) +} + +fn remove_candidate_artifacts(candidate: &Path) -> Result<()> { + for suffix in ["", "-journal", "-shm", "-wal"] { + let artifact = if suffix.is_empty() { + candidate.to_path_buf() + } else { + sibling_path(candidate, suffix)? + }; + match std::fs::remove_file(&artifact) { + Ok(()) => {} + Err(ref error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(error) + .with_context(|| format!("删除候选数据失败: {}", artifact.display())) + } + } + } + Ok(()) +} + +fn replace_with_candidate(path: &Path, candidate: &Path) -> Result<()> { + if !path.exists() { + return std::fs::rename(candidate, path).with_context(|| { + format!( + "替换数据文件失败: {} -> {}", + candidate.display(), + path.display() + ) + }); + } + + // Windows does not allow rename to replace an existing destination. Move + // the live file aside first, then restore it if moving the candidate fails. + let backup = sibling_path(path, ".backup")?; + match std::fs::remove_file(&backup) { + Ok(()) => {} + Err(ref error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(error).with_context(|| format!("删除旧备份失败: {}", backup.display())) + } + } + std::fs::rename(path, &backup).with_context(|| { + format!( + "准备替换数据文件失败: {} -> {}", + path.display(), + backup.display() + ) + })?; + + if let Err(replace_error) = std::fs::rename(candidate, path) { + return match std::fs::rename(&backup, path) { + Ok(()) => Err(replace_error).with_context(|| { + format!( + "替换数据文件失败: {} -> {}", + candidate.display(), + path.display() + ) + }), + Err(restore_error) => Err(anyhow!( + "替换数据文件失败: {} -> {}; 恢复原数据失败: {} -> {}: {}", + candidate.display(), + path.display(), + backup.display(), + path.display(), + restore_error + )), + }; + } + + std::fs::remove_file(&backup) + .with_context(|| format!("删除旧数据备份失败: {}", backup.display()))?; Ok(()) } @@ -190,6 +308,7 @@ pub fn validate_compatibility(conn: &rusqlite::Connection) -> Result Result< }) } +fn require_trigram_fts(conn: &rusqlite::Connection) -> Result<()> { + let _: Option = conn + .query_row( + "SELECT 1 FROM parallels_fts WHERE parallels_fts MATCH ?1 LIMIT 1", + ["x"], + |row| row.get(0), + ) + .optional() + .map_err(|e| { + anyhow!( + "dataset incompatibility: required schema `parallels_fts` does not support FTS5 MATCH: {e}. Run `fojin data update`." + ) + })?; + + let sql: String = conn + .query_row( + "SELECT sql FROM sqlite_schema WHERE type = 'table' AND name = 'parallels_fts'", + [], + |row| row.get(0), + ) + .map_err(|e| { + anyhow!( + "dataset incompatibility: required schema `parallels_fts` has no table declaration: {e}. Run `fojin data update`." + ) + })?; + let declaration: String = sql + .chars() + .filter(|character| !character.is_ascii_whitespace()) + .flat_map(char::to_lowercase) + .collect(); + if !declaration.contains("usingfts5(") || !declaration.contains("tokenize='trigram'") { + return Err(anyhow!( + "dataset incompatibility: required schema `parallels_fts` must be an FTS5 table with tokenize='trigram'. Run `fojin data update`." + )); + } + Ok(()) +} + fn require_expected_meta(conn: &rusqlite::Connection, key: &str, expected: &str) -> Result { let got = conn .query_row("SELECT value FROM meta WHERE key=?1", [key], |row| row.get::<_, String>(0)) diff --git a/tests/data.rs b/tests/data.rs index 1b2bd4e..fe3d08e 100644 --- a/tests/data.rs +++ b/tests/data.rs @@ -1,5 +1,5 @@ use fojin_cli::data::{ - ensure_data, gunzip, open_compatible_db, open_read_only_db, resolve_data_path, + ensure_data, gunzip, open_compatible_db, open_read_only_db, resolve_data_path, update_data, validate_compatibility, verify_dataset, verify_sha256, DataSource, EXPECTED_DATA_VERSION, EXPECTED_NORM_RULESET, }; @@ -258,6 +258,35 @@ fn compatibility_rejects_missing_parallels_fts() { assert!(err.contains("fojin data update"), "got: {err}"); } +#[test] +fn compatibility_rejects_parallels_fts_lookalike_table() { + let conn = compatible_conn(); + conn.execute("DROP TABLE parallels_fts", []).unwrap(); + conn.execute("CREATE TABLE parallels_fts (zh_norm TEXT NOT NULL)", []) + .unwrap(); + + let err = validate_compatibility(&conn).unwrap_err().to_string(); + + assert!(err.contains("parallels_fts"), "got: {err}"); + assert!(err.contains("fojin data update"), "got: {err}"); +} + +#[test] +fn compatibility_rejects_non_trigram_parallels_fts() { + let conn = compatible_conn(); + conn.execute("DROP TABLE parallels_fts", []).unwrap(); + conn.execute( + "CREATE VIRTUAL TABLE parallels_fts USING fts5(zh_norm, content='parallels', content_rowid='id')", + [], + ) + .unwrap(); + + let err = validate_compatibility(&conn).unwrap_err().to_string(); + + assert!(err.contains("parallels_fts"), "got: {err}"); + assert!(err.contains("fojin data update"), "got: {err}"); +} + #[test] fn compatibility_rejects_missing_query_required_parallels_column() { let conn = rusqlite::Connection::open_in_memory().unwrap(); @@ -328,6 +357,98 @@ fn verify_dataset_accepts_compatible_read_only_db() { assert_eq!(got.norm_ruleset, EXPECTED_NORM_RULESET); } +#[test] +fn verify_dataset_rejects_quick_check_diagnostics() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("data.sqlite"); + let conn = rusqlite::Connection::open(&path).unwrap(); + fojin_cli::schema::init_schema(&conn).unwrap(); + insert_compat_meta(&conn); + conn.execute("CREATE TABLE quick_check_probe (value TEXT)", []) + .unwrap(); + let meta_rootpage: i64 = conn + .query_row( + "SELECT rootpage FROM sqlite_schema WHERE name = 'meta'", + [], + |row| row.get(0), + ) + .unwrap(); + conn.execute_batch("PRAGMA writable_schema = ON").unwrap(); + conn.execute( + "UPDATE sqlite_schema SET rootpage = ?1 WHERE name = 'quick_check_probe'", + [meta_rootpage], + ) + .unwrap(); + conn.execute_batch("PRAGMA writable_schema = OFF").unwrap(); + drop(conn); + + let conn = open_read_only_db(&path).unwrap(); + let err = verify_dataset(&conn).unwrap_err().to_string(); + + assert!(err.contains("PRAGMA quick_check failed"), "got: {err}"); + assert!(err.contains("2nd reference to page"), "got: {err}"); +} + +#[test] +fn update_data_preserves_live_dataset_when_candidate_fails_compatibility() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("data.sqlite"); + std::fs::write(&path, b"live dataset").unwrap(); + + let source_path = dir.path().join("candidate.sqlite"); + let conn = rusqlite::Connection::open(&source_path).unwrap(); + fojin_cli::schema::init_schema(&conn).unwrap(); + insert_compat_meta(&conn); + conn.execute("UPDATE meta SET value = 'v0' WHERE key = 'version'", []) + .unwrap(); + drop(conn); + + let mut enc = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + enc.write_all(&std::fs::read(&source_path).unwrap()) + .unwrap(); + let gz = enc.finish().unwrap(); + std::fs::remove_file(&source_path).unwrap(); + + use sha2::{Digest, Sha256}; + let mut hash = Sha256::new(); + hash.update(&gz); + let sha: String = hash + .finalize() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect(); + + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = [0u8; 4096]; + let _ = std::io::Read::read(&mut stream, &mut request); + let head = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + gz.len() + ); + stream.write_all(head.as_bytes()).unwrap(); + stream.write_all(&gz).unwrap(); + }); + let source_url = format!("http://127.0.0.1:{port}/data.gz"); + let source = DataSource { + url: &source_url, + sha256: &sha, + }; + + let err = update_data(&path, &source).unwrap_err().to_string(); + server.join().unwrap(); + + assert!(err.contains("dataset incompatibility"), "got: {err}"); + assert_eq!(std::fs::read(&path).unwrap(), b"live dataset"); + let entries: Vec<_> = std::fs::read_dir(dir.path()) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect(); + assert_eq!(entries, vec![std::ffi::OsString::from("data.sqlite")]); +} + fn compatible_conn() -> rusqlite::Connection { let conn = rusqlite::Connection::open_in_memory().unwrap(); fojin_cli::schema::init_schema(&conn).unwrap(); From f50dc41dc5b26bbd6d8ae69e0dad22ebe6a00fdb Mon Sep 17 00:00:00 2001 From: xr843 <137012659+xr843@users.noreply.github.com> Date: Fri, 10 Jul 2026 15:11:21 +0800 Subject: [PATCH 08/13] fix: make dataset replacement atomic --- .github/workflows/ci.yml | 6 + src/data.rs | 132 ++++++++++------- tests/data.rs | 308 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 396 insertions(+), 50 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 95ce1ec..375a1a6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,12 @@ jobs: - run: cargo fmt --all --check - run: cargo clippy --all-targets -- -D warnings - run: cargo test --all + windows-data: + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - run: cargo test --test data --locked python-parity: runs-on: ubuntu-latest steps: diff --git a/src/data.rs b/src/data.rs index ddfa000..b149bc6 100644 --- a/src/data.rs +++ b/src/data.rs @@ -198,6 +198,20 @@ fn remove_candidate_artifacts(candidate: &Path) -> Result<()> { Ok(()) } +#[cfg(not(windows))] +fn replace_with_candidate(path: &Path, candidate: &Path) -> Result<()> { + // The candidate is a sibling, so rename atomically replaces the live path + // on Unix without first removing or moving it away. + std::fs::rename(candidate, path).with_context(|| { + format!( + "替换数据文件失败: {} -> {}", + candidate.display(), + path.display() + ) + }) +} + +#[cfg(windows)] fn replace_with_candidate(path: &Path, candidate: &Path) -> Result<()> { if !path.exists() { return std::fs::rename(candidate, path).with_context(|| { @@ -209,47 +223,64 @@ fn replace_with_candidate(path: &Path, candidate: &Path) -> Result<()> { }); } - // Windows does not allow rename to replace an existing destination. Move - // the live file aside first, then restore it if moving the candidate fails. - let backup = sibling_path(path, ".backup")?; - match std::fs::remove_file(&backup) { - Ok(()) => {} - Err(ref error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => { - return Err(error).with_context(|| format!("删除旧备份失败: {}", backup.display())) + use std::ffi::c_void; + use std::os::windows::ffi::OsStrExt; + use std::ptr; + + #[link(name = "Kernel32")] + extern "system" { + fn ReplaceFileW( + replaced_file_name: *const u16, + replacement_file_name: *const u16, + backup_file_name: *const u16, + replace_flags: u32, + exclude: *mut c_void, + reserved: *mut c_void, + ) -> i32; + } + + fn wide_path(path: &Path) -> Result> { + let mut encoded: Vec = path.as_os_str().encode_wide().collect(); + if encoded.contains(&0) { + return Err(anyhow!("数据路径包含空字符: {}", path.display())); } + encoded.push(0); + Ok(encoded) } - std::fs::rename(path, &backup).with_context(|| { - format!( - "准备替换数据文件失败: {} -> {}", - path.display(), - backup.display() + + let replaced = wide_path(path)?; + let replacement = wide_path(candidate)?; + // SAFETY: both path buffers are NUL-terminated UTF-16 and remain alive for + // the call; optional and reserved pointers are null as ReplaceFileW requires. + let replaced_ok = unsafe { + ReplaceFileW( + replaced.as_ptr(), + replacement.as_ptr(), + ptr::null(), + 0, + ptr::null_mut(), + ptr::null_mut(), ) - })?; + }; + if replaced_ok != 0 { + return Ok(()); + } - if let Err(replace_error) = std::fs::rename(candidate, path) { - return match std::fs::rename(&backup, path) { - Ok(()) => Err(replace_error).with_context(|| { - format!( - "替换数据文件失败: {} -> {}", - candidate.display(), - path.display() - ) - }), - Err(restore_error) => Err(anyhow!( - "替换数据文件失败: {} -> {}; 恢复原数据失败: {} -> {}: {}", - candidate.display(), - path.display(), - backup.display(), - path.display(), - restore_error - )), - }; + let native_error = std::io::Error::last_os_error(); + if path.exists() && !candidate.exists() { + return Ok(()); + } + if !path.exists() && candidate.exists() && std::fs::rename(candidate, path).is_ok() { + return Ok(()); } - std::fs::remove_file(&backup) - .with_context(|| format!("删除旧数据备份失败: {}", backup.display()))?; - Ok(()) + Err(native_error).with_context(|| { + format!( + "原子替换数据文件失败: {} -> {}", + candidate.display(), + path.display() + ) + }) } fn http_get(url: &str) -> Result> { @@ -415,19 +446,6 @@ fn require_schema(conn: &rusqlite::Connection, name: &str, sql: &str) -> Result< } fn require_trigram_fts(conn: &rusqlite::Connection) -> Result<()> { - let _: Option = conn - .query_row( - "SELECT 1 FROM parallels_fts WHERE parallels_fts MATCH ?1 LIMIT 1", - ["x"], - |row| row.get(0), - ) - .optional() - .map_err(|e| { - anyhow!( - "dataset incompatibility: required schema `parallels_fts` does not support FTS5 MATCH: {e}. Run `fojin data update`." - ) - })?; - let sql: String = conn .query_row( "SELECT sql FROM sqlite_schema WHERE type = 'table' AND name = 'parallels_fts'", @@ -444,11 +462,25 @@ fn require_trigram_fts(conn: &rusqlite::Connection) -> Result<()> { .filter(|character| !character.is_ascii_whitespace()) .flat_map(char::to_lowercase) .collect(); - if !declaration.contains("usingfts5(") || !declaration.contains("tokenize='trigram'") { + const EXPECTED_DECLARATION: &str = "createvirtualtableparallels_ftsusingfts5(zh_norm,content='parallels',content_rowid='id',tokenize='trigram')"; + if declaration != EXPECTED_DECLARATION { return Err(anyhow!( - "dataset incompatibility: required schema `parallels_fts` must be an FTS5 table with tokenize='trigram'. Run `fojin data update`." + "dataset incompatibility: required schema `parallels_fts` does not match the required FTS5 trigram declaration. Run `fojin data update`." )); } + + let _: Option = conn + .query_row( + "SELECT 1 FROM parallels_fts WHERE parallels_fts MATCH ?1 LIMIT 1", + ["x"], + |row| row.get(0), + ) + .optional() + .map_err(|e| { + anyhow!( + "dataset incompatibility: required schema `parallels_fts` does not support FTS5 MATCH: {e}. Run `fojin data update`." + ) + })?; Ok(()) } diff --git a/tests/data.rs b/tests/data.rs index fe3d08e..b3b5678 100644 --- a/tests/data.rs +++ b/tests/data.rs @@ -287,6 +287,43 @@ fn compatibility_rejects_non_trigram_parallels_fts() { assert!(err.contains("fojin data update"), "got: {err}"); } +#[test] +fn compatibility_rejects_default_tokenizer_fts_spoofing_trigram_declaration() { + let conn = compatible_conn(); + conn.execute("DROP TABLE parallels_fts", []).unwrap(); + conn.execute( + "CREATE VIRTUAL TABLE parallels_fts USING fts5( + zh_norm, + \"tokenize='trigram'\", + content='parallels', + content_rowid='id' + )", + [], + ) + .unwrap(); + + let err = validate_compatibility(&conn).unwrap_err().to_string(); + + assert!(err.contains("parallels_fts"), "got: {err}"); + assert!(err.contains("fojin data update"), "got: {err}"); +} + +#[test] +fn compatibility_rejects_parallels_fts_view_lookalike() { + let conn = compatible_conn(); + conn.execute("DROP TABLE parallels_fts", []).unwrap(); + conn.execute( + "CREATE VIEW parallels_fts AS SELECT id AS rowid, zh_norm FROM parallels", + [], + ) + .unwrap(); + + let err = validate_compatibility(&conn).unwrap_err().to_string(); + + assert!(err.contains("parallels_fts"), "got: {err}"); + assert!(err.contains("fojin data update"), "got: {err}"); +} + #[test] fn compatibility_rejects_missing_query_required_parallels_column() { let conn = rusqlite::Connection::open_in_memory().unwrap(); @@ -449,6 +486,277 @@ fn update_data_preserves_live_dataset_when_candidate_fails_compatibility() { assert_eq!(entries, vec![std::ffi::OsString::from("data.sqlite")]); } +#[test] +fn update_data_replaces_valid_existing_dataset() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("data.sqlite"); + std::fs::write(&path, b"old live dataset").unwrap(); + let gz = gzip_bytes(&replacement_database_bytes()); + let sha = sha256_hex(&gz); + + serve_update(&path, gz, &sha).unwrap(); + + assert_replacement_marker(&path); + assert_no_candidate_artifacts(&path); +} + +#[test] +fn update_data_installs_valid_dataset_when_target_is_absent() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("data.sqlite"); + let gz = gzip_bytes(&replacement_database_bytes()); + let sha = sha256_hex(&gz); + + serve_update(&path, gz, &sha).unwrap(); + + assert_replacement_marker(&path); + assert_no_candidate_artifacts(&path); +} + +#[test] +fn update_data_preserves_live_dataset_on_download_failure() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("data.sqlite"); + std::fs::write(&path, b"old live dataset").unwrap(); + + let err = serve_update_response( + &path, + "500 Internal Server Error", + b"download failed".to_vec(), + "unused", + ) + .unwrap_err() + .to_string(); + + assert!(err.contains("下载失败"), "got: {err}"); + assert_eq!(std::fs::read(&path).unwrap(), b"old live dataset"); + assert_no_candidate_artifacts(&path); +} + +#[test] +fn update_data_preserves_live_dataset_on_checksum_mismatch() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("data.sqlite"); + std::fs::write(&path, b"old live dataset").unwrap(); + let gz = gzip_bytes(&replacement_database_bytes()); + + let err = serve_update(&path, gz, &"0".repeat(64)) + .unwrap_err() + .to_string(); + + assert!(err.contains("sha256"), "got: {err}"); + assert_eq!(std::fs::read(&path).unwrap(), b"old live dataset"); + assert_no_candidate_artifacts(&path); +} + +#[test] +fn update_data_preserves_live_dataset_on_decompression_failure() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("data.sqlite"); + std::fs::write(&path, b"old live dataset").unwrap(); + let invalid_gzip = b"not gzip data".to_vec(); + let sha = sha256_hex(&invalid_gzip); + + let err = serve_update(&path, invalid_gzip, &sha) + .unwrap_err() + .to_string(); + + assert!(err.contains("解压 gzip 失败"), "got: {err}"); + assert_eq!(std::fs::read(&path).unwrap(), b"old live dataset"); + assert_no_candidate_artifacts(&path); +} + +#[test] +fn update_data_preserves_live_dataset_when_candidate_is_corrupt() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("data.sqlite"); + std::fs::write(&path, b"old live dataset").unwrap(); + let database = compatible_database_bytes(|conn| { + conn.execute("CREATE TABLE quick_check_probe (value TEXT)", []) + .unwrap(); + let meta_rootpage: i64 = conn + .query_row( + "SELECT rootpage FROM sqlite_schema WHERE name = 'meta'", + [], + |row| row.get(0), + ) + .unwrap(); + conn.execute_batch("PRAGMA writable_schema = ON").unwrap(); + conn.execute( + "UPDATE sqlite_schema SET rootpage = ?1 WHERE name = 'quick_check_probe'", + [meta_rootpage], + ) + .unwrap(); + conn.execute_batch("PRAGMA writable_schema = OFF").unwrap(); + }); + let gz = gzip_bytes(&database); + let sha = sha256_hex(&gz); + + let err = serve_update(&path, gz, &sha).unwrap_err().to_string(); + + assert!(err.contains("PRAGMA quick_check failed"), "got: {err}"); + assert_eq!(std::fs::read(&path).unwrap(), b"old live dataset"); + assert_no_candidate_artifacts(&path); +} + +#[test] +fn update_data_cleans_stale_candidate_artifacts_on_failure() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("data.sqlite"); + std::fs::write(&path, b"old live dataset").unwrap(); + let candidate = sibling_path(&path, ".candidate"); + for suffix in ["", "-journal", "-shm", "-wal"] { + std::fs::write(sibling_path(&candidate, suffix), b"stale").unwrap(); + } + + serve_update_response( + &path, + "500 Internal Server Error", + b"download failed".to_vec(), + "unused", + ) + .unwrap_err(); + + assert_eq!(std::fs::read(&path).unwrap(), b"old live dataset"); + assert_no_candidate_artifacts(&path); +} + +#[test] +fn update_data_replaces_live_dataset_without_backup_path_dependency() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("data.sqlite"); + std::fs::write(&path, b"old live dataset").unwrap(); + let backup_sentinel = dir.path().join("data.sqlite.backup"); + std::fs::create_dir(&backup_sentinel).unwrap(); + + let database = compatible_database_bytes(|conn| { + conn.execute( + "INSERT INTO meta(key, value) VALUES ('marker', 'replacement')", + [], + ) + .unwrap(); + }); + let gz = gzip_bytes(&database); + let sha = sha256_hex(&gz); + + serve_update(&path, gz, &sha).unwrap(); + + let conn = open_read_only_db(&path).unwrap(); + verify_dataset(&conn).unwrap(); + let marker: String = conn + .query_row("SELECT value FROM meta WHERE key = 'marker'", [], |row| { + row.get(0) + }) + .unwrap(); + assert_eq!(marker, "replacement"); + assert!(backup_sentinel.is_dir()); + assert_no_candidate_artifacts(&path); +} + +fn replacement_database_bytes() -> Vec { + compatible_database_bytes(|conn| { + conn.execute( + "INSERT INTO meta(key, value) VALUES ('marker', 'replacement')", + [], + ) + .unwrap(); + }) +} + +fn assert_replacement_marker(path: &std::path::Path) { + let conn = open_read_only_db(path).unwrap(); + verify_dataset(&conn).unwrap(); + let marker: String = conn + .query_row("SELECT value FROM meta WHERE key = 'marker'", [], |row| { + row.get(0) + }) + .unwrap(); + assert_eq!(marker, "replacement"); +} + +fn assert_no_candidate_artifacts(path: &std::path::Path) { + let candidate = sibling_path(path, ".candidate"); + for suffix in ["", "-journal", "-shm", "-wal"] { + let artifact = sibling_path(&candidate, suffix); + assert!( + !artifact.exists(), + "artifact remains: {}", + artifact.display() + ); + } +} + +fn sibling_path(path: &std::path::Path, suffix: &str) -> PathBuf { + let mut name = path.file_name().unwrap().to_os_string(); + name.push(suffix); + path.with_file_name(name) +} + +fn compatible_database_bytes(configure: impl FnOnce(&rusqlite::Connection)) -> Vec { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("candidate.sqlite"); + let conn = rusqlite::Connection::open(&path).unwrap(); + fojin_cli::schema::init_schema(&conn).unwrap(); + insert_compat_meta(&conn); + configure(&conn); + drop(conn); + std::fs::read(path).unwrap() +} + +fn gzip_bytes(bytes: &[u8]) -> Vec { + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder.write_all(bytes).unwrap(); + encoder.finish().unwrap() +} + +fn sha256_hex(bytes: &[u8]) -> String { + use sha2::{Digest, Sha256}; + let mut hash = Sha256::new(); + hash.update(bytes); + hash.finalize() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} + +fn serve_update( + path: &std::path::Path, + body: Vec, + expected_sha256: &str, +) -> anyhow::Result<()> { + serve_update_response(path, "200 OK", body, expected_sha256) +} + +fn serve_update_response( + path: &std::path::Path, + status: &'static str, + body: Vec, + expected_sha256: &str, +) -> anyhow::Result<()> { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = [0u8; 4096]; + let _ = std::io::Read::read(&mut stream, &mut request); + let head = format!( + "HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len(), + ); + stream.write_all(head.as_bytes()).unwrap(); + let _ = stream.write_all(&body); + }); + let source_url = format!("http://127.0.0.1:{port}/data.gz"); + let source = DataSource { + url: &source_url, + sha256: expected_sha256, + }; + + let result = update_data(path, &source); + server.join().unwrap(); + result +} + fn compatible_conn() -> rusqlite::Connection { let conn = rusqlite::Connection::open_in_memory().unwrap(); fojin_cli::schema::init_schema(&conn).unwrap(); From 94038198466b67e7fcb0f9d0bb8d4a7d60348220 Mon Sep 17 00:00:00 2001 From: xr843 <137012659+xr843@users.noreply.github.com> Date: Fri, 10 Jul 2026 15:28:12 +0800 Subject: [PATCH 09/13] fix: handle windows replacement failures --- .github/workflows/ci.yml | 27 ++++- src/data.rs | 244 +++++++++++++++++++++++++++++++++------ tests/data.rs | 44 +++++-- 3 files changed, 270 insertions(+), 45 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 375a1a6..1871ee4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,32 @@ jobs: steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable - - run: cargo test --test data --locked + - run: cargo test --all --locked + release-build: + name: release build (${{ matrix.target }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - target: x86_64-unknown-linux-gnu + os: ubuntu-latest + archive_ext: tar.gz + - target: aarch64-apple-darwin + os: macos-latest + archive_ext: tar.gz + - target: x86_64-apple-darwin + os: macos-latest + archive_ext: tar.gz + - target: x86_64-pc-windows-msvc + os: windows-latest + archive_ext: zip + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - name: Install release target + run: rustup target add ${{ matrix.target }} + - run: cargo build --release --locked --target ${{ matrix.target }} python-parity: runs-on: ubuntu-latest steps: diff --git a/src/data.rs b/src/data.rs index b149bc6..0fd1ae3 100644 --- a/src/data.rs +++ b/src/data.rs @@ -1,7 +1,8 @@ use anyhow::{anyhow, Context, Result}; use rusqlite::{OpenFlags, OptionalExtension}; -use std::io::Read; +use std::io::{Read, Write}; use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::time::Duration; /// Connect timeout for the data download: fails fast if the release host is @@ -11,6 +12,7 @@ const CONNECT_TIMEOUT: Duration = Duration::from_secs(30); /// so this is generous for a slow-but-alive connection while still /// guaranteeing the CLI can never hang indefinitely. const READ_TIMEOUT: Duration = Duration::from_secs(900); +static CANDIDATE_SEQUENCE: AtomicU64 = AtomicU64::new(0); pub const EXPECTED_DATA_VERSION: &str = "v1"; pub const EXPECTED_NORM_RULESET: &str = "t2s-char-1to1-v1"; @@ -126,14 +128,17 @@ pub fn update_data(path: &Path, source: &DataSource) -> Result<()> { std::fs::create_dir_all(parent).context("创建缓存目录失败")?; } - let candidate = sibling_path(path, ".candidate")?; - remove_candidate_artifacts(&candidate)?; + let raw = download_and_unpack(path, source)?; + let (candidate, mut candidate_file) = create_candidate(path)?; + let write_result = candidate_file + .write_all(&raw) + .with_context(|| format!("写入候选数据失败: {}", candidate.display())); + drop(candidate_file); + if let Err(error) = write_result { + return Err(cleanup_candidate_error(&candidate, error)); + } let result = (|| { - let raw = download_and_unpack(path, source)?; - std::fs::write(&candidate, raw) - .with_context(|| format!("写入候选数据失败: {}", candidate.display()))?; - let conn = open_read_only_db(&candidate)?; verify_dataset(&conn)?; drop(conn); @@ -143,12 +148,7 @@ pub fn update_data(path: &Path, source: &DataSource) -> Result<()> { match result { Ok(()) => Ok(()), - Err(error) => { - if let Err(cleanup_error) = remove_candidate_artifacts(&candidate) { - return Err(error.context(format!("清理候选数据失败: {cleanup_error}"))); - } - Err(error) - } + Err(error) => Err(cleanup_candidate_error(&candidate, error)), } } @@ -179,6 +179,33 @@ fn sibling_path(path: &Path, suffix: &str) -> Result { Ok(path.with_file_name(sibling)) } +fn create_candidate(path: &Path) -> Result<(PathBuf, std::fs::File)> { + loop { + let sequence = CANDIDATE_SEQUENCE.fetch_add(1, Ordering::Relaxed); + let suffix = format!(".candidate.{}.{sequence}", std::process::id()); + let candidate = sibling_path(path, &suffix)?; + match std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&candidate) + { + Ok(file) => return Ok((candidate, file)), + Err(ref error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(error) => { + return Err(error) + .with_context(|| format!("创建候选数据失败: {}", candidate.display())) + } + } + } +} + +fn cleanup_candidate_error(candidate: &Path, error: anyhow::Error) -> anyhow::Error { + match remove_candidate_artifacts(candidate) { + Ok(()) => error, + Err(cleanup_error) => error.context(format!("清理候选数据失败: {cleanup_error}")), + } +} + fn remove_candidate_artifacts(candidate: &Path) -> Result<()> { for suffix in ["", "-journal", "-shm", "-wal"] { let artifact = if suffix.is_empty() { @@ -213,14 +240,20 @@ fn replace_with_candidate(path: &Path, candidate: &Path) -> Result<()> { #[cfg(windows)] fn replace_with_candidate(path: &Path, candidate: &Path) -> Result<()> { - if !path.exists() { - return std::fs::rename(candidate, path).with_context(|| { - format!( - "替换数据文件失败: {} -> {}", - candidate.display(), - path.display() - ) - }); + match std::fs::metadata(path) { + Ok(_) => {} + Err(ref error) if error.kind() == std::io::ErrorKind::NotFound => { + return std::fs::rename(candidate, path).with_context(|| { + format!( + "替换数据文件失败: {} -> {}", + candidate.display(), + path.display() + ) + }) + } + Err(error) => { + return Err(error).with_context(|| format!("检查现有数据文件失败: {}", path.display())) + } } use std::ffi::c_void; @@ -267,20 +300,67 @@ fn replace_with_candidate(path: &Path, candidate: &Path) -> Result<()> { } let native_error = std::io::Error::last_os_error(); - if path.exists() && !candidate.exists() { - return Ok(()); - } - if !path.exists() && candidate.exists() && std::fs::rename(candidate, path).is_ok() { - return Ok(()); + handle_windows_replace_failure(path, candidate, native_error, std::fs::rename) +} + +#[cfg(windows)] +fn handle_windows_replace_failure( + path: &Path, + candidate: &Path, + native_error: std::io::Error, + recover: F, +) -> Result<()> +where + F: FnOnce(&Path, &Path) -> std::io::Result<()>, +{ + const ERROR_UNABLE_TO_REMOVE_REPLACED: i32 = 1175; + const ERROR_UNABLE_TO_MOVE_REPLACEMENT: i32 = 1176; + const ERROR_UNABLE_TO_MOVE_REPLACEMENT_2: i32 = 1177; + + match native_error.raw_os_error() { + Some(ERROR_UNABLE_TO_MOVE_REPLACEMENT | ERROR_UNABLE_TO_MOVE_REPLACEMENT_2) => { + match recover(candidate, path) { + Ok(()) => Ok(()), + Err(recovery_error) => Err(anyhow!( + "ReplaceFileW failed for replaced `{}` and replacement `{}` with {}; recovery rename `{} -> {}` failed with {}", + path.display(), + candidate.display(), + describe_windows_error(&native_error), + candidate.display(), + path.display(), + describe_windows_error(&recovery_error) + )), + } + } + Some(ERROR_UNABLE_TO_REMOVE_REPLACED) => Err(windows_replace_error( + path, + candidate, + &native_error, + )), + _ => Err(windows_replace_error(path, candidate, &native_error)), } +} - Err(native_error).with_context(|| { - format!( - "原子替换数据文件失败: {} -> {}", - candidate.display(), - path.display() - ) - }) +#[cfg(windows)] +fn windows_replace_error( + path: &Path, + candidate: &Path, + native_error: &std::io::Error, +) -> anyhow::Error { + anyhow!( + "ReplaceFileW failed for replaced `{}` and replacement `{}` with {}", + path.display(), + candidate.display(), + describe_windows_error(native_error) + ) +} + +#[cfg(windows)] +fn describe_windows_error(error: &std::io::Error) -> String { + match error.raw_os_error() { + Some(code) => format!("Windows error {code}: {error}"), + None => error.to_string(), + } } fn http_get(url: &str) -> Result> { @@ -507,3 +587,101 @@ fn require_expected_meta(conn: &rusqlite::Connection, key: &str, expected: &str) Ok(got) } + +#[cfg(all(test, windows))] +mod windows_replace_failure_tests { + use super::*; + use std::cell::Cell; + use std::io; + + const LIVE: &str = r"C:\cache\data.sqlite"; + const CANDIDATE: &str = r"C:\cache\data.sqlite.candidate.1.1"; + + #[test] + fn error_1175_returns_native_failure_without_recovery() { + assert_names_untouched_error_skips_recovery(1175); + } + + #[test] + fn other_error_returns_native_failure_without_recovery() { + assert_names_untouched_error_skips_recovery(87); + } + + #[test] + fn error_1176_recovers_missing_live_path() { + assert_partial_failure_recovers(1176); + } + + #[test] + fn error_1177_recovers_missing_live_path() { + assert_partial_failure_recovers(1177); + } + + #[test] + fn error_1176_preserves_native_and_recovery_failures() { + assert_partial_failure_preserves_both_errors(1176); + } + + #[test] + fn error_1177_preserves_native_and_recovery_failures() { + assert_partial_failure_preserves_both_errors(1177); + } + + fn assert_names_untouched_error_skips_recovery(code: i32) { + let recovery_called = Cell::new(false); + let error = handle_windows_replace_failure( + Path::new(LIVE), + Path::new(CANDIDATE), + io::Error::from_raw_os_error(code), + |_, _| { + recovery_called.set(true); + Ok(()) + }, + ) + .unwrap_err(); + + assert!(!recovery_called.get()); + let detail = format!("{error:#}"); + assert!( + detail.contains(&format!("Windows error {code}")), + "{detail}" + ); + } + + fn assert_partial_failure_recovers(code: i32) { + let recovery_called = Cell::new(false); + handle_windows_replace_failure( + Path::new(LIVE), + Path::new(CANDIDATE), + io::Error::from_raw_os_error(code), + |from, to| { + recovery_called.set(true); + assert_eq!(from, Path::new(CANDIDATE)); + assert_eq!(to, Path::new(LIVE)); + Ok(()) + }, + ) + .unwrap(); + + assert!(recovery_called.get()); + } + + fn assert_partial_failure_preserves_both_errors(code: i32) { + let error = handle_windows_replace_failure( + Path::new(LIVE), + Path::new(CANDIDATE), + io::Error::from_raw_os_error(code), + |_, _| Err(io::Error::from_raw_os_error(5)), + ) + .unwrap_err(); + + let detail = format!("{error:#}"); + assert!( + detail.contains(&format!("Windows error {code}")), + "{detail}" + ); + assert!(detail.contains("Windows error 5"), "{detail}"); + assert!(detail.contains(CANDIDATE), "{detail}"); + assert!(detail.contains(LIVE), "{detail}"); + } +} diff --git a/tests/data.rs b/tests/data.rs index b3b5678..2355f9f 100644 --- a/tests/data.rs +++ b/tests/data.rs @@ -600,25 +600,31 @@ fn update_data_preserves_live_dataset_when_candidate_is_corrupt() { } #[test] -fn update_data_cleans_stale_candidate_artifacts_on_failure() { +fn update_data_preserves_foreign_candidate_artifacts_and_cleans_its_own() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("data.sqlite"); std::fs::write(&path, b"old live dataset").unwrap(); - let candidate = sibling_path(&path, ".candidate"); + let foreign_candidate = sibling_path(&path, ".candidate"); for suffix in ["", "-journal", "-shm", "-wal"] { - std::fs::write(sibling_path(&candidate, suffix), b"stale").unwrap(); + std::fs::write(sibling_path(&foreign_candidate, suffix), b"foreign").unwrap(); } + let database = compatible_database_bytes(|conn| { + conn.execute("UPDATE meta SET value = 'v0' WHERE key = 'version'", []) + .unwrap(); + }); + let gz = gzip_bytes(&database); + let sha = sha256_hex(&gz); - serve_update_response( - &path, - "500 Internal Server Error", - b"download failed".to_vec(), - "unused", - ) - .unwrap_err(); + serve_update(&path, gz, &sha).unwrap_err(); assert_eq!(std::fs::read(&path).unwrap(), b"old live dataset"); - assert_no_candidate_artifacts(&path); + for suffix in ["", "-journal", "-shm", "-wal"] { + assert_eq!( + std::fs::read(sibling_path(&foreign_candidate, suffix)).unwrap(), + b"foreign" + ); + } + assert_no_owned_candidate_artifacts(&path); } #[test] @@ -684,6 +690,22 @@ fn assert_no_candidate_artifacts(path: &std::path::Path) { artifact.display() ); } + assert_no_owned_candidate_artifacts(path); +} + +fn assert_no_owned_candidate_artifacts(path: &std::path::Path) { + let mut prefix = path.file_name().unwrap().to_os_string(); + prefix.push(".candidate."); + let prefix = prefix.to_string_lossy(); + let owned: Vec<_> = std::fs::read_dir(path.parent().unwrap()) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .filter(|name| name.to_string_lossy().starts_with(prefix.as_ref())) + .collect(); + assert!( + owned.is_empty(), + "owned candidate artifacts remain: {owned:?}" + ); } fn sibling_path(path: &std::path::Path, suffix: &str) -> PathBuf { From 9a378c8997b9157977f722be9d18d8334d281d02 Mon Sep 17 00:00:00 2001 From: xr843 <137012659+xr843@users.noreply.github.com> Date: Fri, 10 Jul 2026 17:57:41 +0800 Subject: [PATCH 10/13] test: model concurrent candidate ownership --- tests/data.rs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/tests/data.rs b/tests/data.rs index 2355f9f..f681b50 100644 --- a/tests/data.rs +++ b/tests/data.rs @@ -604,7 +604,7 @@ fn update_data_preserves_foreign_candidate_artifacts_and_cleans_its_own() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("data.sqlite"); std::fs::write(&path, b"old live dataset").unwrap(); - let foreign_candidate = sibling_path(&path, ".candidate"); + let foreign_candidate = sibling_path(&path, ".candidate.999999.42"); for suffix in ["", "-journal", "-shm", "-wal"] { std::fs::write(sibling_path(&foreign_candidate, suffix), b"foreign").unwrap(); } @@ -624,7 +624,22 @@ fn update_data_preserves_foreign_candidate_artifacts_and_cleans_its_own() { b"foreign" ); } - assert_no_owned_candidate_artifacts(&path); + let candidate_prefix = format!("{}.candidate.", path.file_name().unwrap().to_string_lossy()); + let remaining: std::collections::BTreeSet<_> = std::fs::read_dir(dir.path()) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .filter(|name| name.to_string_lossy().starts_with(&candidate_prefix)) + .collect(); + let expected: std::collections::BTreeSet<_> = ["", "-journal", "-shm", "-wal"] + .into_iter() + .map(|suffix| { + sibling_path(&foreign_candidate, suffix) + .file_name() + .unwrap() + .to_os_string() + }) + .collect(); + assert_eq!(remaining, expected); } #[test] From d51c617823c6a4238702ef40c6532e53066a6b26 Mon Sep 17 00:00:00 2001 From: xr843 <137012659+xr843@users.noreply.github.com> Date: Fri, 10 Jul 2026 18:06:45 +0800 Subject: [PATCH 11/13] docs: lock English cargo install command --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3adbc0a..4c46877 100644 --- a/README.md +++ b/README.md @@ -184,7 +184,7 @@ fojin parallel "<汉文短语>" --json --offline **fojin-cli** is an offline command-line tool: give it a Chinese Buddhist canonical passage, it returns the aligned Sanskrit/Tibetan parallels — from a local SQLite, in ~2 ms, fully offline after a one-time 183 MB data download. Single binary, no account, deterministic output. ```bash -cargo install fojin-cli # or: curl -fsSL https://raw.githubusercontent.com/xr843/fojin-cli/master/install.sh | sh +cargo install fojin-cli --locked # or: curl -fsSL https://raw.githubusercontent.com/xr843/fojin-cli/master/install.sh | sh fojin parallel "色即是空" # Sanskrit + Tibetan parallels with Taishō source refs fojin texts "心经" # fuzzy title search → Taishō numbers fojin cite T0251 # browse one text's alignments in canonical order From df5c4404240ca0725bfd8ca2613c665482c10be9 Mon Sep 17 00:00:00 2001 From: xr843 <137012659+xr843@users.noreply.github.com> Date: Fri, 10 Jul 2026 18:28:40 +0800 Subject: [PATCH 12/13] fix: verify fts content integrity --- README.md | 2 + examples/claude/README.md | 2 +- install.sh | 4 +- src/cli.rs | 7 +- src/data.rs | 203 +++++++++++++++++++++++++++++--------- tests/command.rs | 44 +++++++++ tests/data.rs | 28 ++++++ 7 files changed, 236 insertions(+), 54 deletions(-) diff --git a/README.md b/README.md index 4c46877..528f194 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,7 @@ fojin cite T0251 # 按编号列出一部经的对齐,经文顺序;--jua fojin data status # 本地数据状态(位置/大小/版本/行数统计) fojin data clean # 删除本地数据,释放 561 MB fojin data update # 重新下载数据(覆盖本地) +fojin data verify # 校验版本、SQLite 与 FTS 完整性 ``` `texts` 与 `cite` 支持与 `parallel` 一致的 `--json` / `--data-dir` / `--offline`; @@ -189,6 +190,7 @@ fojin parallel "色即是空" # Sanskrit + Tibetan parallels with Taish fojin texts "心经" # fuzzy title search → Taishō numbers fojin cite T0251 # browse one text's alignments in canonical order fojin data status # local dataset stats +fojin data verify # verify version, SQLite, and FTS integrity ``` - **Input**: Chinese only (traditional/simplified folded, punctuation ignored); literal substring matching over normalized text. 2-to-12-character phrases work best. diff --git a/examples/claude/README.md b/examples/claude/README.md index e5aaf28..df55bdf 100644 --- a/examples/claude/README.md +++ b/examples/claude/README.md @@ -7,7 +7,7 @@ 2. **`CLAUDE-snippet.md`** — 内容粘贴进你项目的 `CLAUDE.md`,让 Claude 在任何任务中 知道本机有 fojin-cli 可用、何时该用它(以及何时不该——语义检索/巴利/翻译请走在线 API)。 -前提:`fojin` 已安装且在 PATH 中(`cargo install fojin-cli` 或仓库根目录的 `install.sh`)。 +前提:`fojin` 已安装且在 PATH 中(`cargo install fojin-cli --locked` 或仓库根目录的 `install.sh`)。 建议先跑一次 `fojin data status` 确认数据就绪,agent 会话中即可全程 `--offline`。 其他 agent 框架(LangChain、OpenAI function calling 等)同理:fojin-cli 的 diff --git a/install.sh b/install.sh index 798245a..e1d4c40 100755 --- a/install.sh +++ b/install.sh @@ -23,7 +23,7 @@ case "$os" in Linux) case "$arch" in x86_64) target="x86_64-unknown-linux-gnu" ;; - *) die "暂无 Linux/$arch 预编译二进制,请改用: cargo install fojin-cli" ;; + *) die "暂无 Linux/$arch 预编译二进制,请改用: cargo install fojin-cli --locked" ;; esac ;; Darwin) case "$arch" in @@ -34,7 +34,7 @@ case "$os" in MINGW*|MSYS*|CYGWIN*) die "Windows 请从 Releases 页下载 zip: https://github.com/$REPO/releases/latest" ;; *) - die "暂不支持 $os,请改用: cargo install fojin-cli" ;; + die "暂不支持 $os,请改用: cargo install fojin-cli --locked" ;; esac # Resolve version: newest v* tag (the repo also publishes data-v* releases, diff --git a/src/cli.rs b/src/cli.rs index dfe1278..9a80688 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -54,7 +54,7 @@ pub enum Command { #[arg(long)] offline: bool, }, - /// 本地数据管理:状态 / 清理 / 重新下载 + /// 本地数据管理:状态 / 校验 / 清理 / 重新下载 Data { #[command(subcommand)] action: DataAction, @@ -135,7 +135,7 @@ pub enum DataAction { #[arg(long)] data_dir: Option, }, - /// 校验本地数据版本 / 规范 / SQLite 完整性(不触发下载) + /// 校验本地数据版本 / 规范 / SQLite 与 FTS 完整性(不触发下载) Verify { /// 机器可读 JSON 输出 #[arg(long)] @@ -370,8 +370,7 @@ fn run_data(action: DataAction) -> Result { path.display() ); } - let conn = data::open_read_only_db(&path)?; - let compatibility = data::verify_dataset(&conn)?; + let compatibility = data::verify_dataset_file(&path)?; if json { let v = serde_json::json!({ "ok": true, diff --git a/src/data.rs b/src/data.rs index 0fd1ae3..b6f172b 100644 --- a/src/data.rs +++ b/src/data.rs @@ -138,18 +138,12 @@ pub fn update_data(path: &Path, source: &DataSource) -> Result<()> { return Err(cleanup_candidate_error(&candidate, error)); } - let result = (|| { - let conn = open_read_only_db(&candidate)?; - verify_dataset(&conn)?; - drop(conn); - - replace_with_candidate(path, &candidate) - })(); - - match result { - Ok(()) => Ok(()), - Err(error) => Err(cleanup_candidate_error(&candidate, error)), + let validation_result = verify_dataset_file(&candidate).map(|_| ()); + if let Err(error) = validation_result { + return Err(cleanup_candidate_error(&candidate, error)); } + + finish_replacement(&candidate, replace_with_candidate(path, &candidate)) } fn download_and_unpack(path: &Path, source: &DataSource) -> Result> { @@ -206,6 +200,54 @@ fn cleanup_candidate_error(candidate: &Path, error: anyhow::Error) -> anyhow::Er } } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum CandidateCleanupPolicy { + Remove, + #[cfg(any(test, windows))] + Preserve, +} + +#[derive(Debug)] +struct ReplacementFailure { + error: anyhow::Error, + cleanup: CandidateCleanupPolicy, +} + +impl ReplacementFailure { + fn remove(error: anyhow::Error) -> Self { + Self { + error, + cleanup: CandidateCleanupPolicy::Remove, + } + } + + #[cfg(any(test, windows))] + fn preserve(error: anyhow::Error) -> Self { + Self { + error, + cleanup: CandidateCleanupPolicy::Preserve, + } + } +} + +type ReplacementResult = std::result::Result<(), ReplacementFailure>; + +fn finish_replacement(candidate: &Path, result: ReplacementResult) -> Result<()> { + match result { + Ok(()) => Ok(()), + Err(failure) => match failure.cleanup { + CandidateCleanupPolicy::Remove => { + Err(cleanup_candidate_error(candidate, failure.error)) + } + #[cfg(any(test, windows))] + CandidateCleanupPolicy::Preserve => Err(failure.error.context(format!( + "validated candidate preserved at `{}`", + candidate.display() + ))), + }, + } +} + fn remove_candidate_artifacts(candidate: &Path) -> Result<()> { for suffix in ["", "-journal", "-shm", "-wal"] { let artifact = if suffix.is_empty() { @@ -226,33 +268,40 @@ fn remove_candidate_artifacts(candidate: &Path) -> Result<()> { } #[cfg(not(windows))] -fn replace_with_candidate(path: &Path, candidate: &Path) -> Result<()> { +fn replace_with_candidate(path: &Path, candidate: &Path) -> ReplacementResult { // The candidate is a sibling, so rename atomically replaces the live path // on Unix without first removing or moving it away. - std::fs::rename(candidate, path).with_context(|| { - format!( - "替换数据文件失败: {} -> {}", - candidate.display(), - path.display() - ) - }) + std::fs::rename(candidate, path) + .with_context(|| { + format!( + "替换数据文件失败: {} -> {}", + candidate.display(), + path.display() + ) + }) + .map_err(ReplacementFailure::remove) } #[cfg(windows)] -fn replace_with_candidate(path: &Path, candidate: &Path) -> Result<()> { +fn replace_with_candidate(path: &Path, candidate: &Path) -> ReplacementResult { match std::fs::metadata(path) { Ok(_) => {} Err(ref error) if error.kind() == std::io::ErrorKind::NotFound => { - return std::fs::rename(candidate, path).with_context(|| { - format!( - "替换数据文件失败: {} -> {}", - candidate.display(), - path.display() - ) - }) + return std::fs::rename(candidate, path) + .with_context(|| { + format!( + "替换数据文件失败: {} -> {}", + candidate.display(), + path.display() + ) + }) + .map_err(ReplacementFailure::remove) } Err(error) => { - return Err(error).with_context(|| format!("检查现有数据文件失败: {}", path.display())) + return Err(ReplacementFailure::remove( + anyhow::Error::new(error) + .context(format!("检查现有数据文件失败: {}", path.display())), + )) } } @@ -281,8 +330,8 @@ fn replace_with_candidate(path: &Path, candidate: &Path) -> Result<()> { Ok(encoded) } - let replaced = wide_path(path)?; - let replacement = wide_path(candidate)?; + let replaced = wide_path(path).map_err(ReplacementFailure::remove)?; + let replacement = wide_path(candidate).map_err(ReplacementFailure::remove)?; // SAFETY: both path buffers are NUL-terminated UTF-16 and remain alive for // the call; optional and reserved pointers are null as ReplaceFileW requires. let replaced_ok = unsafe { @@ -309,7 +358,7 @@ fn handle_windows_replace_failure( candidate: &Path, native_error: std::io::Error, recover: F, -) -> Result<()> +) -> ReplacementResult where F: FnOnce(&Path, &Path) -> std::io::Result<()>, { @@ -321,23 +370,25 @@ where Some(ERROR_UNABLE_TO_MOVE_REPLACEMENT | ERROR_UNABLE_TO_MOVE_REPLACEMENT_2) => { match recover(candidate, path) { Ok(()) => Ok(()), - Err(recovery_error) => Err(anyhow!( - "ReplaceFileW failed for replaced `{}` and replacement `{}` with {}; recovery rename `{} -> {}` failed with {}", - path.display(), - candidate.display(), - describe_windows_error(&native_error), - candidate.display(), - path.display(), - describe_windows_error(&recovery_error) - )), + Err(recovery_error) => Err(ReplacementFailure::preserve(anyhow!( + "ReplaceFileW failed for replaced `{}` and replacement `{}` with {}; recovery rename `{} -> {}` failed with {}", + path.display(), + candidate.display(), + describe_windows_error(&native_error), + candidate.display(), + path.display(), + describe_windows_error(&recovery_error) + ))), } } - Some(ERROR_UNABLE_TO_REMOVE_REPLACED) => Err(windows_replace_error( + Some(ERROR_UNABLE_TO_REMOVE_REPLACED) => Err(ReplacementFailure::remove( + windows_replace_error(path, candidate, &native_error), + )), + _ => Err(ReplacementFailure::remove(windows_replace_error( path, candidate, &native_error, - )), - _ => Err(windows_replace_error(path, candidate, &native_error)), + ))), } } @@ -467,6 +518,33 @@ pub fn verify_dataset(conn: &rusqlite::Connection) -> Result Result { + let compatibility = { + let conn = open_read_only_db(path)?; + verify_dataset(&conn)? + }; + let conn = rusqlite::Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_WRITE) + .map_err(|error| { + anyhow!( + "dataset incompatibility: could not open existing dataset for FTS5 integrity-check: {error}. Run `fojin data update`." + ) + })?; + verify_fts_content_integrity(&conn)?; + Ok(compatibility) +} + +fn verify_fts_content_integrity(conn: &rusqlite::Connection) -> Result<()> { + const FTS_CONTENT_INTEGRITY_CHECK: &str = + "INSERT INTO parallels_fts(parallels_fts, rank) VALUES('integrity-check', 1)"; + conn.execute(FTS_CONTENT_INTEGRITY_CHECK, []) + .map(|_| ()) + .map_err(|error| { + anyhow!( + "dataset incompatibility: FTS5 integrity-check failed: {error}. Run `fojin data update`." + ) + }) +} + pub fn open_compatible_db(path: &Path) -> Result { let conn = open_read_only_db(path)?; validate_compatibility(&conn)?; @@ -588,6 +666,35 @@ fn require_expected_meta(conn: &rusqlite::Connection, key: &str, expected: &str) Ok(got) } +#[cfg(test)] +mod replacement_cleanup_tests { + use super::*; + + #[test] + fn preserve_policy_keeps_validated_candidate_and_reports_path() { + let dir = tempfile::tempdir().unwrap(); + let candidate = dir.path().join("data.sqlite.candidate.1.1"); + std::fs::write(&candidate, b"validated dataset").unwrap(); + + let error = finish_replacement( + &candidate, + Err(ReplacementFailure::preserve(anyhow!( + "replacement recovery failed" + ))), + ) + .unwrap_err(); + + assert_eq!(std::fs::read(&candidate).unwrap(), b"validated dataset"); + let detail = format!("{error:#}"); + assert!(detail.contains("validated candidate preserved"), "{detail}"); + assert!( + detail.contains(candidate.to_string_lossy().as_ref()), + "{detail}" + ); + assert!(detail.contains("replacement recovery failed"), "{detail}"); + } +} + #[cfg(all(test, windows))] mod windows_replace_failure_tests { use super::*; @@ -629,7 +736,7 @@ mod windows_replace_failure_tests { fn assert_names_untouched_error_skips_recovery(code: i32) { let recovery_called = Cell::new(false); - let error = handle_windows_replace_failure( + let failure = handle_windows_replace_failure( Path::new(LIVE), Path::new(CANDIDATE), io::Error::from_raw_os_error(code), @@ -641,7 +748,8 @@ mod windows_replace_failure_tests { .unwrap_err(); assert!(!recovery_called.get()); - let detail = format!("{error:#}"); + assert_eq!(failure.cleanup, CandidateCleanupPolicy::Remove); + let detail = format!("{:#}", failure.error); assert!( detail.contains(&format!("Windows error {code}")), "{detail}" @@ -667,7 +775,7 @@ mod windows_replace_failure_tests { } fn assert_partial_failure_preserves_both_errors(code: i32) { - let error = handle_windows_replace_failure( + let failure = handle_windows_replace_failure( Path::new(LIVE), Path::new(CANDIDATE), io::Error::from_raw_os_error(code), @@ -675,7 +783,8 @@ mod windows_replace_failure_tests { ) .unwrap_err(); - let detail = format!("{error:#}"); + assert_eq!(failure.cleanup, CandidateCleanupPolicy::Preserve); + let detail = format!("{:#}", failure.error); assert!( detail.contains(&format!("Windows error {code}")), "{detail}" diff --git a/tests/command.rs b/tests/command.rs index f60ad9b..cdff623 100644 --- a/tests/command.rs +++ b/tests/command.rs @@ -88,6 +88,10 @@ fn zero_limit_is_rejected_during_clap_parsing_before_data_access() { } fn write_fixture_db(dir: &std::path::Path) { + write_fixture_db_with_fts(dir, true); +} + +fn write_fixture_db_with_fts(dir: &std::path::Path, rebuild_fts: bool) { let conn = Connection::open(dir.join("data.sqlite")).unwrap(); init_schema(&conn).unwrap(); for (k, v) in [ @@ -147,6 +151,19 @@ fn write_fixture_db(dir: &std::path::Path) { ) .unwrap(); } + if rebuild_fts { + conn.execute( + "INSERT INTO parallels_fts(parallels_fts) VALUES('rebuild')", + [], + ) + .unwrap(); + } else { + conn.execute( + "INSERT INTO parallels_fts(parallels_fts) VALUES('delete-all')", + [], + ) + .unwrap(); + } } fn run_fojin(args: &[&str], dir: &std::path::Path) -> std::process::Output { @@ -204,6 +221,33 @@ fn data_verify_human_reports_compatibility_summary() { assert!(stdout.contains("t2s-char-1to1-v1"), "got: {stdout}"); } +#[test] +fn data_verify_rejects_empty_external_content_fts_index() { + let dir = tempfile::tempdir().unwrap(); + write_fixture_db_with_fts(dir.path(), false); + + let out = run_fojin(&["data", "verify"], dir.path()); + + assert_eq!(out.status.code(), Some(1)); + let stdout = String::from_utf8(out.stdout).unwrap(); + let stderr = String::from_utf8(out.stderr).unwrap(); + assert!(stdout.trim().is_empty(), "got stdout: {stdout}"); + assert!(stderr.contains("FTS5 integrity-check"), "got: {stderr}"); +} + +#[test] +fn data_verify_does_not_modify_database_bytes() { + let dir = tempfile::tempdir().unwrap(); + write_fixture_db(dir.path()); + let path = dir.path().join("data.sqlite"); + let before = std::fs::read(&path).unwrap(); + + let out = run_fojin(&["data", "verify"], dir.path()); + + assert_eq!(out.status.code(), Some(0)); + assert_eq!(std::fs::read(path).unwrap(), before); +} + #[test] fn data_verify_missing_data_exits_one_without_creating_file() { let dir = tempfile::tempdir().unwrap(); diff --git a/tests/data.rs b/tests/data.rs index f681b50..27adee2 100644 --- a/tests/data.rs +++ b/tests/data.rs @@ -599,6 +599,34 @@ fn update_data_preserves_live_dataset_when_candidate_is_corrupt() { assert_no_candidate_artifacts(&path); } +#[test] +fn update_data_rejects_candidate_with_empty_external_content_fts_index() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("data.sqlite"); + std::fs::write(&path, b"old live dataset").unwrap(); + let database = compatible_database_bytes(|conn| { + conn.execute( + "INSERT INTO parallels(zh_text, zh_norm, foreign_lang, foreign_text) \ + VALUES ('色即是空', '色即是空', 'sa', 'rupam sunyata')", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO parallels_fts(parallels_fts) VALUES('delete-all')", + [], + ) + .unwrap(); + }); + let gz = gzip_bytes(&database); + let sha = sha256_hex(&gz); + + let err = serve_update(&path, gz, &sha).unwrap_err().to_string(); + + assert!(err.contains("FTS5 integrity-check"), "got: {err}"); + assert_eq!(std::fs::read(&path).unwrap(), b"old live dataset"); + assert_no_candidate_artifacts(&path); +} + #[test] fn update_data_preserves_foreign_candidate_artifacts_and_cleans_its_own() { let dir = tempfile::tempdir().unwrap(); From e52ae9e70e98318987ac7f8b9825c7425b20f7bf Mon Sep 17 00:00:00 2001 From: xr843 <137012659+xr843@users.noreply.github.com> Date: Fri, 10 Jul 2026 18:39:06 +0800 Subject: [PATCH 13/13] fix: compile windows replacement recovery --- src/data.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/data.rs b/src/data.rs index b6f172b..868828d 100644 --- a/src/data.rs +++ b/src/data.rs @@ -349,7 +349,9 @@ fn replace_with_candidate(path: &Path, candidate: &Path) -> ReplacementResult { } let native_error = std::io::Error::last_os_error(); - handle_windows_replace_failure(path, candidate, native_error, std::fs::rename) + handle_windows_replace_failure(path, candidate, native_error, |from, to| { + std::fs::rename(from, to) + }) } #[cfg(windows)]