diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f2e3f1..c198730 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ All notable changes to this project will be documented in this file. Version 0.3.0 is prepared but has not been published. Its stabilization work includes: - Data verification: strengthen `fojin data verify` and dataset compatibility checks. +- Data pipeline: move installs and updates to a bounded, disk-streamed, checksum-first pipeline with hard end-to-end HTTP deadlines and rollback-safe Windows replacement backups. +- Data concurrency: serialize concurrent install, update, and clean operations per data directory, with full candidate validation before publication. - Query correctness: make short-query matching literal and remove duplicate parallel text within a match group and language. - SQLite safety: upgrade the bundled SQLite and verify its runtime version. - Release integrity: validate release versions, locked builds, archive contents, checksums, and installer verification. diff --git a/Cargo.lock b/Cargo.lock index db6e0a7..30b15f5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -85,6 +85,12 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + [[package]] name = "cc" version = "1.2.66" @@ -266,15 +272,6 @@ dependencies = [ "ureq", ] -[[package]] -name = "form_urlencoded" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" -dependencies = [ - "percent-encoding", -] - [[package]] name = "generic-array" version = "0.14.7" @@ -315,15 +312,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] -name = "idna" -version = "0.5.0" +name = "http" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "bytes", + "itoa", ] +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -658,48 +661,18 @@ dependencies = [ "syn", ] -[[package]] -name = "tinyvec" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - [[package]] name = "typenum" version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" -[[package]] -name = "unicode-bidi" -version = "0.3.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" - [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" -[[package]] -name = "unicode-normalization" -version = "0.1.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" -dependencies = [ - "tinyvec", -] - [[package]] name = "untrusted" version = "0.9.0" @@ -708,31 +681,38 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "ureq" -version = "2.12.1" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" +checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0" dependencies = [ "base64", - "flate2", "log", - "once_cell", + "percent-encoding", "rustls", "rustls-pki-types", - "url", - "webpki-roots 0.26.11", + "ureq-proto", + "utf8-zero", + "webpki-roots", ] [[package]] -name = "url" -version = "2.5.0" +name = "ureq-proto" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +checksum = "e994ba84b0bd1b1b0cf92878b7ef898a5c1760108fe7b6010327e274917a808c" dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", + "base64", + "http", + "httparse", + "log", ] +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + [[package]] name = "utf8parse" version = "0.2.2" @@ -766,15 +746,6 @@ dependencies = [ "wit-bindgen", ] -[[package]] -name = "webpki-roots" -version = "0.26.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" -dependencies = [ - "webpki-roots 1.0.8", -] - [[package]] name = "webpki-roots" version = "1.0.8" diff --git a/Cargo.toml b/Cargo.toml index 97bfc3c..61df9c7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,7 +25,7 @@ rusqlite = { version = "=0.40.1", default-features = false, features = ["bundled serde = { version = "1", features = ["derive"] } serde_json = "1" directories = "5" -ureq = "2" +ureq = { version = "=3.3.0", default-features = false, features = ["rustls"] } sha2 = "0.10" flate2 = "1" anyhow = "1" diff --git a/README.md b/README.md index 5b67df1..281b095 100644 --- a/README.md +++ b/README.md @@ -184,6 +184,10 @@ fojin parallel "<汉文短语>" --json --offline - 来源:Dharmamitra 的 [MITRA-parallel](https://github.com/dharmamitra/mitra-parallel) 对齐数据集([Nehrdich & Keutzer, 2026](https://arxiv.org/pdf/2601.06400)),以 GitHub Release(`data-v1`)形式分发;学术使用请引用原论文(BibTeX 见 [`DATA_LICENSE`](DATA_LICENSE))。 - 当前二进制把官方下载地址、SHA-256 与兼容元数据固定在 `data-v1`;`fojin data update` 只会重新获取这份固定数据,不会自动切换到未来的数据主版本。版本、归一化规则或查询所需 schema 不兼容的数据会被拒绝。 - 首次运行时下载,压缩包约 **183 MB**,解压后约 **561 MB**(SQLite)。下载后完全离线可用。 +- 安装和更新采用有界的磁盘流式传输,不再在内存中缓冲完整压缩包或数据库;压缩响应上限为 **256 MiB**,解压后的数据库上限为 **768 MiB**。更新期间可能临时需要现用数据库所占空间,外加约 **744 MiB** 暂存磁盘空间(约 183 MiB 压缩包 + 约 561 MiB 候选数据库)。 +- HTTP DNS 解析与连接超时均为 **30 秒**,响应头和响应体的空闲读取超时均为 **60 秒**,并以跨重定向、覆盖 DNS 到响应体读取的 **15 分钟端到端硬时限** 为上限。 +- 同一数据目录上的首次安装、更新和清理操作按 single-flight 串行执行;等待者最多等待 **20 分钟**。永久保留的 `data.sqlite.lock` 文件是无害的协调文件,`fojin data clean` 会有意保留它。 +- 离线查询行为及固定到 `data-v1` 的校验和契约保持不变。 - 当前不含巴利对齐(上游 MITRA-parallel 尚未覆盖巴利),默认输出不显示巴利行;显式 `--lang pi` 仍可查询(如实答「未找到对齐」)。程序的渲染路径可兼容未来新增语言行,但当前官方下载通道仍固定为 `data-v1`;上游出现新语言不代表当前二进制会自动获得它。**渲染兼容不等于官方更新通道无需升级**,未来数据版本可能要求升级二进制或明确切换数据发布。 - 许可:**CC BY-SA 4.0**(Dharmamitra + fojin)。 - 范围:仅含 MITRA 跨藏平行;fojin 自有的精选对齐(alignment_pairs)**未包含**在本数据集中。 @@ -219,6 +223,10 @@ fojin data verify # verify version, SQLite, and FTS integrity - **Build/install integrity**: building from crates.io or source requires Rust 1.95+ (MSRV 1.95). Starting with v0.3.0, the shell installer requires the target binary release to provide `SHA256SUMS` and verifies the archive before extraction. It fails closed for an older latest or explicitly selected release without that file, including the transition before v0.3.0 is published; use the currently published crates.io version or a source build instead. This does not state that v0.3.0 has been released. - **For AI agents**: pure-JSON stdout, semantic exit codes (`0` ok / `1` runtime / `2` usage), zero network with `--offline`. Ready-made Claude Code integration in [`examples/claude/`](examples/claude/). - **Data**: 908,620 zh↔sa/bo alignments from Dharmamitra's [MITRA-parallel](https://github.com/dharmamitra/mitra-parallel) dataset, redistributed under CC BY-SA 4.0. The official URL, checksum, and compatibility contract remain pinned to `data-v1`; rendering support for future language rows does not mean the official update channel can adopt them without a binary upgrade. Academic use: please cite [Nehrdich & Keutzer (2026)](https://arxiv.org/pdf/2601.06400) — BibTeX in [`DATA_LICENSE`](DATA_LICENSE). +- **Data transfer resources**: installs and updates use bounded, disk-streamed transfers and no longer buffer the complete archive or database in memory. Compressed responses are capped at **256 MiB** and decompressed databases at **768 MiB**. An update can temporarily require the live database plus roughly **744 MiB** of staging disk (about 183 MiB for the archive and 561 MiB for the candidate database). +- **Data timeouts**: HTTP DNS resolution and connection timeouts are both **30 seconds**, response-header and response-body idle-read timeouts are both **60 seconds**, and a **15-minute hard end-to-end deadline** spans redirects from DNS through the final body read. +- **Concurrent data operations**: initial install, update, and clean operations on one data directory are single-flight; a waiter may wait up to **20 minutes**. The permanent `data.sqlite.lock` file is harmless coordination state and intentionally survives `fojin data clean`. +- **Stable query contract**: offline queries and the checksum contract pinned to `data-v1` are unchanged. - **Not in scope**: semantic search, Pāli, translation — use [Dharmamitra](https://dharmamitra.org)'s online APIs for those; the two are complementary. - **License**: code MIT OR Apache-2.0; data CC BY-SA 4.0. diff --git a/docs/superpowers/plans/2026-07-11-streaming-data-download.md b/docs/superpowers/plans/2026-07-11-streaming-data-download.md new file mode 100644 index 0000000..fd44435 --- /dev/null +++ b/docs/superpowers/plans/2026-07-11-streaming-data-download.md @@ -0,0 +1,1177 @@ +# Streaming Data Download Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace the unbounded in-memory data download with a bounded two-stage disk pipeline and serialize data mutations so concurrent first runs perform exactly one download. + +**Architecture:** A private `data::transfer` module streams the authenticated gzip asset to a unique sibling and then streams a `MultiGzDecoder` into a unique SQLite candidate. A private `data::operation_lock` module provides a persistent, time-bounded OS file lock used by ensure, update, and clean; `data.rs` retains SQLite validation and platform-specific atomic publication. + +**Tech Stack:** Rust 2021 on Rust 1.95, ureq 3.3.0, sha2 0.10, flate2 1.1.9, rusqlite 0.40.1, standard-library `File::try_lock`, local `TcpListener` fixtures, GitHub Actions. + +## Global Constraints + +- Keep the MSRV at Rust 1.95 and add no locking dependency. +- Production limits are connect 30 seconds, idle read 60 seconds, total HTTP 15 minutes, lock wait 20 minutes, compressed 256 MiB, decompressed 768 MiB, and buffer 64 KiB. +- Send `Accept-Encoding: identity`; SHA-256 covers the downloaded release-asset bytes. +- First install and update both run schema/version, quick-check, and FTS integrity validation before publish. +- Existing data remains byte-for-byte unchanged on every failure before atomic publish. +- Ordinary queries never wait for the long-running mutation lock. +- Temporary files are same-directory, `create_new` siblings reserved and + cleaned by one cooperating operation; this is not an adversarial same-user + filesystem boundary. +- Do not add retries, resume support, mirrors, data discovery, schema changes, a tag, or a GitHub Release. +- Use TDD, focused commits, independent review, PR checks, and merge to `master`. + +--- + +## File Map + +- Create `src/data/transfer.rs`: download policy, strict response metadata, incremental hashing, bounded disk transfer, multi-member gzip decoding, and owned artifact cleanup. +- Create `src/data/operation_lock.rs`: permanent lock-file acquisition with bounded `try_lock` polling. +- Modify `src/data.rs`: module wiring, unified candidate validation/publication, lock integration, and clean API. +- Modify `src/cli.rs`: delegate `data clean` to the locked data-layer API. +- Modify `tests/data.rs`: end-to-end failure invariants and process-level concurrent installation. +- Modify `tests/command.rs`: clean/lock-file CLI behavior. +- Modify `README.md`: memory, disk, timeout, size, and concurrency contract. +- Modify `CHANGELOG.md`: unreleased bounded-transfer and concurrency entries. + +--- + +### Task 1: Build and adopt the bounded two-stage transfer pipeline + +**Files:** +- Create: `src/data/transfer.rs` +- Modify: `src/data.rs:1-205` +- Modify: `src/data.rs:419-445` +- Test: `src/data/transfer.rs` +- Test: `tests/data.rs:108-166` +- Test: `tests/data.rs:430-570` + +**Interfaces:** +- Consumes: `super::DataSource`, `super::Progress`, `super::download_notice`, `super::sibling_path`, and the existing replacement/verification functions in `data.rs`. +- Produces: `transfer::DownloadPolicy`, `transfer::PRODUCTION_POLICY`, `transfer::StagedCandidate`, and `transfer::stage_candidate(&Path, &DataSource, DownloadPolicy) -> Result`. + +- [ ] **Step 1: Add failing bounded-transfer tests** + +Create `src/data/transfer.rs`, wire it with `mod transfer;` in `src/data.rs`, and start with unit tests that specify exact-limit and over-limit behavior: + +```rust +#[cfg(test)] +mod tests { + use super::*; + use flate2::write::GzEncoder; + use flate2::Compression; + use std::io::{Cursor, Write}; + + #[test] + fn bounded_copy_accepts_exact_limit_and_rejects_one_more() { + let mut exact = Vec::new(); + assert_eq!(copy_bounded(Cursor::new(b"1234"), &mut exact, 4, "test").unwrap(), 4); + assert_eq!(exact, b"1234"); + + let mut oversized = Vec::new(); + let error = copy_bounded(Cursor::new(b"12345"), &mut oversized, 4, "test") + .unwrap_err() + .to_string(); + assert!(error.contains("4"), "got: {error}"); + } + + #[test] + fn unpack_accepts_exact_limit_and_rejects_expansion() { + let gzip = |body: &[u8]| { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(body).unwrap(); + encoder.finish().unwrap() + }; + + let mut exact = Vec::new(); + unpack_gzip(Cursor::new(gzip(b"1234")), &mut exact, 4).unwrap(); + assert_eq!(exact, b"1234"); + + let mut oversized = Vec::new(); + let error = unpack_gzip(Cursor::new(gzip(b"12345")), &mut oversized, 4) + .unwrap_err() + .to_string(); + assert!(error.contains("解压") && error.contains("4"), "got: {error}"); + } +} +``` + +In `tests/data.rs`, change `ensure_data_downloads_verifies_and_unpacks` to +serve `gzip_bytes(&replacement_database_bytes())` and finish by calling +`verify_dataset_file(&path)`. Add `first_install_rejects_incompatible_database` +using the original `b"fake sqlite payload"`; its SHA is valid, but +`ensure_data` must return a dataset-incompatibility error, leave `path` +absent, and leave no owned candidate artifacts. This integration test fails +against the current weak first-install path. + +- [ ] **Step 2: Run the new tests and confirm RED** + +Run: + +```bash +cargo +1.95.0 test --lib data::transfer::tests --locked +``` + +Expected: compilation fails because `copy_bounded` and `unpack_gzip` do not exist. + +- [ ] **Step 3: Implement policy, bounded copy, gzip handling, and owned artifacts** + +Add these concrete interfaces to `src/data/transfer.rs`: + +```rust +use super::{download_notice, sibling_path, DataSource, Progress}; +use anyhow::{anyhow, Context, Result}; +use flate2::read::MultiGzDecoder; +use sha2::{Digest, Sha256}; +use std::fs::{File, OpenOptions}; +use std::io::{self, Read, Seek, Write}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +const MIB: u64 = 1024 * 1024; +const BUFFER_SIZE: usize = 64 * 1024; +static ARTIFACT_SEQUENCE: AtomicU64 = AtomicU64::new(0); + +#[derive(Clone, Copy, Debug)] +pub(super) struct DownloadPolicy { + pub connect_timeout: Duration, + pub idle_read_timeout: Duration, + pub http_timeout: Duration, + pub max_compressed: u64, + pub max_uncompressed: u64, +} + +pub(super) const PRODUCTION_POLICY: DownloadPolicy = DownloadPolicy { + connect_timeout: Duration::from_secs(30), + idle_read_timeout: Duration::from_secs(60), + http_timeout: Duration::from_secs(15 * 60), + max_compressed: 256 * MIB, + max_uncompressed: 768 * MIB, +}; + +pub(super) struct StagedCandidate { + path: PathBuf, + armed: bool, +} + +struct OwnedCompressed { + path: PathBuf, + armed: bool, +} + +impl OwnedCompressed { + fn remove_now(mut self) -> Result<()> { + self.armed = false; + match std::fs::remove_file(&self.path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error) + .with_context(|| format!("删除压缩临时文件失败: {}", self.path.display())), + } + } + + fn cleanup_with(mut self, error: anyhow::Error) -> anyhow::Error { + self.armed = false; + match std::fs::remove_file(&self.path) { + Ok(()) => error, + Err(cleanup) if cleanup.kind() == io::ErrorKind::NotFound => error, + Err(cleanup) => error.context(format!( + "清理压缩临时文件失败: {}: {cleanup}", + self.path.display() + )), + } + } +} + +impl Drop for OwnedCompressed { + fn drop(&mut self) { + if self.armed { + let _ = std::fs::remove_file(&self.path); + } + } +} + +impl StagedCandidate { + pub(super) fn path(&self) -> &Path { + &self.path + } + + pub(super) fn publish_succeeded(mut self) { + self.armed = false; + } + + pub(super) fn preserve(mut self) { + self.armed = false; + } + + pub(super) fn cleanup_with(mut self, error: anyhow::Error) -> anyhow::Error { + self.armed = false; + match remove_artifact_family(&self.path) { + Ok(()) => error, + Err(cleanup) => error.context(format!("清理候选数据失败: {cleanup}")), + } + } +} + +impl Drop for StagedCandidate { + fn drop(&mut self) { + if self.armed { + let _ = remove_artifact_family(&self.path); + } + } +} + +fn unique_path(live_path: &Path, role: &str, extension: &str) -> Result { + let sequence = ARTIFACT_SEQUENCE.fetch_add(1, Ordering::Relaxed); + sibling_path( + live_path, + &format!(".{role}.{}.{sequence}{extension}", std::process::id()), + ) +} + +fn create_compressed_artifact(live_path: &Path) -> Result<(OwnedCompressed, File)> { + loop { + let path = unique_path(live_path, "download", ".gz")?; + match OpenOptions::new() + .read(true) + .write(true) + .create_new(true) + .open(&path) + { + Ok(file) => return Ok((OwnedCompressed { path, armed: true }, file)), + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => continue, + Err(error) => { + return Err(error) + .with_context(|| format!("创建压缩临时文件失败: {}", path.display())); + } + } + } +} + +fn create_candidate_artifact(live_path: &Path) -> Result<(StagedCandidate, File)> { + loop { + let path = unique_path(live_path, "candidate", "")?; + match OpenOptions::new().write(true).create_new(true).open(&path) { + Ok(file) => return Ok((StagedCandidate { path, armed: true }, file)), + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => continue, + Err(error) => { + return Err(error) + .with_context(|| format!("创建候选数据失败: {}", path.display())); + } + } + } +} + +fn remove_artifact_family(path: &Path) -> Result<()> { + for suffix in ["", "-journal", "-shm", "-wal"] { + let artifact = if suffix.is_empty() { + path.to_path_buf() + } else { + sibling_path(path, suffix)? + }; + match std::fs::remove_file(&artifact) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => { + return Err(error) + .with_context(|| format!("删除候选数据失败: {}", artifact.display())); + } + } + } + Ok(()) +} + +fn copy_bounded( + mut reader: impl Read, + mut writer: impl Write, + maximum: u64, + label: &str, +) -> Result { + let mut limited = reader.by_ref().take(maximum.saturating_add(1)); + let copied = io::copy(&mut limited, &mut writer) + .with_context(|| format!("{label}失败"))?; + if copied > maximum { + return Err(anyhow!("{label}超过限制: 最大 {maximum} 字节")); + } + Ok(copied) +} + +fn unpack_gzip(reader: impl Read, writer: impl Write, maximum: u64) -> Result { + let decoder = MultiGzDecoder::new(reader); + copy_bounded(decoder, writer, maximum, "解压 gzip") +} +``` + +The sequence is consumed before each open attempt; an `AlreadyExists` retry +therefore always receives a new name and cannot loop on a stale artifact. + +- [ ] **Step 4: Implement streaming download and candidate staging** + +Add `stage_candidate` and keep the HTTP reader/file loop explicit so SHA, +progress, and the compressed limit share one byte count: + +```rust +pub(super) fn stage_candidate( + live_path: &Path, + source: &DataSource<'_>, + policy: DownloadPolicy, +) -> Result { + let (compressed_guard, mut compressed_file) = create_compressed_artifact(live_path)?; + let staged = stage_candidate_inner( + live_path, + source, + policy, + &mut compressed_file, + ); + match staged { + Ok(candidate) => { + compressed_guard.remove_now()?; + Ok(candidate) + } + Err(error) => Err(compressed_guard.cleanup_with(error)), + } +} + +fn stage_candidate_inner( + live_path: &Path, + source: &DataSource<'_>, + policy: DownloadPolicy, + compressed_file: &mut File, +) -> Result { + let agent = ureq::AgentBuilder::new() + .timeout_connect(policy.connect_timeout) + .timeout_read(policy.idle_read_timeout) + .timeout(policy.http_timeout) + .build(); + let response = agent + .get(source.url) + .set("Accept-Encoding", "identity") + .call() + .map_err(|error| { + anyhow!( + "下载失败: {}: {error}\n请手动下载:\n {}\n解压后放到: {}", + source.url, + source.url, + live_path.display() + ) + })?; + let declared = basic_declared_length(&response, policy.max_compressed)?; + eprintln!("{}", download_notice(declared)); + + let mut reader = response.into_reader(); + let mut progress = Progress::new(declared); + let mut digest = Sha256::new(); + let mut received = 0_u64; + let mut buffer = [0_u8; BUFFER_SIZE]; + loop { + let count = reader.read(&mut buffer).context("读取响应失败")?; + if count == 0 { + break; + } + received = received + .checked_add(count as u64) + .ok_or_else(|| anyhow!("下载大小溢出"))?; + if received > policy.max_compressed { + return Err(anyhow!( + "下载数据超过限制: 最大 {} 字节", + policy.max_compressed + )); + } + digest.update(&buffer[..count]); + compressed_file + .write_all(&buffer[..count]) + .context("写入压缩临时文件失败")?; + if let Some(message) = progress.advance(count as u64) { + eprintln!("{message}"); + } + } + require_declared_length(declared, received)?; + let actual = digest.finalize(); + require_digest(actual.as_ref(), source.sha256).map_err(|error| { + anyhow!( + "{error}\n请手动下载:\n {}\n解压后放到: {}", + source.url, + live_path.display() + ) + })?; + compressed_file.flush().context("刷新压缩临时文件失败")?; + compressed_file.rewind().context("重置压缩临时文件失败")?; + + let (candidate_guard, mut candidate_file) = + create_candidate_artifact(live_path)?; + let unpacked = (|| -> Result<()> { + unpack_gzip( + &mut *compressed_file, + &mut candidate_file, + policy.max_uncompressed, + )?; + candidate_file.flush().context("刷新候选数据失败")?; + candidate_file.sync_all().context("同步候选数据失败")?; + Ok(()) + })(); + drop(candidate_file); + match unpacked { + Ok(()) => Ok(candidate_guard), + Err(error) => Err(candidate_guard.cleanup_with(error)), + } +} +``` + +Define the Task 1 metadata and digest helpers as: + +```rust +fn basic_declared_length(response: &ureq::Response, maximum: u64) -> Result> { + let Some(value) = response.header("Content-Length") else { + return Ok(None); + }; + let parsed = value + .parse::() + .with_context(|| format!("无效 Content-Length: {value}"))?; + if parsed > maximum { + return Err(anyhow!( + "Content-Length 超过下载限制: {parsed} > {maximum}" + )); + } + Ok(Some(parsed)) +} + +fn require_declared_length(declared: Option, received: u64) -> Result<()> { + if let Some(expected) = declared { + if expected != received { + return Err(anyhow!( + "响应长度不符: Content-Length={expected}, received={received}" + )); + } + } + Ok(()) +} + +fn require_digest(actual: &[u8], expected_hex: &str) -> Result<()> { + if expected_hex.len() != 64 || !expected_hex.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return Err(anyhow!("配置的 SHA-256 格式无效")); + } + let actual_hex: String = actual.iter().map(|byte| format!("{byte:02x}")).collect(); + if !actual_hex.eq_ignore_ascii_case(expected_hex) { + return Err(anyhow!("下载校验失败(sha256 不符)")); + } + Ok(()) +} +``` + +Task 2 replaces the single-value lookup with strict duplicate and transfer- +encoding handling while retaining the latter two helpers unchanged. + +- [ ] **Step 5: Replace the Vec pipeline with unified candidate publication** + +In `src/data.rs`, delete production use of `download_and_unpack`, `http_get`, +and fixed-name `write_atomic`. Keep the public small-byte helpers for source +compatibility. Add: + +```rust +mod transfer; + +fn install_candidate(path: &Path, source: &DataSource<'_>) -> Result<()> { + let candidate = + transfer::stage_candidate(path, source, transfer::PRODUCTION_POLICY)?; + if let Err(error) = verify_dataset_file(candidate.path()).map(|_| ()) { + return Err(candidate.cleanup_with(error)); + } + if let Err(error) = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(candidate.path()) + .and_then(|file| file.sync_all()) + .with_context(|| format!("同步候选数据失败: {}", candidate.path().display())) + { + return Err(candidate.cleanup_with(error)); + } + let candidate_path = candidate.path().to_path_buf(); + finish_replacement(candidate, replace_with_candidate(path, &candidate_path)) +} +``` + +Change `finish_replacement` to consume `StagedCandidate`. On success call +`publish_succeeded`; on remove-policy failure call `cleanup_with`; on +preserve-policy failure call `preserve` and retain the exact path in context. +Make both `ensure_data` and `update_data` call `install_candidate`, so first +installation receives full SQLite/FTS verification. + +- [ ] **Step 6: Run targeted and regression tests** + +Run: + +```bash +cargo +1.95.0 test --lib data::transfer::tests --locked +cargo +1.95.0 test --test data --locked +cargo +1.95.0 test --all --locked +``` + +Expected: all tests pass; the Rust total is at least 102 after the two new +bounded-transfer tests. + +- [ ] **Step 7: Commit the bounded pipeline** + +```bash +git add src/data.rs src/data/transfer.rs tests/data.rs +git commit -m "fix(data): stream verified downloads to disk" +``` + +--- + +### Task 2: Enforce strict HTTP metadata and timeout behavior + +**Files:** +- Modify: `src/data/transfer.rs` +- Test: `src/data/transfer.rs` + +**Interfaces:** +- Consumes: `DownloadPolicy` and `stage_candidate` from Task 1. +- Produces: strict `declared_length(&ureq::Response, u64) -> Result>` and stable timeout/HTTP error contexts used by all transfers. + +- [ ] **Step 1: Add failing raw-HTTP tests** + +Add a reusable one-response `TcpListener` fixture inside the transfer test +module and tests with small injected policies for: + +```rust +#[test] +fn duplicate_content_length_is_rejected() { + let response = b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\nContent-Length: 4\r\nConnection: close\r\n\r\ntest"; + let error = stage_from_raw_response(response, policy_for_tests()) + .unwrap_err() + .to_string(); + assert!(error.contains("Content-Length") && error.contains("重复"), "got: {error}"); +} + +#[test] +fn declared_length_must_match_received_body() { + let response = b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\nConnection: close\r\n\r\ntest"; + let error = stage_from_raw_response(response, policy_for_tests()) + .unwrap_err() + .to_string(); + assert!(error.contains("长度") || error.contains("读取响应失败"), "got: {error}"); +} + +#[test] +fn total_timeout_stops_a_non_idle_dribble() { + let policy = DownloadPolicy { + connect_timeout: Duration::from_millis(200), + idle_read_timeout: Duration::from_millis(150), + http_timeout: Duration::from_millis(300), + max_compressed: 1024, + max_uncompressed: 4096, + }; + let error = stage_from_dribbling_server(Duration::from_millis(50), policy) + .unwrap_err() + .to_string(); + assert!(error.contains("读取响应失败") || error.contains("timed out"), "got: {error}"); +} +``` + +Define the test helpers in the same module so no external network is used: + +```rust +fn policy_for_tests() -> DownloadPolicy { + DownloadPolicy { + connect_timeout: Duration::from_millis(200), + idle_read_timeout: Duration::from_millis(200), + http_timeout: Duration::from_secs(2), + max_compressed: 1024, + max_uncompressed: 4096, + } +} + +fn stage_from_raw_response(response: &[u8], policy: DownloadPolicy) -> Result<()> { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let response = response.to_vec(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = [0_u8; 4096]; + let _ = stream.read(&mut request); + stream.write_all(&response).unwrap(); + }); + let directory = tempfile::tempdir().unwrap(); + let live = directory.path().join("data.sqlite"); + let url = format!("http://{address}/data.gz"); + let sha = "0".repeat(64); + let result = stage_candidate(&live, &DataSource { url: &url, sha256: &sha }, policy) + .map(drop); + server.join().unwrap(); + result +} + +fn stage_from_dribbling_server(interval: Duration, policy: DownloadPolicy) -> Result<()> { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = [0_u8; 4096]; + let _ = stream.read(&mut request); + stream + .write_all(b"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n") + .unwrap(); + for _ in 0..20 { + if stream.write_all(b"x").is_err() { + break; + } + if stream.flush().is_err() { + break; + } + std::thread::sleep(interval); + } + }); + let directory = tempfile::tempdir().unwrap(); + let live = directory.path().join("data.sqlite"); + let url = format!("http://{address}/data.gz"); + let sha = "0".repeat(64); + let result = stage_candidate(&live, &DataSource { url: &url, sha256: &sha }, policy) + .map(drop); + server.join().unwrap(); + result +} +``` + +Also add tests for an oversized declared length rejected before body reads, +missing length, chunked `max + 1`, an idle body pause, and capture the request +to assert `Accept-Encoding: identity`. + +- [ ] **Step 2: Run the strict HTTP tests and confirm RED** + +Run: + +```bash +cargo +1.95.0 test --lib data::transfer::tests --locked +``` + +Expected: duplicate-length, chunked-limit, and timeout assertions fail against +the basic Task 1 metadata implementation. + +- [ ] **Step 3: Implement strict response metadata** + +Replace `basic_declared_length` with: + +```rust +fn declared_length(response: &ureq::Response, maximum: u64) -> Result> { + if !response.all("Transfer-Encoding").is_empty() { + return Ok(None); + } + let values = response.all("Content-Length"); + match values.as_slice() { + [] => Ok(None), + [value] => { + let parsed = value + .parse::() + .with_context(|| format!("无效 Content-Length: {value}"))?; + if parsed > maximum { + return Err(anyhow!( + "Content-Length 超过下载限制: {parsed} > {maximum}" + )); + } + Ok(Some(parsed)) + } + _ => Err(anyhow!("响应包含重复 Content-Length")), + } +} + +fn require_declared_length(declared: Option, received: u64) -> Result<()> { + if let Some(expected) = declared { + if expected != received { + return Err(anyhow!( + "响应长度不符: Content-Length={expected}, received={received}" + )); + } + } + Ok(()) +} +``` + +Keep the actual compressed counter authoritative for missing/chunked lengths. +Map body `TimedOut` and `UnexpectedEof` without discarding their source error, +so tests and users can distinguish timeout from truncation. + +- [ ] **Step 4: Add gzip integrity edge tests** + +Add tests for two concatenated gzip members, a truncated trailer, modified +CRC, and trailing non-gzip bytes. Use `MultiGzDecoder`; exact-limit valid data +must pass because the reader probes `maximum + 1`, while each malformed input +must fail before a candidate can be published. + +- [ ] **Step 5: Run transfer tests, Clippy, and formatting** + +Run: + +```bash +cargo +1.95.0 fmt --all -- --check +cargo +1.95.0 clippy --all-targets --locked -- -D warnings +cargo +1.95.0 test --lib data::transfer::tests --locked +cargo +1.95.0 test --test data --locked +``` + +Expected: all commands exit 0. + +- [ ] **Step 6: Commit strict protocol handling** + +```bash +git add src/data/transfer.rs +git commit -m "fix(data): bound HTTP metadata and deadlines" +``` + +--- + +### Task 3: Add the single-flight operation lock and locked clean path + +**Files:** +- Create: `src/data/operation_lock.rs` +- Modify: `src/data.rs:1-150` +- Modify: `src/cli.rs:332-351` +- Test: `src/data/operation_lock.rs` +- Test: `tests/data.rs` +- Test: `tests/command.rs:240-283` + +**Interfaces:** +- Consumes: `super::sibling_path`, `install_candidate`, and Task 1 artifact names. +- Produces: `operation_lock::acquire(&Path, Duration) -> Result` and public `data::clean_data(&Path) -> Result>`. + +- [ ] **Step 1: Add failing lock lifecycle tests** + +Create `src/data/operation_lock.rs`, wire `mod operation_lock;`, and add: + +```rust +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn competing_lock_times_out_then_succeeds_after_drop() { + let directory = tempfile::tempdir().unwrap(); + let data = directory.path().join("data.sqlite"); + let first = acquire(&data, Duration::from_millis(100)).unwrap(); + let error = acquire(&data, Duration::from_millis(20)) + .unwrap_err() + .to_string(); + assert!(error.contains("等待") && error.contains("超时"), "got: {error}"); + drop(first); + acquire(&data, Duration::from_millis(100)).unwrap(); + assert!(data.with_file_name("data.sqlite.lock").exists()); + } +} +``` + +- [ ] **Step 2: Run the lock test and confirm RED** + +Run: + +```bash +cargo +1.95.0 test --lib data::operation_lock::tests --locked +``` + +Expected: compilation fails because `acquire` and `OperationLock` do not exist. + +- [ ] **Step 3: Implement the permanent bounded-wait lock** + +Implement `src/data/operation_lock.rs` as: + +```rust +use super::sibling_path; +use anyhow::{anyhow, Context, Result}; +use std::fs::{File, OpenOptions}; +use std::io::ErrorKind; +use std::path::Path; +use std::time::{Duration, Instant}; + +const POLL_INTERVAL: Duration = Duration::from_millis(100); +pub(super) const LOCK_WAIT_TIMEOUT: Duration = Duration::from_secs(20 * 60); + +#[derive(Debug)] +pub(super) struct OperationLock { + _file: File, +} + +pub(super) fn acquire(data_path: &Path, timeout: Duration) -> Result { + let lock_path = sibling_path(data_path, ".lock")?; + let file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .open(&lock_path) + .with_context(|| format!("打开数据操作锁失败: {}", lock_path.display()))?; + let started = Instant::now(); + let mut reported_wait = false; + loop { + match file.try_lock() { + Ok(()) => return Ok(OperationLock { _file: file }), + Err(error) if error.kind() == ErrorKind::WouldBlock => { + if started.elapsed() >= timeout { + return Err(anyhow!( + "等待其他 fojin 数据操作超时: {}", + lock_path.display() + )); + } + if !reported_wait { + eprintln!("检测到另一个 fojin 数据操作,正在等待..."); + reported_wait = true; + } + let remaining = timeout.saturating_sub(started.elapsed()); + std::thread::sleep(POLL_INTERVAL.min(remaining)); + } + Err(error) => { + return Err(error) + .with_context(|| format!("获取数据操作锁失败: {}", lock_path.display())); + } + } + } +} +``` + +Do not unlink the lock file. Dropping `OperationLock` drops `File` and releases +the OS lock after success, error, unwind, or process exit. + +- [ ] **Step 4: Lock ensure, update, and clean with a second check** + +Change `ensure_data` to preserve the no-op and offline fast paths, then acquire +the lock and check existence again: + +```rust +pub fn ensure_data(path: &Path, offline: bool, source: &DataSource<'_>) -> Result<()> { + if path.exists() { + return Ok(()); + } + if offline { + return Err(anyhow!( + "本地数据不存在且处于 --offline (offline)。请手动下载:\n {}\n解压后放到: {}", + source.url, + path.display() + )); + } + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).context("创建缓存目录失败")?; + } + let _lock = operation_lock::acquire(path, operation_lock::LOCK_WAIT_TIMEOUT)?; + if path.exists() { + return Ok(()); + } + install_candidate(path, source) +} +``` + +Acquire the same lock around all of `update_data`. Add: + +```rust +pub fn clean_data(path: &Path) -> Result> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).context("创建缓存目录失败")?; + } + let _lock = operation_lock::acquire(path, operation_lock::LOCK_WAIT_TIMEOUT)?; + transfer::remove_known_artifacts(path)?; + let size = match std::fs::metadata(path) { + Ok(metadata) => Some(metadata.len()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(error) => return Err(error).context("读取数据文件状态失败"), + }; + if size.is_some() { + std::fs::remove_file(path) + .with_context(|| format!("删除数据失败: {}", path.display()))?; + } + Ok(size) +} +``` + +Implement the locked sweep as: + +```rust +pub(super) fn remove_known_artifacts(live_path: &Path) -> Result<()> { + let legacy = live_path.with_extension("tmp"); + match std::fs::remove_file(&legacy) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => { + return Err(error) + .with_context(|| format!("删除旧临时数据失败: {}", legacy.display())); + } + } + + let directory = live_path + .parent() + .ok_or_else(|| anyhow!("数据路径没有父目录: {}", live_path.display()))?; + let file_name = live_path + .file_name() + .ok_or_else(|| anyhow!("数据路径没有文件名: {}", live_path.display()))? + .to_string_lossy(); + let download_prefix = format!("{file_name}.download."); + let candidate_prefix = format!("{file_name}.candidate."); + for entry in std::fs::read_dir(directory) + .with_context(|| format!("读取数据目录失败: {}", directory.display()))? + { + let entry = entry.context("读取数据目录项失败")?; + let name = entry.file_name(); + let name = name.to_string_lossy(); + if !name.starts_with(&download_prefix) && !name.starts_with(&candidate_prefix) { + continue; + } + if entry.file_type().context("读取临时数据类型失败")?.is_dir() { + continue; + } + std::fs::remove_file(entry.path()) + .with_context(|| format!("删除临时数据失败: {}", entry.path().display()))?; + } + Ok(()) +} +``` + +This code does not traverse directories or delete the `.lock` sibling. +Change `run_data(DataAction::Clean)` in `src/cli.rs` to print from +`data::clean_data` and leave `data.sqlite.lock` untouched. + +- [ ] **Step 5: Add the process-level single-flight test** + +In `tests/data.rs`, make one test act as either the parent or a re-executed +worker. This avoids adding a normal test that exits without assertions: + +```rust +#[test] +fn concurrent_first_install_downloads_once() { + if std::env::var_os("FOJIN_CONCURRENT_WORKER").is_some() { + let path = PathBuf::from(std::env::var_os("FOJIN_WORKER_DATA").unwrap()); + let url = std::env::var("FOJIN_WORKER_URL").unwrap(); + let sha = std::env::var("FOJIN_WORKER_SHA256").unwrap(); + ensure_data(&path, false, &DataSource { url: &url, sha256: &sha }).unwrap(); + return; + } + use std::io::{Read, Write}; + use std::process::Command; + use std::time::{Duration, Instant}; + + let directory = tempfile::tempdir().unwrap(); + let data_path = directory.path().join("data.sqlite"); + let body = gzip_bytes(&replacement_database_bytes()); + let sha = sha256_hex(&body); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}/data.gz", listener.local_addr().unwrap()); + let server_body = body.clone(); + let server = std::thread::spawn(move || { + let mut requests = 0_usize; + let (mut first, _) = listener.accept().unwrap(); + requests += 1; + let mut request = [0_u8; 4096]; + let _ = first.read(&mut request); + write!( + first, + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + server_body.len() + ) + .unwrap(); + let midpoint = server_body.len() / 2; + first.write_all(&server_body[..midpoint]).unwrap(); + first.flush().unwrap(); + std::thread::sleep(Duration::from_millis(200)); + first.write_all(&server_body[midpoint..]).unwrap(); + drop(first); + + listener.set_nonblocking(true).unwrap(); + let deadline = Instant::now() + Duration::from_secs(2); + while Instant::now() < deadline { + match listener.accept() { + Ok((mut stream, _)) => { + requests += 1; + let _ = stream.read(&mut request); + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + server_body.len() + ) + .unwrap(); + stream.write_all(&server_body).unwrap(); + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + std::thread::sleep(Duration::from_millis(10)); + } + Err(error) => panic!("accept failed: {error}"), + } + } + requests + }); + + let spawn_worker = || { + Command::new(std::env::current_exe().unwrap()) + .arg("--exact") + .arg("concurrent_first_install_downloads_once") + .arg("--nocapture") + .env("FOJIN_CONCURRENT_WORKER", "1") + .env("FOJIN_WORKER_DATA", &data_path) + .env("FOJIN_WORKER_URL", &url) + .env("FOJIN_WORKER_SHA256", &sha) + .spawn() + .unwrap() + }; + let mut first = spawn_worker(); + let mut second = spawn_worker(); + assert!(first.wait().unwrap().success()); + assert!(second.wait().unwrap().success()); + assert_eq!(server.join().unwrap(), 1); + verify_dataset_file(&data_path).unwrap(); + assert_no_owned_candidate_artifacts(&data_path); +} +``` + +- [ ] **Step 6: Add clean and lock regression tests** + +In `tests/command.rs`, extend `data_clean_removes_file_and_is_idempotent` to +assert `data.sqlite.lock` remains a regular file after both clean calls. Add a +data-layer test showing an existing live file stays readable while an updater +is downloading; do not make query connections acquire the operation lock. + +- [ ] **Step 7: Run all lock, data, command, and Windows-compilable tests** + +Run: + +```bash +cargo +1.95.0 fmt --all -- --check +cargo +1.95.0 clippy --all-targets --locked -- -D warnings +cargo +1.95.0 test --lib data::operation_lock::tests --locked +cargo +1.95.0 test --test data --locked +cargo +1.95.0 test --test command --locked +cargo +1.95.0 test --all --locked +``` + +Expected: all commands exit 0, the process-level server reports one request, +and no test waits for a production-scale timeout. + +- [ ] **Step 8: Commit single-flight locking** + +```bash +git add src/data.rs src/data/operation_lock.rs src/data/transfer.rs src/cli.rs tests/data.rs tests/command.rs +git commit -m "fix(data): serialize installation and cleanup" +``` + +--- + +### Task 4: Document the resource contract and run release-grade verification + +**Files:** +- Modify: `README.md:155-190` +- Modify: `README.md:330-365` +- Modify: `CHANGELOG.md:5-20` +- Test: all repository test and release-contract files + +**Interfaces:** +- Consumes: the final constants and behavior from Tasks 1-3. +- Produces: user-facing operational documentation and a verified merge-ready branch. + +- [ ] **Step 1: Update the Chinese and English data documentation** + +Document these exact facts in both README language sections: + +- transfers no longer buffer the complete archive or database in memory; +- compressed responses are capped at 256 MiB and decompressed databases at + 768 MiB; +- updates can temporarily require the live database plus roughly 744 MiB of + staging disk; +- connect, idle, and total HTTP timeouts are 30 seconds, 60 seconds, and 15 + minutes; +- concurrent operations on one data directory are single-flight and a waiter + may wait up to 20 minutes; +- the permanent `.lock` file is harmless and intentionally survives clean; +- offline queries and the pinned data-v1 checksum contract are unchanged. + +- [ ] **Step 2: Update CHANGELOG** + +Under `[0.3.0] - Unreleased`, add bullets stating that the data path now uses a +bounded disk-streamed, checksum-first pipeline and that concurrent install, +update, and clean operations are serialized with full candidate validation. + +- [ ] **Step 3: Verify no obsolete production path remains** + +Run: + +```bash +rg -n "download_and_unpack|fn http_get|let raw =" src +``` + +Expected: no matches. The small public `gunzip`, `verify_sha256`, and +`write_atomic` compatibility helpers may remain, but neither `ensure_data` nor +`update_data` may call them. + +- [ ] **Step 4: Run the complete local verification matrix** + +Run: + +```bash +git diff --check +cargo +1.95.0 fmt --all -- --check +cargo +1.95.0 clippy --all-targets --locked -- -D warnings +cargo +1.95.0 test --all --locked +cargo +1.95.0 build --release --locked +cargo +1.95.0 package --locked +python3 -m pytest data-pipeline/tests -q +bash tests/release-scripts.sh +bash tests/install-script.sh +``` + +Expected: every command exits 0; Rust tests include the new transfer and +concurrency cases, Python reports one pass, and both shell suites report their +success messages. + +- [ ] **Step 5: Run static shell/workflow validation when available** + +Run ShellCheck 0.10.0 over `install.sh`, `scripts/*.sh`, and `tests/*.sh`, and +actionlint 1.7.7 over `.github/workflows/*.yml`. Expected: exit 0 with no +diagnostics. If the binaries are absent, use the same checksum-verified pinned +downloads established in the preceding v0.3.0 work. + +- [ ] **Step 6: Commit documentation** + +```bash +git add README.md CHANGELOG.md +git commit -m "docs: describe bounded data downloads" +``` + +--- + +### Task 5: Independent review, PR, CI, and merge + +**Files:** +- Review: `origin/master...HEAD` +- GitHub: branch `agent/streaming-data-download` + +**Interfaces:** +- Consumes: all implementation commits and fresh verification evidence. +- Produces: a merged GitHub PR and synchronized clean local `master`. + +- [ ] **Step 1: Request independent spec and code-quality review** + +Give reviewers the approved design, this plan, the commit range, and test +evidence. Require explicit Critical/Important/Minor findings and a ready-to- +merge verdict. Address substantive findings with TDD and focused fix commits. + +- [ ] **Step 2: Re-run verification after the final review fix** + +At minimum rerun `git diff --check`, fmt, Clippy, all locked Rust tests, Python, +release scripts, and installer scripts. Expected: all exit 0 on the final HEAD. + +- [ ] **Step 3: Push and create a ready pull request** + +```bash +git push -u origin agent/streaming-data-download +gh pr create --base master --head agent/streaming-data-download \ + --title "fix(data): stream and serialize dataset installation" \ + --body-file .superpowers/sdd/pr-body.md +``` + +The PR body summarizes bounded memory, limits/timeouts, full first-install +validation, single-flight concurrency, documentation, and exact test evidence. + +- [ ] **Step 4: Wait for every GitHub check and fix failures systematically** + +Run `gh pr checks --watch --interval 10`. Do not merge with a pending, +cancelled, or failing check. For a failure, inspect the job logs, reproduce it, +apply one root-cause fix, and rerun local verification before pushing. + +- [ ] **Step 5: Merge and verify master** + +Use a merge commit, delete the remote feature branch, fast-forward the main +checkout, wait for the post-merge `master` CI run, rerun the core local tests, +and remove the isolated worktree/local feature branch. Verify no tag or GitHub +Release was created. diff --git a/docs/superpowers/specs/2026-07-11-streaming-data-download-design.md b/docs/superpowers/specs/2026-07-11-streaming-data-download-design.md new file mode 100644 index 0000000..8917663 --- /dev/null +++ b/docs/superpowers/specs/2026-07-11-streaming-data-download-design.md @@ -0,0 +1,267 @@ +# Streaming Data Download and Single-Flight Installation + +Status: approved design + +Date: 2026-07-11 + +## Context + +The current data path keeps the complete gzip response and the complete +decompressed SQLite database in two `Vec` values. With the published +`data-v1` artifact this means at least roughly 183 MiB plus 561 MiB of live +payload memory, excluding allocation capacity and library buffers. + +The current 900-second `ureq::AgentBuilder::timeout_read` is an individual +socket-read timeout, not a deadline for the complete request. The response and +gzip output are also unbounded. Finally, first installation publishes through +the shared name `data.tmp`, so concurrent processes can write and rename the +same temporary file. + +## Goals + +- Keep transfer and decompression memory bounded by small fixed buffers. +- Enforce compressed and decompressed byte limits. +- Enforce connect, idle-read, total HTTP, and lock-wait timeouts. +- Allow only one data download per data directory at a time. +- Make first installation and explicit update use the same fully validated + candidate-publish path. +- Keep an existing live database byte-for-byte unchanged on every failure + before a successful atomic replacement. +- Preserve the existing CLI contract and data-v1 URL/checksum pinning. +- Work on Linux, macOS, and Windows with Rust 1.95. + +## Non-goals + +- Download resumption or range requests. +- Automatic retry or mirror selection. +- Remote data-version discovery. +- Data schema or data-v1 artifact changes. +- Changing normal query ranking, output, or offline behavior. +- Serializing ordinary readers behind a long-running download lock. + +## Chosen Approach + +Use a two-stage disk pipeline protected by a persistent OS file lock: + +```text +HTTP response + -> unique compressed sibling (incremental SHA-256 and compressed limit) + -> MultiGzDecoder + -> unique SQLite candidate (decompressed limit) + -> file sync + -> schema/version + quick-check + FTS integrity verification + -> file sync + -> atomic publish +``` + +This keeps memory O(buffer size) and verifies the authenticated compressed +artifact before spending work on decompression. It uses about 183 MiB more +temporary disk than a one-pass decoder, but has simpler and safer checksum, +trailing-data, and cleanup semantics. + +## Components and Boundaries + +### Transfer module + +A focused private submodule under `data` owns: + +- `DownloadPolicy`, including all byte and time limits; +- strict response-length handling; +- streaming HTTP reads, progress, compressed byte counting, and incremental + SHA-256; +- streaming multi-member gzip decoding and decompressed byte counting; +- uniquely named temporary artifact creation and ordinary-failure cleanup. + +It does not open SQLite, decide whether an install is needed, or publish a +candidate as the live database. + +Production defaults are: + +- connect timeout: 30 seconds; +- idle read timeout: 60 seconds; +- total HTTP timeout: 15 minutes; +- lock wait timeout: 20 minutes; +- compressed limit: 256 MiB; +- decompressed limit: 768 MiB; +- transfer buffer: 64 KiB. + +Tests can inject smaller limits and timeouts without changing production +constants. + +### Operation lock + +The lock path is a permanent sibling named `data.sqlite.lock`. The +implementation opens it with `create(true)` and uses the Rust standard +library's OS file-lock API, available below the project's Rust 1.95 MSRV. No +new locking dependency is needed. + +Lock acquisition uses `try_lock` with bounded polling rather than an +unbounded blocking call. It reports once that another fojin process is active, +then fails clearly after the lock-wait deadline. The lock file is never deleted: +process exit or crash releases the OS lock, so there is no stale-lock breaking +algorithm and no lock-file deletion race. + +- `ensure_data` retains its unlocked existing-file fast path. If data is + missing, it creates the parent directory, acquires the operation lock, and + checks again before downloading. +- `update_data` acquires the same lock for its complete operation, preventing + duplicate downloads and last-finisher version reversal. +- `data clean` acquires the same lock before deleting the live data and known + temporary artifacts. It leaves the lock file in place. +- Ordinary queries do not acquire this lock and can keep using the previous + live database while an update downloads and validates its candidate. + +On Windows, a non-cooperating reader may still prevent final replacement. The +existing replacement recovery contract remains: a candidate is preserved only +when replacement entered an ambiguous recovery state, and the error reports +its exact path. The download lock is not extended to readers because doing so +would block queries for the full transfer duration. + +### Artifact lifecycle + +Compressed and decompressed artifacts are siblings of the live database so +the final rename stays on one filesystem. Names include the live filename, +artifact role, process ID, and an atomic sequence, and are opened with +`create_new(true)`: + +- `data.sqlite.download...gz` +- `data.sqlite.candidate..` + +Within the cooperating-process model enforced by the operation lock, an +ownership guard removes the unique artifact family reserved by the current +operation. It is disarmed only after a successful publish or an intentional +Windows preservation result. Pre-existing family collisions are skipped, so +recoverable errors do not sweep another cooperating operation's files. + +The same coordination model applies to SQLite sidecars and Windows replacement +backups. `create_new` reservations and post-reservation checks are not a +security boundary against another same-user process that can concurrently +modify the data directory; filesystem permissions remain that boundary. + +## Detailed Data Flow + +1. Acquire the operation lock and perform the mode-specific second check. +2. Create a unique read/write compressed artifact. +3. Build a ureq agent with connect, idle-read, and total HTTP timeouts. Send + `Accept-Encoding: identity` so the bytes counted and hashed are the release + asset bytes rather than transparent HTTP content decoding. +4. Treat `Content-Length` as an early rejection/progress hint, never as the + resource limit. A missing length is allowed. When transfer encoding does + not override a present value, reject non-numeric values, duplicates, and + values above the compressed limit. +5. Read the response with a fixed buffer. Before every write, checked-add the + cumulative count, enforce the compressed limit, update SHA-256, write the + chunk, and emit decile progress when a trustworthy total is available. +6. At EOF, require any usable declared length to match the received count and + compare the final digest with the pinned SHA-256. A mismatch deletes the + compressed artifact before decompression starts. +7. Rewind the compressed file and stream a `MultiGzDecoder` into a unique + candidate. Copy at most `max_uncompressed + 1` bytes so an exact-limit gzip + still reads and verifies its trailer while an oversized stream is detected. + Multi-member gzip is supported; truncation, invalid CRC/ISIZE, or trailing + non-gzip data fails closed. +8. Flush, `sync_all`, and close the candidate. Run the existing complete + schema/version, `PRAGMA quick_check`, and FTS content-integrity checks. + First installation now receives the same validation as explicit update. +9. Sync the validated candidate again and publish with the existing + platform-specific atomic replacement logic. +10. Remove the compressed artifact and any ordinary candidate sidecars. + +No production transfer path returns the complete response or decompressed +database as a `Vec`. Existing public byte helpers may remain for source +compatibility and small unit tests, but the installer and updater do not call +them. + +## Error and Recovery Contract + +Errors distinguish these stages: + +- operation-lock wait timeout or lock I/O failure; +- connection, HTTP deadline, idle read, status, or response read failure; +- invalid, duplicate, mismatched, or oversized response length; +- compressed stream limit exceeded; +- SHA-256 mismatch; +- gzip format, trailer, or decompressed stream limit failure; +- candidate creation, write, flush, or sync failure; +- SQLite compatibility, quick-check, or FTS integrity failure; +- final platform-specific replacement failure. + +Network and checksum failures retain the existing manual-download guidance. +Every failure before successful publish satisfies both invariants: + +1. an existing live database is unchanged; and +2. a first installation does not expose a live database path. + +Ordinary failures remove both owned artifacts and SQLite sidecars. If cleanup +also fails, the cleanup error is attached as context without hiding the primary +failure. A Windows replacement rollback failure is the only exception: it +preserves and explicitly reports both the old-live backup and validated +candidate so neither recoverable copy is lost. + +## Testing Strategy + +### Deterministic unit and local HTTP tests + +- compressed size exactly at the limit and one byte over; +- decompressed size exactly at the limit and one byte over; +- known oversized `Content-Length` rejected before body consumption; +- missing length and chunked transfer constrained by the actual byte counter; +- invalid, duplicate, and mismatched declared lengths; +- valid digest and digest mismatch; +- single-member and multi-member gzip; +- truncated gzip, bad trailer, and trailing garbage; +- a small gzip that expands past the configured limit; +- a server that continuously dribbles bytes: idle timeout does not fire but + the injected total HTTP deadline does; +- a body pause longer than the injected idle timeout; +- injected reader/writer failures without platform-specific `/dev/full`; +- every error leaves an existing live database unchanged and removes owned + artifacts. + +### Process-level concurrency tests + +A test re-executes the Rust test binary as two worker processes against one +slow local HTTP server and one temporary data directory. It asserts: + +- both callers succeed; +- the server observes exactly one request; +- the waiter reuses the installed database after the lock-time second check; +- the final SQLite and FTS verification succeeds; +- no owned compressed or candidate artifacts remain. + +Additional tests serialize update versus update and clean versus install, and +verify that the lock is released after worker exit. Windows CI retains the +existing replacement recovery tests and exercises the new lock path. + +### Regression suite + +The acceptance run includes: + +- Rust 1.95 formatting, Clippy with warnings denied, all locked tests, release + build, and package verification; +- Python normalization parity; +- release and installer shell contracts; +- ShellCheck and actionlint when their covered files change. + +## Documentation and Delivery + +README documents the 256 MiB compressed limit, 768 MiB decompressed limit, +timeouts, single-flight behavior, temporary disk requirement, and unchanged +offline behavior. CHANGELOG records the bounded-memory transfer and concurrent +installation fix. + +Delivery uses the established repository process: focused commits on an +isolated branch, independent code review, GitHub pull request, all checks green, +then merge to `master`. This work does not create a tag or GitHub Release. + +## Acceptance Criteria + +- The production download/update path has no full-response or full-database + in-memory allocation. +- Compressed input above 256 MiB and decompressed output above 768 MiB fail + before publish. +- A continuously active response cannot exceed the 15-minute HTTP deadline. +- Two concurrent first installations make one HTTP request and both succeed. +- First installation and update both perform complete SQLite/FTS validation. +- All injected failures preserve the live database and clean owned artifacts. +- Rust, Python, shell-contract, Windows, and GitHub CI checks pass. diff --git a/src/cli.rs b/src/cli.rs index 9a80688..306828e 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -339,14 +339,13 @@ fn run_data(action: DataAction) -> Result { } DataAction::Clean { data_dir } => { let path = data::resolve_data_path(data_dir)?; - // A crashed download can leave a temp sibling; sweep it too. - let _ = std::fs::remove_file(path.with_extension("tmp")); - if path.exists() { - let size = std::fs::metadata(&path)?.len(); - std::fs::remove_file(&path)?; - println!("已删除 {} (释放 {} MB)", path.display(), size / MB); - } else { - println!("本地无数据,无需清理: {}", path.display()); + match data::clean_data(&path)? { + Some(size) => { + println!("已删除 {} (释放 {} MB)", path.display(), size / MB); + } + None => { + println!("本地无数据,无需清理: {}", path.display()); + } } Ok(0) } diff --git a/src/data.rs b/src/data.rs index 74f5a9b..7c2bd98 100644 --- a/src/data.rs +++ b/src/data.rs @@ -1,18 +1,14 @@ use anyhow::{anyhow, Context, Result}; use rusqlite::{OpenFlags, OptionalExtension}; -use std::io::{Read, Write}; +use std::io::Read; use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::time::Duration; - -/// Connect timeout for the data download: fails fast if the release host is -/// unreachable rather than hanging forever. -const CONNECT_TIMEOUT: Duration = Duration::from_secs(30); -/// Overall read ceiling for the data download. The artifact is ~100-200MB, -/// so this is generous for a slow-but-alive connection while still -/// guaranteeing the CLI can never hang indefinitely. -const READ_TIMEOUT: Duration = Duration::from_secs(900); -static CANDIDATE_SEQUENCE: AtomicU64 = AtomicU64::new(0); + +mod operation_lock; +mod transfer; + +#[cfg(windows)] +static REPLACEMENT_BACKUP_SEQUENCE: std::sync::atomic::AtomicU64 = + std::sync::atomic::AtomicU64::new(0); pub const EXPECTED_DATA_VERSION: &str = "v1"; pub const EXPECTED_NORM_RULESET: &str = "t2s-char-1to1-v1"; @@ -117,8 +113,11 @@ pub fn ensure_data(path: &Path, offline: bool, source: &DataSource) -> Result<() if let Some(parent) = path.parent() { std::fs::create_dir_all(parent).context("创建缓存目录失败")?; } - let raw = download_and_unpack(path, source)?; - write_atomic(path, &raw) + let _lock = operation_lock::acquire(path, operation_lock::LOCK_WAIT_TIMEOUT)?; + if path.exists() { + return Ok(()); + } + install_candidate(path, source) } /// Download a replacement dataset to a sibling candidate, validate it, then @@ -128,40 +127,43 @@ pub fn update_data(path: &Path, source: &DataSource) -> Result<()> { std::fs::create_dir_all(parent).context("创建缓存目录失败")?; } - let raw = download_and_unpack(path, source)?; - let (candidate, mut candidate_file) = create_candidate(path)?; - let write_result = candidate_file - .write_all(&raw) - .with_context(|| format!("写入候选数据失败: {}", candidate.display())); - drop(candidate_file); - if let Err(error) = write_result { - return Err(cleanup_candidate_error(&candidate, error)); - } + let _lock = operation_lock::acquire(path, operation_lock::LOCK_WAIT_TIMEOUT)?; + install_candidate(path, source) +} - let validation_result = verify_dataset_file(&candidate).map(|_| ()); - if let Err(error) = validation_result { - return Err(cleanup_candidate_error(&candidate, error)); +pub fn clean_data(path: &Path) -> Result> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).context("创建缓存目录失败")?; } - - finish_replacement(&candidate, replace_with_candidate(path, &candidate)) + let _lock = operation_lock::acquire(path, operation_lock::LOCK_WAIT_TIMEOUT)?; + transfer::remove_known_artifacts(path)?; + let size = match std::fs::metadata(path) { + Ok(metadata) => Some(metadata.len()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(error) => return Err(error).context("读取数据文件状态失败"), + }; + if size.is_some() { + std::fs::remove_file(path).with_context(|| format!("删除数据失败: {}", path.display()))?; + } + Ok(size) } -fn download_and_unpack(path: &Path, source: &DataSource) -> Result> { - let gz = http_get(source.url).map_err(|e| { - anyhow!( - "{e:#}\n请手动下载:\n {}\n解压后放到: {}", - source.url, - path.display() - ) - })?; - if !verify_sha256(&gz, source.sha256) { - return Err(anyhow!( - "下载校验失败(sha256 不符)。请重试或手动下载:\n {}\n解压后放到: {}", - source.url, - path.display() - )); +fn install_candidate(path: &Path, source: &DataSource<'_>) -> Result<()> { + let candidate = transfer::stage_candidate(path, source, transfer::PRODUCTION_POLICY)?; + if let Err(error) = verify_dataset_file(candidate.path()).map(|_| ()) { + return Err(candidate.cleanup_with(error)); } - gunzip(&gz) + if let Err(error) = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(candidate.path()) + .and_then(|file| file.sync_all()) + .with_context(|| format!("同步候选数据失败: {}", candidate.path().display())) + { + return Err(candidate.cleanup_with(error)); + } + let candidate_path = candidate.path().to_path_buf(); + finish_replacement(candidate, replace_with_candidate(path, &candidate_path)) } fn sibling_path(path: &Path, suffix: &str) -> Result { @@ -173,33 +175,6 @@ fn sibling_path(path: &Path, suffix: &str) -> Result { Ok(path.with_file_name(sibling)) } -fn create_candidate(path: &Path) -> Result<(PathBuf, std::fs::File)> { - loop { - let sequence = CANDIDATE_SEQUENCE.fetch_add(1, Ordering::Relaxed); - let suffix = format!(".candidate.{}.{sequence}", std::process::id()); - let candidate = sibling_path(path, &suffix)?; - match std::fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(&candidate) - { - Ok(file) => return Ok((candidate, file)), - Err(ref error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, - Err(error) => { - return Err(error) - .with_context(|| format!("创建候选数据失败: {}", candidate.display())) - } - } - } -} - -fn cleanup_candidate_error(candidate: &Path, error: anyhow::Error) -> anyhow::Error { - match remove_candidate_artifacts(candidate) { - Ok(()) => error, - Err(cleanup_error) => error.context(format!("清理候选数据失败: {cleanup_error}")), - } -} - #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum CandidateCleanupPolicy { Remove, @@ -232,39 +207,28 @@ impl ReplacementFailure { type ReplacementResult = std::result::Result<(), ReplacementFailure>; -fn finish_replacement(candidate: &Path, result: ReplacementResult) -> Result<()> { +fn finish_replacement( + candidate: transfer::StagedCandidate, + result: ReplacementResult, +) -> Result<()> { match result { - Ok(()) => Ok(()), + Ok(()) => { + candidate.publish_succeeded(); + Ok(()) + } Err(failure) => match failure.cleanup { - CandidateCleanupPolicy::Remove => { - Err(cleanup_candidate_error(candidate, failure.error)) - } + CandidateCleanupPolicy::Remove => Err(candidate.cleanup_with(failure.error)), #[cfg(any(test, windows))] - CandidateCleanupPolicy::Preserve => Err(failure.error.context(format!( - "validated candidate preserved at `{}`", - candidate.display() - ))), - }, - } -} - -fn remove_candidate_artifacts(candidate: &Path) -> Result<()> { - for suffix in ["", "-journal", "-shm", "-wal"] { - let artifact = if suffix.is_empty() { - candidate.to_path_buf() - } else { - sibling_path(candidate, suffix)? - }; - match std::fs::remove_file(&artifact) { - Ok(()) => {} - Err(ref error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => { - return Err(error) - .with_context(|| format!("删除候选数据失败: {}", artifact.display())) + CandidateCleanupPolicy::Preserve => { + let candidate_path = candidate.path().to_path_buf(); + candidate.preserve(); + Err(failure.error.context(format!( + "validated candidate preserved at `{}`", + candidate_path.display() + ))) } - } + }, } - Ok(()) } #[cfg(not(windows))] @@ -330,85 +294,273 @@ fn replace_with_candidate(path: &Path, candidate: &Path) -> ReplacementResult { Ok(encoded) } + let mut backup = reserve_replacement_backup(path).map_err(ReplacementFailure::remove)?; let replaced = wide_path(path).map_err(ReplacementFailure::remove)?; let replacement = wide_path(candidate).map_err(ReplacementFailure::remove)?; - // SAFETY: both path buffers are NUL-terminated UTF-16 and remain alive for - // the call; optional and reserved pointers are null as ReplaceFileW requires. + let backup_name = wide_path(backup.path()).map_err(ReplacementFailure::remove)?; + backup + .prepare_for_replace() + .map_err(ReplacementFailure::remove)?; + // SAFETY: all path buffers are NUL-terminated UTF-16 and remain alive for + // the call; the reserved pointers are null as ReplaceFileW requires. let replaced_ok = unsafe { ReplaceFileW( replaced.as_ptr(), replacement.as_ptr(), - ptr::null(), + backup_name.as_ptr(), 0, ptr::null_mut(), ptr::null_mut(), ) }; if replaced_ok != 0 { + cleanup_windows_backup_after_success(backup.path()); return Ok(()); } let native_error = std::io::Error::last_os_error(); - handle_windows_replace_failure(path, candidate, native_error, |from, to| { - std::fs::rename(from, to) - }) + handle_windows_replace_failure( + path, + candidate, + backup.path(), + native_error, + |from, to| std::fs::rename(from, to), + std::thread::sleep, + ) +} + +#[cfg(any(test, windows))] +#[derive(Debug)] +struct ReplacementBackupReservation { + path: PathBuf, + file: Option, + armed: bool, +} + +#[cfg(any(test, windows))] +impl ReplacementBackupReservation { + #[cfg(windows)] + fn path(&self) -> &Path { + &self.path + } + + #[cfg(windows)] + fn prepare_for_replace(&mut self) -> Result<()> { + drop(self.file.take()); + std::fs::remove_file(&self.path) + .with_context(|| format!("释放替换备份路径失败: {}", self.path.display()))?; + self.armed = false; + Ok(()) + } +} + +#[cfg(any(test, windows))] +impl Drop for ReplacementBackupReservation { + fn drop(&mut self) { + drop(self.file.take()); + if self.armed { + let _ = std::fs::remove_file(&self.path); + } + } } #[cfg(windows)] -fn handle_windows_replace_failure( +fn reserve_replacement_backup(live_path: &Path) -> Result { + use std::sync::atomic::Ordering; + + // create_new avoids stale-name collisions between cooperating fojin + // operations. The reservation is not a security boundary against another + // same-user process that can alter this directory concurrently. + loop { + let sequence = REPLACEMENT_BACKUP_SEQUENCE.fetch_add(1, Ordering::Relaxed); + let path = sibling_path( + live_path, + &format!(".replace-backup.{}.{sequence}", std::process::id()), + )?; + if let Some(reservation) = try_reserve_replacement_backup(&path)? { + return Ok(reservation); + } + } +} + +#[cfg(any(test, windows))] +fn try_reserve_replacement_backup(path: &Path) -> Result> { + match std::fs::OpenOptions::new() + .read(true) + .write(true) + .create_new(true) + .open(path) + { + Ok(file) => Ok(Some(ReplacementBackupReservation { + path: path.to_path_buf(), + file: Some(file), + armed: true, + })), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => Ok(None), + Err(error) => { + Err(error).with_context(|| format!("保留替换备份路径失败: {}", path.display())) + } + } +} + +#[cfg(any(test, windows))] +fn cleanup_windows_backup_after_success(backup: &Path) { + match std::fs::remove_file(backup) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => eprintln!( + "警告: 新数据已发布,但无法删除旧数据备份 `{}`: {error}", + backup.display() + ), + } +} + +#[cfg(any(test, windows))] +fn handle_windows_replace_failure( path: &Path, candidate: &Path, + backup: &Path, native_error: std::io::Error, - recover: F, + mut recover: F, + mut sleep: S, ) -> ReplacementResult where - F: FnOnce(&Path, &Path) -> std::io::Result<()>, + F: FnMut(&Path, &Path) -> std::io::Result<()>, + S: FnMut(std::time::Duration), { - const ERROR_UNABLE_TO_REMOVE_REPLACED: i32 = 1175; - const ERROR_UNABLE_TO_MOVE_REPLACEMENT: i32 = 1176; - const ERROR_UNABLE_TO_MOVE_REPLACEMENT_2: i32 = 1177; - - match native_error.raw_os_error() { - Some(ERROR_UNABLE_TO_MOVE_REPLACEMENT | ERROR_UNABLE_TO_MOVE_REPLACEMENT_2) => { - match recover(candidate, path) { - Ok(()) => Ok(()), - Err(recovery_error) => Err(ReplacementFailure::preserve(anyhow!( - "ReplaceFileW failed for replaced `{}` and replacement `{}` with {}; recovery rename `{} -> {}` failed with {}", - path.display(), - candidate.display(), - describe_windows_error(&native_error), - candidate.display(), - path.display(), - describe_windows_error(&recovery_error) - ))), - } + let original = windows_replace_error(path, candidate, backup, &native_error); + let live_exists = match replacement_entry_exists(path) { + Ok(exists) => exists, + Err(error) => { + return Err(ReplacementFailure::preserve(original.context(format!( + "could not inspect live path `{}` after replacement failure: {error}", + path.display() + )))); + } + }; + let backup_exists = match replacement_entry_exists(backup) { + Ok(exists) => exists, + Err(error) => { + return Err(ReplacementFailure::preserve(original.context(format!( + "could not inspect replacement backup `{}`: {error}", + backup.display() + )))); + } + }; + + if live_exists { + let error = if backup_exists { + original.context(format!( + "live path still exists; replacement backup preserved at `{}`", + backup.display() + )) + } else { + original + }; + return Err(ReplacementFailure::remove(error)); + } + + if backup_exists { + return match recover_with_retries(backup, path, &mut recover, &mut sleep) { + Ok(()) => Err(ReplacementFailure::remove(original.context(format!( + "old live data restored from replacement backup `{}`", + backup.display() + )))), + Err(recovery_error) => Err(ReplacementFailure::preserve(anyhow!( + "{original:#}; rollback `{}` -> `{}` failed with {}; validated candidate remains at `{}` and old live remains at `{}`. manual recovery required: restore the backup to the live path before retrying", + backup.display(), + path.display(), + describe_windows_error(&recovery_error), + candidate.display(), + backup.display() + ))), + }; + } + + let candidate_exists = match replacement_entry_exists(candidate) { + Ok(exists) => exists, + Err(error) => { + return Err(ReplacementFailure::preserve(original.context(format!( + "could not inspect validated candidate `{}`: {error}", + candidate.display() + )))); } - Some(ERROR_UNABLE_TO_REMOVE_REPLACED) => Err(ReplacementFailure::remove( - windows_replace_error(path, candidate, &native_error), - )), - _ => Err(ReplacementFailure::remove(windows_replace_error( - path, - candidate, - &native_error, - ))), + }; + if candidate_exists { + return match recover_with_retries(candidate, path, &mut recover, &mut sleep) { + Ok(()) => Err(ReplacementFailure::remove(original.context(format!( + "replacement backup was missing; validated candidate restored to live path `{}`", + path.display() + )))), + Err(recovery_error) => Err(ReplacementFailure::preserve(anyhow!( + "{original:#}; both live `{}` and backup `{}` are missing; recovery rename `{}` -> `{}` failed with {}. manual recovery required", + path.display(), + backup.display(), + candidate.display(), + path.display(), + describe_windows_error(&recovery_error) + ))), + }; } + + Err(ReplacementFailure::preserve(original.context(format!( + "live `{}`, backup `{}`, and candidate `{}` are all missing; manual recovery required", + path.display(), + backup.display(), + candidate.display() + )))) +} + +#[cfg(any(test, windows))] +fn recover_with_retries( + from: &Path, + to: &Path, + recover: &mut F, + sleep: &mut S, +) -> std::io::Result<()> +where + F: FnMut(&Path, &Path) -> std::io::Result<()>, + S: FnMut(std::time::Duration), +{ + let mut last_error = None; + for attempt in 0..3 { + match recover(from, to) { + Ok(()) => return Ok(()), + Err(error) => last_error = Some(error), + } + if attempt < 2 { + sleep(std::time::Duration::from_millis(50)); + } + } + Err(last_error.expect("recovery attempted at least once")) } -#[cfg(windows)] +#[cfg(any(test, windows))] +fn replacement_entry_exists(path: &Path) -> std::io::Result { + match std::fs::symlink_metadata(path) { + Ok(_) => Ok(true), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error), + } +} + +#[cfg(any(test, windows))] fn windows_replace_error( path: &Path, candidate: &Path, + backup: &Path, native_error: &std::io::Error, ) -> anyhow::Error { anyhow!( - "ReplaceFileW failed for replaced `{}` and replacement `{}` with {}", + "ReplaceFileW failed for replaced `{}`, replacement `{}`, and backup `{}` with {}", path.display(), candidate.display(), + backup.display(), describe_windows_error(native_error) ) } -#[cfg(windows)] +#[cfg(any(test, windows))] fn describe_windows_error(error: &std::io::Error) -> String { match error.raw_os_error() { Some(code) => format!("Windows error {code}: {error}"), @@ -416,34 +568,6 @@ fn describe_windows_error(error: &std::io::Error) -> String { } } -fn http_get(url: &str) -> Result> { - let agent = ureq::AgentBuilder::new() - .timeout_connect(CONNECT_TIMEOUT) - .timeout_read(READ_TIMEOUT) - .build(); - let resp = agent - .get(url) - .call() - .with_context(|| format!("下载失败: {url}"))?; - let total: Option = resp.header("Content-Length").and_then(|v| v.parse().ok()); - eprintln!("{}", download_notice(total)); - let mut progress = Progress::new(total); - let mut reader = resp.into_reader(); - let mut buf = Vec::new(); - let mut chunk = [0u8; 64 * 1024]; - loop { - let n = reader.read(&mut chunk).context("读取响应失败")?; - if n == 0 { - break; - } - buf.extend_from_slice(&chunk[..n]); - if let Some(msg) = progress.advance(n as u64) { - eprintln!("{msg}"); - } - } - Ok(buf) -} - pub fn open_db(path: &Path) -> Result { rusqlite::Connection::open(path).with_context(|| format!("打开数据失败: {}", path.display())) } @@ -681,9 +805,10 @@ mod replacement_cleanup_tests { let dir = tempfile::tempdir().unwrap(); let candidate = dir.path().join("data.sqlite.candidate.1.1"); std::fs::write(&candidate, b"validated dataset").unwrap(); + let staged = transfer::StagedCandidate::for_test(candidate.clone()); let error = finish_replacement( - &candidate, + staged, Err(ReplacementFailure::preserve(anyhow!( "replacement recovery failed" ))), @@ -701,102 +826,177 @@ mod replacement_cleanup_tests { } } -#[cfg(all(test, windows))] -mod windows_replace_failure_tests { +#[cfg(test)] +mod windows_replace_state_tests { use super::*; - use std::cell::Cell; use std::io; - const LIVE: &str = r"C:\cache\data.sqlite"; - const CANDIDATE: &str = r"C:\cache\data.sqlite.candidate.1.1"; - #[test] - fn error_1175_returns_native_failure_without_recovery() { - assert_names_untouched_error_skips_recovery(1175); + fn errors_with_an_intact_live_path_preserve_old_content() { + for code in [1175, 1176, 87] { + let directory = tempfile::tempdir().unwrap(); + let live = directory.path().join("data.sqlite"); + let candidate = directory.path().join("data.sqlite.candidate.1.1"); + let backup = directory.path().join("data.sqlite.replace-backup.1.1"); + std::fs::write(&live, format!("old-{code}")).unwrap(); + std::fs::write(&candidate, b"validated replacement").unwrap(); + + let staged = transfer::StagedCandidate::for_test(candidate.clone()); + let result = handle_windows_replace_failure( + &live, + &candidate, + &backup, + io::Error::from_raw_os_error(code), + |from, to| std::fs::rename(from, to), + |_| {}, + ); + let error = finish_replacement(staged, result).unwrap_err(); + + assert_eq!( + std::fs::read(&live).unwrap(), + format!("old-{code}").as_bytes() + ); + assert!(!candidate.exists()); + assert!(!backup.exists()); + assert!(format!("{error:#}").contains(&format!("Windows error {code}"))); + } } #[test] - fn other_error_returns_native_failure_without_recovery() { - assert_names_untouched_error_skips_recovery(87); - } + fn error_1177_restores_old_live_from_backup_and_rejects_update() { + let directory = tempfile::tempdir().unwrap(); + let live = directory.path().join("data.sqlite"); + let candidate = directory.path().join("data.sqlite.candidate.1.1"); + let backup = directory.path().join("data.sqlite.replace-backup.1.1"); + std::fs::write(&candidate, b"validated replacement").unwrap(); + std::fs::write(&backup, b"old live").unwrap(); + + let staged = transfer::StagedCandidate::for_test(candidate.clone()); + let result = handle_windows_replace_failure( + &live, + &candidate, + &backup, + io::Error::from_raw_os_error(1177), + |from, to| std::fs::rename(from, to), + |_| {}, + ); + let error = finish_replacement(staged, result).unwrap_err(); - #[test] - fn error_1176_recovers_missing_live_path() { - assert_partial_failure_recovers(1176); + assert_eq!(std::fs::read(&live).unwrap(), b"old live"); + assert!(!backup.exists()); + assert!(!candidate.exists()); + let detail = format!("{error:#}"); + assert!(detail.contains("Windows error 1177"), "{detail}"); + assert!(detail.contains("restored"), "{detail}"); } #[test] - fn error_1177_recovers_missing_live_path() { - assert_partial_failure_recovers(1177); - } + fn rollback_failure_preserves_backup_and_candidate_for_manual_recovery() { + let directory = tempfile::tempdir().unwrap(); + let live = directory.path().join("data.sqlite"); + let candidate = directory.path().join("data.sqlite.candidate.1.1"); + let backup = directory.path().join("data.sqlite.replace-backup.1.1"); + std::fs::write(&candidate, b"validated replacement").unwrap(); + std::fs::write(&backup, b"old live").unwrap(); + + let staged = transfer::StagedCandidate::for_test(candidate.clone()); + let result = handle_windows_replace_failure( + &live, + &candidate, + &backup, + io::Error::from_raw_os_error(1177), + |_, _| Err(io::Error::from_raw_os_error(5)), + |_| {}, + ); + let error = finish_replacement(staged, result).unwrap_err(); - #[test] - fn error_1176_preserves_native_and_recovery_failures() { - assert_partial_failure_preserves_both_errors(1176); + assert!(!live.exists()); + assert_eq!(std::fs::read(&backup).unwrap(), b"old live"); + assert_eq!(std::fs::read(&candidate).unwrap(), b"validated replacement"); + let detail = format!("{error:#}"); + for expected in [ + "Windows error 1177", + "Windows error 5", + "manual recovery", + live.to_string_lossy().as_ref(), + backup.to_string_lossy().as_ref(), + candidate.to_string_lossy().as_ref(), + ] { + assert!(detail.contains(expected), "missing {expected:?}: {detail}"); + } } #[test] - fn error_1177_preserves_native_and_recovery_failures() { - assert_partial_failure_preserves_both_errors(1177); - } - - fn assert_names_untouched_error_skips_recovery(code: i32) { - let recovery_called = Cell::new(false); - let failure = handle_windows_replace_failure( - Path::new(LIVE), - Path::new(CANDIDATE), - io::Error::from_raw_os_error(code), - |_, _| { - recovery_called.set(true); - Ok(()) - }, - ) - .unwrap_err(); + fn successful_replace_removes_backup_without_touching_new_live() { + let directory = tempfile::tempdir().unwrap(); + let live = directory.path().join("data.sqlite"); + let candidate = directory.path().join("data.sqlite.candidate.1.1"); + let backup = directory.path().join("data.sqlite.replace-backup.1.1"); + std::fs::write(&live, b"new live").unwrap(); + std::fs::write(&backup, b"old live").unwrap(); - assert!(!recovery_called.get()); - assert_eq!(failure.cleanup, CandidateCleanupPolicy::Remove); - let detail = format!("{:#}", failure.error); - assert!( - detail.contains(&format!("Windows error {code}")), - "{detail}" - ); + cleanup_windows_backup_after_success(&backup); + + assert_eq!(std::fs::read(&live).unwrap(), b"new live"); + assert!(!candidate.exists()); + assert!(!backup.exists()); } - fn assert_partial_failure_recovers(code: i32) { - let recovery_called = Cell::new(false); - handle_windows_replace_failure( - Path::new(LIVE), - Path::new(CANDIDATE), - io::Error::from_raw_os_error(code), - |from, to| { - recovery_called.set(true); - assert_eq!(from, Path::new(CANDIDATE)); - assert_eq!(to, Path::new(LIVE)); - Ok(()) - }, - ) - .unwrap(); + #[test] + fn successful_replace_keeps_a_backup_that_cannot_be_cleaned() { + let directory = tempfile::tempdir().unwrap(); + let live = directory.path().join("data.sqlite"); + let backup = directory.path().join("data.sqlite.replace-backup.1.1"); + std::fs::write(&live, b"new live").unwrap(); + std::fs::create_dir(&backup).unwrap(); - assert!(recovery_called.get()); - } + cleanup_windows_backup_after_success(&backup); - fn assert_partial_failure_preserves_both_errors(code: i32) { - let failure = handle_windows_replace_failure( - Path::new(LIVE), - Path::new(CANDIDATE), - io::Error::from_raw_os_error(code), - |_, _| Err(io::Error::from_raw_os_error(5)), - ) - .unwrap_err(); + assert_eq!(std::fs::read(&live).unwrap(), b"new live"); + assert!(backup.is_dir()); + } - assert_eq!(failure.cleanup, CandidateCleanupPolicy::Preserve); - let detail = format!("{:#}", failure.error); - assert!( - detail.contains(&format!("Windows error {code}")), - "{detail}" - ); - assert!(detail.contains("Windows error 5"), "{detail}"); - assert!(detail.contains(CANDIDATE), "{detail}"); - assert!(detail.contains(LIVE), "{detail}"); + #[test] + fn backup_reservation_skips_a_collision_without_overwriting_it() { + let directory = tempfile::tempdir().unwrap(); + let collision = directory.path().join("data.sqlite.replace-backup.1.1"); + let available = directory.path().join("data.sqlite.replace-backup.1.2"); + std::fs::write(&collision, b"foreign backup").unwrap(); + + assert!(try_reserve_replacement_backup(&collision) + .unwrap() + .is_none()); + let reservation = try_reserve_replacement_backup(&available) + .unwrap() + .expect("next unique backup name should be reservable"); + + assert_eq!(std::fs::read(&collision).unwrap(), b"foreign backup"); + assert!(available.is_file()); + drop(reservation); + assert!(!available.exists()); + } + + #[cfg(windows)] + #[test] + fn native_replace_file_success_preserves_new_live_and_removes_backup() { + let directory = tempfile::tempdir().unwrap(); + let live = directory.path().join("data.sqlite"); + let candidate = directory.path().join("data.sqlite.candidate.1.1"); + std::fs::write(&live, b"old live").unwrap(); + std::fs::write(&candidate, b"new live").unwrap(); + + let staged = transfer::StagedCandidate::for_test(candidate.clone()); + finish_replacement(staged, replace_with_candidate(&live, &candidate)).unwrap(); + + assert_eq!(std::fs::read(&live).unwrap(), b"new live"); + assert!(!candidate.exists()); + let prefix = "data.sqlite.replace-backup."; + assert!(std::fs::read_dir(directory.path()).unwrap().all(|entry| { + !entry + .unwrap() + .file_name() + .to_string_lossy() + .starts_with(prefix) + })); } } diff --git a/src/data/operation_lock.rs b/src/data/operation_lock.rs new file mode 100644 index 0000000..8f79242 --- /dev/null +++ b/src/data/operation_lock.rs @@ -0,0 +1,72 @@ +use super::sibling_path; +use anyhow::{anyhow, Context, Result}; +use std::fs::{File, OpenOptions, TryLockError}; +use std::path::Path; +use std::time::{Duration, Instant}; + +const POLL_INTERVAL: Duration = Duration::from_millis(100); +pub(super) const LOCK_WAIT_TIMEOUT: Duration = Duration::from_secs(20 * 60); + +#[derive(Debug)] +pub(super) struct OperationLock { + _file: File, +} + +pub(super) fn acquire(data_path: &Path, timeout: Duration) -> Result { + let lock_path = sibling_path(data_path, ".lock")?; + let file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(&lock_path) + .with_context(|| format!("打开数据操作锁失败: {}", lock_path.display()))?; + let started = Instant::now(); + let mut reported_wait = false; + loop { + match file.try_lock() { + Ok(()) => return Ok(OperationLock { _file: file }), + Err(TryLockError::WouldBlock) => { + if started.elapsed() >= timeout { + return Err(anyhow!( + "等待其他 fojin 数据操作超时: {}", + lock_path.display() + )); + } + if !reported_wait { + eprintln!("检测到另一个 fojin 数据操作,正在等待..."); + reported_wait = true; + } + let remaining = timeout.saturating_sub(started.elapsed()); + std::thread::sleep(POLL_INTERVAL.min(remaining)); + } + Err(TryLockError::Error(error)) => { + return Err(error) + .with_context(|| format!("获取数据操作锁失败: {}", lock_path.display())); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + + #[test] + fn competing_lock_times_out_then_succeeds_after_drop() { + let directory = tempfile::tempdir().unwrap(); + let data = directory.path().join("data.sqlite"); + let first = acquire(&data, Duration::from_millis(100)).unwrap(); + let error = acquire(&data, Duration::from_millis(20)) + .unwrap_err() + .to_string(); + assert!( + error.contains("等待") && error.contains("超时"), + "got: {error}" + ); + drop(first); + acquire(&data, Duration::from_millis(100)).unwrap(); + assert!(data.with_file_name("data.sqlite.lock").exists()); + } +} diff --git a/src/data/transfer.rs b/src/data/transfer.rs new file mode 100644 index 0000000..f0df5e8 --- /dev/null +++ b/src/data/transfer.rs @@ -0,0 +1,2110 @@ +use super::{download_notice, sibling_path, DataSource, Progress}; +use anyhow::{anyhow, Context, Result}; +use flate2::read::MultiGzDecoder; +use sha2::{Digest, Sha256}; +use std::fs::{File, OpenOptions}; +use std::io::{self, Read, Seek, Write}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, Instant as StdInstant}; + +const MIB: u64 = 1024 * 1024; +const BUFFER_SIZE: usize = 64 * 1024; +static ARTIFACT_SEQUENCE: AtomicU64 = AtomicU64::new(0); + +#[derive(Clone, Copy, Debug)] +struct ProxyTunnelConnector; + +#[derive(Debug)] +struct ProxyTunnelTransport { + inner: T, +} + +impl ureq::unversioned::transport::Connector + for ProxyTunnelConnector +{ + type Out = ProxyTunnelTransport; + + fn connect( + &self, + _: &ureq::unversioned::transport::ConnectionDetails<'_>, + chained: Option, + ) -> std::result::Result, ureq::Error> { + Ok(chained.map(|inner| ProxyTunnelTransport { inner })) + } +} + +impl ureq::unversioned::transport::Transport + for ProxyTunnelTransport +{ + fn buffers(&mut self) -> &mut dyn ureq::unversioned::transport::Buffers { + self.inner.buffers() + } + + fn transmit_output( + &mut self, + amount: usize, + timeout: ureq::unversioned::transport::NextTimeout, + ) -> std::result::Result<(), ureq::Error> { + self.inner.transmit_output(amount, timeout) + } + + fn maybe_await_input( + &mut self, + timeout: ureq::unversioned::transport::NextTimeout, + ) -> std::result::Result { + self.inner.maybe_await_input(timeout) + } + + fn await_input( + &mut self, + timeout: ureq::unversioned::transport::NextTimeout, + ) -> std::result::Result { + self.inner.await_input(timeout) + } + + fn is_open(&mut self) -> bool { + self.inner.is_open() + } + + fn is_tls(&self) -> bool { + // The inner transport may be TLS-secured to an HTTPS proxy, but the + // CONNECT tunnel itself is plaintext from the target's perspective. + // Report false so the following RustlsConnector adds target TLS. + false + } +} + +// Ureq's socket adapter turns an already-expired zero timeout into one second +// because operating systems reject a zero socket timeout. TLS can also perform +// several lower-level reads for one HTTP-level read while reusing the original +// timeout. Recompute the global and connection-phase deadlines plus the +// per-read idle timeout at both transport layers so neither buffered protocol +// bytes nor a dribbling TLS record can extend a configured boundary. The +// transport API is unversioned, so ureq is pinned exactly in Cargo.toml. +#[derive(Clone, Copy, Debug)] +struct AbsoluteDeadlineConnector { + deadline: StdInstant, + connect_timeout: Duration, + idle_read_timeout: Duration, +} + +#[derive(Debug)] +struct AbsoluteDeadlineTransport { + inner: T, + deadline: StdInstant, + connect_timeout: Duration, + connect_phase_deadline: StdInstant, + connect_phase_active: bool, + idle_read_timeout: Duration, +} + +impl ureq::unversioned::transport::Connector + for AbsoluteDeadlineConnector +{ + type Out = AbsoluteDeadlineTransport; + + fn connect( + &self, + details: &ureq::unversioned::transport::ConnectionDetails<'_>, + chained: Option, + ) -> std::result::Result, ureq::Error> { + // ConnectionDetails::now is captured before the connector chain runs, + // so this includes TCP and CONNECT-proxy work that preceded this + // wrapper instead of restarting the budget after the socket opened. + let connect_phase_started = match details.now { + ureq::unversioned::transport::time::Instant::Exact(started) => started, + _ => { + return Err(ureq::Error::Io(io::Error::new( + io::ErrorKind::InvalidData, + "HTTP transport did not provide an exact connection start time", + ))); + } + }; + let connect_phase_deadline = checked_deadline(connect_phase_started, self.connect_timeout)?; + Ok(chained.map(|inner| AbsoluteDeadlineTransport { + inner, + deadline: self.deadline, + connect_timeout: self.connect_timeout, + connect_phase_deadline, + connect_phase_active: true, + idle_read_timeout: self.idle_read_timeout, + })) + } +} + +impl AbsoluteDeadlineTransport { + fn bound_timeout( + &mut self, + timeout: ureq::unversioned::transport::NextTimeout, + idle_read_timeout: Option, + ) -> std::result::Result { + self.bound_timeout_at(timeout, idle_read_timeout, StdInstant::now()) + } + + fn bound_timeout_at( + &mut self, + timeout: ureq::unversioned::transport::NextTimeout, + idle_read_timeout: Option, + now: StdInstant, + ) -> std::result::Result { + // ConnectProxyConnector drives its TransportAdapter with this sentinel + // while TLS-to-proxy and CONNECT are still part of the active connect + // phase. It may preserve an existing phase, but must never start one. + let connect_proxy_sentinel = self.connect_phase_active + && timeout.reason == ureq::Timeout::Global + && timeout.after.is_not_happening(); + let connect_deadline = if timeout.reason == ureq::Timeout::Connect { + if !self.connect_phase_active { + self.connect_phase_deadline = checked_deadline(now, self.connect_timeout)?; + self.connect_phase_active = true; + } + Some(self.connect_phase_deadline) + } else if connect_proxy_sentinel { + Some(self.connect_phase_deadline) + } else { + self.connect_phase_active = false; + None + }; + Ok(bound_transport_timeout( + timeout, + self.deadline, + connect_deadline, + idle_read_timeout, + now, + )) + } +} + +impl ureq::unversioned::transport::Transport + for AbsoluteDeadlineTransport +{ + fn buffers(&mut self) -> &mut dyn ureq::unversioned::transport::Buffers { + self.inner.buffers() + } + + fn transmit_output( + &mut self, + amount: usize, + timeout: ureq::unversioned::transport::NextTimeout, + ) -> std::result::Result<(), ureq::Error> { + let timeout = self.bound_timeout(timeout, None)?; + require_remaining_timeout(timeout)?; + self.inner.transmit_output(amount, timeout) + } + + fn maybe_await_input( + &mut self, + timeout: ureq::unversioned::transport::NextTimeout, + ) -> std::result::Result { + let timeout = self.bound_timeout(timeout, Some(self.idle_read_timeout))?; + require_remaining_timeout(timeout)?; + self.inner.maybe_await_input(timeout) + } + + fn await_input( + &mut self, + timeout: ureq::unversioned::transport::NextTimeout, + ) -> std::result::Result { + let timeout = self.bound_timeout(timeout, Some(self.idle_read_timeout))?; + require_remaining_timeout(timeout)?; + self.inner.await_input(timeout) + } + + fn is_open(&mut self) -> bool { + self.inner.is_open() + } + + fn is_tls(&self) -> bool { + self.inner.is_tls() + } +} + +fn bound_transport_timeout( + mut timeout: ureq::unversioned::transport::NextTimeout, + deadline: StdInstant, + connect_deadline: Option, + idle_read_timeout: Option, + now: StdInstant, +) -> ureq::unversioned::transport::NextTimeout { + let remaining = deadline.saturating_duration_since(now); + let global_after: ureq::unversioned::transport::time::Duration = remaining.into(); + if global_after < timeout.after { + timeout = ureq::unversioned::transport::NextTimeout { + after: global_after, + reason: ureq::Timeout::Global, + }; + } + + if let Some(connect_deadline) = connect_deadline { + let remaining = connect_deadline.saturating_duration_since(now); + let connect_after: ureq::unversioned::transport::time::Duration = remaining.into(); + if connect_after < timeout.after { + timeout = ureq::unversioned::transport::NextTimeout { + after: connect_after, + reason: ureq::Timeout::Connect, + }; + } + } + + if let Some(idle_read_timeout) = idle_read_timeout { + let idle_after: ureq::unversioned::transport::time::Duration = idle_read_timeout.into(); + if idle_after < timeout.after { + timeout = ureq::unversioned::transport::NextTimeout { + after: idle_after, + reason: ureq::Timeout::RecvResponse, + }; + } + } + + timeout +} + +fn checked_deadline( + started: StdInstant, + timeout: Duration, +) -> std::result::Result { + started.checked_add(timeout).ok_or_else(|| { + ureq::Error::Io(io::Error::new( + io::ErrorKind::InvalidInput, + "HTTP timeout exceeds the system clock range", + )) + }) +} + +fn require_remaining_timeout( + timeout: ureq::unversioned::transport::NextTimeout, +) -> std::result::Result<(), ureq::Error> { + if !timeout.after.is_not_happening() && timeout.after.is_zero() { + Err(ureq::Error::Timeout(timeout.reason)) + } else { + Ok(()) + } +} + +fn require_absolute_deadline(deadline: StdInstant) -> std::result::Result<(), ureq::Error> { + if StdInstant::now() >= deadline { + Err(ureq::Error::Timeout(ureq::Timeout::Global)) + } else { + Ok(()) + } +} + +#[derive(Clone, Copy, Debug)] +pub(super) struct DownloadPolicy { + pub connect_timeout: Duration, + pub idle_read_timeout: Duration, + pub http_timeout: Duration, + pub max_compressed: u64, + pub max_uncompressed: u64, +} + +pub(super) const PRODUCTION_POLICY: DownloadPolicy = DownloadPolicy { + connect_timeout: Duration::from_secs(30), + idle_read_timeout: Duration::from_secs(60), + http_timeout: Duration::from_secs(15 * 60), + max_compressed: 256 * MIB, + max_uncompressed: 768 * MIB, +}; + +pub(super) struct StagedCandidate { + path: PathBuf, + armed: bool, +} + +struct OwnedCompressed { + path: PathBuf, + armed: bool, +} + +impl OwnedCompressed { + fn remove_now(mut self) -> Result<()> { + self.armed = false; + match std::fs::remove_file(&self.path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(error) => { + Err(error).with_context(|| format!("删除压缩临时文件失败: {}", self.path.display())) + } + } + } + + fn cleanup_with(mut self, error: anyhow::Error) -> anyhow::Error { + self.armed = false; + match std::fs::remove_file(&self.path) { + Ok(()) => error, + Err(cleanup) if cleanup.kind() == io::ErrorKind::NotFound => error, + Err(cleanup) => error.context(format!( + "清理压缩临时文件失败: {}: {cleanup}", + self.path.display() + )), + } + } +} + +impl Drop for OwnedCompressed { + fn drop(&mut self) { + if self.armed { + let _ = std::fs::remove_file(&self.path); + } + } +} + +impl StagedCandidate { + #[cfg(test)] + pub(super) fn for_test(path: PathBuf) -> Self { + Self { path, armed: true } + } + + pub(super) fn path(&self) -> &Path { + &self.path + } + + pub(super) fn publish_succeeded(mut self) { + self.armed = false; + } + + #[cfg(any(test, windows))] + pub(super) fn preserve(mut self) { + self.armed = false; + } + + pub(super) fn cleanup_with(mut self, error: anyhow::Error) -> anyhow::Error { + self.armed = false; + match remove_artifact_family(&self.path) { + Ok(()) => error, + Err(cleanup) => error.context(format!("清理候选数据失败: {cleanup}")), + } + } +} + +impl Drop for StagedCandidate { + fn drop(&mut self) { + if self.armed { + let _ = remove_artifact_family(&self.path); + } + } +} + +fn unique_path(live_path: &Path, role: &str, extension: &str) -> Result { + let sequence = ARTIFACT_SEQUENCE.fetch_add(1, Ordering::Relaxed); + sibling_path( + live_path, + &format!(".{role}.{}.{sequence}{extension}", std::process::id()), + ) +} + +fn create_compressed_artifact(live_path: &Path) -> Result<(OwnedCompressed, File)> { + loop { + let path = unique_path(live_path, "download", ".gz")?; + match OpenOptions::new() + .read(true) + .write(true) + .create_new(true) + .open(&path) + { + Ok(file) => return Ok((OwnedCompressed { path, armed: true }, file)), + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => continue, + Err(error) => { + return Err(error) + .with_context(|| format!("创建压缩临时文件失败: {}", path.display())); + } + } + } +} + +fn create_candidate_artifact(live_path: &Path) -> Result<(StagedCandidate, File)> { + loop { + let path = unique_path(live_path, "candidate", "")?; + if let Some(candidate) = try_create_candidate_artifact(&path)? { + return Ok(candidate); + } + } +} + +fn try_create_candidate_artifact(path: &Path) -> Result> { + try_create_candidate_artifact_with(path, path_entry_exists) +} + +fn try_create_candidate_artifact_with( + path: &Path, + mut entry_exists: F, +) -> Result> +where + F: FnMut(&Path) -> Result, +{ + let sidecars = ["-journal", "-shm", "-wal"] + .map(|suffix| sibling_path(path, suffix)) + .into_iter() + .collect::>>()?; + let family_occupied = sidecars + .iter() + .try_fold(entry_exists(path)?, |occupied, sidecar| { + entry_exists(sidecar).map(|exists| occupied || exists) + })?; + if family_occupied { + return Ok(None); + } + + let file = match OpenOptions::new().write(true).create_new(true).open(path) { + Ok(file) => file, + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => return Ok(None), + Err(error) => { + return Err(error).with_context(|| format!("创建候选数据失败: {}", path.display())); + } + }; + + // The main file reserves this generation for cooperating fojin processes. + // Recheck the sidecars so a stale family is not claimed and later deleted + // by this guard. After this point, the operation lock ensures cooperating + // processes leave this unique family to SQLite. This naming protocol is + // coordination, not isolation from a same-user process that can modify the + // data directory concurrently. + let sidecar_occupied = sidecars.iter().try_fold(false, |occupied, sidecar| { + entry_exists(sidecar).map(|exists| occupied || exists) + }); + let sidecar_occupied = match sidecar_occupied { + Ok(occupied) => occupied, + Err(error) => { + drop(file); + return Err(cleanup_owned_candidate_main(path, error)); + } + }; + if sidecar_occupied { + drop(file); + std::fs::remove_file(path) + .with_context(|| format!("释放冲突候选数据失败: {}", path.display()))?; + return Ok(None); + } + + Ok(Some(( + StagedCandidate { + path: path.to_path_buf(), + armed: true, + }, + file, + ))) +} + +fn cleanup_owned_candidate_main(path: &Path, error: anyhow::Error) -> anyhow::Error { + match std::fs::remove_file(path) { + Ok(()) => error, + Err(cleanup) if cleanup.kind() == io::ErrorKind::NotFound => error, + Err(cleanup) => error.context(format!( + "清理候选数据主文件失败: {}: {cleanup}", + path.display() + )), + } +} + +fn path_entry_exists(path: &Path) -> Result { + match std::fs::symlink_metadata(path) { + Ok(_) => Ok(true), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(false), + Err(error) => { + Err(error).with_context(|| format!("检查临时数据路径失败: {}", path.display())) + } + } +} + +fn remove_artifact_family(path: &Path) -> Result<()> { + for suffix in ["", "-journal", "-shm", "-wal"] { + let artifact = if suffix.is_empty() { + path.to_path_buf() + } else { + sibling_path(path, suffix)? + }; + match std::fs::remove_file(&artifact) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => { + return Err(error) + .with_context(|| format!("删除候选数据失败: {}", artifact.display())); + } + } + } + Ok(()) +} + +pub(super) fn remove_known_artifacts(live_path: &Path) -> Result<()> { + let legacy = live_path.with_extension("tmp"); + match std::fs::remove_file(&legacy) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => { + return Err(error).with_context(|| format!("删除旧临时数据失败: {}", legacy.display())); + } + } + + let directory = live_path + .parent() + .ok_or_else(|| anyhow!("数据路径没有父目录: {}", live_path.display()))?; + let file_name = live_path + .file_name() + .ok_or_else(|| anyhow!("数据路径没有文件名: {}", live_path.display()))? + .to_str() + .ok_or_else(|| anyhow!("数据文件名不是有效 UTF-8: {}", live_path.display()))?; + for entry in std::fs::read_dir(directory) + .with_context(|| format!("读取数据目录失败: {}", directory.display()))? + { + let entry = entry.context("读取数据目录项失败")?; + let name = entry.file_name(); + let Some(name) = name.to_str() else { + continue; + }; + if !is_owned_artifact_name(file_name, name) { + continue; + } + if entry.file_type().context("读取临时数据类型失败")?.is_dir() { + continue; + } + std::fs::remove_file(entry.path()) + .with_context(|| format!("删除临时数据失败: {}", entry.path().display()))?; + } + Ok(()) +} + +fn is_owned_artifact_name(live_name: &str, artifact_name: &str) -> bool { + let Some(suffix) = artifact_name.strip_prefix(live_name) else { + return false; + }; + + if let Some(generation) = suffix + .strip_prefix(".download.") + .and_then(|name| name.strip_suffix(".gz")) + { + return is_numeric_generation(generation); + } + + let Some(candidate) = suffix.strip_prefix(".candidate.") else { + return false; + }; + let generation = ["-journal", "-shm", "-wal"] + .into_iter() + .find_map(|sidecar| candidate.strip_suffix(sidecar)) + .unwrap_or(candidate); + is_numeric_generation(generation) +} + +fn is_numeric_generation(value: &str) -> bool { + let mut parts = value.split('.'); + matches!( + (parts.next(), parts.next(), parts.next()), + (Some(pid), Some(sequence), None) + if !pid.is_empty() + && !sequence.is_empty() + && pid.bytes().all(|byte| byte.is_ascii_digit()) + && sequence.bytes().all(|byte| byte.is_ascii_digit()) + ) +} + +fn copy_bounded( + mut reader: impl Read, + mut writer: impl Write, + maximum: u64, + label: &str, +) -> Result { + let mut limited = reader.by_ref().take(maximum.saturating_add(1)); + let copied = io::copy(&mut limited, &mut writer).with_context(|| format!("{label}失败"))?; + if copied > maximum { + return Err(anyhow!("{label}超过限制: 最大 {maximum} 字节")); + } + Ok(copied) +} + +fn unpack_gzip(reader: impl Read, writer: impl Write, maximum: u64) -> Result { + let decoder = MultiGzDecoder::new(reader); + copy_bounded(decoder, writer, maximum, "解压 gzip") +} + +pub(super) fn stage_candidate( + live_path: &Path, + source: &DataSource<'_>, + policy: DownloadPolicy, +) -> Result { + let (compressed_guard, mut compressed_file) = create_compressed_artifact(live_path)?; + let staged = stage_candidate_inner(live_path, source, policy, &mut compressed_file); + match staged { + Ok(candidate) => { + finish_successful_stage_with(compressed_guard, candidate, OwnedCompressed::remove_now) + } + Err(error) => Err(compressed_guard.cleanup_with(error)), + } +} + +fn finish_successful_stage_with( + compressed_guard: OwnedCompressed, + candidate: StagedCandidate, + remove_compressed: RemoveCompressed, +) -> Result +where + RemoveCompressed: FnOnce(OwnedCompressed) -> Result<()>, +{ + match remove_compressed(compressed_guard) { + Ok(()) => Ok(candidate), + Err(error) => Err(candidate.cleanup_with(error)), + } +} + +fn stage_candidate_inner( + live_path: &Path, + source: &DataSource<'_>, + policy: DownloadPolicy, + compressed_file: &mut File, +) -> Result { + let deadline = StdInstant::now() + .checked_add(policy.http_timeout) + .ok_or_else(|| anyhow!("HTTP 总超时超出系统时间范围"))?; + let config = ureq::Agent::config_builder() + .max_redirects(5) + .max_redirects_will_error(true) + .max_response_header_size(64 * 1024) + .accept_encoding("identity") + .timeout_global(Some(policy.http_timeout)) + .timeout_resolve(Some(policy.connect_timeout)) + .timeout_connect(Some(policy.connect_timeout)) + .timeout_recv_body(Some(policy.idle_read_timeout)) + .build(); + let agent = build_agent(config, policy, deadline); + let mut response = agent + .get(source.url) + .header("Accept-Encoding", "identity") + .call() + .map_err(normalize_ureq_error) + .map_err(|error| download_error_context(error, "下载失败", live_path, source.url))?; + let declared = declared_length(&response, policy.max_compressed)?; + eprintln!("{}", download_notice(declared)); + + let mut reader = response + .body_mut() + .with_config() + .limit(policy.max_compressed.saturating_add(1)) + .reader(); + let mut progress = Progress::new(declared); + let mut digest = Sha256::new(); + let mut received = 0_u64; + let mut buffer = [0_u8; BUFFER_SIZE]; + loop { + require_absolute_deadline(deadline) + .map_err(normalize_ureq_error) + .map_err(|error| { + download_error_context(error, "读取响应失败", live_path, source.url) + })?; + let count = match reader.read(&mut buffer) { + Ok(count) => { + require_absolute_deadline(deadline) + .map_err(normalize_ureq_error) + .map_err(|error| { + download_error_context(error, "读取响应失败", live_path, source.url) + })?; + count + } + Err(error) => { + return Err(download_error_context( + anyhow::Error::new(normalize_body_read_error(error)), + "读取响应失败", + live_path, + source.url, + )); + } + }; + if count == 0 { + break; + } + received = received + .checked_add(count as u64) + .ok_or_else(|| anyhow!("下载大小溢出"))?; + if received > policy.max_compressed { + return Err(anyhow!( + "下载数据超过限制: 最大 {} 字节", + policy.max_compressed + )); + } + digest.update(&buffer[..count]); + compressed_file + .write_all(&buffer[..count]) + .context("写入压缩临时文件失败")?; + if let Some(message) = progress.advance(count as u64) { + eprintln!("{message}"); + } + } + require_declared_length(declared, received) + .map_err(|error| download_error_context(error, "读取响应失败", live_path, source.url))?; + let actual = digest.finalize(); + require_digest(actual.as_ref(), source.sha256).map_err(|error| { + anyhow!( + "{error}\n请手动下载:\n {}\n解压后放到: {}", + source.url, + live_path.display() + ) + })?; + compressed_file.flush().context("刷新压缩临时文件失败")?; + compressed_file.rewind().context("重置压缩临时文件失败")?; + + let (candidate_guard, mut candidate_file) = create_candidate_artifact(live_path)?; + let unpacked = (|| -> Result<()> { + unpack_gzip( + &mut *compressed_file, + &mut candidate_file, + policy.max_uncompressed, + )?; + candidate_file.flush().context("刷新候选数据失败")?; + candidate_file.sync_all().context("同步候选数据失败")?; + Ok(()) + })(); + drop(candidate_file); + match unpacked { + Ok(()) => Ok(candidate_guard), + Err(error) => Err(candidate_guard.cleanup_with(error)), + } +} + +fn build_agent( + config: ureq::config::Config, + policy: DownloadPolicy, + deadline: StdInstant, +) -> ureq::Agent { + use ureq::unversioned::transport::Connector; + let deadline_connector = AbsoluteDeadlineConnector { + deadline, + connect_timeout: policy.connect_timeout, + idle_read_timeout: policy.idle_read_timeout, + }; + let connector = + ().chain(ureq::unversioned::transport::ConnectProxyConnector::default()) + .chain(ProxyTunnelConnector) + .chain(ureq::unversioned::transport::TcpConnector::default()) + .chain(deadline_connector) + .chain(ureq::unversioned::transport::RustlsConnector::default()) + .chain(deadline_connector); + ureq::Agent::with_parts( + config, + connector, + ureq::unversioned::resolver::DefaultResolver::default(), + ) +} + +fn download_error_context( + error: anyhow::Error, + action: &str, + live_path: &Path, + url: &str, +) -> anyhow::Error { + let cause = error.to_string(); + error.context(format!( + "{action}: {url}: {cause}\n请手动下载:\n {url}\n解压后放到: {}", + live_path.display() + )) +} + +fn normalize_ureq_error(error: ureq::Error) -> anyhow::Error { + if matches!(error, ureq::Error::Timeout(_)) { + anyhow::Error::new(io::Error::new(io::ErrorKind::TimedOut, error)) + } else { + anyhow::Error::new(error) + } +} + +fn normalize_body_read_error(error: io::Error) -> io::Error { + let is_timeout = error + .get_ref() + .and_then(|source| source.downcast_ref::()) + .is_some_and(|source| matches!(source, ureq::Error::Timeout(_))); + if is_timeout { + io::Error::new(io::ErrorKind::TimedOut, error) + } else { + error + } +} + +fn declared_length( + response: &ureq::http::Response, + maximum: u64, +) -> Result> { + if response + .headers() + .get_all(ureq::http::header::TRANSFER_ENCODING) + .iter() + .next() + .is_some() + { + return Ok(None); + } + let values: Vec<_> = response + .headers() + .get_all(ureq::http::header::CONTENT_LENGTH) + .iter() + .collect(); + match values.as_slice() { + [] => Ok(None), + [value] => { + let value = value.to_str().context("Content-Length 不是有效 ASCII")?; + let parsed = value + .parse::() + .with_context(|| format!("无效 Content-Length: {value}"))?; + if parsed > maximum { + return Err(anyhow!("Content-Length 超过下载限制: {parsed} > {maximum}")); + } + Ok(Some(parsed)) + } + _ => Err(anyhow!("响应包含重复 Content-Length")), + } +} + +fn require_declared_length(declared: Option, received: u64) -> Result<()> { + if let Some(expected) = declared { + if expected != received { + return Err(anyhow!( + "响应长度不符: Content-Length={expected}, received={received}" + )); + } + } + Ok(()) +} + +fn require_digest(actual: &[u8], expected_hex: &str) -> Result<()> { + if expected_hex.len() != 64 || !expected_hex.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return Err(anyhow!("配置的 SHA-256 格式无效")); + } + let actual_hex: String = actual.iter().map(|byte| format!("{byte:02x}")).collect(); + if !actual_hex.eq_ignore_ascii_case(expected_hex) { + return Err(anyhow!("下载校验失败(sha256 不符)")); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use flate2::write::GzEncoder; + use flate2::Compression; + use std::io::{Cursor, Read, Write}; + use std::time::Instant; + use ureq::unversioned::transport::Buffers; + + #[derive(Debug)] + struct TlsMarkedTransport { + inner: T, + } + + impl ureq::unversioned::transport::Transport + for TlsMarkedTransport + { + fn buffers(&mut self) -> &mut dyn ureq::unversioned::transport::Buffers { + self.inner.buffers() + } + + fn transmit_output( + &mut self, + amount: usize, + timeout: ureq::unversioned::transport::NextTimeout, + ) -> std::result::Result<(), ureq::Error> { + self.inner.transmit_output(amount, timeout) + } + + fn maybe_await_input( + &mut self, + timeout: ureq::unversioned::transport::NextTimeout, + ) -> std::result::Result { + self.inner.maybe_await_input(timeout) + } + + fn await_input( + &mut self, + timeout: ureq::unversioned::transport::NextTimeout, + ) -> std::result::Result { + self.inner.await_input(timeout) + } + + fn is_open(&mut self) -> bool { + self.inner.is_open() + } + + fn is_tls(&self) -> bool { + true + } + } + + #[derive(Debug)] + struct LocalTcpTransport { + stream: std::net::TcpStream, + buffers: ureq::unversioned::transport::LazyBuffers, + } + + impl ureq::unversioned::transport::Transport for LocalTcpTransport { + fn buffers(&mut self) -> &mut dyn ureq::unversioned::transport::Buffers { + &mut self.buffers + } + + fn transmit_output( + &mut self, + amount: usize, + _: ureq::unversioned::transport::NextTimeout, + ) -> std::result::Result<(), ureq::Error> { + let output = &self.buffers.output()[..amount]; + self.stream.write_all(output).map_err(ureq::Error::Io) + } + + fn await_input( + &mut self, + _: ureq::unversioned::transport::NextTimeout, + ) -> std::result::Result { + let input = self.buffers.input_append_buf(); + let amount = self.stream.read(input).map_err(ureq::Error::Io)?; + self.buffers.input_appended(amount); + Ok(amount > 0) + } + + fn is_open(&mut self) -> bool { + true + } + } + + #[derive(Clone, Copy, Debug)] + struct FakeTlsProxyConnector { + address: std::net::SocketAddr, + } + + impl ureq::unversioned::transport::Connector<()> for FakeTlsProxyConnector { + type Out = TlsMarkedTransport; + + fn connect( + &self, + _: &ureq::unversioned::transport::ConnectionDetails<'_>, + _: Option<()>, + ) -> std::result::Result, ureq::Error> { + let stream = std::net::TcpStream::connect(self.address).map_err(ureq::Error::Io)?; + stream.set_nodelay(true).map_err(ureq::Error::Io)?; + Ok(Some(TlsMarkedTransport { + inner: LocalTcpTransport { + stream, + buffers: ureq::unversioned::transport::LazyBuffers::new(16 * 1024, 16 * 1024), + }, + })) + } + } + + fn policy_for_tests() -> DownloadPolicy { + DownloadPolicy { + connect_timeout: Duration::from_millis(200), + idle_read_timeout: Duration::from_millis(200), + http_timeout: Duration::from_secs(2), + max_compressed: 1024, + max_uncompressed: 4096, + } + } + + fn first_wire_prefix_through_tls_marked_proxy(target: &str) -> [u8; 5] { + use ureq::unversioned::transport::Connector; + + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let mut prefix = [0_u8; 5]; + stream.read_exact(&mut prefix).unwrap(); + prefix + }); + let connector = () + .chain(FakeTlsProxyConnector { address }) + .chain(ProxyTunnelConnector) + .chain(ureq::unversioned::transport::RustlsConnector::default()); + let config = ureq::Agent::config_builder() + .proxy(None) + .timeout_global(Some(Duration::from_secs(2))) + .timeout_connect(Some(Duration::from_secs(1))) + .build(); + let agent = ureq::Agent::with_parts( + config, + connector, + ureq::unversioned::resolver::DefaultResolver::default(), + ); + + let _ = agent.get(target).call(); + server.join().unwrap() + } + + #[test] + fn proxy_tunnel_hides_proxy_tls_from_the_target_connector() { + use ureq::unversioned::transport::Transport; + + let proxy_tls = TlsMarkedTransport { inner: () }; + assert!(proxy_tls.is_tls()); + let tunnel = ProxyTunnelTransport { inner: proxy_tls }; + + assert!(!tunnel.is_tls()); + } + + #[test] + fn tls_marked_proxy_still_starts_a_target_tls_handshake() { + let prefix = first_wire_prefix_through_tls_marked_proxy("https://127.0.0.1/data.gz"); + + assert_eq!(prefix[0], 0x16, "expected TLS handshake, got {prefix:02x?}"); + assert_eq!(prefix[1], 0x03, "expected TLS version, got {prefix:02x?}"); + assert!( + (0x01..=0x04).contains(&prefix[2]), + "expected TLS record version, got {prefix:02x?}" + ); + } + + #[test] + fn tls_marked_proxy_keeps_an_http_target_plaintext() { + let prefix = first_wire_prefix_through_tls_marked_proxy("http://127.0.0.1/data.gz"); + + assert_eq!(&prefix, b"GET /"); + } + + #[test] + fn candidate_reservation_rejects_a_preexisting_sidecar_family() { + let directory = tempfile::tempdir().unwrap(); + let candidate = directory.path().join("data.sqlite.candidate.123.4"); + let wal = sibling_path(&candidate, "-wal").unwrap(); + std::fs::write(&wal, b"foreign wal").unwrap(); + + assert!(try_create_candidate_artifact(&candidate).unwrap().is_none()); + assert!(!candidate.exists()); + assert_eq!(std::fs::read(wal).unwrap(), b"foreign wal"); + } + + #[test] + fn candidate_reservation_cleans_its_main_file_when_postcheck_fails() { + let directory = tempfile::tempdir().unwrap(); + let candidate = directory.path().join("data.sqlite.candidate.123.4"); + let checks = std::cell::Cell::new(0_usize); + + let result = try_create_candidate_artifact_with(&candidate, |_| { + let next = checks.get() + 1; + checks.set(next); + if next == 5 { + Err(anyhow!("injected sidecar inspection failure")) + } else { + Ok(false) + } + }); + let error = match result { + Err(error) => error, + Ok(_) => panic!("post-reservation inspection failure was ignored"), + }; + + assert!( + format!("{error:#}").contains("injected sidecar inspection failure"), + "got: {error:#}" + ); + assert!(!candidate.exists(), "owned candidate main leaked"); + } + + #[test] + fn compressed_removal_failure_attaches_candidate_cleanup_error() { + let directory = tempfile::tempdir().unwrap(); + let compressed_path = directory.path().join("data.sqlite.download.123.4.gz"); + std::fs::write(&compressed_path, b"compressed").unwrap(); + let compressed_guard = OwnedCompressed { + path: compressed_path, + armed: true, + }; + let candidate_path = directory.path().join("data.sqlite.candidate.123.4"); + std::fs::create_dir(&candidate_path).unwrap(); + let candidate = StagedCandidate::for_test(candidate_path); + + let result = finish_successful_stage_with(compressed_guard, candidate, |_| { + Err(anyhow!("injected compressed removal failure")) + }); + let error = match result { + Err(error) => error, + Ok(_) => panic!("compressed removal failure was ignored"), + }; + let detail = format!("{error:#}"); + + assert!( + detail.contains("injected compressed removal failure"), + "got: {detail}" + ); + assert!(detail.contains("清理候选数据失败"), "got: {detail}"); + } + + #[test] + fn connect_proxy_sentinel_keeps_the_original_connection_deadline() { + let started = Instant::now(); + let original_connect_deadline = started + Duration::from_millis(250); + let mut transport = AbsoluteDeadlineTransport { + inner: (), + deadline: started + Duration::from_secs(2), + connect_timeout: Duration::from_millis(250), + connect_phase_deadline: original_connect_deadline, + connect_phase_active: true, + idle_read_timeout: Duration::from_secs(1), + }; + let sentinel = ureq::unversioned::transport::NextTimeout { + after: ureq::unversioned::transport::time::Duration::NotHappening, + reason: ureq::Timeout::Global, + }; + + let first = transport + .bound_timeout_at(sentinel, None, started + Duration::from_millis(100)) + .unwrap(); + let expected_first: ureq::unversioned::transport::time::Duration = + Duration::from_millis(150).into(); + assert_eq!(first.reason, ureq::Timeout::Connect); + assert_eq!(first.after, expected_first); + assert!(transport.connect_phase_active); + assert_eq!(transport.connect_phase_deadline, original_connect_deadline); + + let second = transport + .bound_timeout_at(sentinel, None, started + Duration::from_millis(200)) + .unwrap(); + let expected_second: ureq::unversioned::transport::time::Duration = + Duration::from_millis(50).into(); + assert_eq!(second.reason, ureq::Timeout::Connect); + assert_eq!(second.after, expected_second); + assert!(transport.connect_phase_active); + assert_eq!(transport.connect_phase_deadline, original_connect_deadline); + } + + #[test] + fn finite_non_connect_timeout_ends_then_connect_restarts_the_phase() { + let started = Instant::now(); + let mut transport = AbsoluteDeadlineTransport { + inner: (), + deadline: started + Duration::from_secs(2), + connect_timeout: Duration::from_millis(250), + connect_phase_deadline: started + Duration::from_millis(250), + connect_phase_active: true, + idle_read_timeout: Duration::from_secs(1), + }; + let finite_global = ureq::unversioned::transport::NextTimeout { + after: Duration::from_millis(500).into(), + reason: ureq::Timeout::Global, + }; + + let finite = transport + .bound_timeout_at(finite_global, None, started + Duration::from_millis(50)) + .unwrap(); + assert_eq!(finite.reason, ureq::Timeout::Global); + assert!(!transport.connect_phase_active); + + let sentinel = ureq::unversioned::transport::NextTimeout { + after: ureq::unversioned::transport::time::Duration::NotHappening, + reason: ureq::Timeout::Global, + }; + let sentinel = transport + .bound_timeout_at(sentinel, None, started + Duration::from_millis(100)) + .unwrap(); + assert_eq!(sentinel.reason, ureq::Timeout::Global); + assert!(!transport.connect_phase_active); + + let restarted_at = started + Duration::from_millis(200); + let connect = ureq::unversioned::transport::NextTimeout { + after: Duration::from_millis(250).into(), + reason: ureq::Timeout::Connect, + }; + let connect = transport + .bound_timeout_at(connect, None, restarted_at) + .unwrap(); + assert_eq!(connect.reason, ureq::Timeout::Connect); + assert!(transport.connect_phase_active); + assert_eq!( + transport.connect_phase_deadline, + restarted_at + Duration::from_millis(250) + ); + } + + fn sha256_hex(bytes: &[u8]) -> String { + Sha256::digest(bytes) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() + } + + fn gzip(body: &[u8]) -> Vec { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(body).unwrap(); + encoder.finish().unwrap() + } + + fn read_request(stream: &mut std::net::TcpStream) -> Vec { + stream + .set_read_timeout(Some(Duration::from_secs(1))) + .unwrap(); + let mut request = Vec::new(); + let mut buffer = [0_u8; 1024]; + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let count = stream.read(&mut buffer).unwrap(); + if count == 0 { + break; + } + request.extend_from_slice(&buffer[..count]); + } + request + } + + fn stage_from_raw_response_with_sha( + response: &[u8], + sha256: &str, + policy: DownloadPolicy, + ) -> (Result<()>, Vec) { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let response = response.to_vec(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let request = read_request(&mut stream); + stream.write_all(&response).unwrap(); + request + }); + let directory = tempfile::tempdir().unwrap(); + let live = directory.path().join("data.sqlite"); + let url = format!("http://{address}/data.gz"); + let result = stage_candidate(&live, &DataSource { url: &url, sha256 }, policy).map(drop); + let request = server.join().unwrap(); + (result, request) + } + + fn stage_from_raw_response(response: &[u8], policy: DownloadPolicy) -> Result<()> { + stage_from_raw_response_with_sha(response, &"0".repeat(64), policy).0 + } + + fn stage_from_delayed_raw_response( + response: &[u8], + delay: Duration, + policy: DownloadPolicy, + ) -> Result<()> { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let response = response.to_vec(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let _ = read_request(&mut stream); + std::thread::sleep(delay); + stream.write_all(&response).unwrap(); + }); + let directory = tempfile::tempdir().unwrap(); + let live = directory.path().join("data.sqlite"); + let url = format!("http://{address}/data.gz"); + let sha = "0".repeat(64); + let result = stage_candidate( + &live, + &DataSource { + url: &url, + sha256: &sha, + }, + policy, + ) + .map(drop); + server.join().unwrap(); + result + } + + fn stage_from_dribbling_server(interval: Duration, policy: DownloadPolicy) -> Result<()> { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let _ = read_request(&mut stream); + stream + .write_all(b"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n") + .unwrap(); + for _ in 0..20 { + if stream.write_all(b"x").is_err() { + break; + } + if stream.flush().is_err() { + break; + } + std::thread::sleep(interval); + } + }); + let directory = tempfile::tempdir().unwrap(); + let live = directory.path().join("data.sqlite"); + let url = format!("http://{address}/data.gz"); + let sha = "0".repeat(64); + let result = stage_candidate( + &live, + &DataSource { + url: &url, + sha256: &sha, + }, + policy, + ) + .map(drop); + server.join().unwrap(); + result + } + + fn stage_from_pausing_server(pause: Duration, policy: DownloadPolicy) -> Result<()> { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let _ = read_request(&mut stream); + stream + .write_all(b"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\nx") + .unwrap(); + stream.flush().unwrap(); + std::thread::sleep(pause); + let _ = stream.write_all(b"x"); + }); + let directory = tempfile::tempdir().unwrap(); + let live = directory.path().join("data.sqlite"); + let url = format!("http://{address}/data.gz"); + let sha = "0".repeat(64); + let result = stage_candidate( + &live, + &DataSource { + url: &url, + sha256: &sha, + }, + policy, + ) + .map(drop); + server.join().unwrap(); + result + } + + fn stage_from_pausing_headers(pause: Duration, policy: DownloadPolicy) -> Result<()> { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let _ = read_request(&mut stream); + stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n") + .unwrap(); + stream.flush().unwrap(); + std::thread::sleep(pause); + let _ = stream.write_all(b"\r\n"); + }); + let directory = tempfile::tempdir().unwrap(); + let live = directory.path().join("data.sqlite"); + let url = format!("http://{address}/data.gz"); + let sha = "0".repeat(64); + let result = stage_candidate( + &live, + &DataSource { + url: &url, + sha256: &sha, + }, + policy, + ) + .map(drop); + server.join().unwrap(); + result + } + + fn stage_from_protocol_dribble( + prefix: &'static [u8], + dribble: u8, + count: usize, + suffix: &'static [u8], + interval: Duration, + policy: DownloadPolicy, + ) -> (Result<()>, Duration) { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let _ = read_request(&mut stream); + if stream.write_all(prefix).is_err() { + return; + } + for _ in 0..count { + if stream.write_all(&[dribble]).is_err() || stream.flush().is_err() { + return; + } + std::thread::sleep(interval); + } + let _ = stream.write_all(suffix); + }); + let directory = tempfile::tempdir().unwrap(); + let live = directory.path().join("data.sqlite"); + let url = format!("http://{address}/data.gz"); + let sha = "0".repeat(64); + let started = Instant::now(); + let result = stage_candidate( + &live, + &DataSource { + url: &url, + sha256: &sha, + }, + policy, + ) + .map(drop); + let elapsed = started.elapsed(); + server.join().unwrap(); + (result, elapsed) + } + + fn contains_io_kind(error: &anyhow::Error, expected: io::ErrorKind) -> bool { + error.chain().any(|cause| { + cause + .downcast_ref::() + .is_some_and(|error| error.kind() == expected) + }) + } + + fn contains_ureq_timeout(error: &anyhow::Error, expected: ureq::Timeout) -> bool { + fn source_contains_timeout( + source: &(dyn std::error::Error + 'static), + expected: ureq::Timeout, + ) -> bool { + if let Some(source) = source.downcast_ref::() { + return match source { + ureq::Error::Timeout(reason) => *reason == expected, + ureq::Error::Io(source) => source_contains_timeout(source, expected), + _ => false, + }; + } + source + .downcast_ref::() + .and_then(io::Error::get_ref) + .is_some_and(|source| source_contains_timeout(source, expected)) + } + + error + .chain() + .any(|cause| source_contains_timeout(cause, expected)) + } + + fn assert_no_owned_transfer_artifacts(live: &Path) { + let live_name = live.file_name().unwrap().to_string_lossy(); + let owned: Vec<_> = std::fs::read_dir(live.parent().unwrap()) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .filter(|name| { + is_owned_artifact_name(live_name.as_ref(), name.to_string_lossy().as_ref()) + }) + .collect(); + assert!( + owned.is_empty(), + "owned transfer artifacts remain: {owned:?}" + ); + } + + fn chunked_response(body: &[u8], extra_headers: &str) -> Vec { + let mut response = format!( + "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n{extra_headers}Connection: close\r\n\r\n{:x}\r\n", + body.len() + ) + .into_bytes(); + response.extend_from_slice(body); + response.extend_from_slice(b"\r\n0\r\n\r\n"); + response + } + + fn content_length_response(body: &[u8]) -> Vec { + let mut response = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ) + .into_bytes(); + response.extend_from_slice(body); + response + } + + fn stage_compressed(compressed: &[u8], policy: DownloadPolicy) -> Result<()> { + let sha = sha256_hex(compressed); + let response = content_length_response(compressed); + stage_from_raw_response_with_sha(&response, &sha, policy).0 + } + + #[test] + fn duplicate_content_length_is_rejected() { + let response = b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\nContent-Length: 4\r\nConnection: close\r\n\r\ntest"; + let error = stage_from_raw_response(response, policy_for_tests()) + .unwrap_err() + .to_string(); + assert!( + error.contains("Content-Length") && error.contains("重复"), + "got: {error}" + ); + } + + #[test] + fn declared_length_must_match_received_body() { + let response = b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\nConnection: close\r\n\r\ntest"; + let error = stage_from_raw_response(response, policy_for_tests()).unwrap_err(); + assert!( + error.to_string().contains("长度") || error.to_string().contains("读取响应失败"), + "got: {error:#}" + ); + assert!( + contains_io_kind(&error, io::ErrorKind::UnexpectedEof), + "got: {error:#}" + ); + assert!( + format!("{error:#}").contains("请手动下载"), + "got: {error:#}" + ); + } + + #[test] + fn total_timeout_stops_a_non_idle_dribble() { + let policy = DownloadPolicy { + connect_timeout: Duration::from_secs(1), + idle_read_timeout: Duration::from_millis(600), + http_timeout: Duration::from_millis(300), + max_compressed: 1024, + max_uncompressed: 4096, + }; + let error = stage_from_dribbling_server(Duration::from_millis(50), policy).unwrap_err(); + assert!( + error.to_string().contains("读取响应失败") || error.to_string().contains("timed out"), + "got: {error:#}" + ); + assert!( + contains_io_kind(&error, io::ErrorKind::TimedOut), + "got: {error:#}" + ); + assert!( + contains_ureq_timeout(&error, ureq::Timeout::Global), + "got: {error:#}" + ); + } + + #[test] + fn slow_progressing_valid_body_can_outlive_the_idle_window() { + let compressed = gzip(b"slow valid body"); + let sha = sha256_hex(&compressed); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let server_body = compressed.clone(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + stream.set_nodelay(true).unwrap(); + let _ = read_request(&mut stream); + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + server_body.len() + ) + .unwrap(); + for (index, byte) in server_body.iter().enumerate() { + stream.write_all(std::slice::from_ref(byte)).unwrap(); + stream.flush().unwrap(); + if index + 1 < server_body.len() { + std::thread::sleep(Duration::from_millis(20)); + } + } + }); + let directory = tempfile::tempdir().unwrap(); + let live = directory.path().join("data.sqlite"); + let url = format!("http://{address}/data.gz"); + let policy = DownloadPolicy { + connect_timeout: Duration::from_millis(500), + idle_read_timeout: Duration::from_millis(250), + http_timeout: Duration::from_secs(2), + max_compressed: 1024, + max_uncompressed: 4096, + }; + + let candidate = stage_candidate( + &live, + &DataSource { + url: &url, + sha256: &sha, + }, + policy, + ) + .expect("continuously progressing body should not hit the idle timeout"); + server.join().unwrap(); + + assert_eq!(std::fs::read(candidate.path()).unwrap(), b"slow valid body"); + } + + #[test] + fn global_deadline_interrupts_a_dribbling_tls_record() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + stream.set_nodelay(true).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let mut client_hello = [0_u8; 16 * 1024]; + let _ = stream.read(&mut client_hello).unwrap(); + stream.write_all(&[0x16, 0x03, 0x03, 0x40, 0x00]).unwrap(); + stream.flush().unwrap(); + for _ in 0..24 { + if stream.write_all(&[0]).is_err() || stream.flush().is_err() { + break; + } + std::thread::sleep(Duration::from_millis(50)); + } + }); + let directory = tempfile::tempdir().unwrap(); + let live = directory.path().join("data.sqlite"); + let url = format!("https://{address}/data.gz"); + let sha = "0".repeat(64); + let policy = DownloadPolicy { + connect_timeout: Duration::from_secs(1), + idle_read_timeout: Duration::from_secs(1), + http_timeout: Duration::from_millis(250), + max_compressed: 1024, + max_uncompressed: 4096, + }; + + let started = Instant::now(); + let error = stage_candidate( + &live, + &DataSource { + url: &url, + sha256: &sha, + }, + policy, + ) + .map(drop) + .unwrap_err(); + let elapsed = started.elapsed(); + server.join().unwrap(); + + assert!(elapsed < Duration::from_millis(700), "elapsed: {elapsed:?}"); + assert!( + contains_ureq_timeout(&error, ureq::Timeout::Global), + "got: {error:#}" + ); + assert_no_owned_transfer_artifacts(&live); + } + + #[test] + fn connect_deadline_interrupts_a_dribbling_tls_record() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + stream.set_nodelay(true).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let mut client_hello = [0_u8; 16 * 1024]; + let _ = stream.read(&mut client_hello).unwrap(); + stream.write_all(&[0x16, 0x03, 0x03, 0x40, 0x00]).unwrap(); + stream.flush().unwrap(); + for _ in 0..24 { + if stream.write_all(&[0]).is_err() || stream.flush().is_err() { + break; + } + std::thread::sleep(Duration::from_millis(50)); + } + }); + let directory = tempfile::tempdir().unwrap(); + let live = directory.path().join("data.sqlite"); + let url = format!("https://{address}/data.gz"); + let sha = "0".repeat(64); + let policy = DownloadPolicy { + connect_timeout: Duration::from_millis(250), + idle_read_timeout: Duration::from_secs(1), + http_timeout: Duration::from_secs(2), + max_compressed: 1024, + max_uncompressed: 4096, + }; + + let started = Instant::now(); + let error = stage_candidate( + &live, + &DataSource { + url: &url, + sha256: &sha, + }, + policy, + ) + .map(drop) + .unwrap_err(); + let elapsed = started.elapsed(); + server.join().unwrap(); + + assert!(elapsed < Duration::from_millis(700), "elapsed: {elapsed:?}"); + assert!( + contains_ureq_timeout(&error, ureq::Timeout::Connect), + "got: {error:#}" + ); + assert_no_owned_transfer_artifacts(&live); + } + + #[test] + fn https_connect_proxy_first_tls_handshake_obeys_connect_deadline() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + stream.set_nodelay(true).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let mut client_hello = [0_u8; 16 * 1024]; + let _ = stream.read(&mut client_hello).unwrap(); + stream.write_all(&[0x16, 0x03, 0x03, 0x40, 0x00]).unwrap(); + stream.flush().unwrap(); + for _ in 0..24 { + if stream.write_all(&[0]).is_err() || stream.flush().is_err() { + break; + } + std::thread::sleep(Duration::from_millis(50)); + } + }); + let policy = DownloadPolicy { + connect_timeout: Duration::from_millis(250), + idle_read_timeout: Duration::from_secs(1), + http_timeout: Duration::from_secs(2), + max_compressed: 1024, + max_uncompressed: 4096, + }; + let proxy = ureq::Proxy::new(&format!("https://{address}")).unwrap(); + let config = ureq::Agent::config_builder() + .proxy(Some(proxy)) + .max_redirects(5) + .max_redirects_will_error(true) + .max_response_header_size(64 * 1024) + .accept_encoding("identity") + .timeout_global(Some(policy.http_timeout)) + .timeout_resolve(Some(policy.connect_timeout)) + .timeout_connect(Some(policy.connect_timeout)) + .timeout_recv_body(Some(policy.idle_read_timeout)) + .build(); + let started = Instant::now(); + let deadline = started + policy.http_timeout; + let agent = build_agent(config, policy, deadline); + + let error = anyhow::Error::new( + agent + .get("http://example.invalid/data.gz") + .call() + .unwrap_err(), + ); + let elapsed = started.elapsed(); + server.join().unwrap(); + + assert!(elapsed < Duration::from_millis(700), "elapsed: {elapsed:?}"); + assert!( + contains_ureq_timeout(&error, ureq::Timeout::Connect), + "got: {error:#}" + ); + } + + #[test] + fn global_timeout_interrupts_continuously_dribbling_response_headers() { + let policy = DownloadPolicy { + connect_timeout: Duration::from_secs(1), + idle_read_timeout: Duration::from_millis(500), + http_timeout: Duration::from_millis(250), + max_compressed: 1024, + max_uncompressed: 4096, + }; + let (result, elapsed) = stage_from_protocol_dribble( + b"HTTP/1.1 200 OK\r\nX-Slow: ", + b'a', + 40, + b"\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + Duration::from_millis(50), + policy, + ); + let error = result.unwrap_err(); + assert!(elapsed < Duration::from_secs(1), "elapsed: {elapsed:?}"); + assert!( + contains_io_kind(&error, io::ErrorKind::TimedOut), + "got: {error:#}" + ); + assert!( + contains_ureq_timeout(&error, ureq::Timeout::Global), + "got: {error:#}" + ); + assert!( + format!("{error:#}").contains("请手动下载"), + "got: {error:#}" + ); + } + + #[test] + fn global_timeout_interrupts_continuously_dribbling_chunk_framing() { + let policy = DownloadPolicy { + connect_timeout: Duration::from_secs(1), + idle_read_timeout: Duration::from_millis(500), + http_timeout: Duration::from_millis(250), + max_compressed: 1024, + max_uncompressed: 4096, + }; + let (result, elapsed) = stage_from_protocol_dribble( + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n1;", + b'a', + 40, + b"\r\nx\r\n0\r\n\r\n", + Duration::from_millis(50), + policy, + ); + let error = result.unwrap_err(); + assert!(elapsed < Duration::from_secs(1), "elapsed: {elapsed:?}"); + assert!( + contains_io_kind(&error, io::ErrorKind::TimedOut), + "got: {error:#}" + ); + assert!( + contains_ureq_timeout(&error, ureq::Timeout::Global), + "got: {error:#}" + ); + assert!( + format!("{error:#}").contains("请手动下载"), + "got: {error:#}" + ); + } + + #[test] + fn oversized_declared_length_is_rejected_before_body_reads() { + let response = b"HTTP/1.1 200 OK\r\nContent-Length: 1025\r\nConnection: close\r\n\r\n"; + let error = stage_from_raw_response(response, policy_for_tests()) + .unwrap_err() + .to_string(); + assert!( + error.contains("Content-Length") && error.contains("1025") && error.contains("1024"), + "got: {error}" + ); + assert!(!error.contains("读取响应失败"), "got: {error}"); + } + + #[test] + fn invalid_content_length_is_rejected_before_body_reads() { + let response = b"HTTP/1.1 200 OK\r\nContent-Length: nope\r\nConnection: close\r\n\r\n"; + let error = stage_from_raw_response(response, policy_for_tests()) + .unwrap_err() + .to_string(); + let lower = error.to_ascii_lowercase(); + assert!( + lower.contains("content-length") + && (error.contains("无效") || lower.contains("not a number")), + "got: {error}" + ); + assert!(!error.contains("读取响应失败"), "got: {error}"); + } + + #[test] + fn missing_content_length_uses_received_body_size() { + let compressed = gzip(b"test"); + let sha = sha256_hex(&compressed); + let mut response = b"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n".to_vec(); + response.extend_from_slice(&compressed); + stage_from_raw_response_with_sha(&response, &sha, policy_for_tests()) + .0 + .unwrap(); + } + + #[test] + fn transfer_encoding_makes_content_length_non_authoritative() { + let compressed = gzip(b"test"); + let sha = sha256_hex(&compressed); + let response = chunked_response(&compressed, "Content-Length: 1\r\n"); + stage_from_raw_response_with_sha(&response, &sha, policy_for_tests()) + .0 + .unwrap(); + } + + #[test] + fn chunked_body_is_rejected_at_compressed_limit_plus_one() { + let body = vec![b'x'; 1025]; + let response = chunked_response(&body, ""); + let error = stage_from_raw_response(&response, policy_for_tests()) + .unwrap_err() + .to_string(); + assert!( + error.contains("下载数据超过限制") && error.contains("1024"), + "got: {error}" + ); + } + + #[test] + fn idle_body_pause_reports_timeout_with_io_source() { + let policy = DownloadPolicy { + idle_read_timeout: Duration::from_millis(100), + ..policy_for_tests() + }; + let error = stage_from_pausing_server(Duration::from_millis(300), policy).unwrap_err(); + assert!(error.to_string().contains("读取响应失败"), "got: {error:#}"); + assert!( + contains_io_kind(&error, io::ErrorKind::TimedOut), + "got: {error:#}" + ); + assert!( + contains_ureq_timeout(&error, ureq::Timeout::RecvBody), + "got: {error:#}" + ); + let detail = format!("{error:#}"); + assert!(detail.contains("http://"), "got: {detail}"); + assert!(detail.contains("data.sqlite"), "got: {detail}"); + assert!(detail.contains("请手动下载"), "got: {detail}"); + } + + #[test] + fn request_disables_transport_content_decoding() { + let response = b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"; + let (_, request) = + stage_from_raw_response_with_sha(response, &"0".repeat(64), policy_for_tests()); + let request = String::from_utf8(request).unwrap(); + assert!( + request + .lines() + .any(|line| line.eq_ignore_ascii_case("Accept-Encoding: identity")), + "got request: {request}" + ); + } + + #[test] + fn content_encoding_cannot_bypass_wire_length_limit() { + let encoded = gzip(b"x"); + let maximum = encoded.len() as u64 - 1; + let mut response = format!( + "HTTP/1.1 200 OK\r\nContent-Encoding: gzip\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + encoded.len() + ) + .into_bytes(); + response.extend_from_slice(&encoded); + let policy = DownloadPolicy { + max_compressed: maximum, + ..policy_for_tests() + }; + let error = stage_from_raw_response(&response, policy) + .unwrap_err() + .to_string(); + assert!( + error.contains("Content-Length") && error.contains("超过"), + "got: {error}" + ); + } + + #[test] + fn http_status_error_has_stable_download_context() { + let response = + b"HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"; + let error = stage_from_raw_response(response, policy_for_tests()) + .unwrap_err() + .to_string(); + assert!( + error.contains("下载失败") && error.contains("请手动下载"), + "got: {error}" + ); + } + + #[test] + fn total_deadline_is_checked_after_http_status_response() { + let policy = DownloadPolicy { + idle_read_timeout: Duration::from_millis(300), + http_timeout: Duration::from_millis(100), + ..policy_for_tests() + }; + let response = + b"HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"; + let error = stage_from_delayed_raw_response(response, Duration::from_millis(150), policy) + .unwrap_err(); + assert!( + contains_io_kind(&error, io::ErrorKind::TimedOut), + "got: {error:#}" + ); + } + + #[test] + fn header_idle_timeout_preserves_io_source() { + let policy = DownloadPolicy { + idle_read_timeout: Duration::from_millis(100), + ..policy_for_tests() + }; + let error = stage_from_pausing_headers(Duration::from_millis(300), policy).unwrap_err(); + assert!(error.to_string().contains("下载失败"), "got: {error:#}"); + assert!( + contains_io_kind(&error, io::ErrorKind::TimedOut), + "got: {error:#}" + ); + assert!( + contains_ureq_timeout(&error, ureq::Timeout::RecvResponse), + "got: {error:#}" + ); + } + + #[test] + fn unpack_reads_all_concatenated_gzip_members() { + let mut compressed = gzip(b"one"); + compressed.extend_from_slice(&gzip(b"two")); + let mut unpacked = Vec::new(); + assert_eq!( + unpack_gzip(Cursor::new(compressed), &mut unpacked, 6).unwrap(), + 6 + ); + assert_eq!(unpacked, b"onetwo"); + } + + #[test] + fn stage_accepts_gzip_at_exact_uncompressed_limit() { + let policy = DownloadPolicy { + max_uncompressed: 4, + ..policy_for_tests() + }; + stage_compressed(&gzip(b"test"), policy).unwrap(); + } + + #[test] + fn truncated_gzip_trailer_is_rejected_during_staging() { + let mut compressed = gzip(b"test"); + compressed.truncate(compressed.len() - 4); + let error = stage_compressed(&compressed, policy_for_tests()) + .unwrap_err() + .to_string(); + assert!(error.contains("解压 gzip失败"), "got: {error}"); + } + + #[test] + fn modified_gzip_crc_is_rejected_during_staging() { + let mut compressed = gzip(b"test"); + let crc = compressed.len() - 8; + compressed[crc] ^= 0xff; + let error = stage_compressed(&compressed, policy_for_tests()) + .unwrap_err() + .to_string(); + assert!(error.contains("解压 gzip失败"), "got: {error}"); + } + + #[test] + fn trailing_non_gzip_bytes_are_rejected_during_staging() { + let mut compressed = gzip(b"test"); + compressed.extend_from_slice(b"not-gzip"); + let error = stage_compressed(&compressed, policy_for_tests()) + .unwrap_err() + .to_string(); + assert!(error.contains("解压 gzip失败"), "got: {error}"); + } + + #[test] + fn bounded_copy_accepts_exact_limit_and_rejects_one_more() { + let mut exact = Vec::new(); + assert_eq!( + copy_bounded(Cursor::new(b"1234"), &mut exact, 4, "test").unwrap(), + 4 + ); + assert_eq!(exact, b"1234"); + + let mut oversized = Vec::new(); + let error = copy_bounded(Cursor::new(b"12345"), &mut oversized, 4, "test") + .unwrap_err() + .to_string(); + assert!(error.contains("4"), "got: {error}"); + } + + #[test] + fn unpack_accepts_exact_limit_and_rejects_expansion() { + let gzip = |body: &[u8]| { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(body).unwrap(); + encoder.finish().unwrap() + }; + + let mut exact = Vec::new(); + unpack_gzip(Cursor::new(gzip(b"1234")), &mut exact, 4).unwrap(); + assert_eq!(exact, b"1234"); + + let mut oversized = Vec::new(); + let error = unpack_gzip(Cursor::new(gzip(b"12345")), &mut oversized, 4) + .unwrap_err() + .to_string(); + assert!( + error.contains("解压") && error.contains("4"), + "got: {error}" + ); + } +} diff --git a/tests/command.rs b/tests/command.rs index cdff623..1c5be76 100644 --- a/tests/command.rs +++ b/tests/command.rs @@ -293,12 +293,15 @@ fn data_status_reports_missing_data_without_downloading() { fn data_clean_removes_file_and_is_idempotent() { let dir = tempfile::tempdir().unwrap(); write_fixture_db(dir.path()); + let lock_path = dir.path().join("data.sqlite.lock"); let out = run_fojin(&["data", "clean"], dir.path()); assert_eq!(out.status.code(), Some(0)); assert!(!dir.path().join("data.sqlite").exists()); + assert!(lock_path.is_file()); let out2 = run_fojin(&["data", "clean"], dir.path()); assert_eq!(out2.status.code(), Some(0)); assert!(String::from_utf8(out2.stdout).unwrap().contains("无数据")); + assert!(lock_path.is_file()); } #[test] diff --git a/tests/data.rs b/tests/data.rs index 27adee2..5c262b5 100644 --- a/tests/data.rs +++ b/tests/data.rs @@ -1,9 +1,9 @@ use fojin_cli::data::{ - ensure_data, gunzip, open_compatible_db, open_read_only_db, resolve_data_path, update_data, - validate_compatibility, verify_dataset, verify_sha256, DataSource, EXPECTED_DATA_VERSION, - EXPECTED_NORM_RULESET, + clean_data, ensure_data, gunzip, open_compatible_db, open_read_only_db, resolve_data_path, + update_data, validate_compatibility, verify_dataset, verify_dataset_file, verify_sha256, + DataSource, EXPECTED_DATA_VERSION, EXPECTED_NORM_RULESET, }; -use std::io::Write; +use std::io::{Read, Write}; use std::path::PathBuf; #[test] @@ -53,6 +53,181 @@ fn present_file_is_a_noop() { assert!(ensure_data(&path, false, &src).is_ok()); } +#[test] +fn ensure_data_rechecks_existence_after_waiting_for_lock() { + let directory = tempfile::tempdir().unwrap(); + let data = directory.path().join("data.sqlite"); + let lock_path = sibling_path(&data, ".lock"); + let blocker = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(lock_path) + .unwrap(); + blocker.lock().unwrap(); + + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let source_url = format!("http://{}/data.gz", listener.local_addr().unwrap()); + let server = std::thread::spawn(move || { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(1); + loop { + match listener.accept() { + Ok((mut stream, _)) => { + let mut request = [0_u8; 4096]; + let _ = std::io::Read::read(&mut stream, &mut request); + stream + .write_all( + b"HTTP/1.1 500 Internal Server Error\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + ) + .unwrap(); + return 1_usize; + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + if std::time::Instant::now() >= deadline { + return 0; + } + std::thread::sleep(std::time::Duration::from_millis(5)); + } + Err(error) => panic!("accept failed: {error}"), + } + } + }); + + let worker_data = data.clone(); + let (sender, receiver) = std::sync::mpsc::channel(); + let worker = std::thread::spawn(move || { + let source = DataSource { + url: &source_url, + sha256: "unused", + }; + sender + .send(ensure_data(&worker_data, false, &source)) + .unwrap(); + }); + + let early = receiver.recv_timeout(std::time::Duration::from_millis(100)); + let waited_for_lock = matches!(&early, Err(std::sync::mpsc::RecvTimeoutError::Timeout)); + std::fs::write(&data, b"installed by competing process").unwrap(); + drop(blocker); + + let result = match early { + Ok(result) => result, + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => receiver + .recv_timeout(std::time::Duration::from_secs(2)) + .expect("ensure_data remained blocked after the lock was released"), + Err(error) => panic!("ensure_data worker disconnected: {error}"), + }; + worker.join().unwrap(); + let requests = server.join().unwrap(); + assert!( + waited_for_lock, + "ensure_data did not wait for the operation lock: {result:?}" + ); + assert_eq!(requests, 0, "lock waiter attempted a redundant download"); + result.unwrap(); + assert_eq!( + std::fs::read(data).unwrap(), + b"installed by competing process" + ); +} + +#[test] +fn clean_data_waits_for_operation_lock_before_deleting() { + let directory = tempfile::tempdir().unwrap(); + let data = directory.path().join("data.sqlite"); + std::fs::write(&data, b"live").unwrap(); + let lock_path = sibling_path(&data, ".lock"); + let blocker = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(&lock_path) + .unwrap(); + blocker.lock().unwrap(); + + let worker_data = data.clone(); + let (sender, receiver) = std::sync::mpsc::channel(); + let worker = std::thread::spawn(move || { + sender.send(clean_data(&worker_data)).unwrap(); + }); + + assert!(matches!( + receiver.recv_timeout(std::time::Duration::from_millis(100)), + Err(std::sync::mpsc::RecvTimeoutError::Timeout) + )); + assert!(data.is_file(), "clean deleted live data without the lock"); + drop(blocker); + + assert_eq!( + receiver + .recv_timeout(std::time::Duration::from_secs(2)) + .expect("clean remained blocked after the lock was released") + .unwrap(), + Some(4) + ); + worker.join().unwrap(); + assert!(!data.exists()); + assert!(lock_path.is_file()); +} + +#[test] +fn clean_data_removes_known_file_artifacts_and_preserves_other_entries() { + let directory = tempfile::tempdir().unwrap(); + let data = directory.path().join("data.sqlite"); + let legacy = data.with_extension("tmp"); + let download = sibling_path(&data, ".download.123.0.gz"); + let candidate = sibling_path(&data, ".candidate.123.1"); + let candidate_journal = sibling_path(&data, ".candidate.123.1-journal"); + let matching_directory = sibling_path(&data, ".download.keep"); + let lock_path = sibling_path(&data, ".lock"); + let unrelated = sibling_path(&data, ".unrelated"); + let lookalikes = [ + sibling_path(&data, ".candidate.notes"), + sibling_path(&data, ".candidate.123"), + sibling_path(&data, ".candidate.123.x"), + sibling_path(&data, ".candidate.123.1.notes"), + sibling_path(&data, ".candidate.123.1-journal.notes"), + sibling_path(&data, ".download.123.0"), + sibling_path(&data, ".download.123.x.gz"), + sibling_path(&data, ".download.123.0.gz.notes"), + ]; + std::fs::write(&data, b"live").unwrap(); + for artifact in [&legacy, &download, &candidate, &candidate_journal] { + std::fs::write(artifact, b"temporary").unwrap(); + } + std::fs::create_dir(&matching_directory).unwrap(); + std::fs::write(&lock_path, b"permanent lock inode").unwrap(); + std::fs::write(&unrelated, b"unrelated").unwrap(); + for lookalike in &lookalikes { + std::fs::write(lookalike, b"not owned").unwrap(); + } + + assert_eq!(clean_data(&data).unwrap(), Some(4)); + for removed in [&data, &legacy, &download, &candidate, &candidate_journal] { + assert!(!removed.exists(), "artifact remains: {}", removed.display()); + } + assert!(matching_directory.is_dir()); + assert_eq!(std::fs::read(&lock_path).unwrap(), b"permanent lock inode"); + assert_eq!(std::fs::read(&unrelated).unwrap(), b"unrelated"); + for lookalike in &lookalikes { + assert_eq!( + std::fs::read(lookalike).unwrap(), + b"not owned", + "clean removed lookalike: {}", + lookalike.display() + ); + } + + assert_eq!(clean_data(&data).unwrap(), None); + assert!(lock_path.is_file()); + assert!(matching_directory.is_dir()); + assert!(unrelated.is_file()); + assert!(lookalikes.iter().all(|path| path.is_file())); +} + #[test] fn write_atomic_writes_content_and_leaves_no_temp() { use fojin_cli::data::write_atomic; @@ -106,17 +281,11 @@ fn download_notice_mentions_size_and_offline() { } /// Serve one HTTP response on localhost and exercise the full download path: -/// stream -> sha256 verify -> gunzip -> atomic write. No external network. +/// stream -> sha256 verify -> bounded gunzip -> verified publication. No external network. #[test] fn ensure_data_downloads_verifies_and_unpacks() { - let mut enc = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); - enc.write_all(b"fake sqlite payload").unwrap(); - let gz = enc.finish().unwrap(); - - use sha2::{Digest, Sha256}; - let mut h = Sha256::new(); - h.update(&gz); - let sha: String = h.finalize().iter().map(|b| format!("{b:02x}")).collect(); + let gz = gzip_bytes(&replacement_database_bytes()); + let sha = sha256_hex(&gz); let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let port = listener.local_addr().unwrap().port(); @@ -142,11 +311,291 @@ fn ensure_data_downloads_verifies_and_unpacks() { ensure_data(&path, false, &source).unwrap(); server.join().unwrap(); - assert_eq!(std::fs::read(&path).unwrap(), b"fake sqlite payload"); - assert!( - !path.with_extension("tmp").exists(), - "temp file must not linger" - ); + verify_dataset_file(&path).unwrap(); +} + +#[test] +fn concurrent_first_install_downloads_once() { + if std::env::var_os("FOJIN_CONCURRENT_WORKER").is_some() { + let path = PathBuf::from(std::env::var_os("FOJIN_WORKER_DATA").unwrap()); + let url = std::env::var("FOJIN_WORKER_URL").unwrap(); + let sha = std::env::var("FOJIN_WORKER_SHA256").unwrap(); + ensure_data( + &path, + false, + &DataSource { + url: &url, + sha256: &sha, + }, + ) + .unwrap(); + return; + } + use std::fs::OpenOptions; + use std::process::{Child, Command, Stdio}; + use std::time::{Duration, Instant}; + + fn terminate_children(children: [&mut Child; 2]) { + for child in children { + let _ = child.kill(); + let _ = child.wait(); + } + } + + fn wait_for_children(mut children: [&mut Child; 2], timeout: Duration) { + let deadline = Instant::now() + timeout; + let mut finished = [false; 2]; + while !finished.iter().all(|done| *done) { + let mut failed = None; + for (index, child) in children.iter_mut().enumerate() { + if finished[index] { + continue; + } + if let Some(status) = child.try_wait().unwrap() { + finished[index] = true; + if !status.success() { + failed = Some((index, status)); + break; + } + } + } + if let Some((failed_index, status)) = failed { + for (index, child) in children.iter_mut().enumerate() { + if !finished[index] { + let _ = child.kill(); + let _ = child.wait(); + } + } + panic!("worker {failed_index} failed with {status}"); + } + if Instant::now() >= deadline { + for (index, child) in children.iter_mut().enumerate() { + if !finished[index] { + let _ = child.kill(); + let _ = child.wait(); + } + } + panic!("concurrent install workers exceeded {timeout:?}"); + } + std::thread::sleep(Duration::from_millis(10)); + } + } + + let directory = tempfile::tempdir().unwrap(); + let data_path = directory.path().join("data.sqlite"); + let lock_path = sibling_path(&data_path, ".lock"); + let parent_lock = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(&lock_path) + .unwrap(); + parent_lock.lock().unwrap(); + let body = gzip_bytes(&replacement_database_bytes()); + let sha = sha256_hex(&body); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let url = format!("http://{}/data.gz", listener.local_addr().unwrap()); + let server_body = body.clone(); + let (request_started_tx, request_started_rx) = std::sync::mpsc::channel(); + let (early_request_count_tx, early_request_count_rx) = std::sync::mpsc::channel(); + let (release_response_tx, release_response_rx) = std::sync::mpsc::channel(); + let server = std::thread::spawn(move || { + let mut requests = 0_usize; + let accept_deadline = Instant::now() + Duration::from_secs(5); + let mut first = loop { + match listener.accept() { + Ok((stream, _)) => { + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + break stream; + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + assert!( + Instant::now() < accept_deadline, + "no worker began the first download" + ); + std::thread::sleep(Duration::from_millis(10)); + } + Err(error) => panic!("accept failed: {error}"), + } + }; + requests += 1; + let mut request = [0_u8; 4096]; + let _ = first.read(&mut request); + request_started_tx.send(()).unwrap(); + + let early_deadline = Instant::now() + Duration::from_millis(300); + while Instant::now() < early_deadline { + match listener.accept() { + Ok((mut stream, _)) => { + requests += 1; + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let _ = stream.read(&mut request); + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + std::thread::sleep(Duration::from_millis(10)); + } + Err(error) => panic!("accept failed: {error}"), + } + } + early_request_count_tx.send(requests).unwrap(); + release_response_rx + .recv_timeout(Duration::from_secs(5)) + .expect("parent did not release the first response"); + + write!( + first, + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + server_body.len() + ) + .unwrap(); + first.write_all(&server_body).unwrap(); + drop(first); + + let deadline = Instant::now() + Duration::from_secs(1); + while Instant::now() < deadline { + match listener.accept() { + Ok((mut stream, _)) => { + requests += 1; + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let _ = stream.read(&mut request); + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + server_body.len() + ) + .unwrap(); + stream.write_all(&server_body).unwrap(); + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + std::thread::sleep(Duration::from_millis(10)); + } + Err(error) => panic!("accept failed: {error}"), + } + } + requests + }); + + let first_log = directory.path().join("worker-1.stderr"); + let second_log = directory.path().join("worker-2.stderr"); + let spawn_worker = |stderr_path: &std::path::Path| { + Command::new(std::env::current_exe().unwrap()) + .arg("--exact") + .arg("concurrent_first_install_downloads_once") + .arg("--nocapture") + .env("FOJIN_CONCURRENT_WORKER", "1") + .env("FOJIN_WORKER_DATA", &data_path) + .env("FOJIN_WORKER_URL", &url) + .env("FOJIN_WORKER_SHA256", &sha) + .stdout(Stdio::null()) + .stderr(Stdio::from(std::fs::File::create(stderr_path).unwrap())) + .spawn() + .unwrap() + }; + let mut first = spawn_worker(&first_log); + let mut second = spawn_worker(&second_log); + + let barrier_deadline = Instant::now() + Duration::from_secs(5); + loop { + let first_stderr = std::fs::read_to_string(&first_log).unwrap_or_default(); + let second_stderr = std::fs::read_to_string(&second_log).unwrap_or_default(); + let marker = "检测到另一个 fojin 数据操作,正在等待"; + if first_stderr.contains(marker) && second_stderr.contains(marker) { + break; + } + if Instant::now() >= barrier_deadline { + let _ = first.kill(); + let _ = second.kill(); + let _ = first.wait(); + let _ = second.wait(); + panic!( + "workers did not reach the lock barrier; first={first_stderr:?}, second={second_stderr:?}" + ); + } + std::thread::sleep(Duration::from_millis(10)); + } + + drop(parent_lock); + if let Err(error) = request_started_rx.recv_timeout(Duration::from_secs(5)) { + let _ = release_response_tx.send(()); + terminate_children([&mut first, &mut second]); + let _ = server.join(); + panic!("neither worker began downloading after the parent released the lock: {error}"); + } + let early_requests = match early_request_count_rx.recv_timeout(Duration::from_secs(2)) { + Ok(requests) => requests, + Err(error) => { + let _ = release_response_tx.send(()); + terminate_children([&mut first, &mut second]); + let _ = server.join(); + panic!("server did not report the pre-release request count: {error}"); + } + }; + if early_requests != 1 { + let _ = release_response_tx.send(()); + terminate_children([&mut first, &mut second]); + let _ = server.join(); + panic!("both workers downloaded while the first response was blocked"); + } + let first_early = first.try_wait().unwrap(); + let second_early = second.try_wait().unwrap(); + if first_early.is_some() || second_early.is_some() { + let _ = release_response_tx.send(()); + terminate_children([&mut first, &mut second]); + let _ = server.join(); + panic!("a worker exited before the first download was released"); + } + if release_response_tx.send(()).is_err() { + terminate_children([&mut first, &mut second]); + let _ = server.join(); + panic!("server exited before the first response was released"); + } + + wait_for_children([&mut first, &mut second], Duration::from_secs(10)); + assert_eq!(server.join().unwrap(), 1); + verify_dataset_file(&data_path).unwrap(); + assert_no_owned_transfer_artifacts(&data_path); +} + +#[test] +fn first_install_rejects_incompatible_database() { + let gz = gzip_bytes(b"fake sqlite payload"); + let sha = sha256_hex(&gz); + + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + let body = gz.clone(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = [0u8; 4096]; + let _ = std::io::Read::read(&mut stream, &mut request); + let head = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nContent-Type: application/gzip\r\nConnection: close\r\n\r\n", + body.len() + ); + stream.write_all(head.as_bytes()).unwrap(); + stream.write_all(&body).unwrap(); + }); + + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("data.sqlite"); + let source = DataSource { + url: &format!("http://127.0.0.1:{port}/data.gz"), + sha256: &sha, + }; + let error = ensure_data(&path, false, &source).unwrap_err().to_string(); + server.join().unwrap(); + + assert!(error.contains("dataset incompatibility"), "got: {error}"); + assert!(!path.exists(), "incompatible dataset was published"); + assert_no_owned_transfer_artifacts(&path); } #[test] @@ -479,11 +928,15 @@ fn update_data_preserves_live_dataset_when_candidate_fails_compatibility() { assert!(err.contains("dataset incompatibility"), "got: {err}"); assert_eq!(std::fs::read(&path).unwrap(), b"live dataset"); - let entries: Vec<_> = std::fs::read_dir(dir.path()) + let entries: std::collections::BTreeSet<_> = std::fs::read_dir(dir.path()) .unwrap() .map(|entry| entry.unwrap().file_name()) .collect(); - assert_eq!(entries, vec![std::ffi::OsString::from("data.sqlite")]); + let expected: std::collections::BTreeSet<_> = ["data.sqlite", "data.sqlite.lock"] + .into_iter() + .map(std::ffi::OsString::from) + .collect(); + assert_eq!(entries, expected); } #[test] @@ -500,6 +953,103 @@ fn update_data_replaces_valid_existing_dataset() { assert_no_candidate_artifacts(&path); } +#[test] +fn update_download_keeps_live_dataset_readable_without_query_locking() { + let directory = tempfile::tempdir().unwrap(); + let data = directory.path().join("data.sqlite"); + let live = compatible_database_bytes(|connection| { + connection + .execute("INSERT INTO meta(key, value) VALUES ('marker', 'live')", []) + .unwrap(); + }); + std::fs::write(&data, live).unwrap(); + + let replacement = gzip_bytes(&replacement_database_bytes()); + let sha = sha256_hex(&replacement); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let source_url = format!("http://{}/data.gz", listener.local_addr().unwrap()); + let (midpoint_sender, midpoint_receiver) = std::sync::mpsc::channel(); + let (release_sender, release_receiver) = std::sync::mpsc::channel(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = [0_u8; 4096]; + let _ = std::io::Read::read(&mut stream, &mut request); + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + replacement.len() + ) + .unwrap(); + let midpoint = replacement.len() / 2; + stream.write_all(&replacement[..midpoint]).unwrap(); + stream.flush().unwrap(); + midpoint_sender.send(()).unwrap(); + release_receiver.recv().unwrap(); + stream.write_all(&replacement[midpoint..]).unwrap(); + }); + + let update_path = data.clone(); + let updater = std::thread::spawn(move || { + update_data( + &update_path, + &DataSource { + url: &source_url, + sha256: &sha, + }, + ) + }); + midpoint_receiver + .recv_timeout(std::time::Duration::from_secs(2)) + .expect("updater did not begin its download"); + + let lock_path = sibling_path(&data, ".lock"); + let lock_probe = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(lock_path) + .unwrap(); + let update_holds_lock = match lock_probe.try_lock() { + Err(std::fs::TryLockError::WouldBlock) => true, + Ok(()) => false, + Err(std::fs::TryLockError::Error(error)) => { + panic!("probing the update operation lock failed: {error}") + } + }; + drop(lock_probe); + + let query_path = data.clone(); + let (query_sender, query_receiver) = std::sync::mpsc::channel(); + let query = std::thread::spawn(move || { + let result = (|| { + let connection = open_compatible_db(&query_path)?; + let marker = + connection.query_row("SELECT value FROM meta WHERE key = 'marker'", [], |row| { + row.get::<_, String>(0) + })?; + Ok::<_, anyhow::Error>(marker) + })(); + query_sender.send(result).unwrap(); + }); + let live_marker = query_receiver + .recv_timeout(std::time::Duration::from_secs(1)) + .expect("ordinary query waited for the data operation lock") + .unwrap(); + query.join().unwrap(); + + release_sender.send(()).unwrap(); + server.join().unwrap(); + let update_result = updater.join().unwrap(); + assert!( + update_holds_lock, + "update_data did not hold the operation lock while downloading" + ); + assert_eq!(live_marker, "live"); + update_result.unwrap(); + assert_replacement_marker(&data); +} + #[test] fn update_data_installs_valid_dataset_when_target_is_absent() { let dir = tempfile::tempdir().unwrap(); @@ -561,7 +1111,7 @@ fn update_data_preserves_live_dataset_on_decompression_failure() { .unwrap_err() .to_string(); - assert!(err.contains("解压 gzip 失败"), "got: {err}"); + assert!(err.contains("解压 gzip"), "got: {err}"); assert_eq!(std::fs::read(&path).unwrap(), b"old live dataset"); assert_no_candidate_artifacts(&path); } @@ -733,24 +1283,53 @@ fn assert_no_candidate_artifacts(path: &std::path::Path) { artifact.display() ); } - assert_no_owned_candidate_artifacts(path); + assert_no_owned_transfer_artifacts(path); } -fn assert_no_owned_candidate_artifacts(path: &std::path::Path) { - let mut prefix = path.file_name().unwrap().to_os_string(); - prefix.push(".candidate."); - let prefix = prefix.to_string_lossy(); +fn assert_no_owned_transfer_artifacts(path: &std::path::Path) { + let live_name = path.file_name().unwrap().to_string_lossy(); let owned: Vec<_> = std::fs::read_dir(path.parent().unwrap()) .unwrap() .map(|entry| entry.unwrap().file_name()) - .filter(|name| name.to_string_lossy().starts_with(prefix.as_ref())) + .filter(|name| { + let name = name.to_string_lossy(); + let Some(suffix) = name.strip_prefix(live_name.as_ref()) else { + return false; + }; + if let Some(generation) = suffix + .strip_prefix(".download.") + .and_then(|rest| rest.strip_suffix(".gz")) + { + return is_numeric_generation(generation); + } + let Some(candidate) = suffix.strip_prefix(".candidate.") else { + return false; + }; + let generation = ["-journal", "-shm", "-wal"] + .into_iter() + .find_map(|sidecar| candidate.strip_suffix(sidecar)) + .unwrap_or(candidate); + is_numeric_generation(generation) + }) .collect(); assert!( owned.is_empty(), - "owned candidate artifacts remain: {owned:?}" + "owned transfer artifacts remain: {owned:?}" ); } +fn is_numeric_generation(value: &str) -> bool { + let mut parts = value.split('.'); + matches!( + (parts.next(), parts.next(), parts.next()), + (Some(pid), Some(sequence), None) + if !pid.is_empty() + && !sequence.is_empty() + && pid.bytes().all(|byte| byte.is_ascii_digit()) + && sequence.bytes().all(|byte| byte.is_ascii_digit()) + ) +} + fn sibling_path(path: &std::path::Path, suffix: &str) -> PathBuf { let mut name = path.file_name().unwrap().to_os_string(); name.push(suffix);