From e85eb2f159d0782ffb0422dbea84713794015c44 Mon Sep 17 00:00:00 2001 From: xr843 <137012659+xr843@users.noreply.github.com> Date: Sat, 11 Jul 2026 21:33:12 +0800 Subject: [PATCH 01/14] docs: design streaming data installation --- ...26-07-11-streaming-data-download-design.md | 259 ++++++++++++++++++ 1 file changed, 259 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-11-streaming-data-download-design.md diff --git a/docs/superpowers/specs/2026-07-11-streaming-data-download-design.md b/docs/superpowers/specs/2026-07-11-streaming-data-download-design.md new file mode 100644 index 0000000..f283291 --- /dev/null +++ b/docs/superpowers/specs/2026-07-11-streaming-data-download-design.md @@ -0,0 +1,259 @@ +# Streaming Data Download and Single-Flight Installation + +Status: approved design + +Date: 2026-07-11 + +## Context + +The current data path keeps the complete gzip response and the complete +decompressed SQLite database in two `Vec` values. With the published +`data-v1` artifact this means at least roughly 183 MiB plus 561 MiB of live +payload memory, excluding allocation capacity and library buffers. + +The current 900-second `ureq::AgentBuilder::timeout_read` is an individual +socket-read timeout, not a deadline for the complete request. The response and +gzip output are also unbounded. Finally, first installation publishes through +the shared name `data.tmp`, so concurrent processes can write and rename the +same temporary file. + +## Goals + +- Keep transfer and decompression memory bounded by small fixed buffers. +- Enforce compressed and decompressed byte limits. +- Enforce connect, idle-read, total HTTP, and lock-wait timeouts. +- Allow only one data download per data directory at a time. +- Make first installation and explicit update use the same fully validated + candidate-publish path. +- Keep an existing live database byte-for-byte unchanged on every failure + before a successful atomic replacement. +- Preserve the existing CLI contract and data-v1 URL/checksum pinning. +- Work on Linux, macOS, and Windows with Rust 1.95. + +## Non-goals + +- Download resumption or range requests. +- Automatic retry or mirror selection. +- Remote data-version discovery. +- Data schema or data-v1 artifact changes. +- Changing normal query ranking, output, or offline behavior. +- Serializing ordinary readers behind a long-running download lock. + +## Chosen Approach + +Use a two-stage disk pipeline protected by a persistent OS file lock: + +```text +HTTP response + -> unique compressed sibling (incremental SHA-256 and compressed limit) + -> MultiGzDecoder + -> unique SQLite candidate (decompressed limit) + -> file sync + -> schema/version + quick-check + FTS integrity verification + -> file sync + -> atomic publish +``` + +This keeps memory O(buffer size) and verifies the authenticated compressed +artifact before spending work on decompression. It uses about 183 MiB more +temporary disk than a one-pass decoder, but has simpler and safer checksum, +trailing-data, and cleanup semantics. + +## Components and Boundaries + +### Transfer module + +A focused private submodule under `data` owns: + +- `DownloadPolicy`, including all byte and time limits; +- strict response-length handling; +- streaming HTTP reads, progress, compressed byte counting, and incremental + SHA-256; +- streaming multi-member gzip decoding and decompressed byte counting; +- uniquely named temporary artifact creation and ordinary-failure cleanup. + +It does not open SQLite, decide whether an install is needed, or publish a +candidate as the live database. + +Production defaults are: + +- connect timeout: 30 seconds; +- idle read timeout: 60 seconds; +- total HTTP timeout: 15 minutes; +- lock wait timeout: 20 minutes; +- compressed limit: 256 MiB; +- decompressed limit: 768 MiB; +- transfer buffer: 64 KiB. + +Tests can inject smaller limits and timeouts without changing production +constants. + +### Operation lock + +The lock path is a permanent sibling named `data.sqlite.lock`. The +implementation opens it with `create(true)` and uses the Rust standard +library's OS file-lock API, available below the project's Rust 1.95 MSRV. No +new locking dependency is needed. + +Lock acquisition uses `try_lock` with bounded polling rather than an +unbounded blocking call. It reports once that another fojin process is active, +then fails clearly after the lock-wait deadline. The lock file is never deleted: +process exit or crash releases the OS lock, so there is no stale-lock breaking +algorithm and no lock-file deletion race. + +- `ensure_data` retains its unlocked existing-file fast path. If data is + missing, it creates the parent directory, acquires the operation lock, and + checks again before downloading. +- `update_data` acquires the same lock for its complete operation, preventing + duplicate downloads and last-finisher version reversal. +- `data clean` acquires the same lock before deleting the live data and known + temporary artifacts. It leaves the lock file in place. +- Ordinary queries do not acquire this lock and can keep using the previous + live database while an update downloads and validates its candidate. + +On Windows, a non-cooperating reader may still prevent final replacement. The +existing replacement recovery contract remains: a candidate is preserved only +when replacement entered an ambiguous recovery state, and the error reports +its exact path. The download lock is not extended to readers because doing so +would block queries for the full transfer duration. + +### Artifact lifecycle + +Compressed and decompressed artifacts are siblings of the live database so +the final rename stays on one filesystem. Names include the live filename, +artifact role, process ID, and an atomic sequence, and are opened with +`create_new(true)`: + +- `data.sqlite.download...gz` +- `data.sqlite.candidate..` + +An ownership guard removes only artifacts created by the current operation. +It is disarmed only after a successful publish or an intentional Windows +preservation result. Recoverable errors never sweep another process's files. + +## Detailed Data Flow + +1. Acquire the operation lock and perform the mode-specific second check. +2. Create a unique read/write compressed artifact. +3. Build a ureq agent with connect, idle-read, and total HTTP timeouts. Send + `Accept-Encoding: identity` so the bytes counted and hashed are the release + asset bytes rather than transparent HTTP content decoding. +4. Treat `Content-Length` as an early rejection/progress hint, never as the + resource limit. A missing length is allowed. When transfer encoding does + not override a present value, reject non-numeric values, duplicates, and + values above the compressed limit. +5. Read the response with a fixed buffer. Before every write, checked-add the + cumulative count, enforce the compressed limit, update SHA-256, write the + chunk, and emit decile progress when a trustworthy total is available. +6. At EOF, require any usable declared length to match the received count and + compare the final digest with the pinned SHA-256. A mismatch deletes the + compressed artifact before decompression starts. +7. Rewind the compressed file and stream a `MultiGzDecoder` into a unique + candidate. Copy at most `max_uncompressed + 1` bytes so an exact-limit gzip + still reads and verifies its trailer while an oversized stream is detected. + Multi-member gzip is supported; truncation, invalid CRC/ISIZE, or trailing + non-gzip data fails closed. +8. Flush, `sync_all`, and close the candidate. Run the existing complete + schema/version, `PRAGMA quick_check`, and FTS content-integrity checks. + First installation now receives the same validation as explicit update. +9. Sync the validated candidate again and publish with the existing + platform-specific atomic replacement logic. +10. Remove the compressed artifact and any ordinary candidate sidecars. + +No production transfer path returns the complete response or decompressed +database as a `Vec`. Existing public byte helpers may remain for source +compatibility and small unit tests, but the installer and updater do not call +them. + +## Error and Recovery Contract + +Errors distinguish these stages: + +- operation-lock wait timeout or lock I/O failure; +- connection, HTTP deadline, idle read, status, or response read failure; +- invalid, duplicate, mismatched, or oversized response length; +- compressed stream limit exceeded; +- SHA-256 mismatch; +- gzip format, trailer, or decompressed stream limit failure; +- candidate creation, write, flush, or sync failure; +- SQLite compatibility, quick-check, or FTS integrity failure; +- final platform-specific replacement failure. + +Network and checksum failures retain the existing manual-download guidance. +Every failure before successful publish satisfies both invariants: + +1. an existing live database is unchanged; and +2. a first installation does not expose a live database path. + +Ordinary failures remove both owned artifacts and SQLite sidecars. If cleanup +also fails, the cleanup error is attached as context without hiding the primary +failure. The intentionally preserved Windows recovery case is the only +exception, and it explicitly reports the validated candidate path. + +## Testing Strategy + +### Deterministic unit and local HTTP tests + +- compressed size exactly at the limit and one byte over; +- decompressed size exactly at the limit and one byte over; +- known oversized `Content-Length` rejected before body consumption; +- missing length and chunked transfer constrained by the actual byte counter; +- invalid, duplicate, and mismatched declared lengths; +- valid digest and digest mismatch; +- single-member and multi-member gzip; +- truncated gzip, bad trailer, and trailing garbage; +- a small gzip that expands past the configured limit; +- a server that continuously dribbles bytes: idle timeout does not fire but + the injected total HTTP deadline does; +- a body pause longer than the injected idle timeout; +- injected reader/writer failures without platform-specific `/dev/full`; +- every error leaves an existing live database unchanged and removes owned + artifacts. + +### Process-level concurrency tests + +A test re-executes the Rust test binary as two worker processes against one +slow local HTTP server and one temporary data directory. It asserts: + +- both callers succeed; +- the server observes exactly one request; +- the waiter reuses the installed database after the lock-time second check; +- the final SQLite and FTS verification succeeds; +- no owned compressed or candidate artifacts remain. + +Additional tests serialize update versus update and clean versus install, and +verify that the lock is released after worker exit. Windows CI retains the +existing replacement recovery tests and exercises the new lock path. + +### Regression suite + +The acceptance run includes: + +- Rust 1.95 formatting, Clippy with warnings denied, all locked tests, release + build, and package verification; +- Python normalization parity; +- release and installer shell contracts; +- ShellCheck and actionlint when their covered files change. + +## Documentation and Delivery + +README documents the 256 MiB compressed limit, 768 MiB decompressed limit, +timeouts, single-flight behavior, temporary disk requirement, and unchanged +offline behavior. CHANGELOG records the bounded-memory transfer and concurrent +installation fix. + +Delivery uses the established repository process: focused commits on an +isolated branch, independent code review, GitHub pull request, all checks green, +then merge to `master`. This work does not create a tag or GitHub Release. + +## Acceptance Criteria + +- The production download/update path has no full-response or full-database + in-memory allocation. +- Compressed input above 256 MiB and decompressed output above 768 MiB fail + before publish. +- A continuously active response cannot exceed the 15-minute HTTP deadline. +- Two concurrent first installations make one HTTP request and both succeed. +- First installation and update both perform complete SQLite/FTS validation. +- All injected failures preserve the live database and clean owned artifacts. +- Rust, Python, shell-contract, Windows, and GitHub CI checks pass. From a5c1b668b73e5c4298831a93b7eb07fc7c58b96a Mon Sep 17 00:00:00 2001 From: xr843 <137012659+xr843@users.noreply.github.com> Date: Sat, 11 Jul 2026 21:50:15 +0800 Subject: [PATCH 02/14] docs: plan streaming data installation --- .../2026-07-11-streaming-data-download.md | 1175 +++++++++++++++++ 1 file changed, 1175 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-11-streaming-data-download.md diff --git a/docs/superpowers/plans/2026-07-11-streaming-data-download.md b/docs/superpowers/plans/2026-07-11-streaming-data-download.md new file mode 100644 index 0000000..c51096b --- /dev/null +++ b/docs/superpowers/plans/2026-07-11-streaming-data-download.md @@ -0,0 +1,1175 @@ +# Streaming Data Download Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace the unbounded in-memory data download with a bounded two-stage disk pipeline and serialize data mutations so concurrent first runs perform exactly one download. + +**Architecture:** A private `data::transfer` module streams the authenticated gzip asset to a unique sibling and then streams a `MultiGzDecoder` into a unique SQLite candidate. A private `data::operation_lock` module provides a persistent, time-bounded OS file lock used by ensure, update, and clean; `data.rs` retains SQLite validation and platform-specific atomic publication. + +**Tech Stack:** Rust 2021 on Rust 1.95, ureq 2.12.1, sha2 0.10, flate2 1.1.9, rusqlite 0.40.1, standard-library `File::try_lock`, local `TcpListener` fixtures, GitHub Actions. + +## Global Constraints + +- Keep the MSRV at Rust 1.95 and add no locking dependency. +- Production limits are connect 30 seconds, idle read 60 seconds, total HTTP 15 minutes, lock wait 20 minutes, compressed 256 MiB, decompressed 768 MiB, and buffer 64 KiB. +- Send `Accept-Encoding: identity`; SHA-256 covers the downloaded release-asset bytes. +- First install and update both run schema/version, quick-check, and FTS integrity validation before publish. +- Existing data remains byte-for-byte unchanged on every failure before atomic publish. +- Ordinary queries never wait for the long-running mutation lock. +- Temporary files are same-directory, `create_new` siblings owned and cleaned by one operation. +- Do not add retries, resume support, mirrors, data discovery, schema changes, a tag, or a GitHub Release. +- Use TDD, focused commits, independent review, PR checks, and merge to `master`. + +--- + +## File Map + +- Create `src/data/transfer.rs`: download policy, strict response metadata, incremental hashing, bounded disk transfer, multi-member gzip decoding, and owned artifact cleanup. +- Create `src/data/operation_lock.rs`: permanent lock-file acquisition with bounded `try_lock` polling. +- Modify `src/data.rs`: module wiring, unified candidate validation/publication, lock integration, and clean API. +- Modify `src/cli.rs`: delegate `data clean` to the locked data-layer API. +- Modify `tests/data.rs`: end-to-end failure invariants and process-level concurrent installation. +- Modify `tests/command.rs`: clean/lock-file CLI behavior. +- Modify `README.md`: memory, disk, timeout, size, and concurrency contract. +- Modify `CHANGELOG.md`: unreleased bounded-transfer and concurrency entries. + +--- + +### Task 1: Build and adopt the bounded two-stage transfer pipeline + +**Files:** +- Create: `src/data/transfer.rs` +- Modify: `src/data.rs:1-205` +- Modify: `src/data.rs:419-445` +- Test: `src/data/transfer.rs` +- Test: `tests/data.rs:108-166` +- Test: `tests/data.rs:430-570` + +**Interfaces:** +- Consumes: `super::DataSource`, `super::Progress`, `super::download_notice`, `super::sibling_path`, and the existing replacement/verification functions in `data.rs`. +- Produces: `transfer::DownloadPolicy`, `transfer::PRODUCTION_POLICY`, `transfer::StagedCandidate`, and `transfer::stage_candidate(&Path, &DataSource, DownloadPolicy) -> Result`. + +- [ ] **Step 1: Add failing bounded-transfer tests** + +Create `src/data/transfer.rs`, wire it with `mod transfer;` in `src/data.rs`, and start with unit tests that specify exact-limit and over-limit behavior: + +```rust +#[cfg(test)] +mod tests { + use super::*; + use flate2::write::GzEncoder; + use flate2::Compression; + use std::io::{Cursor, Write}; + + #[test] + fn bounded_copy_accepts_exact_limit_and_rejects_one_more() { + let mut exact = Vec::new(); + assert_eq!(copy_bounded(Cursor::new(b"1234"), &mut exact, 4, "test").unwrap(), 4); + assert_eq!(exact, b"1234"); + + let mut oversized = Vec::new(); + let error = copy_bounded(Cursor::new(b"12345"), &mut oversized, 4, "test") + .unwrap_err() + .to_string(); + assert!(error.contains("4"), "got: {error}"); + } + + #[test] + fn unpack_accepts_exact_limit_and_rejects_expansion() { + let gzip = |body: &[u8]| { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(body).unwrap(); + encoder.finish().unwrap() + }; + + let mut exact = Vec::new(); + unpack_gzip(Cursor::new(gzip(b"1234")), &mut exact, 4).unwrap(); + assert_eq!(exact, b"1234"); + + let mut oversized = Vec::new(); + let error = unpack_gzip(Cursor::new(gzip(b"12345")), &mut oversized, 4) + .unwrap_err() + .to_string(); + assert!(error.contains("解压") && error.contains("4"), "got: {error}"); + } +} +``` + +In `tests/data.rs`, change `ensure_data_downloads_verifies_and_unpacks` to +serve `gzip_bytes(&replacement_database_bytes())` and finish by calling +`verify_dataset_file(&path)`. Add `first_install_rejects_incompatible_database` +using the original `b"fake sqlite payload"`; its SHA is valid, but +`ensure_data` must return a dataset-incompatibility error, leave `path` +absent, and leave no owned candidate artifacts. This integration test fails +against the current weak first-install path. + +- [ ] **Step 2: Run the new tests and confirm RED** + +Run: + +```bash +cargo +1.95.0 test --lib data::transfer::tests --locked +``` + +Expected: compilation fails because `copy_bounded` and `unpack_gzip` do not exist. + +- [ ] **Step 3: Implement policy, bounded copy, gzip handling, and owned artifacts** + +Add these concrete interfaces to `src/data/transfer.rs`: + +```rust +use super::{download_notice, sibling_path, DataSource, Progress}; +use anyhow::{anyhow, Context, Result}; +use flate2::read::MultiGzDecoder; +use sha2::{Digest, Sha256}; +use std::fs::{File, OpenOptions}; +use std::io::{self, Read, Seek, Write}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +const MIB: u64 = 1024 * 1024; +const BUFFER_SIZE: usize = 64 * 1024; +static ARTIFACT_SEQUENCE: AtomicU64 = AtomicU64::new(0); + +#[derive(Clone, Copy, Debug)] +pub(super) struct DownloadPolicy { + pub connect_timeout: Duration, + pub idle_read_timeout: Duration, + pub http_timeout: Duration, + pub max_compressed: u64, + pub max_uncompressed: u64, +} + +pub(super) const PRODUCTION_POLICY: DownloadPolicy = DownloadPolicy { + connect_timeout: Duration::from_secs(30), + idle_read_timeout: Duration::from_secs(60), + http_timeout: Duration::from_secs(15 * 60), + max_compressed: 256 * MIB, + max_uncompressed: 768 * MIB, +}; + +pub(super) struct StagedCandidate { + path: PathBuf, + armed: bool, +} + +struct OwnedCompressed { + path: PathBuf, + armed: bool, +} + +impl OwnedCompressed { + fn remove_now(mut self) -> Result<()> { + self.armed = false; + match std::fs::remove_file(&self.path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error) + .with_context(|| format!("删除压缩临时文件失败: {}", self.path.display())), + } + } + + fn cleanup_with(mut self, error: anyhow::Error) -> anyhow::Error { + self.armed = false; + match std::fs::remove_file(&self.path) { + Ok(()) => error, + Err(cleanup) if cleanup.kind() == io::ErrorKind::NotFound => error, + Err(cleanup) => error.context(format!( + "清理压缩临时文件失败: {}: {cleanup}", + self.path.display() + )), + } + } +} + +impl Drop for OwnedCompressed { + fn drop(&mut self) { + if self.armed { + let _ = std::fs::remove_file(&self.path); + } + } +} + +impl StagedCandidate { + pub(super) fn path(&self) -> &Path { + &self.path + } + + pub(super) fn publish_succeeded(mut self) { + self.armed = false; + } + + pub(super) fn preserve(mut self) { + self.armed = false; + } + + pub(super) fn cleanup_with(mut self, error: anyhow::Error) -> anyhow::Error { + self.armed = false; + match remove_artifact_family(&self.path) { + Ok(()) => error, + Err(cleanup) => error.context(format!("清理候选数据失败: {cleanup}")), + } + } +} + +impl Drop for StagedCandidate { + fn drop(&mut self) { + if self.armed { + let _ = remove_artifact_family(&self.path); + } + } +} + +fn unique_path(live_path: &Path, role: &str, extension: &str) -> Result { + let sequence = ARTIFACT_SEQUENCE.fetch_add(1, Ordering::Relaxed); + sibling_path( + live_path, + &format!(".{role}.{}.{sequence}{extension}", std::process::id()), + ) +} + +fn create_compressed_artifact(live_path: &Path) -> Result<(OwnedCompressed, File)> { + loop { + let path = unique_path(live_path, "download", ".gz")?; + match OpenOptions::new() + .read(true) + .write(true) + .create_new(true) + .open(&path) + { + Ok(file) => return Ok((OwnedCompressed { path, armed: true }, file)), + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => continue, + Err(error) => { + return Err(error) + .with_context(|| format!("创建压缩临时文件失败: {}", path.display())); + } + } + } +} + +fn create_candidate_artifact(live_path: &Path) -> Result<(StagedCandidate, File)> { + loop { + let path = unique_path(live_path, "candidate", "")?; + match OpenOptions::new().write(true).create_new(true).open(&path) { + Ok(file) => return Ok((StagedCandidate { path, armed: true }, file)), + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => continue, + Err(error) => { + return Err(error) + .with_context(|| format!("创建候选数据失败: {}", path.display())); + } + } + } +} + +fn remove_artifact_family(path: &Path) -> Result<()> { + for suffix in ["", "-journal", "-shm", "-wal"] { + let artifact = if suffix.is_empty() { + path.to_path_buf() + } else { + sibling_path(path, suffix)? + }; + match std::fs::remove_file(&artifact) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => { + return Err(error) + .with_context(|| format!("删除候选数据失败: {}", artifact.display())); + } + } + } + Ok(()) +} + +fn copy_bounded( + mut reader: impl Read, + mut writer: impl Write, + maximum: u64, + label: &str, +) -> Result { + let mut limited = reader.by_ref().take(maximum.saturating_add(1)); + let copied = io::copy(&mut limited, &mut writer) + .with_context(|| format!("{label}失败"))?; + if copied > maximum { + return Err(anyhow!("{label}超过限制: 最大 {maximum} 字节")); + } + Ok(copied) +} + +fn unpack_gzip(reader: impl Read, writer: impl Write, maximum: u64) -> Result { + let decoder = MultiGzDecoder::new(reader); + copy_bounded(decoder, writer, maximum, "解压 gzip") +} +``` + +The sequence is consumed before each open attempt; an `AlreadyExists` retry +therefore always receives a new name and cannot loop on a stale artifact. + +- [ ] **Step 4: Implement streaming download and candidate staging** + +Add `stage_candidate` and keep the HTTP reader/file loop explicit so SHA, +progress, and the compressed limit share one byte count: + +```rust +pub(super) fn stage_candidate( + live_path: &Path, + source: &DataSource<'_>, + policy: DownloadPolicy, +) -> Result { + let (compressed_guard, mut compressed_file) = create_compressed_artifact(live_path)?; + let staged = stage_candidate_inner( + live_path, + source, + policy, + &mut compressed_file, + ); + match staged { + Ok(candidate) => { + compressed_guard.remove_now()?; + Ok(candidate) + } + Err(error) => Err(compressed_guard.cleanup_with(error)), + } +} + +fn stage_candidate_inner( + live_path: &Path, + source: &DataSource<'_>, + policy: DownloadPolicy, + compressed_file: &mut File, +) -> Result { + let agent = ureq::AgentBuilder::new() + .timeout_connect(policy.connect_timeout) + .timeout_read(policy.idle_read_timeout) + .timeout(policy.http_timeout) + .build(); + let response = agent + .get(source.url) + .set("Accept-Encoding", "identity") + .call() + .map_err(|error| { + anyhow!( + "下载失败: {}: {error}\n请手动下载:\n {}\n解压后放到: {}", + source.url, + source.url, + live_path.display() + ) + })?; + let declared = basic_declared_length(&response, policy.max_compressed)?; + eprintln!("{}", download_notice(declared)); + + let mut reader = response.into_reader(); + let mut progress = Progress::new(declared); + let mut digest = Sha256::new(); + let mut received = 0_u64; + let mut buffer = [0_u8; BUFFER_SIZE]; + loop { + let count = reader.read(&mut buffer).context("读取响应失败")?; + if count == 0 { + break; + } + received = received + .checked_add(count as u64) + .ok_or_else(|| anyhow!("下载大小溢出"))?; + if received > policy.max_compressed { + return Err(anyhow!( + "下载数据超过限制: 最大 {} 字节", + policy.max_compressed + )); + } + digest.update(&buffer[..count]); + compressed_file + .write_all(&buffer[..count]) + .context("写入压缩临时文件失败")?; + if let Some(message) = progress.advance(count as u64) { + eprintln!("{message}"); + } + } + require_declared_length(declared, received)?; + let actual = digest.finalize(); + require_digest(actual.as_ref(), source.sha256).map_err(|error| { + anyhow!( + "{error}\n请手动下载:\n {}\n解压后放到: {}", + source.url, + live_path.display() + ) + })?; + compressed_file.flush().context("刷新压缩临时文件失败")?; + compressed_file.rewind().context("重置压缩临时文件失败")?; + + let (candidate_guard, mut candidate_file) = + create_candidate_artifact(live_path)?; + let unpacked = (|| -> Result<()> { + unpack_gzip( + &mut *compressed_file, + &mut candidate_file, + policy.max_uncompressed, + )?; + candidate_file.flush().context("刷新候选数据失败")?; + candidate_file.sync_all().context("同步候选数据失败")?; + Ok(()) + })(); + drop(candidate_file); + match unpacked { + Ok(()) => Ok(candidate_guard), + Err(error) => Err(candidate_guard.cleanup_with(error)), + } +} +``` + +Define the Task 1 metadata and digest helpers as: + +```rust +fn basic_declared_length(response: &ureq::Response, maximum: u64) -> Result> { + let Some(value) = response.header("Content-Length") else { + return Ok(None); + }; + let parsed = value + .parse::() + .with_context(|| format!("无效 Content-Length: {value}"))?; + if parsed > maximum { + return Err(anyhow!( + "Content-Length 超过下载限制: {parsed} > {maximum}" + )); + } + Ok(Some(parsed)) +} + +fn require_declared_length(declared: Option, received: u64) -> Result<()> { + if let Some(expected) = declared { + if expected != received { + return Err(anyhow!( + "响应长度不符: Content-Length={expected}, received={received}" + )); + } + } + Ok(()) +} + +fn require_digest(actual: &[u8], expected_hex: &str) -> Result<()> { + if expected_hex.len() != 64 || !expected_hex.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return Err(anyhow!("配置的 SHA-256 格式无效")); + } + let actual_hex: String = actual.iter().map(|byte| format!("{byte:02x}")).collect(); + if !actual_hex.eq_ignore_ascii_case(expected_hex) { + return Err(anyhow!("下载校验失败(sha256 不符)")); + } + Ok(()) +} +``` + +Task 2 replaces the single-value lookup with strict duplicate and transfer- +encoding handling while retaining the latter two helpers unchanged. + +- [ ] **Step 5: Replace the Vec pipeline with unified candidate publication** + +In `src/data.rs`, delete production use of `download_and_unpack`, `http_get`, +and fixed-name `write_atomic`. Keep the public small-byte helpers for source +compatibility. Add: + +```rust +mod transfer; + +fn install_candidate(path: &Path, source: &DataSource<'_>) -> Result<()> { + let candidate = + transfer::stage_candidate(path, source, transfer::PRODUCTION_POLICY)?; + if let Err(error) = verify_dataset_file(candidate.path()).map(|_| ()) { + return Err(candidate.cleanup_with(error)); + } + if let Err(error) = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(candidate.path()) + .and_then(|file| file.sync_all()) + .with_context(|| format!("同步候选数据失败: {}", candidate.path().display())) + { + return Err(candidate.cleanup_with(error)); + } + let candidate_path = candidate.path().to_path_buf(); + finish_replacement(candidate, replace_with_candidate(path, &candidate_path)) +} +``` + +Change `finish_replacement` to consume `StagedCandidate`. On success call +`publish_succeeded`; on remove-policy failure call `cleanup_with`; on +preserve-policy failure call `preserve` and retain the exact path in context. +Make both `ensure_data` and `update_data` call `install_candidate`, so first +installation receives full SQLite/FTS verification. + +- [ ] **Step 6: Run targeted and regression tests** + +Run: + +```bash +cargo +1.95.0 test --lib data::transfer::tests --locked +cargo +1.95.0 test --test data --locked +cargo +1.95.0 test --all --locked +``` + +Expected: all tests pass; the Rust total is at least 102 after the two new +bounded-transfer tests. + +- [ ] **Step 7: Commit the bounded pipeline** + +```bash +git add src/data.rs src/data/transfer.rs tests/data.rs +git commit -m "fix(data): stream verified downloads to disk" +``` + +--- + +### Task 2: Enforce strict HTTP metadata and timeout behavior + +**Files:** +- Modify: `src/data/transfer.rs` +- Test: `src/data/transfer.rs` + +**Interfaces:** +- Consumes: `DownloadPolicy` and `stage_candidate` from Task 1. +- Produces: strict `declared_length(&ureq::Response, u64) -> Result>` and stable timeout/HTTP error contexts used by all transfers. + +- [ ] **Step 1: Add failing raw-HTTP tests** + +Add a reusable one-response `TcpListener` fixture inside the transfer test +module and tests with small injected policies for: + +```rust +#[test] +fn duplicate_content_length_is_rejected() { + let response = b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\nContent-Length: 4\r\nConnection: close\r\n\r\ntest"; + let error = stage_from_raw_response(response, policy_for_tests()) + .unwrap_err() + .to_string(); + assert!(error.contains("Content-Length") && error.contains("重复"), "got: {error}"); +} + +#[test] +fn declared_length_must_match_received_body() { + let response = b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\nConnection: close\r\n\r\ntest"; + let error = stage_from_raw_response(response, policy_for_tests()) + .unwrap_err() + .to_string(); + assert!(error.contains("长度") || error.contains("读取响应失败"), "got: {error}"); +} + +#[test] +fn total_timeout_stops_a_non_idle_dribble() { + let policy = DownloadPolicy { + connect_timeout: Duration::from_millis(200), + idle_read_timeout: Duration::from_millis(150), + http_timeout: Duration::from_millis(300), + max_compressed: 1024, + max_uncompressed: 4096, + }; + let error = stage_from_dribbling_server(Duration::from_millis(50), policy) + .unwrap_err() + .to_string(); + assert!(error.contains("读取响应失败") || error.contains("timed out"), "got: {error}"); +} +``` + +Define the test helpers in the same module so no external network is used: + +```rust +fn policy_for_tests() -> DownloadPolicy { + DownloadPolicy { + connect_timeout: Duration::from_millis(200), + idle_read_timeout: Duration::from_millis(200), + http_timeout: Duration::from_secs(2), + max_compressed: 1024, + max_uncompressed: 4096, + } +} + +fn stage_from_raw_response(response: &[u8], policy: DownloadPolicy) -> Result<()> { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let response = response.to_vec(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = [0_u8; 4096]; + let _ = stream.read(&mut request); + stream.write_all(&response).unwrap(); + }); + let directory = tempfile::tempdir().unwrap(); + let live = directory.path().join("data.sqlite"); + let url = format!("http://{address}/data.gz"); + let sha = "0".repeat(64); + let result = stage_candidate(&live, &DataSource { url: &url, sha256: &sha }, policy) + .map(drop); + server.join().unwrap(); + result +} + +fn stage_from_dribbling_server(interval: Duration, policy: DownloadPolicy) -> Result<()> { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = [0_u8; 4096]; + let _ = stream.read(&mut request); + stream + .write_all(b"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n") + .unwrap(); + for _ in 0..20 { + if stream.write_all(b"x").is_err() { + break; + } + if stream.flush().is_err() { + break; + } + std::thread::sleep(interval); + } + }); + let directory = tempfile::tempdir().unwrap(); + let live = directory.path().join("data.sqlite"); + let url = format!("http://{address}/data.gz"); + let sha = "0".repeat(64); + let result = stage_candidate(&live, &DataSource { url: &url, sha256: &sha }, policy) + .map(drop); + server.join().unwrap(); + result +} +``` + +Also add tests for an oversized declared length rejected before body reads, +missing length, chunked `max + 1`, an idle body pause, and capture the request +to assert `Accept-Encoding: identity`. + +- [ ] **Step 2: Run the strict HTTP tests and confirm RED** + +Run: + +```bash +cargo +1.95.0 test --lib data::transfer::tests --locked +``` + +Expected: duplicate-length, chunked-limit, and timeout assertions fail against +the basic Task 1 metadata implementation. + +- [ ] **Step 3: Implement strict response metadata** + +Replace `basic_declared_length` with: + +```rust +fn declared_length(response: &ureq::Response, maximum: u64) -> Result> { + if !response.all("Transfer-Encoding").is_empty() { + return Ok(None); + } + let values = response.all("Content-Length"); + match values.as_slice() { + [] => Ok(None), + [value] => { + let parsed = value + .parse::() + .with_context(|| format!("无效 Content-Length: {value}"))?; + if parsed > maximum { + return Err(anyhow!( + "Content-Length 超过下载限制: {parsed} > {maximum}" + )); + } + Ok(Some(parsed)) + } + _ => Err(anyhow!("响应包含重复 Content-Length")), + } +} + +fn require_declared_length(declared: Option, received: u64) -> Result<()> { + if let Some(expected) = declared { + if expected != received { + return Err(anyhow!( + "响应长度不符: Content-Length={expected}, received={received}" + )); + } + } + Ok(()) +} +``` + +Keep the actual compressed counter authoritative for missing/chunked lengths. +Map body `TimedOut` and `UnexpectedEof` without discarding their source error, +so tests and users can distinguish timeout from truncation. + +- [ ] **Step 4: Add gzip integrity edge tests** + +Add tests for two concatenated gzip members, a truncated trailer, modified +CRC, and trailing non-gzip bytes. Use `MultiGzDecoder`; exact-limit valid data +must pass because the reader probes `maximum + 1`, while each malformed input +must fail before a candidate can be published. + +- [ ] **Step 5: Run transfer tests, Clippy, and formatting** + +Run: + +```bash +cargo +1.95.0 fmt --all -- --check +cargo +1.95.0 clippy --all-targets --locked -- -D warnings +cargo +1.95.0 test --lib data::transfer::tests --locked +cargo +1.95.0 test --test data --locked +``` + +Expected: all commands exit 0. + +- [ ] **Step 6: Commit strict protocol handling** + +```bash +git add src/data/transfer.rs +git commit -m "fix(data): bound HTTP metadata and deadlines" +``` + +--- + +### Task 3: Add the single-flight operation lock and locked clean path + +**Files:** +- Create: `src/data/operation_lock.rs` +- Modify: `src/data.rs:1-150` +- Modify: `src/cli.rs:332-351` +- Test: `src/data/operation_lock.rs` +- Test: `tests/data.rs` +- Test: `tests/command.rs:240-283` + +**Interfaces:** +- Consumes: `super::sibling_path`, `install_candidate`, and Task 1 artifact names. +- Produces: `operation_lock::acquire(&Path, Duration) -> Result` and public `data::clean_data(&Path) -> Result>`. + +- [ ] **Step 1: Add failing lock lifecycle tests** + +Create `src/data/operation_lock.rs`, wire `mod operation_lock;`, and add: + +```rust +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn competing_lock_times_out_then_succeeds_after_drop() { + let directory = tempfile::tempdir().unwrap(); + let data = directory.path().join("data.sqlite"); + let first = acquire(&data, Duration::from_millis(100)).unwrap(); + let error = acquire(&data, Duration::from_millis(20)) + .unwrap_err() + .to_string(); + assert!(error.contains("等待") && error.contains("超时"), "got: {error}"); + drop(first); + acquire(&data, Duration::from_millis(100)).unwrap(); + assert!(data.with_file_name("data.sqlite.lock").exists()); + } +} +``` + +- [ ] **Step 2: Run the lock test and confirm RED** + +Run: + +```bash +cargo +1.95.0 test --lib data::operation_lock::tests --locked +``` + +Expected: compilation fails because `acquire` and `OperationLock` do not exist. + +- [ ] **Step 3: Implement the permanent bounded-wait lock** + +Implement `src/data/operation_lock.rs` as: + +```rust +use super::sibling_path; +use anyhow::{anyhow, Context, Result}; +use std::fs::{File, OpenOptions}; +use std::io::ErrorKind; +use std::path::Path; +use std::time::{Duration, Instant}; + +const POLL_INTERVAL: Duration = Duration::from_millis(100); +pub(super) const LOCK_WAIT_TIMEOUT: Duration = Duration::from_secs(20 * 60); + +#[derive(Debug)] +pub(super) struct OperationLock { + _file: File, +} + +pub(super) fn acquire(data_path: &Path, timeout: Duration) -> Result { + let lock_path = sibling_path(data_path, ".lock")?; + let file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .open(&lock_path) + .with_context(|| format!("打开数据操作锁失败: {}", lock_path.display()))?; + let started = Instant::now(); + let mut reported_wait = false; + loop { + match file.try_lock() { + Ok(()) => return Ok(OperationLock { _file: file }), + Err(error) if error.kind() == ErrorKind::WouldBlock => { + if started.elapsed() >= timeout { + return Err(anyhow!( + "等待其他 fojin 数据操作超时: {}", + lock_path.display() + )); + } + if !reported_wait { + eprintln!("检测到另一个 fojin 数据操作,正在等待..."); + reported_wait = true; + } + let remaining = timeout.saturating_sub(started.elapsed()); + std::thread::sleep(POLL_INTERVAL.min(remaining)); + } + Err(error) => { + return Err(error) + .with_context(|| format!("获取数据操作锁失败: {}", lock_path.display())); + } + } + } +} +``` + +Do not unlink the lock file. Dropping `OperationLock` drops `File` and releases +the OS lock after success, error, unwind, or process exit. + +- [ ] **Step 4: Lock ensure, update, and clean with a second check** + +Change `ensure_data` to preserve the no-op and offline fast paths, then acquire +the lock and check existence again: + +```rust +pub fn ensure_data(path: &Path, offline: bool, source: &DataSource<'_>) -> Result<()> { + if path.exists() { + return Ok(()); + } + if offline { + return Err(anyhow!( + "本地数据不存在且处于 --offline (offline)。请手动下载:\n {}\n解压后放到: {}", + source.url, + path.display() + )); + } + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).context("创建缓存目录失败")?; + } + let _lock = operation_lock::acquire(path, operation_lock::LOCK_WAIT_TIMEOUT)?; + if path.exists() { + return Ok(()); + } + install_candidate(path, source) +} +``` + +Acquire the same lock around all of `update_data`. Add: + +```rust +pub fn clean_data(path: &Path) -> Result> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).context("创建缓存目录失败")?; + } + let _lock = operation_lock::acquire(path, operation_lock::LOCK_WAIT_TIMEOUT)?; + transfer::remove_known_artifacts(path)?; + let size = match std::fs::metadata(path) { + Ok(metadata) => Some(metadata.len()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(error) => return Err(error).context("读取数据文件状态失败"), + }; + if size.is_some() { + std::fs::remove_file(path) + .with_context(|| format!("删除数据失败: {}", path.display()))?; + } + Ok(size) +} +``` + +Implement the locked sweep as: + +```rust +pub(super) fn remove_known_artifacts(live_path: &Path) -> Result<()> { + let legacy = live_path.with_extension("tmp"); + match std::fs::remove_file(&legacy) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => { + return Err(error) + .with_context(|| format!("删除旧临时数据失败: {}", legacy.display())); + } + } + + let directory = live_path + .parent() + .ok_or_else(|| anyhow!("数据路径没有父目录: {}", live_path.display()))?; + let file_name = live_path + .file_name() + .ok_or_else(|| anyhow!("数据路径没有文件名: {}", live_path.display()))? + .to_string_lossy(); + let download_prefix = format!("{file_name}.download."); + let candidate_prefix = format!("{file_name}.candidate."); + for entry in std::fs::read_dir(directory) + .with_context(|| format!("读取数据目录失败: {}", directory.display()))? + { + let entry = entry.context("读取数据目录项失败")?; + let name = entry.file_name(); + let name = name.to_string_lossy(); + if !name.starts_with(&download_prefix) && !name.starts_with(&candidate_prefix) { + continue; + } + if entry.file_type().context("读取临时数据类型失败")?.is_dir() { + continue; + } + std::fs::remove_file(entry.path()) + .with_context(|| format!("删除临时数据失败: {}", entry.path().display()))?; + } + Ok(()) +} +``` + +This code does not traverse directories or delete the `.lock` sibling. +Change `run_data(DataAction::Clean)` in `src/cli.rs` to print from +`data::clean_data` and leave `data.sqlite.lock` untouched. + +- [ ] **Step 5: Add the process-level single-flight test** + +In `tests/data.rs`, make one test act as either the parent or a re-executed +worker. This avoids adding a normal test that exits without assertions: + +```rust +#[test] +fn concurrent_first_install_downloads_once() { + if std::env::var_os("FOJIN_CONCURRENT_WORKER").is_some() { + let path = PathBuf::from(std::env::var_os("FOJIN_WORKER_DATA").unwrap()); + let url = std::env::var("FOJIN_WORKER_URL").unwrap(); + let sha = std::env::var("FOJIN_WORKER_SHA256").unwrap(); + ensure_data(&path, false, &DataSource { url: &url, sha256: &sha }).unwrap(); + return; + } + use std::io::{Read, Write}; + use std::process::Command; + use std::time::{Duration, Instant}; + + let directory = tempfile::tempdir().unwrap(); + let data_path = directory.path().join("data.sqlite"); + let body = gzip_bytes(&replacement_database_bytes()); + let sha = sha256_hex(&body); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}/data.gz", listener.local_addr().unwrap()); + let server_body = body.clone(); + let server = std::thread::spawn(move || { + let mut requests = 0_usize; + let (mut first, _) = listener.accept().unwrap(); + requests += 1; + let mut request = [0_u8; 4096]; + let _ = first.read(&mut request); + write!( + first, + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + server_body.len() + ) + .unwrap(); + let midpoint = server_body.len() / 2; + first.write_all(&server_body[..midpoint]).unwrap(); + first.flush().unwrap(); + std::thread::sleep(Duration::from_millis(200)); + first.write_all(&server_body[midpoint..]).unwrap(); + drop(first); + + listener.set_nonblocking(true).unwrap(); + let deadline = Instant::now() + Duration::from_secs(2); + while Instant::now() < deadline { + match listener.accept() { + Ok((mut stream, _)) => { + requests += 1; + let _ = stream.read(&mut request); + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + server_body.len() + ) + .unwrap(); + stream.write_all(&server_body).unwrap(); + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + std::thread::sleep(Duration::from_millis(10)); + } + Err(error) => panic!("accept failed: {error}"), + } + } + requests + }); + + let spawn_worker = || { + Command::new(std::env::current_exe().unwrap()) + .arg("--exact") + .arg("concurrent_first_install_downloads_once") + .arg("--nocapture") + .env("FOJIN_CONCURRENT_WORKER", "1") + .env("FOJIN_WORKER_DATA", &data_path) + .env("FOJIN_WORKER_URL", &url) + .env("FOJIN_WORKER_SHA256", &sha) + .spawn() + .unwrap() + }; + let mut first = spawn_worker(); + let mut second = spawn_worker(); + assert!(first.wait().unwrap().success()); + assert!(second.wait().unwrap().success()); + assert_eq!(server.join().unwrap(), 1); + verify_dataset_file(&data_path).unwrap(); + assert_no_owned_candidate_artifacts(&data_path); +} +``` + +- [ ] **Step 6: Add clean and lock regression tests** + +In `tests/command.rs`, extend `data_clean_removes_file_and_is_idempotent` to +assert `data.sqlite.lock` remains a regular file after both clean calls. Add a +data-layer test showing an existing live file stays readable while an updater +is downloading; do not make query connections acquire the operation lock. + +- [ ] **Step 7: Run all lock, data, command, and Windows-compilable tests** + +Run: + +```bash +cargo +1.95.0 fmt --all -- --check +cargo +1.95.0 clippy --all-targets --locked -- -D warnings +cargo +1.95.0 test --lib data::operation_lock::tests --locked +cargo +1.95.0 test --test data --locked +cargo +1.95.0 test --test command --locked +cargo +1.95.0 test --all --locked +``` + +Expected: all commands exit 0, the process-level server reports one request, +and no test waits for a production-scale timeout. + +- [ ] **Step 8: Commit single-flight locking** + +```bash +git add src/data.rs src/data/operation_lock.rs src/data/transfer.rs src/cli.rs tests/data.rs tests/command.rs +git commit -m "fix(data): serialize installation and cleanup" +``` + +--- + +### Task 4: Document the resource contract and run release-grade verification + +**Files:** +- Modify: `README.md:155-190` +- Modify: `README.md:330-365` +- Modify: `CHANGELOG.md:5-20` +- Test: all repository test and release-contract files + +**Interfaces:** +- Consumes: the final constants and behavior from Tasks 1-3. +- Produces: user-facing operational documentation and a verified merge-ready branch. + +- [ ] **Step 1: Update the Chinese and English data documentation** + +Document these exact facts in both README language sections: + +- transfers no longer buffer the complete archive or database in memory; +- compressed responses are capped at 256 MiB and decompressed databases at + 768 MiB; +- updates can temporarily require the live database plus roughly 744 MiB of + staging disk; +- connect, idle, and total HTTP timeouts are 30 seconds, 60 seconds, and 15 + minutes; +- concurrent operations on one data directory are single-flight and a waiter + may wait up to 20 minutes; +- the permanent `.lock` file is harmless and intentionally survives clean; +- offline queries and the pinned data-v1 checksum contract are unchanged. + +- [ ] **Step 2: Update CHANGELOG** + +Under `[0.3.0] - Unreleased`, add bullets stating that the data path now uses a +bounded disk-streamed, checksum-first pipeline and that concurrent install, +update, and clean operations are serialized with full candidate validation. + +- [ ] **Step 3: Verify no obsolete production path remains** + +Run: + +```bash +rg -n "download_and_unpack|fn http_get|let raw =" src +``` + +Expected: no matches. The small public `gunzip`, `verify_sha256`, and +`write_atomic` compatibility helpers may remain, but neither `ensure_data` nor +`update_data` may call them. + +- [ ] **Step 4: Run the complete local verification matrix** + +Run: + +```bash +git diff --check +cargo +1.95.0 fmt --all -- --check +cargo +1.95.0 clippy --all-targets --locked -- -D warnings +cargo +1.95.0 test --all --locked +cargo +1.95.0 build --release --locked +cargo +1.95.0 package --locked +python3 -m pytest data-pipeline/tests -q +bash tests/release-scripts.sh +bash tests/install-script.sh +``` + +Expected: every command exits 0; Rust tests include the new transfer and +concurrency cases, Python reports one pass, and both shell suites report their +success messages. + +- [ ] **Step 5: Run static shell/workflow validation when available** + +Run ShellCheck 0.10.0 over `install.sh`, `scripts/*.sh`, and `tests/*.sh`, and +actionlint 1.7.7 over `.github/workflows/*.yml`. Expected: exit 0 with no +diagnostics. If the binaries are absent, use the same checksum-verified pinned +downloads established in the preceding v0.3.0 work. + +- [ ] **Step 6: Commit documentation** + +```bash +git add README.md CHANGELOG.md +git commit -m "docs: describe bounded data downloads" +``` + +--- + +### Task 5: Independent review, PR, CI, and merge + +**Files:** +- Review: `origin/master...HEAD` +- GitHub: branch `agent/streaming-data-download` + +**Interfaces:** +- Consumes: all implementation commits and fresh verification evidence. +- Produces: a merged GitHub PR and synchronized clean local `master`. + +- [ ] **Step 1: Request independent spec and code-quality review** + +Give reviewers the approved design, this plan, the commit range, and test +evidence. Require explicit Critical/Important/Minor findings and a ready-to- +merge verdict. Address substantive findings with TDD and focused fix commits. + +- [ ] **Step 2: Re-run verification after the final review fix** + +At minimum rerun `git diff --check`, fmt, Clippy, all locked Rust tests, Python, +release scripts, and installer scripts. Expected: all exit 0 on the final HEAD. + +- [ ] **Step 3: Push and create a ready pull request** + +```bash +git push -u origin agent/streaming-data-download +gh pr create --base master --head agent/streaming-data-download \ + --title "fix(data): stream and serialize dataset installation" \ + --body-file .superpowers/sdd/pr-body.md +``` + +The PR body summarizes bounded memory, limits/timeouts, full first-install +validation, single-flight concurrency, documentation, and exact test evidence. + +- [ ] **Step 4: Wait for every GitHub check and fix failures systematically** + +Run `gh pr checks --watch --interval 10`. Do not merge with a pending, +cancelled, or failing check. For a failure, inspect the job logs, reproduce it, +apply one root-cause fix, and rerun local verification before pushing. + +- [ ] **Step 5: Merge and verify master** + +Use a merge commit, delete the remote feature branch, fast-forward the main +checkout, wait for the post-merge `master` CI run, rerun the core local tests, +and remove the isolated worktree/local feature branch. Verify no tag or GitHub +Release was created. From e92ca77d611ebf94dc1e04e4c9fdd9ac746ecd97 Mon Sep 17 00:00:00 2001 From: xr843 <137012659+xr843@users.noreply.github.com> Date: Sat, 11 Jul 2026 22:05:33 +0800 Subject: [PATCH 03/14] fix(data): stream verified downloads to disk --- src/data.rs | 168 +++++--------------- src/data/transfer.rs | 367 +++++++++++++++++++++++++++++++++++++++++++ tests/data.rs | 58 +++++-- 3 files changed, 447 insertions(+), 146 deletions(-) create mode 100644 src/data/transfer.rs diff --git a/src/data.rs b/src/data.rs index 74f5a9b..bc1d43e 100644 --- a/src/data.rs +++ b/src/data.rs @@ -1,18 +1,9 @@ use anyhow::{anyhow, Context, Result}; use rusqlite::{OpenFlags, OptionalExtension}; -use std::io::{Read, Write}; +use std::io::Read; use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::time::Duration; - -/// Connect timeout for the data download: fails fast if the release host is -/// unreachable rather than hanging forever. -const CONNECT_TIMEOUT: Duration = Duration::from_secs(30); -/// Overall read ceiling for the data download. The artifact is ~100-200MB, -/// so this is generous for a slow-but-alive connection while still -/// guaranteeing the CLI can never hang indefinitely. -const READ_TIMEOUT: Duration = Duration::from_secs(900); -static CANDIDATE_SEQUENCE: AtomicU64 = AtomicU64::new(0); + +mod transfer; pub const EXPECTED_DATA_VERSION: &str = "v1"; pub const EXPECTED_NORM_RULESET: &str = "t2s-char-1to1-v1"; @@ -117,8 +108,7 @@ pub fn ensure_data(path: &Path, offline: bool, source: &DataSource) -> Result<() if let Some(parent) = path.parent() { std::fs::create_dir_all(parent).context("创建缓存目录失败")?; } - let raw = download_and_unpack(path, source)?; - write_atomic(path, &raw) + install_candidate(path, source) } /// Download a replacement dataset to a sibling candidate, validate it, then @@ -128,40 +118,25 @@ pub fn update_data(path: &Path, source: &DataSource) -> Result<()> { std::fs::create_dir_all(parent).context("创建缓存目录失败")?; } - let raw = download_and_unpack(path, source)?; - let (candidate, mut candidate_file) = create_candidate(path)?; - let write_result = candidate_file - .write_all(&raw) - .with_context(|| format!("写入候选数据失败: {}", candidate.display())); - drop(candidate_file); - if let Err(error) = write_result { - return Err(cleanup_candidate_error(&candidate, error)); - } - - let validation_result = verify_dataset_file(&candidate).map(|_| ()); - if let Err(error) = validation_result { - return Err(cleanup_candidate_error(&candidate, error)); - } - - finish_replacement(&candidate, replace_with_candidate(path, &candidate)) + install_candidate(path, source) } -fn download_and_unpack(path: &Path, source: &DataSource) -> Result> { - let gz = http_get(source.url).map_err(|e| { - anyhow!( - "{e:#}\n请手动下载:\n {}\n解压后放到: {}", - source.url, - path.display() - ) - })?; - if !verify_sha256(&gz, source.sha256) { - return Err(anyhow!( - "下载校验失败(sha256 不符)。请重试或手动下载:\n {}\n解压后放到: {}", - source.url, - path.display() - )); +fn install_candidate(path: &Path, source: &DataSource<'_>) -> Result<()> { + let candidate = transfer::stage_candidate(path, source, transfer::PRODUCTION_POLICY)?; + if let Err(error) = verify_dataset_file(candidate.path()).map(|_| ()) { + return Err(candidate.cleanup_with(error)); + } + if let Err(error) = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(candidate.path()) + .and_then(|file| file.sync_all()) + .with_context(|| format!("同步候选数据失败: {}", candidate.path().display())) + { + return Err(candidate.cleanup_with(error)); } - gunzip(&gz) + let candidate_path = candidate.path().to_path_buf(); + finish_replacement(candidate, replace_with_candidate(path, &candidate_path)) } fn sibling_path(path: &Path, suffix: &str) -> Result { @@ -173,33 +148,6 @@ fn sibling_path(path: &Path, suffix: &str) -> Result { Ok(path.with_file_name(sibling)) } -fn create_candidate(path: &Path) -> Result<(PathBuf, std::fs::File)> { - loop { - let sequence = CANDIDATE_SEQUENCE.fetch_add(1, Ordering::Relaxed); - let suffix = format!(".candidate.{}.{sequence}", std::process::id()); - let candidate = sibling_path(path, &suffix)?; - match std::fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(&candidate) - { - Ok(file) => return Ok((candidate, file)), - Err(ref error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, - Err(error) => { - return Err(error) - .with_context(|| format!("创建候选数据失败: {}", candidate.display())) - } - } - } -} - -fn cleanup_candidate_error(candidate: &Path, error: anyhow::Error) -> anyhow::Error { - match remove_candidate_artifacts(candidate) { - Ok(()) => error, - Err(cleanup_error) => error.context(format!("清理候选数据失败: {cleanup_error}")), - } -} - #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum CandidateCleanupPolicy { Remove, @@ -232,39 +180,28 @@ impl ReplacementFailure { type ReplacementResult = std::result::Result<(), ReplacementFailure>; -fn finish_replacement(candidate: &Path, result: ReplacementResult) -> Result<()> { +fn finish_replacement( + candidate: transfer::StagedCandidate, + result: ReplacementResult, +) -> Result<()> { match result { - Ok(()) => Ok(()), + Ok(()) => { + candidate.publish_succeeded(); + Ok(()) + } Err(failure) => match failure.cleanup { - CandidateCleanupPolicy::Remove => { - Err(cleanup_candidate_error(candidate, failure.error)) - } + CandidateCleanupPolicy::Remove => Err(candidate.cleanup_with(failure.error)), #[cfg(any(test, windows))] - CandidateCleanupPolicy::Preserve => Err(failure.error.context(format!( - "validated candidate preserved at `{}`", - candidate.display() - ))), - }, - } -} - -fn remove_candidate_artifacts(candidate: &Path) -> Result<()> { - for suffix in ["", "-journal", "-shm", "-wal"] { - let artifact = if suffix.is_empty() { - candidate.to_path_buf() - } else { - sibling_path(candidate, suffix)? - }; - match std::fs::remove_file(&artifact) { - Ok(()) => {} - Err(ref error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => { - return Err(error) - .with_context(|| format!("删除候选数据失败: {}", artifact.display())) + CandidateCleanupPolicy::Preserve => { + let candidate_path = candidate.path().to_path_buf(); + candidate.preserve(); + Err(failure.error.context(format!( + "validated candidate preserved at `{}`", + candidate_path.display() + ))) } - } + }, } - Ok(()) } #[cfg(not(windows))] @@ -416,34 +353,6 @@ fn describe_windows_error(error: &std::io::Error) -> String { } } -fn http_get(url: &str) -> Result> { - let agent = ureq::AgentBuilder::new() - .timeout_connect(CONNECT_TIMEOUT) - .timeout_read(READ_TIMEOUT) - .build(); - let resp = agent - .get(url) - .call() - .with_context(|| format!("下载失败: {url}"))?; - let total: Option = resp.header("Content-Length").and_then(|v| v.parse().ok()); - eprintln!("{}", download_notice(total)); - let mut progress = Progress::new(total); - let mut reader = resp.into_reader(); - let mut buf = Vec::new(); - let mut chunk = [0u8; 64 * 1024]; - loop { - let n = reader.read(&mut chunk).context("读取响应失败")?; - if n == 0 { - break; - } - buf.extend_from_slice(&chunk[..n]); - if let Some(msg) = progress.advance(n as u64) { - eprintln!("{msg}"); - } - } - Ok(buf) -} - pub fn open_db(path: &Path) -> Result { rusqlite::Connection::open(path).with_context(|| format!("打开数据失败: {}", path.display())) } @@ -681,9 +590,10 @@ mod replacement_cleanup_tests { let dir = tempfile::tempdir().unwrap(); let candidate = dir.path().join("data.sqlite.candidate.1.1"); std::fs::write(&candidate, b"validated dataset").unwrap(); + let staged = transfer::StagedCandidate::for_test(candidate.clone()); let error = finish_replacement( - &candidate, + staged, Err(ReplacementFailure::preserve(anyhow!( "replacement recovery failed" ))), diff --git a/src/data/transfer.rs b/src/data/transfer.rs new file mode 100644 index 0000000..cc0732f --- /dev/null +++ b/src/data/transfer.rs @@ -0,0 +1,367 @@ +use super::{download_notice, sibling_path, DataSource, Progress}; +use anyhow::{anyhow, Context, Result}; +use flate2::read::MultiGzDecoder; +use sha2::{Digest, Sha256}; +use std::fs::{File, OpenOptions}; +use std::io::{self, Read, Seek, Write}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +const MIB: u64 = 1024 * 1024; +const BUFFER_SIZE: usize = 64 * 1024; +static ARTIFACT_SEQUENCE: AtomicU64 = AtomicU64::new(0); + +#[derive(Clone, Copy, Debug)] +pub(super) struct DownloadPolicy { + pub connect_timeout: Duration, + pub idle_read_timeout: Duration, + pub http_timeout: Duration, + pub max_compressed: u64, + pub max_uncompressed: u64, +} + +pub(super) const PRODUCTION_POLICY: DownloadPolicy = DownloadPolicy { + connect_timeout: Duration::from_secs(30), + idle_read_timeout: Duration::from_secs(60), + http_timeout: Duration::from_secs(15 * 60), + max_compressed: 256 * MIB, + max_uncompressed: 768 * MIB, +}; + +pub(super) struct StagedCandidate { + path: PathBuf, + armed: bool, +} + +struct OwnedCompressed { + path: PathBuf, + armed: bool, +} + +impl OwnedCompressed { + fn remove_now(mut self) -> Result<()> { + self.armed = false; + match std::fs::remove_file(&self.path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(error) => { + Err(error).with_context(|| format!("删除压缩临时文件失败: {}", self.path.display())) + } + } + } + + fn cleanup_with(mut self, error: anyhow::Error) -> anyhow::Error { + self.armed = false; + match std::fs::remove_file(&self.path) { + Ok(()) => error, + Err(cleanup) if cleanup.kind() == io::ErrorKind::NotFound => error, + Err(cleanup) => error.context(format!( + "清理压缩临时文件失败: {}: {cleanup}", + self.path.display() + )), + } + } +} + +impl Drop for OwnedCompressed { + fn drop(&mut self) { + if self.armed { + let _ = std::fs::remove_file(&self.path); + } + } +} + +impl StagedCandidate { + #[cfg(test)] + pub(super) fn for_test(path: PathBuf) -> Self { + Self { path, armed: true } + } + + pub(super) fn path(&self) -> &Path { + &self.path + } + + pub(super) fn publish_succeeded(mut self) { + self.armed = false; + } + + #[cfg(any(test, windows))] + pub(super) fn preserve(mut self) { + self.armed = false; + } + + pub(super) fn cleanup_with(mut self, error: anyhow::Error) -> anyhow::Error { + self.armed = false; + match remove_artifact_family(&self.path) { + Ok(()) => error, + Err(cleanup) => error.context(format!("清理候选数据失败: {cleanup}")), + } + } +} + +impl Drop for StagedCandidate { + fn drop(&mut self) { + if self.armed { + let _ = remove_artifact_family(&self.path); + } + } +} + +fn unique_path(live_path: &Path, role: &str, extension: &str) -> Result { + let sequence = ARTIFACT_SEQUENCE.fetch_add(1, Ordering::Relaxed); + sibling_path( + live_path, + &format!(".{role}.{}.{sequence}{extension}", std::process::id()), + ) +} + +fn create_compressed_artifact(live_path: &Path) -> Result<(OwnedCompressed, File)> { + loop { + let path = unique_path(live_path, "download", ".gz")?; + match OpenOptions::new() + .read(true) + .write(true) + .create_new(true) + .open(&path) + { + Ok(file) => return Ok((OwnedCompressed { path, armed: true }, file)), + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => continue, + Err(error) => { + return Err(error) + .with_context(|| format!("创建压缩临时文件失败: {}", path.display())); + } + } + } +} + +fn create_candidate_artifact(live_path: &Path) -> Result<(StagedCandidate, File)> { + loop { + let path = unique_path(live_path, "candidate", "")?; + match OpenOptions::new().write(true).create_new(true).open(&path) { + Ok(file) => return Ok((StagedCandidate { path, armed: true }, file)), + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => continue, + Err(error) => { + return Err(error).with_context(|| format!("创建候选数据失败: {}", path.display())); + } + } + } +} + +fn remove_artifact_family(path: &Path) -> Result<()> { + for suffix in ["", "-journal", "-shm", "-wal"] { + let artifact = if suffix.is_empty() { + path.to_path_buf() + } else { + sibling_path(path, suffix)? + }; + match std::fs::remove_file(&artifact) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => { + return Err(error) + .with_context(|| format!("删除候选数据失败: {}", artifact.display())); + } + } + } + Ok(()) +} + +fn copy_bounded( + mut reader: impl Read, + mut writer: impl Write, + maximum: u64, + label: &str, +) -> Result { + let mut limited = reader.by_ref().take(maximum.saturating_add(1)); + let copied = io::copy(&mut limited, &mut writer).with_context(|| format!("{label}失败"))?; + if copied > maximum { + return Err(anyhow!("{label}超过限制: 最大 {maximum} 字节")); + } + Ok(copied) +} + +fn unpack_gzip(reader: impl Read, writer: impl Write, maximum: u64) -> Result { + let decoder = MultiGzDecoder::new(reader); + copy_bounded(decoder, writer, maximum, "解压 gzip") +} + +pub(super) fn stage_candidate( + live_path: &Path, + source: &DataSource<'_>, + policy: DownloadPolicy, +) -> Result { + let (compressed_guard, mut compressed_file) = create_compressed_artifact(live_path)?; + let staged = stage_candidate_inner(live_path, source, policy, &mut compressed_file); + match staged { + Ok(candidate) => { + compressed_guard.remove_now()?; + Ok(candidate) + } + Err(error) => Err(compressed_guard.cleanup_with(error)), + } +} + +fn stage_candidate_inner( + live_path: &Path, + source: &DataSource<'_>, + policy: DownloadPolicy, + compressed_file: &mut File, +) -> Result { + let agent = ureq::AgentBuilder::new() + .timeout_connect(policy.connect_timeout) + .timeout_read(policy.idle_read_timeout) + .timeout(policy.http_timeout) + .build(); + let response = agent + .get(source.url) + .set("Accept-Encoding", "identity") + .call() + .map_err(|error| { + anyhow!( + "下载失败: {}: {error}\n请手动下载:\n {}\n解压后放到: {}", + source.url, + source.url, + live_path.display() + ) + })?; + let declared = basic_declared_length(&response, policy.max_compressed)?; + eprintln!("{}", download_notice(declared)); + + let mut reader = response.into_reader(); + let mut progress = Progress::new(declared); + let mut digest = Sha256::new(); + let mut received = 0_u64; + let mut buffer = [0_u8; BUFFER_SIZE]; + loop { + let count = reader.read(&mut buffer).context("读取响应失败")?; + if count == 0 { + break; + } + received = received + .checked_add(count as u64) + .ok_or_else(|| anyhow!("下载大小溢出"))?; + if received > policy.max_compressed { + return Err(anyhow!( + "下载数据超过限制: 最大 {} 字节", + policy.max_compressed + )); + } + digest.update(&buffer[..count]); + compressed_file + .write_all(&buffer[..count]) + .context("写入压缩临时文件失败")?; + if let Some(message) = progress.advance(count as u64) { + eprintln!("{message}"); + } + } + require_declared_length(declared, received)?; + let actual = digest.finalize(); + require_digest(actual.as_ref(), source.sha256).map_err(|error| { + anyhow!( + "{error}\n请手动下载:\n {}\n解压后放到: {}", + source.url, + live_path.display() + ) + })?; + compressed_file.flush().context("刷新压缩临时文件失败")?; + compressed_file.rewind().context("重置压缩临时文件失败")?; + + let (candidate_guard, mut candidate_file) = create_candidate_artifact(live_path)?; + let unpacked = (|| -> Result<()> { + unpack_gzip( + &mut *compressed_file, + &mut candidate_file, + policy.max_uncompressed, + )?; + candidate_file.flush().context("刷新候选数据失败")?; + candidate_file.sync_all().context("同步候选数据失败")?; + Ok(()) + })(); + drop(candidate_file); + match unpacked { + Ok(()) => Ok(candidate_guard), + Err(error) => Err(candidate_guard.cleanup_with(error)), + } +} + +fn basic_declared_length(response: &ureq::Response, maximum: u64) -> Result> { + let Some(value) = response.header("Content-Length") else { + return Ok(None); + }; + let parsed = value + .parse::() + .with_context(|| format!("无效 Content-Length: {value}"))?; + if parsed > maximum { + return Err(anyhow!("Content-Length 超过下载限制: {parsed} > {maximum}")); + } + Ok(Some(parsed)) +} + +fn require_declared_length(declared: Option, received: u64) -> Result<()> { + if let Some(expected) = declared { + if expected != received { + return Err(anyhow!( + "响应长度不符: Content-Length={expected}, received={received}" + )); + } + } + Ok(()) +} + +fn require_digest(actual: &[u8], expected_hex: &str) -> Result<()> { + if expected_hex.len() != 64 || !expected_hex.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return Err(anyhow!("配置的 SHA-256 格式无效")); + } + let actual_hex: String = actual.iter().map(|byte| format!("{byte:02x}")).collect(); + if !actual_hex.eq_ignore_ascii_case(expected_hex) { + return Err(anyhow!("下载校验失败(sha256 不符)")); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use flate2::write::GzEncoder; + use flate2::Compression; + use std::io::{Cursor, Write}; + + #[test] + fn bounded_copy_accepts_exact_limit_and_rejects_one_more() { + let mut exact = Vec::new(); + assert_eq!( + copy_bounded(Cursor::new(b"1234"), &mut exact, 4, "test").unwrap(), + 4 + ); + assert_eq!(exact, b"1234"); + + let mut oversized = Vec::new(); + let error = copy_bounded(Cursor::new(b"12345"), &mut oversized, 4, "test") + .unwrap_err() + .to_string(); + assert!(error.contains("4"), "got: {error}"); + } + + #[test] + fn unpack_accepts_exact_limit_and_rejects_expansion() { + let gzip = |body: &[u8]| { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(body).unwrap(); + encoder.finish().unwrap() + }; + + let mut exact = Vec::new(); + unpack_gzip(Cursor::new(gzip(b"1234")), &mut exact, 4).unwrap(); + assert_eq!(exact, b"1234"); + + let mut oversized = Vec::new(); + let error = unpack_gzip(Cursor::new(gzip(b"12345")), &mut oversized, 4) + .unwrap_err() + .to_string(); + assert!( + error.contains("解压") && error.contains("4"), + "got: {error}" + ); + } +} diff --git a/tests/data.rs b/tests/data.rs index 27adee2..d5e5b74 100644 --- a/tests/data.rs +++ b/tests/data.rs @@ -1,7 +1,7 @@ use fojin_cli::data::{ ensure_data, gunzip, open_compatible_db, open_read_only_db, resolve_data_path, update_data, - validate_compatibility, verify_dataset, verify_sha256, DataSource, EXPECTED_DATA_VERSION, - EXPECTED_NORM_RULESET, + validate_compatibility, verify_dataset, verify_dataset_file, verify_sha256, DataSource, + EXPECTED_DATA_VERSION, EXPECTED_NORM_RULESET, }; use std::io::Write; use std::path::PathBuf; @@ -106,17 +106,11 @@ fn download_notice_mentions_size_and_offline() { } /// Serve one HTTP response on localhost and exercise the full download path: -/// stream -> sha256 verify -> gunzip -> atomic write. No external network. +/// stream -> sha256 verify -> bounded gunzip -> verified publication. No external network. #[test] fn ensure_data_downloads_verifies_and_unpacks() { - let mut enc = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); - enc.write_all(b"fake sqlite payload").unwrap(); - let gz = enc.finish().unwrap(); - - use sha2::{Digest, Sha256}; - let mut h = Sha256::new(); - h.update(&gz); - let sha: String = h.finalize().iter().map(|b| format!("{b:02x}")).collect(); + let gz = gzip_bytes(&replacement_database_bytes()); + let sha = sha256_hex(&gz); let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let port = listener.local_addr().unwrap().port(); @@ -142,11 +136,41 @@ fn ensure_data_downloads_verifies_and_unpacks() { ensure_data(&path, false, &source).unwrap(); server.join().unwrap(); - assert_eq!(std::fs::read(&path).unwrap(), b"fake sqlite payload"); - assert!( - !path.with_extension("tmp").exists(), - "temp file must not linger" - ); + verify_dataset_file(&path).unwrap(); +} + +#[test] +fn first_install_rejects_incompatible_database() { + let gz = gzip_bytes(b"fake sqlite payload"); + let sha = sha256_hex(&gz); + + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + let body = gz.clone(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = [0u8; 4096]; + let _ = std::io::Read::read(&mut stream, &mut request); + let head = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nContent-Type: application/gzip\r\nConnection: close\r\n\r\n", + body.len() + ); + stream.write_all(head.as_bytes()).unwrap(); + stream.write_all(&body).unwrap(); + }); + + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("data.sqlite"); + let source = DataSource { + url: &format!("http://127.0.0.1:{port}/data.gz"), + sha256: &sha, + }; + let error = ensure_data(&path, false, &source).unwrap_err().to_string(); + server.join().unwrap(); + + assert!(error.contains("dataset incompatibility"), "got: {error}"); + assert!(!path.exists(), "incompatible dataset was published"); + assert_no_owned_candidate_artifacts(&path); } #[test] @@ -561,7 +585,7 @@ fn update_data_preserves_live_dataset_on_decompression_failure() { .unwrap_err() .to_string(); - assert!(err.contains("解压 gzip 失败"), "got: {err}"); + assert!(err.contains("解压 gzip"), "got: {err}"); assert_eq!(std::fs::read(&path).unwrap(), b"old live dataset"); assert_no_candidate_artifacts(&path); } From f7e44e963144837539dc87701ad98ad31ac4f5c7 Mon Sep 17 00:00:00 2001 From: xr843 <137012659+xr843@users.noreply.github.com> Date: Sat, 11 Jul 2026 22:29:19 +0800 Subject: [PATCH 04/14] fix(data): bound HTTP metadata and deadlines --- src/data/transfer.rs | 482 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 459 insertions(+), 23 deletions(-) diff --git a/src/data/transfer.rs b/src/data/transfer.rs index cc0732f..375e77a 100644 --- a/src/data/transfer.rs +++ b/src/data/transfer.rs @@ -6,7 +6,7 @@ use std::fs::{File, OpenOptions}; use std::io::{self, Read, Seek, Write}; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; -use std::time::Duration; +use std::time::{Duration, Instant}; const MIB: u64 = 1024 * 1024; const BUFFER_SIZE: usize = 64 * 1024; @@ -208,24 +208,27 @@ fn stage_candidate_inner( policy: DownloadPolicy, compressed_file: &mut File, ) -> Result { + let deadline = Instant::now() + .checked_add(policy.http_timeout) + .ok_or_else(|| anyhow!("HTTP 总超时配置溢出"))?; let agent = ureq::AgentBuilder::new() .timeout_connect(policy.connect_timeout) .timeout_read(policy.idle_read_timeout) - .timeout(policy.http_timeout) .build(); let response = agent .get(source.url) .set("Accept-Encoding", "identity") - .call() - .map_err(|error| { - anyhow!( - "下载失败: {}: {error}\n请手动下载:\n {}\n解压后放到: {}", - source.url, - source.url, - live_path.display() - ) - })?; - let declared = basic_declared_length(&response, policy.max_compressed)?; + .call(); + require_http_deadline(deadline)?; + let response = response.map_err(|error| { + anyhow!( + "下载失败: {}: {error}\n请手动下载:\n {}\n解压后放到: {}", + source.url, + source.url, + live_path.display() + ) + })?; + let declared = declared_length(&response, policy.max_compressed)?; eprintln!("{}", download_notice(declared)); let mut reader = response.into_reader(); @@ -234,7 +237,9 @@ fn stage_candidate_inner( let mut received = 0_u64; let mut buffer = [0_u8; BUFFER_SIZE]; loop { + require_http_deadline(deadline)?; let count = reader.read(&mut buffer).context("读取响应失败")?; + require_http_deadline(deadline)?; if count == 0 { break; } @@ -285,17 +290,35 @@ fn stage_candidate_inner( } } -fn basic_declared_length(response: &ureq::Response, maximum: u64) -> Result> { - let Some(value) = response.header("Content-Length") else { +fn require_http_deadline(deadline: Instant) -> Result<()> { + if Instant::now() >= deadline { + return Err(io::Error::new( + io::ErrorKind::TimedOut, + "HTTP total deadline timed out", + )) + .context("读取响应失败"); + } + Ok(()) +} + +fn declared_length(response: &ureq::Response, maximum: u64) -> Result> { + if !response.all("Transfer-Encoding").is_empty() { return Ok(None); - }; - let parsed = value - .parse::() - .with_context(|| format!("无效 Content-Length: {value}"))?; - if parsed > maximum { - return Err(anyhow!("Content-Length 超过下载限制: {parsed} > {maximum}")); - } - Ok(Some(parsed)) + } + let values = response.all("Content-Length"); + match values.as_slice() { + [] => Ok(None), + [value] => { + let parsed = value + .parse::() + .with_context(|| format!("无效 Content-Length: {value}"))?; + if parsed > maximum { + return Err(anyhow!("Content-Length 超过下载限制: {parsed} > {maximum}")); + } + Ok(Some(parsed)) + } + _ => Err(anyhow!("响应包含重复 Content-Length")), + } } fn require_declared_length(declared: Option, received: u64) -> Result<()> { @@ -325,7 +348,420 @@ mod tests { use super::*; use flate2::write::GzEncoder; use flate2::Compression; - use std::io::{Cursor, Write}; + use std::io::{Cursor, Read, Write}; + + fn policy_for_tests() -> DownloadPolicy { + DownloadPolicy { + connect_timeout: Duration::from_millis(200), + idle_read_timeout: Duration::from_millis(200), + http_timeout: Duration::from_secs(2), + max_compressed: 1024, + max_uncompressed: 4096, + } + } + + fn sha256_hex(bytes: &[u8]) -> String { + Sha256::digest(bytes) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() + } + + fn gzip(body: &[u8]) -> Vec { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(body).unwrap(); + encoder.finish().unwrap() + } + + fn read_request(stream: &mut std::net::TcpStream) -> Vec { + stream + .set_read_timeout(Some(Duration::from_secs(1))) + .unwrap(); + let mut request = Vec::new(); + let mut buffer = [0_u8; 1024]; + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let count = stream.read(&mut buffer).unwrap(); + if count == 0 { + break; + } + request.extend_from_slice(&buffer[..count]); + } + request + } + + fn stage_from_raw_response_with_sha( + response: &[u8], + sha256: &str, + policy: DownloadPolicy, + ) -> (Result<()>, Vec) { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let response = response.to_vec(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let request = read_request(&mut stream); + stream.write_all(&response).unwrap(); + request + }); + let directory = tempfile::tempdir().unwrap(); + let live = directory.path().join("data.sqlite"); + let url = format!("http://{address}/data.gz"); + let result = stage_candidate(&live, &DataSource { url: &url, sha256 }, policy).map(drop); + let request = server.join().unwrap(); + (result, request) + } + + fn stage_from_raw_response(response: &[u8], policy: DownloadPolicy) -> Result<()> { + stage_from_raw_response_with_sha(response, &"0".repeat(64), policy).0 + } + + fn stage_from_delayed_raw_response( + response: &[u8], + delay: Duration, + policy: DownloadPolicy, + ) -> Result<()> { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let response = response.to_vec(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let _ = read_request(&mut stream); + std::thread::sleep(delay); + stream.write_all(&response).unwrap(); + }); + let directory = tempfile::tempdir().unwrap(); + let live = directory.path().join("data.sqlite"); + let url = format!("http://{address}/data.gz"); + let sha = "0".repeat(64); + let result = stage_candidate( + &live, + &DataSource { + url: &url, + sha256: &sha, + }, + policy, + ) + .map(drop); + server.join().unwrap(); + result + } + + fn stage_from_dribbling_server(interval: Duration, policy: DownloadPolicy) -> Result<()> { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let _ = read_request(&mut stream); + stream + .write_all(b"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n") + .unwrap(); + for _ in 0..20 { + if stream.write_all(b"x").is_err() { + break; + } + if stream.flush().is_err() { + break; + } + std::thread::sleep(interval); + } + }); + let directory = tempfile::tempdir().unwrap(); + let live = directory.path().join("data.sqlite"); + let url = format!("http://{address}/data.gz"); + let sha = "0".repeat(64); + let result = stage_candidate( + &live, + &DataSource { + url: &url, + sha256: &sha, + }, + policy, + ) + .map(drop); + server.join().unwrap(); + result + } + + fn stage_from_pausing_server(pause: Duration, policy: DownloadPolicy) -> Result<()> { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let _ = read_request(&mut stream); + stream + .write_all(b"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\nx") + .unwrap(); + stream.flush().unwrap(); + std::thread::sleep(pause); + let _ = stream.write_all(b"x"); + }); + let directory = tempfile::tempdir().unwrap(); + let live = directory.path().join("data.sqlite"); + let url = format!("http://{address}/data.gz"); + let sha = "0".repeat(64); + let result = stage_candidate( + &live, + &DataSource { + url: &url, + sha256: &sha, + }, + policy, + ) + .map(drop); + server.join().unwrap(); + result + } + + fn contains_io_kind(error: &anyhow::Error, expected: io::ErrorKind) -> bool { + error.chain().any(|cause| { + cause + .downcast_ref::() + .is_some_and(|error| error.kind() == expected) + }) + } + + fn chunked_response(body: &[u8], extra_headers: &str) -> Vec { + let mut response = format!( + "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n{extra_headers}Connection: close\r\n\r\n{:x}\r\n", + body.len() + ) + .into_bytes(); + response.extend_from_slice(body); + response.extend_from_slice(b"\r\n0\r\n\r\n"); + response + } + + fn content_length_response(body: &[u8]) -> Vec { + let mut response = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ) + .into_bytes(); + response.extend_from_slice(body); + response + } + + fn stage_compressed(compressed: &[u8], policy: DownloadPolicy) -> Result<()> { + let sha = sha256_hex(compressed); + let response = content_length_response(compressed); + stage_from_raw_response_with_sha(&response, &sha, policy).0 + } + + #[test] + fn duplicate_content_length_is_rejected() { + let response = b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\nContent-Length: 4\r\nConnection: close\r\n\r\ntest"; + let error = stage_from_raw_response(response, policy_for_tests()) + .unwrap_err() + .to_string(); + assert!( + error.contains("Content-Length") && error.contains("重复"), + "got: {error}" + ); + } + + #[test] + fn declared_length_must_match_received_body() { + let response = b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\nConnection: close\r\n\r\ntest"; + let error = stage_from_raw_response(response, policy_for_tests()).unwrap_err(); + assert!( + error.to_string().contains("长度") || error.to_string().contains("读取响应失败"), + "got: {error:#}" + ); + assert!( + contains_io_kind(&error, io::ErrorKind::UnexpectedEof), + "got: {error:#}" + ); + } + + #[test] + fn total_timeout_stops_a_non_idle_dribble() { + let policy = DownloadPolicy { + connect_timeout: Duration::from_millis(200), + idle_read_timeout: Duration::from_millis(150), + http_timeout: Duration::from_millis(300), + max_compressed: 1024, + max_uncompressed: 4096, + }; + let error = stage_from_dribbling_server(Duration::from_millis(50), policy).unwrap_err(); + assert!( + error.to_string().contains("读取响应失败") || error.to_string().contains("timed out"), + "got: {error:#}" + ); + assert!( + contains_io_kind(&error, io::ErrorKind::TimedOut), + "got: {error:#}" + ); + } + + #[test] + fn oversized_declared_length_is_rejected_before_body_reads() { + let response = b"HTTP/1.1 200 OK\r\nContent-Length: 1025\r\nConnection: close\r\n\r\n"; + let error = stage_from_raw_response(response, policy_for_tests()) + .unwrap_err() + .to_string(); + assert!( + error.contains("Content-Length") && error.contains("1025") && error.contains("1024"), + "got: {error}" + ); + assert!(!error.contains("读取响应失败"), "got: {error}"); + } + + #[test] + fn invalid_content_length_is_rejected_before_body_reads() { + let response = b"HTTP/1.1 200 OK\r\nContent-Length: nope\r\nConnection: close\r\n\r\n"; + let error = stage_from_raw_response(response, policy_for_tests()) + .unwrap_err() + .to_string(); + assert!( + error.contains("无效 Content-Length") && error.contains("nope"), + "got: {error}" + ); + assert!(!error.contains("读取响应失败"), "got: {error}"); + } + + #[test] + fn missing_content_length_uses_received_body_size() { + let compressed = gzip(b"test"); + let sha = sha256_hex(&compressed); + let mut response = b"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n".to_vec(); + response.extend_from_slice(&compressed); + stage_from_raw_response_with_sha(&response, &sha, policy_for_tests()) + .0 + .unwrap(); + } + + #[test] + fn transfer_encoding_makes_content_length_non_authoritative() { + let compressed = gzip(b"test"); + let sha = sha256_hex(&compressed); + let response = chunked_response(&compressed, "Content-Length: 1\r\n"); + stage_from_raw_response_with_sha(&response, &sha, policy_for_tests()) + .0 + .unwrap(); + } + + #[test] + fn chunked_body_is_rejected_at_compressed_limit_plus_one() { + let body = vec![b'x'; 1025]; + let response = chunked_response(&body, ""); + let error = stage_from_raw_response(&response, policy_for_tests()) + .unwrap_err() + .to_string(); + assert!( + error.contains("下载数据超过限制") && error.contains("1024"), + "got: {error}" + ); + } + + #[test] + fn idle_body_pause_reports_timeout_with_io_source() { + let policy = DownloadPolicy { + idle_read_timeout: Duration::from_millis(100), + ..policy_for_tests() + }; + let error = stage_from_pausing_server(Duration::from_millis(300), policy).unwrap_err(); + assert!(error.to_string().contains("读取响应失败"), "got: {error:#}"); + assert!( + contains_io_kind(&error, io::ErrorKind::TimedOut), + "got: {error:#}" + ); + } + + #[test] + fn request_disables_transport_content_decoding() { + let response = b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"; + let (_, request) = + stage_from_raw_response_with_sha(response, &"0".repeat(64), policy_for_tests()); + let request = String::from_utf8(request).unwrap(); + assert!( + request + .lines() + .any(|line| line.eq_ignore_ascii_case("Accept-Encoding: identity")), + "got request: {request}" + ); + } + + #[test] + fn http_status_error_has_stable_download_context() { + let response = + b"HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"; + let error = stage_from_raw_response(response, policy_for_tests()) + .unwrap_err() + .to_string(); + assert!( + error.contains("下载失败") && error.contains("请手动下载"), + "got: {error}" + ); + } + + #[test] + fn total_deadline_is_checked_after_http_status_response() { + let policy = DownloadPolicy { + idle_read_timeout: Duration::from_millis(300), + http_timeout: Duration::from_millis(100), + ..policy_for_tests() + }; + let response = + b"HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"; + let error = stage_from_delayed_raw_response(response, Duration::from_millis(150), policy) + .unwrap_err(); + assert!( + contains_io_kind(&error, io::ErrorKind::TimedOut), + "got: {error:#}" + ); + } + + #[test] + fn unpack_reads_all_concatenated_gzip_members() { + let mut compressed = gzip(b"one"); + compressed.extend_from_slice(&gzip(b"two")); + let mut unpacked = Vec::new(); + assert_eq!( + unpack_gzip(Cursor::new(compressed), &mut unpacked, 6).unwrap(), + 6 + ); + assert_eq!(unpacked, b"onetwo"); + } + + #[test] + fn stage_accepts_gzip_at_exact_uncompressed_limit() { + let policy = DownloadPolicy { + max_uncompressed: 4, + ..policy_for_tests() + }; + stage_compressed(&gzip(b"test"), policy).unwrap(); + } + + #[test] + fn truncated_gzip_trailer_is_rejected_during_staging() { + let mut compressed = gzip(b"test"); + compressed.truncate(compressed.len() - 4); + let error = stage_compressed(&compressed, policy_for_tests()) + .unwrap_err() + .to_string(); + assert!(error.contains("解压 gzip失败"), "got: {error}"); + } + + #[test] + fn modified_gzip_crc_is_rejected_during_staging() { + let mut compressed = gzip(b"test"); + let crc = compressed.len() - 8; + compressed[crc] ^= 0xff; + let error = stage_compressed(&compressed, policy_for_tests()) + .unwrap_err() + .to_string(); + assert!(error.contains("解压 gzip失败"), "got: {error}"); + } + + #[test] + fn trailing_non_gzip_bytes_are_rejected_during_staging() { + let mut compressed = gzip(b"test"); + compressed.extend_from_slice(b"not-gzip"); + let error = stage_compressed(&compressed, policy_for_tests()) + .unwrap_err() + .to_string(); + assert!(error.contains("解压 gzip失败"), "got: {error}"); + } #[test] fn bounded_copy_accepts_exact_limit_and_rejects_one_more() { From ef340544074db2f25b298b3031413687e90c859b Mon Sep 17 00:00:00 2001 From: xr843 <137012659+xr843@users.noreply.github.com> Date: Sat, 11 Jul 2026 22:40:51 +0800 Subject: [PATCH 05/14] fix(data): preserve raw HTTP response bytes --- Cargo.lock | 1 - Cargo.toml | 2 +- src/data/transfer.rs | 72 ++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 71 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index db6e0a7..dd0bbf9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -713,7 +713,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" dependencies = [ "base64", - "flate2", "log", "once_cell", "rustls", diff --git a/Cargo.toml b/Cargo.toml index 97bfc3c..d09e000 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,7 +25,7 @@ rusqlite = { version = "=0.40.1", default-features = false, features = ["bundled serde = { version = "1", features = ["derive"] } serde_json = "1" directories = "5" -ureq = "2" +ureq = { version = "2", default-features = false, features = ["tls"] } sha2 = "0.10" flate2 = "1" anyhow = "1" diff --git a/src/data/transfer.rs b/src/data/transfer.rs index 375e77a..f800e7c 100644 --- a/src/data/transfer.rs +++ b/src/data/transfer.rs @@ -221,12 +221,13 @@ fn stage_candidate_inner( .call(); require_http_deadline(deadline)?; let response = response.map_err(|error| { - anyhow!( + let context = format!( "下载失败: {}: {error}\n请手动下载:\n {}\n解压后放到: {}", source.url, source.url, live_path.display() - ) + ); + anyhow::Error::new(error).context(context) })?; let declared = declared_length(&response, policy.max_compressed)?; eprintln!("{}", download_notice(declared)); @@ -512,6 +513,36 @@ mod tests { result } + fn stage_from_pausing_headers(pause: Duration, policy: DownloadPolicy) -> Result<()> { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let _ = read_request(&mut stream); + stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n") + .unwrap(); + stream.flush().unwrap(); + std::thread::sleep(pause); + let _ = stream.write_all(b"\r\n"); + }); + let directory = tempfile::tempdir().unwrap(); + let live = directory.path().join("data.sqlite"); + let url = format!("http://{address}/data.gz"); + let sha = "0".repeat(64); + let result = stage_candidate( + &live, + &DataSource { + url: &url, + sha256: &sha, + }, + policy, + ) + .map(drop); + server.join().unwrap(); + result + } + fn contains_io_kind(error: &anyhow::Error, expected: io::ErrorKind) -> bool { error.chain().any(|cause| { cause @@ -681,6 +712,29 @@ mod tests { ); } + #[test] + fn content_encoding_cannot_bypass_wire_length_limit() { + let encoded = gzip(b"x"); + let maximum = encoded.len() as u64 - 1; + let mut response = format!( + "HTTP/1.1 200 OK\r\nContent-Encoding: gzip\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + encoded.len() + ) + .into_bytes(); + response.extend_from_slice(&encoded); + let policy = DownloadPolicy { + max_compressed: maximum, + ..policy_for_tests() + }; + let error = stage_from_raw_response(&response, policy) + .unwrap_err() + .to_string(); + assert!( + error.contains("Content-Length") && error.contains("超过"), + "got: {error}" + ); + } + #[test] fn http_status_error_has_stable_download_context() { let response = @@ -711,6 +765,20 @@ mod tests { ); } + #[test] + fn header_idle_timeout_preserves_io_source() { + let policy = DownloadPolicy { + idle_read_timeout: Duration::from_millis(100), + ..policy_for_tests() + }; + let error = stage_from_pausing_headers(Duration::from_millis(300), policy).unwrap_err(); + assert!(error.to_string().contains("下载失败"), "got: {error:#}"); + assert!( + contains_io_kind(&error, io::ErrorKind::TimedOut), + "got: {error:#}" + ); + } + #[test] fn unpack_reads_all_concatenated_gzip_members() { let mut compressed = gzip(b"one"); From 0bb23bfd8a0d8815e0130769a704f140e8ae7b6f Mon Sep 17 00:00:00 2001 From: xr843 <137012659+xr843@users.noreply.github.com> Date: Sat, 11 Jul 2026 23:03:34 +0800 Subject: [PATCH 06/14] fix(data): serialize installation and cleanup --- src/cli.rs | 15 +- src/data.rs | 23 +++ src/data/operation_lock.rs | 72 ++++++++ src/data/transfer.rs | 37 ++++ tests/command.rs | 3 + tests/data.rs | 357 ++++++++++++++++++++++++++++++++++++- 6 files changed, 493 insertions(+), 14 deletions(-) create mode 100644 src/data/operation_lock.rs diff --git a/src/cli.rs b/src/cli.rs index 9a80688..306828e 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -339,14 +339,13 @@ fn run_data(action: DataAction) -> Result { } DataAction::Clean { data_dir } => { let path = data::resolve_data_path(data_dir)?; - // A crashed download can leave a temp sibling; sweep it too. - let _ = std::fs::remove_file(path.with_extension("tmp")); - if path.exists() { - let size = std::fs::metadata(&path)?.len(); - std::fs::remove_file(&path)?; - println!("已删除 {} (释放 {} MB)", path.display(), size / MB); - } else { - println!("本地无数据,无需清理: {}", path.display()); + match data::clean_data(&path)? { + Some(size) => { + println!("已删除 {} (释放 {} MB)", path.display(), size / MB); + } + None => { + println!("本地无数据,无需清理: {}", path.display()); + } } Ok(0) } diff --git a/src/data.rs b/src/data.rs index bc1d43e..9612361 100644 --- a/src/data.rs +++ b/src/data.rs @@ -3,6 +3,7 @@ use rusqlite::{OpenFlags, OptionalExtension}; use std::io::Read; use std::path::{Path, PathBuf}; +mod operation_lock; mod transfer; pub const EXPECTED_DATA_VERSION: &str = "v1"; @@ -108,6 +109,10 @@ pub fn ensure_data(path: &Path, offline: bool, source: &DataSource) -> Result<() if let Some(parent) = path.parent() { std::fs::create_dir_all(parent).context("创建缓存目录失败")?; } + let _lock = operation_lock::acquire(path, operation_lock::LOCK_WAIT_TIMEOUT)?; + if path.exists() { + return Ok(()); + } install_candidate(path, source) } @@ -118,9 +123,27 @@ pub fn update_data(path: &Path, source: &DataSource) -> Result<()> { std::fs::create_dir_all(parent).context("创建缓存目录失败")?; } + let _lock = operation_lock::acquire(path, operation_lock::LOCK_WAIT_TIMEOUT)?; install_candidate(path, source) } +pub fn clean_data(path: &Path) -> Result> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).context("创建缓存目录失败")?; + } + let _lock = operation_lock::acquire(path, operation_lock::LOCK_WAIT_TIMEOUT)?; + transfer::remove_known_artifacts(path)?; + let size = match std::fs::metadata(path) { + Ok(metadata) => Some(metadata.len()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(error) => return Err(error).context("读取数据文件状态失败"), + }; + if size.is_some() { + std::fs::remove_file(path).with_context(|| format!("删除数据失败: {}", path.display()))?; + } + Ok(size) +} + fn install_candidate(path: &Path, source: &DataSource<'_>) -> Result<()> { let candidate = transfer::stage_candidate(path, source, transfer::PRODUCTION_POLICY)?; if let Err(error) = verify_dataset_file(candidate.path()).map(|_| ()) { diff --git a/src/data/operation_lock.rs b/src/data/operation_lock.rs new file mode 100644 index 0000000..8f79242 --- /dev/null +++ b/src/data/operation_lock.rs @@ -0,0 +1,72 @@ +use super::sibling_path; +use anyhow::{anyhow, Context, Result}; +use std::fs::{File, OpenOptions, TryLockError}; +use std::path::Path; +use std::time::{Duration, Instant}; + +const POLL_INTERVAL: Duration = Duration::from_millis(100); +pub(super) const LOCK_WAIT_TIMEOUT: Duration = Duration::from_secs(20 * 60); + +#[derive(Debug)] +pub(super) struct OperationLock { + _file: File, +} + +pub(super) fn acquire(data_path: &Path, timeout: Duration) -> Result { + let lock_path = sibling_path(data_path, ".lock")?; + let file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(&lock_path) + .with_context(|| format!("打开数据操作锁失败: {}", lock_path.display()))?; + let started = Instant::now(); + let mut reported_wait = false; + loop { + match file.try_lock() { + Ok(()) => return Ok(OperationLock { _file: file }), + Err(TryLockError::WouldBlock) => { + if started.elapsed() >= timeout { + return Err(anyhow!( + "等待其他 fojin 数据操作超时: {}", + lock_path.display() + )); + } + if !reported_wait { + eprintln!("检测到另一个 fojin 数据操作,正在等待..."); + reported_wait = true; + } + let remaining = timeout.saturating_sub(started.elapsed()); + std::thread::sleep(POLL_INTERVAL.min(remaining)); + } + Err(TryLockError::Error(error)) => { + return Err(error) + .with_context(|| format!("获取数据操作锁失败: {}", lock_path.display())); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + + #[test] + fn competing_lock_times_out_then_succeeds_after_drop() { + let directory = tempfile::tempdir().unwrap(); + let data = directory.path().join("data.sqlite"); + let first = acquire(&data, Duration::from_millis(100)).unwrap(); + let error = acquire(&data, Duration::from_millis(20)) + .unwrap_err() + .to_string(); + assert!( + error.contains("等待") && error.contains("超时"), + "got: {error}" + ); + drop(first); + acquire(&data, Duration::from_millis(100)).unwrap(); + assert!(data.with_file_name("data.sqlite.lock").exists()); + } +} diff --git a/src/data/transfer.rs b/src/data/transfer.rs index f800e7c..7bab4d2 100644 --- a/src/data/transfer.rs +++ b/src/data/transfer.rs @@ -167,6 +167,43 @@ fn remove_artifact_family(path: &Path) -> Result<()> { Ok(()) } +pub(super) fn remove_known_artifacts(live_path: &Path) -> Result<()> { + let legacy = live_path.with_extension("tmp"); + match std::fs::remove_file(&legacy) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => { + return Err(error).with_context(|| format!("删除旧临时数据失败: {}", legacy.display())); + } + } + + let directory = live_path + .parent() + .ok_or_else(|| anyhow!("数据路径没有父目录: {}", live_path.display()))?; + let file_name = live_path + .file_name() + .ok_or_else(|| anyhow!("数据路径没有文件名: {}", live_path.display()))? + .to_string_lossy(); + let download_prefix = format!("{file_name}.download."); + let candidate_prefix = format!("{file_name}.candidate."); + for entry in std::fs::read_dir(directory) + .with_context(|| format!("读取数据目录失败: {}", directory.display()))? + { + let entry = entry.context("读取数据目录项失败")?; + let name = entry.file_name(); + let name = name.to_string_lossy(); + if !name.starts_with(&download_prefix) && !name.starts_with(&candidate_prefix) { + continue; + } + if entry.file_type().context("读取临时数据类型失败")?.is_dir() { + continue; + } + std::fs::remove_file(entry.path()) + .with_context(|| format!("删除临时数据失败: {}", entry.path().display()))?; + } + Ok(()) +} + fn copy_bounded( mut reader: impl Read, mut writer: impl Write, diff --git a/tests/command.rs b/tests/command.rs index cdff623..1c5be76 100644 --- a/tests/command.rs +++ b/tests/command.rs @@ -293,12 +293,15 @@ fn data_status_reports_missing_data_without_downloading() { fn data_clean_removes_file_and_is_idempotent() { let dir = tempfile::tempdir().unwrap(); write_fixture_db(dir.path()); + let lock_path = dir.path().join("data.sqlite.lock"); let out = run_fojin(&["data", "clean"], dir.path()); assert_eq!(out.status.code(), Some(0)); assert!(!dir.path().join("data.sqlite").exists()); + assert!(lock_path.is_file()); let out2 = run_fojin(&["data", "clean"], dir.path()); assert_eq!(out2.status.code(), Some(0)); assert!(String::from_utf8(out2.stdout).unwrap().contains("无数据")); + assert!(lock_path.is_file()); } #[test] diff --git a/tests/data.rs b/tests/data.rs index d5e5b74..8d29033 100644 --- a/tests/data.rs +++ b/tests/data.rs @@ -1,9 +1,9 @@ use fojin_cli::data::{ - ensure_data, gunzip, open_compatible_db, open_read_only_db, resolve_data_path, update_data, - validate_compatibility, verify_dataset, verify_dataset_file, verify_sha256, DataSource, - EXPECTED_DATA_VERSION, EXPECTED_NORM_RULESET, + clean_data, ensure_data, gunzip, open_compatible_db, open_read_only_db, resolve_data_path, + update_data, validate_compatibility, verify_dataset, verify_dataset_file, verify_sha256, + DataSource, EXPECTED_DATA_VERSION, EXPECTED_NORM_RULESET, }; -use std::io::Write; +use std::io::{Read, Write}; use std::path::PathBuf; #[test] @@ -53,6 +53,159 @@ fn present_file_is_a_noop() { assert!(ensure_data(&path, false, &src).is_ok()); } +#[test] +fn ensure_data_rechecks_existence_after_waiting_for_lock() { + let directory = tempfile::tempdir().unwrap(); + let data = directory.path().join("data.sqlite"); + let lock_path = sibling_path(&data, ".lock"); + let blocker = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(lock_path) + .unwrap(); + blocker.lock().unwrap(); + + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let source_url = format!("http://{}/data.gz", listener.local_addr().unwrap()); + let server = std::thread::spawn(move || { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(1); + loop { + match listener.accept() { + Ok((mut stream, _)) => { + let mut request = [0_u8; 4096]; + let _ = std::io::Read::read(&mut stream, &mut request); + stream + .write_all( + b"HTTP/1.1 500 Internal Server Error\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + ) + .unwrap(); + return 1_usize; + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + if std::time::Instant::now() >= deadline { + return 0; + } + std::thread::sleep(std::time::Duration::from_millis(5)); + } + Err(error) => panic!("accept failed: {error}"), + } + } + }); + + let worker_data = data.clone(); + let (sender, receiver) = std::sync::mpsc::channel(); + let worker = std::thread::spawn(move || { + let source = DataSource { + url: &source_url, + sha256: "unused", + }; + sender + .send(ensure_data(&worker_data, false, &source)) + .unwrap(); + }); + + let early = receiver.recv_timeout(std::time::Duration::from_millis(100)); + let waited_for_lock = matches!(&early, Err(std::sync::mpsc::RecvTimeoutError::Timeout)); + std::fs::write(&data, b"installed by competing process").unwrap(); + drop(blocker); + + let result = match early { + Ok(result) => result, + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => receiver + .recv_timeout(std::time::Duration::from_secs(2)) + .expect("ensure_data remained blocked after the lock was released"), + Err(error) => panic!("ensure_data worker disconnected: {error}"), + }; + worker.join().unwrap(); + let requests = server.join().unwrap(); + assert!( + waited_for_lock, + "ensure_data did not wait for the operation lock: {result:?}" + ); + assert_eq!(requests, 0, "lock waiter attempted a redundant download"); + result.unwrap(); + assert_eq!( + std::fs::read(data).unwrap(), + b"installed by competing process" + ); +} + +#[test] +fn clean_data_waits_for_operation_lock_before_deleting() { + let directory = tempfile::tempdir().unwrap(); + let data = directory.path().join("data.sqlite"); + std::fs::write(&data, b"live").unwrap(); + let lock_path = sibling_path(&data, ".lock"); + let blocker = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(&lock_path) + .unwrap(); + blocker.lock().unwrap(); + + let worker_data = data.clone(); + let (sender, receiver) = std::sync::mpsc::channel(); + let worker = std::thread::spawn(move || { + sender.send(clean_data(&worker_data)).unwrap(); + }); + + assert!(matches!( + receiver.recv_timeout(std::time::Duration::from_millis(100)), + Err(std::sync::mpsc::RecvTimeoutError::Timeout) + )); + assert!(data.is_file(), "clean deleted live data without the lock"); + drop(blocker); + + assert_eq!( + receiver + .recv_timeout(std::time::Duration::from_secs(2)) + .expect("clean remained blocked after the lock was released") + .unwrap(), + Some(4) + ); + worker.join().unwrap(); + assert!(!data.exists()); + assert!(lock_path.is_file()); +} + +#[test] +fn clean_data_removes_known_file_artifacts_and_preserves_other_entries() { + let directory = tempfile::tempdir().unwrap(); + let data = directory.path().join("data.sqlite"); + let legacy = data.with_extension("tmp"); + let download = sibling_path(&data, ".download.123.0.gz"); + let candidate = sibling_path(&data, ".candidate.123.1"); + let candidate_journal = sibling_path(&data, ".candidate.123.1-journal"); + let matching_directory = sibling_path(&data, ".download.keep"); + let lock_path = sibling_path(&data, ".lock"); + let unrelated = sibling_path(&data, ".unrelated"); + std::fs::write(&data, b"live").unwrap(); + for artifact in [&legacy, &download, &candidate, &candidate_journal] { + std::fs::write(artifact, b"temporary").unwrap(); + } + std::fs::create_dir(&matching_directory).unwrap(); + std::fs::write(&lock_path, b"permanent lock inode").unwrap(); + std::fs::write(&unrelated, b"unrelated").unwrap(); + + assert_eq!(clean_data(&data).unwrap(), Some(4)); + for removed in [&data, &legacy, &download, &candidate, &candidate_journal] { + assert!(!removed.exists(), "artifact remains: {}", removed.display()); + } + assert!(matching_directory.is_dir()); + assert_eq!(std::fs::read(&lock_path).unwrap(), b"permanent lock inode"); + assert_eq!(std::fs::read(&unrelated).unwrap(), b"unrelated"); + + assert_eq!(clean_data(&data).unwrap(), None); + assert!(lock_path.is_file()); + assert!(matching_directory.is_dir()); + assert!(unrelated.is_file()); +} + #[test] fn write_atomic_writes_content_and_leaves_no_temp() { use fojin_cli::data::write_atomic; @@ -139,6 +292,97 @@ fn ensure_data_downloads_verifies_and_unpacks() { verify_dataset_file(&path).unwrap(); } +#[test] +fn concurrent_first_install_downloads_once() { + if std::env::var_os("FOJIN_CONCURRENT_WORKER").is_some() { + let path = PathBuf::from(std::env::var_os("FOJIN_WORKER_DATA").unwrap()); + let url = std::env::var("FOJIN_WORKER_URL").unwrap(); + let sha = std::env::var("FOJIN_WORKER_SHA256").unwrap(); + ensure_data( + &path, + false, + &DataSource { + url: &url, + sha256: &sha, + }, + ) + .unwrap(); + return; + } + use std::process::Command; + use std::time::{Duration, Instant}; + + let directory = tempfile::tempdir().unwrap(); + let data_path = directory.path().join("data.sqlite"); + let body = gzip_bytes(&replacement_database_bytes()); + let sha = sha256_hex(&body); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}/data.gz", listener.local_addr().unwrap()); + let server_body = body.clone(); + let server = std::thread::spawn(move || { + let mut requests = 0_usize; + let (mut first, _) = listener.accept().unwrap(); + requests += 1; + let mut request = [0_u8; 4096]; + let _ = first.read(&mut request); + write!( + first, + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + server_body.len() + ) + .unwrap(); + let midpoint = server_body.len() / 2; + first.write_all(&server_body[..midpoint]).unwrap(); + first.flush().unwrap(); + std::thread::sleep(Duration::from_millis(200)); + first.write_all(&server_body[midpoint..]).unwrap(); + drop(first); + + listener.set_nonblocking(true).unwrap(); + let deadline = Instant::now() + Duration::from_secs(2); + while Instant::now() < deadline { + match listener.accept() { + Ok((mut stream, _)) => { + requests += 1; + let _ = stream.read(&mut request); + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + server_body.len() + ) + .unwrap(); + stream.write_all(&server_body).unwrap(); + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + std::thread::sleep(Duration::from_millis(10)); + } + Err(error) => panic!("accept failed: {error}"), + } + } + requests + }); + + let spawn_worker = || { + Command::new(std::env::current_exe().unwrap()) + .arg("--exact") + .arg("concurrent_first_install_downloads_once") + .arg("--nocapture") + .env("FOJIN_CONCURRENT_WORKER", "1") + .env("FOJIN_WORKER_DATA", &data_path) + .env("FOJIN_WORKER_URL", &url) + .env("FOJIN_WORKER_SHA256", &sha) + .spawn() + .unwrap() + }; + let mut first = spawn_worker(); + let mut second = spawn_worker(); + assert!(first.wait().unwrap().success()); + assert!(second.wait().unwrap().success()); + assert_eq!(server.join().unwrap(), 1); + verify_dataset_file(&data_path).unwrap(); + assert_no_owned_candidate_artifacts(&data_path); +} + #[test] fn first_install_rejects_incompatible_database() { let gz = gzip_bytes(b"fake sqlite payload"); @@ -503,11 +747,15 @@ fn update_data_preserves_live_dataset_when_candidate_fails_compatibility() { assert!(err.contains("dataset incompatibility"), "got: {err}"); assert_eq!(std::fs::read(&path).unwrap(), b"live dataset"); - let entries: Vec<_> = std::fs::read_dir(dir.path()) + let entries: std::collections::BTreeSet<_> = std::fs::read_dir(dir.path()) .unwrap() .map(|entry| entry.unwrap().file_name()) .collect(); - assert_eq!(entries, vec![std::ffi::OsString::from("data.sqlite")]); + let expected: std::collections::BTreeSet<_> = ["data.sqlite", "data.sqlite.lock"] + .into_iter() + .map(std::ffi::OsString::from) + .collect(); + assert_eq!(entries, expected); } #[test] @@ -524,6 +772,103 @@ fn update_data_replaces_valid_existing_dataset() { assert_no_candidate_artifacts(&path); } +#[test] +fn update_download_keeps_live_dataset_readable_without_query_locking() { + let directory = tempfile::tempdir().unwrap(); + let data = directory.path().join("data.sqlite"); + let live = compatible_database_bytes(|connection| { + connection + .execute("INSERT INTO meta(key, value) VALUES ('marker', 'live')", []) + .unwrap(); + }); + std::fs::write(&data, live).unwrap(); + + let replacement = gzip_bytes(&replacement_database_bytes()); + let sha = sha256_hex(&replacement); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let source_url = format!("http://{}/data.gz", listener.local_addr().unwrap()); + let (midpoint_sender, midpoint_receiver) = std::sync::mpsc::channel(); + let (release_sender, release_receiver) = std::sync::mpsc::channel(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = [0_u8; 4096]; + let _ = std::io::Read::read(&mut stream, &mut request); + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + replacement.len() + ) + .unwrap(); + let midpoint = replacement.len() / 2; + stream.write_all(&replacement[..midpoint]).unwrap(); + stream.flush().unwrap(); + midpoint_sender.send(()).unwrap(); + release_receiver.recv().unwrap(); + stream.write_all(&replacement[midpoint..]).unwrap(); + }); + + let update_path = data.clone(); + let updater = std::thread::spawn(move || { + update_data( + &update_path, + &DataSource { + url: &source_url, + sha256: &sha, + }, + ) + }); + midpoint_receiver + .recv_timeout(std::time::Duration::from_secs(2)) + .expect("updater did not begin its download"); + + let lock_path = sibling_path(&data, ".lock"); + let lock_probe = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(lock_path) + .unwrap(); + let update_holds_lock = match lock_probe.try_lock() { + Err(std::fs::TryLockError::WouldBlock) => true, + Ok(()) => false, + Err(std::fs::TryLockError::Error(error)) => { + panic!("probing the update operation lock failed: {error}") + } + }; + drop(lock_probe); + + let query_path = data.clone(); + let (query_sender, query_receiver) = std::sync::mpsc::channel(); + let query = std::thread::spawn(move || { + let result = (|| { + let connection = open_compatible_db(&query_path)?; + let marker = + connection.query_row("SELECT value FROM meta WHERE key = 'marker'", [], |row| { + row.get::<_, String>(0) + })?; + Ok::<_, anyhow::Error>(marker) + })(); + query_sender.send(result).unwrap(); + }); + let live_marker = query_receiver + .recv_timeout(std::time::Duration::from_secs(1)) + .expect("ordinary query waited for the data operation lock") + .unwrap(); + query.join().unwrap(); + + release_sender.send(()).unwrap(); + server.join().unwrap(); + let update_result = updater.join().unwrap(); + assert!( + update_holds_lock, + "update_data did not hold the operation lock while downloading" + ); + assert_eq!(live_marker, "live"); + update_result.unwrap(); + assert_replacement_marker(&data); +} + #[test] fn update_data_installs_valid_dataset_when_target_is_absent() { let dir = tempfile::tempdir().unwrap(); From 10f3c3bfcdd03f5fc429d9134d63041320339477 Mon Sep 17 00:00:00 2001 From: xr843 <137012659+xr843@users.noreply.github.com> Date: Sat, 11 Jul 2026 23:17:17 +0800 Subject: [PATCH 07/14] docs: describe bounded data downloads --- CHANGELOG.md | 2 ++ README.md | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f2e3f1..e2a4b85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ All notable changes to this project will be documented in this file. Version 0.3.0 is prepared but has not been published. Its stabilization work includes: - Data verification: strengthen `fojin data verify` and dataset compatibility checks. +- Data pipeline: move installs and updates to a bounded, disk-streamed, checksum-first pipeline. +- Data concurrency: serialize concurrent install, update, and clean operations per data directory, with full candidate validation before publication. - Query correctness: make short-query matching literal and remove duplicate parallel text within a match group and language. - SQLite safety: upgrade the bundled SQLite and verify its runtime version. - Release integrity: validate release versions, locked builds, archive contents, checksums, and installer verification. diff --git a/README.md b/README.md index 5b67df1..e4046fc 100644 --- a/README.md +++ b/README.md @@ -184,6 +184,10 @@ fojin parallel "<汉文短语>" --json --offline - 来源:Dharmamitra 的 [MITRA-parallel](https://github.com/dharmamitra/mitra-parallel) 对齐数据集([Nehrdich & Keutzer, 2026](https://arxiv.org/pdf/2601.06400)),以 GitHub Release(`data-v1`)形式分发;学术使用请引用原论文(BibTeX 见 [`DATA_LICENSE`](DATA_LICENSE))。 - 当前二进制把官方下载地址、SHA-256 与兼容元数据固定在 `data-v1`;`fojin data update` 只会重新获取这份固定数据,不会自动切换到未来的数据主版本。版本、归一化规则或查询所需 schema 不兼容的数据会被拒绝。 - 首次运行时下载,压缩包约 **183 MB**,解压后约 **561 MB**(SQLite)。下载后完全离线可用。 +- 安装和更新采用有界的磁盘流式传输,不再在内存中缓冲完整压缩包或数据库;压缩响应上限为 **256 MiB**,解压后的数据库上限为 **768 MiB**。更新期间可能临时需要现用数据库所占空间,外加约 **744 MiB** 暂存磁盘空间(约 183 MiB 压缩包 + 约 561 MiB 候选数据库)。 +- HTTP 连接超时为 **30 秒**,空闲读取超时为 **60 秒**,总时限为 **15 分钟**。总时限是协作式检查,并非不可突破的硬截止:同步 DNS、响应头处理或单次读取最多可能额外越过一个 60 秒空闲超时窗口。 +- 同一数据目录上的首次安装、更新和清理操作按 single-flight 串行执行;等待者最多等待 **20 分钟**。永久保留的 `data.sqlite.lock` 文件是无害的协调文件,`fojin data clean` 会有意保留它。 +- 离线查询行为及固定到 `data-v1` 的校验和契约保持不变。 - 当前不含巴利对齐(上游 MITRA-parallel 尚未覆盖巴利),默认输出不显示巴利行;显式 `--lang pi` 仍可查询(如实答「未找到对齐」)。程序的渲染路径可兼容未来新增语言行,但当前官方下载通道仍固定为 `data-v1`;上游出现新语言不代表当前二进制会自动获得它。**渲染兼容不等于官方更新通道无需升级**,未来数据版本可能要求升级二进制或明确切换数据发布。 - 许可:**CC BY-SA 4.0**(Dharmamitra + fojin)。 - 范围:仅含 MITRA 跨藏平行;fojin 自有的精选对齐(alignment_pairs)**未包含**在本数据集中。 @@ -219,6 +223,10 @@ fojin data verify # verify version, SQLite, and FTS integrity - **Build/install integrity**: building from crates.io or source requires Rust 1.95+ (MSRV 1.95). Starting with v0.3.0, the shell installer requires the target binary release to provide `SHA256SUMS` and verifies the archive before extraction. It fails closed for an older latest or explicitly selected release without that file, including the transition before v0.3.0 is published; use the currently published crates.io version or a source build instead. This does not state that v0.3.0 has been released. - **For AI agents**: pure-JSON stdout, semantic exit codes (`0` ok / `1` runtime / `2` usage), zero network with `--offline`. Ready-made Claude Code integration in [`examples/claude/`](examples/claude/). - **Data**: 908,620 zh↔sa/bo alignments from Dharmamitra's [MITRA-parallel](https://github.com/dharmamitra/mitra-parallel) dataset, redistributed under CC BY-SA 4.0. The official URL, checksum, and compatibility contract remain pinned to `data-v1`; rendering support for future language rows does not mean the official update channel can adopt them without a binary upgrade. Academic use: please cite [Nehrdich & Keutzer (2026)](https://arxiv.org/pdf/2601.06400) — BibTeX in [`DATA_LICENSE`](DATA_LICENSE). +- **Data transfer resources**: installs and updates use bounded, disk-streamed transfers and no longer buffer the complete archive or database in memory. Compressed responses are capped at **256 MiB** and decompressed databases at **768 MiB**. An update can temporarily require the live database plus roughly **744 MiB** of staging disk (about 183 MiB for the archive and 561 MiB for the candidate database). +- **Data timeouts**: HTTP connect and idle-read timeouts are **30 seconds** and **60 seconds**, with a **15-minute** cooperative total deadline. The total deadline is not an unbreakable hard cutoff: synchronous DNS, response-header processing, or one read can overrun it by at most one additional 60-second idle-timeout window. +- **Concurrent data operations**: initial install, update, and clean operations on one data directory are single-flight; a waiter may wait up to **20 minutes**. The permanent `data.sqlite.lock` file is harmless coordination state and intentionally survives `fojin data clean`. +- **Stable query contract**: offline queries and the checksum contract pinned to `data-v1` are unchanged. - **Not in scope**: semantic search, Pāli, translation — use [Dharmamitra](https://dharmamitra.org)'s online APIs for those; the two are complementary. - **License**: code MIT OR Apache-2.0; data CC BY-SA 4.0. From 2d6f393cadec88511e9970d35cd27ef13289773e Mon Sep 17 00:00:00 2001 From: xr843 <137012659+xr843@users.noreply.github.com> Date: Sun, 12 Jul 2026 00:14:23 +0800 Subject: [PATCH 08/14] fix(data): enforce transfer deadlines and ownership --- Cargo.lock | 100 ++++----- Cargo.toml | 2 +- src/data/transfer.rs | 489 ++++++++++++++++++++++++++++++++++++++----- tests/data.rs | 256 ++++++++++++++++++++-- 4 files changed, 711 insertions(+), 136 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dd0bbf9..30b15f5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -85,6 +85,12 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + [[package]] name = "cc" version = "1.2.66" @@ -266,15 +272,6 @@ dependencies = [ "ureq", ] -[[package]] -name = "form_urlencoded" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" -dependencies = [ - "percent-encoding", -] - [[package]] name = "generic-array" version = "0.14.7" @@ -315,15 +312,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] -name = "idna" -version = "0.5.0" +name = "http" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "bytes", + "itoa", ] +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -658,48 +661,18 @@ dependencies = [ "syn", ] -[[package]] -name = "tinyvec" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - [[package]] name = "typenum" version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" -[[package]] -name = "unicode-bidi" -version = "0.3.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" - [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" -[[package]] -name = "unicode-normalization" -version = "0.1.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" -dependencies = [ - "tinyvec", -] - [[package]] name = "untrusted" version = "0.9.0" @@ -708,30 +681,38 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "ureq" -version = "2.12.1" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" +checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0" dependencies = [ "base64", "log", - "once_cell", + "percent-encoding", "rustls", "rustls-pki-types", - "url", - "webpki-roots 0.26.11", + "ureq-proto", + "utf8-zero", + "webpki-roots", ] [[package]] -name = "url" -version = "2.5.0" +name = "ureq-proto" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +checksum = "e994ba84b0bd1b1b0cf92878b7ef898a5c1760108fe7b6010327e274917a808c" dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", + "base64", + "http", + "httparse", + "log", ] +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + [[package]] name = "utf8parse" version = "0.2.2" @@ -765,15 +746,6 @@ dependencies = [ "wit-bindgen", ] -[[package]] -name = "webpki-roots" -version = "0.26.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" -dependencies = [ - "webpki-roots 1.0.8", -] - [[package]] name = "webpki-roots" version = "1.0.8" diff --git a/Cargo.toml b/Cargo.toml index d09e000..61df9c7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,7 +25,7 @@ rusqlite = { version = "=0.40.1", default-features = false, features = ["bundled serde = { version = "1", features = ["derive"] } serde_json = "1" directories = "5" -ureq = { version = "2", default-features = false, features = ["tls"] } +ureq = { version = "=3.3.0", default-features = false, features = ["rustls"] } sha2 = "0.10" flate2 = "1" anyhow = "1" diff --git a/src/data/transfer.rs b/src/data/transfer.rs index 7bab4d2..da4eb64 100644 --- a/src/data/transfer.rs +++ b/src/data/transfer.rs @@ -6,12 +6,90 @@ use std::fs::{File, OpenOptions}; use std::io::{self, Read, Seek, Write}; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; -use std::time::{Duration, Instant}; +use std::time::Duration; const MIB: u64 = 1024 * 1024; const BUFFER_SIZE: usize = 64 * 1024; static ARTIFACT_SEQUENCE: AtomicU64 = AtomicU64::new(0); +// Ureq's socket adapter turns an already-expired zero timeout into one second +// because operating systems reject a zero socket timeout. Reject it before the +// adapter instead, including when protocol bytes are already buffered, so the +// configured global deadline remains a hard boundary under byte dribbling. +// The transport API is unversioned, so ureq is pinned exactly in Cargo.toml. +#[derive(Debug)] +struct StrictTimeoutConnector; + +#[derive(Debug)] +struct StrictTimeoutTransport { + inner: T, +} + +impl ureq::unversioned::transport::Connector + for StrictTimeoutConnector +{ + type Out = StrictTimeoutTransport; + + fn connect( + &self, + _: &ureq::unversioned::transport::ConnectionDetails<'_>, + chained: Option, + ) -> std::result::Result, ureq::Error> { + Ok(chained.map(|inner| StrictTimeoutTransport { inner })) + } +} + +impl ureq::unversioned::transport::Transport + for StrictTimeoutTransport +{ + fn buffers(&mut self) -> &mut dyn ureq::unversioned::transport::Buffers { + self.inner.buffers() + } + + fn transmit_output( + &mut self, + amount: usize, + timeout: ureq::unversioned::transport::NextTimeout, + ) -> std::result::Result<(), ureq::Error> { + require_remaining_timeout(timeout)?; + self.inner.transmit_output(amount, timeout) + } + + fn maybe_await_input( + &mut self, + timeout: ureq::unversioned::transport::NextTimeout, + ) -> std::result::Result { + require_remaining_timeout(timeout)?; + self.inner.maybe_await_input(timeout) + } + + fn await_input( + &mut self, + timeout: ureq::unversioned::transport::NextTimeout, + ) -> std::result::Result { + require_remaining_timeout(timeout)?; + self.inner.await_input(timeout) + } + + fn is_open(&mut self) -> bool { + self.inner.is_open() + } + + fn is_tls(&self) -> bool { + self.inner.is_tls() + } +} + +fn require_remaining_timeout( + timeout: ureq::unversioned::transport::NextTimeout, +) -> std::result::Result<(), ureq::Error> { + if !timeout.after.is_not_happening() && timeout.after.is_zero() { + Err(ureq::Error::Timeout(timeout.reason)) + } else { + Ok(()) + } +} + #[derive(Clone, Copy, Debug)] pub(super) struct DownloadPolicy { pub connect_timeout: Duration, @@ -138,12 +216,92 @@ fn create_compressed_artifact(live_path: &Path) -> Result<(OwnedCompressed, File fn create_candidate_artifact(live_path: &Path) -> Result<(StagedCandidate, File)> { loop { let path = unique_path(live_path, "candidate", "")?; - match OpenOptions::new().write(true).create_new(true).open(&path) { - Ok(file) => return Ok((StagedCandidate { path, armed: true }, file)), - Err(error) if error.kind() == io::ErrorKind::AlreadyExists => continue, - Err(error) => { - return Err(error).with_context(|| format!("创建候选数据失败: {}", path.display())); - } + if let Some(candidate) = try_create_candidate_artifact(&path)? { + return Ok(candidate); + } + } +} + +fn try_create_candidate_artifact(path: &Path) -> Result> { + try_create_candidate_artifact_with(path, path_entry_exists) +} + +fn try_create_candidate_artifact_with( + path: &Path, + mut entry_exists: F, +) -> Result> +where + F: FnMut(&Path) -> Result, +{ + let sidecars = ["-journal", "-shm", "-wal"] + .map(|suffix| sibling_path(path, suffix)) + .into_iter() + .collect::>>()?; + let family_occupied = sidecars + .iter() + .try_fold(entry_exists(path)?, |occupied, sidecar| { + entry_exists(sidecar).map(|exists| occupied || exists) + })?; + if family_occupied { + return Ok(None); + } + + let file = match OpenOptions::new().write(true).create_new(true).open(path) { + Ok(file) => file, + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => return Ok(None), + Err(error) => { + return Err(error).with_context(|| format!("创建候选数据失败: {}", path.display())); + } + }; + + // The main file reserves this generation. Recheck the sidecars after that + // reservation so a stale family can never be claimed and later deleted by + // this guard. Under the data-operation lock, any sidecar that appears after + // this point can only be produced by SQLite while operating on this unique + // candidate. + let sidecar_occupied = sidecars.iter().try_fold(false, |occupied, sidecar| { + entry_exists(sidecar).map(|exists| occupied || exists) + }); + let sidecar_occupied = match sidecar_occupied { + Ok(occupied) => occupied, + Err(error) => { + drop(file); + return Err(cleanup_owned_candidate_main(path, error)); + } + }; + if sidecar_occupied { + drop(file); + std::fs::remove_file(path) + .with_context(|| format!("释放冲突候选数据失败: {}", path.display()))?; + return Ok(None); + } + + Ok(Some(( + StagedCandidate { + path: path.to_path_buf(), + armed: true, + }, + file, + ))) +} + +fn cleanup_owned_candidate_main(path: &Path, error: anyhow::Error) -> anyhow::Error { + match std::fs::remove_file(path) { + Ok(()) => error, + Err(cleanup) if cleanup.kind() == io::ErrorKind::NotFound => error, + Err(cleanup) => error.context(format!( + "清理候选数据主文件失败: {}: {cleanup}", + path.display() + )), + } +} + +fn path_entry_exists(path: &Path) -> Result { + match std::fs::symlink_metadata(path) { + Ok(_) => Ok(true), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(false), + Err(error) => { + Err(error).with_context(|| format!("检查临时数据路径失败: {}", path.display())) } } } @@ -183,16 +341,17 @@ pub(super) fn remove_known_artifacts(live_path: &Path) -> Result<()> { let file_name = live_path .file_name() .ok_or_else(|| anyhow!("数据路径没有文件名: {}", live_path.display()))? - .to_string_lossy(); - let download_prefix = format!("{file_name}.download."); - let candidate_prefix = format!("{file_name}.candidate."); + .to_str() + .ok_or_else(|| anyhow!("数据文件名不是有效 UTF-8: {}", live_path.display()))?; for entry in std::fs::read_dir(directory) .with_context(|| format!("读取数据目录失败: {}", directory.display()))? { let entry = entry.context("读取数据目录项失败")?; let name = entry.file_name(); - let name = name.to_string_lossy(); - if !name.starts_with(&download_prefix) && !name.starts_with(&candidate_prefix) { + let Some(name) = name.to_str() else { + continue; + }; + if !is_owned_artifact_name(file_name, name) { continue; } if entry.file_type().context("读取临时数据类型失败")?.is_dir() { @@ -204,6 +363,40 @@ pub(super) fn remove_known_artifacts(live_path: &Path) -> Result<()> { Ok(()) } +fn is_owned_artifact_name(live_name: &str, artifact_name: &str) -> bool { + let Some(suffix) = artifact_name.strip_prefix(live_name) else { + return false; + }; + + if let Some(generation) = suffix + .strip_prefix(".download.") + .and_then(|name| name.strip_suffix(".gz")) + { + return is_numeric_generation(generation); + } + + let Some(candidate) = suffix.strip_prefix(".candidate.") else { + return false; + }; + let generation = ["-journal", "-shm", "-wal"] + .into_iter() + .find_map(|sidecar| candidate.strip_suffix(sidecar)) + .unwrap_or(candidate); + is_numeric_generation(generation) +} + +fn is_numeric_generation(value: &str) -> bool { + let mut parts = value.split('.'); + matches!( + (parts.next(), parts.next(), parts.next()), + (Some(pid), Some(sequence), None) + if !pid.is_empty() + && !sequence.is_empty() + && pid.bytes().all(|byte| byte.is_ascii_digit()) + && sequence.bytes().all(|byte| byte.is_ascii_digit()) + ) +} + fn copy_bounded( mut reader: impl Read, mut writer: impl Write, @@ -245,39 +438,51 @@ fn stage_candidate_inner( policy: DownloadPolicy, compressed_file: &mut File, ) -> Result { - let deadline = Instant::now() - .checked_add(policy.http_timeout) - .ok_or_else(|| anyhow!("HTTP 总超时配置溢出"))?; - let agent = ureq::AgentBuilder::new() - .timeout_connect(policy.connect_timeout) - .timeout_read(policy.idle_read_timeout) + let config = ureq::Agent::config_builder() + .max_redirects(5) + .max_redirects_will_error(true) + .max_response_header_size(64 * 1024) + .accept_encoding("identity") + .timeout_global(Some(policy.http_timeout)) + .timeout_resolve(Some(policy.connect_timeout)) + .timeout_connect(Some(policy.connect_timeout)) + .timeout_recv_response(Some(policy.idle_read_timeout)) + .timeout_recv_body(Some(policy.idle_read_timeout)) .build(); - let response = agent + use ureq::unversioned::transport::Connector; + let connector = + ureq::unversioned::transport::DefaultConnector::new().chain(StrictTimeoutConnector); + let agent = ureq::Agent::with_parts( + config, + connector, + ureq::unversioned::resolver::DefaultResolver::default(), + ); + let mut response = agent .get(source.url) - .set("Accept-Encoding", "identity") - .call(); - require_http_deadline(deadline)?; - let response = response.map_err(|error| { - let context = format!( - "下载失败: {}: {error}\n请手动下载:\n {}\n解压后放到: {}", - source.url, - source.url, - live_path.display() - ); - anyhow::Error::new(error).context(context) - })?; + .header("Accept-Encoding", "identity") + .call() + .map_err(normalize_ureq_error) + .map_err(|error| download_error_context(error, "下载失败", live_path, source.url))?; let declared = declared_length(&response, policy.max_compressed)?; eprintln!("{}", download_notice(declared)); - let mut reader = response.into_reader(); + let mut reader = response + .body_mut() + .with_config() + .limit(policy.max_compressed.saturating_add(1)) + .reader(); let mut progress = Progress::new(declared); let mut digest = Sha256::new(); let mut received = 0_u64; let mut buffer = [0_u8; BUFFER_SIZE]; loop { - require_http_deadline(deadline)?; - let count = reader.read(&mut buffer).context("读取响应失败")?; - require_http_deadline(deadline)?; + let count = reader + .read(&mut buffer) + .map_err(normalize_body_read_error) + .map_err(anyhow::Error::new) + .map_err(|error| { + download_error_context(error, "读取响应失败", live_path, source.url) + })?; if count == 0 { break; } @@ -298,7 +503,8 @@ fn stage_candidate_inner( eprintln!("{message}"); } } - require_declared_length(declared, received)?; + require_declared_length(declared, received) + .map_err(|error| download_error_context(error, "读取响应失败", live_path, source.url))?; let actual = digest.finalize(); require_digest(actual.as_ref(), source.sha256).map_err(|error| { anyhow!( @@ -328,25 +534,61 @@ fn stage_candidate_inner( } } -fn require_http_deadline(deadline: Instant) -> Result<()> { - if Instant::now() >= deadline { - return Err(io::Error::new( - io::ErrorKind::TimedOut, - "HTTP total deadline timed out", - )) - .context("读取响应失败"); +fn download_error_context( + error: anyhow::Error, + action: &str, + live_path: &Path, + url: &str, +) -> anyhow::Error { + let cause = error.to_string(); + error.context(format!( + "{action}: {url}: {cause}\n请手动下载:\n {url}\n解压后放到: {}", + live_path.display() + )) +} + +fn normalize_ureq_error(error: ureq::Error) -> anyhow::Error { + if matches!(error, ureq::Error::Timeout(_)) { + anyhow::Error::new(io::Error::new(io::ErrorKind::TimedOut, error)) + } else { + anyhow::Error::new(error) } - Ok(()) } -fn declared_length(response: &ureq::Response, maximum: u64) -> Result> { - if !response.all("Transfer-Encoding").is_empty() { +fn normalize_body_read_error(error: io::Error) -> io::Error { + let is_timeout = error + .get_ref() + .and_then(|source| source.downcast_ref::()) + .is_some_and(|source| matches!(source, ureq::Error::Timeout(_))); + if is_timeout { + io::Error::new(io::ErrorKind::TimedOut, error) + } else { + error + } +} + +fn declared_length( + response: &ureq::http::Response, + maximum: u64, +) -> Result> { + if response + .headers() + .get_all(ureq::http::header::TRANSFER_ENCODING) + .iter() + .next() + .is_some() + { return Ok(None); } - let values = response.all("Content-Length"); + let values: Vec<_> = response + .headers() + .get_all(ureq::http::header::CONTENT_LENGTH) + .iter() + .collect(); match values.as_slice() { [] => Ok(None), [value] => { + let value = value.to_str().context("Content-Length 不是有效 ASCII")?; let parsed = value .parse::() .with_context(|| format!("无效 Content-Length: {value}"))?; @@ -387,6 +629,7 @@ mod tests { use flate2::write::GzEncoder; use flate2::Compression; use std::io::{Cursor, Read, Write}; + use std::time::Instant; fn policy_for_tests() -> DownloadPolicy { DownloadPolicy { @@ -398,6 +641,45 @@ mod tests { } } + #[test] + fn candidate_reservation_rejects_a_preexisting_sidecar_family() { + let directory = tempfile::tempdir().unwrap(); + let candidate = directory.path().join("data.sqlite.candidate.123.4"); + let wal = sibling_path(&candidate, "-wal").unwrap(); + std::fs::write(&wal, b"foreign wal").unwrap(); + + assert!(try_create_candidate_artifact(&candidate).unwrap().is_none()); + assert!(!candidate.exists()); + assert_eq!(std::fs::read(wal).unwrap(), b"foreign wal"); + } + + #[test] + fn candidate_reservation_cleans_its_main_file_when_postcheck_fails() { + let directory = tempfile::tempdir().unwrap(); + let candidate = directory.path().join("data.sqlite.candidate.123.4"); + let checks = std::cell::Cell::new(0_usize); + + let result = try_create_candidate_artifact_with(&candidate, |_| { + let next = checks.get() + 1; + checks.set(next); + if next == 5 { + Err(anyhow!("injected sidecar inspection failure")) + } else { + Ok(false) + } + }); + let error = match result { + Err(error) => error, + Ok(_) => panic!("post-reservation inspection failure was ignored"), + }; + + assert!( + format!("{error:#}").contains("injected sidecar inspection failure"), + "got: {error:#}" + ); + assert!(!candidate.exists(), "owned candidate main leaked"); + } + fn sha256_hex(bytes: &[u8]) -> String { Sha256::digest(bytes) .iter() @@ -580,6 +862,49 @@ mod tests { result } + fn stage_from_protocol_dribble( + prefix: &'static [u8], + dribble: u8, + count: usize, + suffix: &'static [u8], + interval: Duration, + policy: DownloadPolicy, + ) -> (Result<()>, Duration) { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let _ = read_request(&mut stream); + if stream.write_all(prefix).is_err() { + return; + } + for _ in 0..count { + if stream.write_all(&[dribble]).is_err() || stream.flush().is_err() { + return; + } + std::thread::sleep(interval); + } + let _ = stream.write_all(suffix); + }); + let directory = tempfile::tempdir().unwrap(); + let live = directory.path().join("data.sqlite"); + let url = format!("http://{address}/data.gz"); + let sha = "0".repeat(64); + let started = Instant::now(); + let result = stage_candidate( + &live, + &DataSource { + url: &url, + sha256: &sha, + }, + policy, + ) + .map(drop); + let elapsed = started.elapsed(); + server.join().unwrap(); + (result, elapsed) + } + fn contains_io_kind(error: &anyhow::Error, expected: io::ErrorKind) -> bool { error.chain().any(|cause| { cause @@ -639,6 +964,10 @@ mod tests { contains_io_kind(&error, io::ErrorKind::UnexpectedEof), "got: {error:#}" ); + assert!( + format!("{error:#}").contains("请手动下载"), + "got: {error:#}" + ); } #[test] @@ -661,6 +990,64 @@ mod tests { ); } + #[test] + fn global_timeout_interrupts_continuously_dribbling_response_headers() { + let policy = DownloadPolicy { + connect_timeout: Duration::from_millis(200), + idle_read_timeout: Duration::from_millis(200), + http_timeout: Duration::from_millis(250), + max_compressed: 1024, + max_uncompressed: 4096, + }; + let (result, elapsed) = stage_from_protocol_dribble( + b"HTTP/1.1 200 OK\r\nX-Slow: ", + b'a', + 40, + b"\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + Duration::from_millis(50), + policy, + ); + let error = result.unwrap_err(); + assert!(elapsed < Duration::from_secs(1), "elapsed: {elapsed:?}"); + assert!( + contains_io_kind(&error, io::ErrorKind::TimedOut), + "got: {error:#}" + ); + assert!( + format!("{error:#}").contains("请手动下载"), + "got: {error:#}" + ); + } + + #[test] + fn global_timeout_interrupts_continuously_dribbling_chunk_framing() { + let policy = DownloadPolicy { + connect_timeout: Duration::from_millis(200), + idle_read_timeout: Duration::from_millis(200), + http_timeout: Duration::from_millis(250), + max_compressed: 1024, + max_uncompressed: 4096, + }; + let (result, elapsed) = stage_from_protocol_dribble( + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n1;", + b'a', + 40, + b"\r\nx\r\n0\r\n\r\n", + Duration::from_millis(50), + policy, + ); + let error = result.unwrap_err(); + assert!(elapsed < Duration::from_secs(1), "elapsed: {elapsed:?}"); + assert!( + contains_io_kind(&error, io::ErrorKind::TimedOut), + "got: {error:#}" + ); + assert!( + format!("{error:#}").contains("请手动下载"), + "got: {error:#}" + ); + } + #[test] fn oversized_declared_length_is_rejected_before_body_reads() { let response = b"HTTP/1.1 200 OK\r\nContent-Length: 1025\r\nConnection: close\r\n\r\n"; @@ -680,8 +1067,10 @@ mod tests { let error = stage_from_raw_response(response, policy_for_tests()) .unwrap_err() .to_string(); + let lower = error.to_ascii_lowercase(); assert!( - error.contains("无效 Content-Length") && error.contains("nope"), + lower.contains("content-length") + && (error.contains("无效") || lower.contains("not a number")), "got: {error}" ); assert!(!error.contains("读取响应失败"), "got: {error}"); @@ -733,6 +1122,10 @@ mod tests { contains_io_kind(&error, io::ErrorKind::TimedOut), "got: {error:#}" ); + let detail = format!("{error:#}"); + assert!(detail.contains("http://"), "got: {detail}"); + assert!(detail.contains("data.sqlite"), "got: {detail}"); + assert!(detail.contains("请手动下载"), "got: {detail}"); } #[test] diff --git a/tests/data.rs b/tests/data.rs index 8d29033..5c262b5 100644 --- a/tests/data.rs +++ b/tests/data.rs @@ -184,6 +184,16 @@ fn clean_data_removes_known_file_artifacts_and_preserves_other_entries() { let matching_directory = sibling_path(&data, ".download.keep"); let lock_path = sibling_path(&data, ".lock"); let unrelated = sibling_path(&data, ".unrelated"); + let lookalikes = [ + sibling_path(&data, ".candidate.notes"), + sibling_path(&data, ".candidate.123"), + sibling_path(&data, ".candidate.123.x"), + sibling_path(&data, ".candidate.123.1.notes"), + sibling_path(&data, ".candidate.123.1-journal.notes"), + sibling_path(&data, ".download.123.0"), + sibling_path(&data, ".download.123.x.gz"), + sibling_path(&data, ".download.123.0.gz.notes"), + ]; std::fs::write(&data, b"live").unwrap(); for artifact in [&legacy, &download, &candidate, &candidate_journal] { std::fs::write(artifact, b"temporary").unwrap(); @@ -191,6 +201,9 @@ fn clean_data_removes_known_file_artifacts_and_preserves_other_entries() { std::fs::create_dir(&matching_directory).unwrap(); std::fs::write(&lock_path, b"permanent lock inode").unwrap(); std::fs::write(&unrelated, b"unrelated").unwrap(); + for lookalike in &lookalikes { + std::fs::write(lookalike, b"not owned").unwrap(); + } assert_eq!(clean_data(&data).unwrap(), Some(4)); for removed in [&data, &legacy, &download, &candidate, &candidate_journal] { @@ -199,11 +212,20 @@ fn clean_data_removes_known_file_artifacts_and_preserves_other_entries() { assert!(matching_directory.is_dir()); assert_eq!(std::fs::read(&lock_path).unwrap(), b"permanent lock inode"); assert_eq!(std::fs::read(&unrelated).unwrap(), b"unrelated"); + for lookalike in &lookalikes { + assert_eq!( + std::fs::read(lookalike).unwrap(), + b"not owned", + "clean removed lookalike: {}", + lookalike.display() + ); + } assert_eq!(clean_data(&data).unwrap(), None); assert!(lock_path.is_file()); assert!(matching_directory.is_dir()); assert!(unrelated.is_file()); + assert!(lookalikes.iter().all(|path| path.is_file())); } #[test] @@ -309,41 +331,140 @@ fn concurrent_first_install_downloads_once() { .unwrap(); return; } - use std::process::Command; + use std::fs::OpenOptions; + use std::process::{Child, Command, Stdio}; use std::time::{Duration, Instant}; + fn terminate_children(children: [&mut Child; 2]) { + for child in children { + let _ = child.kill(); + let _ = child.wait(); + } + } + + fn wait_for_children(mut children: [&mut Child; 2], timeout: Duration) { + let deadline = Instant::now() + timeout; + let mut finished = [false; 2]; + while !finished.iter().all(|done| *done) { + let mut failed = None; + for (index, child) in children.iter_mut().enumerate() { + if finished[index] { + continue; + } + if let Some(status) = child.try_wait().unwrap() { + finished[index] = true; + if !status.success() { + failed = Some((index, status)); + break; + } + } + } + if let Some((failed_index, status)) = failed { + for (index, child) in children.iter_mut().enumerate() { + if !finished[index] { + let _ = child.kill(); + let _ = child.wait(); + } + } + panic!("worker {failed_index} failed with {status}"); + } + if Instant::now() >= deadline { + for (index, child) in children.iter_mut().enumerate() { + if !finished[index] { + let _ = child.kill(); + let _ = child.wait(); + } + } + panic!("concurrent install workers exceeded {timeout:?}"); + } + std::thread::sleep(Duration::from_millis(10)); + } + } + let directory = tempfile::tempdir().unwrap(); let data_path = directory.path().join("data.sqlite"); + let lock_path = sibling_path(&data_path, ".lock"); + let parent_lock = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(&lock_path) + .unwrap(); + parent_lock.lock().unwrap(); let body = gzip_bytes(&replacement_database_bytes()); let sha = sha256_hex(&body); let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); let url = format!("http://{}/data.gz", listener.local_addr().unwrap()); let server_body = body.clone(); + let (request_started_tx, request_started_rx) = std::sync::mpsc::channel(); + let (early_request_count_tx, early_request_count_rx) = std::sync::mpsc::channel(); + let (release_response_tx, release_response_rx) = std::sync::mpsc::channel(); let server = std::thread::spawn(move || { let mut requests = 0_usize; - let (mut first, _) = listener.accept().unwrap(); + let accept_deadline = Instant::now() + Duration::from_secs(5); + let mut first = loop { + match listener.accept() { + Ok((stream, _)) => { + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + break stream; + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + assert!( + Instant::now() < accept_deadline, + "no worker began the first download" + ); + std::thread::sleep(Duration::from_millis(10)); + } + Err(error) => panic!("accept failed: {error}"), + } + }; requests += 1; let mut request = [0_u8; 4096]; let _ = first.read(&mut request); + request_started_tx.send(()).unwrap(); + + let early_deadline = Instant::now() + Duration::from_millis(300); + while Instant::now() < early_deadline { + match listener.accept() { + Ok((mut stream, _)) => { + requests += 1; + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let _ = stream.read(&mut request); + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + std::thread::sleep(Duration::from_millis(10)); + } + Err(error) => panic!("accept failed: {error}"), + } + } + early_request_count_tx.send(requests).unwrap(); + release_response_rx + .recv_timeout(Duration::from_secs(5)) + .expect("parent did not release the first response"); + write!( first, "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", server_body.len() ) .unwrap(); - let midpoint = server_body.len() / 2; - first.write_all(&server_body[..midpoint]).unwrap(); - first.flush().unwrap(); - std::thread::sleep(Duration::from_millis(200)); - first.write_all(&server_body[midpoint..]).unwrap(); + first.write_all(&server_body).unwrap(); drop(first); - listener.set_nonblocking(true).unwrap(); - let deadline = Instant::now() + Duration::from_secs(2); + let deadline = Instant::now() + Duration::from_secs(1); while Instant::now() < deadline { match listener.accept() { Ok((mut stream, _)) => { requests += 1; + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); let _ = stream.read(&mut request); write!( stream, @@ -362,7 +483,9 @@ fn concurrent_first_install_downloads_once() { requests }); - let spawn_worker = || { + let first_log = directory.path().join("worker-1.stderr"); + let second_log = directory.path().join("worker-2.stderr"); + let spawn_worker = |stderr_path: &std::path::Path| { Command::new(std::env::current_exe().unwrap()) .arg("--exact") .arg("concurrent_first_install_downloads_once") @@ -371,16 +494,74 @@ fn concurrent_first_install_downloads_once() { .env("FOJIN_WORKER_DATA", &data_path) .env("FOJIN_WORKER_URL", &url) .env("FOJIN_WORKER_SHA256", &sha) + .stdout(Stdio::null()) + .stderr(Stdio::from(std::fs::File::create(stderr_path).unwrap())) .spawn() .unwrap() }; - let mut first = spawn_worker(); - let mut second = spawn_worker(); - assert!(first.wait().unwrap().success()); - assert!(second.wait().unwrap().success()); + let mut first = spawn_worker(&first_log); + let mut second = spawn_worker(&second_log); + + let barrier_deadline = Instant::now() + Duration::from_secs(5); + loop { + let first_stderr = std::fs::read_to_string(&first_log).unwrap_or_default(); + let second_stderr = std::fs::read_to_string(&second_log).unwrap_or_default(); + let marker = "检测到另一个 fojin 数据操作,正在等待"; + if first_stderr.contains(marker) && second_stderr.contains(marker) { + break; + } + if Instant::now() >= barrier_deadline { + let _ = first.kill(); + let _ = second.kill(); + let _ = first.wait(); + let _ = second.wait(); + panic!( + "workers did not reach the lock barrier; first={first_stderr:?}, second={second_stderr:?}" + ); + } + std::thread::sleep(Duration::from_millis(10)); + } + + drop(parent_lock); + if let Err(error) = request_started_rx.recv_timeout(Duration::from_secs(5)) { + let _ = release_response_tx.send(()); + terminate_children([&mut first, &mut second]); + let _ = server.join(); + panic!("neither worker began downloading after the parent released the lock: {error}"); + } + let early_requests = match early_request_count_rx.recv_timeout(Duration::from_secs(2)) { + Ok(requests) => requests, + Err(error) => { + let _ = release_response_tx.send(()); + terminate_children([&mut first, &mut second]); + let _ = server.join(); + panic!("server did not report the pre-release request count: {error}"); + } + }; + if early_requests != 1 { + let _ = release_response_tx.send(()); + terminate_children([&mut first, &mut second]); + let _ = server.join(); + panic!("both workers downloaded while the first response was blocked"); + } + let first_early = first.try_wait().unwrap(); + let second_early = second.try_wait().unwrap(); + if first_early.is_some() || second_early.is_some() { + let _ = release_response_tx.send(()); + terminate_children([&mut first, &mut second]); + let _ = server.join(); + panic!("a worker exited before the first download was released"); + } + if release_response_tx.send(()).is_err() { + terminate_children([&mut first, &mut second]); + let _ = server.join(); + panic!("server exited before the first response was released"); + } + + wait_for_children([&mut first, &mut second], Duration::from_secs(10)); assert_eq!(server.join().unwrap(), 1); verify_dataset_file(&data_path).unwrap(); - assert_no_owned_candidate_artifacts(&data_path); + assert_no_owned_transfer_artifacts(&data_path); } #[test] @@ -414,7 +595,7 @@ fn first_install_rejects_incompatible_database() { assert!(error.contains("dataset incompatibility"), "got: {error}"); assert!(!path.exists(), "incompatible dataset was published"); - assert_no_owned_candidate_artifacts(&path); + assert_no_owned_transfer_artifacts(&path); } #[test] @@ -1102,24 +1283,53 @@ fn assert_no_candidate_artifacts(path: &std::path::Path) { artifact.display() ); } - assert_no_owned_candidate_artifacts(path); + assert_no_owned_transfer_artifacts(path); } -fn assert_no_owned_candidate_artifacts(path: &std::path::Path) { - let mut prefix = path.file_name().unwrap().to_os_string(); - prefix.push(".candidate."); - let prefix = prefix.to_string_lossy(); +fn assert_no_owned_transfer_artifacts(path: &std::path::Path) { + let live_name = path.file_name().unwrap().to_string_lossy(); let owned: Vec<_> = std::fs::read_dir(path.parent().unwrap()) .unwrap() .map(|entry| entry.unwrap().file_name()) - .filter(|name| name.to_string_lossy().starts_with(prefix.as_ref())) + .filter(|name| { + let name = name.to_string_lossy(); + let Some(suffix) = name.strip_prefix(live_name.as_ref()) else { + return false; + }; + if let Some(generation) = suffix + .strip_prefix(".download.") + .and_then(|rest| rest.strip_suffix(".gz")) + { + return is_numeric_generation(generation); + } + let Some(candidate) = suffix.strip_prefix(".candidate.") else { + return false; + }; + let generation = ["-journal", "-shm", "-wal"] + .into_iter() + .find_map(|sidecar| candidate.strip_suffix(sidecar)) + .unwrap_or(candidate); + is_numeric_generation(generation) + }) .collect(); assert!( owned.is_empty(), - "owned candidate artifacts remain: {owned:?}" + "owned transfer artifacts remain: {owned:?}" ); } +fn is_numeric_generation(value: &str) -> bool { + let mut parts = value.split('.'); + matches!( + (parts.next(), parts.next(), parts.next()), + (Some(pid), Some(sequence), None) + if !pid.is_empty() + && !sequence.is_empty() + && pid.bytes().all(|byte| byte.is_ascii_digit()) + && sequence.bytes().all(|byte| byte.is_ascii_digit()) + ) +} + fn sibling_path(path: &std::path::Path, suffix: &str) -> PathBuf { let mut name = path.file_name().unwrap().to_os_string(); name.push(suffix); From 6d5f1f42b80ababfb28112cad39033d93369962a Mon Sep 17 00:00:00 2001 From: xr843 <137012659+xr843@users.noreply.github.com> Date: Sun, 12 Jul 2026 00:14:31 +0800 Subject: [PATCH 09/14] fix(data): preserve Windows replacement backups --- src/data.rs | 490 ++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 377 insertions(+), 113 deletions(-) diff --git a/src/data.rs b/src/data.rs index 9612361..a72416c 100644 --- a/src/data.rs +++ b/src/data.rs @@ -6,6 +6,10 @@ use std::path::{Path, PathBuf}; mod operation_lock; mod transfer; +#[cfg(windows)] +static REPLACEMENT_BACKUP_SEQUENCE: std::sync::atomic::AtomicU64 = + std::sync::atomic::AtomicU64::new(0); + pub const EXPECTED_DATA_VERSION: &str = "v1"; pub const EXPECTED_NORM_RULESET: &str = "t2s-char-1to1-v1"; @@ -290,85 +294,270 @@ fn replace_with_candidate(path: &Path, candidate: &Path) -> ReplacementResult { Ok(encoded) } + let mut backup = reserve_replacement_backup(path).map_err(ReplacementFailure::remove)?; let replaced = wide_path(path).map_err(ReplacementFailure::remove)?; let replacement = wide_path(candidate).map_err(ReplacementFailure::remove)?; - // SAFETY: both path buffers are NUL-terminated UTF-16 and remain alive for - // the call; optional and reserved pointers are null as ReplaceFileW requires. + let backup_name = wide_path(backup.path()).map_err(ReplacementFailure::remove)?; + backup + .prepare_for_replace() + .map_err(ReplacementFailure::remove)?; + // SAFETY: all path buffers are NUL-terminated UTF-16 and remain alive for + // the call; the reserved pointers are null as ReplaceFileW requires. let replaced_ok = unsafe { ReplaceFileW( replaced.as_ptr(), replacement.as_ptr(), - ptr::null(), + backup_name.as_ptr(), 0, ptr::null_mut(), ptr::null_mut(), ) }; if replaced_ok != 0 { + cleanup_windows_backup_after_success(backup.path()); return Ok(()); } let native_error = std::io::Error::last_os_error(); - handle_windows_replace_failure(path, candidate, native_error, |from, to| { - std::fs::rename(from, to) - }) + handle_windows_replace_failure( + path, + candidate, + backup.path(), + native_error, + |from, to| std::fs::rename(from, to), + std::thread::sleep, + ) +} + +#[cfg(any(test, windows))] +#[derive(Debug)] +struct ReplacementBackupReservation { + path: PathBuf, + file: Option, + armed: bool, +} + +#[cfg(any(test, windows))] +impl ReplacementBackupReservation { + #[cfg(windows)] + fn path(&self) -> &Path { + &self.path + } + + #[cfg(windows)] + fn prepare_for_replace(&mut self) -> Result<()> { + drop(self.file.take()); + std::fs::remove_file(&self.path) + .with_context(|| format!("释放替换备份路径失败: {}", self.path.display()))?; + self.armed = false; + Ok(()) + } +} + +#[cfg(any(test, windows))] +impl Drop for ReplacementBackupReservation { + fn drop(&mut self) { + drop(self.file.take()); + if self.armed { + let _ = std::fs::remove_file(&self.path); + } + } } #[cfg(windows)] -fn handle_windows_replace_failure( +fn reserve_replacement_backup(live_path: &Path) -> Result { + use std::sync::atomic::Ordering; + + loop { + let sequence = REPLACEMENT_BACKUP_SEQUENCE.fetch_add(1, Ordering::Relaxed); + let path = sibling_path( + live_path, + &format!(".replace-backup.{}.{sequence}", std::process::id()), + )?; + if let Some(reservation) = try_reserve_replacement_backup(&path)? { + return Ok(reservation); + } + } +} + +#[cfg(any(test, windows))] +fn try_reserve_replacement_backup(path: &Path) -> Result> { + match std::fs::OpenOptions::new() + .read(true) + .write(true) + .create_new(true) + .open(path) + { + Ok(file) => Ok(Some(ReplacementBackupReservation { + path: path.to_path_buf(), + file: Some(file), + armed: true, + })), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => Ok(None), + Err(error) => { + Err(error).with_context(|| format!("保留替换备份路径失败: {}", path.display())) + } + } +} + +#[cfg(any(test, windows))] +fn cleanup_windows_backup_after_success(backup: &Path) { + match std::fs::remove_file(backup) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => eprintln!( + "警告: 新数据已发布,但无法删除旧数据备份 `{}`: {error}", + backup.display() + ), + } +} + +#[cfg(any(test, windows))] +fn handle_windows_replace_failure( path: &Path, candidate: &Path, + backup: &Path, native_error: std::io::Error, - recover: F, + mut recover: F, + mut sleep: S, ) -> ReplacementResult where - F: FnOnce(&Path, &Path) -> std::io::Result<()>, + F: FnMut(&Path, &Path) -> std::io::Result<()>, + S: FnMut(std::time::Duration), { - const ERROR_UNABLE_TO_REMOVE_REPLACED: i32 = 1175; - const ERROR_UNABLE_TO_MOVE_REPLACEMENT: i32 = 1176; - const ERROR_UNABLE_TO_MOVE_REPLACEMENT_2: i32 = 1177; - - match native_error.raw_os_error() { - Some(ERROR_UNABLE_TO_MOVE_REPLACEMENT | ERROR_UNABLE_TO_MOVE_REPLACEMENT_2) => { - match recover(candidate, path) { - Ok(()) => Ok(()), - Err(recovery_error) => Err(ReplacementFailure::preserve(anyhow!( - "ReplaceFileW failed for replaced `{}` and replacement `{}` with {}; recovery rename `{} -> {}` failed with {}", - path.display(), - candidate.display(), - describe_windows_error(&native_error), - candidate.display(), - path.display(), - describe_windows_error(&recovery_error) - ))), - } + let original = windows_replace_error(path, candidate, backup, &native_error); + let live_exists = match replacement_entry_exists(path) { + Ok(exists) => exists, + Err(error) => { + return Err(ReplacementFailure::preserve(original.context(format!( + "could not inspect live path `{}` after replacement failure: {error}", + path.display() + )))); + } + }; + let backup_exists = match replacement_entry_exists(backup) { + Ok(exists) => exists, + Err(error) => { + return Err(ReplacementFailure::preserve(original.context(format!( + "could not inspect replacement backup `{}`: {error}", + backup.display() + )))); + } + }; + + if live_exists { + let error = if backup_exists { + original.context(format!( + "live path still exists; replacement backup preserved at `{}`", + backup.display() + )) + } else { + original + }; + return Err(ReplacementFailure::remove(error)); + } + + if backup_exists { + return match recover_with_retries(backup, path, &mut recover, &mut sleep) { + Ok(()) => Err(ReplacementFailure::remove(original.context(format!( + "old live data restored from replacement backup `{}`", + backup.display() + )))), + Err(recovery_error) => Err(ReplacementFailure::preserve(anyhow!( + "{original:#}; rollback `{}` -> `{}` failed with {}; validated candidate remains at `{}` and old live remains at `{}`. manual recovery required: restore the backup to the live path before retrying", + backup.display(), + path.display(), + describe_windows_error(&recovery_error), + candidate.display(), + backup.display() + ))), + }; + } + + let candidate_exists = match replacement_entry_exists(candidate) { + Ok(exists) => exists, + Err(error) => { + return Err(ReplacementFailure::preserve(original.context(format!( + "could not inspect validated candidate `{}`: {error}", + candidate.display() + )))); + } + }; + if candidate_exists { + return match recover_with_retries(candidate, path, &mut recover, &mut sleep) { + Ok(()) => Err(ReplacementFailure::remove(original.context(format!( + "replacement backup was missing; validated candidate restored to live path `{}`", + path.display() + )))), + Err(recovery_error) => Err(ReplacementFailure::preserve(anyhow!( + "{original:#}; both live `{}` and backup `{}` are missing; recovery rename `{}` -> `{}` failed with {}. manual recovery required", + path.display(), + backup.display(), + candidate.display(), + path.display(), + describe_windows_error(&recovery_error) + ))), + }; + } + + Err(ReplacementFailure::preserve(original.context(format!( + "live `{}`, backup `{}`, and candidate `{}` are all missing; manual recovery required", + path.display(), + backup.display(), + candidate.display() + )))) +} + +#[cfg(any(test, windows))] +fn recover_with_retries( + from: &Path, + to: &Path, + recover: &mut F, + sleep: &mut S, +) -> std::io::Result<()> +where + F: FnMut(&Path, &Path) -> std::io::Result<()>, + S: FnMut(std::time::Duration), +{ + let mut last_error = None; + for attempt in 0..3 { + match recover(from, to) { + Ok(()) => return Ok(()), + Err(error) => last_error = Some(error), + } + if attempt < 2 { + sleep(std::time::Duration::from_millis(50)); } - Some(ERROR_UNABLE_TO_REMOVE_REPLACED) => Err(ReplacementFailure::remove( - windows_replace_error(path, candidate, &native_error), - )), - _ => Err(ReplacementFailure::remove(windows_replace_error( - path, - candidate, - &native_error, - ))), } + Err(last_error.expect("recovery attempted at least once")) } -#[cfg(windows)] +#[cfg(any(test, windows))] +fn replacement_entry_exists(path: &Path) -> std::io::Result { + match std::fs::symlink_metadata(path) { + Ok(_) => Ok(true), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error), + } +} + +#[cfg(any(test, windows))] fn windows_replace_error( path: &Path, candidate: &Path, + backup: &Path, native_error: &std::io::Error, ) -> anyhow::Error { anyhow!( - "ReplaceFileW failed for replaced `{}` and replacement `{}` with {}", + "ReplaceFileW failed for replaced `{}`, replacement `{}`, and backup `{}` with {}", path.display(), candidate.display(), + backup.display(), describe_windows_error(native_error) ) } -#[cfg(windows)] +#[cfg(any(test, windows))] fn describe_windows_error(error: &std::io::Error) -> String { match error.raw_os_error() { Some(code) => format!("Windows error {code}: {error}"), @@ -634,102 +823,177 @@ mod replacement_cleanup_tests { } } -#[cfg(all(test, windows))] -mod windows_replace_failure_tests { +#[cfg(test)] +mod windows_replace_state_tests { use super::*; - use std::cell::Cell; use std::io; - const LIVE: &str = r"C:\cache\data.sqlite"; - const CANDIDATE: &str = r"C:\cache\data.sqlite.candidate.1.1"; - #[test] - fn error_1175_returns_native_failure_without_recovery() { - assert_names_untouched_error_skips_recovery(1175); + fn errors_with_an_intact_live_path_preserve_old_content() { + for code in [1175, 1176, 87] { + let directory = tempfile::tempdir().unwrap(); + let live = directory.path().join("data.sqlite"); + let candidate = directory.path().join("data.sqlite.candidate.1.1"); + let backup = directory.path().join("data.sqlite.replace-backup.1.1"); + std::fs::write(&live, format!("old-{code}")).unwrap(); + std::fs::write(&candidate, b"validated replacement").unwrap(); + + let staged = transfer::StagedCandidate::for_test(candidate.clone()); + let result = handle_windows_replace_failure( + &live, + &candidate, + &backup, + io::Error::from_raw_os_error(code), + |from, to| std::fs::rename(from, to), + |_| {}, + ); + let error = finish_replacement(staged, result).unwrap_err(); + + assert_eq!( + std::fs::read(&live).unwrap(), + format!("old-{code}").as_bytes() + ); + assert!(!candidate.exists()); + assert!(!backup.exists()); + assert!(format!("{error:#}").contains(&format!("Windows error {code}"))); + } } #[test] - fn other_error_returns_native_failure_without_recovery() { - assert_names_untouched_error_skips_recovery(87); - } + fn error_1177_restores_old_live_from_backup_and_rejects_update() { + let directory = tempfile::tempdir().unwrap(); + let live = directory.path().join("data.sqlite"); + let candidate = directory.path().join("data.sqlite.candidate.1.1"); + let backup = directory.path().join("data.sqlite.replace-backup.1.1"); + std::fs::write(&candidate, b"validated replacement").unwrap(); + std::fs::write(&backup, b"old live").unwrap(); - #[test] - fn error_1176_recovers_missing_live_path() { - assert_partial_failure_recovers(1176); - } + let staged = transfer::StagedCandidate::for_test(candidate.clone()); + let result = handle_windows_replace_failure( + &live, + &candidate, + &backup, + io::Error::from_raw_os_error(1177), + |from, to| std::fs::rename(from, to), + |_| {}, + ); + let error = finish_replacement(staged, result).unwrap_err(); - #[test] - fn error_1177_recovers_missing_live_path() { - assert_partial_failure_recovers(1177); + assert_eq!(std::fs::read(&live).unwrap(), b"old live"); + assert!(!backup.exists()); + assert!(!candidate.exists()); + let detail = format!("{error:#}"); + assert!(detail.contains("Windows error 1177"), "{detail}"); + assert!(detail.contains("restored"), "{detail}"); } #[test] - fn error_1176_preserves_native_and_recovery_failures() { - assert_partial_failure_preserves_both_errors(1176); + fn rollback_failure_preserves_backup_and_candidate_for_manual_recovery() { + let directory = tempfile::tempdir().unwrap(); + let live = directory.path().join("data.sqlite"); + let candidate = directory.path().join("data.sqlite.candidate.1.1"); + let backup = directory.path().join("data.sqlite.replace-backup.1.1"); + std::fs::write(&candidate, b"validated replacement").unwrap(); + std::fs::write(&backup, b"old live").unwrap(); + + let staged = transfer::StagedCandidate::for_test(candidate.clone()); + let result = handle_windows_replace_failure( + &live, + &candidate, + &backup, + io::Error::from_raw_os_error(1177), + |_, _| Err(io::Error::from_raw_os_error(5)), + |_| {}, + ); + let error = finish_replacement(staged, result).unwrap_err(); + + assert!(!live.exists()); + assert_eq!(std::fs::read(&backup).unwrap(), b"old live"); + assert_eq!(std::fs::read(&candidate).unwrap(), b"validated replacement"); + let detail = format!("{error:#}"); + for expected in [ + "Windows error 1177", + "Windows error 5", + "manual recovery", + live.to_string_lossy().as_ref(), + backup.to_string_lossy().as_ref(), + candidate.to_string_lossy().as_ref(), + ] { + assert!(detail.contains(expected), "missing {expected:?}: {detail}"); + } } #[test] - fn error_1177_preserves_native_and_recovery_failures() { - assert_partial_failure_preserves_both_errors(1177); - } - - fn assert_names_untouched_error_skips_recovery(code: i32) { - let recovery_called = Cell::new(false); - let failure = handle_windows_replace_failure( - Path::new(LIVE), - Path::new(CANDIDATE), - io::Error::from_raw_os_error(code), - |_, _| { - recovery_called.set(true); - Ok(()) - }, - ) - .unwrap_err(); + fn successful_replace_removes_backup_without_touching_new_live() { + let directory = tempfile::tempdir().unwrap(); + let live = directory.path().join("data.sqlite"); + let candidate = directory.path().join("data.sqlite.candidate.1.1"); + let backup = directory.path().join("data.sqlite.replace-backup.1.1"); + std::fs::write(&live, b"new live").unwrap(); + std::fs::write(&backup, b"old live").unwrap(); - assert!(!recovery_called.get()); - assert_eq!(failure.cleanup, CandidateCleanupPolicy::Remove); - let detail = format!("{:#}", failure.error); - assert!( - detail.contains(&format!("Windows error {code}")), - "{detail}" - ); + cleanup_windows_backup_after_success(&backup); + + assert_eq!(std::fs::read(&live).unwrap(), b"new live"); + assert!(!candidate.exists()); + assert!(!backup.exists()); } - fn assert_partial_failure_recovers(code: i32) { - let recovery_called = Cell::new(false); - handle_windows_replace_failure( - Path::new(LIVE), - Path::new(CANDIDATE), - io::Error::from_raw_os_error(code), - |from, to| { - recovery_called.set(true); - assert_eq!(from, Path::new(CANDIDATE)); - assert_eq!(to, Path::new(LIVE)); - Ok(()) - }, - ) - .unwrap(); + #[test] + fn successful_replace_keeps_a_backup_that_cannot_be_cleaned() { + let directory = tempfile::tempdir().unwrap(); + let live = directory.path().join("data.sqlite"); + let backup = directory.path().join("data.sqlite.replace-backup.1.1"); + std::fs::write(&live, b"new live").unwrap(); + std::fs::create_dir(&backup).unwrap(); + + cleanup_windows_backup_after_success(&backup); - assert!(recovery_called.get()); + assert_eq!(std::fs::read(&live).unwrap(), b"new live"); + assert!(backup.is_dir()); } - fn assert_partial_failure_preserves_both_errors(code: i32) { - let failure = handle_windows_replace_failure( - Path::new(LIVE), - Path::new(CANDIDATE), - io::Error::from_raw_os_error(code), - |_, _| Err(io::Error::from_raw_os_error(5)), - ) - .unwrap_err(); + #[test] + fn backup_reservation_skips_a_collision_without_overwriting_it() { + let directory = tempfile::tempdir().unwrap(); + let collision = directory.path().join("data.sqlite.replace-backup.1.1"); + let available = directory.path().join("data.sqlite.replace-backup.1.2"); + std::fs::write(&collision, b"foreign backup").unwrap(); + + assert!(try_reserve_replacement_backup(&collision) + .unwrap() + .is_none()); + let reservation = try_reserve_replacement_backup(&available) + .unwrap() + .expect("next unique backup name should be reservable"); + + assert_eq!(std::fs::read(&collision).unwrap(), b"foreign backup"); + assert!(available.is_file()); + drop(reservation); + assert!(!available.exists()); + } + + #[cfg(windows)] + #[test] + fn native_replace_file_success_preserves_new_live_and_removes_backup() { + let directory = tempfile::tempdir().unwrap(); + let live = directory.path().join("data.sqlite"); + let candidate = directory.path().join("data.sqlite.candidate.1.1"); + std::fs::write(&live, b"old live").unwrap(); + std::fs::write(&candidate, b"new live").unwrap(); - assert_eq!(failure.cleanup, CandidateCleanupPolicy::Preserve); - let detail = format!("{:#}", failure.error); - assert!( - detail.contains(&format!("Windows error {code}")), - "{detail}" - ); - assert!(detail.contains("Windows error 5"), "{detail}"); - assert!(detail.contains(CANDIDATE), "{detail}"); - assert!(detail.contains(LIVE), "{detail}"); + let staged = transfer::StagedCandidate::for_test(candidate.clone()); + finish_replacement(staged, replace_with_candidate(&live, &candidate)).unwrap(); + + assert_eq!(std::fs::read(&live).unwrap(), b"new live"); + assert!(!candidate.exists()); + let prefix = "data.sqlite.replace-backup."; + assert!(std::fs::read_dir(directory.path()).unwrap().all(|entry| { + !entry + .unwrap() + .file_name() + .to_string_lossy() + .starts_with(prefix) + })); } } From 5913a1b105b8d5a1b81ad53ca0cd2d1a853a2a38 Mon Sep 17 00:00:00 2001 From: xr843 <137012659+xr843@users.noreply.github.com> Date: Sun, 12 Jul 2026 00:14:51 +0800 Subject: [PATCH 10/14] docs: describe hard data download deadlines --- CHANGELOG.md | 2 +- README.md | 4 ++-- docs/superpowers/plans/2026-07-11-streaming-data-download.md | 2 +- .../specs/2026-07-11-streaming-data-download-design.md | 5 +++-- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2a4b85..c198730 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ All notable changes to this project will be documented in this file. Version 0.3.0 is prepared but has not been published. Its stabilization work includes: - Data verification: strengthen `fojin data verify` and dataset compatibility checks. -- Data pipeline: move installs and updates to a bounded, disk-streamed, checksum-first pipeline. +- Data pipeline: move installs and updates to a bounded, disk-streamed, checksum-first pipeline with hard end-to-end HTTP deadlines and rollback-safe Windows replacement backups. - Data concurrency: serialize concurrent install, update, and clean operations per data directory, with full candidate validation before publication. - Query correctness: make short-query matching literal and remove duplicate parallel text within a match group and language. - SQLite safety: upgrade the bundled SQLite and verify its runtime version. diff --git a/README.md b/README.md index e4046fc..281b095 100644 --- a/README.md +++ b/README.md @@ -185,7 +185,7 @@ fojin parallel "<汉文短语>" --json --offline - 当前二进制把官方下载地址、SHA-256 与兼容元数据固定在 `data-v1`;`fojin data update` 只会重新获取这份固定数据,不会自动切换到未来的数据主版本。版本、归一化规则或查询所需 schema 不兼容的数据会被拒绝。 - 首次运行时下载,压缩包约 **183 MB**,解压后约 **561 MB**(SQLite)。下载后完全离线可用。 - 安装和更新采用有界的磁盘流式传输,不再在内存中缓冲完整压缩包或数据库;压缩响应上限为 **256 MiB**,解压后的数据库上限为 **768 MiB**。更新期间可能临时需要现用数据库所占空间,外加约 **744 MiB** 暂存磁盘空间(约 183 MiB 压缩包 + 约 561 MiB 候选数据库)。 -- HTTP 连接超时为 **30 秒**,空闲读取超时为 **60 秒**,总时限为 **15 分钟**。总时限是协作式检查,并非不可突破的硬截止:同步 DNS、响应头处理或单次读取最多可能额外越过一个 60 秒空闲超时窗口。 +- HTTP DNS 解析与连接超时均为 **30 秒**,响应头和响应体的空闲读取超时均为 **60 秒**,并以跨重定向、覆盖 DNS 到响应体读取的 **15 分钟端到端硬时限** 为上限。 - 同一数据目录上的首次安装、更新和清理操作按 single-flight 串行执行;等待者最多等待 **20 分钟**。永久保留的 `data.sqlite.lock` 文件是无害的协调文件,`fojin data clean` 会有意保留它。 - 离线查询行为及固定到 `data-v1` 的校验和契约保持不变。 - 当前不含巴利对齐(上游 MITRA-parallel 尚未覆盖巴利),默认输出不显示巴利行;显式 `--lang pi` 仍可查询(如实答「未找到对齐」)。程序的渲染路径可兼容未来新增语言行,但当前官方下载通道仍固定为 `data-v1`;上游出现新语言不代表当前二进制会自动获得它。**渲染兼容不等于官方更新通道无需升级**,未来数据版本可能要求升级二进制或明确切换数据发布。 @@ -224,7 +224,7 @@ fojin data verify # verify version, SQLite, and FTS integrity - **For AI agents**: pure-JSON stdout, semantic exit codes (`0` ok / `1` runtime / `2` usage), zero network with `--offline`. Ready-made Claude Code integration in [`examples/claude/`](examples/claude/). - **Data**: 908,620 zh↔sa/bo alignments from Dharmamitra's [MITRA-parallel](https://github.com/dharmamitra/mitra-parallel) dataset, redistributed under CC BY-SA 4.0. The official URL, checksum, and compatibility contract remain pinned to `data-v1`; rendering support for future language rows does not mean the official update channel can adopt them without a binary upgrade. Academic use: please cite [Nehrdich & Keutzer (2026)](https://arxiv.org/pdf/2601.06400) — BibTeX in [`DATA_LICENSE`](DATA_LICENSE). - **Data transfer resources**: installs and updates use bounded, disk-streamed transfers and no longer buffer the complete archive or database in memory. Compressed responses are capped at **256 MiB** and decompressed databases at **768 MiB**. An update can temporarily require the live database plus roughly **744 MiB** of staging disk (about 183 MiB for the archive and 561 MiB for the candidate database). -- **Data timeouts**: HTTP connect and idle-read timeouts are **30 seconds** and **60 seconds**, with a **15-minute** cooperative total deadline. The total deadline is not an unbreakable hard cutoff: synchronous DNS, response-header processing, or one read can overrun it by at most one additional 60-second idle-timeout window. +- **Data timeouts**: HTTP DNS resolution and connection timeouts are both **30 seconds**, response-header and response-body idle-read timeouts are both **60 seconds**, and a **15-minute hard end-to-end deadline** spans redirects from DNS through the final body read. - **Concurrent data operations**: initial install, update, and clean operations on one data directory are single-flight; a waiter may wait up to **20 minutes**. The permanent `data.sqlite.lock` file is harmless coordination state and intentionally survives `fojin data clean`. - **Stable query contract**: offline queries and the checksum contract pinned to `data-v1` are unchanged. - **Not in scope**: semantic search, Pāli, translation — use [Dharmamitra](https://dharmamitra.org)'s online APIs for those; the two are complementary. diff --git a/docs/superpowers/plans/2026-07-11-streaming-data-download.md b/docs/superpowers/plans/2026-07-11-streaming-data-download.md index c51096b..e17ad7b 100644 --- a/docs/superpowers/plans/2026-07-11-streaming-data-download.md +++ b/docs/superpowers/plans/2026-07-11-streaming-data-download.md @@ -6,7 +6,7 @@ **Architecture:** A private `data::transfer` module streams the authenticated gzip asset to a unique sibling and then streams a `MultiGzDecoder` into a unique SQLite candidate. A private `data::operation_lock` module provides a persistent, time-bounded OS file lock used by ensure, update, and clean; `data.rs` retains SQLite validation and platform-specific atomic publication. -**Tech Stack:** Rust 2021 on Rust 1.95, ureq 2.12.1, sha2 0.10, flate2 1.1.9, rusqlite 0.40.1, standard-library `File::try_lock`, local `TcpListener` fixtures, GitHub Actions. +**Tech Stack:** Rust 2021 on Rust 1.95, ureq 3.3.0, sha2 0.10, flate2 1.1.9, rusqlite 0.40.1, standard-library `File::try_lock`, local `TcpListener` fixtures, GitHub Actions. ## Global Constraints diff --git a/docs/superpowers/specs/2026-07-11-streaming-data-download-design.md b/docs/superpowers/specs/2026-07-11-streaming-data-download-design.md index f283291..00e72b8 100644 --- a/docs/superpowers/specs/2026-07-11-streaming-data-download-design.md +++ b/docs/superpowers/specs/2026-07-11-streaming-data-download-design.md @@ -187,8 +187,9 @@ Every failure before successful publish satisfies both invariants: Ordinary failures remove both owned artifacts and SQLite sidecars. If cleanup also fails, the cleanup error is attached as context without hiding the primary -failure. The intentionally preserved Windows recovery case is the only -exception, and it explicitly reports the validated candidate path. +failure. A Windows replacement rollback failure is the only exception: it +preserves and explicitly reports both the old-live backup and validated +candidate so neither recoverable copy is lost. ## Testing Strategy From 4b0e237eddddf89799ece6a4e5e7599e31a1216f Mon Sep 17 00:00:00 2001 From: xr843 <137012659+xr843@users.noreply.github.com> Date: Sun, 12 Jul 2026 00:56:43 +0800 Subject: [PATCH 11/14] fix(data): enforce absolute transfer deadlines --- .../2026-07-11-streaming-data-download.md | 4 +- ...26-07-11-streaming-data-download-design.md | 13 +- src/data.rs | 3 + src/data/transfer.rs | 309 ++++++++++++++++-- 4 files changed, 300 insertions(+), 29 deletions(-) diff --git a/docs/superpowers/plans/2026-07-11-streaming-data-download.md b/docs/superpowers/plans/2026-07-11-streaming-data-download.md index e17ad7b..fd44435 100644 --- a/docs/superpowers/plans/2026-07-11-streaming-data-download.md +++ b/docs/superpowers/plans/2026-07-11-streaming-data-download.md @@ -16,7 +16,9 @@ - First install and update both run schema/version, quick-check, and FTS integrity validation before publish. - Existing data remains byte-for-byte unchanged on every failure before atomic publish. - Ordinary queries never wait for the long-running mutation lock. -- Temporary files are same-directory, `create_new` siblings owned and cleaned by one operation. +- Temporary files are same-directory, `create_new` siblings reserved and + cleaned by one cooperating operation; this is not an adversarial same-user + filesystem boundary. - Do not add retries, resume support, mirrors, data discovery, schema changes, a tag, or a GitHub Release. - Use TDD, focused commits, independent review, PR checks, and merge to `master`. diff --git a/docs/superpowers/specs/2026-07-11-streaming-data-download-design.md b/docs/superpowers/specs/2026-07-11-streaming-data-download-design.md index 00e72b8..8917663 100644 --- a/docs/superpowers/specs/2026-07-11-streaming-data-download-design.md +++ b/docs/superpowers/specs/2026-07-11-streaming-data-download-design.md @@ -127,9 +127,16 @@ artifact role, process ID, and an atomic sequence, and are opened with - `data.sqlite.download...gz` - `data.sqlite.candidate..` -An ownership guard removes only artifacts created by the current operation. -It is disarmed only after a successful publish or an intentional Windows -preservation result. Recoverable errors never sweep another process's files. +Within the cooperating-process model enforced by the operation lock, an +ownership guard removes the unique artifact family reserved by the current +operation. It is disarmed only after a successful publish or an intentional +Windows preservation result. Pre-existing family collisions are skipped, so +recoverable errors do not sweep another cooperating operation's files. + +The same coordination model applies to SQLite sidecars and Windows replacement +backups. `create_new` reservations and post-reservation checks are not a +security boundary against another same-user process that can concurrently +modify the data directory; filesystem permissions remain that boundary. ## Detailed Data Flow diff --git a/src/data.rs b/src/data.rs index a72416c..7c2bd98 100644 --- a/src/data.rs +++ b/src/data.rs @@ -368,6 +368,9 @@ impl Drop for ReplacementBackupReservation { fn reserve_replacement_backup(live_path: &Path) -> Result { use std::sync::atomic::Ordering; + // create_new avoids stale-name collisions between cooperating fojin + // operations. The reservation is not a security boundary against another + // same-user process that can alter this directory concurrently. loop { let sequence = REPLACEMENT_BACKUP_SEQUENCE.fetch_add(1, Ordering::Relaxed); let path = sibling_path( diff --git a/src/data/transfer.rs b/src/data/transfer.rs index da4eb64..645780b 100644 --- a/src/data/transfer.rs +++ b/src/data/transfer.rs @@ -6,41 +6,52 @@ use std::fs::{File, OpenOptions}; use std::io::{self, Read, Seek, Write}; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; -use std::time::Duration; +use std::time::{Duration, Instant as StdInstant}; const MIB: u64 = 1024 * 1024; const BUFFER_SIZE: usize = 64 * 1024; static ARTIFACT_SEQUENCE: AtomicU64 = AtomicU64::new(0); // Ureq's socket adapter turns an already-expired zero timeout into one second -// because operating systems reject a zero socket timeout. Reject it before the -// adapter instead, including when protocol bytes are already buffered, so the -// configured global deadline remains a hard boundary under byte dribbling. -// The transport API is unversioned, so ureq is pinned exactly in Cargo.toml. -#[derive(Debug)] -struct StrictTimeoutConnector; +// because operating systems reject a zero socket timeout. TLS can also perform +// several lower-level reads for one HTTP-level read while reusing the original +// timeout. Recompute the absolute deadline and the per-read idle timeout at +// both transport layers so neither buffered protocol bytes nor a dribbling TLS +// record can extend the configured boundary. The transport API is unversioned, +// so ureq is pinned exactly in Cargo.toml. +#[derive(Clone, Copy, Debug)] +struct AbsoluteDeadlineConnector { + deadline: StdInstant, + idle_read_timeout: Duration, +} #[derive(Debug)] -struct StrictTimeoutTransport { +struct AbsoluteDeadlineTransport { inner: T, + deadline: StdInstant, + idle_read_timeout: Duration, } impl ureq::unversioned::transport::Connector - for StrictTimeoutConnector + for AbsoluteDeadlineConnector { - type Out = StrictTimeoutTransport; + type Out = AbsoluteDeadlineTransport; fn connect( &self, _: &ureq::unversioned::transport::ConnectionDetails<'_>, chained: Option, ) -> std::result::Result, ureq::Error> { - Ok(chained.map(|inner| StrictTimeoutTransport { inner })) + Ok(chained.map(|inner| AbsoluteDeadlineTransport { + inner, + deadline: self.deadline, + idle_read_timeout: self.idle_read_timeout, + })) } } impl ureq::unversioned::transport::Transport - for StrictTimeoutTransport + for AbsoluteDeadlineTransport { fn buffers(&mut self) -> &mut dyn ureq::unversioned::transport::Buffers { self.inner.buffers() @@ -51,6 +62,7 @@ impl ureq::unversioned::transport::T amount: usize, timeout: ureq::unversioned::transport::NextTimeout, ) -> std::result::Result<(), ureq::Error> { + let timeout = bound_transport_timeout(timeout, self.deadline, None); require_remaining_timeout(timeout)?; self.inner.transmit_output(amount, timeout) } @@ -59,6 +71,7 @@ impl ureq::unversioned::transport::T &mut self, timeout: ureq::unversioned::transport::NextTimeout, ) -> std::result::Result { + let timeout = bound_transport_timeout(timeout, self.deadline, Some(self.idle_read_timeout)); require_remaining_timeout(timeout)?; self.inner.maybe_await_input(timeout) } @@ -67,6 +80,7 @@ impl ureq::unversioned::transport::T &mut self, timeout: ureq::unversioned::transport::NextTimeout, ) -> std::result::Result { + let timeout = bound_transport_timeout(timeout, self.deadline, Some(self.idle_read_timeout)); require_remaining_timeout(timeout)?; self.inner.await_input(timeout) } @@ -80,6 +94,33 @@ impl ureq::unversioned::transport::T } } +fn bound_transport_timeout( + mut timeout: ureq::unversioned::transport::NextTimeout, + deadline: StdInstant, + idle_read_timeout: Option, +) -> ureq::unversioned::transport::NextTimeout { + let remaining = deadline.saturating_duration_since(StdInstant::now()); + let global_after: ureq::unversioned::transport::time::Duration = remaining.into(); + if global_after < timeout.after { + timeout = ureq::unversioned::transport::NextTimeout { + after: global_after, + reason: ureq::Timeout::Global, + }; + } + + if let Some(idle_read_timeout) = idle_read_timeout { + let idle_after: ureq::unversioned::transport::time::Duration = idle_read_timeout.into(); + if idle_after < timeout.after { + timeout = ureq::unversioned::transport::NextTimeout { + after: idle_after, + reason: ureq::Timeout::RecvResponse, + }; + } + } + + timeout +} + fn require_remaining_timeout( timeout: ureq::unversioned::transport::NextTimeout, ) -> std::result::Result<(), ureq::Error> { @@ -90,6 +131,14 @@ fn require_remaining_timeout( } } +fn require_absolute_deadline(deadline: StdInstant) -> std::result::Result<(), ureq::Error> { + if StdInstant::now() >= deadline { + Err(ureq::Error::Timeout(ureq::Timeout::Global)) + } else { + Ok(()) + } +} + #[derive(Clone, Copy, Debug)] pub(super) struct DownloadPolicy { pub connect_timeout: Duration, @@ -254,11 +303,12 @@ where } }; - // The main file reserves this generation. Recheck the sidecars after that - // reservation so a stale family can never be claimed and later deleted by - // this guard. Under the data-operation lock, any sidecar that appears after - // this point can only be produced by SQLite while operating on this unique - // candidate. + // The main file reserves this generation for cooperating fojin processes. + // Recheck the sidecars so a stale family is not claimed and later deleted + // by this guard. After this point, the operation lock ensures cooperating + // processes leave this unique family to SQLite. This naming protocol is + // coordination, not isolation from a same-user process that can modify the + // data directory concurrently. let sidecar_occupied = sidecars.iter().try_fold(false, |occupied, sidecar| { entry_exists(sidecar).map(|exists| occupied || exists) }); @@ -425,19 +475,35 @@ pub(super) fn stage_candidate( let staged = stage_candidate_inner(live_path, source, policy, &mut compressed_file); match staged { Ok(candidate) => { - compressed_guard.remove_now()?; - Ok(candidate) + finish_successful_stage_with(compressed_guard, candidate, OwnedCompressed::remove_now) } Err(error) => Err(compressed_guard.cleanup_with(error)), } } +fn finish_successful_stage_with( + compressed_guard: OwnedCompressed, + candidate: StagedCandidate, + remove_compressed: RemoveCompressed, +) -> Result +where + RemoveCompressed: FnOnce(OwnedCompressed) -> Result<()>, +{ + match remove_compressed(compressed_guard) { + Ok(()) => Ok(candidate), + Err(error) => Err(candidate.cleanup_with(error)), + } +} + fn stage_candidate_inner( live_path: &Path, source: &DataSource<'_>, policy: DownloadPolicy, compressed_file: &mut File, ) -> Result { + let deadline = StdInstant::now() + .checked_add(policy.http_timeout) + .ok_or_else(|| anyhow!("HTTP 总超时超出系统时间范围"))?; let config = ureq::Agent::config_builder() .max_redirects(5) .max_redirects_will_error(true) @@ -446,12 +512,19 @@ fn stage_candidate_inner( .timeout_global(Some(policy.http_timeout)) .timeout_resolve(Some(policy.connect_timeout)) .timeout_connect(Some(policy.connect_timeout)) - .timeout_recv_response(Some(policy.idle_read_timeout)) .timeout_recv_body(Some(policy.idle_read_timeout)) .build(); use ureq::unversioned::transport::Connector; + let deadline_connector = AbsoluteDeadlineConnector { + deadline, + idle_read_timeout: policy.idle_read_timeout, + }; let connector = - ureq::unversioned::transport::DefaultConnector::new().chain(StrictTimeoutConnector); + ().chain(ureq::unversioned::transport::ConnectProxyConnector::default()) + .chain(ureq::unversioned::transport::TcpConnector::default()) + .chain(deadline_connector) + .chain(ureq::unversioned::transport::RustlsConnector::default()) + .chain(deadline_connector); let agent = ureq::Agent::with_parts( config, connector, @@ -476,13 +549,29 @@ fn stage_candidate_inner( let mut received = 0_u64; let mut buffer = [0_u8; BUFFER_SIZE]; loop { - let count = reader - .read(&mut buffer) - .map_err(normalize_body_read_error) - .map_err(anyhow::Error::new) + require_absolute_deadline(deadline) + .map_err(normalize_ureq_error) .map_err(|error| { download_error_context(error, "读取响应失败", live_path, source.url) })?; + let count = match reader.read(&mut buffer) { + Ok(count) => { + require_absolute_deadline(deadline) + .map_err(normalize_ureq_error) + .map_err(|error| { + download_error_context(error, "读取响应失败", live_path, source.url) + })?; + count + } + Err(error) => { + return Err(download_error_context( + anyhow::Error::new(normalize_body_read_error(error)), + "读取响应失败", + live_path, + source.url, + )); + } + }; if count == 0 { break; } @@ -680,6 +769,35 @@ mod tests { assert!(!candidate.exists(), "owned candidate main leaked"); } + #[test] + fn compressed_removal_failure_attaches_candidate_cleanup_error() { + let directory = tempfile::tempdir().unwrap(); + let compressed_path = directory.path().join("data.sqlite.download.123.4.gz"); + std::fs::write(&compressed_path, b"compressed").unwrap(); + let compressed_guard = OwnedCompressed { + path: compressed_path, + armed: true, + }; + let candidate_path = directory.path().join("data.sqlite.candidate.123.4"); + std::fs::create_dir(&candidate_path).unwrap(); + let candidate = StagedCandidate::for_test(candidate_path); + + let result = finish_successful_stage_with(compressed_guard, candidate, |_| { + Err(anyhow!("injected compressed removal failure")) + }); + let error = match result { + Err(error) => error, + Ok(_) => panic!("compressed removal failure was ignored"), + }; + let detail = format!("{error:#}"); + + assert!( + detail.contains("injected compressed removal failure"), + "got: {detail}" + ); + assert!(detail.contains("清理候选数据失败"), "got: {detail}"); + } + fn sha256_hex(bytes: &[u8]) -> String { Sha256::digest(bytes) .iter() @@ -913,6 +1031,38 @@ mod tests { }) } + fn contains_ureq_timeout(error: &anyhow::Error, expected: ureq::Timeout) -> bool { + error.chain().any(|cause| { + let direct = matches!( + cause.downcast_ref::(), + Some(ureq::Error::Timeout(reason)) if *reason == expected + ); + let wrapped = cause + .downcast_ref::() + .and_then(io::Error::get_ref) + .and_then(|source| source.downcast_ref::()) + .is_some_and( + |source| matches!(source, ureq::Error::Timeout(reason) if *reason == expected), + ); + direct || wrapped + }) + } + + fn assert_no_owned_transfer_artifacts(live: &Path) { + let live_name = live.file_name().unwrap().to_string_lossy(); + let owned: Vec<_> = std::fs::read_dir(live.parent().unwrap()) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .filter(|name| { + is_owned_artifact_name(live_name.as_ref(), name.to_string_lossy().as_ref()) + }) + .collect(); + assert!( + owned.is_empty(), + "owned transfer artifacts remain: {owned:?}" + ); + } + fn chunked_response(body: &[u8], extra_headers: &str) -> Vec { let mut response = format!( "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n{extra_headers}Connection: close\r\n\r\n{:x}\r\n", @@ -990,6 +1140,111 @@ mod tests { ); } + #[test] + fn slow_progressing_valid_body_can_outlive_the_idle_window() { + let compressed = gzip(b"slow valid body"); + let sha = sha256_hex(&compressed); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let server_body = compressed.clone(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + stream.set_nodelay(true).unwrap(); + let _ = read_request(&mut stream); + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + server_body.len() + ) + .unwrap(); + for (index, byte) in server_body.iter().enumerate() { + stream.write_all(std::slice::from_ref(byte)).unwrap(); + stream.flush().unwrap(); + if index + 1 < server_body.len() { + std::thread::sleep(Duration::from_millis(20)); + } + } + }); + let directory = tempfile::tempdir().unwrap(); + let live = directory.path().join("data.sqlite"); + let url = format!("http://{address}/data.gz"); + let policy = DownloadPolicy { + connect_timeout: Duration::from_millis(500), + idle_read_timeout: Duration::from_millis(120), + http_timeout: Duration::from_secs(2), + max_compressed: 1024, + max_uncompressed: 4096, + }; + + let candidate = stage_candidate( + &live, + &DataSource { + url: &url, + sha256: &sha, + }, + policy, + ) + .expect("continuously progressing body should not hit the idle timeout"); + server.join().unwrap(); + + assert_eq!(std::fs::read(candidate.path()).unwrap(), b"slow valid body"); + } + + #[test] + fn global_deadline_interrupts_a_dribbling_tls_record() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + stream.set_nodelay(true).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let mut client_hello = [0_u8; 16 * 1024]; + let _ = stream.read(&mut client_hello).unwrap(); + stream.write_all(&[0x16, 0x03, 0x03, 0x40, 0x00]).unwrap(); + stream.flush().unwrap(); + for _ in 0..24 { + if stream.write_all(&[0]).is_err() || stream.flush().is_err() { + break; + } + std::thread::sleep(Duration::from_millis(50)); + } + }); + let directory = tempfile::tempdir().unwrap(); + let live = directory.path().join("data.sqlite"); + let url = format!("https://{address}/data.gz"); + let sha = "0".repeat(64); + let policy = DownloadPolicy { + connect_timeout: Duration::from_secs(1), + idle_read_timeout: Duration::from_secs(1), + http_timeout: Duration::from_millis(250), + max_compressed: 1024, + max_uncompressed: 4096, + }; + + let started = Instant::now(); + let error = stage_candidate( + &live, + &DataSource { + url: &url, + sha256: &sha, + }, + policy, + ) + .map(drop) + .unwrap_err(); + let elapsed = started.elapsed(); + server.join().unwrap(); + + assert!(elapsed < Duration::from_millis(700), "elapsed: {elapsed:?}"); + assert!( + contains_ureq_timeout(&error, ureq::Timeout::Global), + "got: {error:#}" + ); + assert_no_owned_transfer_artifacts(&live); + } + #[test] fn global_timeout_interrupts_continuously_dribbling_response_headers() { let policy = DownloadPolicy { @@ -1207,6 +1462,10 @@ mod tests { contains_io_kind(&error, io::ErrorKind::TimedOut), "got: {error:#}" ); + assert!( + contains_ureq_timeout(&error, ureq::Timeout::RecvResponse), + "got: {error:#}" + ); } #[test] From 106206e9f267fc8fd6876ffbbf2b2471bea75072 Mon Sep 17 00:00:00 2001 From: xr843 <137012659+xr843@users.noreply.github.com> Date: Sun, 12 Jul 2026 01:15:00 +0800 Subject: [PATCH 12/14] fix(data): bound connection phase deadlines --- src/data/transfer.rs | 206 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 178 insertions(+), 28 deletions(-) diff --git a/src/data/transfer.rs b/src/data/transfer.rs index 645780b..15fda1b 100644 --- a/src/data/transfer.rs +++ b/src/data/transfer.rs @@ -15,13 +15,14 @@ static ARTIFACT_SEQUENCE: AtomicU64 = AtomicU64::new(0); // Ureq's socket adapter turns an already-expired zero timeout into one second // because operating systems reject a zero socket timeout. TLS can also perform // several lower-level reads for one HTTP-level read while reusing the original -// timeout. Recompute the absolute deadline and the per-read idle timeout at -// both transport layers so neither buffered protocol bytes nor a dribbling TLS -// record can extend the configured boundary. The transport API is unversioned, -// so ureq is pinned exactly in Cargo.toml. +// timeout. Recompute the global and connection-phase deadlines plus the +// per-read idle timeout at both transport layers so neither buffered protocol +// bytes nor a dribbling TLS record can extend a configured boundary. The +// transport API is unversioned, so ureq is pinned exactly in Cargo.toml. #[derive(Clone, Copy, Debug)] struct AbsoluteDeadlineConnector { deadline: StdInstant, + connect_timeout: Duration, idle_read_timeout: Duration, } @@ -29,6 +30,9 @@ struct AbsoluteDeadlineConnector { struct AbsoluteDeadlineTransport { inner: T, deadline: StdInstant, + connect_timeout: Duration, + connect_phase_deadline: StdInstant, + connect_phase_active: bool, idle_read_timeout: Duration, } @@ -39,17 +43,60 @@ impl ureq::unversioned::transport::C fn connect( &self, - _: &ureq::unversioned::transport::ConnectionDetails<'_>, + details: &ureq::unversioned::transport::ConnectionDetails<'_>, chained: Option, ) -> std::result::Result, ureq::Error> { + // ConnectionDetails::now is captured before the connector chain runs, + // so this includes TCP and CONNECT-proxy work that preceded this + // wrapper instead of restarting the budget after the socket opened. + let connect_phase_started = match details.now { + ureq::unversioned::transport::time::Instant::Exact(started) => started, + _ => { + return Err(ureq::Error::Io(io::Error::new( + io::ErrorKind::InvalidData, + "HTTP transport did not provide an exact connection start time", + ))); + } + }; + let connect_phase_deadline = checked_deadline(connect_phase_started, self.connect_timeout)?; Ok(chained.map(|inner| AbsoluteDeadlineTransport { inner, deadline: self.deadline, + connect_timeout: self.connect_timeout, + connect_phase_deadline, + connect_phase_active: true, idle_read_timeout: self.idle_read_timeout, })) } } +impl AbsoluteDeadlineTransport { + fn bound_timeout( + &mut self, + timeout: ureq::unversioned::transport::NextTimeout, + idle_read_timeout: Option, + ) -> std::result::Result { + let now = StdInstant::now(); + let connect_deadline = if timeout.reason == ureq::Timeout::Connect { + if !self.connect_phase_active { + self.connect_phase_deadline = checked_deadline(now, self.connect_timeout)?; + self.connect_phase_active = true; + } + Some(self.connect_phase_deadline) + } else { + self.connect_phase_active = false; + None + }; + Ok(bound_transport_timeout( + timeout, + self.deadline, + connect_deadline, + idle_read_timeout, + now, + )) + } +} + impl ureq::unversioned::transport::Transport for AbsoluteDeadlineTransport { @@ -62,7 +109,7 @@ impl ureq::unversioned::transport::T amount: usize, timeout: ureq::unversioned::transport::NextTimeout, ) -> std::result::Result<(), ureq::Error> { - let timeout = bound_transport_timeout(timeout, self.deadline, None); + let timeout = self.bound_timeout(timeout, None)?; require_remaining_timeout(timeout)?; self.inner.transmit_output(amount, timeout) } @@ -71,7 +118,7 @@ impl ureq::unversioned::transport::T &mut self, timeout: ureq::unversioned::transport::NextTimeout, ) -> std::result::Result { - let timeout = bound_transport_timeout(timeout, self.deadline, Some(self.idle_read_timeout)); + let timeout = self.bound_timeout(timeout, Some(self.idle_read_timeout))?; require_remaining_timeout(timeout)?; self.inner.maybe_await_input(timeout) } @@ -80,7 +127,7 @@ impl ureq::unversioned::transport::T &mut self, timeout: ureq::unversioned::transport::NextTimeout, ) -> std::result::Result { - let timeout = bound_transport_timeout(timeout, self.deadline, Some(self.idle_read_timeout)); + let timeout = self.bound_timeout(timeout, Some(self.idle_read_timeout))?; require_remaining_timeout(timeout)?; self.inner.await_input(timeout) } @@ -97,9 +144,11 @@ impl ureq::unversioned::transport::T fn bound_transport_timeout( mut timeout: ureq::unversioned::transport::NextTimeout, deadline: StdInstant, + connect_deadline: Option, idle_read_timeout: Option, + now: StdInstant, ) -> ureq::unversioned::transport::NextTimeout { - let remaining = deadline.saturating_duration_since(StdInstant::now()); + let remaining = deadline.saturating_duration_since(now); let global_after: ureq::unversioned::transport::time::Duration = remaining.into(); if global_after < timeout.after { timeout = ureq::unversioned::transport::NextTimeout { @@ -108,6 +157,17 @@ fn bound_transport_timeout( }; } + if let Some(connect_deadline) = connect_deadline { + let remaining = connect_deadline.saturating_duration_since(now); + let connect_after: ureq::unversioned::transport::time::Duration = remaining.into(); + if connect_after < timeout.after { + timeout = ureq::unversioned::transport::NextTimeout { + after: connect_after, + reason: ureq::Timeout::Connect, + }; + } + } + if let Some(idle_read_timeout) = idle_read_timeout { let idle_after: ureq::unversioned::transport::time::Duration = idle_read_timeout.into(); if idle_after < timeout.after { @@ -121,6 +181,18 @@ fn bound_transport_timeout( timeout } +fn checked_deadline( + started: StdInstant, + timeout: Duration, +) -> std::result::Result { + started.checked_add(timeout).ok_or_else(|| { + ureq::Error::Io(io::Error::new( + io::ErrorKind::InvalidInput, + "HTTP timeout exceeds the system clock range", + )) + }) +} + fn require_remaining_timeout( timeout: ureq::unversioned::transport::NextTimeout, ) -> std::result::Result<(), ureq::Error> { @@ -517,6 +589,7 @@ fn stage_candidate_inner( use ureq::unversioned::transport::Connector; let deadline_connector = AbsoluteDeadlineConnector { deadline, + connect_timeout: policy.connect_timeout, idle_read_timeout: policy.idle_read_timeout, }; let connector = @@ -1032,20 +1105,26 @@ mod tests { } fn contains_ureq_timeout(error: &anyhow::Error, expected: ureq::Timeout) -> bool { - error.chain().any(|cause| { - let direct = matches!( - cause.downcast_ref::(), - Some(ureq::Error::Timeout(reason)) if *reason == expected - ); - let wrapped = cause + fn source_contains_timeout( + source: &(dyn std::error::Error + 'static), + expected: ureq::Timeout, + ) -> bool { + if let Some(source) = source.downcast_ref::() { + return match source { + ureq::Error::Timeout(reason) => *reason == expected, + ureq::Error::Io(source) => source_contains_timeout(source, expected), + _ => false, + }; + } + source .downcast_ref::() .and_then(io::Error::get_ref) - .and_then(|source| source.downcast_ref::()) - .is_some_and( - |source| matches!(source, ureq::Error::Timeout(reason) if *reason == expected), - ); - direct || wrapped - }) + .is_some_and(|source| source_contains_timeout(source, expected)) + } + + error + .chain() + .any(|cause| source_contains_timeout(cause, expected)) } fn assert_no_owned_transfer_artifacts(live: &Path) { @@ -1123,8 +1202,8 @@ mod tests { #[test] fn total_timeout_stops_a_non_idle_dribble() { let policy = DownloadPolicy { - connect_timeout: Duration::from_millis(200), - idle_read_timeout: Duration::from_millis(150), + connect_timeout: Duration::from_secs(1), + idle_read_timeout: Duration::from_millis(600), http_timeout: Duration::from_millis(300), max_compressed: 1024, max_uncompressed: 4096, @@ -1138,6 +1217,10 @@ mod tests { contains_io_kind(&error, io::ErrorKind::TimedOut), "got: {error:#}" ); + assert!( + contains_ureq_timeout(&error, ureq::Timeout::Global), + "got: {error:#}" + ); } #[test] @@ -1170,7 +1253,7 @@ mod tests { let url = format!("http://{address}/data.gz"); let policy = DownloadPolicy { connect_timeout: Duration::from_millis(500), - idle_read_timeout: Duration::from_millis(120), + idle_read_timeout: Duration::from_millis(250), http_timeout: Duration::from_secs(2), max_compressed: 1024, max_uncompressed: 4096, @@ -1245,11 +1328,66 @@ mod tests { assert_no_owned_transfer_artifacts(&live); } + #[test] + fn connect_deadline_interrupts_a_dribbling_tls_record() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + stream.set_nodelay(true).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let mut client_hello = [0_u8; 16 * 1024]; + let _ = stream.read(&mut client_hello).unwrap(); + stream.write_all(&[0x16, 0x03, 0x03, 0x40, 0x00]).unwrap(); + stream.flush().unwrap(); + for _ in 0..24 { + if stream.write_all(&[0]).is_err() || stream.flush().is_err() { + break; + } + std::thread::sleep(Duration::from_millis(50)); + } + }); + let directory = tempfile::tempdir().unwrap(); + let live = directory.path().join("data.sqlite"); + let url = format!("https://{address}/data.gz"); + let sha = "0".repeat(64); + let policy = DownloadPolicy { + connect_timeout: Duration::from_millis(250), + idle_read_timeout: Duration::from_secs(1), + http_timeout: Duration::from_secs(2), + max_compressed: 1024, + max_uncompressed: 4096, + }; + + let started = Instant::now(); + let error = stage_candidate( + &live, + &DataSource { + url: &url, + sha256: &sha, + }, + policy, + ) + .map(drop) + .unwrap_err(); + let elapsed = started.elapsed(); + server.join().unwrap(); + + assert!(elapsed < Duration::from_millis(700), "elapsed: {elapsed:?}"); + assert!( + contains_ureq_timeout(&error, ureq::Timeout::Connect), + "got: {error:#}" + ); + assert_no_owned_transfer_artifacts(&live); + } + #[test] fn global_timeout_interrupts_continuously_dribbling_response_headers() { let policy = DownloadPolicy { - connect_timeout: Duration::from_millis(200), - idle_read_timeout: Duration::from_millis(200), + connect_timeout: Duration::from_secs(1), + idle_read_timeout: Duration::from_millis(500), http_timeout: Duration::from_millis(250), max_compressed: 1024, max_uncompressed: 4096, @@ -1268,6 +1406,10 @@ mod tests { contains_io_kind(&error, io::ErrorKind::TimedOut), "got: {error:#}" ); + assert!( + contains_ureq_timeout(&error, ureq::Timeout::Global), + "got: {error:#}" + ); assert!( format!("{error:#}").contains("请手动下载"), "got: {error:#}" @@ -1277,8 +1419,8 @@ mod tests { #[test] fn global_timeout_interrupts_continuously_dribbling_chunk_framing() { let policy = DownloadPolicy { - connect_timeout: Duration::from_millis(200), - idle_read_timeout: Duration::from_millis(200), + connect_timeout: Duration::from_secs(1), + idle_read_timeout: Duration::from_millis(500), http_timeout: Duration::from_millis(250), max_compressed: 1024, max_uncompressed: 4096, @@ -1297,6 +1439,10 @@ mod tests { contains_io_kind(&error, io::ErrorKind::TimedOut), "got: {error:#}" ); + assert!( + contains_ureq_timeout(&error, ureq::Timeout::Global), + "got: {error:#}" + ); assert!( format!("{error:#}").contains("请手动下载"), "got: {error:#}" @@ -1377,6 +1523,10 @@ mod tests { contains_io_kind(&error, io::ErrorKind::TimedOut), "got: {error:#}" ); + assert!( + contains_ureq_timeout(&error, ureq::Timeout::RecvBody), + "got: {error:#}" + ); let detail = format!("{error:#}"); assert!(detail.contains("http://"), "got: {detail}"); assert!(detail.contains("data.sqlite"), "got: {detail}"); From be2000f4c449eb967e16a7725564788d83b90ab4 Mon Sep 17 00:00:00 2001 From: xr843 <137012659+xr843@users.noreply.github.com> Date: Sun, 12 Jul 2026 01:28:07 +0800 Subject: [PATCH 13/14] fix(data): preserve proxy connection deadlines --- src/data/transfer.rs | 206 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 188 insertions(+), 18 deletions(-) diff --git a/src/data/transfer.rs b/src/data/transfer.rs index 15fda1b..85f75fd 100644 --- a/src/data/transfer.rs +++ b/src/data/transfer.rs @@ -76,13 +76,29 @@ impl AbsoluteDeadlineTransport { timeout: ureq::unversioned::transport::NextTimeout, idle_read_timeout: Option, ) -> std::result::Result { - let now = StdInstant::now(); + self.bound_timeout_at(timeout, idle_read_timeout, StdInstant::now()) + } + + fn bound_timeout_at( + &mut self, + timeout: ureq::unversioned::transport::NextTimeout, + idle_read_timeout: Option, + now: StdInstant, + ) -> std::result::Result { + // ConnectProxyConnector drives its TransportAdapter with this sentinel + // while TLS-to-proxy and CONNECT are still part of the active connect + // phase. It may preserve an existing phase, but must never start one. + let connect_proxy_sentinel = self.connect_phase_active + && timeout.reason == ureq::Timeout::Global + && timeout.after.is_not_happening(); let connect_deadline = if timeout.reason == ureq::Timeout::Connect { if !self.connect_phase_active { self.connect_phase_deadline = checked_deadline(now, self.connect_timeout)?; self.connect_phase_active = true; } Some(self.connect_phase_deadline) + } else if connect_proxy_sentinel { + Some(self.connect_phase_deadline) } else { self.connect_phase_active = false; None @@ -586,23 +602,7 @@ fn stage_candidate_inner( .timeout_connect(Some(policy.connect_timeout)) .timeout_recv_body(Some(policy.idle_read_timeout)) .build(); - use ureq::unversioned::transport::Connector; - let deadline_connector = AbsoluteDeadlineConnector { - deadline, - connect_timeout: policy.connect_timeout, - idle_read_timeout: policy.idle_read_timeout, - }; - let connector = - ().chain(ureq::unversioned::transport::ConnectProxyConnector::default()) - .chain(ureq::unversioned::transport::TcpConnector::default()) - .chain(deadline_connector) - .chain(ureq::unversioned::transport::RustlsConnector::default()) - .chain(deadline_connector); - let agent = ureq::Agent::with_parts( - config, - connector, - ureq::unversioned::resolver::DefaultResolver::default(), - ); + let agent = build_agent(config, policy, deadline); let mut response = agent .get(source.url) .header("Accept-Encoding", "identity") @@ -696,6 +696,30 @@ fn stage_candidate_inner( } } +fn build_agent( + config: ureq::config::Config, + policy: DownloadPolicy, + deadline: StdInstant, +) -> ureq::Agent { + use ureq::unversioned::transport::Connector; + let deadline_connector = AbsoluteDeadlineConnector { + deadline, + connect_timeout: policy.connect_timeout, + idle_read_timeout: policy.idle_read_timeout, + }; + let connector = + ().chain(ureq::unversioned::transport::ConnectProxyConnector::default()) + .chain(ureq::unversioned::transport::TcpConnector::default()) + .chain(deadline_connector) + .chain(ureq::unversioned::transport::RustlsConnector::default()) + .chain(deadline_connector); + ureq::Agent::with_parts( + config, + connector, + ureq::unversioned::resolver::DefaultResolver::default(), + ) +} + fn download_error_context( error: anyhow::Error, action: &str, @@ -871,6 +895,92 @@ mod tests { assert!(detail.contains("清理候选数据失败"), "got: {detail}"); } + #[test] + fn connect_proxy_sentinel_keeps_the_original_connection_deadline() { + let started = Instant::now(); + let original_connect_deadline = started + Duration::from_millis(250); + let mut transport = AbsoluteDeadlineTransport { + inner: (), + deadline: started + Duration::from_secs(2), + connect_timeout: Duration::from_millis(250), + connect_phase_deadline: original_connect_deadline, + connect_phase_active: true, + idle_read_timeout: Duration::from_secs(1), + }; + let sentinel = ureq::unversioned::transport::NextTimeout { + after: ureq::unversioned::transport::time::Duration::NotHappening, + reason: ureq::Timeout::Global, + }; + + let first = transport + .bound_timeout_at(sentinel, None, started + Duration::from_millis(100)) + .unwrap(); + let expected_first: ureq::unversioned::transport::time::Duration = + Duration::from_millis(150).into(); + assert_eq!(first.reason, ureq::Timeout::Connect); + assert_eq!(first.after, expected_first); + assert!(transport.connect_phase_active); + assert_eq!(transport.connect_phase_deadline, original_connect_deadline); + + let second = transport + .bound_timeout_at(sentinel, None, started + Duration::from_millis(200)) + .unwrap(); + let expected_second: ureq::unversioned::transport::time::Duration = + Duration::from_millis(50).into(); + assert_eq!(second.reason, ureq::Timeout::Connect); + assert_eq!(second.after, expected_second); + assert!(transport.connect_phase_active); + assert_eq!(transport.connect_phase_deadline, original_connect_deadline); + } + + #[test] + fn finite_non_connect_timeout_ends_then_connect_restarts_the_phase() { + let started = Instant::now(); + let mut transport = AbsoluteDeadlineTransport { + inner: (), + deadline: started + Duration::from_secs(2), + connect_timeout: Duration::from_millis(250), + connect_phase_deadline: started + Duration::from_millis(250), + connect_phase_active: true, + idle_read_timeout: Duration::from_secs(1), + }; + let finite_global = ureq::unversioned::transport::NextTimeout { + after: Duration::from_millis(500).into(), + reason: ureq::Timeout::Global, + }; + + let finite = transport + .bound_timeout_at(finite_global, None, started + Duration::from_millis(50)) + .unwrap(); + assert_eq!(finite.reason, ureq::Timeout::Global); + assert!(!transport.connect_phase_active); + + let sentinel = ureq::unversioned::transport::NextTimeout { + after: ureq::unversioned::transport::time::Duration::NotHappening, + reason: ureq::Timeout::Global, + }; + let sentinel = transport + .bound_timeout_at(sentinel, None, started + Duration::from_millis(100)) + .unwrap(); + assert_eq!(sentinel.reason, ureq::Timeout::Global); + assert!(!transport.connect_phase_active); + + let restarted_at = started + Duration::from_millis(200); + let connect = ureq::unversioned::transport::NextTimeout { + after: Duration::from_millis(250).into(), + reason: ureq::Timeout::Connect, + }; + let connect = transport + .bound_timeout_at(connect, None, restarted_at) + .unwrap(); + assert_eq!(connect.reason, ureq::Timeout::Connect); + assert!(transport.connect_phase_active); + assert_eq!( + transport.connect_phase_deadline, + restarted_at + Duration::from_millis(250) + ); + } + fn sha256_hex(bytes: &[u8]) -> String { Sha256::digest(bytes) .iter() @@ -1383,6 +1493,66 @@ mod tests { assert_no_owned_transfer_artifacts(&live); } + #[test] + fn https_connect_proxy_tls_handshake_obeys_connect_deadline() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + stream.set_nodelay(true).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let mut client_hello = [0_u8; 16 * 1024]; + let _ = stream.read(&mut client_hello).unwrap(); + stream.write_all(&[0x16, 0x03, 0x03, 0x40, 0x00]).unwrap(); + stream.flush().unwrap(); + for _ in 0..24 { + if stream.write_all(&[0]).is_err() || stream.flush().is_err() { + break; + } + std::thread::sleep(Duration::from_millis(50)); + } + }); + let policy = DownloadPolicy { + connect_timeout: Duration::from_millis(250), + idle_read_timeout: Duration::from_secs(1), + http_timeout: Duration::from_secs(2), + max_compressed: 1024, + max_uncompressed: 4096, + }; + let proxy = ureq::Proxy::new(&format!("https://{address}")).unwrap(); + let config = ureq::Agent::config_builder() + .proxy(Some(proxy)) + .max_redirects(5) + .max_redirects_will_error(true) + .max_response_header_size(64 * 1024) + .accept_encoding("identity") + .timeout_global(Some(policy.http_timeout)) + .timeout_resolve(Some(policy.connect_timeout)) + .timeout_connect(Some(policy.connect_timeout)) + .timeout_recv_body(Some(policy.idle_read_timeout)) + .build(); + let started = Instant::now(); + let deadline = started + policy.http_timeout; + let agent = build_agent(config, policy, deadline); + + let error = anyhow::Error::new( + agent + .get("http://example.invalid/data.gz") + .call() + .unwrap_err(), + ); + let elapsed = started.elapsed(); + server.join().unwrap(); + + assert!(elapsed < Duration::from_millis(700), "elapsed: {elapsed:?}"); + assert!( + contains_ureq_timeout(&error, ureq::Timeout::Connect), + "got: {error:#}" + ); + } + #[test] fn global_timeout_interrupts_continuously_dribbling_response_headers() { let policy = DownloadPolicy { From bf43a5bf812ea98cae9e7aa53620f38db694b5a0 Mon Sep 17 00:00:00 2001 From: xr843 <137012659+xr843@users.noreply.github.com> Date: Sun, 12 Jul 2026 01:39:15 +0800 Subject: [PATCH 14/14] fix(data): preserve target TLS through proxies --- src/data/transfer.rs | 232 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 231 insertions(+), 1 deletion(-) diff --git a/src/data/transfer.rs b/src/data/transfer.rs index 85f75fd..f0df5e8 100644 --- a/src/data/transfer.rs +++ b/src/data/transfer.rs @@ -12,6 +12,69 @@ const MIB: u64 = 1024 * 1024; const BUFFER_SIZE: usize = 64 * 1024; static ARTIFACT_SEQUENCE: AtomicU64 = AtomicU64::new(0); +#[derive(Clone, Copy, Debug)] +struct ProxyTunnelConnector; + +#[derive(Debug)] +struct ProxyTunnelTransport { + inner: T, +} + +impl ureq::unversioned::transport::Connector + for ProxyTunnelConnector +{ + type Out = ProxyTunnelTransport; + + fn connect( + &self, + _: &ureq::unversioned::transport::ConnectionDetails<'_>, + chained: Option, + ) -> std::result::Result, ureq::Error> { + Ok(chained.map(|inner| ProxyTunnelTransport { inner })) + } +} + +impl ureq::unversioned::transport::Transport + for ProxyTunnelTransport +{ + fn buffers(&mut self) -> &mut dyn ureq::unversioned::transport::Buffers { + self.inner.buffers() + } + + fn transmit_output( + &mut self, + amount: usize, + timeout: ureq::unversioned::transport::NextTimeout, + ) -> std::result::Result<(), ureq::Error> { + self.inner.transmit_output(amount, timeout) + } + + fn maybe_await_input( + &mut self, + timeout: ureq::unversioned::transport::NextTimeout, + ) -> std::result::Result { + self.inner.maybe_await_input(timeout) + } + + fn await_input( + &mut self, + timeout: ureq::unversioned::transport::NextTimeout, + ) -> std::result::Result { + self.inner.await_input(timeout) + } + + fn is_open(&mut self) -> bool { + self.inner.is_open() + } + + fn is_tls(&self) -> bool { + // The inner transport may be TLS-secured to an HTTPS proxy, but the + // CONNECT tunnel itself is plaintext from the target's perspective. + // Report false so the following RustlsConnector adds target TLS. + false + } +} + // Ureq's socket adapter turns an already-expired zero timeout into one second // because operating systems reject a zero socket timeout. TLS can also perform // several lower-level reads for one HTTP-level read while reusing the original @@ -709,6 +772,7 @@ fn build_agent( }; let connector = ().chain(ureq::unversioned::transport::ConnectProxyConnector::default()) + .chain(ProxyTunnelConnector) .chain(ureq::unversioned::transport::TcpConnector::default()) .chain(deadline_connector) .chain(ureq::unversioned::transport::RustlsConnector::default()) @@ -816,6 +880,109 @@ mod tests { use flate2::Compression; use std::io::{Cursor, Read, Write}; use std::time::Instant; + use ureq::unversioned::transport::Buffers; + + #[derive(Debug)] + struct TlsMarkedTransport { + inner: T, + } + + impl ureq::unversioned::transport::Transport + for TlsMarkedTransport + { + fn buffers(&mut self) -> &mut dyn ureq::unversioned::transport::Buffers { + self.inner.buffers() + } + + fn transmit_output( + &mut self, + amount: usize, + timeout: ureq::unversioned::transport::NextTimeout, + ) -> std::result::Result<(), ureq::Error> { + self.inner.transmit_output(amount, timeout) + } + + fn maybe_await_input( + &mut self, + timeout: ureq::unversioned::transport::NextTimeout, + ) -> std::result::Result { + self.inner.maybe_await_input(timeout) + } + + fn await_input( + &mut self, + timeout: ureq::unversioned::transport::NextTimeout, + ) -> std::result::Result { + self.inner.await_input(timeout) + } + + fn is_open(&mut self) -> bool { + self.inner.is_open() + } + + fn is_tls(&self) -> bool { + true + } + } + + #[derive(Debug)] + struct LocalTcpTransport { + stream: std::net::TcpStream, + buffers: ureq::unversioned::transport::LazyBuffers, + } + + impl ureq::unversioned::transport::Transport for LocalTcpTransport { + fn buffers(&mut self) -> &mut dyn ureq::unversioned::transport::Buffers { + &mut self.buffers + } + + fn transmit_output( + &mut self, + amount: usize, + _: ureq::unversioned::transport::NextTimeout, + ) -> std::result::Result<(), ureq::Error> { + let output = &self.buffers.output()[..amount]; + self.stream.write_all(output).map_err(ureq::Error::Io) + } + + fn await_input( + &mut self, + _: ureq::unversioned::transport::NextTimeout, + ) -> std::result::Result { + let input = self.buffers.input_append_buf(); + let amount = self.stream.read(input).map_err(ureq::Error::Io)?; + self.buffers.input_appended(amount); + Ok(amount > 0) + } + + fn is_open(&mut self) -> bool { + true + } + } + + #[derive(Clone, Copy, Debug)] + struct FakeTlsProxyConnector { + address: std::net::SocketAddr, + } + + impl ureq::unversioned::transport::Connector<()> for FakeTlsProxyConnector { + type Out = TlsMarkedTransport; + + fn connect( + &self, + _: &ureq::unversioned::transport::ConnectionDetails<'_>, + _: Option<()>, + ) -> std::result::Result, ureq::Error> { + let stream = std::net::TcpStream::connect(self.address).map_err(ureq::Error::Io)?; + stream.set_nodelay(true).map_err(ureq::Error::Io)?; + Ok(Some(TlsMarkedTransport { + inner: LocalTcpTransport { + stream, + buffers: ureq::unversioned::transport::LazyBuffers::new(16 * 1024, 16 * 1024), + }, + })) + } + } fn policy_for_tests() -> DownloadPolicy { DownloadPolicy { @@ -827,6 +994,69 @@ mod tests { } } + fn first_wire_prefix_through_tls_marked_proxy(target: &str) -> [u8; 5] { + use ureq::unversioned::transport::Connector; + + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let mut prefix = [0_u8; 5]; + stream.read_exact(&mut prefix).unwrap(); + prefix + }); + let connector = () + .chain(FakeTlsProxyConnector { address }) + .chain(ProxyTunnelConnector) + .chain(ureq::unversioned::transport::RustlsConnector::default()); + let config = ureq::Agent::config_builder() + .proxy(None) + .timeout_global(Some(Duration::from_secs(2))) + .timeout_connect(Some(Duration::from_secs(1))) + .build(); + let agent = ureq::Agent::with_parts( + config, + connector, + ureq::unversioned::resolver::DefaultResolver::default(), + ); + + let _ = agent.get(target).call(); + server.join().unwrap() + } + + #[test] + fn proxy_tunnel_hides_proxy_tls_from_the_target_connector() { + use ureq::unversioned::transport::Transport; + + let proxy_tls = TlsMarkedTransport { inner: () }; + assert!(proxy_tls.is_tls()); + let tunnel = ProxyTunnelTransport { inner: proxy_tls }; + + assert!(!tunnel.is_tls()); + } + + #[test] + fn tls_marked_proxy_still_starts_a_target_tls_handshake() { + let prefix = first_wire_prefix_through_tls_marked_proxy("https://127.0.0.1/data.gz"); + + assert_eq!(prefix[0], 0x16, "expected TLS handshake, got {prefix:02x?}"); + assert_eq!(prefix[1], 0x03, "expected TLS version, got {prefix:02x?}"); + assert!( + (0x01..=0x04).contains(&prefix[2]), + "expected TLS record version, got {prefix:02x?}" + ); + } + + #[test] + fn tls_marked_proxy_keeps_an_http_target_plaintext() { + let prefix = first_wire_prefix_through_tls_marked_proxy("http://127.0.0.1/data.gz"); + + assert_eq!(&prefix, b"GET /"); + } + #[test] fn candidate_reservation_rejects_a_preexisting_sidecar_family() { let directory = tempfile::tempdir().unwrap(); @@ -1494,7 +1724,7 @@ mod tests { } #[test] - fn https_connect_proxy_tls_handshake_obeys_connect_deadline() { + fn https_connect_proxy_first_tls_handshake_obeys_connect_deadline() { let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let address = listener.local_addr().unwrap(); let server = std::thread::spawn(move || {