From 967c39e55798ae163f4dba380b7ccaa4f6ad9672 Mon Sep 17 00:00:00 2001 From: Cg8 <5712.cg8@gmail.com> Date: Mon, 13 Jul 2026 17:24:15 +0800 Subject: [PATCH 1/7] feat: implement Cloudreve to AsterDrive migration --- Cargo.lock | 40 + Cargo.toml | 10 +- README.md | 45 ++ crates/cloudreve-entities/src/dav_accounts.rs | 2 +- crates/cloudreve-entities/src/direct_links.rs | 2 +- crates/cloudreve-entities/src/entities.rs | 2 +- .../cloudreve-entities/src/file_entities.rs | 2 +- crates/cloudreve-entities/src/files.rs | 3 +- crates/cloudreve-entities/src/fs_events.rs | 2 +- crates/cloudreve-entities/src/groups.rs | 2 +- crates/cloudreve-entities/src/metadata.rs | 2 +- crates/cloudreve-entities/src/nodes.rs | 2 +- .../cloudreve-entities/src/oauth_clients.rs | 2 +- crates/cloudreve-entities/src/oauth_grants.rs | 2 +- crates/cloudreve-entities/src/passkeys.rs | 2 +- crates/cloudreve-entities/src/settings.rs | 2 +- crates/cloudreve-entities/src/shares.rs | 2 +- .../src/storage_policies.rs | 2 +- crates/cloudreve-entities/src/tasks.rs | 2 +- crates/cloudreve-entities/src/users.rs | 2 +- src/lib.rs | 3 +- src/main.rs | 70 +- src/migration/mod.rs | 751 ++++++++++++++++++ src/migration/phases.rs | 524 ++++++++++++ 24 files changed, 1451 insertions(+), 27 deletions(-) create mode 100644 src/migration/mod.rs create mode 100644 src/migration/phases.rs diff --git a/Cargo.lock b/Cargo.lock index 75673ba..058321e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -319,13 +319,19 @@ name = "aster_drive_migration" version = "0.0.0-alpha.0" dependencies = [ "argon2", + "asterdrive-entities", "chrono", "clap", + "cloudreve-entities", "color-eyre", "crossterm 0.29.0", + "mime_guess", "ratatui", "sea-orm", "serde", + "serde_json", + "sha2 0.10.9", + "tokio", "uuid", ] @@ -1847,6 +1853,22 @@ dependencies = [ "autocfg", ] +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -3529,9 +3551,21 @@ dependencies = [ "mio", "pin-project-lite", "socket2", + "tokio-macros", "windows-sys 0.61.2", ] +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "tokio-stream" version = "0.1.18" @@ -3639,6 +3673,12 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + [[package]] name = "unicode-bidi" version = "0.3.18" diff --git a/Cargo.toml b/Cargo.toml index e428fe6..42cfa28 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,19 +35,25 @@ members = [ [features] default = ["cli", "tui"] -cli = ["dep:clap"] +cli = [] tui = ["cli", "dep:ratatui"] full = ["cli", "tui"] [dependencies] +asterdrive-entities = { path = "crates/asterdrive-entities" } argon2 = "0.5.3" chrono = { version = "0.4.45", default-features = false, features = ["clock", "serde"] } -clap = { version = "4", features = ["derive", "env"], optional = true } +clap = { version = "4", features = ["derive", "env"] } +cloudreve-entities = { path = "crates/cloudreve-entities" } color-eyre = "0.6.5" crossterm = "0.29.0" +mime_guess = "2" ratatui = { version = "0.30.2", optional = true, features = ["serde"] } sea-orm = { version = "2.0.0-rc.42", features = ["sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", "macros", "runtime-tokio-rustls"] } serde = { version = "1.0.228", features = ["derive"] } +serde_json = "1" +sha2 = "0.10" +tokio = { version = "1", features = ["macros", "rt-multi-thread"] } uuid = { version = "1", features = ["v4"] } [lib] diff --git a/README.md b/README.md index 0f04cbc..420a13b 100644 --- a/README.md +++ b/README.md @@ -1 +1,46 @@ # AsterDriveMigration + +Database migration tool for moving a Cloudreve v4 installation to AsterDrive (AD). + +## Supported data + +- Cloudreve groups to AD storage policy groups +- users and profiles, including quotas and disabled status +- local, S3, OSS, KS3, OBS and Tencent COS storage policies +- folders, files, physical entities/blobs and version history +- file metadata and public shares + +Cloudreve password hashes are not compatible with AD. The migration assigns the supplied temporary password to every migrated user and sets `must_change_password = true`. Passkeys, WebDAV credentials, two-factor secrets, OAuth grants, sessions, tasks and filesystem events are not migrated. + +The tool reuses existing storage objects. For local storage, pass the directory from which Cloudreve entity paths should be resolved. Object-storage policies keep their existing bucket, endpoint, credentials and object keys. + +## Prerequisites + +1. Stop writes to Cloudreve and back up both databases. +2. Create an empty AD database and run the AD database migrations against it. +3. Do not start the AD application against the target database until this migration completes. + +## Check compatibility + +```powershell +cargo run -- check ` + --source-url "sqlite://C:/cloudreve/cloudreve.db?mode=ro" ` + --target-url "sqlite://C:/asterdrive/asterdrive.db" +``` + +The URLs can also be provided through `CLOUDREVE_DATABASE_URL` and `ASTERDRIVE_DATABASE_URL`. SQLite, MySQL and PostgreSQL URLs supported by SeaORM can be used. + +## Run migration + +```powershell +$env:ASTER_MIGRATION_DEFAULT_PASSWORD = "replace-with-a-strong-temporary-password" + +cargo run -- migrate ` + --source-url "sqlite://C:/cloudreve/cloudreve.db?mode=ro" ` + --target-url "sqlite://C:/asterdrive/asterdrive.db" ` + --local-base-path "C:/cloudreve" +``` + +Use `--dry-run` to perform preflight checks without writing the target. The target core tables must be empty by default. `--allow-non-empty-target` disables that guard but may still fail on unique values or conflicting data. + +Cloudreve Qiniu, Upyun, remote-node, OneDrive and encrypted storage policies cannot be reused safely by AD. The migration stops when they are present. `--skip-unsupported-policies` explicitly omits those policies and all dependent files. diff --git a/crates/cloudreve-entities/src/dav_accounts.rs b/crates/cloudreve-entities/src/dav_accounts.rs index 53b248c..a9290da 100644 --- a/crates/cloudreve-entities/src/dav_accounts.rs +++ b/crates/cloudreve-entities/src/dav_accounts.rs @@ -4,7 +4,7 @@ use sea_orm::entity::prelude::*; #[sea_orm::model] #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] -#[sea_orm(schema_name = "public", table_name = "dav_accounts")] +#[sea_orm(table_name = "dav_accounts")] pub struct Model { #[sea_orm(primary_key)] pub id: i64, diff --git a/crates/cloudreve-entities/src/direct_links.rs b/crates/cloudreve-entities/src/direct_links.rs index 4b8c9be..a083a3c 100644 --- a/crates/cloudreve-entities/src/direct_links.rs +++ b/crates/cloudreve-entities/src/direct_links.rs @@ -4,7 +4,7 @@ use sea_orm::entity::prelude::*; #[sea_orm::model] #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] -#[sea_orm(schema_name = "public", table_name = "direct_links")] +#[sea_orm(table_name = "direct_links")] pub struct Model { #[sea_orm(primary_key)] pub id: i64, diff --git a/crates/cloudreve-entities/src/entities.rs b/crates/cloudreve-entities/src/entities.rs index 4ff3052..d1044c4 100644 --- a/crates/cloudreve-entities/src/entities.rs +++ b/crates/cloudreve-entities/src/entities.rs @@ -4,7 +4,7 @@ use sea_orm::entity::prelude::*; #[sea_orm::model] #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] -#[sea_orm(schema_name = "public", table_name = "entities")] +#[sea_orm(table_name = "entities")] pub struct Model { #[sea_orm(primary_key)] pub id: i64, diff --git a/crates/cloudreve-entities/src/file_entities.rs b/crates/cloudreve-entities/src/file_entities.rs index accde17..62b47ce 100644 --- a/crates/cloudreve-entities/src/file_entities.rs +++ b/crates/cloudreve-entities/src/file_entities.rs @@ -4,7 +4,7 @@ use sea_orm::entity::prelude::*; #[sea_orm::model] #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] -#[sea_orm(schema_name = "public", table_name = "file_entities")] +#[sea_orm(table_name = "file_entities")] pub struct Model { #[sea_orm(primary_key, auto_increment = false)] pub file_id: i64, diff --git a/crates/cloudreve-entities/src/files.rs b/crates/cloudreve-entities/src/files.rs index f4503a8..dca9b02 100644 --- a/crates/cloudreve-entities/src/files.rs +++ b/crates/cloudreve-entities/src/files.rs @@ -4,13 +4,12 @@ use sea_orm::entity::prelude::*; #[sea_orm::model] #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] -#[sea_orm(schema_name = "public", table_name = "files")] +#[sea_orm(table_name = "files")] pub struct Model { #[sea_orm(primary_key)] pub id: i64, pub created_at: DateTimeWithTimeZone, pub updated_at: DateTimeWithTimeZone, - pub deleted_at: Option, pub r#type: i64, #[sea_orm(unique_key = "file_file_children_name")] pub name: String, diff --git a/crates/cloudreve-entities/src/fs_events.rs b/crates/cloudreve-entities/src/fs_events.rs index 216db30..a71fbe7 100644 --- a/crates/cloudreve-entities/src/fs_events.rs +++ b/crates/cloudreve-entities/src/fs_events.rs @@ -4,7 +4,7 @@ use sea_orm::entity::prelude::*; #[sea_orm::model] #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] -#[sea_orm(schema_name = "public", table_name = "fs_events")] +#[sea_orm(table_name = "fs_events")] pub struct Model { #[sea_orm(primary_key)] pub id: i64, diff --git a/crates/cloudreve-entities/src/groups.rs b/crates/cloudreve-entities/src/groups.rs index 961c75a..29f0ec1 100644 --- a/crates/cloudreve-entities/src/groups.rs +++ b/crates/cloudreve-entities/src/groups.rs @@ -4,7 +4,7 @@ use sea_orm::entity::prelude::*; #[sea_orm::model] #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] -#[sea_orm(schema_name = "public", table_name = "groups")] +#[sea_orm(table_name = "groups")] pub struct Model { #[sea_orm(primary_key)] pub id: i64, diff --git a/crates/cloudreve-entities/src/metadata.rs b/crates/cloudreve-entities/src/metadata.rs index 085c420..6784f7b 100644 --- a/crates/cloudreve-entities/src/metadata.rs +++ b/crates/cloudreve-entities/src/metadata.rs @@ -4,7 +4,7 @@ use sea_orm::entity::prelude::*; #[sea_orm::model] #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] -#[sea_orm(schema_name = "public", table_name = "metadata")] +#[sea_orm(table_name = "metadata")] pub struct Model { #[sea_orm(primary_key)] pub id: i64, diff --git a/crates/cloudreve-entities/src/nodes.rs b/crates/cloudreve-entities/src/nodes.rs index 81178d9..fafef8f 100644 --- a/crates/cloudreve-entities/src/nodes.rs +++ b/crates/cloudreve-entities/src/nodes.rs @@ -4,7 +4,7 @@ use sea_orm::entity::prelude::*; #[sea_orm::model] #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] -#[sea_orm(schema_name = "public", table_name = "nodes")] +#[sea_orm(table_name = "nodes")] pub struct Model { #[sea_orm(primary_key)] pub id: i64, diff --git a/crates/cloudreve-entities/src/oauth_clients.rs b/crates/cloudreve-entities/src/oauth_clients.rs index 4d28025..2934c22 100644 --- a/crates/cloudreve-entities/src/oauth_clients.rs +++ b/crates/cloudreve-entities/src/oauth_clients.rs @@ -4,7 +4,7 @@ use sea_orm::entity::prelude::*; #[sea_orm::model] #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] -#[sea_orm(schema_name = "public", table_name = "oauth_clients")] +#[sea_orm(table_name = "oauth_clients")] pub struct Model { #[sea_orm(primary_key)] pub id: i64, diff --git a/crates/cloudreve-entities/src/oauth_grants.rs b/crates/cloudreve-entities/src/oauth_grants.rs index 5063856..0230643 100644 --- a/crates/cloudreve-entities/src/oauth_grants.rs +++ b/crates/cloudreve-entities/src/oauth_grants.rs @@ -4,7 +4,7 @@ use sea_orm::entity::prelude::*; #[sea_orm::model] #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] -#[sea_orm(schema_name = "public", table_name = "oauth_grants")] +#[sea_orm(table_name = "oauth_grants")] pub struct Model { #[sea_orm(primary_key)] pub id: i64, diff --git a/crates/cloudreve-entities/src/passkeys.rs b/crates/cloudreve-entities/src/passkeys.rs index 3e57935..09de791 100644 --- a/crates/cloudreve-entities/src/passkeys.rs +++ b/crates/cloudreve-entities/src/passkeys.rs @@ -4,7 +4,7 @@ use sea_orm::entity::prelude::*; #[sea_orm::model] #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] -#[sea_orm(schema_name = "public", table_name = "passkeys")] +#[sea_orm(table_name = "passkeys")] pub struct Model { #[sea_orm(primary_key)] pub id: i64, diff --git a/crates/cloudreve-entities/src/settings.rs b/crates/cloudreve-entities/src/settings.rs index 2ebcc4e..5093ab6 100644 --- a/crates/cloudreve-entities/src/settings.rs +++ b/crates/cloudreve-entities/src/settings.rs @@ -4,7 +4,7 @@ use sea_orm::entity::prelude::*; #[sea_orm::model] #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] -#[sea_orm(schema_name = "public", table_name = "settings")] +#[sea_orm(table_name = "settings")] pub struct Model { #[sea_orm(primary_key)] pub id: i64, diff --git a/crates/cloudreve-entities/src/shares.rs b/crates/cloudreve-entities/src/shares.rs index 5fd10f2..1b74540 100644 --- a/crates/cloudreve-entities/src/shares.rs +++ b/crates/cloudreve-entities/src/shares.rs @@ -4,7 +4,7 @@ use sea_orm::entity::prelude::*; #[sea_orm::model] #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] -#[sea_orm(schema_name = "public", table_name = "shares")] +#[sea_orm(table_name = "shares")] pub struct Model { #[sea_orm(primary_key)] pub id: i64, diff --git a/crates/cloudreve-entities/src/storage_policies.rs b/crates/cloudreve-entities/src/storage_policies.rs index 0fffcc3..b563933 100644 --- a/crates/cloudreve-entities/src/storage_policies.rs +++ b/crates/cloudreve-entities/src/storage_policies.rs @@ -4,7 +4,7 @@ use sea_orm::entity::prelude::*; #[sea_orm::model] #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] -#[sea_orm(schema_name = "public", table_name = "storage_policies")] +#[sea_orm(table_name = "storage_policies")] pub struct Model { #[sea_orm(primary_key)] pub id: i64, diff --git a/crates/cloudreve-entities/src/tasks.rs b/crates/cloudreve-entities/src/tasks.rs index 1c655f4..9b73397 100644 --- a/crates/cloudreve-entities/src/tasks.rs +++ b/crates/cloudreve-entities/src/tasks.rs @@ -4,7 +4,7 @@ use sea_orm::entity::prelude::*; #[sea_orm::model] #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] -#[sea_orm(schema_name = "public", table_name = "tasks")] +#[sea_orm(table_name = "tasks")] pub struct Model { #[sea_orm(primary_key)] pub id: i64, diff --git a/crates/cloudreve-entities/src/users.rs b/crates/cloudreve-entities/src/users.rs index 6eaad1a..b63daff 100644 --- a/crates/cloudreve-entities/src/users.rs +++ b/crates/cloudreve-entities/src/users.rs @@ -4,7 +4,7 @@ use sea_orm::entity::prelude::*; #[sea_orm::model] #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] -#[sea_orm(schema_name = "public", table_name = "users")] +#[sea_orm(table_name = "users")] pub struct Model { #[sea_orm(primary_key)] pub id: i64, diff --git a/src/lib.rs b/src/lib.rs index ab5cfeb..a571eaa 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,2 +1 @@ -mod entities; -mod interfaces; +pub mod migration; diff --git a/src/main.rs b/src/main.rs index fcd38bd..d29ec8c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,8 +1,68 @@ -/// AsterDriveMigration is a tool for migrating data from other storage solution to AsterDrive. +use aster_drive_migration::migration::{MigrationOptions, inspect, migrate}; +use clap::{Parser, Subcommand}; +use color_eyre::eyre::Result; -mod entities; -mod interfaces; +#[derive(Debug, Parser)] +#[command(name = "aster-drive-migration", version, about)] +struct Cli { + #[command(subcommand)] + command: Command, +} + +#[derive(Debug, Subcommand)] +enum Command { + Check(ConnectionArgs), + Migrate(MigrateArgs), +} + +#[derive(Debug, clap::Args)] +struct ConnectionArgs { + #[arg(long, env = "CLOUDREVE_DATABASE_URL")] + source_url: String, + #[arg(long, env = "ASTERDRIVE_DATABASE_URL")] + target_url: String, + #[arg(long)] + include_deleted: bool, +} + +#[derive(Debug, clap::Args)] +struct MigrateArgs { + #[command(flatten)] + connection: ConnectionArgs, + #[arg(long, env = "ASTER_MIGRATION_DEFAULT_PASSWORD")] + default_password: String, + #[arg(long, default_value = ".")] + local_base_path: String, + #[arg(long)] + allow_non_empty_target: bool, + #[arg(long)] + skip_unsupported_policies: bool, + #[arg(long)] + dry_run: bool, +} -fn main() { - println!("Hello, world!"); +#[tokio::main] +async fn main() -> Result<()> { + color_eyre::install()?; + match Cli::parse().command { + Command::Check(args) => { + let report = inspect(&args.source_url, &args.target_url, args.include_deleted).await?; + println!("{report}"); + } + Command::Migrate(args) => { + let report = migrate(MigrationOptions { + source_url: args.connection.source_url, + target_url: args.connection.target_url, + default_password: args.default_password, + local_base_path: args.local_base_path, + include_deleted: args.connection.include_deleted, + allow_non_empty_target: args.allow_non_empty_target, + skip_unsupported_policies: args.skip_unsupported_policies, + dry_run: args.dry_run, + }) + .await?; + println!("{report}"); + } + } + Ok(()) } diff --git a/src/migration/mod.rs b/src/migration/mod.rs new file mode 100644 index 0000000..94fa910 --- /dev/null +++ b/src/migration/mod.rs @@ -0,0 +1,751 @@ +use std::collections::{HashMap, HashSet}; +use std::fmt::{self, Write}; + +use argon2::Argon2; +use argon2::password_hash::{PasswordHasher, SaltString}; +use color_eyre::eyre::{Result, WrapErr, bail}; +use sea_orm::{ + ActiveModelTrait, Database, DatabaseConnection, EntityTrait, PaginatorTrait, Set, + TransactionTrait, +}; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; + +use asterdrive_entities as ad; +use cloudreve_entities as cr; + +#[derive(Debug, Clone)] +pub struct MigrationOptions { + pub source_url: String, + pub target_url: String, + pub default_password: String, + pub local_base_path: String, + pub include_deleted: bool, + pub allow_non_empty_target: bool, + pub skip_unsupported_policies: bool, + pub dry_run: bool, +} + +#[derive(Debug, Default, Clone)] +pub struct MigrationReport { + pub source_users: u64, + pub source_groups: u64, + pub source_policies: u64, + pub source_folders: u64, + pub source_files: u64, + pub source_entities: u64, + pub source_shares: u64, + pub migrated_users: usize, + pub migrated_policy_groups: usize, + pub migrated_policies: usize, + pub migrated_folders: usize, + pub migrated_files: usize, + pub migrated_blobs: usize, + pub migrated_versions: usize, + pub migrated_shares: usize, + pub migrated_properties: usize, + pub skipped: usize, + pub dry_run: bool, + pub warnings: Vec, +} + +impl fmt::Display for MigrationReport { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut output = String::new(); + writeln!(output, "Cloudreve -> AsterDrive migration report")?; + writeln!( + output, + "source: users={}, groups={}, policies={}, folders={}, files={}, entities={}, shares={}", + self.source_users, + self.source_groups, + self.source_policies, + self.source_folders, + self.source_files, + self.source_entities, + self.source_shares + )?; + if self.dry_run { + writeln!(output, "mode: dry-run (target was not modified)")?; + } else { + writeln!( + output, + "migrated: users={}, policy_groups={}, policies={}, folders={}, files={}, blobs={}, versions={}, shares={}, properties={}", + self.migrated_users, + self.migrated_policy_groups, + self.migrated_policies, + self.migrated_folders, + self.migrated_files, + self.migrated_blobs, + self.migrated_versions, + self.migrated_shares, + self.migrated_properties + )?; + writeln!(output, "skipped: {}", self.skipped)?; + } + if !self.warnings.is_empty() { + writeln!(output, "warnings:")?; + for warning in &self.warnings { + writeln!(output, "- {warning}")?; + } + } + formatter.write_str(output.trim_end()) + } +} + +pub async fn inspect( + source_url: &str, + target_url: &str, + include_deleted: bool, +) -> Result { + let source = connect(source_url, "Cloudreve").await?; + let target = connect(target_url, "AsterDrive").await?; + let source_data = SourceData::load(&source, include_deleted).await?; + validate_target_schema(&target).await?; + let mut report = source_data.report(); + report.dry_run = true; + report.warnings.extend(source_data.compatibility_warnings()); + Ok(report) +} + +pub async fn migrate(options: MigrationOptions) -> Result { + if options.default_password.chars().count() < 8 { + bail!("--default-password must contain at least 8 characters"); + } + + let source = connect(&options.source_url, "Cloudreve").await?; + let target = connect(&options.target_url, "AsterDrive").await?; + let source_data = SourceData::load(&source, options.include_deleted).await?; + validate_target_schema(&target).await?; + ensure_target_safe(&target, options.allow_non_empty_target).await?; + + let unsupported = source_data.unsupported_policy_types(); + if !unsupported.is_empty() && !options.skip_unsupported_policies { + bail!( + "unsupported Cloudreve storage policy types: {}; rerun with --skip-unsupported-policies to omit their files", + unsupported.join(", ") + ); + } + + let mut report = source_data.report(); + report.dry_run = options.dry_run; + report.warnings.extend(source_data.compatibility_warnings()); + if options.dry_run { + return Ok(report); + } + + let password_hash = hash_password(&options.default_password)?; + let transaction = target + .begin() + .await + .wrap_err("begin AsterDrive transaction")?; + let mut context = MigrationContext::default(); + + migrate_policies( + &transaction, + &source_data, + &options, + &mut context, + &mut report, + ) + .await?; + migrate_policy_groups(&transaction, &source_data, &mut context, &mut report).await?; + migrate_users( + &transaction, + &source_data, + &password_hash, + &mut context, + &mut report, + ) + .await?; + migrate_folders(&transaction, &source_data, &mut context, &mut report).await?; + migrate_blobs(&transaction, &source_data, &mut context, &mut report).await?; + migrate_files(&transaction, &source_data, &mut context, &mut report).await?; + migrate_metadata(&transaction, &source_data, &context, &mut report).await?; + migrate_shares(&transaction, &source_data, &context, &mut report).await?; + + transaction + .commit() + .await + .wrap_err("commit AsterDrive migration transaction")?; + Ok(report) +} + +async fn connect(url: &str, label: &str) -> Result { + Database::connect(url) + .await + .wrap_err_with(|| format!("connect to {label} database")) +} + +async fn validate_target_schema(db: &DatabaseConnection) -> Result<()> { + ad::users::Entity::find() + .count(db) + .await + .wrap_err("AsterDrive schema is unavailable; run AD database migrations first")?; + ad::storage_policies::Entity::find() + .count(db) + .await + .wrap_err("AsterDrive storage_policies table is unavailable")?; + ad::files::Entity::find() + .count(db) + .await + .wrap_err("AsterDrive files table is unavailable")?; + Ok(()) +} + +async fn ensure_target_safe(db: &DatabaseConnection, allow_non_empty: bool) -> Result<()> { + if allow_non_empty { + return Ok(()); + } + let counts = [ + ("users", ad::users::Entity::find().count(db).await?), + ("folders", ad::folders::Entity::find().count(db).await?), + ("files", ad::files::Entity::find().count(db).await?), + ( + "file_blobs", + ad::file_blobs::Entity::find().count(db).await?, + ), + ("shares", ad::shares::Entity::find().count(db).await?), + ]; + let occupied: Vec = counts + .into_iter() + .filter(|(_, count)| *count > 0) + .map(|(table, count)| format!("{table}={count}")) + .collect(); + if !occupied.is_empty() { + bail!( + "target AD database is not empty ({}); use a freshly migrated database or pass --allow-non-empty-target", + occupied.join(", ") + ); + } + Ok(()) +} + +fn hash_password(password: &str) -> Result { + let salt = SaltString::encode_b64(uuid::Uuid::new_v4().as_bytes()) + .map_err(|error| color_eyre::eyre::eyre!("create password salt: {error}"))?; + Argon2::default() + .hash_password(password.as_bytes(), &salt) + .map(|hash| hash.to_string()) + .map_err(|error| color_eyre::eyre::eyre!("hash temporary AD password: {error}")) +} + +#[derive(Default)] +struct MigrationContext { + policies: HashMap, + policy_groups: HashMap, + users: HashMap, + usernames: HashMap, + folders: HashMap, + blobs: HashMap, + files: HashMap, +} + +struct SourceData { + groups: Vec, + users: Vec, + policies: Vec, + files: Vec, + entities: Vec, + file_entities: Vec, + shares: Vec, + metadata: Vec, +} + +impl SourceData { + async fn load(db: &DatabaseConnection, include_deleted: bool) -> Result { + let groups = cr::groups::Entity::find().all(db).await?; + let users = cr::users::Entity::find().all(db).await?; + let policies = cr::storage_policies::Entity::find().all(db).await?; + let files = cr::files::Entity::find().all(db).await?; + let entities = cr::entities::Entity::find().all(db).await?; + let file_entities = cr::file_entities::Entity::find().all(db).await?; + let shares = cr::shares::Entity::find().all(db).await?; + let metadata = cr::metadata::Entity::find().all(db).await?; + + Ok(Self { + groups: filter_deleted(groups, include_deleted, |model| model.deleted_at.is_some()), + users: filter_deleted(users, include_deleted, |model| model.deleted_at.is_some()), + policies: filter_deleted(policies, include_deleted, |model| { + model.deleted_at.is_some() + }), + files, + entities: filter_deleted(entities, include_deleted, |model| { + model.deleted_at.is_some() + }), + file_entities, + shares: filter_deleted(shares, include_deleted, |model| model.deleted_at.is_some()), + metadata: filter_deleted(metadata, include_deleted, |model| { + model.deleted_at.is_some() + }), + }) + } + + fn report(&self) -> MigrationReport { + MigrationReport { + source_users: self.users.len() as u64, + source_groups: self.groups.len() as u64, + source_policies: self.policies.len() as u64, + source_folders: self.files.iter().filter(|file| file.r#type == 1).count() as u64, + source_files: self.files.iter().filter(|file| file.r#type == 0).count() as u64, + source_entities: self.entities.len() as u64, + source_shares: self.shares.len() as u64, + ..Default::default() + } + } + + fn unsupported_policy_types(&self) -> Vec { + let mut values: Vec = self + .policies + .iter() + .filter_map(unsupported_policy_reason) + .collect(); + values.sort(); + values.dedup(); + values + } + + fn compatibility_warnings(&self) -> Vec { + let mut warnings = vec![ + "Cloudreve user passwords use SHA/legacy MD5 formats and are replaced by the supplied temporary Argon2 password; every migrated user is marked must_change_password".to_string(), + "OAuth grants, login sessions, background tasks and Cloudreve filesystem events are intentionally not migrated".to_string(), + "Cloudreve Passkeys, WebDAV credentials and two-factor secrets are not portable to AD and must be enrolled again".to_string(), + "file objects are reused in their existing local/object-storage locations; the migration does not duplicate object bytes".to_string(), + ]; + let symbolic = self.files.iter().filter(|file| file.is_symbolic).count(); + if symbolic > 0 { + warnings.push(format!( + "{symbolic} symbolic/placeholder Cloudreve files cannot be represented in AD and will be skipped" + )); + } + let unsupported = self.unsupported_policy_types(); + if !unsupported.is_empty() { + warnings.push(format!( + "unsupported storage policy types detected: {}", + unsupported.join(", ") + )); + } + warnings + } +} + +fn filter_deleted(items: Vec, include_deleted: bool, deleted: F) -> Vec +where + F: Fn(&T) -> bool, +{ + if include_deleted { + items + } else { + items.into_iter().filter(|item| !deleted(item)).collect() + } +} + +fn map_driver_type(source: &str) -> Option<&'static str> { + match source { + "local" => Some("local"), + "s3" | "oss" | "ks3" | "obs" => Some("s3"), + "cos" => Some("tencent_cos"), + _ => None, + } +} + +fn unsupported_policy_reason(policy: &cr::storage_policies::Model) -> Option { + if map_driver_type(&policy.r#type).is_none() { + return Some(policy.r#type.clone()); + } + if source_settings(&policy.settings) + .get("encryption") + .and_then(Value::as_bool) + .unwrap_or(false) + { + return Some(format!("{} (Cloudreve encryption enabled)", policy.r#type)); + } + None +} + +fn source_settings(value: &Option) -> Value { + value.clone().unwrap_or_else(|| json!({})) +} + +fn policy_options(policy: &cr::storage_policies::Model) -> String { + let settings = source_settings(&policy.settings); + let path_style = settings + .get("s3_path_style") + .and_then(Value::as_bool) + .unwrap_or(true); + json!({ + "s3_path_style": path_style, + "object_storage_upload_strategy": "relay_stream", + "object_storage_download_strategy": "relay_stream", + "cloudreve_source": settings, + "cloudreve_policy_type": policy.r#type, + }) + .to_string() +} + +fn allowed_types(policy: &cr::storage_policies::Model) -> String { + source_settings(&policy.settings) + .get("file_type") + .cloned() + .unwrap_or_else(|| json!([])) + .to_string() +} + +fn chunk_size(policy: &cr::storage_policies::Model) -> i64 { + source_settings(&policy.settings) + .get("chunk_size") + .and_then(Value::as_i64) + .unwrap_or(0) +} + +fn group_is_admin(group: &cr::groups::Model) -> bool { + group + .permissions + .first() + .is_some_and(|permissions| permissions & 1 == 1) +} + +fn opaque_blob_key(entity_id: i64) -> String { + format!("cloudreve-{entity_id:016x}") +} + +fn share_token(share_id: i64) -> String { + let digest = Sha256::digest(format!("cloudreve-share-{share_id}").as_bytes()); + format!("cr-{share_id}-{}", &format!("{digest:x}")[..16]) +} + +fn unique_username(source: &str, source_id: i64, used: &mut HashSet) -> String { + let mut base: String = source + .trim() + .chars() + .filter(|character| !character.is_control()) + .take(64) + .collect(); + if base.is_empty() { + base = format!("cloudreve-user-{source_id}"); + } + if used.insert(base.clone()) { + return base; + } + let suffix = format!("-{source_id}"); + let keep = 64usize.saturating_sub(suffix.chars().count()); + let mut candidate: String = base.chars().take(keep).collect(); + candidate.push_str(&suffix); + let mut discriminator = 2; + while !used.insert(candidate.clone()) { + let suffix = format!("-{source_id}-{discriminator}"); + let keep = 64usize.saturating_sub(suffix.chars().count()); + candidate = base.chars().take(keep).collect(); + candidate.push_str(&suffix); + discriminator += 1; + } + candidate +} + +fn file_classification(name: &str) -> (String, Option, String, String) { + let mime = mime_guess::from_path(name) + .first_or_octet_stream() + .essence_str() + .to_string(); + let lowercase = name.to_ascii_lowercase(); + let extension = lowercase + .rsplit_once('.') + .map(|(_, extension)| extension.to_string()) + .unwrap_or_default(); + let compound_extension = ["tar.gz", "tar.bz2", "tar.xz", "user.js"] + .into_iter() + .find(|candidate| lowercase.ends_with(candidate)) + .map(str::to_string); + let category = if mime.starts_with("image/") { + "image" + } else if mime.starts_with("video/") { + "video" + } else if mime.starts_with("audio/") { + "audio" + } else if ["zip", "rar", "7z", "gz", "bz2", "xz", "tar"].contains(&extension.as_str()) { + "archive" + } else if ["xls", "xlsx", "csv", "ods"].contains(&extension.as_str()) { + "spreadsheet" + } else if ["ppt", "pptx", "odp"].contains(&extension.as_str()) { + "presentation" + } else if [ + "rs", "go", "js", "ts", "py", "java", "c", "cpp", "html", "css", "json", "yaml", "yml", + "toml", + ] + .contains(&extension.as_str()) + { + "code" + } else if mime.starts_with("text/") + || ["pdf", "doc", "docx", "odt", "md"].contains(&extension.as_str()) + { + "document" + } else { + "other" + }; + (mime, compound_extension, extension, category.to_string()) +} + +mod phases; +use phases::*; + +#[cfg(test)] +mod tests { + use super::*; + use sea_orm::{ConnectionTrait, DbBackend, Schema}; + + #[test] + fn classifies_common_file_types() { + assert_eq!(file_classification("photo.JPG").3, "image"); + assert_eq!( + file_classification("backup.tar.gz").1.as_deref(), + Some("tar.gz") + ); + assert_eq!(file_classification("main.rs").3, "code"); + } + + #[test] + fn maps_supported_storage_drivers_conservatively() { + assert_eq!(map_driver_type("local"), Some("local")); + assert_eq!(map_driver_type("oss"), Some("s3")); + assert_eq!(map_driver_type("cos"), Some("tencent_cos")); + assert_eq!(map_driver_type("onedrive"), None); + assert_eq!(map_driver_type("qiniu"), None); + } + + #[tokio::test] + async fn migrates_minimal_cloudreve_database() -> Result<()> { + let suffix = uuid::Uuid::new_v4(); + let source_path = std::env::temp_dir().join(format!("cloudreve-{suffix}.db")); + let target_path = std::env::temp_dir().join(format!("asterdrive-{suffix}.db")); + let source_url = sqlite_url(&source_path); + let target_url = sqlite_url(&target_path); + + let source = Database::connect(&source_url).await?; + create_source_schema(&source).await?; + seed_source(&source).await?; + source.close().await?; + + let target = Database::connect(&target_url).await?; + create_target_schema(&target).await?; + target.close().await?; + + let report = migrate(MigrationOptions { + source_url: source_url.clone(), + target_url: target_url.clone(), + default_password: "temporary-password".to_string(), + local_base_path: "C:/cloudreve".to_string(), + include_deleted: false, + allow_non_empty_target: false, + skip_unsupported_policies: false, + dry_run: false, + }) + .await?; + + assert_eq!(report.migrated_users, 1); + assert_eq!(report.migrated_folders, 1); + assert_eq!(report.migrated_files, 1); + assert_eq!(report.migrated_blobs, 1); + assert_eq!(report.migrated_shares, 1); + assert_eq!(report.migrated_properties, 1); + + let target = Database::connect(&target_url).await?; + assert_eq!(ad::users::Entity::find().count(&target).await?, 1); + assert_eq!(ad::folders::Entity::find().count(&target).await?, 1); + assert_eq!(ad::files::Entity::find().count(&target).await?, 1); + assert_eq!(ad::file_blobs::Entity::find().count(&target).await?, 1); + assert_eq!(ad::shares::Entity::find().count(&target).await?, 1); + assert_eq!( + ad::entity_properties::Entity::find().count(&target).await?, + 1 + ); + let blob = ad::file_blobs::Entity::find().one(&target).await?.unwrap(); + assert_eq!(blob.storage_path, "uploads/object.bin"); + let user = ad::users::Entity::find().one(&target).await?.unwrap(); + assert!(user.must_change_password); + assert_eq!(user.role, "admin"); + target.close().await?; + + let _ = std::fs::remove_file(source_path); + let _ = std::fs::remove_file(target_path); + Ok(()) + } + + fn sqlite_url(path: &std::path::Path) -> String { + format!( + "sqlite://{}?mode=rwc", + path.to_string_lossy().replace('\\', "/") + ) + } + + async fn create_table(db: &DatabaseConnection, entity: E) -> Result<()> { + let schema = Schema::new(DbBackend::Sqlite); + db.execute(&schema.create_table_from_entity(entity)).await?; + Ok(()) + } + + async fn create_source_schema(db: &DatabaseConnection) -> Result<()> { + create_table(db, cr::nodes::Entity).await?; + create_table(db, cr::groups::Entity).await?; + create_table(db, cr::users::Entity).await?; + create_table(db, cr::storage_policies::Entity).await?; + create_table(db, cr::files::Entity).await?; + create_table(db, cr::entities::Entity).await?; + create_table(db, cr::file_entities::Entity).await?; + create_table(db, cr::shares::Entity).await?; + create_table(db, cr::metadata::Entity).await?; + Ok(()) + } + + async fn create_target_schema(db: &DatabaseConnection) -> Result<()> { + create_table(db, ad::managed_followers::Entity).await?; + create_table(db, ad::storage_policy_groups::Entity).await?; + create_table(db, ad::storage_policies::Entity).await?; + create_table(db, ad::storage_policy_group_items::Entity).await?; + create_table(db, ad::users::Entity).await?; + create_table(db, ad::user_profiles::Entity).await?; + create_table(db, ad::folders::Entity).await?; + create_table(db, ad::file_blobs::Entity).await?; + create_table(db, ad::files::Entity).await?; + create_table(db, ad::file_versions::Entity).await?; + create_table(db, ad::entity_properties::Entity).await?; + create_table(db, ad::shares::Entity).await?; + Ok(()) + } + + async fn seed_source(db: &DatabaseConnection) -> Result<()> { + let now = chrono::Utc::now().fixed_offset(); + let policy = cr::storage_policies::ActiveModel { + created_at: Set(now), + updated_at: Set(now), + deleted_at: Set(None), + name: Set("Local".to_string()), + r#type: Set("local".to_string()), + server: Set(None), + bucket_name: Set(None), + is_private: Set(Some(true)), + access_key: Set(None), + secret_key: Set(None), + max_size: Set(None), + dir_name_rule: Set(None), + file_name_rule: Set(None), + settings: Set(Some(json!({"chunk_size": 0}))), + node_id: Set(None), + ..Default::default() + } + .insert(db) + .await?; + let group = cr::groups::ActiveModel { + created_at: Set(now), + updated_at: Set(now), + deleted_at: Set(None), + name: Set("Administrators".to_string()), + max_storage: Set(Some(1024 * 1024)), + speed_limit: Set(None), + permissions: Set(vec![1]), + settings: Set(None), + storage_policy_id: Set(Some(policy.id)), + ..Default::default() + } + .insert(db) + .await?; + let user = cr::users::ActiveModel { + created_at: Set(now), + updated_at: Set(now), + deleted_at: Set(None), + email: Set("admin@example.test".to_string()), + nick: Set("admin".to_string()), + password: Set(Some("legacy:hash".to_string())), + status: Set("active".to_string()), + storage: Set(128), + two_factor_secret: Set(None), + avatar: Set(None), + settings: Set(None), + group_users: Set(group.id), + ..Default::default() + } + .insert(db) + .await?; + let folder = cr::files::ActiveModel { + created_at: Set(now), + updated_at: Set(now), + r#type: Set(1), + name: Set("Documents".to_string()), + size: Set(0), + primary_entity: Set(None), + is_symbolic: Set(false), + props: Set(None), + file_children: Set(None), + storage_policy_files: Set(Some(policy.id)), + owner_id: Set(user.id), + ..Default::default() + } + .insert(db) + .await?; + let entity = cr::entities::ActiveModel { + created_at: Set(now), + updated_at: Set(now), + deleted_at: Set(None), + r#type: Set(0), + source: Set("uploads/object.bin".to_string()), + size: Set(128), + reference_count: Set(1), + upload_session_id: Set(None), + recycle_options: Set(None), + storage_policy_entities: Set(policy.id), + created_by: Set(Some(user.id)), + ..Default::default() + } + .insert(db) + .await?; + let file = cr::files::ActiveModel { + created_at: Set(now), + updated_at: Set(now), + r#type: Set(0), + name: Set("hello.txt".to_string()), + size: Set(128), + primary_entity: Set(Some(entity.id)), + is_symbolic: Set(false), + props: Set(None), + file_children: Set(Some(folder.id)), + storage_policy_files: Set(Some(policy.id)), + owner_id: Set(user.id), + ..Default::default() + } + .insert(db) + .await?; + cr::file_entities::ActiveModel { + file_id: Set(file.id), + entity_id: Set(entity.id), + } + .insert(db) + .await?; + cr::metadata::ActiveModel { + created_at: Set(now), + updated_at: Set(now), + deleted_at: Set(None), + name: Set("author".to_string()), + value: Set("Cloudreve".to_string()), + is_public: Set(true), + file_id: Set(file.id), + ..Default::default() + } + .insert(db) + .await?; + cr::shares::ActiveModel { + created_at: Set(now), + updated_at: Set(now), + deleted_at: Set(None), + password: Set(Some("share-password".to_string())), + views: Set(4), + downloads: Set(2), + expires: Set(None), + remain_downloads: Set(Some(3)), + file_shares: Set(Some(file.id)), + user_shares: Set(Some(user.id)), + props: Set(None), + ..Default::default() + } + .insert(db) + .await?; + Ok(()) + } +} diff --git a/src/migration/phases.rs b/src/migration/phases.rs new file mode 100644 index 0000000..929c3a5 --- /dev/null +++ b/src/migration/phases.rs @@ -0,0 +1,524 @@ +use super::*; +use sea_orm::DatabaseTransaction; + +pub(super) async fn migrate_policies( + transaction: &DatabaseTransaction, + source: &SourceData, + options: &MigrationOptions, + context: &mut MigrationContext, + report: &mut MigrationReport, +) -> Result<()> { + let mut default_assigned = false; + for policy in &source.policies { + let Some(driver_type) = map_driver_type(&policy.r#type).filter(|_| { + !source_settings(&policy.settings) + .get("encryption") + .and_then(Value::as_bool) + .unwrap_or(false) + }) else { + report.skipped += 1; + continue; + }; + let base_path = if policy.r#type == "local" { + options.local_base_path.clone() + } else { + String::new() + }; + let model = ad::storage_policies::ActiveModel { + name: Set(policy.name.clone()), + driver_type: Set(driver_type.to_string()), + endpoint: Set(policy.server.clone().unwrap_or_default()), + bucket: Set(policy.bucket_name.clone().unwrap_or_default()), + access_key: Set(policy.access_key.clone().unwrap_or_default()), + secret_key: Set(policy.secret_key.clone().unwrap_or_default()), + base_path: Set(base_path), + remote_node_id: Set(None), + max_file_size: Set(policy.max_size.unwrap_or(0)), + allowed_types: Set(allowed_types(policy)), + options: Set(policy_options(policy)), + is_default: Set(!default_assigned), + chunk_size: Set(chunk_size(policy)), + created_at: Set(policy.created_at), + updated_at: Set(policy.updated_at), + remote_storage_target_key: Set(None), + ..Default::default() + } + .insert(transaction) + .await + .wrap_err_with(|| format!("migrate storage policy {}", policy.id))?; + default_assigned = true; + context.policies.insert(policy.id, model.id); + report.migrated_policies += 1; + } + Ok(()) +} + +pub(super) async fn migrate_policy_groups( + transaction: &DatabaseTransaction, + source: &SourceData, + context: &mut MigrationContext, + report: &mut MigrationReport, +) -> Result<()> { + for group in &source.groups { + let target_group = ad::storage_policy_groups::ActiveModel { + name: Set(group.name.clone()), + description: Set(format!("Migrated from Cloudreve group {}", group.id)), + is_enabled: Set(true), + is_default: Set(false), + created_at: Set(group.created_at), + updated_at: Set(group.updated_at), + ..Default::default() + } + .insert(transaction) + .await + .wrap_err_with(|| format!("migrate Cloudreve group {}", group.id))?; + context.policy_groups.insert(group.id, target_group.id); + report.migrated_policy_groups += 1; + + if let Some(source_policy_id) = group.storage_policy_id + && let Some(target_policy_id) = context.policies.get(&source_policy_id) + { + ad::storage_policy_group_items::ActiveModel { + group_id: Set(target_group.id), + policy_id: Set(*target_policy_id), + priority: Set(0), + min_file_size: Set(0), + max_file_size: Set(0), + created_at: Set(group.created_at), + ..Default::default() + } + .insert(transaction) + .await + .wrap_err_with(|| format!("link Cloudreve group {} storage policy", group.id))?; + } + } + Ok(()) +} + +pub(super) async fn migrate_users( + transaction: &DatabaseTransaction, + source: &SourceData, + password_hash: &str, + context: &mut MigrationContext, + report: &mut MigrationReport, +) -> Result<()> { + let groups: HashMap = source + .groups + .iter() + .map(|group| (group.id, group)) + .collect(); + let mut used_usernames = HashSet::new(); + for user in &source.users { + let username = unique_username(&user.nick, user.id, &mut used_usernames); + let group = groups.get(&user.group_users).copied(); + let role = if group.is_some_and(group_is_admin) { + "admin" + } else { + "user" + }; + let status = if user.status == "active" && user.deleted_at.is_none() { + "active" + } else { + "disabled" + }; + let target = ad::users::ActiveModel { + username: Set(username.clone()), + email: Set(user.email.clone()), + password_hash: Set(password_hash.to_string()), + role: Set(role.to_string()), + status: Set(status.to_string()), + session_version: Set(1), + email_verified_at: Set((status == "active").then_some(user.created_at)), + pending_email: Set(None), + storage_used: Set(user.storage), + storage_quota: Set(group.and_then(|group| group.max_storage).unwrap_or(0)), + policy_group_id: Set(context.policy_groups.get(&user.group_users).copied()), + created_at: Set(user.created_at), + updated_at: Set(user.updated_at), + config: Set(user.settings.as_ref().map(Value::to_string)), + must_change_password: Set(true), + ..Default::default() + } + .insert(transaction) + .await + .wrap_err_with(|| format!("migrate Cloudreve user {}", user.id))?; + + let avatar = user.avatar.clone().unwrap_or_default(); + let avatar_source = if avatar.is_empty() { + "none" + } else if avatar.to_ascii_lowercase().contains("gravatar") { + "gravatar" + } else { + "upload" + }; + ad::user_profiles::ActiveModel { + user_id: Set(target.id), + display_name: Set(Some(user.nick.clone())), + wopi_user_info: Set(None), + avatar_source: Set(avatar_source.to_string()), + avatar_key: Set((!avatar.is_empty()).then_some(avatar)), + avatar_version: Set(0), + created_at: Set(user.created_at), + updated_at: Set(user.updated_at), + } + .insert(transaction) + .await + .wrap_err_with(|| format!("create profile for Cloudreve user {}", user.id))?; + + context.users.insert(user.id, target.id); + context.usernames.insert(user.id, username); + report.migrated_users += 1; + } + Ok(()) +} + +pub(super) async fn migrate_folders( + transaction: &DatabaseTransaction, + source: &SourceData, + context: &mut MigrationContext, + report: &mut MigrationReport, +) -> Result<()> { + let mut pending: Vec<&cr::files::Model> = source + .files + .iter() + .filter(|file| file.r#type == 1) + .collect(); + pending.sort_by_key(|folder| folder.id); + + while !pending.is_empty() { + let mut progress = false; + let mut next = Vec::new(); + for folder in pending { + let parent = folder.file_children; + if parent.is_some_and(|parent_id| { + source + .files + .iter() + .any(|file| file.id == parent_id && file.r#type == 1) + && !context.folders.contains_key(&parent_id) + }) { + next.push(folder); + continue; + } + let Some(owner_id) = context.users.get(&folder.owner_id).copied() else { + report.skipped += 1; + continue; + }; + let target = ad::folders::ActiveModel { + name: Set(folder.name.clone()), + parent_id: Set(parent.and_then(|id| context.folders.get(&id).copied())), + team_id: Set(None), + owner_user_id: Set(Some(owner_id)), + created_by_user_id: Set(Some(owner_id)), + created_by_username: Set(context + .usernames + .get(&folder.owner_id) + .cloned() + .unwrap_or_default()), + policy_id: Set(folder + .storage_policy_files + .and_then(|id| context.policies.get(&id).copied())), + created_at: Set(folder.created_at), + updated_at: Set(folder.updated_at), + deleted_at: Set(None), + is_locked: Set(false), + ..Default::default() + } + .insert(transaction) + .await + .wrap_err_with(|| format!("migrate Cloudreve folder {}", folder.id))?; + context.folders.insert(folder.id, target.id); + report.migrated_folders += 1; + progress = true; + } + if !progress && !next.is_empty() { + bail!("Cloudreve folder hierarchy contains a cycle or a missing parent"); + } + pending = next; + } + Ok(()) +} + +pub(super) async fn migrate_blobs( + transaction: &DatabaseTransaction, + source: &SourceData, + context: &mut MigrationContext, + report: &mut MigrationReport, +) -> Result<()> { + let associations = associations(source); + let entities: HashMap = source + .entities + .iter() + .map(|entity| (entity.id, entity)) + .collect(); + let mut reference_counts: HashMap = HashMap::new(); + let mut thumbnail_paths: HashMap = HashMap::new(); + for entity_ids in associations.values() { + let thumbnail_path = entity_ids.iter().find_map(|entity_id| { + entities + .get(entity_id) + .filter(|entity| entity.r#type == 1) + .map(|entity| entity.source.clone()) + }); + for entity_id in entity_ids { + if entities + .get(entity_id) + .is_some_and(|entity| entity.r#type == 0) + { + *reference_counts.entry(*entity_id).or_default() += 1; + if let Some(path) = &thumbnail_path { + thumbnail_paths + .entry(*entity_id) + .or_insert_with(|| path.clone()); + } + } + } + } + for entity in source.entities.iter().filter(|entity| entity.r#type == 0) { + let Some(policy_id) = context + .policies + .get(&entity.storage_policy_entities) + .copied() + else { + report.skipped += 1; + continue; + }; + let reference_count = reference_counts.get(&entity.id).copied().unwrap_or(1); + let thumbnail_path = thumbnail_paths.get(&entity.id).cloned(); + let target = ad::file_blobs::ActiveModel { + hash: Set(opaque_blob_key(entity.id)), + size: Set(entity.size), + policy_id: Set(policy_id), + storage_path: Set(entity.source.clone()), + thumbnail_path: Set(thumbnail_path), + thumbnail_processor: Set(None), + thumbnail_version: Set(None), + ref_count: Set(reference_count), + created_at: Set(entity.created_at), + updated_at: Set(entity.updated_at), + ..Default::default() + } + .insert(transaction) + .await + .wrap_err_with(|| format!("migrate Cloudreve entity {}", entity.id))?; + context.blobs.insert(entity.id, target.id); + report.migrated_blobs += 1; + } + Ok(()) +} + +pub(super) async fn migrate_files( + transaction: &DatabaseTransaction, + source: &SourceData, + context: &mut MigrationContext, + report: &mut MigrationReport, +) -> Result<()> { + let associations = associations(source); + let entities: HashMap = source + .entities + .iter() + .map(|entity| (entity.id, entity)) + .collect(); + let mut files: Vec<&cr::files::Model> = source + .files + .iter() + .filter(|file| file.r#type == 0) + .collect(); + files.sort_by_key(|file| file.id); + + for file in files { + if file.is_symbolic { + report.skipped += 1; + continue; + } + let Some(owner_id) = context.users.get(&file.owner_id).copied() else { + report.skipped += 1; + continue; + }; + let mut version_entities: Vec<&cr::entities::Model> = associations + .get(&file.id) + .into_iter() + .flatten() + .filter_map(|entity_id| entities.get(entity_id).copied()) + .filter(|entity| entity.r#type == 0 && context.blobs.contains_key(&entity.id)) + .collect(); + version_entities.sort_by_key(|entity| entity.created_at); + let primary_entity_id = file + .primary_entity + .filter(|id| context.blobs.contains_key(id)) + .or_else(|| version_entities.last().map(|entity| entity.id)); + let Some(primary_entity_id) = primary_entity_id else { + report.skipped += 1; + report + .warnings + .push(format!("file {} has no migratable version entity", file.id)); + continue; + }; + let blob_id = context.blobs[&primary_entity_id]; + let (mime_type, compound_extension, extension, category) = file_classification(&file.name); + let target = ad::files::ActiveModel { + name: Set(file.name.clone()), + folder_id: Set(file + .file_children + .and_then(|folder_id| context.folders.get(&folder_id).copied())), + team_id: Set(None), + blob_id: Set(blob_id), + size: Set(file.size), + owner_user_id: Set(Some(owner_id)), + created_by_user_id: Set(Some(owner_id)), + created_by_username: Set(context + .usernames + .get(&file.owner_id) + .cloned() + .unwrap_or_default()), + mime_type: Set(mime_type), + created_at: Set(file.created_at), + updated_at: Set(file.updated_at), + deleted_at: Set(None), + is_locked: Set(false), + extension: Set(extension), + compound_extension: Set(compound_extension), + file_category: Set(category), + ..Default::default() + } + .insert(transaction) + .await + .wrap_err_with(|| format!("migrate Cloudreve file {}", file.id))?; + context.files.insert(file.id, target.id); + report.migrated_files += 1; + + let historical: Vec<&cr::entities::Model> = version_entities + .into_iter() + .filter(|entity| entity.id != primary_entity_id) + .collect(); + for (index, entity) in historical.into_iter().enumerate() { + ad::file_versions::ActiveModel { + file_id: Set(target.id), + blob_id: Set(context.blobs[&entity.id]), + version: Set((index + 1) as i64), + size: Set(entity.size), + created_at: Set(entity.created_at), + ..Default::default() + } + .insert(transaction) + .await + .wrap_err_with(|| format!("migrate version {} for file {}", entity.id, file.id))?; + report.migrated_versions += 1; + } + } + Ok(()) +} + +pub(super) async fn migrate_metadata( + transaction: &DatabaseTransaction, + source: &SourceData, + context: &MigrationContext, + report: &mut MigrationReport, +) -> Result<()> { + for metadata in &source.metadata { + let Some(file_id) = context.files.get(&metadata.file_id).copied() else { + report.skipped += 1; + continue; + }; + let namespace = if metadata.is_public { + "cloudreve.public" + } else { + "cloudreve.private" + }; + ad::entity_properties::ActiveModel { + entity_type: Set("file".to_string()), + entity_id: Set(file_id), + namespace: Set(namespace.to_string()), + name: Set(metadata.name.clone()), + value: Set(Some(metadata.value.clone())), + ..Default::default() + } + .insert(transaction) + .await + .wrap_err_with(|| format!("migrate metadata {}", metadata.id))?; + report.migrated_properties += 1; + } + Ok(()) +} + +pub(super) async fn migrate_shares( + transaction: &DatabaseTransaction, + source: &SourceData, + context: &MigrationContext, + report: &mut MigrationReport, +) -> Result<()> { + let source_files: HashMap = + source.files.iter().map(|file| (file.id, file)).collect(); + for share in &source.shares { + let Some(source_user_id) = share.user_shares else { + report.skipped += 1; + continue; + }; + let Some(user_id) = context.users.get(&source_user_id).copied() else { + report.skipped += 1; + continue; + }; + let Some(source_file_id) = share.file_shares else { + report.skipped += 1; + continue; + }; + let source_file = source_files.get(&source_file_id).copied(); + let file_id = source_file + .filter(|file| file.r#type == 0) + .and_then(|_| context.files.get(&source_file_id).copied()); + let folder_id = source_file + .filter(|file| file.r#type == 1) + .and_then(|_| context.folders.get(&source_file_id).copied()); + if file_id.is_none() && folder_id.is_none() { + report.skipped += 1; + continue; + } + let password = match share.password.as_deref().filter(|value| !value.is_empty()) { + Some(password) => Some(hash_password(password)?), + None => None, + }; + let max_downloads = share + .remain_downloads + .map(|remaining| share.downloads.saturating_add(remaining)) + .unwrap_or(0); + ad::shares::ActiveModel { + token: Set(share_token(share.id)), + user_id: Set(user_id), + team_id: Set(None), + file_id: Set(file_id), + folder_id: Set(folder_id), + password: Set(password), + expires_at: Set(share.expires), + max_downloads: Set(max_downloads), + download_count: Set(share.downloads), + view_count: Set(share.views), + created_at: Set(share.created_at), + updated_at: Set(share.updated_at), + ..Default::default() + } + .insert(transaction) + .await + .wrap_err_with(|| format!("migrate share {}", share.id))?; + report.migrated_shares += 1; + } + Ok(()) +} + +fn associations(source: &SourceData) -> HashMap> { + let mut result: HashMap> = HashMap::new(); + for relation in &source.file_entities { + result + .entry(relation.file_id) + .or_default() + .push(relation.entity_id); + } + for file in &source.files { + if let Some(primary_entity) = file.primary_entity { + let values = result.entry(file.id).or_default(); + if !values.contains(&primary_entity) { + values.push(primary_entity); + } + } + } + result +} From 8110984d8f35168af9439edaefbdc0ef1d3418e6 Mon Sep 17 00:00:00 2001 From: Cg8 <5712.cg8@gmail.com> Date: Mon, 13 Jul 2026 19:31:45 +0800 Subject: [PATCH 2/7] feat: migrate Cloudreve links tags and task history --- Cargo.lock | 36 +- Cargo.toml | 3 + README.md | 27 +- docs/cloudreve-to-asterdrive-field-mapping.md | 518 +++++++++++ docs/migration-architecture.md | 814 ++++++++++++++++++ src/main.rs | 3 + src/migration/mod.rs | 289 ++++++- src/migration/phases.rs | 234 ++++- 8 files changed, 1903 insertions(+), 21 deletions(-) create mode 100644 docs/cloudreve-to-asterdrive-field-mapping.md create mode 100644 docs/migration-architecture.md diff --git a/Cargo.lock b/Cargo.lock index 058321e..e02e561 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -320,11 +320,13 @@ version = "0.0.0-alpha.0" dependencies = [ "argon2", "asterdrive-entities", + "base64", "chrono", "clap", "cloudreve-entities", "color-eyre", "crossterm 0.29.0", + "hmac 0.12.1", "mime_guess", "ratatui", "sea-orm", @@ -332,6 +334,7 @@ dependencies = [ "serde_json", "sha2 0.10.9", "tokio", + "urlencoding", "uuid", ] @@ -867,7 +870,7 @@ dependencies = [ "document-features", "mio", "parking_lot", - "rustix 1.1.4", + "rustix 1.1.3", "serde", "signal-hook", "signal-hook-mio", @@ -1427,7 +1430,16 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4aaa26c720c68b866f2c96ef5c1264b3e6f473fe5d4ce61cd44bbe913e553018" dependencies = [ - "hmac", + "hmac 0.13.0", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", ] [[package]] @@ -1771,9 +1783,9 @@ checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] name = "linux-raw-sys" -version = "0.12.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" [[package]] name = "litemap" @@ -2672,14 +2684,14 @@ dependencies = [ [[package]] name = "rustix" -version = "1.1.4" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" dependencies = [ "bitflags 2.13.0", "errno", "libc", - "linux-raw-sys 0.12.1", + "linux-raw-sys 0.11.0", "windows-sys 0.61.2", ] @@ -3195,7 +3207,7 @@ dependencies = [ "futures-util", "hex", "hkdf", - "hmac", + "hmac 0.13.0", "itoa", "log", "md-5", @@ -3345,7 +3357,7 @@ checksum = "9048a889effe34a5cddee0af7f53285198b16dca3be510858d38dfdb3e62a04e" dependencies = [ "bitflags 2.13.0", "parking_lot", - "rustix 1.1.4", + "rustix 1.1.3", "signal-hook", "windows-sys 0.61.2", ] @@ -3753,6 +3765,12 @@ dependencies = [ "serde", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf8_iter" version = "1.0.4" diff --git a/Cargo.toml b/Cargo.toml index 42cfa28..5d64e98 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,11 +42,13 @@ full = ["cli", "tui"] [dependencies] asterdrive-entities = { path = "crates/asterdrive-entities" } argon2 = "0.5.3" +base64 = "0.22" chrono = { version = "0.4.45", default-features = false, features = ["clock", "serde"] } clap = { version = "4", features = ["derive", "env"] } cloudreve-entities = { path = "crates/cloudreve-entities" } color-eyre = "0.6.5" crossterm = "0.29.0" +hmac = "0.12" mime_guess = "2" ratatui = { version = "0.30.2", optional = true, features = ["serde"] } sea-orm = { version = "2.0.0-rc.42", features = ["sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", "macros", "runtime-tokio-rustls"] } @@ -54,6 +56,7 @@ serde = { version = "1.0.228", features = ["derive"] } serde_json = "1" sha2 = "0.10" tokio = { version = "1", features = ["macros", "rt-multi-thread"] } +urlencoding = "2" uuid = { version = "1", features = ["v4"] } [lib] diff --git a/README.md b/README.md index 420a13b..c748dba 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,10 @@ # AsterDriveMigration +Design documents: + +- [Cloudreve and AsterDrive field mapping](docs/cloudreve-to-asterdrive-field-mapping.md) +- [Migration architecture](docs/migration-architecture.md) + Database migration tool for moving a Cloudreve v4 installation to AsterDrive (AD). ## Supported data @@ -8,9 +13,13 @@ Database migration tool for moving a Cloudreve v4 installation to AsterDrive (AD - users and profiles, including quotas and disabled status - local, S3, OSS, KS3, OBS and Tencent COS storage policies - folders, files, physical entities/blobs and version history -- file metadata and public shares +- file/folder metadata, including Cloudreve v4 `tag:*` metadata as AD tags +- public shares and regenerated AD v2 direct-link mappings +- Cloudreve background tasks as non-executable terminal history + +Cloudreve password hashes are not compatible with AD. The migration assigns the supplied temporary password to every migrated user and sets `must_change_password = true`. Passkeys, WebDAV credentials, two-factor secrets, OAuth grants, sessions and filesystem events are not migrated. -Cloudreve password hashes are not compatible with AD. The migration assigns the supplied temporary password to every migrated user and sets `must_change_password = true`. Passkeys, WebDAV credentials, two-factor secrets, OAuth grants, sessions, tasks and filesystem events are not migrated. +Cloudreve tasks are preserved only as terminal AD `system_runtime` history. Completed/error/canceled statuses are mapped to terminal AD statuses; queued/processing/suspending tasks are archived as canceled and are never resumed. The tool reuses existing storage objects. For local storage, pass the directory from which Cloudreve entity paths should be resolved. Object-storage policies keep their existing bucket, endpoint, credentials and object keys. @@ -34,6 +43,7 @@ The URLs can also be provided through `CLOUDREVE_DATABASE_URL` and `ASTERDRIVE_D ```powershell $env:ASTER_MIGRATION_DEFAULT_PASSWORD = "replace-with-a-strong-temporary-password" +$env:ASTER_DIRECT_LINK_SECRET = "use-the-same-auth.direct_link_secret-as-ad" cargo run -- migrate ` --source-url "sqlite://C:/cloudreve/cloudreve.db?mode=ro" ` @@ -41,6 +51,19 @@ cargo run -- migrate ` --local-base-path "C:/cloudreve" ``` +`ASTER_DIRECT_LINK_SECRET` is optional only when there are no direct links to regenerate. When supplied, each active Cloudreve `direct_links` row is mapped to a newly signed AD `/d/v2...` URL and stored under the target file's `cloudreve.direct_links` properties. Existing Cloudreve `/f/...` URLs cannot remain valid because the URL path, ID and signing algorithms differ. + Use `--dry-run` to perform preflight checks without writing the target. The target core tables must be empty by default. `--allow-non-empty-target` disables that guard but may still fail on unique values or conflicting data. Cloudreve Qiniu, Upyun, remote-node, OneDrive and encrypted storage policies cannot be reused safely by AD. The migration stops when they are present. `--skip-unsupported-policies` explicitly omits those policies and all dependent files. + +## Verification before push + +Every migration feature must include focused tests plus end-to-end coverage where it writes database state. Before pushing, run and require all of these commands to pass: + +```powershell +cargo fmt -- --check +cargo check --offline +cargo clippy --offline --all-targets --all-features -- -D warnings +cargo test --offline --all-targets --all-features +``` diff --git a/docs/cloudreve-to-asterdrive-field-mapping.md b/docs/cloudreve-to-asterdrive-field-mapping.md new file mode 100644 index 0000000..21b951a --- /dev/null +++ b/docs/cloudreve-to-asterdrive-field-mapping.md @@ -0,0 +1,518 @@ +# Cloudreve v4 到 AsterDrive 字段映射与差异 + +本文档基于以下代码快照整理： + +- Cloudreve v4：[`cloudreve/cloudreve`](https://github.com/cloudreve/cloudreve) ，Ent 实际表定义位于 `ent/migrate/schema.go` +- AsterDrive：[`AsterCommunity/AsterDrive`](https://github.com/AsterCommunity/AsterDrive)，SeaORM Entity 位于 `src/entities/` +- 迁移工具：当前仓库 `crates/cloudreve-entities` 和 `crates/asterdrive-entities` + +本文档的目标不是假设所有数据都应原样复制，而是明确： + +1. 两边分别有什么表和字段。 +2. 哪些字段可以直接映射。 +3. 哪些字段需要转换、拆分或生成。 +4. 哪些数据无法安全迁移，需要重新配置或由用户决策。 + +## 1. 总体结构差异 + +| 主题 | Cloudreve v4 | AsterDrive | 迁移影响 | +|---|---|---|---| +| 表数量 | 17 张业务表 | 45 张业务表 | AD 对认证、MFA、团队、任务、远端节点和存储凭据拆分更细 | +| 文件树 | `files` 同时保存文件和目录，通过 `type` 区分 | `files` 与 `folders` 分表 | 必须拆分 Cloudreve `files` | +| 物理对象 | `entities` 保存物理对象，`file_entities` 关联逻辑文件 | `file_blobs` 保存物理对象，`file_versions` 保存历史版本 | 需要重建当前 blob 和历史版本关系 | +| 用户组 | `groups` 同时承载权限、配额和默认存储策略 | `storage_policy_groups` 只负责存储路由；团队使用 `teams` | Cloudreve 用户组不应直接迁移为 AD 团队 | +| 用户密码 | Cloudreve SHA256/SHA1/旧 MD5 加盐格式 | AD Argon2 PHC 字符串 | 不能直接复用，必须设置临时密码或走重置流程 | +| 分享标识 | 分享 URL token 由 ID 和 Cloudreve HashID 密钥计算 | `shares.token` 为独立持久化 token | 必须生成新 token，旧分享 URL 不能保持不变 | +| 存储策略 | 多个 Cloudreve 专用驱动，配置集中在一张表 | 驱动类型更少，OAuth 凭据和应用配置拆表 | 仅部分驱动可直接复用 | +| 系统配置 | `settings(name, value)` 键值表 | `system_config` 包含类型、命名空间、可见性和敏感标记 | 不能盲目全量复制，需要配置键映射表 | +| 软删除 | 大部分 Cloudreve 表有 `deleted_at`，但当前 v4 `files` 没有 | AD 仅部分资源有 `deleted_at` | 需要决定是否包含已删除数据 | +| 认证数据 | OAuth Client/Grant、Passkey、TOTP secret 混合保存 | 外部认证、Passkey、MFA 流程分别建模 | 凭据结构和加密方式不兼容，不应直接复制 | + +## 2. 迁移状态说明 + +| 标记 | 含义 | +|---|---| +| 直接 | 类型和语义基本一致，只需替换外键 ID | +| 转换 | 可以迁移，但需要格式、枚举、类型或语义转换 | +| 生成 | AD 必填，但 Cloudreve 没有，需要迁移工具生成 | +| 拆分 | 一个 Cloudreve 记录需要写入多张 AD 表 | +| 合并 | 多张 Cloudreve 表共同生成一张 AD 表或一组记录 | +| 跳过 | 运行时或临时数据，不建议迁移 | +| 不兼容 | 凭据、加密或协议语义不同，不能安全直接迁移 | +| 决策 | 没有唯一正确映射，需要迁移前由管理员确定 | + +## 3. 用户组映射 + +Cloudreve `groups` 更接近“用户套餐/权限组”，而不是 AD 的协作团队。当前建议映射为： + +- `groups` -> `storage_policy_groups` +- `groups.storage_policy_id` -> `storage_policy_group_items.policy_id` +- `users.group_users` -> `users.policy_group_id` +- `groups.permissions` 中的管理员位 -> `users.role = admin` + +### `groups` -> `storage_policy_groups` + +| Cloudreve 字段 | AD 字段 | 状态 | 规则或差异 | +|---|---|---|---| +| `id` | 新生成 `storage_policy_groups.id` | 转换 | 保存旧 ID -> 新 ID 映射，不建议强行复用 ID | +| `name` | `name` | 直接 | 注意目标库可能已有同名组 | +| `created_at` | `created_at` | 直接 | 保留原时间 | +| `updated_at` | `updated_at` | 直接 | 保留原时间 | +| `deleted_at` | 无直接字段 | 转换 | 默认跳过已删除组；若包含则建议 `is_enabled = false` | +| `max_storage` | `users.storage_quota` | 转换 | AD 策略组自身没有配额，需要下放到组内每个用户 | +| `speed_limit` | 无直接字段 | 决策 | AD 当前用户/组模型没有等价限速列，可写入用户 `config` 或忽略 | +| `permissions` | `users.role`，部分可能进入 `config` | 转换 | 管理员位映射为 `admin`；其他 Cloudreve 权限没有一一对应字段 | +| `settings` | 无直接字段 | 决策 | 压缩、离线下载、回收站配置可选择写入 AD 配置扩展，不宜直接复制 | +| `storage_policy_id` | `storage_policy_group_items.policy_id` | 拆分 | 先迁移存储策略，再创建 group item | +| 无 | `description` | 生成 | 可写 `Migrated from Cloudreve group ` | +| 无 | `is_enabled` | 生成 | 活跃组为 `true`，已删除组为 `false` | +| 无 | `is_default` | 决策 | 选择 Cloudreve 默认用户组对应的 AD 默认策略组 | + +### `groups.storage_policy_id` -> `storage_policy_group_items` + +| AD 字段 | 来源 | 规则 | +|---|---|---| +| `id` | 生成 | AD 自增 ID | +| `group_id` | Cloudreve group ID 映射 | 指向新 `storage_policy_groups.id` | +| `policy_id` | `groups.storage_policy_id` 映射 | 指向新 `storage_policies.id` | +| `priority` | 生成 | 单策略组可设为 `0` | +| `min_file_size` | 生成 | `0` | +| `max_file_size` | 生成 | `0` 表示不额外限制 | +| `created_at` | `groups.created_at` | 保留原时间 | + +## 4. 用户与资料映射 + +Cloudreve 一条 `users` 记录需要拆成 AD 的 `users` 和 `user_profiles`。 + +### `users` -> AD `users` + +| Cloudreve 字段 | AD 字段 | 状态 | 规则或差异 | +|---|---|---|---| +| `id` | 新生成 `id` | 转换 | 保存用户 ID 映射，所有 owner/user 外键都要替换 | +| `email` | `email` | 直接 | AD 同样要求唯一；迁移前检查重复和空值 | +| `nick` | `username` | 转换 | AD `username` 唯一且最长 64；重名时追加 Cloudreve ID | +| `nick` | `user_profiles.display_name` | 拆分 | 原昵称同时作为展示名 | +| `password` | `password_hash` | 不兼容 | Cloudreve SHA/MD5 格式不能被 AD Argon2 验证 | +| `status=active` | `status=active` | 转换 | 直接映射 | +| `status=inactive/manual_banned/sys_banned` | `status=disabled` | 转换 | AD 当前只有 `active`/`disabled` | +| `storage` | `storage_used` | 直接 | 保留已用空间统计，迁移后应重新校验 blob 汇总 | +| `group_users -> groups.max_storage` | `storage_quota` | 转换 | 配额来自用户所属 Cloudreve 组 | +| `group_users` | `policy_group_id` | 转换 | 使用 group ID 映射 | +| `settings` | `config` | 转换 | 可保存原 JSON，但 AD 不会自动理解 Cloudreve 键 | +| `two_factor_secret` | 无安全直接映射 | 不兼容 | AD 需要加密后的 `mfa_factors.secret_ciphertext`，必须重新绑定 MFA | +| `avatar` | `user_profiles.avatar_source/avatar_key` | 转换 | 根据值判断 `none`、`gravatar` 或 `upload` | +| `created_at` | `created_at` | 直接 | 保留原时间 | +| `updated_at` | `updated_at` | 直接 | 保留原时间 | +| `deleted_at` | `status=disabled` 或跳过 | 决策 | 默认跳过软删除用户；包含时应禁用 | +| `groups.permissions` 管理员位 | `role` | 转换 | 管理员组用户 -> `admin`，其他 -> `user` | +| 无 | `session_version` | 生成 | 建议初始化为 `1` | +| 无 | `email_verified_at` | 生成/决策 | Cloudreve 没有等价字段；可对活跃用户设为 `created_at`，或要求重新验证 | +| 无 | `pending_email` | 生成 | `NULL` | +| 无 | `must_change_password` | 生成 | 设置为 `true` | + +### AD `user_profiles` 字段 + +| AD 字段 | Cloudreve 来源 | 规则 | +|---|---|---| +| `user_id` | 新 AD 用户 ID | 与 `users.id` 一对一 | +| `display_name` | `users.nick` | 保留原昵称 | +| `wopi_user_info` | 无 | `NULL` | +| `avatar_source` | `users.avatar` | `none` / `gravatar` / `upload` | +| `avatar_key` | `users.avatar` | 非空时保存原值；之后可能需要单独搬迁头像文件 | +| `avatar_version` | 无 | `0` | +| `created_at` | `users.created_at` | 保留 | +| `updated_at` | `users.updated_at` | 保留 | + +## 5. 存储策略映射 + +### 驱动类型 + +| Cloudreve `type` | AD `driver_type` | 状态 | 说明 | +|---|---|---|---| +| `local` | `local` | 可迁移 | `entity.source` 作为 `storage_path`，需指定正确 AD `base_path` | +| `s3` | `s3` | 可迁移 | 复用 endpoint、bucket、key 和 secret | +| `oss` | `s3` | 需验证 | 通过 S3 兼容接口接入；需验证 endpoint 和 path-style | +| `ks3` | `s3` | 需验证 | 通过 S3 兼容接口接入 | +| `obs` | `s3` | 需验证 | 通过 S3 兼容接口接入；签名/endpoint 兼容性需实测 | +| `cos` | `tencent_cos` | 可迁移 | 使用 AD 原生腾讯 COS 驱动 | +| `qiniu` | 无 | 不兼容 | AD 当前没有七牛驱动，也不能保证 S3 兼容 | +| `upyun` | 无 | 不兼容 | AD 当前没有又拍云驱动 | +| `onedrive` | `onedrive` 但凭据结构不同 | 不兼容 | AD 将 OAuth token 拆到 credential/config 表，不能只复制策略字段 | +| `remote` | `remote` 但节点协议不同 | 不兼容 | Cloudreve Slave 与 AD follower 协议完全不同 | + +启用了 Cloudreve `settings.encryption=true` 的策略不能只迁移数据库元数据。Cloudreve 对象内容仍是 Cloudreve 加密格式，AD 无法直接读取，必须先通过 Cloudreve 解密导出再上传到 AD。 + +### `storage_policies` 字段 + +| Cloudreve 字段 | AD 字段 | 状态 | 规则或差异 | +|---|---|---|---| +| `id` | 新生成 `id` | 转换 | 保存策略 ID 映射 | +| `name` | `name` | 直接 | 保留名称 | +| `type` | `driver_type` | 转换 | 按上表映射 | +| `server` | `endpoint` | 转换 | `NULL` 转为空字符串；不同驱动 endpoint 语义需核实 | +| `bucket_name` | `bucket` | 转换 | `NULL` 转为空字符串 | +| `access_key` | `access_key` | 直接/敏感 | 仅对兼容驱动复制；不得写日志 | +| `secret_key` | `secret_key` | 直接/敏感 | 仅对兼容驱动复制；不得写日志 | +| `max_size` | `max_file_size` | 转换 | Cloudreve `NULL` -> AD `0`，两边都表示无限制时才成立 | +| `settings.file_type` | `allowed_types` | 转换 | 保存为 JSON 数组；还需处理 deny-list 语义 | +| `settings.chunk_size` | `chunk_size` | 转换 | 保留字节数；`0` 表示单次上传 | +| `settings.s3_path_style` | `options.s3_path_style` | 转换 | 保留布尔值 | +| `settings.relay` | `options.object_storage_upload_strategy` | 转换 | `true` 倾向 `relay_stream`；否则可考虑 `presigned` | +| `is_private` | 无单独字段 | 合并 | AD 下载策略由 `options` 控制，不应机械复制 | +| `dir_name_rule` | 无直接字段 | 决策 | AD 对象 key 规则不同；旧对象迁移时保留现有 `entity.source` | +| `file_name_rule` | 无直接字段 | 决策 | 同上 | +| `settings` 其他键 | `options.cloudreve_source` 或忽略 | 决策 | 可存档原 JSON，但 AD 不会自动使用未知键 | +| `node_id` | `remote_node_id` | 不兼容 | Cloudreve node 不能直接变成 AD managed follower | +| `created_at` | `created_at` | 直接 | 保留 | +| `updated_at` | `updated_at` | 直接 | 保留 | +| `deleted_at` | 无 | 转换 | 默认跳过已删除策略 | +| 无 | `base_path` | 生成/配置 | 本地策略必须由管理员提供 Cloudreve 数据根目录 | +| 无 | `remote_storage_target_key` | 生成 | 普通策略为 `NULL` | +| 无 | `is_default` | 决策 | 选择一个迁移策略作为默认策略 | + +## 6. 文件树拆分 + +Cloudreve `files.type`： + +| 值 | 含义 | AD 目标 | +|---|---|---| +| `0` | 文件 | `files` | +| `1` | 目录 | `folders` | + +### Cloudreve 目录记录 -> AD `folders` + +| Cloudreve `files` 字段 | AD `folders` 字段 | 状态 | 规则或差异 | +|---|---|---|---| +| `id` | 新生成 `id` | 转换 | 保存 folder ID 映射 | +| `type=1` | 决定目标表 | 拆分 | 不写入 AD `files` | +| `name` | `name` | 直接 | 保留 | +| `file_children` | `parent_id` | 转换 | Cloudreve 字段名实际表示父目录 ID | +| `owner_id` | `owner_user_id` | 转换 | 使用用户 ID 映射 | +| `owner_id` | `created_by_user_id` | 生成/转换 | Cloudreve 没有独立创建者时使用 owner | +| `owner_id -> users.nick` | `created_by_username` | 生成 | 保存迁移后的 username | +| `storage_policy_files` | `policy_id` | 转换 | 使用策略 ID 映射 | +| `created_at` | `created_at` | 直接 | 保留 | +| `updated_at` | `updated_at` | 直接 | 保留 | +| `props` | 无直接字段 | 决策 | 可按需要拆到 `entity_properties` | +| `size` | 无 | 忽略 | 目录大小不写入 AD folder | +| `primary_entity` | 无 | 忽略 | 目录不应有主实体 | +| `is_symbolic` | 无 | 不兼容 | Cloudreve 占位/符号对象需要单独处理 | +| 无 | `team_id` | 生成 | 个人空间迁移为 `NULL`；若要转团队需另行规划 | +| 无 | `deleted_at` | 生成/决策 | 当前 Cloudreve v4 `files` 无此字段，通常为 `NULL` | +| 无 | `is_locked` | 生成 | `false` | + +### Cloudreve 文件记录 -> AD `files` + +| Cloudreve `files` 字段 | AD `files` 字段 | 状态 | 规则或差异 | +|---|---|---|---| +| `id` | 新生成 `id` | 转换 | 保存 file ID 映射 | +| `type=0` | 决定目标表 | 拆分 | 不写入 AD `folders` | +| `name` | `name` | 直接 | 保留 | +| `file_children` | `folder_id` | 转换 | 使用 folder ID 映射 | +| `owner_id` | `owner_user_id` | 转换 | 使用用户 ID 映射 | +| `owner_id` | `created_by_user_id` | 生成/转换 | 没有独立创建者时使用 owner | +| `owner_id -> username` | `created_by_username` | 生成 | 保存迁移后的 AD username | +| `primary_entity` | `blob_id` | 转换 | 先将主 entity 迁移为 `file_blobs` | +| `size` | `size` | 直接 | 建议同时与主 entity.size 校验 | +| `created_at` | `created_at` | 直接 | 保留 | +| `updated_at` | `updated_at` | 直接 | 保留 | +| `props` | 无直接字段 | 决策 | 可存入 `entity_properties` | +| `storage_policy_files` | 无直接字段 | 间接 | 文件实际策略由 `blob_id -> file_blobs.policy_id` 决定 | +| `is_symbolic=true` | 无直接模型 | 不兼容 | 默认跳过；若是占位文件需先在 Cloudreve 物化 | +| 无 | `mime_type` | 生成 | 根据文件名扩展名推断，建议迁移后重新扫描 | +| 无 | `extension` | 生成 | 小写末级扩展名 | +| 无 | `compound_extension` | 生成 | 如 `tar.gz` | +| 无 | `file_category` | 生成 | image/video/audio/document 等分类 | +| 无 | `team_id` | 生成 | 个人空间为 `NULL` | +| 无 | `deleted_at` | 生成 | 当前 Cloudreve v4 文件表没有该字段，通常为 `NULL` | +| 无 | `is_locked` | 生成 | `false`；Cloudreve 运行时锁不迁移 | + +## 7. 物理对象、当前版本与历史版本 + +Cloudreve 使用： + +- `entities`：物理对象记录 +- `file_entities`：逻辑文件与物理对象多对多关系 +- `files.primary_entity`：当前使用的物理对象 +- `entities.type=0`：文件版本 +- `entities.type=1`：缩略图 +- `entities.type=2`：Live Photo 关联对象 + +AD 使用： + +- `file_blobs`：物理对象 +- `files.blob_id`：当前 blob +- `file_versions`：历史 blob +- `file_blobs.thumbnail_path`：缩略图路径 + +### `entities` -> `file_blobs` + +| Cloudreve 字段 | AD 字段 | 状态 | 规则或差异 | +|---|---|---|---| +| `id` | 新生成 `id` | 转换 | 保存 entity ID -> blob ID 映射 | +| `type=0` | 创建 `file_blobs` | 转换 | 普通版本实体 | +| `type=1` | `thumbnail_path` | 合并 | 找到同一文件的缩略图 entity，将其 `source` 写入主 blob | +| `type=2` | 无直接字段 | 决策 | 可作为额外 blob + property，当前 AD 无 Live Photo 专用关系 | +| `source` | `storage_path` | 直接/关键 | 复用旧对象时必须原样保留 | +| `size` | `size` | 直接 | 保留 | +| `storage_policy_entities` | `policy_id` | 转换 | 使用策略 ID 映射 | +| `reference_count` | `ref_count` | 校验后转换 | 建议根据 `file_entities` 重新计算，不完全信任旧计数 | +| `created_at` | `created_at` | 直接 | 保留 | +| `updated_at` | `updated_at` | 直接 | 保留 | +| `deleted_at` | 无直接字段 | 决策 | 默认跳过已删除实体 | +| `upload_session_id` | 无 | 跳过 | 历史上传会话不应迁移 | +| `recycle_options` | 无直接字段 | 决策 | 回收/恢复信息可归档到 property，但 AD 不会自动使用 | +| `created_by` | 无 blob 创建者字段 | 间接 | 文件记录仍保存 owner/creator | +| 无 | `hash` | 生成 | Cloudreve 没有可靠内容 SHA256；应生成非 SHA256 形态的 opaque key，例如 `cloudreve-` | +| 无 | `thumbnail_processor` | 生成 | `NULL`，等待 AD 重新处理 | +| 无 | `thumbnail_version` | 生成 | `NULL` | + +不能把 `SHA256(policy_id + source + size)` 伪装成内容哈希。AD 会将 64 位十六进制值视为真实内容 SHA256，在校验或存储迁移时会产生错误语义。 + +### `file_entities` 和 `primary_entity` -> `file_versions` + +| Cloudreve 来源 | AD 字段 | 规则 | +|---|---|---| +| `file_entities.file_id` | `file_versions.file_id` | 使用逻辑文件 ID 映射 | +| `file_entities.entity_id` | `file_versions.blob_id` | 使用 entity -> blob 映射 | +| `files.primary_entity` | `files.blob_id` | 当前版本，不写入历史版本表 | +| 非 primary 的 `entity.type=0` | 一条 `file_versions` | 按创建时间升序编号 | +| `entities.size` | `file_versions.size` | 保留 | +| `entities.created_at` | `file_versions.created_at` | 保留 | +| 无明确版本号 | `file_versions.version` | 按历史 entity 创建时间生成 `1..N` | + +## 8. 元数据映射 + +### `metadata` -> `entity_properties` + +| Cloudreve 字段 | AD 字段 | 状态 | 规则 | +|---|---|---|---| +| `id` | 新生成 `id` | 转换 | 无需保留原 ID | +| `file_id` | `entity_id` | 转换 | 根据 Cloudreve `files.type` 使用 AD file 或 folder ID 映射 | +| `files.type` | `entity_type` | 转换 | `type=0 -> file`，`type=1 -> folder` | +| `is_public=true` | `namespace` | 转换 | `cloudreve.public` | +| `is_public=false` | `namespace` | 转换 | `cloudreve.private` | +| `name` | `name` | 直接 | 保留 | +| `value` | `value` | 直接 | 转为 `Some(value)` | +| `created_at` | 无 | 差异 | AD `entity_properties` 当前没有时间字段 | +| `updated_at` | 无 | 差异 | 无法保留 | +| `deleted_at` | 无 | 转换 | 默认跳过已删除元数据 | + +### Cloudreve v4 标签 -> AD `tags` + `entity_properties` + +Cloudreve v4 确实有标签功能，但没有独立 `tags` 表。标签保存在 `metadata.name = tag:<标签名>` 中，`metadata.value` 保存可选颜色。AD 使用 `tags` 保存标签定义，并使用 `entity_properties(namespace=system.tags, name=)` 关联文件或目录。 + +| Cloudreve 来源 | AD 目标 | 迁移规则 | +|---|---|---| +| `metadata.name = tag:` | `tags.name` | 去掉 `tag:`；去除首尾空白；最长保留 64 个字符 | +| 标签名小写值 | `tags.normalized_name` | 按 AD 当前规则 `trim + lowercase`，并按用户个人空间去重 | +| `metadata.value` | `tags.color` | `#RRGGBB` 直接规范化；`#RGB` 展开；空值或非法值使用 `#3b82f6` | +| `files.owner_id` | `tags.owner_user_id` | 使用用户 ID 映射，`scope_type=personal`，`team_id=NULL` | +| `metadata.created_at/updated_at` | `tags.created_at/updated_at` | 创建标签定义时保留首条标签元数据时间 | +| 目标 file/folder ID | `entity_properties.entity_id` | 根据源 `files.type` 关联文件或目录 | +| 无 | `entity_properties.namespace` | 固定为 AD 原生标签命名空间 `system.tags` | +| 新 AD tag ID | `entity_properties.name` | 十进制字符串；`value=NULL` | + +同一用户下大小写不同但规范化名称相同的 Cloudreve 标签会合并为一个 AD 标签定义。Cloudreve v3 曾有独立标签模型，但 Cloudreve 官方 v3 -> v4 迁移阶段没有对应标签迁移步骤；若数据仍停留在 v3 表中，需要直接读取 v3 数据库，不能从 v4 `metadata` 反推出已丢失记录。 + +Cloudreve `files.props` 是另一组 JSON 属性，不在 `metadata` 表中。是否拆成多个 `entity_properties` 需要先定义允许迁移的键，避免将 Cloudreve 内部状态直接暴露给 AD。 + +## 9. 分享映射 + +### `shares` -> AD `shares` + +| Cloudreve 字段 | AD 字段 | 状态 | 规则或差异 | +|---|---|---|---| +| `id` | 新生成 `id` | 转换 | 保存旧 ID仅用于报告 | +| 无持久化 token | `token` | 生成 | 生成新唯一 token，旧 URL 会失效 | +| `user_shares` | `user_id` | 转换 | 使用用户 ID 映射 | +| `file_shares` 指向文件 | `file_id` | 转换 | 根据源 `files.type=0` 判断 | +| `file_shares` 指向目录 | `folder_id` | 转换 | 根据源 `files.type=1` 判断 | +| `password` | `password` | 转换 | Cloudreve 保存明文分享密码；AD 必须保存 Argon2 hash | +| `expires` | `expires_at` | 直接 | 保留 | +| `downloads` | `download_count` | 直接 | 保留已下载次数 | +| `views` | `view_count` | 直接 | 保留浏览次数 | +| `remain_downloads` | `max_downloads` | 转换 | 有剩余次数时：`downloads + remain_downloads`；`NULL` 通常表示无限制，映射为 `0` | +| `props` | 无直接字段 | 决策 | Cloudreve 分享展示/权限属性需逐键分析 | +| `created_at` | `created_at` | 直接 | 保留 | +| `updated_at` | `updated_at` | 直接 | 保留 | +| `deleted_at` | 无 | 转换 | 默认跳过已删除分享 | +| 无 | `team_id` | 生成 | 个人分享为 `NULL` | + +### `direct_links` -> AD v2 直链 + 迁移映射属性 + +Cloudreve 直链并非“完全不能迁移”，但不能原样复制。Cloudreve 表只保存 `direct_link.id/name/downloads/speed/file_id`，旧 URL `/f/{hashid}/{name}` 中的 HashID 是运行时由 `[direct_link.id, SourceLinkID]` 和 Cloudreve salt 计算的。AD 没有 `direct_links` 表，而是使用 `auth.direct_link_secret` 对目标 file ID 和个人/团队 scope 做 HMAC-SHA256，生成 `/d/v2.{base62(file_id)}.{base64url_hmac}/{filename}`。 + +| Cloudreve 字段/语义 | AD 迁移结果 | 差异 | +|---|---|---| +| `file_id` | 映射后的 AD file ID | 文件必须已成功迁移 | +| `id` | `entity_properties.name` | 保存源 direct-link ID，便于查回映射 | +| `name` | 映射 JSON 的 `source_name` | AD URL 必须使用目标文件真实名称，不能继续使用任意旧 link name | +| `downloads` | 映射 JSON 的 `source_downloads` | AD 无持久化直链行，不能继续累计每条旧链接的计数 | +| `speed` | 映射 JSON 的 `source_speed_limit` | AD 没有每条直链限速字段，仅作历史存档 | +| Cloudreve HashID URL | 新 AD v2 URL | 通过 `--direct-link-secret` / `ASTER_DIRECT_LINK_SECRET` 重新签发 | +| 无 | `entity_properties.namespace` | `cloudreve.direct_links` | + +多个指向同一文件的 Cloudreve 直链会得到同一个确定性 AD token，但每个源 `direct_link.id` 都保留一条映射属性。旧 `/f/...` URL、逐条撤销语义、逐条下载计数和限速无法自动延续；若必须保留旧 URL，需要额外实现兼容重定向路由和持久化映射表。 + +即使使用 `--include-deleted`，已软删除的 Cloudreve direct-link 也不会重新签发，以免把已经撤销的公开入口重新激活。 + +### `tasks` -> AD 终态历史归档 + +Cloudreve 任务不能恢复到 AD 执行器中，但可以保存为不可执行的历史记录。迁移使用 AD `kind=system_runtime` 的合法 payload，并把 Cloudreve 原始字段保存在 `runtime_json`；所有记录均写成终态，`lease_expires_at=NULL`，不会被 worker 领取。 + +| Cloudreve `status` | AD `status` | 规则 | +|---|---|---| +| `completed` | `succeeded` | 保存为已完成历史 | +| `error` | `failed` | `failure_can_retry=false`，不允许在 AD 重试源任务 | +| `canceled` | `canceled` | 保存原终态 | +| `queued` | `canceled` | 仅归档，明确不恢复执行 | +| `processing` | `canceled` | 仅归档，明确不恢复执行 | +| `suspending` | `canceled` | 仅归档，明确不恢复执行 | + +`runtime_json` 保存源 task ID、type、status、public_state、private_state、correlation_id、deleted_at 和 `archived_without_resume=true`。这保留了排查历史，但不承诺 Cloudreve payload 能被 AD UI 或执行器理解。 + +## 10. 不能原样迁移的 Cloudreve 表 + +| Cloudreve 表 | 最接近的 AD 表 | 建议 | 原因 | +|---|---|---|---| +| `dav_accounts` | `webdav_accounts` | 不直接迁移 | URI/root 语义不同；Cloudreve 密码不是 AD Argon2 hash；options 位集需重建 | +| `direct_links` | AD stateless direct link + `entity_properties` | 重新签发 | 不能复用旧 HashID URL；提供 AD direct-link secret 后生成新 v2 URL | +| `fs_events` | 无持久化对应表 | 跳过 | 事件订阅属于运行时状态，subscriber UUID 对 AD 无意义 | +| `nodes` | `managed_followers` / `remote_storage_targets` | 不直接迁移 | Cloudreve master/slave 协议与 AD primary/follower 协议不同，密钥也不能复用 | +| `oauth_clients` | `external_auth_providers` 并不等价 | 跳过或导出报告 | Cloudreve 表是“第三方应用访问 Cloudreve”，AD external provider 是“用户通过外部 IdP 登录 AD” | +| `oauth_grants` | 无直接等价 | 跳过 | grant、scope 和 client 都是 Cloudreve 安全域数据 | +| `passkeys` | `passkeys` | 要求重新注册 | credential JSON、user_handle、签名计数和序列化格式不同，直接复制可能导致认证失败或安全风险 | +| `settings` | `system_config` | 只迁移白名单键 | AD 需要 value type、namespace、category、visibility、sensitive 等元数据 | +| `tasks` | `background_tasks` | 只归档终态历史 | 不复制成可执行任务；活动任务统一归档为 canceled | + +## 11. AD 新增但 Cloudreve 没有的数据域 + +以下 AD 表没有可直接对应的 Cloudreve 数据。迁移时通常保持为空，由 AD 运行时重新生成或由管理员配置。 + +| AD 数据域 | AD 表 | 迁移策略 | +|---|---|---| +| 登录会话 | `auth_sessions` | 不迁移，所有用户重新登录 | +| 审计 | `audit_logs` | 可选生成一条“从 Cloudreve 迁移”审计记录，不伪造历史审计 | +| 后台任务 | `background_tasks`、`storage_migration_checkpoints` | Cloudreve 旧任务仅写成 `system_runtime` 终态历史；不创建 checkpoint，不恢复执行 | +| 邮件 | `mail_outbox` | 不迁移 | +| 邮箱验证 | `contact_verification_tokens`、`external_auth_email_verification_flows` | 不迁移临时 token | +| 外部登录 | `external_auth_providers`、`external_auth_identities`、`external_auth_login_flows` | 由管理员重新配置 IdP，用户重新绑定 | +| MFA | `mfa_factors`、`mfa_recovery_codes`、`mfa_login_flows`、`mfa_email_codes`、`mfa_totp_setup_flows` | 不复制 Cloudreve secret，要求重新绑定 | +| 团队空间 | `teams`、`team_members` | Cloudreve group 不等于 team；需要单独的组织映射方案 | +| 标签 | `tags` + `entity_properties(system.tags)` | 从 Cloudreve v4 `metadata.name=tag:*` 创建标签定义和文件/目录关联 | +| 锁 | `resource_locks` | 不迁移运行时锁 | +| WOPI | `wopi_sessions` | 不迁移会话 | +| 上传 | `upload_sessions`、`upload_session_parts` | 不迁移未完成上传 | +| 用户邀请 | `user_invitations` | 不迁移 | +| 远端节点 | `managed_followers`、`follower_enrollment_sessions`、`master_bindings`、`remote_storage_targets` | 重新注册 AD follower | +| 存储 OAuth | `storage_policy_credentials`、`storage_policy_authorization_flows`、`storage_connector_application_configs` | OneDrive 等策略必须在 AD 中重新授权 | +| 媒体解析 | `blob_media_metadata` | 可在迁移后由 AD 重新扫描生成 | + +## 12. 完整 Cloudreve 表与字段清单 + +| 表 | 字段 | +|---|---| +| `dav_accounts` | `id`, `created_at`, `updated_at`, `deleted_at`, `name`, `uri`, `password`, `options`, `props`, `owner_id` | +| `direct_links` | `id`, `created_at`, `updated_at`, `deleted_at`, `name`, `downloads`, `speed`, `file_id` | +| `entities` | `id`, `created_at`, `updated_at`, `deleted_at`, `type`, `source`, `size`, `reference_count`, `upload_session_id`, `recycle_options`, `storage_policy_entities`, `created_by` | +| `file_entities` | `file_id`, `entity_id` | +| `files` | `id`, `created_at`, `updated_at`, `type`, `name`, `size`, `primary_entity`, `is_symbolic`, `props`, `file_children`, `storage_policy_files`, `owner_id` | +| `fs_events` | `id`, `created_at`, `updated_at`, `deleted_at`, `event`, `subscriber`, `user_fsevent` | +| `groups` | `id`, `created_at`, `updated_at`, `deleted_at`, `name`, `max_storage`, `speed_limit`, `permissions`, `settings`, `storage_policy_id` | +| `metadata` | `id`, `created_at`, `updated_at`, `deleted_at`, `name`, `value`, `is_public`, `file_id` | +| `nodes` | `id`, `created_at`, `updated_at`, `deleted_at`, `status`, `name`, `type`, `server`, `slave_key`, `capabilities`, `settings`, `weight` | +| `oauth_clients` | `id`, `created_at`, `updated_at`, `deleted_at`, `guid`, `secret`, `name`, `homepage_url`, `redirect_uris`, `scopes`, `props`, `is_enabled` | +| `oauth_grants` | `id`, `created_at`, `updated_at`, `deleted_at`, `scopes`, `last_used_at`, `client_id`, `user_id` | +| `passkeys` | `id`, `created_at`, `updated_at`, `deleted_at`, `credential_id`, `name`, `credential`, `used_at`, `user_id` | +| `settings` | `id`, `created_at`, `updated_at`, `deleted_at`, `name`, `value` | +| `shares` | `id`, `created_at`, `updated_at`, `deleted_at`, `password`, `views`, `downloads`, `expires`, `remain_downloads`, `file_shares`, `user_shares`, `props` | +| `storage_policies` | `id`, `created_at`, `updated_at`, `deleted_at`, `name`, `type`, `server`, `bucket_name`, `is_private`, `access_key`, `secret_key`, `max_size`, `dir_name_rule`, `file_name_rule`, `settings`, `node_id` | +| `tasks` | `id`, `created_at`, `updated_at`, `deleted_at`, `type`, `status`, `public_state`, `private_state`, `correlation_id`, `user_tasks` | +| `users` | `id`, `created_at`, `updated_at`, `deleted_at`, `email`, `nick`, `password`, `status`, `storage`, `two_factor_secret`, `avatar`, `settings`, `group_users` | + +## 13. 完整 AsterDrive 表与字段清单 + +| 表 | 字段 | +|---|---| +| `audit_logs` | `id`, `user_id`, `action`, `entity_type`, `entity_id`, `entity_name`, `details`, `ip_address`, `user_agent`, `created_at` | +| `auth_sessions` | `id`, `user_id`, `current_refresh_jti`, `previous_refresh_jti`, `refresh_expires_at`, `ip_address`, `user_agent`, `created_at`, `last_seen_at`, `revoked_at` | +| `background_tasks` | `id`, `kind`, `status`, `creator_user_id`, `team_id`, `share_id`, `display_name`, `payload_json`, `result_json`, `steps_json`, `progress_current`, `progress_total`, `status_text`, `attempt_count`, `max_attempts`, `next_run_at`, `processing_token`, `processing_started_at`, `last_heartbeat_at`, `lease_expires_at`, `started_at`, `finished_at`, `last_error`, `failure_can_retry`, `expires_at`, `created_at`, `updated_at`, `runtime_json` | +| `blob_media_metadata` | `id`, `blob_id`, `blob_hash`, `kind`, `status`, `metadata_json`, `error_message`, `parser`, `parser_version`, `created_at`, `updated_at` | +| `contact_verification_tokens` | `id`, `user_id`, `channel`, `purpose`, `target`, `token_hash`, `expires_at`, `consumed_at`, `created_at` | +| `entity_properties` | `id`, `entity_type`, `entity_id`, `namespace`, `name`, `value` | +| `external_auth_email_verification_flows` | `id`, `provider_id`, `identity_namespace`, `subject`, `target_email`, `display_name_snapshot`, `preferred_username_snapshot`, `return_path`, `flow_token_hash`, `verification_token_hash`, `email_requested_at`, `created_at`, `expires_at`, `consumed_at` | +| `external_auth_identities` | `id`, `user_id`, `provider_id`, `identity_namespace`, `subject`, `email_snapshot`, `display_name_snapshot`, `created_at`, `updated_at`, `last_login_at` | +| `external_auth_login_flows` | `id`, `provider_id`, `state_hash`, `nonce`, `pkce_verifier`, `redirect_uri`, `return_path`, `created_at`, `expires_at`, `consumed_at` | +| `external_auth_providers` | `id`, `key`, `display_name`, `icon_url`, `provider_kind`, `protocol`, `issuer_url`, `authorization_url`, `token_url`, `userinfo_url`, `client_id`, `client_secret`, `scopes`, `enabled`, `auto_provision_enabled`, `auto_link_verified_email_enabled`, `require_email_verified`, `subject_claim`, `username_claim`, `display_name_claim`, `email_claim`, `email_verified_claim`, `groups_claim`, `avatar_url_claim`, `allowed_domains`, `created_at`, `updated_at`, `options` | +| `file_blobs` | `id`, `hash`, `size`, `policy_id`, `storage_path`, `thumbnail_path`, `thumbnail_processor`, `thumbnail_version`, `ref_count`, `created_at`, `updated_at` | +| `file_versions` | `id`, `file_id`, `blob_id`, `version`, `size`, `created_at` | +| `files` | `id`, `name`, `folder_id`, `team_id`, `blob_id`, `size`, `owner_user_id`, `created_by_user_id`, `created_by_username`, `mime_type`, `created_at`, `updated_at`, `deleted_at`, `is_locked`, `extension`, `compound_extension`, `file_category` | +| `folders` | `id`, `name`, `parent_id`, `team_id`, `owner_user_id`, `created_by_user_id`, `created_by_username`, `policy_id`, `created_at`, `updated_at`, `deleted_at`, `is_locked` | +| `follower_enrollment_sessions` | `id`, `managed_follower_id`, `token_hash`, `ack_token_hash`, `expires_at`, `redeemed_at`, `acked_at`, `invalidated_at`, `created_at` | +| `mail_outbox` | `id`, `template_code`, `to_address`, `to_name`, `payload_json`, `status`, `attempt_count`, `next_attempt_at`, `processing_started_at`, `sent_at`, `last_error`, `created_at`, `updated_at` | +| `managed_followers` | `id`, `name`, `base_url`, `access_key`, `secret_key`, `is_enabled`, `last_capabilities`, `last_error`, `last_checked_at`, `created_at`, `updated_at`, `transport_mode`, `tunnel_last_error`, `tunnel_last_seen_at` | +| `master_bindings` | `id`, `name`, `master_url`, `access_key`, `secret_key`, `storage_namespace`, `is_enabled`, `created_at`, `updated_at` | +| `mfa_email_codes` | `id`, `flow_id`, `user_id`, `code_hash`, `expires_at`, `consumed_at`, `created_at` | +| `mfa_factors` | `id`, `user_id`, `method`, `name`, `secret_ciphertext`, `secret_version`, `enabled_at`, `last_used_at`, `created_at`, `updated_at` | +| `mfa_login_flows` | `id`, `flow_token_hash`, `user_id`, `user_session_version`, `first_factor`, `return_path`, `ip_address`, `user_agent`, `attempt_count`, `expires_at`, `consumed_at`, `created_at` | +| `mfa_recovery_codes` | `id`, `user_id`, `code_hash`, `used_at`, `created_at` | +| `mfa_totp_setup_flows` | `id`, `flow_token_hash`, `user_id`, `secret_ciphertext`, `secret_version`, `expires_at`, `consumed_at`, `created_at` | +| `passkeys` | `id`, `user_id`, `credential_id`, `user_handle`, `credential`, `name`, `transports`, `backup_eligible`, `backed_up`, `sign_count`, `created_at`, `updated_at`, `last_used_at` | +| `remote_storage_targets` | `id`, `master_binding_id`, `target_key`, `name`, `driver_type`, `endpoint`, `bucket`, `access_key`, `secret_key`, `base_path`, `is_default`, `desired_revision`, `applied_revision`, `last_error`, `created_at`, `updated_at` | +| `resource_locks` | `id`, `token`, `entity_type`, `entity_id`, `path`, `owner_id`, `owner_info`, `timeout_at`, `shared`, `deep`, `created_at` | +| `shares` | `id`, `token`, `user_id`, `team_id`, `file_id`, `folder_id`, `password`, `expires_at`, `max_downloads`, `download_count`, `view_count`, `created_at`, `updated_at` | +| `storage_connector_application_configs` | `id`, `policy_id`, `provider`, `tenant_id`, `scopes`, `client_id`, `client_secret_ciphertext`, `metadata`, `created_at`, `updated_at` | +| `storage_migration_checkpoints` | `task_id`, `source_policy_id`, `target_policy_id`, `plan_hash`, `stage`, `last_processed_blob_id`, `scanned_blobs`, `migrated_blobs`, `merged_blobs`, `skipped_blobs`, `failed_blobs`, `migrated_bytes`, `last_error`, `created_at`, `updated_at`, `renamed_opaque_blobs` | +| `storage_policies` | `id`, `name`, `driver_type`, `endpoint`, `bucket`, `access_key`, `secret_key`, `base_path`, `remote_node_id`, `max_file_size`, `allowed_types`, `options`, `is_default`, `chunk_size`, `created_at`, `updated_at`, `remote_storage_target_key` | +| `storage_policy_authorization_flows` | `id`, `provider`, `policy_id`, `created_by_user_id`, `state_hash`, `pkce_verifier`, `redirect_uri`, `scopes`, `context`, `status`, `created_at`, `expires_at`, `consumed_at` | +| `storage_policy_credentials` | `id`, `policy_id`, `provider`, `credential_kind`, `account_label`, `subject`, `tenant_id`, `scopes`, `access_token_ciphertext`, `refresh_token_ciphertext`, `metadata`, `status`, `status_reason`, `expires_at`, `authorized_at`, `last_refreshed_at`, `last_validated_at`, `created_at`, `updated_at` | +| `storage_policy_group_items` | `id`, `group_id`, `policy_id`, `priority`, `min_file_size`, `max_file_size`, `created_at` | +| `storage_policy_groups` | `id`, `name`, `description`, `is_enabled`, `is_default`, `created_at`, `updated_at` | +| `system_config` | `id`, `key`, `value`, `value_type`, `requires_restart`, `is_sensitive`, `source`, `namespace`, `category`, `description`, `updated_at`, `updated_by`, `visibility` | +| `tags` | `id`, `scope_type`, `owner_user_id`, `team_id`, `name`, `normalized_name`, `color`, `sort_order`, `created_at`, `updated_at` | +| `team_members` | `id`, `team_id`, `user_id`, `role`, `created_at`, `updated_at` | +| `teams` | `id`, `name`, `description`, `created_by`, `storage_used`, `storage_quota`, `policy_group_id`, `created_at`, `updated_at`, `archived_at` | +| `upload_session_parts` | `id`, `upload_id`, `part_number`, `etag`, `size`, `created_at`, `updated_at` | +| `upload_sessions` | `id`, `user_id`, `team_id`, `filename`, `total_size`, `chunk_size`, `total_chunks`, `received_count`, `folder_id`, `policy_id`, `status`, `object_temp_key`, `object_multipart_id`, `file_id`, `created_at`, `expires_at`, `updated_at`, `frontend_client_id` | +| `user_invitations` | `id`, `email`, `token_hash`, `status`, `invited_by`, `accepted_user_id`, `expires_at`, `created_at`, `updated_at`, `accepted_at`, `revoked_at` | +| `user_profiles` | `user_id`, `display_name`, `wopi_user_info`, `avatar_source`, `avatar_key`, `avatar_version`, `created_at`, `updated_at` | +| `users` | `id`, `username`, `email`, `password_hash`, `role`, `status`, `session_version`, `email_verified_at`, `pending_email`, `storage_used`, `storage_quota`, `policy_group_id`, `created_at`, `updated_at`, `config`, `must_change_password` | +| `webdav_accounts` | `id`, `user_id`, `username`, `password_hash`, `root_folder_id`, `is_active`, `created_at`, `updated_at`, `team_id` | +| `wopi_sessions` | `id`, `token_hash`, `actor_user_id`, `session_version`, `team_id`, `file_id`, `app_key`, `expires_at`, `created_at` | + +## 14. 建议在执行迁移前确认的决策 + +| 决策项 | 可选方案 | 当前迁移实现默认值 | +|---|---|---| +| 已删除数据 | 跳过 / 迁移并标记删除或禁用 | 默认跳过；可显式包含部分软删除表 | +| 用户密码 | 统一临时密码 / 每人随机密码并导出 / 邮件重置 | 统一临时 Argon2 密码，`must_change_password=true` | +| 邮箱验证 | 视为已验证 / 全部重新验证 | 活跃用户按已验证处理 | +| Cloudreve group | AD 策略组 / AD team / 两者都建 | 映射为存储策略组，不自动创建 team | +| 本地存储根目录 | 复用 Cloudreve 工作目录 / 搬迁文件到 AD 新目录 | 通过 `--local-base-path` 指定并复用旧对象路径 | +| 对象存储 | 复用原 bucket / 复制到新 bucket | 复用原 endpoint、bucket 和 object key | +| 历史版本 | 全部迁移 / 只迁移当前版本 | 当前实现迁移全部 `type=0` entity | +| 分享 URL | 生成新 token / 尝试兼容旧 HashID | 生成新 token，旧 URL 失效 | +| Direct link | 重新签发 AD v2 URL / 跳过 / 建旧 URL 兼容层 | 提供 `--direct-link-secret` 时重新签发并存档 source ID -> URL 映射；旧 `/f/...` 失效 | +| Cloudreve metadata | 全量存档 / 白名单迁移 | `metadata` 表全量迁移到带命名空间的 property；`files.props` 暂不迁移 | +| Cloudreve 标签 | 从 `tag:*` 生成 AD 原生标签 / 仅按普通 property 存档 | 当前生成 `tags` 定义和 `system.tags` 关联 | +| Cloudreve 任务 | 跳过 / 历史归档 / 尝试恢复执行 | 当前全部历史归档；活动任务归档为 canceled，绝不恢复执行 | +| 不兼容策略 | 中止 / 跳过策略和依赖文件 / 解密导出后再迁移 | 默认中止；显式参数可跳过 | +| Passkey/MFA/WebDAV | 尝试转换 / 要求重新绑定 | 要求重新绑定 | +| 系统设置 | 全量复制 / 白名单映射 / 不迁移 | 当前不迁移，建议单独制作配置键白名单 | + +## 15. 建议的验证顺序 + +| 阶段 | 检查内容 | +|---|---| +| 迁移前预检 | 表是否存在、目标核心表是否为空、源外键是否完整、是否存在不兼容存储策略 | +| 用户迁移后 | 用户数、管理员数、禁用用户数、邮箱唯一性、用户名冲突、配额合计 | +| 文件树迁移后 | 根目录数、目录数、文件数、孤儿 parent、孤儿 owner、同目录重名 | +| blob 迁移后 | entity 数、blob 数、当前 blob 关联、历史版本数、ref_count 重算结果 | +| 存储验证 | 随机抽样每种策略执行 metadata、读取、范围读取，校验文件大小 | +| 分享验证 | 文件分享、目录分享、密码分享、过期分享、下载次数限制 | +| 标签验证 | 每个 `tag:*` 元数据有对应 AD tag；`system.tags` 关联指向正确 file/folder 和 scope | +| 直链验证 | 使用与 AD 相同的 `auth.direct_link_secret` 请求新 `/d/v2...` URL；核对映射属性数量 | +| 任务验证 | 所有导入任务均为 `succeeded/failed/canceled`，无 `pending/retry/processing`，`lease_expires_at` 为空 | +| AD 启动后 | 重新生成媒体元数据、缩略图和搜索索引；检查 storage_used 汇总 | +| 切换前 | 冻结 Cloudreve 写入，再跑最终迁移或增量校验，备份目标库 | diff --git a/docs/migration-architecture.md b/docs/migration-architecture.md new file mode 100644 index 0000000..487a506 --- /dev/null +++ b/docs/migration-architecture.md @@ -0,0 +1,814 @@ +# AsterDriveMigration 目标架构设计 + +本文档将现有架构图和文件/对象关系图转写为可实施的文字设计。它描述的是迁移工具的目标架构，同时明确当前代码已经覆盖的部分和后续仍需实现的部分。 + +配套字段文档：[`cloudreve-to-asterdrive-field-mapping.md`](cloudreve-to-asterdrive-field-mapping.md)。 + +## 1. 设计目标 + +AsterDriveMigration 不应成为一组把源表逐行复制到目标表的脚本，而应成为一个可恢复、可校验、可扩展的数据迁移引擎。 + +核心目标： + +| 目标 | 说明 | +|---|---| +| 无直接表复制 | 读取源系统对象，转换为统一领域模型，再通过 AD writer 写入目标 | +| 源系统隔离 | Cloudreve 特有字段、状态和存储规则只存在于 Cloudreve Adapter | +| 目标系统隔离 | AD schema、事务、去重、配额和 ref_count 规则只存在于 AsterDrive Writer | +| 可恢复 | 每个 stage 保存 cursor，进程中断后可以从 checkpoint 继续 | +| 幂等 | 同一个源对象重复处理时，通过 object mapping 找到已有目标对象，不重复创建 | +| 可校验 | 迁移后能够重新读取源和目标，验证数量、关系、大小、哈希、配额和引用计数 | +| 可审计 | 每次运行保留状态、统计、错误、警告和对象映射 | +| 可扩展 | 后续增加 Alist、FileBrowser、Nextcloud 等 source adapter 时不修改 AD writer 核心 | +| 大文件友好 | 文件内容使用流式读写，不将整个对象载入内存 | + +## 2. 核心原则 + +### 2.1 禁止直接复制表 + +不采用以下模式： + +```text +Cloudreve users row -> INSERT INTO AsterDrive users +Cloudreve files row -> INSERT INTO AsterDrive files +``` + +采用以下模式： + +```text +Cloudreve DB/Storage + -> Cloudreve Adapter + -> ExternalUser / ExternalFolder / ExternalFile / ExternalBlob + -> Migration Core + -> AsterDrive Writer + -> AsterDrive DB/Storage +``` + +原因是 Cloudreve 和 AD 的表语义不同。例如 Cloudreve `files` 同时表示文件和目录，而 AD 使用 `files`、`folders`、`file_blobs` 和 `file_versions` 分别建模。 + +### 2.2 映射驱动，而不是 ID 假设驱动 + +源 ID 和目标 ID 不能假设相等。所有跨对象关系必须通过持久化 object mapping 解析： + +```text +(source_type, source_id) -> (target_type, target_id) +``` + +典型映射： + +| 源对象 | 目标对象 | +|---|---| +| Cloudreve user ID | AD user ID | +| Cloudreve folder/file record ID | AD folder ID 或 file ID | +| Cloudreve entity ID/object key | AD file_blob ID | +| Cloudreve share ID | AD share ID | +| Cloudreve policy ID | AD storage_policy ID | + +### 2.3 文件记录与物理对象分离 + +迁移模型必须区分： + +- `ExternalFile`：用户看到的逻辑文件，包含名称、父目录、owner、大小、MIME。 +- `ExternalBlob`：存储系统中的物理对象，包含 object key、策略、大小、可选哈希和读取方式。 + +多个逻辑文件可以复用一个 AD `file_blob`。同一逻辑文件的旧 blob 应写入 `file_versions`，当前 blob 写入 `files.blob_id`。 + +## 3. 分层架构 + +## 3.1 Layer 1：Interface Layer + +负责用户交互、配置和输出，不承载源系统或 AD 业务规则。 + +### CLI Commands + +目标命令集合： + +| 命令 | 作用 | +|---|---| +| `migrate start` | 创建迁移运行，执行预检、规划和迁移 | +| `migrate list` | 列出历史运行及状态 | +| `migrate resume` | 从 checkpoint 恢复中断运行 | +| `migrate verify` | 对已完成或部分完成的运行执行一致性校验 | +| `migrate report` | 导出人类可读或 JSON 报告 | + +当前代码提供的是简化命令： + +| 当前命令 | 作用 | +|---|---| +| `check` | 连接源/目标数据库，检查 schema 和源数据概况 | +| `migrate` | 在一个目标数据库事务中执行一次性迁移 | + +### TUI Progress View（可选） + +目标能力： + +- 实时迁移进度 +- 当前 stage 和状态 +- 对象吞吐量、字节吞吐量 +- 错误与重试次数 +- 预计剩余时间 ETA + +TUI 只订阅 engine progress event，不直接访问数据库或实现迁移逻辑。 + +### Config Loader + +配置来源： + +- 配置文件，例如 `config.yaml` +- 环境变量 +- secret provider 或运行时安全输入 +- source 和 target 数据库连接 +- source 和 target 存储连接 +- 冲突处理、并发、重试和验证选项 + +敏感字段不得出现在普通日志、迁移报告或 TUI 中。 + +### Output Renderer + +至少支持： + +- Human Text：供管理员直接阅读 +- JSON：供自动化、CI 或外部控制器消费 +- Machine-readable exit code：区分成功、警告完成、校验失败和运行失败 + +## 3.2 Layer 2：Migration Core + +Migration Core 编排完整流程，但不直接理解 Cloudreve 表或 AD Entity。 + +### Engine + +职责： + +- 创建或恢复 migration run +- 控制完整生命周期 +- 管理取消信号和优雅停止 +- 协调并发任务 +- 汇总错误和最终状态 +- 确保阶段切换符合状态机 + +建议运行状态： + +```text +planned -> running -> verifying -> completed + -> completed_with_warnings + -> failed + -> cancelled +``` + +### Stage Runner + +职责： + +- 按依赖顺序执行 stages +- 控制每个 stage 的并行度 +- 实现可配置 retry 和 exponential backoff +- 在每批数据完成后保存 cursor +- 支持安全停止和 resume + +建议 stage 顺序： + +| 顺序 | Stage | 主要输出 | +|---:|---|---| +| 1 | `preflight` | schema、版本、连接、权限和存储能力检查 | +| 2 | `inventory` | 源对象数量、字节数和不兼容能力清单 | +| 3 | `policies` | AD storage policy 及 ID mapping | +| 4 | `policy_groups` | AD policy group、group item 及 mapping | +| 5 | `users` | AD user/profile 及 mapping | +| 6 | `folders` | 目录树及 parent mapping | +| 7 | `blobs` | 对象复制/复用、哈希和 blob mapping | +| 8 | `files` | 当前文件和历史版本 | +| 9 | `shares` | 分享和新 token | +| 10 | `metadata_tags` | properties、Cloudreve `tag:*` 到 AD 原生标签及关联 | +| 11 | `direct_links` | 使用 AD secret 重新签发 v2 URL，保存 source link -> URL 映射 | +| 12 | `task_history` | 将 Cloudreve 任务写成不可领取的终态历史记录 | +| 13 | `recalculate` | ref_count、storage_used、配额和统计重算 | +| 14 | `verify` | 数量、关系、哈希、直链签名、任务终态和可读性校验 | +| 15 | `finalize` | 最终报告和运行状态 | + +### Plan Builder + +职责： + +- 扫描源 inventory +- 识别源版本和 capabilities +- 判断哪些资源可迁移、跳过或必须中止 +- 生成 `MigrationPlan` +- 估算对象数、总字节数、阶段成本和风险 + +计划中应明确： + +- 用户、目录、文件、blob、版本、分享数量 +- 各存储策略的对象数量和字节数 +- 不兼容的存储驱动 +- Cloudreve 加密对象数量 +- symbolic/placeholder 文件数量 +- 重名冲突和孤儿关系数量 +- 预计需要的目标存储空间 + +### Checkpoint Manager + +职责： + +- 保存和加载 run state +- 保存每个 stage 的 cursor/offset +- 保存 object mapping +- 提供 resume 支持 +- 判断重复执行是否安全 + +checkpoint 必须在目标记录成功提交后更新，不能先推进 cursor 再写目标数据。 + +### Object Mapping Registry + +职责： + +- 根据 source type + ID 查询目标对象 +- 创建或更新映射 +- 保存源对象哈希、元数据摘要和状态 +- 支持 lookup、upsert 和验证 +- 为断点续传、去重和冲突处理提供依据 + +映射不只保存整数 ID。Cloudreve blob 还可能使用 object key 作为稳定源标识。 + +### Conflict Resolver + +需要检测： + +- 用户名或邮箱冲突 +- 同目录同名文件/目录冲突 +- 目标对象已经存在但来源不同 +- mapping 指向的目标对象已被删除或修改 +- 相同 hash+policy 已有 blob +- 源文件在迁移过程中发生变化 + +`ImportDecision` 至少支持： + +| 决策 | 行为 | +|---|---| +| `fail` | 遇到冲突立即停止，适合严格迁移 | +| `rename` | 生成唯一目标名，例如 `name (1).ext` | +| `skip` | 保留目标数据，记录源对象被跳过 | +| `reuse` | 已验证为同一对象时复用目标记录或 blob | +| `update` | 明确允许时更新已映射的目标对象 | + +所有自动决策必须写入报告，不能静默改名或跳过。 + +### Verification Orchestrator + +职责： + +- 重新读取源 inventory +- 读取 object mapping +- 读取目标状态 +- 执行一致性检查 +- 将校验结果交给 Report Builder + +### Report Builder + +报告内容： + +- 源和目标对象统计 +- 成功、复用、跳过、重命名、失败数量 +- 已迁移字节数和吞吐量 +- 不兼容能力和人工处理项 +- 校验结果 +- 错误和警告摘要 +- 可选对象级错误明细 + +## 3.3 Layer 3：Source Adapter Layer + +Source Adapter 读取源系统并产出统一领域对象。所有源系统专有逻辑必须留在 adapter 内。 + +建议 trait： + +```rust +trait SourceAdapter { + async fn capabilities(&self) -> Result; + async fn inventory(&self) -> Result; + async fn list_users(&self, cursor: Cursor) -> Result>; + async fn list_folders(&self, cursor: Cursor) -> Result>; + async fn list_files(&self, cursor: Cursor) -> Result>; + async fn list_shares(&self, cursor: Cursor) -> Result>; + async fn get_blob(&self, blob: &ExternalBlobRef) -> Result; +} +``` + +### Cloudreve Adapter + +内部组件： + +| 组件 | 职责 | +|---|---| +| Cloudreve DB Reader | 读取 users、groups、files、entities、file_entities、shares、metadata、direct_links、tasks 和 policies | +| Cloudreve Storage Resolver | 解析 policy、driver、physical location 和对象是否存在 | +| Cloudreve Model Decoder | 解码 JSON、boolset、版本关系、文件类型和软删除状态 | +| Cloudreve -> Domain Converter | 转成 ExternalUser/Folder/File/Blob/Share，并规范化字段 | + +Cloudreve 特有逻辑示例： + +- `files.type=0/1` 的文件/目录拆分 +- `file_children` 实际是 parent ID +- `primary_entity` 与 `file_entities` 的当前/历史版本关系 +- `entities.type=0/1/2` 的版本、缩略图和 Live Photo 语义 +- Cloudreve policy type 到通用 storage capability 的转换 +- Cloudreve password、MFA、Passkey 和 share password 格式识别 +- 从 `metadata.name=tag:*` 提取标签，在用户个人 scope 内去重，并生成 AD `system.tags` 关联 +- 将 Cloudreve `/f/{hashid}/{name}` 语义转换为 AD `/d/v2...` 重新签发映射，而不是复制旧 token +- 将 Cloudreve 任务作为终态历史归档；queued/processing/suspending 不进入 AD 可执行队列 + +### Future Adapters + +Alist、FileBrowser、Nextcloud 等 adapter 只需实现 SourceAdapter 并产出相同领域模型，不应修改 AsterDrive Writer。 + +## 3.4 Layer 4：Domain Model Layer + +领域模型必须与 Cloudreve 和 AD 的 ORM Entity 解耦。 + +### ExternalUser + +建议字段： + +```text +source_user_id +source_username +source_email +display_name +status +role_hint +storage_used +storage_quota +source_group_id +created_at +updated_at +metadata +``` + +### ExternalFolder + +```text +source_folder_id +source_name +source_parent_id +source_owner_id +source_policy_id +created_at +updated_at +metadata +``` + +### ExternalFile + +```text +source_file_id +source_name +source_folder_id +source_owner_id +source_current_blob +source_historical_blobs +source_size +mime_type +created_at +updated_at +metadata +``` + +### ExternalBlob + +```text +source_blob_id +source_object_key +source_policy_id +source_size +source_hash (optional) +source_thumbnail +created_at +updated_at +``` + +### ExternalShare + +```text +source_share_id +source_owner_id +source_target_type +source_target_id +source_password +view_count +download_count +remaining_downloads +expires_at +metadata +``` + +### WorkspaceScope + +描述目标写入个人空间还是团队空间： + +```text +Personal { owner_user_id } +Team { team_id, actor_user_id } +``` + +Cloudreve group 默认不自动转换成 WorkspaceScope::Team。只有管理员提供明确组织映射时才创建 AD team。 + +### MigrationPlan + +保存： + +- source/target 标识 +- enabled stages +- storage policy decisions +- conflict policy +- include_deleted +- copy/reuse storage mode +- concurrency/retry 参数 +- verification level +- inventory estimate + +### ImportDecision + +保存 conflict resolver 对某个对象的最终决策，以及决策原因和是否需要人工确认。 + +## 3.5 Layer 5：Target Layer / AsterDrive Writer + +Target Writer 是唯一允许理解 AD schema 和 AD 存储规则的层。 + +建议 trait： + +```rust +trait TargetWriter { + async fn write_user(&self, user: &ExternalUser) -> Result; + async fn write_folder(&self, folder: &ExternalFolder) -> Result; + async fn write_blob(&self, blob: &ExternalBlob) -> Result; + async fn write_file(&self, file: &ExternalFile) -> Result; + async fn write_share(&self, share: &ExternalShare) -> Result; + async fn finalize(&self, run: &MigrationRun) -> Result<()>; +} +``` + +### User Writer + +- Create/update user +- 创建 user profile +- 映射 role/status +- 设置临时 Argon2 密码 +- 设置 `must_change_password` +- 写入 quota 和 policy group + +### Folder Writer + +- Create/update folder +- 通过 object mapping 解析 parent +- 保证父目录先于子目录创建 +- 处理同目录重名 + +### Blob Writer + +- 根据 hash + policy 查找可复用 blob +- 流式上传对象 +- 生成 AD storage path +- 校验 size 和 checksum +- 插入或复用 `file_blobs` + +### File Writer + +- Create/update `files` +- 写入 metadata/classification +- 指向当前 blob +- 将历史 blob 写入 `file_versions` +- 处理 rename/skip/fail 决策 + +### Share Writer（可选） + +- 创建或更新分享 +- 生成新的 AD token/link code +- 将 Cloudreve 明文分享密码转为 Argon2 hash +- 保留过期时间、访问和下载计数 + +### Quota Recalculator + +- 按用户或 workspace 重新计算 `storage_used` +- 不只信任 Cloudreve 旧统计 +- 明确去重 blob 是否按逻辑文件大小还是物理占用计入配额 + +### RefCount Recalculator + +- 根据 `files.blob_id` 和 `file_versions.blob_id` 重算 ref_count +- 找出孤儿 blob +- 找出 ref_count 与实际引用不一致的 blob + +### AsterDrive Storage Writer + +- Stream/chunk 上传 +- 计算 SHA-256 +- 写入 AD 规范 storage path +- 支持本地和对象存储 +- 完成后重新读取 metadata/size 验证 + +### AsterDrive DB Writer + +- 事务边界 +- 批量插入 +- index-aware update +- 与 checkpoint 更新保持提交顺序一致 + +## 3.6 Layer 6：Persistence Layer + +### Source System + +- Cloudreve Database：用户、目录、文件、分享、设置、自定义字段 +- Cloudreve Storage：文件 blob、缩略图、附件和其他物理对象 + +### Target System + +- AsterDrive Database：用户、workspace、目录、文件、blob、分享、配额和 metadata +- AsterDrive Storage：规范化 blob、去重对象和备份/归档对象 + +### Checkpoint Tables + +建议 checkpoint 表保存在 AD 数据库中，以便迁移进程重启后恢复。 + +#### `aster_external_migration_runs` + +| 字段 | 建议类型 | 说明 | +|---|---|---| +| `id` | UUID/String PK | Run ID | +| `source_type` | String | `cloudreve`、`nextcloud` 等 | +| `source_fingerprint` | String | 源数据库/实例指纹，避免连错源 | +| `target_fingerprint` | String | 目标 AD 实例指纹 | +| `status` | String | planned/running/verifying/completed/failed/cancelled | +| `started_at` | DateTime | 开始时间 | +| `updated_at` | DateTime | 最后更新时间 | +| `finished_at` | DateTime nullable | 完成时间 | +| `plan_json` | Text | MigrationPlan | +| `stats_json` | Text | 数量、字节和阶段统计 | +| `summary_json` | Text nullable | 最终摘要 | +| `last_error` | Text nullable | 最后错误，不包含 secret | + +#### `aster_external_migration_stage_cursors` + +| 字段 | 建议类型 | 说明 | +|---|---|---| +| `run_id` | FK | 所属 run | +| `stage` | String | users/folders/blobs/files 等 | +| `cursor_json` | Text | source cursor/offset/last ID | +| `processed_count` | BigInt | 已处理对象数 | +| `processed_bytes` | BigInt | 已处理字节数 | +| `updated_at` | DateTime | checkpoint 时间 | +| `state` | String | pending/running/completed/failed | + +建议 `(run_id, stage)` 为联合主键或唯一键。 + +#### `aster_external_migration_object_map` + +| 字段 | 建议类型 | 说明 | +|---|---|---| +| `id` | BigInt PK | 映射记录 ID | +| `run_id` | FK | 所属 run | +| `source_type` | String | user/folder/file/blob/share/policy | +| `source_id` | String | 支持整数 ID 或 object key | +| `target_type` | String | AD 对象类型 | +| `target_id` | String | 目标 ID | +| `source_hash` | String nullable | 源对象摘要或内容哈希 | +| `target_hash` | String nullable | 目标对象摘要或内容哈希 | +| `metadata_json` | Text | 名称、大小、parent 等校验上下文 | +| `state` | String | created/reused/skipped/renamed/failed | +| `created_at` | DateTime | 创建时间 | +| `updated_at` | DateTime | 更新时间 | + +建议唯一键： + +```text +(run_id, source_type, source_id) +``` + +## 4. 文件与对象关系 + +## 4.1 源模型到领域模型 + +| Cloudreve 对象 | 领域对象 | 关键字段 | +|---|---|---| +| User | ExternalUser | source_user_id、username、email、used_storage | +| Folder/Directory | ExternalFolder | source_folder_id、name、parent_id、owner_id | +| File | ExternalFile | source_file_id、name、folder_id、object/entity、size、MIME | +| Policy | Storage capability/plan decision | policy ID、driver、rules | +| Physical Entity | ExternalBlob | object key、size、policy、storage path | +| Share | ExternalShare | source_share_id、target、password、权限、expires_at | + +## 4.2 领域模型到 AD + +| 领域对象 | AD 表 | 关键关系 | +|---|---|---| +| ExternalUser | `users` + `user_profiles` | 一个 source user 对应一个 target user | +| ExternalFolder | `folders` | parent_id 通过 mapping 解析 | +| ExternalBlob | `file_blobs` | hash+policy 去重或创建 | +| ExternalFile | `files` | `blob_id` 指向当前 blob | +| Historical ExternalBlob | `file_versions` | `file_id` + `blob_id` + version | +| ExternalShare | `shares` | 指向 file_id 或 folder_id | + +## 4.3 关键不变量 + +| 不变量 | 说明 | +|---|---| +| 文件与 blob 分离 | `files` 只表示逻辑文件，`file_blobs` 表示物理对象 | +| blob 可复用 | 多个 files 可以指向同一 blob | +| 同策略内容去重 | 只有内容哈希可信且 policy 相同才复用 blob | +| parent 先创建 | folder parent mapping 必须在 child 写入前存在 | +| file 依赖 blob | 当前 blob 写入成功后才能插入 files | +| mapping 持久化 | source->target 映射是 resume、验证和幂等的核心 | +| ref_count 可重算 | ref_count 必须能从 files + file_versions 反向计算 | +| storage_used 可重算 | 用户/workspace 使用量必须能从目标文件关系反向校验 | + +## 5. 物理对象复制流程 + +目标流程不是只复用 Cloudreve `entity.source`，而是支持将对象实际复制到 AD 管理的存储空间。 + +| 步骤 | 动作 | 关键要求 | +|---:|---|---| +| 1 | Read source physical object | SourceAdapter 根据 policy 和 object key 打开流 | +| 2 | Stream bytes | 固定大小 buffer，支持 backpressure，不整文件载入内存 | +| 3 | Calculate SHA-256 | 在流式传输过程中计算真实内容哈希 | +| 4 | Write to AD storage | 使用 AD storage writer 上传到目标策略 | +| 5 | Generate storage path | 使用 AD 规范，例如 `ab/cd/` | +| 6 | Insert or reuse blob | 按可信 hash + target policy 查找或插入 `file_blobs` | +| 7 | Insert file | 创建 `files` 并设置 `blob_id` | +| 8 | Increment/recalculate ref_count | 避免并发下计数漂移，最终统一重算 | +| 9 | Update storage_used | 按 AD 配额语义更新或最终统一重算 | + +### 对象复制模式 + +建议支持两种模式： + +| 模式 | 行为 | 适用场景 | +|---|---|---| +| `reuse_source_storage` | AD policy 指向原 bucket/path，不复制字节 | 快速切换、源存储长期保留、驱动完全兼容 | +| `copy_to_target_storage` | 流式读取源对象并写入 AD 新存储 | 真正脱离 Cloudreve、统一对象布局、计算真实 SHA-256 | + +当前实现属于 `reuse_source_storage`：保留 Cloudreve object key，并使用 opaque blob key，避免把非内容摘要伪装成 SHA-256。 + +## 6. 冲突处理 + +### 文件和目录重名 + +| 策略 | 行为 | +|---|---| +| fail | 停止运行，要求管理员清理目标或选择策略 | +| rename | 生成 `name (1).ext`、`name (2).ext` 等唯一名称 | +| skip | 不修改目标，mapping 标记 skipped | + +### 用户冲突 + +需要分别处理： + +- username 相同但 email 不同 +- email 相同但 username 不同 +- 已存在 mapping，但目标用户被修改 +- 多个源用户规范化后得到同一 username + +自动 rename username 时必须保存原 nick 到 `user_profiles.display_name` 和 mapping metadata。 + +### Blob 冲突 + +只有以下条件同时成立才可以自动复用： + +```text +可信内容 SHA-256 相同 +目标 policy 相同 +目标 size 相同 +目标对象可读且校验通过 +``` + +Cloudreve entity ID、object key、路径或 `policy+path+size` 摘要都不能当作真实内容哈希。 + +## 7. 断点续传与幂等 + +每个 batch 的安全顺序： + +```text +1. 读取 source page +2. 查询 object mapping +3. 转换并写入 AD +4. 提交目标事务 +5. upsert object mapping +6. 更新 stage cursor +``` + +如果目标写入、mapping 和 cursor 使用同一个 AD 数据库，建议放入同一事务。对象存储上传无法纳入数据库事务，因此需要： + +- 上传使用确定性临时 key 或 upload ID +- DB 提交失败时记录待清理对象 +- resume 时检测目标对象是否已存在并校验 +- finalize 或后台任务清理孤儿对象 + +幂等判断不能只依赖“目标表已有同名记录”，必须依赖 object mapping 和对象摘要。 + +## 8. 验证设计 + +### 数量校验 + +- 用户、策略组、目录、文件、分享数量 +- blob、历史版本、metadata 数量 +- skipped/renamed/reused 数量与计划一致 + +### 关系校验 + +- 每个 folder parent 存在 +- 每个 file folder/owner/blob 存在 +- 每个 file_version file/blob 存在 +- 每个 share 的 file/folder 和 owner 存在 +- 每个 object mapping 的目标对象存在 + +### 存储校验 + +- 对象存在 +- metadata size 与 DB size 一致 +- 随机抽样完整读取 +- 支持 range read 的策略执行范围读取 +- copy 模式校验 SHA-256 + +### 可重算校验 + +- `file_blobs.ref_count` 与实际引用数一致 +- `users.storage_used` 与 AD 配额算法计算结果一致 +- 没有未映射的目标对象或孤儿 blob + +### 迁移期间源变化检测 + +迁移前后重新读取 source inventory 或关键对象摘要。如果源仍在写入： + +- 标记 run 为 verification failed 或 source_changed +- 不宣称迁移完成 +- 要求冻结 Cloudreve 写入后执行最终同步 + +## 9. 当前实现与目标架构差距 + +| 能力 | 当前状态 | 目标状态 | +|---|---|---| +| Cloudreve/AD schema 预检 | 已实现 | 扩展版本、权限、capability 和存储连通性检查 | +| 用户、策略组、目录、文件、blob、版本、分享、metadata | 已实现基础迁移 | 改为 adapter/domain/writer 分层和分页 stage | +| Cloudreve v4 标签 | 已从 `tag:*` metadata 生成 AD `tags` 和 `system.tags` 文件/目录关联 | 增加跨批次持久化去重和冲突报告 | +| Direct links | 提供 AD `direct_link_secret` 时生成 v2 URL，并以 `cloudreve.direct_links` property 保存源 ID 映射 | 增加独立 JSON/CSV 报告和可选旧 URL 重定向层 | +| Cloudreve 任务 | 已全部写成 AD `system_runtime` 终态历史；活动任务归档为 canceled | 后续可增加独立 legacy task archive 表，避免占用运维任务列表 | +| 目标事务 | 已实现单次全局事务 | 每 batch 事务 + checkpoint，适合大数据集 | +| ID mapping | 已实现内存 HashMap | 持久化 object mapping table | +| 断点续传 | 未实现 | stage cursor + resume | +| 幂等重复运行 | 未完整实现 | mapping 驱动 upsert/reuse | +| 冲突策略 | 仅 username 自动后缀，其他冲突依赖 DB 报错 | fail/rename/skip/reuse/update 可配置 | +| 物理对象复制 | 未实现，当前复用源 storage path | 流式 copy、SHA-256、目标路径和校验 | +| 真正内容去重 | 未实现 | hash + policy 去重 | +| ref_count 重算 | 基于源关联计算初值 | 迁移后从 AD 关系统一重算 | +| storage_used 重算 | 当前复制 Cloudreve 用户统计 | 迁移后按 AD 语义统一重算 | +| 验证编排 | 有标签、直链、任务独立规则测试和 SQLite 端到端测试 | 对实际运行执行 inventory/关系/存储一致性验证 | +| 报告 | 当前输出基础统计和警告 | run/stage/object 级报告，Human + JSON | +| TUI | 未接入新迁移核心 | 可选进度视图 | +| 多源系统插件 | 未实现 | SourceAdapter 插件化 | + +## 10. 推荐实施阶段 + +### Phase 1：稳定当前一次性迁移 + +- 增加真实 Cloudreve 样本库测试 +- 覆盖 SQLite、MySQL、PostgreSQL +- 增加 orphan、cycle、重名和不兼容策略预检 +- 增加迁移后 ref_count/storage_used 验证 + +### Phase 2：领域模型和 Adapter/Writer 重构 + +- 定义 SourceAdapter、TargetWriter +- 引入 ExternalUser/Folder/File/Blob/Share +- 将 Cloudreve ORM 查询移动到 Cloudreve Adapter +- 将 AD ActiveModel 写入移动到 AsterDrive Writer + +### Phase 3：持久化运行状态 + +- 新增 migration runs、stage cursors 和 object map 表 +- 分页 stage runner +- resume 和幂等 upsert +- `migrate list/resume/report` + +### Phase 4：物理对象复制 + +- StorageResolver 和 BlobReader +- AsterDrive Storage Writer +- 流式 SHA-256 +- copy/reuse 两种模式 +- 孤儿对象清理 + +### Phase 5：验证和运维体验 + +- Verification Orchestrator +- Human/JSON report +- TUI progress +- 吞吐量、ETA、失败重试和人工冲突处理 + +## 11. 最终完成标准 + +一次迁移只有同时满足以下条件，才能标记为 completed： + +1. 所有必需 stages 完成并持久化 cursor。 +2. 所有迁移对象都有 created/reused/skipped/renamed 等明确 mapping 状态。 +3. 目标关系完整，没有孤儿 owner、parent、blob、version 或 share。 +4. copy 模式下对象大小和 SHA-256 校验通过。 +5. ref_count 和 storage_used 重算通过。 +6. 源在最终校验窗口内没有发生未处理变化。 +7. 所有 skipped 和 warning 都出现在最终报告中。 +8. 不存在包含明文密码、token、secret 或 access key 的日志和报告。 diff --git a/src/main.rs b/src/main.rs index d29ec8c..367401d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -33,6 +33,8 @@ struct MigrateArgs { default_password: String, #[arg(long, default_value = ".")] local_base_path: String, + #[arg(long, env = "ASTER_DIRECT_LINK_SECRET", hide_env_values = true)] + direct_link_secret: Option, #[arg(long)] allow_non_empty_target: bool, #[arg(long)] @@ -55,6 +57,7 @@ async fn main() -> Result<()> { target_url: args.connection.target_url, default_password: args.default_password, local_base_path: args.local_base_path, + direct_link_secret: args.direct_link_secret, include_deleted: args.connection.include_deleted, allow_non_empty_target: args.allow_non_empty_target, skip_unsupported_policies: args.skip_unsupported_policies, diff --git a/src/migration/mod.rs b/src/migration/mod.rs index 94fa910..e6a7f7d 100644 --- a/src/migration/mod.rs +++ b/src/migration/mod.rs @@ -3,7 +3,9 @@ use std::fmt::{self, Write}; use argon2::Argon2; use argon2::password_hash::{PasswordHasher, SaltString}; +use base64::Engine; use color_eyre::eyre::{Result, WrapErr, bail}; +use hmac::{Hmac, Mac}; use sea_orm::{ ActiveModelTrait, Database, DatabaseConnection, EntityTrait, PaginatorTrait, Set, TransactionTrait, @@ -20,6 +22,7 @@ pub struct MigrationOptions { pub target_url: String, pub default_password: String, pub local_base_path: String, + pub direct_link_secret: Option, pub include_deleted: bool, pub allow_non_empty_target: bool, pub skip_unsupported_policies: bool, @@ -35,6 +38,9 @@ pub struct MigrationReport { pub source_files: u64, pub source_entities: u64, pub source_shares: u64, + pub source_direct_links: u64, + pub source_tag_assignments: u64, + pub source_tasks: u64, pub migrated_users: usize, pub migrated_policy_groups: usize, pub migrated_policies: usize, @@ -44,6 +50,10 @@ pub struct MigrationReport { pub migrated_versions: usize, pub migrated_shares: usize, pub migrated_properties: usize, + pub migrated_tags: usize, + pub migrated_tag_assignments: usize, + pub migrated_direct_links: usize, + pub migrated_tasks: usize, pub skipped: usize, pub dry_run: bool, pub warnings: Vec, @@ -55,21 +65,24 @@ impl fmt::Display for MigrationReport { writeln!(output, "Cloudreve -> AsterDrive migration report")?; writeln!( output, - "source: users={}, groups={}, policies={}, folders={}, files={}, entities={}, shares={}", + "source: users={}, groups={}, policies={}, folders={}, files={}, entities={}, shares={}, direct_links={}, tag_assignments={}, tasks={}", self.source_users, self.source_groups, self.source_policies, self.source_folders, self.source_files, self.source_entities, - self.source_shares + self.source_shares, + self.source_direct_links, + self.source_tag_assignments, + self.source_tasks )?; if self.dry_run { writeln!(output, "mode: dry-run (target was not modified)")?; } else { writeln!( output, - "migrated: users={}, policy_groups={}, policies={}, folders={}, files={}, blobs={}, versions={}, shares={}, properties={}", + "migrated: users={}, policy_groups={}, policies={}, folders={}, files={}, blobs={}, versions={}, shares={}, properties={}, tags={}, tag_assignments={}, direct_links={}, archived_tasks={}", self.migrated_users, self.migrated_policy_groups, self.migrated_policies, @@ -78,7 +91,11 @@ impl fmt::Display for MigrationReport { self.migrated_blobs, self.migrated_versions, self.migrated_shares, - self.migrated_properties + self.migrated_properties, + self.migrated_tags, + self.migrated_tag_assignments, + self.migrated_direct_links, + self.migrated_tasks )?; writeln!(output, "skipped: {}", self.skipped)?; } @@ -111,6 +128,13 @@ pub async fn migrate(options: MigrationOptions) -> Result { if options.default_password.chars().count() < 8 { bail!("--default-password must contain at least 8 characters"); } + if options + .direct_link_secret + .as_deref() + .is_some_and(|secret| secret.chars().count() < 16) + { + bail!("--direct-link-secret must contain at least 16 characters"); + } let source = connect(&options.source_url, "Cloudreve").await?; let target = connect(&options.target_url, "AsterDrive").await?; @@ -162,6 +186,15 @@ pub async fn migrate(options: MigrationOptions) -> Result { migrate_files(&transaction, &source_data, &mut context, &mut report).await?; migrate_metadata(&transaction, &source_data, &context, &mut report).await?; migrate_shares(&transaction, &source_data, &context, &mut report).await?; + migrate_direct_links( + &transaction, + &source_data, + &context, + options.direct_link_secret.as_deref(), + &mut report, + ) + .await?; + migrate_tasks(&transaction, &source_data, &context, &mut report).await?; transaction .commit() @@ -189,6 +222,18 @@ async fn validate_target_schema(db: &DatabaseConnection) -> Result<()> { .count(db) .await .wrap_err("AsterDrive files table is unavailable")?; + ad::entity_properties::Entity::find() + .count(db) + .await + .wrap_err("AsterDrive entity_properties table is unavailable")?; + ad::tags::Entity::find() + .count(db) + .await + .wrap_err("AsterDrive tags table is unavailable")?; + ad::background_tasks::Entity::find() + .count(db) + .await + .wrap_err("AsterDrive background_tasks table is unavailable")?; Ok(()) } @@ -205,6 +250,15 @@ async fn ensure_target_safe(db: &DatabaseConnection, allow_non_empty: bool) -> R ad::file_blobs::Entity::find().count(db).await?, ), ("shares", ad::shares::Entity::find().count(db).await?), + ( + "entity_properties", + ad::entity_properties::Entity::find().count(db).await?, + ), + ("tags", ad::tags::Entity::find().count(db).await?), + ( + "background_tasks", + ad::background_tasks::Entity::find().count(db).await?, + ), ]; let occupied: Vec = counts .into_iter() @@ -249,6 +303,8 @@ struct SourceData { file_entities: Vec, shares: Vec, metadata: Vec, + direct_links: Vec, + tasks: Vec, } impl SourceData { @@ -261,6 +317,8 @@ impl SourceData { let file_entities = cr::file_entities::Entity::find().all(db).await?; let shares = cr::shares::Entity::find().all(db).await?; let metadata = cr::metadata::Entity::find().all(db).await?; + let direct_links = cr::direct_links::Entity::find().all(db).await?; + let tasks = cr::tasks::Entity::find().all(db).await?; Ok(Self { groups: filter_deleted(groups, include_deleted, |model| model.deleted_at.is_some()), @@ -277,6 +335,10 @@ impl SourceData { metadata: filter_deleted(metadata, include_deleted, |model| { model.deleted_at.is_some() }), + direct_links: filter_deleted(direct_links, include_deleted, |model| { + model.deleted_at.is_some() + }), + tasks: filter_deleted(tasks, include_deleted, |model| model.deleted_at.is_some()), }) } @@ -289,6 +351,13 @@ impl SourceData { source_files: self.files.iter().filter(|file| file.r#type == 0).count() as u64, source_entities: self.entities.len() as u64, source_shares: self.shares.len() as u64, + source_direct_links: self.direct_links.len() as u64, + source_tag_assignments: self + .metadata + .iter() + .filter(|metadata| tag_name(&metadata.name).is_some()) + .count() as u64, + source_tasks: self.tasks.len() as u64, ..Default::default() } } @@ -307,9 +376,10 @@ impl SourceData { fn compatibility_warnings(&self) -> Vec { let mut warnings = vec![ "Cloudreve user passwords use SHA/legacy MD5 formats and are replaced by the supplied temporary Argon2 password; every migrated user is marked must_change_password".to_string(), - "OAuth grants, login sessions, background tasks and Cloudreve filesystem events are intentionally not migrated".to_string(), + "OAuth grants, login sessions and Cloudreve filesystem events are intentionally not migrated".to_string(), "Cloudreve Passkeys, WebDAV credentials and two-factor secrets are not portable to AD and must be enrolled again".to_string(), "file objects are reused in their existing local/object-storage locations; the migration does not duplicate object bytes".to_string(), + "Cloudreve tasks are archived as terminal AD system_runtime records; queued, processing and suspending tasks are canceled instead of resumed".to_string(), ]; let symbolic = self.files.iter().filter(|file| file.is_symbolic).count(); if symbolic > 0 { @@ -324,6 +394,9 @@ impl SourceData { unsupported.join(", ") )); } + if !self.direct_links.is_empty() { + warnings.push("Cloudreve direct links require --direct-link-secret to regenerate AD v2 URLs; old /f/... URLs, per-link counters, speed limits and revocation semantics cannot be preserved".to_string()); + } warnings } } @@ -413,6 +486,93 @@ fn share_token(share_id: i64) -> String { format!("cr-{share_id}-{}", &format!("{digest:x}")[..16]) } +fn tag_name(metadata_name: &str) -> Option<&str> { + metadata_name + .strip_prefix("tag:") + .map(str::trim) + .filter(|name| !name.is_empty()) +} + +fn normalize_tag_name(name: &str) -> String { + name.trim().to_lowercase() +} + +fn target_tag_name(name: &str) -> String { + name.trim().chars().take(64).collect() +} + +fn target_tag_color(color: &str) -> String { + let color = color.trim().to_ascii_lowercase(); + if color.len() == 7 + && color.starts_with('#') + && color[1..].bytes().all(|byte| byte.is_ascii_hexdigit()) + { + return color; + } + if color.len() == 4 + && color.starts_with('#') + && color[1..].bytes().all(|byte| byte.is_ascii_hexdigit()) + { + let mut expanded = String::with_capacity(7); + expanded.push('#'); + for character in color[1..].chars() { + expanded.push(character); + expanded.push(character); + } + return expanded; + } + "#3b82f6".to_string() +} + +fn encode_base62(mut value: u64) -> String { + const BASE62: &[u8] = b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + if value == 0 { + return "a".to_string(); + } + let mut encoded = Vec::new(); + while value > 0 { + encoded.push(char::from(BASE62[(value % 62) as usize])); + value /= 62; + } + encoded.iter().rev().collect() +} + +fn direct_link_url( + file_id: i64, + owner_user_id: i64, + file_name: &str, + secret: &str, +) -> Result { + let file_id = u64::try_from(file_id).wrap_err("AD direct link file ID must be non-negative")?; + let mut mac = Hmac::::new_from_slice(secret.as_bytes()) + .map_err(|error| color_eyre::eyre::eyre!("initialize direct link HMAC: {error}"))?; + mac.update(b"direct_link:v2:"); + mac.update(format!("user:{owner_user_id}").as_bytes()); + mac.update(b":"); + mac.update(file_id.to_string().as_bytes()); + let signature = + base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(mac.finalize().into_bytes()); + Ok(format!( + "/d/v2.{}.{}/{}", + encode_base62(file_id), + signature, + urlencoding::encode(file_name) + )) +} + +fn archived_task_status(source_status: &str) -> &'static str { + match source_status { + "completed" => "succeeded", + "error" => "failed", + "canceled" | "queued" | "processing" | "suspending" => "canceled", + _ => "canceled", + } +} + +fn source_task_was_active(source_status: &str) -> bool { + matches!(source_status, "queued" | "processing" | "suspending") +} + fn unique_username(source: &str, source_id: i64, used: &mut HashSet) -> String { let mut base: String = source .trim() @@ -511,6 +671,46 @@ mod tests { assert_eq!(map_driver_type("qiniu"), None); } + #[test] + fn parses_and_normalizes_cloudreve_tags_for_ad() { + assert_eq!(tag_name("tag:Important"), Some("Important")); + assert_eq!(tag_name("tag: Project A "), Some("Project A")); + assert_eq!(tag_name("author"), None); + assert_eq!(normalize_tag_name(" Important "), "important"); + assert_eq!(target_tag_color("#AbC"), "#aabbcc"); + assert_eq!(target_tag_color("#3B82F6"), "#3b82f6"); + assert_eq!(target_tag_color(""), "#3b82f6"); + assert_eq!(target_tag_name(&"x".repeat(80)).chars().count(), 64); + } + + #[test] + fn builds_asterdrive_v2_direct_link_urls() -> Result<()> { + let url = direct_link_url(1, 7, "hello world.txt", "test-direct-link-secret")?; + assert!(url.starts_with("/d/v2.b.")); + assert!(url.ends_with("/hello%20world.txt")); + assert_eq!( + url, + direct_link_url(1, 7, "hello world.txt", "test-direct-link-secret")? + ); + assert_ne!( + url, + direct_link_url(1, 8, "hello world.txt", "test-direct-link-secret")? + ); + Ok(()) + } + + #[test] + fn maps_cloudreve_tasks_to_non_executable_terminal_statuses() { + assert_eq!(archived_task_status("completed"), "succeeded"); + assert_eq!(archived_task_status("error"), "failed"); + assert_eq!(archived_task_status("canceled"), "canceled"); + for status in ["queued", "processing", "suspending"] { + assert!(source_task_was_active(status)); + assert_eq!(archived_task_status(status), "canceled"); + } + assert_eq!(archived_task_status("unknown"), "canceled"); + } + #[tokio::test] async fn migrates_minimal_cloudreve_database() -> Result<()> { let suffix = uuid::Uuid::new_v4(); @@ -533,6 +733,7 @@ mod tests { target_url: target_url.clone(), default_password: "temporary-password".to_string(), local_base_path: "C:/cloudreve".to_string(), + direct_link_secret: Some("test-direct-link-secret".to_string()), include_deleted: false, allow_non_empty_target: false, skip_unsupported_policies: false, @@ -545,7 +746,11 @@ mod tests { assert_eq!(report.migrated_files, 1); assert_eq!(report.migrated_blobs, 1); assert_eq!(report.migrated_shares, 1); - assert_eq!(report.migrated_properties, 1); + assert_eq!(report.migrated_properties, 3); + assert_eq!(report.migrated_tags, 1); + assert_eq!(report.migrated_tag_assignments, 1); + assert_eq!(report.migrated_direct_links, 1); + assert_eq!(report.migrated_tasks, 2); let target = Database::connect(&target_url).await?; assert_eq!(ad::users::Entity::find().count(&target).await?, 1); @@ -553,10 +758,35 @@ mod tests { assert_eq!(ad::files::Entity::find().count(&target).await?, 1); assert_eq!(ad::file_blobs::Entity::find().count(&target).await?, 1); assert_eq!(ad::shares::Entity::find().count(&target).await?, 1); + assert_eq!(ad::tags::Entity::find().count(&target).await?, 1); + assert_eq!( + ad::background_tasks::Entity::find().count(&target).await?, + 2 + ); assert_eq!( ad::entity_properties::Entity::find().count(&target).await?, - 1 + 3 ); + let properties = ad::entity_properties::Entity::find().all(&target).await?; + assert!( + properties.iter().any(|property| { + property.namespace == "system.tags" && property.value.is_none() + }) + ); + let direct_link = properties + .iter() + .find(|property| property.namespace == "cloudreve.direct_links") + .and_then(|property| property.value.as_deref()) + .expect("migrated direct link mapping"); + assert!(direct_link.contains("/d/v2.")); + let tasks = ad::background_tasks::Entity::find().all(&target).await?; + assert!(tasks.iter().all(|task| { + matches!(task.status.as_str(), "succeeded" | "failed" | "canceled") + && task.kind == "system_runtime" + && task.lease_expires_at.is_none() + })); + assert!(tasks.iter().any(|task| task.status == "succeeded")); + assert!(tasks.iter().any(|task| task.status == "canceled")); let blob = ad::file_blobs::Entity::find().one(&target).await?.unwrap(); assert_eq!(blob.storage_path, "uploads/object.bin"); let user = ad::users::Entity::find().one(&target).await?.unwrap(); @@ -592,6 +822,8 @@ mod tests { create_table(db, cr::file_entities::Entity).await?; create_table(db, cr::shares::Entity).await?; create_table(db, cr::metadata::Entity).await?; + create_table(db, cr::direct_links::Entity).await?; + create_table(db, cr::tasks::Entity).await?; Ok(()) } @@ -608,6 +840,9 @@ mod tests { create_table(db, ad::file_versions::Entity).await?; create_table(db, ad::entity_properties::Entity).await?; create_table(db, ad::shares::Entity).await?; + create_table(db, ad::teams::Entity).await?; + create_table(db, ad::tags::Entity).await?; + create_table(db, ad::background_tasks::Entity).await?; Ok(()) } @@ -730,6 +965,18 @@ mod tests { } .insert(db) .await?; + cr::metadata::ActiveModel { + created_at: Set(now), + updated_at: Set(now), + deleted_at: Set(None), + name: Set("tag:Important".to_string()), + value: Set("#abc".to_string()), + is_public: Set(true), + file_id: Set(file.id), + ..Default::default() + } + .insert(db) + .await?; cr::shares::ActiveModel { created_at: Set(now), updated_at: Set(now), @@ -746,6 +993,34 @@ mod tests { } .insert(db) .await?; + cr::direct_links::ActiveModel { + created_at: Set(now), + updated_at: Set(now), + deleted_at: Set(None), + name: Set("legacy-name.txt".to_string()), + downloads: Set(7), + speed: Set(1024), + file_id: Set(file.id), + ..Default::default() + } + .insert(db) + .await?; + for status in ["completed", "processing"] { + cr::tasks::ActiveModel { + created_at: Set(now), + updated_at: Set(now), + deleted_at: Set(None), + r#type: Set("remote_download".to_string()), + status: Set(status.to_string()), + public_state: Set(json!({"progress": 50})), + private_state: Set(Some("legacy-private-state".to_string())), + correlation_id: Set(None), + user_tasks: Set(Some(user.id)), + ..Default::default() + } + .insert(db) + .await?; + } Ok(()) } } diff --git a/src/migration/phases.rs b/src/migration/phases.rs index 929c3a5..32ae831 100644 --- a/src/migration/phases.rs +++ b/src/migration/phases.rs @@ -415,19 +415,90 @@ pub(super) async fn migrate_metadata( context: &MigrationContext, report: &mut MigrationReport, ) -> Result<()> { + let source_files: HashMap = + source.files.iter().map(|file| (file.id, file)).collect(); + let mut tags: HashMap<(i64, String), i64> = HashMap::new(); for metadata in &source.metadata { - let Some(file_id) = context.files.get(&metadata.file_id).copied() else { + let Some(source_file) = source_files.get(&metadata.file_id).copied() else { + report.skipped += 1; + continue; + }; + let target_entity = if source_file.r#type == 0 { + context + .files + .get(&metadata.file_id) + .copied() + .map(|id| ("file", id)) + } else { + context + .folders + .get(&metadata.file_id) + .copied() + .map(|id| ("folder", id)) + }; + let Some((entity_type, entity_id)) = target_entity else { report.skipped += 1; continue; }; + + if let Some(source_tag_name) = tag_name(&metadata.name) { + let Some(owner_user_id) = context.users.get(&source_file.owner_id).copied() else { + report.skipped += 1; + continue; + }; + let name = target_tag_name(source_tag_name); + if name.is_empty() { + report.skipped += 1; + continue; + } + let normalized_name = normalize_tag_name(&name); + let tag_id = match tags.get(&(owner_user_id, normalized_name.clone())) { + Some(tag_id) => *tag_id, + None => { + let tag = ad::tags::ActiveModel { + scope_type: Set("personal".to_string()), + owner_user_id: Set(Some(owner_user_id)), + team_id: Set(None), + name: Set(name), + normalized_name: Set(normalized_name.clone()), + color: Set(target_tag_color(&metadata.value)), + sort_order: Set(0), + created_at: Set(metadata.created_at), + updated_at: Set(metadata.updated_at), + ..Default::default() + } + .insert(transaction) + .await + .wrap_err_with(|| format!("migrate tag metadata {}", metadata.id))?; + tags.insert((owner_user_id, normalized_name), tag.id); + report.migrated_tags += 1; + tag.id + } + }; + ad::entity_properties::ActiveModel { + entity_type: Set(entity_type.to_string()), + entity_id: Set(entity_id), + namespace: Set("system.tags".to_string()), + name: Set(tag_id.to_string()), + value: Set(None), + ..Default::default() + } + .insert(transaction) + .await + .wrap_err_with(|| format!("attach migrated tag for metadata {}", metadata.id))?; + report.migrated_properties += 1; + report.migrated_tag_assignments += 1; + continue; + } + let namespace = if metadata.is_public { "cloudreve.public" } else { "cloudreve.private" }; ad::entity_properties::ActiveModel { - entity_type: Set("file".to_string()), - entity_id: Set(file_id), + entity_type: Set(entity_type.to_string()), + entity_id: Set(entity_id), namespace: Set(namespace.to_string()), name: Set(metadata.name.clone()), value: Set(Some(metadata.value.clone())), @@ -441,6 +512,163 @@ pub(super) async fn migrate_metadata( Ok(()) } +pub(super) async fn migrate_direct_links( + transaction: &DatabaseTransaction, + source: &SourceData, + context: &MigrationContext, + direct_link_secret: Option<&str>, + report: &mut MigrationReport, +) -> Result<()> { + if source.direct_links.is_empty() { + return Ok(()); + } + let Some(secret) = direct_link_secret else { + report.skipped += source.direct_links.len(); + report.warnings.push(format!( + "{} Cloudreve direct links were not regenerated because --direct-link-secret was not supplied", + source.direct_links.len() + )); + return Ok(()); + }; + let source_files: HashMap = + source.files.iter().map(|file| (file.id, file)).collect(); + for link in &source.direct_links { + if link.deleted_at.is_some() { + report.skipped += 1; + continue; + } + let Some(source_file) = source_files.get(&link.file_id).copied() else { + report.skipped += 1; + continue; + }; + let Some(file_id) = context.files.get(&link.file_id).copied() else { + report.skipped += 1; + continue; + }; + let Some(owner_user_id) = context.users.get(&source_file.owner_id).copied() else { + report.skipped += 1; + continue; + }; + let url = direct_link_url(file_id, owner_user_id, &source_file.name, secret)?; + ad::entity_properties::ActiveModel { + entity_type: Set("file".to_string()), + entity_id: Set(file_id), + namespace: Set("cloudreve.direct_links".to_string()), + name: Set(link.id.to_string()), + value: Set(Some( + json!({ + "url": url, + "source_direct_link_id": link.id, + "source_file_id": link.file_id, + "source_name": link.name, + "source_downloads": link.downloads, + "source_speed_limit": link.speed, + }) + .to_string(), + )), + ..Default::default() + } + .insert(transaction) + .await + .wrap_err_with(|| format!("archive direct link {} mapping", link.id))?; + report.migrated_properties += 1; + report.migrated_direct_links += 1; + } + Ok(()) +} + +pub(super) async fn migrate_tasks( + transaction: &DatabaseTransaction, + source: &SourceData, + context: &MigrationContext, + report: &mut MigrationReport, +) -> Result<()> { + let active_count = source + .tasks + .iter() + .filter(|task| source_task_was_active(&task.status)) + .count(); + if active_count > 0 { + report.warnings.push(format!( + "{active_count} active Cloudreve tasks were archived as canceled terminal records and were not resumed" + )); + } + + for task in &source.tasks { + let status = archived_task_status(&task.status); + let duration_ms = (task.updated_at - task.created_at) + .num_milliseconds() + .max(0); + let task_name = format!( + "cloudreve-legacy-{}", + task.r#type + .replace(|character: char| !character.is_ascii_alphanumeric(), "-") + ); + let result = json!({ + "duration_ms": duration_ms, + "summary": format!("Archived Cloudreve {} task with source status {}", task.r#type, task.status), + }); + let runtime = json!({ + "source": "cloudreve", + "source_task_id": task.id, + "source_type": task.r#type, + "source_status": task.status, + "source_public_state": task.public_state, + "source_private_state": task.private_state, + "source_correlation_id": task.correlation_id, + "source_deleted_at": task.deleted_at, + "archived_without_resume": true, + }); + let started_at = (!matches!(task.status.as_str(), "queued")).then_some(task.created_at); + let expires_at = task + .updated_at + .checked_add_signed(chrono::Duration::days(36_500)) + .unwrap_or(task.updated_at); + ad::background_tasks::ActiveModel { + kind: Set("system_runtime".to_string()), + status: Set(status.to_string()), + creator_user_id: Set(task + .user_tasks + .and_then(|user_id| context.users.get(&user_id).copied())), + team_id: Set(None), + share_id: Set(None), + display_name: Set(format!("Cloudreve task: {}", task.r#type)), + payload_json: Set(json!({"task_name": task_name}).to_string()), + result_json: Set(Some(result.to_string())), + steps_json: Set(Some("[]".to_string())), + progress_current: Set(i64::from(status == "succeeded")), + progress_total: Set(1), + status_text: Set(Some(format!( + "Archived from Cloudreve with source status {}; execution was not resumed", + task.status + ))), + attempt_count: Set(0), + max_attempts: Set(1), + next_run_at: Set(task.updated_at), + processing_token: Set(0), + processing_started_at: Set(None), + last_heartbeat_at: Set(None), + lease_expires_at: Set(None), + started_at: Set(started_at), + finished_at: Set(Some(task.updated_at)), + last_error: Set( + (status == "failed").then(|| "Cloudreve task ended with status error".to_string()) + ), + failure_can_retry: Set(Some(false)), + expires_at: Set(expires_at), + created_at: Set(task.created_at), + updated_at: Set(task.updated_at), + runtime_json: Set(Some(runtime.to_string())), + ..Default::default() + } + .insert(transaction) + .await + .wrap_err_with(|| format!("archive Cloudreve task {}", task.id))?; + report.migrated_tasks += 1; + } + Ok(()) +} + pub(super) async fn migrate_shares( transaction: &DatabaseTransaction, source: &SourceData, From 6cc083cf6586bba0627416820124c22b376df8eb Mon Sep 17 00:00:00 2001 From: Cg8 <5712.cg8@gmail.com> Date: Mon, 13 Jul 2026 19:58:46 +0800 Subject: [PATCH 3/7] feat: add structured migration reports --- README.md | 19 +- docs/cloudreve-to-asterdrive-field-mapping.md | 17 + docs/migration-architecture.md | 25 +- src/main.rs | 26 +- src/migration/mod.rs | 563 +++++++++++++++++- src/migration/phases.rs | 145 ++++- 6 files changed, 750 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index c748dba..59221e2 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,8 @@ The tool reuses existing storage objects. For local storage, pass the directory ```powershell cargo run -- check ` --source-url "sqlite://C:/cloudreve/cloudreve.db?mode=ro" ` - --target-url "sqlite://C:/asterdrive/asterdrive.db" + --target-url "sqlite://C:/asterdrive/asterdrive.db" ` + --report-path "C:/migration/cloudreve-preflight.json" ``` The URLs can also be provided through `CLOUDREVE_DATABASE_URL` and `ASTERDRIVE_DATABASE_URL`. SQLite, MySQL and PostgreSQL URLs supported by SeaORM can be used. @@ -48,7 +49,8 @@ $env:ASTER_DIRECT_LINK_SECRET = "use-the-same-auth.direct_link_secret-as-ad" cargo run -- migrate ` --source-url "sqlite://C:/cloudreve/cloudreve.db?mode=ro" ` --target-url "sqlite://C:/asterdrive/asterdrive.db" ` - --local-base-path "C:/cloudreve" + --local-base-path "C:/cloudreve" ` + --report-path "C:/migration/cloudreve-to-ad.json" ``` `ASTER_DIRECT_LINK_SECRET` is optional only when there are no direct links to regenerate. When supplied, each active Cloudreve `direct_links` row is mapped to a newly signed AD `/d/v2...` URL and stored under the target file's `cloudreve.direct_links` properties. Existing Cloudreve `/f/...` URLs cannot remain valid because the URL path, ID and signing algorithms differ. @@ -57,6 +59,19 @@ Use `--dry-run` to perform preflight checks without writing the target. The targ Cloudreve Qiniu, Upyun, remote-node, OneDrive and encrypted storage policies cannot be reused safely by AD. The migration stops when they are present. `--skip-unsupported-policies` explicitly omits those policies and all dependent files. +## JSON migration report + +`--report-path` writes a pretty-printed JSON report for both `check` and `migrate`. A completed migration report contains: + +- source and migrated counts for every supported object type +- skipped counts grouped by type, plus source ID and reason for every skipped object +- sorted source-to-target ID mappings for policies, groups, users, folders, blobs, files, shares and archived tasks +- every Cloudreve tag metadata assignment and its AD tag/entity IDs +- every regenerated direct-link URL and its Cloudreve/AD file IDs +- post-commit database count checks, imported-task terminal-state checks, tag-binding checks and direct-link property checks + +The CLI writes the JSON report before returning a validation error. A failed post-migration check therefore produces a usable report and exits with a non-zero status. Direct-link URLs are bearer-style public capabilities, so the report file must be stored with restricted access. + ## Verification before push Every migration feature must include focused tests plus end-to-end coverage where it writes database state. Before pushing, run and require all of these commands to pass: diff --git a/docs/cloudreve-to-asterdrive-field-mapping.md b/docs/cloudreve-to-asterdrive-field-mapping.md index 21b951a..faedd47 100644 --- a/docs/cloudreve-to-asterdrive-field-mapping.md +++ b/docs/cloudreve-to-asterdrive-field-mapping.md @@ -516,3 +516,20 @@ Cloudreve 任务不能恢复到 AD 执行器中，但可以保存为不可执行 | 任务验证 | 所有导入任务均为 `succeeded/failed/canceled`，无 `pending/retry/processing`，`lease_expires_at` 为空 | | AD 启动后 | 重新生成媒体元数据、缩略图和搜索索引；检查 storage_used 汇总 | | 切换前 | 冻结 Cloudreve 写入，再跑最终迁移或增量校验，备份目标库 | + +## 16. JSON 报告字段 + +通过 `--report-path ` 可为 `check` 或 `migrate` 输出 `schema_version=1` 的 JSON 报告。 + +| 报告字段 | 内容 | +|---|---| +| `source_*` | Cloudreve 各类源对象数量 | +| `migrated_*` | 本次写入 AD 的各类对象数量 | +| `skipped_by_type` | 按 `file`、`blob`、`share`、`direct_link` 等类型聚合的跳过数量 | +| `skipped_objects` | 每条跳过记录的对象类型、Cloudreve source ID 和明确原因 | +| `mappings` | policy、policy group、user、folder、blob、file、share、task 的排序 source ID -> target ID | +| `tag_assignments` | Cloudreve metadata ID、源 file/folder ID、AD entity ID、AD tag ID 和标签名 | +| `direct_links` | Cloudreve direct-link/file ID、AD file ID、新 URL、原名称、下载次数和限速 | +| `validation` | 是否执行/通过，以及每项检查的 expected、actual 和失败信息 | + +当前提交后校验覆盖核心表增量数量、导入任务是否全为终态且无 lease、`system.tags` 关联是否存在、`cloudreve.direct_links` 属性中的 URL 是否与报告一致。报告不会保存数据库密码、存储密钥或 Cloudreve task private state，但会包含新 direct-link URL，因此必须限制报告文件访问权限。 diff --git a/docs/migration-architecture.md b/docs/migration-architecture.md index 487a506..6af8033 100644 --- a/docs/migration-architecture.md +++ b/docs/migration-architecture.md @@ -121,7 +121,7 @@ TUI 只订阅 engine progress event，不直接访问数据库或实现迁移逻 - source 和 target 存储连接 - 冲突处理、并发、重试和验证选项 -敏感字段不得出现在普通日志、迁移报告或 TUI 中。 +密码、数据库凭据、OAuth token、存储 secret 和任务 private state 不得出现在普通日志、迁移报告或 TUI 中。JSON 报告按需求包含重新签发的 direct-link URL；该 URL 本身是公开访问能力，报告文件必须按敏感文件保护。 ### Output Renderer @@ -264,15 +264,22 @@ checkpoint 必须在目标记录成功提交后更新，不能先推进 cursor ### Report Builder -报告内容： +当前实现通过 `--report-path` 为 `check` 和 `migrate` 输出 `schema_version=1` 的 pretty JSON，同时保留终端摘要。报告内容： - 源和目标对象统计 -- 成功、复用、跳过、重命名、失败数量 -- 已迁移字节数和吞吐量 +- 各对象类型迁移数量 +- 按对象类型聚合的跳过数量 +- 每个跳过对象的 source ID 和原因 +- policy、policy group、user、folder、blob、file、share、task 的 source ID -> target ID +- Cloudreve tag metadata -> AD tag/entity 关联 +- Cloudreve direct-link ID -> AD file ID/new URL - 不兼容能力和人工处理项 -- 校验结果 +- 提交后重新查询 AD 得到的数量校验 +- 导入任务终态/lease 校验 +- `system.tags` 和 `cloudreve.direct_links` property 校验 - 错误和警告摘要 -- 可选对象级错误明细 + +校验失败时 CLI 必须先写报告和终端摘要，再以非零状态退出。当前报告只覆盖成功提交后的校验；事务提交前发生的致命错误仍直接返回错误，后续可引入 migration run 持久化来保存失败中间态。 ## 3.3 Layer 3：Source Adapter Layer @@ -749,7 +756,7 @@ Cloudreve entity ID、object key、路径或 `policy+path+size` 摘要都不能 | Direct links | 提供 AD `direct_link_secret` 时生成 v2 URL，并以 `cloudreve.direct_links` property 保存源 ID 映射 | 增加独立 JSON/CSV 报告和可选旧 URL 重定向层 | | Cloudreve 任务 | 已全部写成 AD `system_runtime` 终态历史；活动任务归档为 canceled | 后续可增加独立 legacy task archive 表，避免占用运维任务列表 | | 目标事务 | 已实现单次全局事务 | 每 batch 事务 + checkpoint，适合大数据集 | -| ID mapping | 已实现内存 HashMap | 持久化 object mapping table | +| ID mapping | 已实现内存 HashMap，并在 JSON 报告中导出排序后的核心对象映射 | 持久化 object mapping table，支持跨进程 resume | | 断点续传 | 未实现 | stage cursor + resume | | 幂等重复运行 | 未完整实现 | mapping 驱动 upsert/reuse | | 冲突策略 | 仅 username 自动后缀，其他冲突依赖 DB 报错 | fail/rename/skip/reuse/update 可配置 | @@ -757,8 +764,8 @@ Cloudreve entity ID、object key、路径或 `policy+path+size` 摘要都不能 | 真正内容去重 | 未实现 | hash + policy 去重 | | ref_count 重算 | 基于源关联计算初值 | 迁移后从 AD 关系统一重算 | | storage_used 重算 | 当前复制 Cloudreve 用户统计 | 迁移后按 AD 语义统一重算 | -| 验证编排 | 有标签、直链、任务独立规则测试和 SQLite 端到端测试 | 对实际运行执行 inventory/关系/存储一致性验证 | -| 报告 | 当前输出基础统计和警告 | run/stage/object 级报告，Human + JSON | +| 验证编排 | 提交后重查核心表数量、导入任务终态、标签绑定和直链 property；有 SQLite 端到端测试 | 增加 parent/owner/blob/ref_count/storage_used 和真实对象可读性验证 | +| 报告 | 已支持终端摘要和 `--report-path` JSON，包含分类跳过原因、核心 ID 映射、标签/直链记录和校验结果 | 增加失败中间态、stage 进度、吞吐量、字节数和独立 CSV 导出 | | TUI | 未接入新迁移核心 | 可选进度视图 | | 多源系统插件 | 未实现 | SourceAdapter 插件化 | diff --git a/src/main.rs b/src/main.rs index 367401d..b276227 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,10 @@ -use aster_drive_migration::migration::{MigrationOptions, inspect, migrate}; +use std::path::{Path, PathBuf}; + +use aster_drive_migration::migration::{ + MigrationOptions, MigrationReport, inspect, migrate, write_json_report, +}; use clap::{Parser, Subcommand}; -use color_eyre::eyre::Result; +use color_eyre::eyre::{Result, bail}; #[derive(Debug, Parser)] #[command(name = "aster-drive-migration", version, about)] @@ -23,6 +27,8 @@ struct ConnectionArgs { target_url: String, #[arg(long)] include_deleted: bool, + #[arg(long, value_name = "PATH")] + report_path: Option, } #[derive(Debug, clap::Args)] @@ -49,9 +55,10 @@ async fn main() -> Result<()> { match Cli::parse().command { Command::Check(args) => { let report = inspect(&args.source_url, &args.target_url, args.include_deleted).await?; - println!("{report}"); + emit_report(args.report_path.as_deref(), &report)?; } Command::Migrate(args) => { + let report_path = args.connection.report_path.clone(); let report = migrate(MigrationOptions { source_url: args.connection.source_url, target_url: args.connection.target_url, @@ -64,8 +71,19 @@ async fn main() -> Result<()> { dry_run: args.dry_run, }) .await?; - println!("{report}"); + emit_report(report_path.as_deref(), &report)?; } } Ok(()) } + +fn emit_report(report_path: Option<&Path>, report: &MigrationReport) -> Result<()> { + if let Some(path) = report_path { + write_json_report(path, report)?; + } + println!("{report}"); + if report.validation.performed && !report.validation.passed { + bail!("migration committed but post-migration validation failed; inspect the JSON report"); + } + Ok(()) +} diff --git a/src/migration/mod.rs b/src/migration/mod.rs index e6a7f7d..72843ce 100644 --- a/src/migration/mod.rs +++ b/src/migration/mod.rs @@ -1,5 +1,6 @@ -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::fmt::{self, Write}; +use std::path::Path; use argon2::Argon2; use argon2::password_hash::{PasswordHasher, SaltString}; @@ -7,9 +8,10 @@ use base64::Engine; use color_eyre::eyre::{Result, WrapErr, bail}; use hmac::{Hmac, Mac}; use sea_orm::{ - ActiveModelTrait, Database, DatabaseConnection, EntityTrait, PaginatorTrait, Set, - TransactionTrait, + ActiveModelTrait, ColumnTrait, Database, DatabaseConnection, EntityTrait, PaginatorTrait, + QueryFilter, Set, TransactionTrait, }; +use serde::Serialize; use serde_json::{Value, json}; use sha2::{Digest, Sha256}; @@ -29,8 +31,10 @@ pub struct MigrationOptions { pub dry_run: bool, } -#[derive(Debug, Default, Clone)] +#[derive(Debug, Clone, Serialize)] pub struct MigrationReport { + pub schema_version: u32, + pub generated_at: chrono::DateTime, pub source_users: u64, pub source_groups: u64, pub source_policies: u64, @@ -57,6 +61,158 @@ pub struct MigrationReport { pub skipped: usize, pub dry_run: bool, pub warnings: Vec, + pub skipped_by_type: BTreeMap, + pub skipped_objects: Vec, + pub mappings: MigrationMappings, + pub direct_links: Vec, + pub tag_assignments: Vec, + pub validation: MigrationValidation, +} + +impl Default for MigrationReport { + fn default() -> Self { + Self { + schema_version: 1, + generated_at: chrono::Utc::now(), + source_users: 0, + source_groups: 0, + source_policies: 0, + source_folders: 0, + source_files: 0, + source_entities: 0, + source_shares: 0, + source_direct_links: 0, + source_tag_assignments: 0, + source_tasks: 0, + migrated_users: 0, + migrated_policy_groups: 0, + migrated_policies: 0, + migrated_folders: 0, + migrated_files: 0, + migrated_blobs: 0, + migrated_versions: 0, + migrated_shares: 0, + migrated_properties: 0, + migrated_tags: 0, + migrated_tag_assignments: 0, + migrated_direct_links: 0, + migrated_tasks: 0, + skipped: 0, + dry_run: false, + warnings: Vec::new(), + skipped_by_type: BTreeMap::new(), + skipped_objects: Vec::new(), + mappings: MigrationMappings::default(), + direct_links: Vec::new(), + tag_assignments: Vec::new(), + validation: MigrationValidation::default(), + } + } +} + +#[derive(Debug, Clone, Serialize)] +pub struct SkippedObject { + pub object_type: String, + pub source_id: Option, + pub reason: String, +} + +#[derive(Debug, Default, Clone, Serialize)] +pub struct MigrationMappings { + pub policies: Vec, + pub policy_groups: Vec, + pub users: Vec, + pub folders: Vec, + pub blobs: Vec, + pub files: Vec, + pub shares: Vec, + pub tasks: Vec, +} + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct IdMapping { + pub source_id: i64, + pub target_id: i64, +} + +#[derive(Debug, Clone, Serialize)] +pub struct DirectLinkReport { + pub source_direct_link_id: i64, + pub source_file_id: i64, + pub target_file_id: i64, + pub source_name: String, + pub source_downloads: i64, + pub source_speed_limit: i64, + pub url: String, +} + +#[derive(Debug, Clone, Serialize)] +pub struct TagAssignmentReport { + pub source_metadata_id: i64, + pub source_entity_id: i64, + pub target_entity_type: String, + pub target_entity_id: i64, + pub target_tag_id: i64, + pub tag_name: String, +} + +#[derive(Debug, Clone, Serialize)] +pub struct MigrationValidation { + pub performed: bool, + pub passed: bool, + pub checks: Vec, +} + +impl Default for MigrationValidation { + fn default() -> Self { + Self { + performed: false, + passed: true, + checks: Vec::new(), + } + } +} + +#[derive(Debug, Clone, Serialize)] +pub struct ValidationCheck { + pub name: String, + pub passed: bool, + pub expected: String, + pub actual: String, + pub message: Option, +} + +impl MigrationReport { + fn record_skip( + &mut self, + object_type: &str, + source_id: Option, + reason: impl Into, + ) { + self.skipped += 1; + *self + .skipped_by_type + .entry(object_type.to_string()) + .or_default() += 1; + self.skipped_objects.push(SkippedObject { + object_type: object_type.to_string(), + source_id, + reason: reason.into(), + }); + } + + fn set_mappings(&mut self, context: &MigrationContext) { + self.mappings = MigrationMappings { + policies: sorted_id_mappings(&context.policies), + policy_groups: sorted_id_mappings(&context.policy_groups), + users: sorted_id_mappings(&context.users), + folders: sorted_id_mappings(&context.folders), + blobs: sorted_id_mappings(&context.blobs), + files: sorted_id_mappings(&context.files), + shares: sorted_id_mappings(&context.shares), + tasks: sorted_id_mappings(&context.tasks), + }; + } } impl fmt::Display for MigrationReport { @@ -98,6 +254,27 @@ impl fmt::Display for MigrationReport { self.migrated_tasks )?; writeln!(output, "skipped: {}", self.skipped)?; + if !self.skipped_by_type.is_empty() { + let categories = self + .skipped_by_type + .iter() + .map(|(object_type, count)| format!("{object_type}={count}")) + .collect::>() + .join(", "); + writeln!(output, "skipped_by_type: {categories}")?; + } + if self.validation.performed { + writeln!( + output, + "validation: {} ({} checks)", + if self.validation.passed { + "passed" + } else { + "failed" + }, + self.validation.checks.len() + )?; + } } if !self.warnings.is_empty() { writeln!(output, "warnings:")?; @@ -109,6 +286,20 @@ impl fmt::Display for MigrationReport { } } +pub fn write_json_report(path: impl AsRef, report: &MigrationReport) -> Result<()> { + let path = path.as_ref(); + if let Some(parent) = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + std::fs::create_dir_all(parent) + .wrap_err_with(|| format!("create report directory {}", parent.display()))?; + } + let contents = serde_json::to_vec_pretty(report).wrap_err("serialize migration report")?; + std::fs::write(path, contents) + .wrap_err_with(|| format!("write migration report {}", path.display())) +} + pub async fn inspect( source_url: &str, target_url: &str, @@ -141,6 +332,7 @@ pub async fn migrate(options: MigrationOptions) -> Result { let source_data = SourceData::load(&source, options.include_deleted).await?; validate_target_schema(&target).await?; ensure_target_safe(&target, options.allow_non_empty_target).await?; + let target_before = TargetCounts::load(&target).await?; let unsupported = source_data.unsupported_policy_types(); if !unsupported.is_empty() && !options.skip_unsupported_policies { @@ -185,7 +377,7 @@ pub async fn migrate(options: MigrationOptions) -> Result { migrate_blobs(&transaction, &source_data, &mut context, &mut report).await?; migrate_files(&transaction, &source_data, &mut context, &mut report).await?; migrate_metadata(&transaction, &source_data, &context, &mut report).await?; - migrate_shares(&transaction, &source_data, &context, &mut report).await?; + migrate_shares(&transaction, &source_data, &mut context, &mut report).await?; migrate_direct_links( &transaction, &source_data, @@ -194,12 +386,15 @@ pub async fn migrate(options: MigrationOptions) -> Result { &mut report, ) .await?; - migrate_tasks(&transaction, &source_data, &context, &mut report).await?; + migrate_tasks(&transaction, &source_data, &mut context, &mut report).await?; transaction .commit() .await .wrap_err("commit AsterDrive migration transaction")?; + report.set_mappings(&context); + report.validation = validate_migration_result(&target, &target_before, &report).await?; + report.generated_at = chrono::Utc::now(); Ok(report) } @@ -274,6 +469,283 @@ async fn ensure_target_safe(db: &DatabaseConnection, allow_non_empty: bool) -> R Ok(()) } +#[derive(Debug, Clone)] +struct TargetCounts { + policies: u64, + policy_groups: u64, + users: u64, + user_profiles: u64, + folders: u64, + blobs: u64, + files: u64, + versions: u64, + shares: u64, + properties: u64, + tags: u64, + tasks: u64, +} + +impl TargetCounts { + async fn load(db: &DatabaseConnection) -> Result { + Ok(Self { + policies: ad::storage_policies::Entity::find().count(db).await?, + policy_groups: ad::storage_policy_groups::Entity::find().count(db).await?, + users: ad::users::Entity::find().count(db).await?, + user_profiles: ad::user_profiles::Entity::find().count(db).await?, + folders: ad::folders::Entity::find().count(db).await?, + blobs: ad::file_blobs::Entity::find().count(db).await?, + files: ad::files::Entity::find().count(db).await?, + versions: ad::file_versions::Entity::find().count(db).await?, + shares: ad::shares::Entity::find().count(db).await?, + properties: ad::entity_properties::Entity::find().count(db).await?, + tags: ad::tags::Entity::find().count(db).await?, + tasks: ad::background_tasks::Entity::find().count(db).await?, + }) + } +} + +fn count_check(name: &str, before: u64, migrated: usize, actual: u64) -> ValidationCheck { + let migrated = u64::try_from(migrated).unwrap_or(u64::MAX); + let expected = before.saturating_add(migrated); + ValidationCheck { + name: name.to_string(), + passed: actual == expected, + expected: expected.to_string(), + actual: actual.to_string(), + message: (actual != expected) + .then(|| format!("expected baseline {before} plus {migrated} migrated records")), + } +} + +fn invariant_check(name: &str, expected: usize, actual: usize, message: &str) -> ValidationCheck { + ValidationCheck { + name: name.to_string(), + passed: actual == expected, + expected: expected.to_string(), + actual: actual.to_string(), + message: (actual != expected).then(|| message.to_string()), + } +} + +async fn validate_migration_result( + db: &DatabaseConnection, + before: &TargetCounts, + report: &MigrationReport, +) -> Result { + let after = TargetCounts::load(db).await?; + let mut checks = vec![ + count_check( + "storage_policies_count", + before.policies, + report.migrated_policies, + after.policies, + ), + count_check( + "storage_policy_groups_count", + before.policy_groups, + report.migrated_policy_groups, + after.policy_groups, + ), + count_check( + "users_count", + before.users, + report.migrated_users, + after.users, + ), + count_check( + "user_profiles_count", + before.user_profiles, + report.migrated_users, + after.user_profiles, + ), + count_check( + "folders_count", + before.folders, + report.migrated_folders, + after.folders, + ), + count_check( + "file_blobs_count", + before.blobs, + report.migrated_blobs, + after.blobs, + ), + count_check( + "files_count", + before.files, + report.migrated_files, + after.files, + ), + count_check( + "file_versions_count", + before.versions, + report.migrated_versions, + after.versions, + ), + count_check( + "shares_count", + before.shares, + report.migrated_shares, + after.shares, + ), + count_check( + "entity_properties_count", + before.properties, + report.migrated_properties, + after.properties, + ), + count_check("tags_count", before.tags, report.migrated_tags, after.tags), + count_check( + "background_tasks_count", + before.tasks, + report.migrated_tasks, + after.tasks, + ), + invariant_check( + "policy_mappings_count", + report.migrated_policies, + report.mappings.policies.len(), + "storage policy source-to-target mappings are incomplete", + ), + invariant_check( + "policy_group_mappings_count", + report.migrated_policy_groups, + report.mappings.policy_groups.len(), + "policy group source-to-target mappings are incomplete", + ), + invariant_check( + "user_mappings_count", + report.migrated_users, + report.mappings.users.len(), + "user source-to-target mappings are incomplete", + ), + invariant_check( + "folder_mappings_count", + report.migrated_folders, + report.mappings.folders.len(), + "folder source-to-target mappings are incomplete", + ), + invariant_check( + "blob_mappings_count", + report.migrated_blobs, + report.mappings.blobs.len(), + "blob source-to-target mappings are incomplete", + ), + invariant_check( + "file_mappings_count", + report.migrated_files, + report.mappings.files.len(), + "file source-to-target mappings are incomplete", + ), + invariant_check( + "share_mappings_count", + report.migrated_shares, + report.mappings.shares.len(), + "share source-to-target mappings are incomplete", + ), + invariant_check( + "task_mappings_count", + report.migrated_tasks, + report.mappings.tasks.len(), + "task source-to-target mappings are incomplete", + ), + ]; + + let task_ids = report + .mappings + .tasks + .iter() + .map(|mapping| mapping.target_id) + .collect::>(); + let mut imported_tasks = Vec::new(); + for chunk in task_ids.chunks(500) { + imported_tasks.extend( + ad::background_tasks::Entity::find() + .filter(ad::background_tasks::Column::Id.is_in(chunk.iter().copied())) + .all(db) + .await?, + ); + } + checks.push(invariant_check( + "imported_tasks_exist", + task_ids.len(), + imported_tasks.len(), + "one or more imported task IDs are missing", + )); + let terminal_tasks = imported_tasks + .iter() + .filter(|task| { + matches!(task.status.as_str(), "succeeded" | "failed" | "canceled") + && task.lease_expires_at.is_none() + }) + .count(); + checks.push(invariant_check( + "imported_tasks_are_terminal", + imported_tasks.len(), + terminal_tasks, + "imported tasks must be terminal and have no active lease", + )); + + let tag_properties = ad::entity_properties::Entity::find() + .filter(ad::entity_properties::Column::Namespace.eq("system.tags")) + .all(db) + .await?; + let tag_binding_keys = tag_properties + .into_iter() + .map(|property| (property.entity_type, property.entity_id, property.name)) + .collect::>(); + let valid_tag_assignments = report + .tag_assignments + .iter() + .filter(|assignment| { + tag_binding_keys.contains(&( + assignment.target_entity_type.clone(), + assignment.target_entity_id, + assignment.target_tag_id.to_string(), + )) + }) + .count(); + checks.push(invariant_check( + "tag_assignments_exist", + report.tag_assignments.len(), + valid_tag_assignments, + "one or more system.tags bindings are missing", + )); + + let direct_link_properties = ad::entity_properties::Entity::find() + .filter(ad::entity_properties::Column::Namespace.eq("cloudreve.direct_links")) + .all(db) + .await?; + let direct_link_values = direct_link_properties + .into_iter() + .map(|property| ((property.entity_id, property.name), property.value)) + .collect::>(); + let valid_direct_links = report + .direct_links + .iter() + .filter(|link| { + direct_link_values + .get(&(link.target_file_id, link.source_direct_link_id.to_string())) + .and_then(|value| value.as_deref()) + .and_then(|value| serde_json::from_str::(value).ok()) + .and_then(|value| value.get("url").and_then(Value::as_str).map(str::to_string)) + .is_some_and(|url| url == link.url) + }) + .count(); + checks.push(invariant_check( + "direct_link_mappings_exist", + report.direct_links.len(), + valid_direct_links, + "one or more cloudreve.direct_links properties are missing or changed", + )); + + Ok(MigrationValidation { + performed: true, + passed: checks.iter().all(|check| check.passed), + checks, + }) +} + fn hash_password(password: &str) -> Result { let salt = SaltString::encode_b64(uuid::Uuid::new_v4().as_bytes()) .map_err(|error| color_eyre::eyre::eyre!("create password salt: {error}"))?; @@ -292,6 +764,20 @@ struct MigrationContext { folders: HashMap, blobs: HashMap, files: HashMap, + shares: HashMap, + tasks: HashMap, +} + +fn sorted_id_mappings(values: &HashMap) -> Vec { + let mut mappings = values + .iter() + .map(|(source_id, target_id)| IdMapping { + source_id: *source_id, + target_id: *target_id, + }) + .collect::>(); + mappings.sort_by_key(|mapping| mapping.source_id); + mappings } struct SourceData { @@ -711,6 +1197,58 @@ mod tests { assert_eq!(archived_task_status("unknown"), "canceled"); } + #[test] + fn records_skipped_objects_by_type() { + let mut report = MigrationReport::default(); + report.record_skip("file", Some(42), "missing blob"); + report.record_skip("file", Some(43), "symbolic file"); + report.record_skip("share", None, "missing target"); + + assert_eq!(report.skipped, 3); + assert_eq!(report.skipped_by_type.get("file"), Some(&2)); + assert_eq!(report.skipped_by_type.get("share"), Some(&1)); + assert_eq!(report.skipped_objects[0].source_id, Some(42)); + assert_eq!(report.skipped_objects[0].reason, "missing blob"); + } + + #[test] + fn writes_structured_json_report() -> Result<()> { + let report_path = std::env::temp_dir().join(format!( + "asterdrive-migration-report-{}.json", + uuid::Uuid::new_v4() + )); + let mut report = MigrationReport { + migrated_users: 1, + validation: MigrationValidation { + performed: true, + passed: true, + checks: vec![ValidationCheck { + name: "users_count".to_string(), + passed: true, + expected: "1".to_string(), + actual: "1".to_string(), + message: None, + }], + }, + ..Default::default() + }; + report.mappings.users.push(IdMapping { + source_id: 7, + target_id: 11, + }); + + write_json_report(&report_path, &report)?; + let value: Value = serde_json::from_slice(&std::fs::read(&report_path)?)?; + assert_eq!(value["schema_version"], 1); + assert_eq!(value["migrated_users"], 1); + assert_eq!(value["mappings"]["users"][0]["source_id"], 7); + assert_eq!(value["mappings"]["users"][0]["target_id"], 11); + assert_eq!(value["validation"]["passed"], true); + + let _ = std::fs::remove_file(report_path); + Ok(()) + } + #[tokio::test] async fn migrates_minimal_cloudreve_database() -> Result<()> { let suffix = uuid::Uuid::new_v4(); @@ -751,6 +1289,19 @@ mod tests { assert_eq!(report.migrated_tag_assignments, 1); assert_eq!(report.migrated_direct_links, 1); assert_eq!(report.migrated_tasks, 2); + assert!(report.validation.performed); + assert!(report.validation.passed); + assert!(report.validation.checks.iter().all(|check| check.passed)); + assert_eq!(report.mappings.users.len(), 1); + assert_eq!(report.mappings.folders.len(), 1); + assert_eq!(report.mappings.files.len(), 1); + assert_eq!(report.mappings.blobs.len(), 1); + assert_eq!(report.mappings.shares.len(), 1); + assert_eq!(report.mappings.tasks.len(), 2); + assert_eq!(report.direct_links.len(), 1); + assert!(report.direct_links[0].url.starts_with("/d/v2.")); + assert_eq!(report.tag_assignments.len(), 1); + assert_eq!(report.tag_assignments[0].target_entity_type, "file"); let target = Database::connect(&target_url).await?; assert_eq!(ad::users::Entity::find().count(&target).await?, 1); diff --git a/src/migration/phases.rs b/src/migration/phases.rs index 32ae831..347bf35 100644 --- a/src/migration/phases.rs +++ b/src/migration/phases.rs @@ -16,7 +16,12 @@ pub(super) async fn migrate_policies( .and_then(Value::as_bool) .unwrap_or(false) }) else { - report.skipped += 1; + report.record_skip( + "storage_policy", + Some(policy.id), + unsupported_policy_reason(policy) + .unwrap_or_else(|| "storage policy is not compatible with AD".to_string()), + ); continue; }; let base_path = if policy.r#type == "local" { @@ -201,7 +206,11 @@ pub(super) async fn migrate_folders( continue; } let Some(owner_id) = context.users.get(&folder.owner_id).copied() else { - report.skipped += 1; + report.record_skip( + "folder", + Some(folder.id), + format!("owner user {} was not migrated", folder.owner_id), + ); continue; }; let target = ad::folders::ActiveModel { @@ -280,7 +289,14 @@ pub(super) async fn migrate_blobs( .get(&entity.storage_policy_entities) .copied() else { - report.skipped += 1; + report.record_skip( + "blob", + Some(entity.id), + format!( + "storage policy {} was not migrated", + entity.storage_policy_entities + ), + ); continue; }; let reference_count = reference_counts.get(&entity.id).copied().unwrap_or(1); @@ -328,11 +344,19 @@ pub(super) async fn migrate_files( for file in files { if file.is_symbolic { - report.skipped += 1; + report.record_skip( + "file", + Some(file.id), + "symbolic/placeholder files are not representable in AD", + ); continue; } let Some(owner_id) = context.users.get(&file.owner_id).copied() else { - report.skipped += 1; + report.record_skip( + "file", + Some(file.id), + format!("owner user {} was not migrated", file.owner_id), + ); continue; }; let mut version_entities: Vec<&cr::entities::Model> = associations @@ -348,7 +372,11 @@ pub(super) async fn migrate_files( .filter(|id| context.blobs.contains_key(id)) .or_else(|| version_entities.last().map(|entity| entity.id)); let Some(primary_entity_id) = primary_entity_id else { - report.skipped += 1; + report.record_skip( + "file", + Some(file.id), + "file has no migratable version entity", + ); report .warnings .push(format!("file {} has no migratable version entity", file.id)); @@ -420,7 +448,11 @@ pub(super) async fn migrate_metadata( let mut tags: HashMap<(i64, String), i64> = HashMap::new(); for metadata in &source.metadata { let Some(source_file) = source_files.get(&metadata.file_id).copied() else { - report.skipped += 1; + report.record_skip( + "metadata", + Some(metadata.id), + format!("source file {} does not exist", metadata.file_id), + ); continue; }; let target_entity = if source_file.r#type == 0 { @@ -437,18 +469,30 @@ pub(super) async fn migrate_metadata( .map(|id| ("folder", id)) }; let Some((entity_type, entity_id)) = target_entity else { - report.skipped += 1; + report.record_skip( + "metadata", + Some(metadata.id), + format!("source entity {} was not migrated", metadata.file_id), + ); continue; }; if let Some(source_tag_name) = tag_name(&metadata.name) { let Some(owner_user_id) = context.users.get(&source_file.owner_id).copied() else { - report.skipped += 1; + report.record_skip( + "tag_assignment", + Some(metadata.id), + format!("owner user {} was not migrated", source_file.owner_id), + ); continue; }; let name = target_tag_name(source_tag_name); if name.is_empty() { - report.skipped += 1; + report.record_skip( + "tag_assignment", + Some(metadata.id), + "tag name is empty after normalization", + ); continue; } let normalized_name = normalize_tag_name(&name); @@ -488,6 +532,14 @@ pub(super) async fn migrate_metadata( .wrap_err_with(|| format!("attach migrated tag for metadata {}", metadata.id))?; report.migrated_properties += 1; report.migrated_tag_assignments += 1; + report.tag_assignments.push(TagAssignmentReport { + source_metadata_id: metadata.id, + source_entity_id: metadata.file_id, + target_entity_type: entity_type.to_string(), + target_entity_id: entity_id, + target_tag_id: tag_id, + tag_name: source_tag_name.to_string(), + }); continue; } @@ -523,7 +575,17 @@ pub(super) async fn migrate_direct_links( return Ok(()); } let Some(secret) = direct_link_secret else { - report.skipped += source.direct_links.len(); + for link in &source.direct_links { + report.record_skip( + "direct_link", + Some(link.id), + if link.deleted_at.is_some() { + "source direct link is deleted and must not be reactivated" + } else { + "AD direct_link_secret was not supplied" + }, + ); + } report.warnings.push(format!( "{} Cloudreve direct links were not regenerated because --direct-link-secret was not supplied", source.direct_links.len() @@ -534,19 +596,35 @@ pub(super) async fn migrate_direct_links( source.files.iter().map(|file| (file.id, file)).collect(); for link in &source.direct_links { if link.deleted_at.is_some() { - report.skipped += 1; + report.record_skip( + "direct_link", + Some(link.id), + "source direct link is deleted and must not be reactivated", + ); continue; } let Some(source_file) = source_files.get(&link.file_id).copied() else { - report.skipped += 1; + report.record_skip( + "direct_link", + Some(link.id), + format!("source file {} does not exist", link.file_id), + ); continue; }; let Some(file_id) = context.files.get(&link.file_id).copied() else { - report.skipped += 1; + report.record_skip( + "direct_link", + Some(link.id), + format!("source file {} was not migrated", link.file_id), + ); continue; }; let Some(owner_user_id) = context.users.get(&source_file.owner_id).copied() else { - report.skipped += 1; + report.record_skip( + "direct_link", + Some(link.id), + format!("owner user {} was not migrated", source_file.owner_id), + ); continue; }; let url = direct_link_url(file_id, owner_user_id, &source_file.name, secret)?; @@ -557,7 +635,7 @@ pub(super) async fn migrate_direct_links( name: Set(link.id.to_string()), value: Set(Some( json!({ - "url": url, + "url": url.clone(), "source_direct_link_id": link.id, "source_file_id": link.file_id, "source_name": link.name, @@ -573,6 +651,15 @@ pub(super) async fn migrate_direct_links( .wrap_err_with(|| format!("archive direct link {} mapping", link.id))?; report.migrated_properties += 1; report.migrated_direct_links += 1; + report.direct_links.push(DirectLinkReport { + source_direct_link_id: link.id, + source_file_id: link.file_id, + target_file_id: file_id, + source_name: link.name.clone(), + source_downloads: link.downloads, + source_speed_limit: link.speed, + url, + }); } Ok(()) } @@ -580,7 +667,7 @@ pub(super) async fn migrate_direct_links( pub(super) async fn migrate_tasks( transaction: &DatabaseTransaction, source: &SourceData, - context: &MigrationContext, + context: &mut MigrationContext, report: &mut MigrationReport, ) -> Result<()> { let active_count = source @@ -624,7 +711,7 @@ pub(super) async fn migrate_tasks( .updated_at .checked_add_signed(chrono::Duration::days(36_500)) .unwrap_or(task.updated_at); - ad::background_tasks::ActiveModel { + let target = ad::background_tasks::ActiveModel { kind: Set("system_runtime".to_string()), status: Set(status.to_string()), creator_user_id: Set(task @@ -664,6 +751,7 @@ pub(super) async fn migrate_tasks( .insert(transaction) .await .wrap_err_with(|| format!("archive Cloudreve task {}", task.id))?; + context.tasks.insert(task.id, target.id); report.migrated_tasks += 1; } Ok(()) @@ -672,22 +760,26 @@ pub(super) async fn migrate_tasks( pub(super) async fn migrate_shares( transaction: &DatabaseTransaction, source: &SourceData, - context: &MigrationContext, + context: &mut MigrationContext, report: &mut MigrationReport, ) -> Result<()> { let source_files: HashMap = source.files.iter().map(|file| (file.id, file)).collect(); for share in &source.shares { let Some(source_user_id) = share.user_shares else { - report.skipped += 1; + report.record_skip("share", Some(share.id), "share has no owner user"); continue; }; let Some(user_id) = context.users.get(&source_user_id).copied() else { - report.skipped += 1; + report.record_skip( + "share", + Some(share.id), + format!("owner user {source_user_id} was not migrated"), + ); continue; }; let Some(source_file_id) = share.file_shares else { - report.skipped += 1; + report.record_skip("share", Some(share.id), "share has no file/folder target"); continue; }; let source_file = source_files.get(&source_file_id).copied(); @@ -698,7 +790,11 @@ pub(super) async fn migrate_shares( .filter(|file| file.r#type == 1) .and_then(|_| context.folders.get(&source_file_id).copied()); if file_id.is_none() && folder_id.is_none() { - report.skipped += 1; + report.record_skip( + "share", + Some(share.id), + format!("source target {source_file_id} was not migrated"), + ); continue; } let password = match share.password.as_deref().filter(|value| !value.is_empty()) { @@ -709,7 +805,7 @@ pub(super) async fn migrate_shares( .remain_downloads .map(|remaining| share.downloads.saturating_add(remaining)) .unwrap_or(0); - ad::shares::ActiveModel { + let target = ad::shares::ActiveModel { token: Set(share_token(share.id)), user_id: Set(user_id), team_id: Set(None), @@ -727,6 +823,7 @@ pub(super) async fn migrate_shares( .insert(transaction) .await .wrap_err_with(|| format!("migrate share {}", share.id))?; + context.shares.insert(share.id, target.id); report.migrated_shares += 1; } Ok(()) From d7623ea190a81ef247b053889e82ec7b2b1aabdc Mon Sep 17 00:00:00 2001 From: Cg8 <5712.cg8@gmail.com> Date: Mon, 13 Jul 2026 20:36:29 +0800 Subject: [PATCH 4/7] feat: add stage-level migration resume --- README.md | 22 + docs/cloudreve-to-asterdrive-field-mapping.md | 22 + docs/migration-architecture.md | 36 +- src/main.rs | 6 + src/migration/checkpoint.rs | 195 +++++++ src/migration/mod.rs | 479 ++++++++++++++++-- 6 files changed, 691 insertions(+), 69 deletions(-) create mode 100644 src/migration/checkpoint.rs diff --git a/README.md b/README.md index 59221e2..ae9ea10 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,7 @@ cargo run -- migrate ` --source-url "sqlite://C:/cloudreve/cloudreve.db?mode=ro" ` --target-url "sqlite://C:/asterdrive/asterdrive.db" ` --local-base-path "C:/cloudreve" ` + --run-id "cloudreve-cutover-2026-07-13" ` --report-path "C:/migration/cloudreve-to-ad.json" ``` @@ -59,6 +60,26 @@ Use `--dry-run` to perform preflight checks without writing the target. The targ Cloudreve Qiniu, Upyun, remote-node, OneDrive and encrypted storage policies cannot be reused safely by AD. The migration stops when they are present. `--skip-unsupported-policies` explicitly omits those policies and all dependent files. +## Limited stage resume + +Migration stages are committed independently in this order: policies, policy groups, users, folders, blobs, files, metadata, shares, direct links and tasks. The target database stores the run state in `aster_external_migration_runs`. Each stage writes its target records and advances the checkpoint in the same database transaction. + +Use a stable run ID for an operational migration. If a stage fails, fix the external cause and rerun the same command with `--resume`: + +```powershell +cargo run -- migrate ` + --source-url "sqlite://C:/cloudreve/cloudreve.db?mode=ro" ` + --target-url "sqlite://C:/asterdrive/asterdrive.db" ` + --local-base-path "C:/cloudreve" ` + --run-id "cloudreve-cutover-2026-07-13" ` + --resume ` + --report-path "C:/migration/cloudreve-to-ad.json" +``` + +Resume verifies the source URL/count fingerprint, target URL fingerprint and migration-plan fingerprint. Repeat the same password, direct-link secret and migration flags used by the original run. Completed stages are skipped and their source-to-target mappings are restored from the checkpoint. + +This is stage-level resume only. A failed stage rolls back and restarts from the beginning of that stage; there is no page/batch cursor inside a large files or blobs stage yet. The source inventory is still loaded into memory, so very large installations still need the future paginated runner. + ## JSON migration report `--report-path` writes a pretty-printed JSON report for both `check` and `migrate`. A completed migration report contains: @@ -69,6 +90,7 @@ Cloudreve Qiniu, Upyun, remote-node, OneDrive and encrypted storage policies can - every Cloudreve tag metadata assignment and its AD tag/entity IDs - every regenerated direct-link URL and its Cloudreve/AD file IDs - post-commit database count checks, imported-task terminal-state checks, tag-binding checks and direct-link property checks +- run ID, whether the execution resumed, and the list of completed stages The CLI writes the JSON report before returning a validation error. A failed post-migration check therefore produces a usable report and exits with a non-zero status. Direct-link URLs are bearer-style public capabilities, so the report file must be stored with restricted access. diff --git a/docs/cloudreve-to-asterdrive-field-mapping.md b/docs/cloudreve-to-asterdrive-field-mapping.md index faedd47..f27f8ed 100644 --- a/docs/cloudreve-to-asterdrive-field-mapping.md +++ b/docs/cloudreve-to-asterdrive-field-mapping.md @@ -531,5 +531,27 @@ Cloudreve 任务不能恢复到 AD 执行器中，但可以保存为不可执行 | `tag_assignments` | Cloudreve metadata ID、源 file/folder ID、AD entity ID、AD tag ID 和标签名 | | `direct_links` | Cloudreve direct-link/file ID、AD file ID、新 URL、原名称、下载次数和限速 | | `validation` | 是否执行/通过，以及每项检查的 expected、actual 和失败信息 | +| `run_id` | checkpoint run ID；未指定时由迁移工具生成 UUID | +| `resumed` | 本次执行是否从已有 checkpoint 恢复 | +| `completed_stages` | 已原子提交完成的迁移阶段列表 | 当前提交后校验覆盖核心表增量数量、导入任务是否全为终态且无 lease、`system.tags` 关联是否存在、`cloudreve.direct_links` 属性中的 URL 是否与报告一致。报告不会保存数据库密码、存储密钥或 Cloudreve task private state，但会包含新 direct-link URL，因此必须限制报告文件访问权限。 + +## 17. 有限断点续传语义 + +迁移工具会在 AD 数据库自动创建 `aster_external_migration_runs`。首次迁移建议显式指定 `--run-id`；失败后使用相同参数和 `--resume --run-id `。 + +| 行为 | 当前实现 | +|---|---| +| checkpoint 粒度 | stage 级 | +| 原子性 | stage 的目标记录、context、report 和 last completed stage 在同一事务提交 | +| 已完成 stage | resume 时跳过，不重复插入 | +| 失败 stage | 整个 stage 回滚，resume 时从该 stage 开头重新执行 | +| ID mapping | 序列化保存在 run 的 `context_json` | +| 初始目标计数 | 保存在 `baseline_json`，恢复完成后仍可执行正确的增量数量校验 | +| 源校验 | URL 摘要 + 源表数量指纹 | +| 计划校验 | local path、迁移 flags、临时密码摘要和 direct-link secret 摘要 | +| stage 内分页 cursor | 尚未实现 | +| 对象上传断点 | 尚未实现 | + +由于源指纹不能检测数量不变的内容修改，断点恢复期间仍必须冻结 Cloudreve 写入。对于单个 stage 内拥有大量文件或 blob 的实例，当前恢复会从该 stage 起点重跑；后续需要增加 page cursor 和独立 object mapping 表。 diff --git a/docs/migration-architecture.md b/docs/migration-architecture.md index 6af8033..045fc05 100644 --- a/docs/migration-architecture.md +++ b/docs/migration-architecture.md @@ -532,27 +532,33 @@ trait TargetWriter { ### Checkpoint Tables -建议 checkpoint 表保存在 AD 数据库中，以便迁移进程重启后恢复。 +checkpoint 表保存在 AD 数据库中，以便迁移进程重启后恢复。当前实现先提供阶段级有限恢复。 #### `aster_external_migration_runs` -| 字段 | 建议类型 | 说明 | +| 字段 | 当前类型 | 说明 | |---|---|---| | `id` | UUID/String PK | Run ID | -| `source_type` | String | `cloudreve`、`nextcloud` 等 | | `source_fingerprint` | String | 源数据库/实例指纹，避免连错源 | | `target_fingerprint` | String | 目标 AD 实例指纹 | -| `status` | String | planned/running/verifying/completed/failed/cancelled | -| `started_at` | DateTime | 开始时间 | +| `plan_fingerprint` | String | local path、flags 和 secret/password 摘要组成的计划指纹 | +| `status` | String | running/failed/completed/validation_failed | +| `last_completed_stage` | String nullable | 最后一个与数据一起原子提交的阶段 | +| `context_json` | JSON/JSONB | 当前内存 ID mapping 和 username mapping 的持久化快照 | +| `report_json` | JSON/JSONB | 已完成阶段的完整结构化报告 | +| `baseline_json` | JSON/JSONB | 新 run 开始前的目标表计数，用于恢复后的最终增量校验 | +| `last_error` | Text nullable | 最近一次阶段失败摘要，不包含 migration secret | +| `created_at` | DateTime | 创建时间 | | `updated_at` | DateTime | 最后更新时间 | -| `finished_at` | DateTime nullable | 完成时间 | -| `plan_json` | Text | MigrationPlan | -| `stats_json` | Text | 数量、字节和阶段统计 | -| `summary_json` | Text nullable | 最终摘要 | -| `last_error` | Text nullable | 最后错误，不包含 secret | + +当前阶段顺序为 policies、policy_groups、users、folders、blobs、files、metadata、shares、direct_links、tasks。每个阶段使用独立目标事务；目标数据和 `last_completed_stage/context_json/report_json` 在同一事务中提交。失败阶段整体回滚，恢复时从下一个未完成阶段开始。 + +当前 source fingerprint 使用数据库 URL 摘要和各源表记录数量，只能防止明显连错源或数量变化，不能检测“数量不变但内容变化”。生产迁移仍应冻结 Cloudreve 写入。 #### `aster_external_migration_stage_cursors` +尚未实现。它是下一步批内分页恢复所需的表。 + | 字段 | 建议类型 | 说明 | |---|---|---| | `run_id` | FK | 所属 run | @@ -567,6 +573,8 @@ trait TargetWriter { #### `aster_external_migration_object_map` +尚未实现独立行式映射表。当前有限实现将完整 HashMap 序列化到 `aster_external_migration_runs.context_json`；这能跨进程恢复阶段依赖，但不能高效查询单个对象，也不适合超大数据集。 + | 字段 | 建议类型 | 说明 | |---|---|---| | `id` | BigInt PK | 映射记录 ID | @@ -755,10 +763,10 @@ Cloudreve entity ID、object key、路径或 `policy+path+size` 摘要都不能 | Cloudreve v4 标签 | 已从 `tag:*` metadata 生成 AD `tags` 和 `system.tags` 文件/目录关联 | 增加跨批次持久化去重和冲突报告 | | Direct links | 提供 AD `direct_link_secret` 时生成 v2 URL，并以 `cloudreve.direct_links` property 保存源 ID 映射 | 增加独立 JSON/CSV 报告和可选旧 URL 重定向层 | | Cloudreve 任务 | 已全部写成 AD `system_runtime` 终态历史；活动任务归档为 canceled | 后续可增加独立 legacy task archive 表，避免占用运维任务列表 | -| 目标事务 | 已实现单次全局事务 | 每 batch 事务 + checkpoint，适合大数据集 | -| ID mapping | 已实现内存 HashMap，并在 JSON 报告中导出排序后的核心对象映射 | 持久化 object mapping table，支持跨进程 resume | -| 断点续传 | 未实现 | stage cursor + resume | -| 幂等重复运行 | 未完整实现 | mapping 驱动 upsert/reuse | +| 目标事务 | 已改为每 stage 独立事务，stage 数据和 checkpoint 原子提交 | 每 page/batch 事务，降低单个 files/blobs stage 的事务规模 | +| ID mapping | 内存 HashMap 同时持久化到 run `context_json`，并导出到 JSON 报告 | 独立 object mapping table，支持单对象查询和超大数据集 | +| 断点续传 | 已支持 `--run-id` + `--resume` 的阶段级恢复 | stage 内 page cursor、对象存储上传恢复和并行 worker checkpoint | +| 幂等重复运行 | 同一 run 会跳过已完成 stage；失败 stage 因事务回滚可安全重跑 | 不依赖 run checkpoint 的对象级 upsert/reuse 和完整重复运行 | | 冲突策略 | 仅 username 自动后缀，其他冲突依赖 DB 报错 | fail/rename/skip/reuse/update 可配置 | | 物理对象复制 | 未实现，当前复用源 storage path | 流式 copy、SHA-256、目标路径和校验 | | 真正内容去重 | 未实现 | hash + policy 去重 | diff --git a/src/main.rs b/src/main.rs index b276227..90a6a30 100644 --- a/src/main.rs +++ b/src/main.rs @@ -47,6 +47,10 @@ struct MigrateArgs { skip_unsupported_policies: bool, #[arg(long)] dry_run: bool, + #[arg(long, value_name = "ID")] + run_id: Option, + #[arg(long, requires = "run_id")] + resume: bool, } #[tokio::main] @@ -69,6 +73,8 @@ async fn main() -> Result<()> { allow_non_empty_target: args.allow_non_empty_target, skip_unsupported_policies: args.skip_unsupported_policies, dry_run: args.dry_run, + run_id: args.run_id, + resume: args.resume, }) .await?; emit_report(report_path.as_deref(), &report)?; diff --git a/src/migration/checkpoint.rs b/src/migration/checkpoint.rs new file mode 100644 index 0000000..b1c4bc2 --- /dev/null +++ b/src/migration/checkpoint.rs @@ -0,0 +1,195 @@ +use color_eyre::eyre::{Result, WrapErr, bail}; +use sea_orm::entity::prelude::*; +use sea_orm::{ActiveModelTrait, ConnectionTrait, EntityTrait, Schema, Set, Unchanged}; + +use super::{MigrationContext, MigrationReport, TargetCounts}; + +#[sea_orm::model] +#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] +#[sea_orm(table_name = "aster_external_migration_runs")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub id: String, + pub source_fingerprint: String, + pub target_fingerprint: String, + pub plan_fingerprint: String, + pub status: String, + pub last_completed_stage: Option, + #[sea_orm(column_type = "JsonBinary")] + pub context_json: Json, + #[sea_orm(column_type = "JsonBinary")] + pub report_json: Json, + #[sea_orm(column_type = "JsonBinary")] + pub baseline_json: Json, + #[sea_orm(column_type = "Text", nullable)] + pub last_error: Option, + pub created_at: DateTimeWithTimeZone, + pub updated_at: DateTimeWithTimeZone, +} + +impl ActiveModelBehavior for ActiveModel {} + +pub(super) struct LoadedCheckpoint { + pub status: String, + pub last_completed_stage: Option, + pub context: MigrationContext, + pub report: MigrationReport, + pub baseline: TargetCounts, +} + +pub(super) struct NewCheckpoint<'a> { + pub run_id: &'a str, + pub source_fingerprint: &'a str, + pub target_fingerprint: &'a str, + pub plan_fingerprint: &'a str, + pub context: &'a MigrationContext, + pub report: &'a MigrationReport, + pub baseline: &'a TargetCounts, +} + +pub(super) async fn ensure_table(db: &C) -> Result<()> { + let schema = Schema::new(db.get_database_backend()); + let mut statement = schema.create_table_from_entity(Entity); + statement.if_not_exists(); + db.execute(&statement) + .await + .wrap_err("create external migration checkpoint table")?; + Ok(()) +} + +pub(super) async fn create( + db: &C, + checkpoint: NewCheckpoint<'_>, +) -> Result<()> { + let NewCheckpoint { + run_id, + source_fingerprint, + target_fingerprint, + plan_fingerprint, + context, + report, + baseline, + } = checkpoint; + if Entity::find_by_id(run_id.to_string()) + .one(db) + .await? + .is_some() + { + bail!("migration run {run_id} already exists; use --resume or choose another --run-id"); + } + let now = chrono::Utc::now().fixed_offset(); + ActiveModel { + id: Set(run_id.to_string()), + source_fingerprint: Set(source_fingerprint.to_string()), + target_fingerprint: Set(target_fingerprint.to_string()), + plan_fingerprint: Set(plan_fingerprint.to_string()), + status: Set("running".to_string()), + last_completed_stage: Set(None), + context_json: Set(serde_json::to_value(context).wrap_err("serialize migration context")?), + report_json: Set(serde_json::to_value(report).wrap_err("serialize migration report")?), + baseline_json: Set(serde_json::to_value(baseline).wrap_err("serialize target baseline")?), + last_error: Set(None), + created_at: Set(now), + updated_at: Set(now), + } + .insert(db) + .await + .wrap_err_with(|| format!("create migration run {run_id}"))?; + Ok(()) +} + +pub(super) async fn load( + db: &C, + run_id: &str, + source_fingerprint: &str, + target_fingerprint: &str, + plan_fingerprint: &str, +) -> Result { + let model = Entity::find_by_id(run_id.to_string()) + .one(db) + .await? + .ok_or_else(|| color_eyre::eyre::eyre!("migration run {run_id} does not exist"))?; + if model.source_fingerprint != source_fingerprint { + bail!("migration run {run_id} belongs to a different Cloudreve source"); + } + if model.target_fingerprint != target_fingerprint { + bail!("migration run {run_id} belongs to a different AD target"); + } + if model.plan_fingerprint != plan_fingerprint { + bail!("migration run {run_id} options differ from the original migration plan"); + } + Ok(LoadedCheckpoint { + status: model.status, + last_completed_stage: model.last_completed_stage, + context: serde_json::from_value(model.context_json) + .wrap_err("decode checkpoint migration context")?, + report: serde_json::from_value(model.report_json) + .wrap_err("decode checkpoint migration report")?, + baseline: serde_json::from_value(model.baseline_json) + .wrap_err("decode checkpoint target baseline")?, + }) +} + +pub(super) async fn save_stage( + db: &C, + run_id: &str, + stage: &str, + context: &MigrationContext, + report: &MigrationReport, +) -> Result<()> { + ActiveModel { + id: Unchanged(run_id.to_string()), + status: Set("running".to_string()), + last_completed_stage: Set(Some(stage.to_string())), + context_json: Set(serde_json::to_value(context).wrap_err("serialize migration context")?), + report_json: Set(serde_json::to_value(report).wrap_err("serialize migration report")?), + last_error: Set(None), + updated_at: Set(chrono::Utc::now().fixed_offset()), + ..Default::default() + } + .update(db) + .await + .wrap_err_with(|| format!("save migration run {run_id} stage {stage}"))?; + Ok(()) +} + +pub(super) async fn finish( + db: &C, + run_id: &str, + status: &str, + context: &MigrationContext, + report: &MigrationReport, +) -> Result<()> { + ActiveModel { + id: Unchanged(run_id.to_string()), + status: Set(status.to_string()), + context_json: Set(serde_json::to_value(context).wrap_err("serialize migration context")?), + report_json: Set(serde_json::to_value(report).wrap_err("serialize migration report")?), + last_error: Set(None), + updated_at: Set(chrono::Utc::now().fixed_offset()), + ..Default::default() + } + .update(db) + .await + .wrap_err_with(|| format!("finish migration run {run_id}"))?; + Ok(()) +} + +pub(super) async fn mark_failed( + db: &C, + run_id: &str, + error: &str, +) -> Result<()> { + let error = error.chars().take(2048).collect::(); + ActiveModel { + id: Unchanged(run_id.to_string()), + status: Set("failed".to_string()), + last_error: Set(Some(error)), + updated_at: Set(chrono::Utc::now().fixed_offset()), + ..Default::default() + } + .update(db) + .await + .wrap_err_with(|| format!("mark migration run {run_id} failed"))?; + Ok(()) +} diff --git a/src/migration/mod.rs b/src/migration/mod.rs index 72843ce..1be1f2a 100644 --- a/src/migration/mod.rs +++ b/src/migration/mod.rs @@ -11,7 +11,7 @@ use sea_orm::{ ActiveModelTrait, ColumnTrait, Database, DatabaseConnection, EntityTrait, PaginatorTrait, QueryFilter, Set, TransactionTrait, }; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use sha2::{Digest, Sha256}; @@ -29,9 +29,11 @@ pub struct MigrationOptions { pub allow_non_empty_target: bool, pub skip_unsupported_policies: bool, pub dry_run: bool, + pub run_id: Option, + pub resume: bool, } -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct MigrationReport { pub schema_version: u32, pub generated_at: chrono::DateTime, @@ -67,6 +69,9 @@ pub struct MigrationReport { pub direct_links: Vec, pub tag_assignments: Vec, pub validation: MigrationValidation, + pub run_id: Option, + pub resumed: bool, + pub completed_stages: Vec, } impl Default for MigrationReport { @@ -106,18 +111,21 @@ impl Default for MigrationReport { direct_links: Vec::new(), tag_assignments: Vec::new(), validation: MigrationValidation::default(), + run_id: None, + resumed: false, + completed_stages: Vec::new(), } } } -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct SkippedObject { pub object_type: String, pub source_id: Option, pub reason: String, } -#[derive(Debug, Default, Clone, Serialize)] +#[derive(Debug, Default, Clone, Serialize, Deserialize)] pub struct MigrationMappings { pub policies: Vec, pub policy_groups: Vec, @@ -129,13 +137,13 @@ pub struct MigrationMappings { pub tasks: Vec, } -#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct IdMapping { pub source_id: i64, pub target_id: i64, } -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct DirectLinkReport { pub source_direct_link_id: i64, pub source_file_id: i64, @@ -146,7 +154,7 @@ pub struct DirectLinkReport { pub url: String, } -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct TagAssignmentReport { pub source_metadata_id: i64, pub source_entity_id: i64, @@ -156,7 +164,7 @@ pub struct TagAssignmentReport { pub tag_name: String, } -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct MigrationValidation { pub performed: bool, pub passed: bool, @@ -173,7 +181,7 @@ impl Default for MigrationValidation { } } -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct ValidationCheck { pub name: String, pub passed: bool, @@ -219,6 +227,13 @@ impl fmt::Display for MigrationReport { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { let mut output = String::new(); writeln!(output, "Cloudreve -> AsterDrive migration report")?; + if let Some(run_id) = &self.run_id { + writeln!( + output, + "run: {run_id}{}", + if self.resumed { " (resumed)" } else { "" } + )?; + } writeln!( output, "source: users={}, groups={}, policies={}, folders={}, files={}, entities={}, shares={}, direct_links={}, tag_assignments={}, tasks={}", @@ -275,6 +290,13 @@ impl fmt::Display for MigrationReport { self.validation.checks.len() )?; } + if !self.completed_stages.is_empty() { + writeln!( + output, + "completed_stages: {}", + self.completed_stages.join(",") + )?; + } } if !self.warnings.is_empty() { writeln!(output, "warnings:")?; @@ -315,6 +337,67 @@ pub async fn inspect( Ok(report) } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum MigrationStage { + Policies, + PolicyGroups, + Users, + Folders, + Blobs, + Files, + Metadata, + Shares, + DirectLinks, + Tasks, +} + +impl MigrationStage { + const ALL: [Self; 10] = [ + Self::Policies, + Self::PolicyGroups, + Self::Users, + Self::Folders, + Self::Blobs, + Self::Files, + Self::Metadata, + Self::Shares, + Self::DirectLinks, + Self::Tasks, + ]; + + const fn as_str(self) -> &'static str { + match self { + Self::Policies => "policies", + Self::PolicyGroups => "policy_groups", + Self::Users => "users", + Self::Folders => "folders", + Self::Blobs => "blobs", + Self::Files => "files", + Self::Metadata => "metadata", + Self::Shares => "shares", + Self::DirectLinks => "direct_links", + Self::Tasks => "tasks", + } + } + + fn should_run_after(self, last_completed_stage: Option<&str>) -> Result { + let Some(last_completed_stage) = last_completed_stage else { + return Ok(true); + }; + let last_index = Self::ALL + .iter() + .position(|stage| stage.as_str() == last_completed_stage) + .ok_or_else(|| { + color_eyre::eyre::eyre!("checkpoint contains unknown stage {last_completed_stage}") + })?; + let current_index = Self::ALL + .iter() + .position(|stage| *stage == self) + .expect("migration stage must exist in ALL"); + Ok(current_index > last_index) + } +} + pub async fn migrate(options: MigrationOptions) -> Result { if options.default_password.chars().count() < 8 { bail!("--default-password must contain at least 8 characters"); @@ -326,13 +409,20 @@ pub async fn migrate(options: MigrationOptions) -> Result { { bail!("--direct-link-secret must contain at least 16 characters"); } + if options.resume && options.run_id.is_none() { + bail!("--resume requires --run-id"); + } + if options.resume && options.dry_run { + bail!("--resume cannot be combined with --dry-run"); + } + if let Some(run_id) = options.run_id.as_deref() { + validate_run_id(run_id)?; + } let source = connect(&options.source_url, "Cloudreve").await?; let target = connect(&options.target_url, "AsterDrive").await?; let source_data = SourceData::load(&source, options.include_deleted).await?; validate_target_schema(&target).await?; - ensure_target_safe(&target, options.allow_non_empty_target).await?; - let target_before = TargetCounts::load(&target).await?; let unsupported = source_data.unsupported_policy_types(); if !unsupported.is_empty() && !options.skip_unsupported_policies { @@ -346,58 +436,225 @@ pub async fn migrate(options: MigrationOptions) -> Result { report.dry_run = options.dry_run; report.warnings.extend(source_data.compatibility_warnings()); if options.dry_run { + report.run_id = options.run_id.clone(); return Ok(report); } + checkpoint::ensure_table(&target).await?; + let run_id = options + .run_id + .clone() + .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()); + let source_fingerprint = source_fingerprint(&options.source_url, &source_data); + let target_fingerprint = hash_fingerprint(&options.target_url); + let plan_fingerprint = plan_fingerprint(&options); + + let (mut context, target_before, mut last_completed_stage) = if options.resume { + let loaded = checkpoint::load( + &target, + &run_id, + &source_fingerprint, + &target_fingerprint, + &plan_fingerprint, + ) + .await?; + if !matches!( + loaded.status.as_str(), + "running" | "failed" | "validation_failed" | "completed" + ) { + bail!( + "migration run {run_id} has unsupported status {}", + loaded.status + ); + } + report = loaded.report; + report.resumed = true; + report.run_id = Some(run_id.clone()); + (loaded.context, loaded.baseline, loaded.last_completed_stage) + } else { + ensure_target_safe(&target, options.allow_non_empty_target).await?; + let baseline = TargetCounts::load(&target).await?; + report.run_id = Some(run_id.clone()); + let context = MigrationContext::default(); + checkpoint::create( + &target, + checkpoint::NewCheckpoint { + run_id: &run_id, + source_fingerprint: &source_fingerprint, + target_fingerprint: &target_fingerprint, + plan_fingerprint: &plan_fingerprint, + context: &context, + report: &report, + baseline: &baseline, + }, + ) + .await?; + (context, baseline, None) + }; + let password_hash = hash_password(&options.default_password)?; - let transaction = target - .begin() - .await - .wrap_err("begin AsterDrive transaction")?; - let mut context = MigrationContext::default(); - - migrate_policies( - &transaction, - &source_data, - &options, - &mut context, - &mut report, - ) - .await?; - migrate_policy_groups(&transaction, &source_data, &mut context, &mut report).await?; - migrate_users( - &transaction, - &source_data, - &password_hash, - &mut context, - &mut report, - ) - .await?; - migrate_folders(&transaction, &source_data, &mut context, &mut report).await?; - migrate_blobs(&transaction, &source_data, &mut context, &mut report).await?; - migrate_files(&transaction, &source_data, &mut context, &mut report).await?; - migrate_metadata(&transaction, &source_data, &context, &mut report).await?; - migrate_shares(&transaction, &source_data, &mut context, &mut report).await?; - migrate_direct_links( - &transaction, - &source_data, - &context, - options.direct_link_secret.as_deref(), - &mut report, - ) - .await?; - migrate_tasks(&transaction, &source_data, &mut context, &mut report).await?; - transaction - .commit() - .await - .wrap_err("commit AsterDrive migration transaction")?; + for stage in MigrationStage::ALL { + if !stage.should_run_after(last_completed_stage.as_deref())? { + continue; + } + let transaction = target + .begin() + .await + .wrap_err_with(|| format!("begin migration stage {}", stage.as_str()))?; + let stage_result: Result<()> = async { + execute_stage( + stage, + &transaction, + &source_data, + &options, + &password_hash, + &mut context, + &mut report, + ) + .await?; + report.set_mappings(&context); + if !report + .completed_stages + .iter() + .any(|completed| completed == stage.as_str()) + { + report.completed_stages.push(stage.as_str().to_string()); + } + checkpoint::save_stage(&transaction, &run_id, stage.as_str(), &context, &report).await + } + .await; + if let Err(error) = stage_result { + drop(transaction); + let _ = checkpoint::mark_failed(&target, &run_id, &error.to_string()).await; + return Err(error).wrap_err_with(|| { + format!( + "migration run {run_id} failed at stage {}; rerun with --resume --run-id {run_id}", + stage.as_str() + ) + }); + } + if let Err(error) = transaction.commit().await { + let _ = checkpoint::mark_failed(&target, &run_id, &error.to_string()).await; + return Err(error).wrap_err_with(|| { + format!( + "commit migration run {run_id} stage {}; resume with the same run ID", + stage.as_str() + ) + }); + } + last_completed_stage = Some(stage.as_str().to_string()); + } + report.set_mappings(&context); - report.validation = validate_migration_result(&target, &target_before, &report).await?; + report.validation = match validate_migration_result(&target, &target_before, &report).await { + Ok(validation) => validation, + Err(error) => { + let _ = checkpoint::mark_failed(&target, &run_id, &error.to_string()).await; + return Err(error) + .wrap_err_with(|| format!("validate completed migration run {run_id}")); + } + }; report.generated_at = chrono::Utc::now(); + checkpoint::finish( + &target, + &run_id, + if report.validation.passed { + "completed" + } else { + "validation_failed" + }, + &context, + &report, + ) + .await?; Ok(report) } +async fn execute_stage( + stage: MigrationStage, + transaction: &sea_orm::DatabaseTransaction, + source_data: &SourceData, + options: &MigrationOptions, + password_hash: &str, + context: &mut MigrationContext, + report: &mut MigrationReport, +) -> Result<()> { + match stage { + MigrationStage::Policies => { + migrate_policies(transaction, source_data, options, context, report).await + } + MigrationStage::PolicyGroups => { + migrate_policy_groups(transaction, source_data, context, report).await + } + MigrationStage::Users => { + migrate_users(transaction, source_data, password_hash, context, report).await + } + MigrationStage::Folders => migrate_folders(transaction, source_data, context, report).await, + MigrationStage::Blobs => migrate_blobs(transaction, source_data, context, report).await, + MigrationStage::Files => migrate_files(transaction, source_data, context, report).await, + MigrationStage::Metadata => { + migrate_metadata(transaction, source_data, context, report).await + } + MigrationStage::Shares => migrate_shares(transaction, source_data, context, report).await, + MigrationStage::DirectLinks => { + migrate_direct_links( + transaction, + source_data, + context, + options.direct_link_secret.as_deref(), + report, + ) + .await + } + MigrationStage::Tasks => migrate_tasks(transaction, source_data, context, report).await, + } +} + +fn validate_run_id(run_id: &str) -> Result<()> { + let run_id = run_id.trim(); + if run_id.is_empty() || run_id.chars().count() > 128 || run_id.chars().any(char::is_control) { + bail!("--run-id must contain 1-128 non-control characters"); + } + Ok(()) +} + +fn hash_fingerprint(value: &str) -> String { + format!("{:x}", Sha256::digest(value.as_bytes())) +} + +fn source_fingerprint(source_url: &str, source: &SourceData) -> String { + hash_fingerprint(&format!( + "{source_url}|{}|{}|{}|{}|{}|{}|{}|{}|{}|{}", + source.users.len(), + source.groups.len(), + source.policies.len(), + source.files.len(), + source.entities.len(), + source.file_entities.len(), + source.shares.len(), + source.metadata.len(), + source.direct_links.len(), + source.tasks.len(), + )) +} + +fn plan_fingerprint(options: &MigrationOptions) -> String { + hash_fingerprint(&format!( + "{}|{}|{}|{}|{}|{}", + options.local_base_path, + options.include_deleted, + options.allow_non_empty_target, + options.skip_unsupported_policies, + hash_fingerprint(&options.default_password), + options + .direct_link_secret + .as_deref() + .map(hash_fingerprint) + .unwrap_or_default(), + )) +} + async fn connect(url: &str, label: &str) -> Result { Database::connect(url) .await @@ -437,13 +694,29 @@ async fn ensure_target_safe(db: &DatabaseConnection, allow_non_empty: bool) -> R return Ok(()); } let counts = [ + ( + "storage_policies", + ad::storage_policies::Entity::find().count(db).await?, + ), + ( + "storage_policy_groups", + ad::storage_policy_groups::Entity::find().count(db).await?, + ), ("users", ad::users::Entity::find().count(db).await?), + ( + "user_profiles", + ad::user_profiles::Entity::find().count(db).await?, + ), ("folders", ad::folders::Entity::find().count(db).await?), ("files", ad::files::Entity::find().count(db).await?), ( "file_blobs", ad::file_blobs::Entity::find().count(db).await?, ), + ( + "file_versions", + ad::file_versions::Entity::find().count(db).await?, + ), ("shares", ad::shares::Entity::find().count(db).await?), ( "entity_properties", @@ -469,7 +742,7 @@ async fn ensure_target_safe(db: &DatabaseConnection, allow_non_empty: bool) -> R Ok(()) } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] struct TargetCounts { policies: u64, policy_groups: u64, @@ -755,7 +1028,7 @@ fn hash_password(password: &str) -> Result { .map_err(|error| color_eyre::eyre::eyre!("hash temporary AD password: {error}")) } -#[derive(Default)] +#[derive(Debug, Default, Clone, Serialize, Deserialize)] struct MigrationContext { policies: HashMap, policy_groups: HashMap, @@ -1130,6 +1403,7 @@ fn file_classification(name: &str) -> (String, Option, String, String) { (mime, compound_extension, extension, category.to_string()) } +mod checkpoint; mod phases; use phases::*; @@ -1276,6 +1550,8 @@ mod tests { allow_non_empty_target: false, skip_unsupported_policies: false, dry_run: false, + run_id: None, + resume: false, }) .await?; @@ -1350,6 +1626,99 @@ mod tests { Ok(()) } + #[tokio::test] + async fn resumes_from_last_completed_stage() -> Result<()> { + let suffix = uuid::Uuid::new_v4(); + let run_id = format!("resume-test-{suffix}"); + let source_path = std::env::temp_dir().join(format!("cloudreve-resume-{suffix}.db")); + let target_path = std::env::temp_dir().join(format!("asterdrive-resume-{suffix}.db")); + let source_url = sqlite_url(&source_path); + let target_url = sqlite_url(&target_path); + + let source = Database::connect(&source_url).await?; + create_source_schema(&source).await?; + seed_source(&source).await?; + source.close().await?; + + let target = Database::connect(&target_url).await?; + create_target_schema(&target).await?; + target + .execute_unprepared( + "CREATE TRIGGER fail_folder_insert BEFORE INSERT ON folders \ + BEGIN SELECT RAISE(ABORT, 'forced folder stage failure'); END", + ) + .await?; + target.close().await?; + + let options = MigrationOptions { + source_url: source_url.clone(), + target_url: target_url.clone(), + default_password: "temporary-password".to_string(), + local_base_path: "C:/cloudreve".to_string(), + direct_link_secret: Some("test-direct-link-secret".to_string()), + include_deleted: false, + allow_non_empty_target: false, + skip_unsupported_policies: false, + dry_run: false, + run_id: Some(run_id.clone()), + resume: false, + }; + + let error = migrate(options.clone()).await.unwrap_err(); + assert!(error.to_string().contains(&run_id)); + + let target = Database::connect(&target_url).await?; + assert_eq!( + ad::storage_policies::Entity::find().count(&target).await?, + 1 + ); + assert_eq!(ad::users::Entity::find().count(&target).await?, 1); + assert_eq!(ad::folders::Entity::find().count(&target).await?, 0); + let failed_checkpoint = checkpoint::Entity::find_by_id(run_id.clone()) + .one(&target) + .await? + .expect("failed migration checkpoint"); + assert_eq!(failed_checkpoint.status, "failed"); + assert_eq!( + failed_checkpoint.last_completed_stage.as_deref(), + Some("users") + ); + target + .execute_unprepared("DROP TRIGGER fail_folder_insert") + .await?; + target.close().await?; + + let report = migrate(MigrationOptions { + resume: true, + ..options + }) + .await?; + assert!(report.resumed); + assert_eq!(report.run_id.as_deref(), Some(run_id.as_str())); + assert!(report.validation.passed); + assert_eq!(report.migrated_users, 1); + assert_eq!(report.migrated_folders, 1); + + let target = Database::connect(&target_url).await?; + assert_eq!(ad::users::Entity::find().count(&target).await?, 1); + assert_eq!(ad::folders::Entity::find().count(&target).await?, 1); + assert_eq!(ad::files::Entity::find().count(&target).await?, 1); + let completed_checkpoint = checkpoint::Entity::find_by_id(run_id) + .one(&target) + .await? + .expect("completed migration checkpoint"); + assert_eq!(completed_checkpoint.status, "completed"); + assert_eq!( + completed_checkpoint.last_completed_stage.as_deref(), + Some("tasks") + ); + target.close().await?; + + let _ = std::fs::remove_file(source_path); + let _ = std::fs::remove_file(target_path); + Ok(()) + } + fn sqlite_url(path: &std::path::Path) -> String { format!( "sqlite://{}?mode=rwc", From 811638405d8cfee21e89e62047b524b467787570 Mon Sep 17 00:00:00 2001 From: Cg8 <5712.cg8@gmail.com> Date: Mon, 13 Jul 2026 22:37:55 +0800 Subject: [PATCH 5/7] feat: add batched blob migration resume --- README.md | 11 +- docs/cloudreve-to-asterdrive-field-mapping.md | 10 +- docs/migration-architecture.md | 40 +- src/main.rs | 3 + src/migration/checkpoint.rs | 155 ++++++ src/migration/mod.rs | 498 ++++++++++++++++-- src/migration/phases.rs | 77 ++- 7 files changed, 676 insertions(+), 118 deletions(-) diff --git a/README.md b/README.md index ae9ea10..f1fa6d4 100644 --- a/README.md +++ b/README.md @@ -60,9 +60,9 @@ Use `--dry-run` to perform preflight checks without writing the target. The targ Cloudreve Qiniu, Upyun, remote-node, OneDrive and encrypted storage policies cannot be reused safely by AD. The migration stops when they are present. `--skip-unsupported-policies` explicitly omits those policies and all dependent files. -## Limited stage resume +## Limited resume -Migration stages are committed independently in this order: policies, policy groups, users, folders, blobs, files, metadata, shares, direct links and tasks. The target database stores the run state in `aster_external_migration_runs`. Each stage writes its target records and advances the checkpoint in the same database transaction. +Migration stages run in this order: policies, policy groups, users, folders, blobs, files, metadata, shares, direct links and tasks. Most stages are committed as one transaction. The blobs stage uses keyset pages ordered by Cloudreve `entities.id`; each page writes AD `file_blobs`, object mappings, its cursor and report progress in one target transaction. Use a stable run ID for an operational migration. If a stage fails, fix the external cause and rerun the same command with `--resume`: @@ -72,13 +72,16 @@ cargo run -- migrate ` --target-url "sqlite://C:/asterdrive/asterdrive.db" ` --local-base-path "C:/cloudreve" ` --run-id "cloudreve-cutover-2026-07-13" ` + --blob-batch-size 500 ` --resume ` --report-path "C:/migration/cloudreve-to-ad.json" ``` -Resume verifies the source URL/count fingerprint, target URL fingerprint and migration-plan fingerprint. Repeat the same password, direct-link secret and migration flags used by the original run. Completed stages are skipped and their source-to-target mappings are restored from the checkpoint. +Resume verifies the source URL/count fingerprint, target URL fingerprint and migration-plan fingerprint. Repeat the same password, direct-link secret and migration flags used by the original run. Completed stages are skipped and their source-to-target mappings are restored from the checkpoint. Blob batch size is operational only and may be adjusted when resuming. -This is stage-level resume only. A failed stage rolls back and restarts from the beginning of that stage; there is no page/batch cursor inside a large files or blobs stage yet. The source inventory is still loaded into memory, so very large installations still need the future paginated runner. +`--blob-batch-size` defaults to 500 and accepts 1-10000. A failed blob page rolls back only that page; resume continues after the last committed entity ID. Blob source rows are no longer loaded into memory as one complete collection, and blob mappings are stored row-by-row in `aster_external_migration_object_map`. + +Other stages, including files, are still stage-level only: a failure restarts that entire stage. The files stage currently loads entity/version relationships into memory, and physical object bytes are still reused in place rather than copied or uploaded. ## JSON migration report diff --git a/docs/cloudreve-to-asterdrive-field-mapping.md b/docs/cloudreve-to-asterdrive-field-mapping.md index f27f8ed..81a9ea4 100644 --- a/docs/cloudreve-to-asterdrive-field-mapping.md +++ b/docs/cloudreve-to-asterdrive-field-mapping.md @@ -543,15 +543,15 @@ Cloudreve 任务不能恢复到 AD 执行器中，但可以保存为不可执行 | 行为 | 当前实现 | |---|---| -| checkpoint 粒度 | stage 级 | -| 原子性 | stage 的目标记录、context、report 和 last completed stage 在同一事务提交 | +| checkpoint 粒度 | blobs 为 batch 级；其他 stage 为 stage 级 | +| 原子性 | blob page 的目标记录、object mapping、cursor 和 report 同事务；其他 stage 的目标记录、context、report 和 last completed stage 同事务 | | 已完成 stage | resume 时跳过，不重复插入 | | 失败 stage | 整个 stage 回滚，resume 时从该 stage 开头重新执行 | -| ID mapping | 序列化保存在 run 的 `context_json` | +| ID mapping | blob 映射逐行保存在 `aster_external_migration_object_map`；其他映射仍在 `context_json` | | 初始目标计数 | 保存在 `baseline_json`，恢复完成后仍可执行正确的增量数量校验 | | 源校验 | URL 摘要 + 源表数量指纹 | | 计划校验 | local path、迁移 flags、临时密码摘要和 direct-link secret 摘要 | -| stage 内分页 cursor | 尚未实现 | +| stage 内分页 cursor | blobs 已使用 `entities.id` keyset cursor；其他 stage 尚未实现 | | 对象上传断点 | 尚未实现 | -由于源指纹不能检测数量不变的内容修改，断点恢复期间仍必须冻结 Cloudreve 写入。对于单个 stage 内拥有大量文件或 blob 的实例，当前恢复会从该 stage 起点重跑；后续需要增加 page cursor 和独立 object mapping 表。 +由于源指纹不能检测数量不变的内容修改，断点恢复期间仍必须冻结 Cloudreve 写入。blobs 可从最后提交的 entity ID 继续，批大小由 `--blob-batch-size` 控制，默认 500；files 等其他 stage 仍会从 stage 起点重跑。对象字节复制/上传仍未实现，因此也没有对象存储分片上传续传。 diff --git a/docs/migration-architecture.md b/docs/migration-architecture.md index 045fc05..c2e4965 100644 --- a/docs/migration-architecture.md +++ b/docs/migration-architecture.md @@ -551,41 +551,35 @@ checkpoint 表保存在 AD 数据库中，以便迁移进程重启后恢复。 | `created_at` | DateTime | 创建时间 | | `updated_at` | DateTime | 最后更新时间 | -当前阶段顺序为 policies、policy_groups、users、folders、blobs、files、metadata、shares、direct_links、tasks。每个阶段使用独立目标事务；目标数据和 `last_completed_stage/context_json/report_json` 在同一事务中提交。失败阶段整体回滚，恢复时从下一个未完成阶段开始。 +当前阶段顺序为 policies、policy_groups、users、folders、blobs、files、metadata、shares、direct_links、tasks。除 blobs 外，每个阶段使用独立目标事务；目标数据和 `last_completed_stage/context_json/report_json` 在同一事务中提交。blobs 按 Cloudreve `entities.id` keyset 分页，每页独立事务提交目标 blob、逐行 mapping、cursor 和报告进度。 当前 source fingerprint 使用数据库 URL 摘要和各源表记录数量，只能防止明显连错源或数量变化，不能检测“数量不变但内容变化”。生产迁移仍应冻结 Cloudreve 写入。 #### `aster_external_migration_stage_cursors` -尚未实现。它是下一步批内分页恢复所需的表。 +已用于 blobs；其他 stage 尚未接入。 -| 字段 | 建议类型 | 说明 | +| 字段 | 当前类型 | 说明 | |---|---|---| -| `run_id` | FK | 所属 run | -| `stage` | String | users/folders/blobs/files 等 | -| `cursor_json` | Text | source cursor/offset/last ID | -| `processed_count` | BigInt | 已处理对象数 | -| `processed_bytes` | BigInt | 已处理字节数 | +| `run_id` | String PK | 所属 run | +| `stage` | String PK | 当前为 `blobs` | +| `cursor_value` | BigInt | 最后提交的 Cloudreve entity ID | +| `processed_count` | BigInt | 已扫描并提交的 blob entity 数量，包含明确跳过项 | | `updated_at` | DateTime | checkpoint 时间 | -| `state` | String | pending/running/completed/failed | 建议 `(run_id, stage)` 为联合主键或唯一键。 #### `aster_external_migration_object_map` -尚未实现独立行式映射表。当前有限实现将完整 HashMap 序列化到 `aster_external_migration_runs.context_json`；这能跨进程恢复阶段依赖，但不能高效查询单个对象，也不适合超大数据集。 +已用于 blob 映射；其他对象类型仍使用 run `context_json` 中的 HashMap。 -| 字段 | 建议类型 | 说明 | +| 字段 | 当前类型 | 说明 | |---|---|---| -| `id` | BigInt PK | 映射记录 ID | -| `run_id` | FK | 所属 run | -| `source_type` | String | user/folder/file/blob/share/policy | -| `source_id` | String | 支持整数 ID 或 object key | -| `target_type` | String | AD 对象类型 | -| `target_id` | String | 目标 ID | -| `source_hash` | String nullable | 源对象摘要或内容哈希 | -| `target_hash` | String nullable | 目标对象摘要或内容哈希 | -| `metadata_json` | Text | 名称、大小、parent 等校验上下文 | +| `run_id` | String PK | 所属 run | +| `object_type` | String PK | 当前为 `blob` | +| `source_id` | BigInt PK | Cloudreve entity ID | +| `target_id` | BigInt | AD file_blob ID | +| `created_at` | DateTime | 映射创建时间 | | `state` | String | created/reused/skipped/renamed/failed | | `created_at` | DateTime | 创建时间 | | `updated_at` | DateTime | 更新时间 | @@ -763,9 +757,9 @@ Cloudreve entity ID、object key、路径或 `policy+path+size` 摘要都不能 | Cloudreve v4 标签 | 已从 `tag:*` metadata 生成 AD `tags` 和 `system.tags` 文件/目录关联 | 增加跨批次持久化去重和冲突报告 | | Direct links | 提供 AD `direct_link_secret` 时生成 v2 URL，并以 `cloudreve.direct_links` property 保存源 ID 映射 | 增加独立 JSON/CSV 报告和可选旧 URL 重定向层 | | Cloudreve 任务 | 已全部写成 AD `system_runtime` 终态历史；活动任务归档为 canceled | 后续可增加独立 legacy task archive 表，避免占用运维任务列表 | -| 目标事务 | 已改为每 stage 独立事务，stage 数据和 checkpoint 原子提交 | 每 page/batch 事务，降低单个 files/blobs stage 的事务规模 | -| ID mapping | 内存 HashMap 同时持久化到 run `context_json`，并导出到 JSON 报告 | 独立 object mapping table，支持单对象查询和超大数据集 | -| 断点续传 | 已支持 `--run-id` + `--resume` 的阶段级恢复 | stage 内 page cursor、对象存储上传恢复和并行 worker checkpoint | +| 目标事务 | blobs 已按 page 提交；其他 stage 独立事务 | files 等 stage 继续分页，增加对象上传事务补偿 | +| ID mapping | blob 使用独立 object mapping 表；其他映射仍在 `context_json` | 将 files/folders 等映射也迁入独立表 | +| 断点续传 | blobs 已支持 entity ID cursor；其他 stage 为阶段级恢复 | files cursor、对象存储上传恢复和并行 worker checkpoint | | 幂等重复运行 | 同一 run 会跳过已完成 stage；失败 stage 因事务回滚可安全重跑 | 不依赖 run checkpoint 的对象级 upsert/reuse 和完整重复运行 | | 冲突策略 | 仅 username 自动后缀，其他冲突依赖 DB 报错 | fail/rename/skip/reuse/update 可配置 | | 物理对象复制 | 未实现，当前复用源 storage path | 流式 copy、SHA-256、目标路径和校验 | diff --git a/src/main.rs b/src/main.rs index 90a6a30..84b9b40 100644 --- a/src/main.rs +++ b/src/main.rs @@ -51,6 +51,8 @@ struct MigrateArgs { run_id: Option, #[arg(long, requires = "run_id")] resume: bool, + #[arg(long, default_value_t = 500, value_name = "COUNT")] + blob_batch_size: usize, } #[tokio::main] @@ -75,6 +77,7 @@ async fn main() -> Result<()> { dry_run: args.dry_run, run_id: args.run_id, resume: args.resume, + blob_batch_size: args.blob_batch_size, }) .await?; emit_report(report_path.as_deref(), &report)?; diff --git a/src/migration/checkpoint.rs b/src/migration/checkpoint.rs index b1c4bc2..e3abd3a 100644 --- a/src/migration/checkpoint.rs +++ b/src/migration/checkpoint.rs @@ -4,6 +4,45 @@ use sea_orm::{ActiveModelTrait, ConnectionTrait, EntityTrait, Schema, Set, Uncha use super::{MigrationContext, MigrationReport, TargetCounts}; +pub(super) mod stage_cursor { + use sea_orm::entity::prelude::*; + + #[sea_orm::model] + #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] + #[sea_orm(table_name = "aster_external_migration_stage_cursors")] + pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub run_id: String, + #[sea_orm(primary_key, auto_increment = false)] + pub stage: String, + pub cursor_value: i64, + pub processed_count: i64, + pub updated_at: DateTimeWithTimeZone, + } + + impl ActiveModelBehavior for ActiveModel {} +} + +pub(super) mod object_map { + use sea_orm::entity::prelude::*; + + #[sea_orm::model] + #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] + #[sea_orm(table_name = "aster_external_migration_object_map")] + pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub run_id: String, + #[sea_orm(primary_key, auto_increment = false)] + pub object_type: String, + #[sea_orm(primary_key, auto_increment = false)] + pub source_id: i64, + pub target_id: i64, + pub created_at: DateTimeWithTimeZone, + } + + impl ActiveModelBehavior for ActiveModel {} +} + #[sea_orm::model] #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] #[sea_orm(table_name = "aster_external_migration_runs")] @@ -54,9 +93,104 @@ pub(super) async fn ensure_table(db: &C) -> Result<()> { db.execute(&statement) .await .wrap_err("create external migration checkpoint table")?; + let mut statement = schema.create_table_from_entity(stage_cursor::Entity); + statement.if_not_exists(); + db.execute(&statement) + .await + .wrap_err("create external migration stage cursor table")?; + let mut statement = schema.create_table_from_entity(object_map::Entity); + statement.if_not_exists(); + db.execute(&statement) + .await + .wrap_err("create external migration object map table")?; + Ok(()) +} + +pub(super) async fn load_stage_cursor( + db: &C, + run_id: &str, + stage: &str, +) -> Result> { + Ok( + stage_cursor::Entity::find_by_id((run_id.to_string(), stage.to_string())) + .one(db) + .await?, + ) +} + +pub(super) async fn save_stage_cursor( + db: &C, + run_id: &str, + stage: &str, + cursor_value: i64, + processed_count: i64, +) -> Result<()> { + use sea_orm::sea_query::OnConflict; + + stage_cursor::Entity::insert(stage_cursor::ActiveModel { + run_id: Set(run_id.to_string()), + stage: Set(stage.to_string()), + cursor_value: Set(cursor_value), + processed_count: Set(processed_count), + updated_at: Set(chrono::Utc::now().fixed_offset()), + }) + .on_conflict( + OnConflict::columns([stage_cursor::Column::RunId, stage_cursor::Column::Stage]) + .update_columns([ + stage_cursor::Column::CursorValue, + stage_cursor::Column::ProcessedCount, + stage_cursor::Column::UpdatedAt, + ]) + .to_owned(), + ) + .exec(db) + .await + .wrap_err_with(|| format!("save migration run {run_id} stage {stage} cursor"))?; + Ok(()) +} + +pub(super) async fn save_object_mappings( + db: &C, + run_id: &str, + object_type: &str, + mappings: &[(i64, i64)], +) -> Result<()> { + if mappings.is_empty() { + return Ok(()); + } + let now = chrono::Utc::now().fixed_offset(); + object_map::Entity::insert_many(mappings.iter().map(|(source_id, target_id)| { + object_map::ActiveModel { + run_id: Set(run_id.to_string()), + object_type: Set(object_type.to_string()), + source_id: Set(*source_id), + target_id: Set(*target_id), + created_at: Set(now), + } + })) + .exec(db) + .await + .wrap_err_with(|| format!("save migration run {run_id} {object_type} mappings"))?; Ok(()) } +pub(super) async fn load_object_mappings( + db: &C, + run_id: &str, + object_type: &str, +) -> Result> { + use sea_orm::{ColumnTrait, QueryFilter}; + + Ok(object_map::Entity::find() + .filter(object_map::Column::RunId.eq(run_id)) + .filter(object_map::Column::ObjectType.eq(object_type)) + .all(db) + .await? + .into_iter() + .map(|mapping| (mapping.source_id, mapping.target_id)) + .collect()) +} + pub(super) async fn create( db: &C, checkpoint: NewCheckpoint<'_>, @@ -153,6 +287,27 @@ pub(super) async fn save_stage( Ok(()) } +pub(super) async fn save_progress( + db: &C, + run_id: &str, + context: &MigrationContext, + report: &MigrationReport, +) -> Result<()> { + ActiveModel { + id: Unchanged(run_id.to_string()), + status: Set("running".to_string()), + context_json: Set(serde_json::to_value(context).wrap_err("serialize migration context")?), + report_json: Set(serde_json::to_value(report).wrap_err("serialize migration report")?), + last_error: Set(None), + updated_at: Set(chrono::Utc::now().fixed_offset()), + ..Default::default() + } + .update(db) + .await + .wrap_err_with(|| format!("save migration run {run_id} progress"))?; + Ok(()) +} + pub(super) async fn finish( db: &C, run_id: &str, diff --git a/src/migration/mod.rs b/src/migration/mod.rs index 1be1f2a..d27160c 100644 --- a/src/migration/mod.rs +++ b/src/migration/mod.rs @@ -9,7 +9,7 @@ use color_eyre::eyre::{Result, WrapErr, bail}; use hmac::{Hmac, Mac}; use sea_orm::{ ActiveModelTrait, ColumnTrait, Database, DatabaseConnection, EntityTrait, PaginatorTrait, - QueryFilter, Set, TransactionTrait, + QueryFilter, QueryOrder, QuerySelect, Set, TransactionTrait, }; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; @@ -31,6 +31,7 @@ pub struct MigrationOptions { pub dry_run: bool, pub run_id: Option, pub resume: bool, + pub blob_batch_size: usize, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -209,13 +210,13 @@ impl MigrationReport { }); } - fn set_mappings(&mut self, context: &MigrationContext) { + fn set_mappings(&mut self, context: &MigrationContext, blobs: &HashMap) { self.mappings = MigrationMappings { policies: sorted_id_mappings(&context.policies), policy_groups: sorted_id_mappings(&context.policy_groups), users: sorted_id_mappings(&context.users), folders: sorted_id_mappings(&context.folders), - blobs: sorted_id_mappings(&context.blobs), + blobs: sorted_id_mappings(blobs), files: sorted_id_mappings(&context.files), shares: sorted_id_mappings(&context.shares), tasks: sorted_id_mappings(&context.tasks), @@ -418,6 +419,9 @@ pub async fn migrate(options: MigrationOptions) -> Result { if let Some(run_id) = options.run_id.as_deref() { validate_run_id(run_id)?; } + if !(1..=10_000).contains(&options.blob_batch_size) { + bail!("--blob-batch-size must be between 1 and 10000"); + } let source = connect(&options.source_url, "Cloudreve").await?; let target = connect(&options.target_url, "AsterDrive").await?; @@ -491,6 +495,24 @@ pub async fn migrate(options: MigrationOptions) -> Result { .await?; (context, baseline, None) }; + let mut blob_mappings = checkpoint::load_object_mappings(&target, &run_id, "blob").await?; + if blob_mappings.is_empty() && !context.blobs.is_empty() { + let legacy_mappings = context + .blobs + .iter() + .map(|(source_id, target_id)| (*source_id, *target_id)) + .collect::>(); + checkpoint::save_object_mappings(&target, &run_id, "blob", &legacy_mappings).await?; + blob_mappings.extend(legacy_mappings); + context.blobs.clear(); + } + if report.migrated_blobs != blob_mappings.len() { + bail!( + "migration run {run_id} blob mapping count {} does not match migrated blob count {}; restore the checkpoint tables before resuming", + blob_mappings.len(), + report.migrated_blobs + ); + } let password_hash = hash_password(&options.default_password)?; @@ -498,22 +520,43 @@ pub async fn migrate(options: MigrationOptions) -> Result { if !stage.should_run_after(last_completed_stage.as_deref())? { continue; } + if stage == MigrationStage::Blobs { + let inputs = BlobBatchInputs { + source: &source, + target: &target, + run_id: &run_id, + source_data: &source_data, + options: &options, + context: &context, + }; + if let Err(error) = + migrate_blobs_batched(&inputs, &mut blob_mappings, &mut report).await + { + let _ = checkpoint::mark_failed(&target, &run_id, &error.to_string()).await; + return Err(error).wrap_err_with(|| { + format!( + "migration run {run_id} failed at stage {}; rerun with --resume --run-id {run_id}", + stage.as_str() + ) + }); + } + last_completed_stage = Some(stage.as_str().to_string()); + continue; + } let transaction = target .begin() .await .wrap_err_with(|| format!("begin migration stage {}", stage.as_str()))?; let stage_result: Result<()> = async { - execute_stage( - stage, - &transaction, - &source_data, - &options, - &password_hash, - &mut context, - &mut report, - ) - .await?; - report.set_mappings(&context); + let inputs = StageInputs { + source_db: &source, + source_data: &source_data, + options: &options, + password_hash: &password_hash, + blob_mappings: &blob_mappings, + }; + execute_stage(stage, &transaction, &inputs, &mut context, &mut report).await?; + report.set_mappings(&context, &blob_mappings); if !report .completed_stages .iter() @@ -546,7 +589,7 @@ pub async fn migrate(options: MigrationOptions) -> Result { last_completed_stage = Some(stage.as_str().to_string()); } - report.set_mappings(&context); + report.set_mappings(&context, &blob_mappings); report.validation = match validate_migration_result(&target, &target_before, &report).await { Ok(validation) => validation, Err(error) => { @@ -571,46 +614,269 @@ pub async fn migrate(options: MigrationOptions) -> Result { Ok(report) } +struct StageInputs<'a> { + source_db: &'a DatabaseConnection, + source_data: &'a SourceData, + options: &'a MigrationOptions, + password_hash: &'a str, + blob_mappings: &'a HashMap, +} + async fn execute_stage( stage: MigrationStage, transaction: &sea_orm::DatabaseTransaction, - source_data: &SourceData, - options: &MigrationOptions, - password_hash: &str, + inputs: &StageInputs<'_>, context: &mut MigrationContext, report: &mut MigrationReport, ) -> Result<()> { match stage { MigrationStage::Policies => { - migrate_policies(transaction, source_data, options, context, report).await + migrate_policies( + transaction, + inputs.source_data, + inputs.options, + context, + report, + ) + .await } MigrationStage::PolicyGroups => { - migrate_policy_groups(transaction, source_data, context, report).await + migrate_policy_groups(transaction, inputs.source_data, context, report).await } MigrationStage::Users => { - migrate_users(transaction, source_data, password_hash, context, report).await + migrate_users( + transaction, + inputs.source_data, + inputs.password_hash, + context, + report, + ) + .await + } + MigrationStage::Folders => { + migrate_folders(transaction, inputs.source_data, context, report).await + } + MigrationStage::Blobs => bail!("blobs stage must use the batched runner"), + MigrationStage::Files => { + migrate_files( + transaction, + inputs.source_db, + inputs.source_data, + inputs.blob_mappings, + context, + report, + ) + .await } - MigrationStage::Folders => migrate_folders(transaction, source_data, context, report).await, - MigrationStage::Blobs => migrate_blobs(transaction, source_data, context, report).await, - MigrationStage::Files => migrate_files(transaction, source_data, context, report).await, MigrationStage::Metadata => { - migrate_metadata(transaction, source_data, context, report).await + migrate_metadata(transaction, inputs.source_data, context, report).await + } + MigrationStage::Shares => { + migrate_shares(transaction, inputs.source_data, context, report).await } - MigrationStage::Shares => migrate_shares(transaction, source_data, context, report).await, MigrationStage::DirectLinks => { migrate_direct_links( transaction, - source_data, + inputs.source_data, context, - options.direct_link_secret.as_deref(), + inputs.options.direct_link_secret.as_deref(), report, ) .await } - MigrationStage::Tasks => migrate_tasks(transaction, source_data, context, report).await, + MigrationStage::Tasks => { + migrate_tasks(transaction, inputs.source_data, context, report).await + } } } +struct BlobBatchInputs<'a> { + source: &'a DatabaseConnection, + target: &'a DatabaseConnection, + run_id: &'a str, + source_data: &'a SourceData, + options: &'a MigrationOptions, + context: &'a MigrationContext, +} + +async fn migrate_blobs_batched( + inputs: &BlobBatchInputs<'_>, + blob_mappings: &mut HashMap, + report: &mut MigrationReport, +) -> Result<()> { + let cursor = + checkpoint::load_stage_cursor(inputs.target, inputs.run_id, MigrationStage::Blobs.as_str()) + .await?; + let mut cursor_value = cursor.as_ref().map_or(0, |cursor| cursor.cursor_value); + let mut processed_count = cursor.as_ref().map_or(0, |cursor| cursor.processed_count); + + loop { + let mut query = cr::entities::Entity::find() + .filter(cr::entities::Column::Type.eq(0)) + .filter(cr::entities::Column::Id.gt(cursor_value)) + .order_by_asc(cr::entities::Column::Id) + .limit(inputs.options.blob_batch_size as u64); + if !inputs.source_data.include_deleted { + query = query.filter(cr::entities::Column::DeletedAt.is_null()); + } + let entities = query.all(inputs.source).await?; + if entities.is_empty() { + let transaction = inputs + .target + .begin() + .await + .wrap_err("begin blobs completion")?; + report.set_mappings(inputs.context, blob_mappings); + if !report + .completed_stages + .iter() + .any(|completed| completed == MigrationStage::Blobs.as_str()) + { + report + .completed_stages + .push(MigrationStage::Blobs.as_str().to_string()); + } + checkpoint::save_stage( + &transaction, + inputs.run_id, + MigrationStage::Blobs.as_str(), + inputs.context, + report, + ) + .await?; + transaction + .commit() + .await + .wrap_err("commit blobs completion")?; + return Ok(()); + } + + let entity_ids = entities.iter().map(|entity| entity.id).collect::>(); + let association_info = load_blob_association_info(inputs.source, &entity_ids).await?; + let last_entity_id = entities.last().expect("non-empty blob batch").id; + let transaction = inputs.target.begin().await.wrap_err("begin blobs batch")?; + let mappings = migrate_blob_batch( + &transaction, + &entities, + &association_info.reference_counts, + &association_info.thumbnail_paths, + inputs.context, + report, + ) + .await?; + checkpoint::save_object_mappings(&transaction, inputs.run_id, "blob", &mappings).await?; + processed_count += entities.len() as i64; + checkpoint::save_stage_cursor( + &transaction, + inputs.run_id, + MigrationStage::Blobs.as_str(), + last_entity_id, + processed_count, + ) + .await?; + checkpoint::save_progress(&transaction, inputs.run_id, inputs.context, report).await?; + transaction.commit().await.wrap_err("commit blobs batch")?; + blob_mappings.extend(mappings); + cursor_value = last_entity_id; + } +} + +struct BlobAssociationInfo { + reference_counts: HashMap, + thumbnail_paths: HashMap, +} + +async fn load_blob_association_info( + source: &DatabaseConnection, + blob_ids: &[i64], +) -> Result { + if blob_ids.is_empty() { + return Ok(BlobAssociationInfo { + reference_counts: HashMap::new(), + thumbnail_paths: HashMap::new(), + }); + } + let relations = cr::file_entities::Entity::find() + .filter(cr::file_entities::Column::EntityId.is_in(blob_ids.iter().copied())) + .all(source) + .await?; + let primary_files = cr::files::Entity::find() + .filter(cr::files::Column::PrimaryEntity.is_in(blob_ids.iter().copied())) + .all(source) + .await?; + let file_ids = relations + .iter() + .map(|relation| relation.file_id) + .chain(primary_files.iter().map(|file| file.id)) + .collect::>(); + if file_ids.is_empty() { + return Ok(BlobAssociationInfo { + reference_counts: HashMap::new(), + thumbnail_paths: HashMap::new(), + }); + } + let all_relations = cr::file_entities::Entity::find() + .filter(cr::file_entities::Column::FileId.is_in(file_ids.iter().copied())) + .all(source) + .await?; + let thumbnail_ids = all_relations + .iter() + .map(|relation| relation.entity_id) + .collect::>(); + let thumbnails = cr::entities::Entity::find() + .filter(cr::entities::Column::Id.is_in(thumbnail_ids.iter().copied())) + .filter(cr::entities::Column::Type.eq(1)) + .all(source) + .await? + .into_iter() + .map(|entity| (entity.id, entity.source)) + .collect::>(); + let relations_by_file = + all_relations + .into_iter() + .fold(HashMap::>::new(), |mut values, relation| { + values + .entry(relation.file_id) + .or_default() + .push(relation.entity_id); + values + }); + let blob_id_set = blob_ids.iter().copied().collect::>(); + let mut file_blob_pairs = HashSet::new(); + let mut thumbnail_paths = HashMap::new(); + for (blob_id, file_id) in relations + .into_iter() + .map(|relation| (relation.entity_id, relation.file_id)) + .chain( + primary_files + .into_iter() + .filter_map(|file| file.primary_entity.map(|entity_id| (entity_id, file.id))), + ) + { + if blob_id_set.contains(&blob_id) { + file_blob_pairs.insert((file_id, blob_id)); + } + if let Some(path) = relations_by_file + .get(&file_id) + .into_iter() + .flatten() + .find_map(|entity_id| thumbnails.get(entity_id)) + { + thumbnail_paths + .entry(blob_id) + .or_insert_with(|| path.clone()); + } + } + let mut reference_counts = HashMap::new(); + for (_, blob_id) in file_blob_pairs { + *reference_counts.entry(blob_id).or_insert(0) += 1; + } + Ok(BlobAssociationInfo { + reference_counts, + thumbnail_paths, + }) +} + fn validate_run_id(run_id: &str) -> Result<()> { let run_id = run_id.trim(); if run_id.is_empty() || run_id.chars().count() > 128 || run_id.chars().any(char::is_control) { @@ -630,8 +896,8 @@ fn source_fingerprint(source_url: &str, source: &SourceData) -> String { source.groups.len(), source.policies.len(), source.files.len(), - source.entities.len(), - source.file_entities.len(), + source.source_entities, + source.source_file_entities, source.shares.len(), source.metadata.len(), source.direct_links.len(), @@ -1035,6 +1301,7 @@ struct MigrationContext { users: HashMap, usernames: HashMap, folders: HashMap, + #[serde(default, skip_serializing_if = "HashMap::is_empty")] blobs: HashMap, files: HashMap, shares: HashMap, @@ -1058,8 +1325,9 @@ struct SourceData { users: Vec, policies: Vec, files: Vec, - entities: Vec, - file_entities: Vec, + source_entities: u64, + source_file_entities: u64, + include_deleted: bool, shares: Vec, metadata: Vec, direct_links: Vec, @@ -1072,8 +1340,13 @@ impl SourceData { let users = cr::users::Entity::find().all(db).await?; let policies = cr::storage_policies::Entity::find().all(db).await?; let files = cr::files::Entity::find().all(db).await?; - let entities = cr::entities::Entity::find().all(db).await?; - let file_entities = cr::file_entities::Entity::find().all(db).await?; + let entity_query = if include_deleted { + cr::entities::Entity::find() + } else { + cr::entities::Entity::find().filter(cr::entities::Column::DeletedAt.is_null()) + }; + let source_entities = entity_query.count(db).await?; + let source_file_entities = cr::file_entities::Entity::find().count(db).await?; let shares = cr::shares::Entity::find().all(db).await?; let metadata = cr::metadata::Entity::find().all(db).await?; let direct_links = cr::direct_links::Entity::find().all(db).await?; @@ -1086,10 +1359,9 @@ impl SourceData { model.deleted_at.is_some() }), files, - entities: filter_deleted(entities, include_deleted, |model| { - model.deleted_at.is_some() - }), - file_entities, + source_entities, + source_file_entities, + include_deleted, shares: filter_deleted(shares, include_deleted, |model| model.deleted_at.is_some()), metadata: filter_deleted(metadata, include_deleted, |model| { model.deleted_at.is_some() @@ -1108,7 +1380,7 @@ impl SourceData { source_policies: self.policies.len() as u64, source_folders: self.files.iter().filter(|file| file.r#type == 1).count() as u64, source_files: self.files.iter().filter(|file| file.r#type == 0).count() as u64, - source_entities: self.entities.len() as u64, + source_entities: self.source_entities, source_shares: self.shares.len() as u64, source_direct_links: self.direct_links.len() as u64, source_tag_assignments: self @@ -1552,6 +1824,7 @@ mod tests { dry_run: false, run_id: None, resume: false, + blob_batch_size: 500, }) .await?; @@ -1662,6 +1935,7 @@ mod tests { dry_run: false, run_id: Some(run_id.clone()), resume: false, + blob_batch_size: 500, }; let error = migrate(options.clone()).await.unwrap_err(); @@ -1719,6 +1993,115 @@ mod tests { Ok(()) } + #[tokio::test] + async fn resumes_blobs_from_last_committed_batch() -> Result<()> { + let suffix = uuid::Uuid::new_v4(); + let run_id = format!("blob-resume-test-{suffix}"); + let source_path = std::env::temp_dir().join(format!("cloudreve-blob-resume-{suffix}.db")); + let target_path = std::env::temp_dir().join(format!("asterdrive-blob-resume-{suffix}.db")); + let source_url = sqlite_url(&source_path); + let target_url = sqlite_url(&target_path); + + let source = Database::connect(&source_url).await?; + create_source_schema(&source).await?; + seed_source(&source).await?; + let extra_blob_ids = seed_extra_blob_entities(&source, 3).await?; + source.close().await?; + + let failing_blob_id = extra_blob_ids[1]; + let target = Database::connect(&target_url).await?; + create_target_schema(&target).await?; + target + .execute_unprepared(&format!( + "CREATE TRIGGER fail_blob_batch BEFORE INSERT ON file_blobs \ + WHEN NEW.hash = '{}' \ + BEGIN SELECT RAISE(ABORT, 'forced blob batch failure'); END", + opaque_blob_key(failing_blob_id) + )) + .await?; + target.close().await?; + + let options = MigrationOptions { + source_url: source_url.clone(), + target_url: target_url.clone(), + default_password: "temporary-password".to_string(), + local_base_path: "C:/cloudreve".to_string(), + direct_link_secret: Some("test-direct-link-secret".to_string()), + include_deleted: false, + allow_non_empty_target: false, + skip_unsupported_policies: false, + dry_run: false, + run_id: Some(run_id.clone()), + resume: false, + blob_batch_size: 2, + }; + + let error = migrate(options.clone()).await.unwrap_err(); + assert!(error.to_string().contains("blobs")); + + let target = Database::connect(&target_url).await?; + assert_eq!(ad::file_blobs::Entity::find().count(&target).await?, 2); + let cursor = checkpoint::load_stage_cursor(&target, &run_id, "blobs") + .await? + .expect("committed blob cursor"); + assert_eq!(cursor.cursor_value, extra_blob_ids[0]); + assert_eq!(cursor.processed_count, 2); + assert_eq!( + checkpoint::object_map::Entity::find() + .filter(checkpoint::object_map::Column::RunId.eq(&run_id)) + .filter(checkpoint::object_map::Column::ObjectType.eq("blob")) + .count(&target) + .await?, + 2 + ); + let failed_checkpoint = checkpoint::Entity::find_by_id(run_id.clone()) + .one(&target) + .await? + .expect("failed blob migration checkpoint"); + assert_eq!(failed_checkpoint.status, "failed"); + assert_eq!( + failed_checkpoint.last_completed_stage.as_deref(), + Some("folders") + ); + let failed_report: MigrationReport = serde_json::from_value(failed_checkpoint.report_json)?; + assert_eq!(failed_report.migrated_blobs, 2); + target + .execute_unprepared("DROP TRIGGER fail_blob_batch") + .await?; + target.close().await?; + + let report = migrate(MigrationOptions { + resume: true, + ..options + }) + .await?; + assert!(report.resumed); + assert_eq!(report.migrated_blobs, 4); + assert_eq!(report.mappings.blobs.len(), 4); + assert!(report.validation.passed); + + let target = Database::connect(&target_url).await?; + assert_eq!(ad::file_blobs::Entity::find().count(&target).await?, 4); + assert_eq!( + checkpoint::object_map::Entity::find() + .filter(checkpoint::object_map::Column::RunId.eq(&run_id)) + .filter(checkpoint::object_map::Column::ObjectType.eq("blob")) + .count(&target) + .await?, + 4 + ); + let completed_cursor = checkpoint::load_stage_cursor(&target, &run_id, "blobs") + .await? + .expect("completed blob cursor"); + assert_eq!(completed_cursor.cursor_value, extra_blob_ids[2]); + assert_eq!(completed_cursor.processed_count, 4); + target.close().await?; + + let _ = std::fs::remove_file(source_path); + let _ = std::fs::remove_file(target_path); + Ok(()) + } + fn sqlite_url(path: &std::path::Path) -> String { format!( "sqlite://{}?mode=rwc", @@ -1943,4 +2326,39 @@ mod tests { } Ok(()) } + + async fn seed_extra_blob_entities(db: &DatabaseConnection, count: usize) -> Result> { + let now = chrono::Utc::now().fixed_offset(); + let policy_id = cr::storage_policies::Entity::find() + .one(db) + .await? + .expect("seeded storage policy") + .id; + let user_id = cr::users::Entity::find() + .one(db) + .await? + .expect("seeded user") + .id; + let mut ids = Vec::with_capacity(count); + for index in 0..count { + let entity = cr::entities::ActiveModel { + created_at: Set(now), + updated_at: Set(now), + deleted_at: Set(None), + r#type: Set(0), + source: Set(format!("uploads/extra-{index}.bin")), + size: Set(256 + index as i64), + reference_count: Set(1), + upload_session_id: Set(None), + recycle_options: Set(None), + storage_policy_entities: Set(policy_id), + created_by: Set(Some(user_id)), + ..Default::default() + } + .insert(db) + .await?; + ids.push(entity.id); + } + Ok(ids) + } } diff --git a/src/migration/phases.rs b/src/migration/phases.rs index 347bf35..e7739c4 100644 --- a/src/migration/phases.rs +++ b/src/migration/phases.rs @@ -248,42 +248,16 @@ pub(super) async fn migrate_folders( Ok(()) } -pub(super) async fn migrate_blobs( +pub(super) async fn migrate_blob_batch( transaction: &DatabaseTransaction, - source: &SourceData, - context: &mut MigrationContext, + entities: &[cr::entities::Model], + reference_counts: &HashMap, + thumbnail_paths: &HashMap, + context: &MigrationContext, report: &mut MigrationReport, -) -> Result<()> { - let associations = associations(source); - let entities: HashMap = source - .entities - .iter() - .map(|entity| (entity.id, entity)) - .collect(); - let mut reference_counts: HashMap = HashMap::new(); - let mut thumbnail_paths: HashMap = HashMap::new(); - for entity_ids in associations.values() { - let thumbnail_path = entity_ids.iter().find_map(|entity_id| { - entities - .get(entity_id) - .filter(|entity| entity.r#type == 1) - .map(|entity| entity.source.clone()) - }); - for entity_id in entity_ids { - if entities - .get(entity_id) - .is_some_and(|entity| entity.r#type == 0) - { - *reference_counts.entry(*entity_id).or_default() += 1; - if let Some(path) = &thumbnail_path { - thumbnail_paths - .entry(*entity_id) - .or_insert_with(|| path.clone()); - } - } - } - } - for entity in source.entities.iter().filter(|entity| entity.r#type == 0) { +) -> Result> { + let mut mappings = Vec::with_capacity(entities.len()); + for entity in entities { let Some(policy_id) = context .policies .get(&entity.storage_policy_entities) @@ -317,21 +291,29 @@ pub(super) async fn migrate_blobs( .insert(transaction) .await .wrap_err_with(|| format!("migrate Cloudreve entity {}", entity.id))?; - context.blobs.insert(entity.id, target.id); + mappings.push((entity.id, target.id)); report.migrated_blobs += 1; } - Ok(()) + Ok(mappings) } pub(super) async fn migrate_files( transaction: &DatabaseTransaction, + source_db: &DatabaseConnection, source: &SourceData, + blob_mappings: &HashMap, context: &mut MigrationContext, report: &mut MigrationReport, ) -> Result<()> { - let associations = associations(source); - let entities: HashMap = source - .entities + let entity_query = if source.include_deleted { + cr::entities::Entity::find() + } else { + cr::entities::Entity::find().filter(cr::entities::Column::DeletedAt.is_null()) + }; + let source_entities = entity_query.all(source_db).await?; + let source_file_entities = cr::file_entities::Entity::find().all(source_db).await?; + let associations = associations(&source.files, &source_file_entities); + let entities: HashMap = source_entities .iter() .map(|entity| (entity.id, entity)) .collect(); @@ -364,12 +346,12 @@ pub(super) async fn migrate_files( .into_iter() .flatten() .filter_map(|entity_id| entities.get(entity_id).copied()) - .filter(|entity| entity.r#type == 0 && context.blobs.contains_key(&entity.id)) + .filter(|entity| entity.r#type == 0 && blob_mappings.contains_key(&entity.id)) .collect(); version_entities.sort_by_key(|entity| entity.created_at); let primary_entity_id = file .primary_entity - .filter(|id| context.blobs.contains_key(id)) + .filter(|id| blob_mappings.contains_key(id)) .or_else(|| version_entities.last().map(|entity| entity.id)); let Some(primary_entity_id) = primary_entity_id else { report.record_skip( @@ -382,7 +364,7 @@ pub(super) async fn migrate_files( .push(format!("file {} has no migratable version entity", file.id)); continue; }; - let blob_id = context.blobs[&primary_entity_id]; + let blob_id = blob_mappings[&primary_entity_id]; let (mime_type, compound_extension, extension, category) = file_classification(&file.name); let target = ad::files::ActiveModel { name: Set(file.name.clone()), @@ -422,7 +404,7 @@ pub(super) async fn migrate_files( for (index, entity) in historical.into_iter().enumerate() { ad::file_versions::ActiveModel { file_id: Set(target.id), - blob_id: Set(context.blobs[&entity.id]), + blob_id: Set(blob_mappings[&entity.id]), version: Set((index + 1) as i64), size: Set(entity.size), created_at: Set(entity.created_at), @@ -829,15 +811,18 @@ pub(super) async fn migrate_shares( Ok(()) } -fn associations(source: &SourceData) -> HashMap> { +fn associations( + files: &[cr::files::Model], + file_entities: &[cr::file_entities::Model], +) -> HashMap> { let mut result: HashMap> = HashMap::new(); - for relation in &source.file_entities { + for relation in file_entities { result .entry(relation.file_id) .or_default() .push(relation.entity_id); } - for file in &source.files { + for file in files { if let Some(primary_entity) = file.primary_entity { let values = result.entry(file.id).or_default(); if !values.contains(&primary_entity) { From 661b2b4f6f8ea274b4631bbb8b8e1be32db5fb3c Mon Sep 17 00:00:00 2001 From: Cg8 <5712.cg8@gmail.com> Date: Tue, 14 Jul 2026 09:00:54 +0800 Subject: [PATCH 6/7] feat: batch migrate files and versions --- README.md | 7 +- docs/cloudreve-to-asterdrive-field-mapping.md | 10 +- docs/migration-architecture.md | 24 +- src/main.rs | 3 + src/migration/mod.rs | 422 +++++++++++++++++- src/migration/phases.rs | 107 +++-- 6 files changed, 489 insertions(+), 84 deletions(-) diff --git a/README.md b/README.md index f1fa6d4..bf5bbb3 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ Cloudreve Qiniu, Upyun, remote-node, OneDrive and encrypted storage policies can ## Limited resume -Migration stages run in this order: policies, policy groups, users, folders, blobs, files, metadata, shares, direct links and tasks. Most stages are committed as one transaction. The blobs stage uses keyset pages ordered by Cloudreve `entities.id`; each page writes AD `file_blobs`, object mappings, its cursor and report progress in one target transaction. +Migration stages run in this order: policies, policy groups, users, folders, blobs, files, metadata, shares, direct links and tasks. Most stages are committed as one transaction. Blobs use keyset pages ordered by Cloudreve `entities.id`, while files use keyset pages ordered by Cloudreve `files.id`; each page writes target records, object mappings, its cursor and report progress in one target transaction. Use a stable run ID for an operational migration. If a stage fails, fix the external cause and rerun the same command with `--resume`: @@ -73,15 +73,16 @@ cargo run -- migrate ` --local-base-path "C:/cloudreve" ` --run-id "cloudreve-cutover-2026-07-13" ` --blob-batch-size 500 ` + --file-batch-size 500 ` --resume ` --report-path "C:/migration/cloudreve-to-ad.json" ``` Resume verifies the source URL/count fingerprint, target URL fingerprint and migration-plan fingerprint. Repeat the same password, direct-link secret and migration flags used by the original run. Completed stages are skipped and their source-to-target mappings are restored from the checkpoint. Blob batch size is operational only and may be adjusted when resuming. -`--blob-batch-size` defaults to 500 and accepts 1-10000. A failed blob page rolls back only that page; resume continues after the last committed entity ID. Blob source rows are no longer loaded into memory as one complete collection, and blob mappings are stored row-by-row in `aster_external_migration_object_map`. +`--blob-batch-size` and `--file-batch-size` default to 500 and accept 1-10000. A failed blob or file page rolls back only that page; resume continues after the last committed entity or file ID. Blob and normal-file source rows are no longer loaded into memory as complete collections, and their mappings are stored row-by-row in `aster_external_migration_object_map`. -Other stages, including files, are still stage-level only: a failure restarts that entire stage. The files stage currently loads entity/version relationships into memory, and physical object bytes are still reused in place rather than copied or uploaded. +Folders, metadata, shares, direct links and tasks are still stage-level only: a failure restarts that entire stage. A file page loads only its related entity/version rows into memory. Physical object bytes are still reused in place rather than copied or uploaded. ## JSON migration report diff --git a/docs/cloudreve-to-asterdrive-field-mapping.md b/docs/cloudreve-to-asterdrive-field-mapping.md index 81a9ea4..2c4db7b 100644 --- a/docs/cloudreve-to-asterdrive-field-mapping.md +++ b/docs/cloudreve-to-asterdrive-field-mapping.md @@ -543,15 +543,15 @@ Cloudreve 任务不能恢复到 AD 执行器中，但可以保存为不可执行 | 行为 | 当前实现 | |---|---| -| checkpoint 粒度 | blobs 为 batch 级；其他 stage 为 stage 级 | -| 原子性 | blob page 的目标记录、object mapping、cursor 和 report 同事务；其他 stage 的目标记录、context、report 和 last completed stage 同事务 | +| checkpoint 粒度 | blobs/files 为 batch 级；其他 stage 为 stage 级 | +| 原子性 | blob/file page 的目标记录、object mapping、cursor 和 report 同事务；其他 stage 的目标记录、context、report 和 last completed stage 同事务 | | 已完成 stage | resume 时跳过，不重复插入 | | 失败 stage | 整个 stage 回滚，resume 时从该 stage 开头重新执行 | -| ID mapping | blob 映射逐行保存在 `aster_external_migration_object_map`；其他映射仍在 `context_json` | +| ID mapping | blob/file 映射逐行保存在 `aster_external_migration_object_map`；其他映射仍在 `context_json` | | 初始目标计数 | 保存在 `baseline_json`，恢复完成后仍可执行正确的增量数量校验 | | 源校验 | URL 摘要 + 源表数量指纹 | | 计划校验 | local path、迁移 flags、临时密码摘要和 direct-link secret 摘要 | -| stage 内分页 cursor | blobs 已使用 `entities.id` keyset cursor；其他 stage 尚未实现 | +| stage 内分页 cursor | blobs 使用 `entities.id`、files 使用 `files.id` keyset cursor；其他 stage 尚未实现 | | 对象上传断点 | 尚未实现 | -由于源指纹不能检测数量不变的内容修改，断点恢复期间仍必须冻结 Cloudreve 写入。blobs 可从最后提交的 entity ID 继续，批大小由 `--blob-batch-size` 控制，默认 500；files 等其他 stage 仍会从 stage 起点重跑。对象字节复制/上传仍未实现，因此也没有对象存储分片上传续传。 +由于源指纹不能检测数量不变的内容修改，断点恢复期间仍必须冻结 Cloudreve 写入。blobs 可从最后提交的 entity ID 继续，files 可从最后提交的 file ID 继续，批大小由 `--blob-batch-size` 和 `--file-batch-size` 控制，默认均为 500；folders、metadata、shares、direct links 和 tasks 等其他 stage 仍会从 stage 起点重跑。对象字节复制/上传仍未实现，因此也没有对象存储分片上传续传。 diff --git a/docs/migration-architecture.md b/docs/migration-architecture.md index c2e4965..2586959 100644 --- a/docs/migration-architecture.md +++ b/docs/migration-architecture.md @@ -551,34 +551,34 @@ checkpoint 表保存在 AD 数据库中，以便迁移进程重启后恢复。 | `created_at` | DateTime | 创建时间 | | `updated_at` | DateTime | 最后更新时间 | -当前阶段顺序为 policies、policy_groups、users、folders、blobs、files、metadata、shares、direct_links、tasks。除 blobs 外，每个阶段使用独立目标事务；目标数据和 `last_completed_stage/context_json/report_json` 在同一事务中提交。blobs 按 Cloudreve `entities.id` keyset 分页，每页独立事务提交目标 blob、逐行 mapping、cursor 和报告进度。 +当前阶段顺序为 policies、policy_groups、users、folders、blobs、files、metadata、shares、direct_links、tasks。除 blobs/files 外，每个阶段使用独立目标事务；目标数据和 `last_completed_stage/context_json/report_json` 在同一事务中提交。blobs 按 Cloudreve `entities.id`、files 按 Cloudreve `files.id` keyset 分页；每页独立事务提交目标记录、逐行 mapping、cursor 和报告进度。 当前 source fingerprint 使用数据库 URL 摘要和各源表记录数量，只能防止明显连错源或数量变化，不能检测“数量不变但内容变化”。生产迁移仍应冻结 Cloudreve 写入。 #### `aster_external_migration_stage_cursors` -已用于 blobs；其他 stage 尚未接入。 +已用于 blobs 和 files；其他 stage 尚未接入。 | 字段 | 当前类型 | 说明 | |---|---|---| | `run_id` | String PK | 所属 run | -| `stage` | String PK | 当前为 `blobs` | -| `cursor_value` | BigInt | 最后提交的 Cloudreve entity ID | -| `processed_count` | BigInt | 已扫描并提交的 blob entity 数量，包含明确跳过项 | +| `stage` | String PK | 当前为 `blobs` 或 `files` | +| `cursor_value` | BigInt | 最后提交的 Cloudreve entity ID 或 file ID | +| `processed_count` | BigInt | 已扫描并提交的 blob entity 或 file 数量，包含明确跳过项 | | `updated_at` | DateTime | checkpoint 时间 | 建议 `(run_id, stage)` 为联合主键或唯一键。 #### `aster_external_migration_object_map` -已用于 blob 映射；其他对象类型仍使用 run `context_json` 中的 HashMap。 +已用于 blob 和 file 映射；其他对象类型仍使用 run `context_json` 中的 HashMap。 | 字段 | 当前类型 | 说明 | |---|---|---| | `run_id` | String PK | 所属 run | -| `object_type` | String PK | 当前为 `blob` | -| `source_id` | BigInt PK | Cloudreve entity ID | -| `target_id` | BigInt | AD file_blob ID | +| `object_type` | String PK | 当前为 `blob` 或 `file` | +| `source_id` | BigInt PK | Cloudreve entity ID 或 file ID | +| `target_id` | BigInt | AD file_blob 或 file ID | | `created_at` | DateTime | 映射创建时间 | | `state` | String | created/reused/skipped/renamed/failed | | `created_at` | DateTime | 创建时间 | @@ -757,9 +757,9 @@ Cloudreve entity ID、object key、路径或 `policy+path+size` 摘要都不能 | Cloudreve v4 标签 | 已从 `tag:*` metadata 生成 AD `tags` 和 `system.tags` 文件/目录关联 | 增加跨批次持久化去重和冲突报告 | | Direct links | 提供 AD `direct_link_secret` 时生成 v2 URL，并以 `cloudreve.direct_links` property 保存源 ID 映射 | 增加独立 JSON/CSV 报告和可选旧 URL 重定向层 | | Cloudreve 任务 | 已全部写成 AD `system_runtime` 终态历史；活动任务归档为 canceled | 后续可增加独立 legacy task archive 表，避免占用运维任务列表 | -| 目标事务 | blobs 已按 page 提交；其他 stage 独立事务 | files 等 stage 继续分页，增加对象上传事务补偿 | -| ID mapping | blob 使用独立 object mapping 表；其他映射仍在 `context_json` | 将 files/folders 等映射也迁入独立表 | -| 断点续传 | blobs 已支持 entity ID cursor；其他 stage 为阶段级恢复 | files cursor、对象存储上传恢复和并行 worker checkpoint | +| 目标事务 | blobs/files 已按 page 提交；其他 stage 独立事务 | folders、metadata、shares 等 stage 继续分页，增加对象上传事务补偿 | +| ID mapping | blob/file 使用独立 object mapping 表；其他映射仍在 `context_json` | 将 folders、shares 等映射也迁入独立表 | +| 断点续传 | blobs/files 已支持 source ID cursor；其他 stage 为阶段级恢复 | folders 等 cursor、对象存储上传恢复和并行 worker checkpoint | | 幂等重复运行 | 同一 run 会跳过已完成 stage；失败 stage 因事务回滚可安全重跑 | 不依赖 run checkpoint 的对象级 upsert/reuse 和完整重复运行 | | 冲突策略 | 仅 username 自动后缀，其他冲突依赖 DB 报错 | fail/rename/skip/reuse/update 可配置 | | 物理对象复制 | 未实现，当前复用源 storage path | 流式 copy、SHA-256、目标路径和校验 | diff --git a/src/main.rs b/src/main.rs index 84b9b40..2e23ba6 100644 --- a/src/main.rs +++ b/src/main.rs @@ -53,6 +53,8 @@ struct MigrateArgs { resume: bool, #[arg(long, default_value_t = 500, value_name = "COUNT")] blob_batch_size: usize, + #[arg(long, default_value_t = 500, value_name = "COUNT")] + file_batch_size: usize, } #[tokio::main] @@ -78,6 +80,7 @@ async fn main() -> Result<()> { run_id: args.run_id, resume: args.resume, blob_batch_size: args.blob_batch_size, + file_batch_size: args.file_batch_size, }) .await?; emit_report(report_path.as_deref(), &report)?; diff --git a/src/migration/mod.rs b/src/migration/mod.rs index d27160c..51460e5 100644 --- a/src/migration/mod.rs +++ b/src/migration/mod.rs @@ -32,6 +32,7 @@ pub struct MigrationOptions { pub run_id: Option, pub resume: bool, pub blob_batch_size: usize, + pub file_batch_size: usize, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -210,14 +211,19 @@ impl MigrationReport { }); } - fn set_mappings(&mut self, context: &MigrationContext, blobs: &HashMap) { + fn set_mappings( + &mut self, + context: &MigrationContext, + blobs: &HashMap, + files: &HashMap, + ) { self.mappings = MigrationMappings { policies: sorted_id_mappings(&context.policies), policy_groups: sorted_id_mappings(&context.policy_groups), users: sorted_id_mappings(&context.users), folders: sorted_id_mappings(&context.folders), blobs: sorted_id_mappings(blobs), - files: sorted_id_mappings(&context.files), + files: sorted_id_mappings(files), shares: sorted_id_mappings(&context.shares), tasks: sorted_id_mappings(&context.tasks), }; @@ -422,6 +428,9 @@ pub async fn migrate(options: MigrationOptions) -> Result { if !(1..=10_000).contains(&options.blob_batch_size) { bail!("--blob-batch-size must be between 1 and 10000"); } + if !(1..=10_000).contains(&options.file_batch_size) { + bail!("--file-batch-size must be between 1 and 10000"); + } let source = connect(&options.source_url, "Cloudreve").await?; let target = connect(&options.target_url, "AsterDrive").await?; @@ -513,6 +522,24 @@ pub async fn migrate(options: MigrationOptions) -> Result { report.migrated_blobs ); } + let mut file_mappings = checkpoint::load_object_mappings(&target, &run_id, "file").await?; + if file_mappings.is_empty() && !context.files.is_empty() { + let legacy_mappings = context + .files + .iter() + .map(|(source_id, target_id)| (*source_id, *target_id)) + .collect::>(); + checkpoint::save_object_mappings(&target, &run_id, "file", &legacy_mappings).await?; + file_mappings.extend(legacy_mappings); + context.files.clear(); + } + if report.migrated_files != file_mappings.len() { + bail!( + "migration run {run_id} file mapping count {} does not match migrated file count {}; restore the checkpoint tables before resuming", + file_mappings.len(), + report.migrated_files + ); + } let password_hash = hash_password(&options.default_password)?; @@ -528,6 +555,7 @@ pub async fn migrate(options: MigrationOptions) -> Result { source_data: &source_data, options: &options, context: &context, + file_mappings: &file_mappings, }; if let Err(error) = migrate_blobs_batched(&inputs, &mut blob_mappings, &mut report).await @@ -543,6 +571,30 @@ pub async fn migrate(options: MigrationOptions) -> Result { last_completed_stage = Some(stage.as_str().to_string()); continue; } + if stage == MigrationStage::Files { + let inputs = FileBatchInputs { + source: &source, + target: &target, + run_id: &run_id, + source_data: &source_data, + options: &options, + context: &context, + blob_mappings: &blob_mappings, + }; + if let Err(error) = + migrate_files_batched(&inputs, &mut file_mappings, &mut report).await + { + let _ = checkpoint::mark_failed(&target, &run_id, &error.to_string()).await; + return Err(error).wrap_err_with(|| { + format!( + "migration run {run_id} failed at stage {}; rerun with --resume --run-id {run_id}", + stage.as_str() + ) + }); + } + last_completed_stage = Some(stage.as_str().to_string()); + continue; + } let transaction = target .begin() .await @@ -553,10 +605,10 @@ pub async fn migrate(options: MigrationOptions) -> Result { source_data: &source_data, options: &options, password_hash: &password_hash, - blob_mappings: &blob_mappings, + file_mappings: &file_mappings, }; execute_stage(stage, &transaction, &inputs, &mut context, &mut report).await?; - report.set_mappings(&context, &blob_mappings); + report.set_mappings(&context, &blob_mappings, &file_mappings); if !report .completed_stages .iter() @@ -589,7 +641,7 @@ pub async fn migrate(options: MigrationOptions) -> Result { last_completed_stage = Some(stage.as_str().to_string()); } - report.set_mappings(&context, &blob_mappings); + report.set_mappings(&context, &blob_mappings, &file_mappings); report.validation = match validate_migration_result(&target, &target_before, &report).await { Ok(validation) => validation, Err(error) => { @@ -619,7 +671,7 @@ struct StageInputs<'a> { source_data: &'a SourceData, options: &'a MigrationOptions, password_hash: &'a str, - blob_mappings: &'a HashMap, + file_mappings: &'a HashMap, } async fn execute_stage( @@ -657,27 +709,35 @@ async fn execute_stage( migrate_folders(transaction, inputs.source_data, context, report).await } MigrationStage::Blobs => bail!("blobs stage must use the batched runner"), - MigrationStage::Files => { - migrate_files( + MigrationStage::Files => bail!("files stage must use the batched runner"), + MigrationStage::Metadata => { + migrate_metadata( transaction, inputs.source_db, inputs.source_data, - inputs.blob_mappings, + inputs.file_mappings, context, report, ) .await } - MigrationStage::Metadata => { - migrate_metadata(transaction, inputs.source_data, context, report).await - } MigrationStage::Shares => { - migrate_shares(transaction, inputs.source_data, context, report).await + migrate_shares( + transaction, + inputs.source_db, + inputs.source_data, + inputs.file_mappings, + context, + report, + ) + .await } MigrationStage::DirectLinks => { migrate_direct_links( transaction, + inputs.source_db, inputs.source_data, + inputs.file_mappings, context, inputs.options.direct_link_secret.as_deref(), report, @@ -697,6 +757,7 @@ struct BlobBatchInputs<'a> { source_data: &'a SourceData, options: &'a MigrationOptions, context: &'a MigrationContext, + file_mappings: &'a HashMap, } async fn migrate_blobs_batched( @@ -726,7 +787,7 @@ async fn migrate_blobs_batched( .begin() .await .wrap_err("begin blobs completion")?; - report.set_mappings(inputs.context, blob_mappings); + report.set_mappings(inputs.context, blob_mappings, inputs.file_mappings); if !report .completed_stages .iter() @@ -781,6 +842,135 @@ async fn migrate_blobs_batched( } } +struct FileBatchInputs<'a> { + source: &'a DatabaseConnection, + target: &'a DatabaseConnection, + run_id: &'a str, + source_data: &'a SourceData, + options: &'a MigrationOptions, + context: &'a MigrationContext, + blob_mappings: &'a HashMap, +} + +async fn migrate_files_batched( + inputs: &FileBatchInputs<'_>, + file_mappings: &mut HashMap, + report: &mut MigrationReport, +) -> Result<()> { + let cursor = + checkpoint::load_stage_cursor(inputs.target, inputs.run_id, MigrationStage::Files.as_str()) + .await?; + let mut cursor_value = cursor.as_ref().map_or(0, |cursor| cursor.cursor_value); + let mut processed_count = cursor.as_ref().map_or(0, |cursor| cursor.processed_count); + + loop { + let files = cr::files::Entity::find() + .filter(cr::files::Column::Type.eq(0)) + .filter(cr::files::Column::Id.gt(cursor_value)) + .order_by_asc(cr::files::Column::Id) + .limit(inputs.options.file_batch_size as u64) + .all(inputs.source) + .await?; + if files.is_empty() { + let transaction = inputs + .target + .begin() + .await + .wrap_err("begin files completion")?; + report.set_mappings(inputs.context, inputs.blob_mappings, file_mappings); + if !report + .completed_stages + .iter() + .any(|completed| completed == MigrationStage::Files.as_str()) + { + report + .completed_stages + .push(MigrationStage::Files.as_str().to_string()); + } + checkpoint::save_stage( + &transaction, + inputs.run_id, + MigrationStage::Files.as_str(), + inputs.context, + report, + ) + .await?; + transaction + .commit() + .await + .wrap_err("commit files completion")?; + return Ok(()); + } + + let (associations, entities) = + load_file_batch_data(inputs.source, inputs.source_data.include_deleted, &files).await?; + let last_file_id = files.last().expect("non-empty file batch").id; + let transaction = inputs.target.begin().await.wrap_err("begin files batch")?; + let mappings = migrate_file_batch( + &transaction, + &files, + &associations, + &entities, + inputs.blob_mappings, + inputs.context, + report, + ) + .await?; + checkpoint::save_object_mappings(&transaction, inputs.run_id, "file", &mappings).await?; + processed_count += files.len() as i64; + checkpoint::save_stage_cursor( + &transaction, + inputs.run_id, + MigrationStage::Files.as_str(), + last_file_id, + processed_count, + ) + .await?; + checkpoint::save_progress(&transaction, inputs.run_id, inputs.context, report).await?; + transaction.commit().await.wrap_err("commit files batch")?; + file_mappings.extend(mappings); + cursor_value = last_file_id; + } +} + +async fn load_file_batch_data( + source: &DatabaseConnection, + include_deleted: bool, + files: &[cr::files::Model], +) -> Result<(HashMap>, HashMap)> { + const QUERY_ID_BATCH_SIZE: usize = 500; + + let file_ids = files.iter().map(|file| file.id).collect::>(); + let mut file_entities = Vec::new(); + for file_ids in file_ids.chunks(QUERY_ID_BATCH_SIZE) { + file_entities.extend( + cr::file_entities::Entity::find() + .filter(cr::file_entities::Column::FileId.is_in(file_ids.iter().copied())) + .all(source) + .await?, + ); + } + let mut entity_ids = file_entities + .iter() + .map(|relation| relation.entity_id) + .collect::>(); + entity_ids.extend(files.iter().filter_map(|file| file.primary_entity)); + let entity_ids = entity_ids.into_iter().collect::>(); + let mut entities = HashMap::with_capacity(entity_ids.len()); + for entity_ids in entity_ids.chunks(QUERY_ID_BATCH_SIZE) { + let mut query = cr::entities::Entity::find() + .filter(cr::entities::Column::Id.is_in(entity_ids.iter().copied())) + .filter(cr::entities::Column::Type.eq(0)); + if !include_deleted { + query = query.filter(cr::entities::Column::DeletedAt.is_null()); + } + for entity in query.all(source).await? { + entities.insert(entity.id, entity); + } + } + Ok((associations(files, &file_entities), entities)) +} + struct BlobAssociationInfo { reference_counts: HashMap, thumbnail_paths: HashMap, @@ -895,7 +1085,7 @@ fn source_fingerprint(source_url: &str, source: &SourceData) -> String { source.users.len(), source.groups.len(), source.policies.len(), - source.files.len(), + source.source_file_records, source.source_entities, source.source_file_entities, source.shares.len(), @@ -1324,7 +1514,10 @@ struct SourceData { groups: Vec, users: Vec, policies: Vec, - files: Vec, + folders: Vec, + source_file_records: u64, + source_files: u64, + symbolic_files: u64, source_entities: u64, source_file_entities: u64, include_deleted: bool, @@ -1339,7 +1532,20 @@ impl SourceData { let groups = cr::groups::Entity::find().all(db).await?; let users = cr::users::Entity::find().all(db).await?; let policies = cr::storage_policies::Entity::find().all(db).await?; - let files = cr::files::Entity::find().all(db).await?; + let folders = cr::files::Entity::find() + .filter(cr::files::Column::Type.eq(1)) + .all(db) + .await?; + let source_file_records = cr::files::Entity::find().count(db).await?; + let source_files = cr::files::Entity::find() + .filter(cr::files::Column::Type.eq(0)) + .count(db) + .await?; + let symbolic_files = cr::files::Entity::find() + .filter(cr::files::Column::Type.eq(0)) + .filter(cr::files::Column::IsSymbolic.eq(true)) + .count(db) + .await?; let entity_query = if include_deleted { cr::entities::Entity::find() } else { @@ -1358,7 +1564,10 @@ impl SourceData { policies: filter_deleted(policies, include_deleted, |model| { model.deleted_at.is_some() }), - files, + folders, + source_file_records, + source_files, + symbolic_files, source_entities, source_file_entities, include_deleted, @@ -1378,8 +1587,8 @@ impl SourceData { source_users: self.users.len() as u64, source_groups: self.groups.len() as u64, source_policies: self.policies.len() as u64, - source_folders: self.files.iter().filter(|file| file.r#type == 1).count() as u64, - source_files: self.files.iter().filter(|file| file.r#type == 0).count() as u64, + source_folders: self.folders.len() as u64, + source_files: self.source_files, source_entities: self.source_entities, source_shares: self.shares.len() as u64, source_direct_links: self.direct_links.len() as u64, @@ -1412,7 +1621,7 @@ impl SourceData { "file objects are reused in their existing local/object-storage locations; the migration does not duplicate object bytes".to_string(), "Cloudreve tasks are archived as terminal AD system_runtime records; queued, processing and suspending tasks are canceled instead of resumed".to_string(), ]; - let symbolic = self.files.iter().filter(|file| file.is_symbolic).count(); + let symbolic = self.symbolic_files; if symbolic > 0 { warnings.push(format!( "{symbolic} symbolic/placeholder Cloudreve files cannot be represented in AD and will be skipped" @@ -1825,6 +2034,7 @@ mod tests { run_id: None, resume: false, blob_batch_size: 500, + file_batch_size: 500, }) .await?; @@ -1936,6 +2146,7 @@ mod tests { run_id: Some(run_id.clone()), resume: false, blob_batch_size: 500, + file_batch_size: 500, }; let error = migrate(options.clone()).await.unwrap_err(); @@ -2034,6 +2245,7 @@ mod tests { run_id: Some(run_id.clone()), resume: false, blob_batch_size: 2, + file_batch_size: 500, }; let error = migrate(options.clone()).await.unwrap_err(); @@ -2102,6 +2314,119 @@ mod tests { Ok(()) } + #[tokio::test] + async fn resumes_files_from_last_committed_batch() -> Result<()> { + let suffix = uuid::Uuid::new_v4(); + let run_id = format!("file-resume-test-{suffix}"); + let source_path = std::env::temp_dir().join(format!("cloudreve-file-resume-{suffix}.db")); + let target_path = std::env::temp_dir().join(format!("asterdrive-file-resume-{suffix}.db")); + let source_url = sqlite_url(&source_path); + let target_url = sqlite_url(&target_path); + + let source = Database::connect(&source_url).await?; + create_source_schema(&source).await?; + seed_source(&source).await?; + let extra_blob_ids = seed_extra_blob_entities(&source, 3).await?; + let extra_file_ids = seed_extra_files(&source, &extra_blob_ids).await?; + source.close().await?; + + let target = Database::connect(&target_url).await?; + create_target_schema(&target).await?; + target + .execute_unprepared( + "CREATE TRIGGER fail_file_batch BEFORE INSERT ON files \ + WHEN NEW.name = 'extra-1.txt' \ + BEGIN SELECT RAISE(ABORT, 'forced file batch failure'); END", + ) + .await?; + target.close().await?; + + let options = MigrationOptions { + source_url: source_url.clone(), + target_url: target_url.clone(), + default_password: "temporary-password".to_string(), + local_base_path: "C:/cloudreve".to_string(), + direct_link_secret: Some("test-direct-link-secret".to_string()), + include_deleted: false, + allow_non_empty_target: false, + skip_unsupported_policies: false, + dry_run: false, + run_id: Some(run_id.clone()), + resume: false, + blob_batch_size: 500, + file_batch_size: 2, + }; + + let error = migrate(options.clone()).await.unwrap_err(); + assert!(error.to_string().contains("files")); + + let target = Database::connect(&target_url).await?; + assert_eq!(ad::files::Entity::find().count(&target).await?, 2); + assert_eq!(ad::file_versions::Entity::find().count(&target).await?, 1); + let cursor = checkpoint::load_stage_cursor(&target, &run_id, "files") + .await? + .expect("committed file cursor"); + assert_eq!(cursor.cursor_value, extra_file_ids[0]); + assert_eq!(cursor.processed_count, 2); + assert_eq!( + checkpoint::object_map::Entity::find() + .filter(checkpoint::object_map::Column::RunId.eq(&run_id)) + .filter(checkpoint::object_map::Column::ObjectType.eq("file")) + .count(&target) + .await?, + 2 + ); + let failed_checkpoint = checkpoint::Entity::find_by_id(run_id.clone()) + .one(&target) + .await? + .expect("failed file migration checkpoint"); + assert_eq!(failed_checkpoint.status, "failed"); + assert_eq!( + failed_checkpoint.last_completed_stage.as_deref(), + Some("blobs") + ); + let failed_report: MigrationReport = serde_json::from_value(failed_checkpoint.report_json)?; + assert_eq!(failed_report.migrated_files, 2); + assert_eq!(failed_report.migrated_versions, 1); + target + .execute_unprepared("DROP TRIGGER fail_file_batch") + .await?; + target.close().await?; + + let report = migrate(MigrationOptions { + resume: true, + ..options + }) + .await?; + assert!(report.resumed); + assert_eq!(report.migrated_files, 4); + assert_eq!(report.migrated_versions, 1); + assert_eq!(report.mappings.files.len(), 4); + assert!(report.validation.passed); + + let target = Database::connect(&target_url).await?; + assert_eq!(ad::files::Entity::find().count(&target).await?, 4); + assert_eq!(ad::file_versions::Entity::find().count(&target).await?, 1); + assert_eq!( + checkpoint::object_map::Entity::find() + .filter(checkpoint::object_map::Column::RunId.eq(&run_id)) + .filter(checkpoint::object_map::Column::ObjectType.eq("file")) + .count(&target) + .await?, + 4 + ); + let completed_cursor = checkpoint::load_stage_cursor(&target, &run_id, "files") + .await? + .expect("completed file cursor"); + assert_eq!(completed_cursor.cursor_value, extra_file_ids[2]); + assert_eq!(completed_cursor.processed_count, 4); + target.close().await?; + + let _ = std::fs::remove_file(source_path); + let _ = std::fs::remove_file(target_path); + Ok(()) + } + fn sqlite_url(path: &std::path::Path) -> String { format!( "sqlite://{}?mode=rwc", @@ -2361,4 +2686,59 @@ mod tests { } Ok(ids) } + + async fn seed_extra_files(db: &DatabaseConnection, entity_ids: &[i64]) -> Result> { + let now = chrono::Utc::now().fixed_offset(); + let policy_id = cr::storage_policies::Entity::find() + .one(db) + .await? + .expect("seeded storage policy") + .id; + let user_id = cr::users::Entity::find() + .one(db) + .await? + .expect("seeded user") + .id; + let folder_id = cr::files::Entity::find() + .filter(cr::files::Column::Type.eq(1)) + .one(db) + .await? + .expect("seeded folder") + .id; + let mut ids = Vec::with_capacity(entity_ids.len()); + for (index, entity_id) in entity_ids.iter().copied().enumerate() { + let file = cr::files::ActiveModel { + created_at: Set(now), + updated_at: Set(now), + r#type: Set(0), + name: Set(format!("extra-{index}.txt")), + size: Set(256 + index as i64), + primary_entity: Set(Some(entity_id)), + is_symbolic: Set(false), + props: Set(None), + file_children: Set(Some(folder_id)), + storage_policy_files: Set(Some(policy_id)), + owner_id: Set(user_id), + ..Default::default() + } + .insert(db) + .await?; + cr::file_entities::ActiveModel { + file_id: Set(file.id), + entity_id: Set(entity_id), + } + .insert(db) + .await?; + ids.push(file.id); + } + if entity_ids.len() >= 2 { + cr::file_entities::ActiveModel { + file_id: Set(ids[0]), + entity_id: Set(entity_ids[1]), + } + .insert(db) + .await?; + } + Ok(ids) + } } diff --git a/src/migration/phases.rs b/src/migration/phases.rs index e7739c4..0177a55 100644 --- a/src/migration/phases.rs +++ b/src/migration/phases.rs @@ -184,7 +184,7 @@ pub(super) async fn migrate_folders( report: &mut MigrationReport, ) -> Result<()> { let mut pending: Vec<&cr::files::Model> = source - .files + .folders .iter() .filter(|file| file.r#type == 1) .collect(); @@ -197,7 +197,7 @@ pub(super) async fn migrate_folders( let parent = folder.file_children; if parent.is_some_and(|parent_id| { source - .files + .folders .iter() .any(|file| file.id == parent_id && file.r#type == 1) && !context.folders.contains_key(&parent_id) @@ -297,33 +297,16 @@ pub(super) async fn migrate_blob_batch( Ok(mappings) } -pub(super) async fn migrate_files( +pub(super) async fn migrate_file_batch( transaction: &DatabaseTransaction, - source_db: &DatabaseConnection, - source: &SourceData, + files: &[cr::files::Model], + associations: &HashMap>, + entities: &HashMap, blob_mappings: &HashMap, - context: &mut MigrationContext, + context: &MigrationContext, report: &mut MigrationReport, -) -> Result<()> { - let entity_query = if source.include_deleted { - cr::entities::Entity::find() - } else { - cr::entities::Entity::find().filter(cr::entities::Column::DeletedAt.is_null()) - }; - let source_entities = entity_query.all(source_db).await?; - let source_file_entities = cr::file_entities::Entity::find().all(source_db).await?; - let associations = associations(&source.files, &source_file_entities); - let entities: HashMap = source_entities - .iter() - .map(|entity| (entity.id, entity)) - .collect(); - let mut files: Vec<&cr::files::Model> = source - .files - .iter() - .filter(|file| file.r#type == 0) - .collect(); - files.sort_by_key(|file| file.id); - +) -> Result> { + let mut mappings = Vec::with_capacity(files.len()); for file in files { if file.is_symbolic { report.record_skip( @@ -345,7 +328,7 @@ pub(super) async fn migrate_files( .get(&file.id) .into_iter() .flatten() - .filter_map(|entity_id| entities.get(entity_id).copied()) + .filter_map(|entity_id| entities.get(entity_id)) .filter(|entity| entity.r#type == 0 && blob_mappings.contains_key(&entity.id)) .collect(); version_entities.sort_by_key(|entity| entity.created_at); @@ -394,7 +377,7 @@ pub(super) async fn migrate_files( .insert(transaction) .await .wrap_err_with(|| format!("migrate Cloudreve file {}", file.id))?; - context.files.insert(file.id, target.id); + mappings.push((file.id, target.id)); report.migrated_files += 1; let historical: Vec<&cr::entities::Model> = version_entities @@ -416,20 +399,26 @@ pub(super) async fn migrate_files( report.migrated_versions += 1; } } - Ok(()) + Ok(mappings) } pub(super) async fn migrate_metadata( transaction: &DatabaseTransaction, + source_db: &DatabaseConnection, source: &SourceData, + file_mappings: &HashMap, context: &MigrationContext, report: &mut MigrationReport, ) -> Result<()> { - let source_files: HashMap = - source.files.iter().map(|file| (file.id, file)).collect(); + let source_file_ids = source + .metadata + .iter() + .map(|metadata| metadata.file_id) + .collect::>(); + let source_files = load_source_files(source_db, &source_file_ids).await?; let mut tags: HashMap<(i64, String), i64> = HashMap::new(); for metadata in &source.metadata { - let Some(source_file) = source_files.get(&metadata.file_id).copied() else { + let Some(source_file) = source_files.get(&metadata.file_id) else { report.record_skip( "metadata", Some(metadata.id), @@ -438,8 +427,7 @@ pub(super) async fn migrate_metadata( continue; }; let target_entity = if source_file.r#type == 0 { - context - .files + file_mappings .get(&metadata.file_id) .copied() .map(|id| ("file", id)) @@ -548,7 +536,9 @@ pub(super) async fn migrate_metadata( pub(super) async fn migrate_direct_links( transaction: &DatabaseTransaction, + source_db: &DatabaseConnection, source: &SourceData, + file_mappings: &HashMap, context: &MigrationContext, direct_link_secret: Option<&str>, report: &mut MigrationReport, @@ -574,8 +564,12 @@ pub(super) async fn migrate_direct_links( )); return Ok(()); }; - let source_files: HashMap = - source.files.iter().map(|file| (file.id, file)).collect(); + let source_file_ids = source + .direct_links + .iter() + .map(|link| link.file_id) + .collect::>(); + let source_files = load_source_files(source_db, &source_file_ids).await?; for link in &source.direct_links { if link.deleted_at.is_some() { report.record_skip( @@ -585,7 +579,7 @@ pub(super) async fn migrate_direct_links( ); continue; } - let Some(source_file) = source_files.get(&link.file_id).copied() else { + let Some(source_file) = source_files.get(&link.file_id) else { report.record_skip( "direct_link", Some(link.id), @@ -593,7 +587,7 @@ pub(super) async fn migrate_direct_links( ); continue; }; - let Some(file_id) = context.files.get(&link.file_id).copied() else { + let Some(file_id) = file_mappings.get(&link.file_id).copied() else { report.record_skip( "direct_link", Some(link.id), @@ -741,12 +735,18 @@ pub(super) async fn migrate_tasks( pub(super) async fn migrate_shares( transaction: &DatabaseTransaction, + source_db: &DatabaseConnection, source: &SourceData, + file_mappings: &HashMap, context: &mut MigrationContext, report: &mut MigrationReport, ) -> Result<()> { - let source_files: HashMap = - source.files.iter().map(|file| (file.id, file)).collect(); + let source_file_ids = source + .shares + .iter() + .filter_map(|share| share.file_shares) + .collect::>(); + let source_files = load_source_files(source_db, &source_file_ids).await?; for share in &source.shares { let Some(source_user_id) = share.user_shares else { report.record_skip("share", Some(share.id), "share has no owner user"); @@ -764,10 +764,10 @@ pub(super) async fn migrate_shares( report.record_skip("share", Some(share.id), "share has no file/folder target"); continue; }; - let source_file = source_files.get(&source_file_id).copied(); + let source_file = source_files.get(&source_file_id); let file_id = source_file .filter(|file| file.r#type == 0) - .and_then(|_| context.files.get(&source_file_id).copied()); + .and_then(|_| file_mappings.get(&source_file_id).copied()); let folder_id = source_file .filter(|file| file.r#type == 1) .and_then(|_| context.folders.get(&source_file_id).copied()); @@ -811,7 +811,28 @@ pub(super) async fn migrate_shares( Ok(()) } -fn associations( +pub(super) async fn load_source_files( + source_db: &DatabaseConnection, + source_ids: &[i64], +) -> Result> { + const QUERY_ID_BATCH_SIZE: usize = 500; + + let source_ids = source_ids.iter().copied().collect::>(); + let mut files = HashMap::with_capacity(source_ids.len()); + let source_ids = source_ids.into_iter().collect::>(); + for source_ids in source_ids.chunks(QUERY_ID_BATCH_SIZE) { + for file in cr::files::Entity::find() + .filter(cr::files::Column::Id.is_in(source_ids.iter().copied())) + .all(source_db) + .await? + { + files.insert(file.id, file); + } + } + Ok(files) +} + +pub(super) fn associations( files: &[cr::files::Model], file_entities: &[cr::file_entities::Model], ) -> HashMap> { From c1a495bfb9099cf0825d653e82a78b9f7f0bed1f Mon Sep 17 00:00:00 2001 From: Cg8 <5712.cg8@gmail.com> Date: Tue, 14 Jul 2026 10:21:16 +0800 Subject: [PATCH 7/7] feat: verify reused local storage --- README.md | 19 + docs/cloudreve-to-asterdrive-field-mapping.md | 2 + docs/migration-architecture.md | 2 + src/main.rs | 56 +++ src/migration/mod.rs | 391 +++++++++++++++++- src/migration/phases.rs | 2 +- 6 files changed, 469 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index bf5bbb3..b4c8c16 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,25 @@ Use `--dry-run` to perform preflight checks without writing the target. The targ Cloudreve Qiniu, Upyun, remote-node, OneDrive and encrypted storage policies cannot be reused safely by AD. The migration stops when they are present. `--skip-unsupported-policies` explicitly omits those policies and all dependent files. +## Reuse local storage + +For compatible Cloudreve local storage, migration keeps each `entities.source` path unchanged and configures AD to read the same files through the target policy base path. This avoids copying object bytes and therefore does not require a second full data volume. + +`--local-base-path` is the fallback root for every local policy. When policies use different volumes, pass one or more explicit source-policy mappings: + +```powershell +cargo run -- migrate ` + --source-url "sqlite://C:/cloudreve/cloudreve.db?mode=ro" ` + --target-url "sqlite://C:/asterdrive/asterdrive.db" ` + --default-password "change-this-password" ` + --local-base-path "D:/cloudreve-default" ` + --local-policy-root "1=D:/cloudreve-default" ` + --local-policy-root "2=E:/cloudreve-archive" ` + --verify-local-storage +``` + +`--verify-local-storage` checks each compatible local policy root and every migrated local blob's resolved path, regular-file status, open permission and byte length. With `--dry-run`, it scans all eligible local blobs before any target writes. It does not verify S3-compatible buckets or prove that a running AD instance can access a Docker mount; validate those separately before cutover. + ## Limited resume Migration stages run in this order: policies, policy groups, users, folders, blobs, files, metadata, shares, direct links and tasks. Most stages are committed as one transaction. Blobs use keyset pages ordered by Cloudreve `entities.id`, while files use keyset pages ordered by Cloudreve `files.id`; each page writes target records, object mappings, its cursor and report progress in one target transaction. diff --git a/docs/cloudreve-to-asterdrive-field-mapping.md b/docs/cloudreve-to-asterdrive-field-mapping.md index 2c4db7b..a8c0a50 100644 --- a/docs/cloudreve-to-asterdrive-field-mapping.md +++ b/docs/cloudreve-to-asterdrive-field-mapping.md @@ -490,6 +490,8 @@ Cloudreve 任务不能恢复到 AD 执行器中，但可以保存为不可执行 | 邮箱验证 | 视为已验证 / 全部重新验证 | 活跃用户按已验证处理 | | Cloudreve group | AD 策略组 / AD team / 两者都建 | 映射为存储策略组，不自动创建 team | | 本地存储根目录 | 复用 Cloudreve 工作目录 / 搬迁文件到 AD 新目录 | 通过 `--local-base-path` 指定并复用旧对象路径 | +| 多本地策略根目录 | 对所有策略使用同一根目录 / 按 source policy 分别指定 | `--local-policy-root =` 覆盖指定策略，未指定策略回退 `--local-base-path` | +| 本地对象预检 | 仅检查根目录 / 逐对象路径、权限和大小检查 | `--verify-local-storage`；配合 `--dry-run` 可在写入前分页扫描全部可复用 local blob | | 对象存储 | 复用原 bucket / 复制到新 bucket | 复用原 endpoint、bucket 和 object key | | 历史版本 | 全部迁移 / 只迁移当前版本 | 当前实现迁移全部 `type=0` entity | | 分享 URL | 生成新 token / 尝试兼容旧 HashID | 生成新 token，旧 URL 失效 | diff --git a/docs/migration-architecture.md b/docs/migration-architecture.md index 2586959..dfd8b06 100644 --- a/docs/migration-architecture.md +++ b/docs/migration-architecture.md @@ -555,6 +555,8 @@ checkpoint 表保存在 AD 数据库中，以便迁移进程重启后恢复。 当前 source fingerprint 使用数据库 URL 摘要和各源表记录数量，只能防止明显连错源或数量变化，不能检测“数量不变但内容变化”。生产迁移仍应冻结 Cloudreve 写入。 +对于 `reuse_source_storage` 的 local policy，当前实现支持用全局 `local_base_path` 或按 Cloudreve source policy ID 覆盖的 local root 配置 AD `base_path`。启用 `verify_local_storage` 后，预检会检查 local root；正式 blob batch 会检查每个复用对象解析后的路径、常规文件状态、打开权限和大小。`dry-run` 会在写入前分页扫描所有兼容 local blob。S3/COS 等远程复用策略仍需要后续的 provider-level `HeadObject`/range-read 验证。 + #### `aster_external_migration_stage_cursors` 已用于 blobs 和 files；其他 stage 尚未接入。 diff --git a/src/main.rs b/src/main.rs index 2e23ba6..1add820 100644 --- a/src/main.rs +++ b/src/main.rs @@ -39,6 +39,10 @@ struct MigrateArgs { default_password: String, #[arg(long, default_value = ".")] local_base_path: String, + #[arg(long = "local-policy-root", value_name = "SOURCE_POLICY_ID=PATH")] + local_policy_roots: Vec, + #[arg(long)] + verify_local_storage: bool, #[arg(long, env = "ASTER_DIRECT_LINK_SECRET", hide_env_values = true)] direct_link_secret: Option, #[arg(long)] @@ -72,6 +76,8 @@ async fn main() -> Result<()> { target_url: args.connection.target_url, default_password: args.default_password, local_base_path: args.local_base_path, + local_policy_roots: parse_local_policy_roots(args.local_policy_roots)?, + verify_local_storage: args.verify_local_storage, direct_link_secret: args.direct_link_secret, include_deleted: args.connection.include_deleted, allow_non_empty_target: args.allow_non_empty_target, @@ -89,6 +95,27 @@ async fn main() -> Result<()> { Ok(()) } +fn parse_local_policy_roots( + values: Vec, +) -> Result> { + let mut roots = std::collections::BTreeMap::new(); + for value in values { + let Some((source_policy_id, path)) = value.split_once('=') else { + bail!("--local-policy-root must use SOURCE_POLICY_ID=PATH"); + }; + let source_policy_id = source_policy_id.parse::().map_err(|_| { + color_eyre::eyre::eyre!("invalid local storage policy ID {source_policy_id}") + })?; + if source_policy_id <= 0 || path.trim().is_empty() { + bail!("--local-policy-root must use a positive policy ID and non-empty path"); + } + if roots.insert(source_policy_id, path.to_string()).is_some() { + bail!("local storage policy {source_policy_id} was configured more than once"); + } + } + Ok(roots) +} + fn emit_report(report_path: Option<&Path>, report: &MigrationReport) -> Result<()> { if let Some(path) = report_path { write_json_report(path, report)?; @@ -99,3 +126,32 @@ fn emit_report(report_path: Option<&Path>, report: &MigrationReport) -> Result<( } Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_local_policy_roots() -> Result<()> { + let roots = parse_local_policy_roots(vec![ + "1=D:/cloudreve-data".to_string(), + "2=E:/cloudreve-archive".to_string(), + ])?; + assert_eq!(roots.get(&1).map(String::as_str), Some("D:/cloudreve-data")); + assert_eq!( + roots.get(&2).map(String::as_str), + Some("E:/cloudreve-archive") + ); + Ok(()) + } + + #[test] + fn rejects_invalid_local_policy_roots() { + assert!(parse_local_policy_roots(vec!["missing-separator".to_string()]).is_err()); + assert!(parse_local_policy_roots(vec!["0=C:/data".to_string()]).is_err()); + assert!( + parse_local_policy_roots(vec!["1=C:/data".to_string(), "1=D:/data".to_string()]) + .is_err() + ); + } +} diff --git a/src/migration/mod.rs b/src/migration/mod.rs index 51460e5..89d2ea2 100644 --- a/src/migration/mod.rs +++ b/src/migration/mod.rs @@ -24,6 +24,8 @@ pub struct MigrationOptions { pub target_url: String, pub default_password: String, pub local_base_path: String, + pub local_policy_roots: BTreeMap, + pub verify_local_storage: bool, pub direct_link_secret: Option, pub include_deleted: bool, pub allow_non_empty_target: bool, @@ -105,7 +107,7 @@ impl Default for MigrationReport { migrated_direct_links: 0, migrated_tasks: 0, skipped: 0, - dry_run: false, + dry_run: true, warnings: Vec::new(), skipped_by_type: BTreeMap::new(), skipped_objects: Vec::new(), @@ -444,6 +446,13 @@ pub async fn migrate(options: MigrationOptions) -> Result { unsupported.join(", ") ); } + validate_local_policy_roots(&source_data, &options)?; + if options.verify_local_storage { + verify_local_storage_roots(&source_data, &options)?; + if options.dry_run { + verify_all_local_source_objects(&source, &source_data, &options).await?; + } + } let mut report = source_data.report(); report.dry_run = options.dry_run; @@ -812,6 +821,15 @@ async fn migrate_blobs_batched( return Ok(()); } + if inputs.options.verify_local_storage { + verify_local_blob_batch( + &entities, + inputs.source_data, + inputs.options, + inputs.context, + )?; + } + let entity_ids = entities.iter().map(|entity| entity.id).collect::>(); let association_info = load_blob_association_info(inputs.source, &entity_ids).await?; let last_entity_id = entities.last().expect("non-empty blob batch").id; @@ -1097,8 +1115,15 @@ fn source_fingerprint(source_url: &str, source: &SourceData) -> String { fn plan_fingerprint(options: &MigrationOptions) -> String { hash_fingerprint(&format!( - "{}|{}|{}|{}|{}|{}", + "{}|{}|{}|{}|{}|{}|{}|{}", options.local_base_path, + options + .local_policy_roots + .iter() + .map(|(policy_id, path)| format!("{policy_id}={path}")) + .collect::>() + .join("|"), + options.verify_local_storage, options.include_deleted, options.allow_non_empty_target, options.skip_unsupported_policies, @@ -1111,6 +1136,190 @@ fn plan_fingerprint(options: &MigrationOptions) -> String { )) } +fn local_policy_root(options: &MigrationOptions, source_policy_id: i64) -> &str { + options + .local_policy_roots + .get(&source_policy_id) + .map(String::as_str) + .unwrap_or(&options.local_base_path) +} + +fn local_storage_path(root: &str, storage_path: &str) -> std::path::PathBuf { + let storage_path = std::path::Path::new(storage_path); + if storage_path.is_absolute() { + storage_path.to_path_buf() + } else { + std::path::Path::new(root).join(storage_path) + } +} + +fn validate_local_policy_roots(source: &SourceData, options: &MigrationOptions) -> Result<()> { + let policies = source + .policies + .iter() + .map(|policy| (policy.id, policy)) + .collect::>(); + for source_policy_id in options.local_policy_roots.keys() { + let Some(policy) = policies.get(source_policy_id) else { + bail!( + "--local-policy-root references Cloudreve policy {source_policy_id}, which was not found" + ); + }; + if policy.r#type != "local" { + bail!( + "--local-policy-root references Cloudreve policy {source_policy_id}, which is not a local policy" + ); + } + } + Ok(()) +} + +fn verify_local_storage_roots(source: &SourceData, options: &MigrationOptions) -> Result<()> { + for policy in source.policies.iter().filter(|policy| { + policy.r#type == "local" + && map_driver_type(&policy.r#type).is_some() + && !source_settings(&policy.settings) + .get("encryption") + .and_then(Value::as_bool) + .unwrap_or(false) + }) { + let root = local_policy_root(options, policy.id); + let metadata = std::fs::metadata(root).wrap_err_with(|| { + format!( + "read local storage root for Cloudreve policy {}: {root}", + policy.id + ) + })?; + if !metadata.is_dir() { + bail!( + "local storage root for Cloudreve policy {} is not a directory: {root}", + policy.id + ); + } + } + Ok(()) +} + +async fn verify_all_local_source_objects( + source_db: &DatabaseConnection, + source: &SourceData, + options: &MigrationOptions, +) -> Result<()> { + const VERIFY_BATCH_SIZE: u64 = 500; + + let policies = source + .policies + .iter() + .map(|policy| (policy.id, policy)) + .collect::>(); + let mut cursor_value = 0; + loop { + let mut query = cr::entities::Entity::find() + .filter(cr::entities::Column::Type.eq(0)) + .filter(cr::entities::Column::Id.gt(cursor_value)) + .order_by_asc(cr::entities::Column::Id) + .limit(VERIFY_BATCH_SIZE); + if !source.include_deleted { + query = query.filter(cr::entities::Column::DeletedAt.is_null()); + } + let entities = query.all(source_db).await?; + let Some(last_entity) = entities.last() else { + return Ok(()); + }; + for entity in &entities { + let Some(policy) = policies.get(&entity.storage_policy_entities) else { + continue; + }; + if policy.r#type == "local" + && map_driver_type(&policy.r#type).is_some() + && !source_settings(&policy.settings) + .get("encryption") + .and_then(Value::as_bool) + .unwrap_or(false) + { + verify_local_entity(entity, policy, options)?; + } + } + cursor_value = last_entity.id; + } +} + +fn verify_local_blob_batch( + entities: &[cr::entities::Model], + source: &SourceData, + options: &MigrationOptions, + context: &MigrationContext, +) -> Result<()> { + let policies = source + .policies + .iter() + .map(|policy| (policy.id, policy)) + .collect::>(); + for entity in entities { + if !context + .policies + .contains_key(&entity.storage_policy_entities) + { + continue; + } + let Some(policy) = policies.get(&entity.storage_policy_entities) else { + continue; + }; + if policy.r#type != "local" { + continue; + } + verify_local_entity(entity, policy, options)?; + } + Ok(()) +} + +fn verify_local_entity( + entity: &cr::entities::Model, + policy: &cr::storage_policies::Model, + options: &MigrationOptions, +) -> Result<()> { + let root = local_policy_root(options, policy.id); + let path = local_storage_path(root, &entity.source); + let metadata = std::fs::metadata(&path).wrap_err_with(|| { + format!( + "read Cloudreve local entity {} at {}", + entity.id, + path.display() + ) + })?; + if !metadata.is_file() { + bail!( + "Cloudreve local entity {} is not a regular file: {}", + entity.id, + path.display() + ); + } + std::fs::File::open(&path).wrap_err_with(|| { + format!( + "open Cloudreve local entity {} at {}", + entity.id, + path.display() + ) + })?; + let expected_size = u64::try_from(entity.size).map_err(|_| { + color_eyre::eyre::eyre!( + "Cloudreve local entity {} has negative size {}", + entity.id, + entity.size + ) + })?; + if metadata.len() != expected_size { + bail!( + "Cloudreve local entity {} size mismatch at {}: database={}, filesystem={}", + entity.id, + path.display(), + expected_size, + metadata.len() + ); + } + Ok(()) +} + async fn connect(url: &str, label: &str) -> Result { Database::connect(url) .await @@ -2026,6 +2235,8 @@ mod tests { target_url: target_url.clone(), default_password: "temporary-password".to_string(), local_base_path: "C:/cloudreve".to_string(), + local_policy_roots: std::collections::BTreeMap::new(), + verify_local_storage: false, direct_link_secret: Some("test-direct-link-secret".to_string()), include_deleted: false, allow_non_empty_target: false, @@ -2109,6 +2320,176 @@ mod tests { Ok(()) } + #[tokio::test] + async fn reuses_and_verifies_local_storage_per_policy_root() -> Result<()> { + let suffix = uuid::Uuid::new_v4(); + let source_path = std::env::temp_dir().join(format!("cloudreve-local-reuse-{suffix}.db")); + let target_path = std::env::temp_dir().join(format!("asterdrive-local-reuse-{suffix}.db")); + let storage_root = std::env::temp_dir().join(format!("cloudreve-local-storage-{suffix}")); + let source_url = sqlite_url(&source_path); + let target_url = sqlite_url(&target_path); + + let source = Database::connect(&source_url).await?; + create_source_schema(&source).await?; + seed_source(&source).await?; + let source_policy_id = cr::storage_policies::Entity::find() + .one(&source) + .await? + .expect("seeded storage policy") + .id; + source.close().await?; + std::fs::create_dir_all(storage_root.join("uploads"))?; + std::fs::write(storage_root.join("uploads/object.bin"), vec![0_u8; 128])?; + + let target = Database::connect(&target_url).await?; + create_target_schema(&target).await?; + target.close().await?; + + let report = migrate(MigrationOptions { + source_url: source_url.clone(), + target_url: target_url.clone(), + default_password: "temporary-password".to_string(), + local_base_path: "unused-local-root".to_string(), + local_policy_roots: std::collections::BTreeMap::from([( + source_policy_id, + storage_root.to_string_lossy().to_string(), + )]), + verify_local_storage: true, + direct_link_secret: Some("test-direct-link-secret".to_string()), + include_deleted: false, + allow_non_empty_target: false, + skip_unsupported_policies: false, + dry_run: false, + run_id: None, + resume: false, + blob_batch_size: 500, + file_batch_size: 500, + }) + .await?; + assert!(report.validation.passed); + + let target = Database::connect(&target_url).await?; + let policy = ad::storage_policies::Entity::find() + .one(&target) + .await? + .expect("migrated storage policy"); + assert_eq!(policy.base_path, storage_root.to_string_lossy()); + target.close().await?; + + let _ = std::fs::remove_file(source_path); + let _ = std::fs::remove_file(target_path); + let _ = std::fs::remove_dir_all(storage_root); + Ok(()) + } + + #[tokio::test] + async fn rejects_local_storage_size_mismatch() -> Result<()> { + let suffix = uuid::Uuid::new_v4(); + let source_path = + std::env::temp_dir().join(format!("cloudreve-local-mismatch-{suffix}.db")); + let target_path = + std::env::temp_dir().join(format!("asterdrive-local-mismatch-{suffix}.db")); + let storage_root = + std::env::temp_dir().join(format!("cloudreve-local-mismatch-storage-{suffix}")); + let source_url = sqlite_url(&source_path); + let target_url = sqlite_url(&target_path); + + let source = Database::connect(&source_url).await?; + create_source_schema(&source).await?; + seed_source(&source).await?; + let source_policy_id = cr::storage_policies::Entity::find() + .one(&source) + .await? + .expect("seeded storage policy") + .id; + source.close().await?; + std::fs::create_dir_all(storage_root.join("uploads"))?; + std::fs::write(storage_root.join("uploads/object.bin"), vec![0_u8; 127])?; + + let target = Database::connect(&target_url).await?; + create_target_schema(&target).await?; + target.close().await?; + + let error = migrate(MigrationOptions { + source_url: source_url.clone(), + target_url: target_url.clone(), + default_password: "temporary-password".to_string(), + local_base_path: "unused-local-root".to_string(), + local_policy_roots: std::collections::BTreeMap::from([( + source_policy_id, + storage_root.to_string_lossy().to_string(), + )]), + verify_local_storage: true, + direct_link_secret: Some("test-direct-link-secret".to_string()), + include_deleted: false, + allow_non_empty_target: false, + skip_unsupported_policies: false, + dry_run: true, + run_id: None, + resume: false, + blob_batch_size: 500, + file_batch_size: 500, + }) + .await + .unwrap_err(); + assert!(format!("{error:?}").contains("size mismatch")); + + let target = Database::connect(&target_url).await?; + assert_eq!( + ad::storage_policies::Entity::find().count(&target).await?, + 0 + ); + target.close().await?; + + let _ = std::fs::remove_file(source_path); + let _ = std::fs::remove_file(target_path); + let _ = std::fs::remove_dir_all(storage_root); + Ok(()) + } + + #[tokio::test] + async fn rejects_unknown_local_policy_root() -> Result<()> { + let suffix = uuid::Uuid::new_v4(); + let source_path = std::env::temp_dir().join(format!("cloudreve-local-policy-{suffix}.db")); + let target_path = std::env::temp_dir().join(format!("asterdrive-local-policy-{suffix}.db")); + let source_url = sqlite_url(&source_path); + let target_url = sqlite_url(&target_path); + + let source = Database::connect(&source_url).await?; + create_source_schema(&source).await?; + seed_source(&source).await?; + source.close().await?; + + let target = Database::connect(&target_url).await?; + create_target_schema(&target).await?; + target.close().await?; + + let error = migrate(MigrationOptions { + source_url: source_url.clone(), + target_url: target_url.clone(), + default_password: "temporary-password".to_string(), + local_base_path: "unused-local-root".to_string(), + local_policy_roots: std::collections::BTreeMap::from([(999, "C:/missing".to_string())]), + verify_local_storage: false, + direct_link_secret: Some("test-direct-link-secret".to_string()), + include_deleted: false, + allow_non_empty_target: false, + skip_unsupported_policies: false, + dry_run: true, + run_id: None, + resume: false, + blob_batch_size: 500, + file_batch_size: 500, + }) + .await + .unwrap_err(); + assert!(error.to_string().contains("policy 999")); + + let _ = std::fs::remove_file(source_path); + let _ = std::fs::remove_file(target_path); + Ok(()) + } + #[tokio::test] async fn resumes_from_last_completed_stage() -> Result<()> { let suffix = uuid::Uuid::new_v4(); @@ -2138,6 +2519,8 @@ mod tests { target_url: target_url.clone(), default_password: "temporary-password".to_string(), local_base_path: "C:/cloudreve".to_string(), + local_policy_roots: std::collections::BTreeMap::new(), + verify_local_storage: false, direct_link_secret: Some("test-direct-link-secret".to_string()), include_deleted: false, allow_non_empty_target: false, @@ -2237,6 +2620,8 @@ mod tests { target_url: target_url.clone(), default_password: "temporary-password".to_string(), local_base_path: "C:/cloudreve".to_string(), + local_policy_roots: std::collections::BTreeMap::new(), + verify_local_storage: false, direct_link_secret: Some("test-direct-link-secret".to_string()), include_deleted: false, allow_non_empty_target: false, @@ -2346,6 +2731,8 @@ mod tests { target_url: target_url.clone(), default_password: "temporary-password".to_string(), local_base_path: "C:/cloudreve".to_string(), + local_policy_roots: std::collections::BTreeMap::new(), + verify_local_storage: false, direct_link_secret: Some("test-direct-link-secret".to_string()), include_deleted: false, allow_non_empty_target: false, diff --git a/src/migration/phases.rs b/src/migration/phases.rs index 0177a55..2abbbe1 100644 --- a/src/migration/phases.rs +++ b/src/migration/phases.rs @@ -25,7 +25,7 @@ pub(super) async fn migrate_policies( continue; }; let base_path = if policy.r#type == "local" { - options.local_base_path.clone() + local_policy_root(options, policy.id).to_string() } else { String::new() };