diff --git a/Cargo.lock b/Cargo.lock index 75673ba..e02e561 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -319,13 +319,22 @@ name = "aster_drive_migration" version = "0.0.0-alpha.0" dependencies = [ "argon2", + "asterdrive-entities", + "base64", "chrono", "clap", + "cloudreve-entities", "color-eyre", "crossterm 0.29.0", + "hmac 0.12.1", + "mime_guess", "ratatui", "sea-orm", "serde", + "serde_json", + "sha2 0.10.9", + "tokio", + "urlencoding", "uuid", ] @@ -861,7 +870,7 @@ dependencies = [ "document-features", "mio", "parking_lot", - "rustix 1.1.4", + "rustix 1.1.3", "serde", "signal-hook", "signal-hook-mio", @@ -1421,7 +1430,16 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4aaa26c720c68b866f2c96ef5c1264b3e6f473fe5d4ce61cd44bbe913e553018" dependencies = [ - "hmac", + "hmac 0.13.0", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", ] [[package]] @@ -1765,9 +1783,9 @@ checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] name = "linux-raw-sys" -version = "0.12.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" [[package]] name = "litemap" @@ -1847,6 +1865,22 @@ dependencies = [ "autocfg", ] +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -2650,14 +2684,14 @@ dependencies = [ [[package]] name = "rustix" -version = "1.1.4" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" dependencies = [ "bitflags 2.13.0", "errno", "libc", - "linux-raw-sys 0.12.1", + "linux-raw-sys 0.11.0", "windows-sys 0.61.2", ] @@ -3173,7 +3207,7 @@ dependencies = [ "futures-util", "hex", "hkdf", - "hmac", + "hmac 0.13.0", "itoa", "log", "md-5", @@ -3323,7 +3357,7 @@ checksum = "9048a889effe34a5cddee0af7f53285198b16dca3be510858d38dfdb3e62a04e" dependencies = [ "bitflags 2.13.0", "parking_lot", - "rustix 1.1.4", + "rustix 1.1.3", "signal-hook", "windows-sys 0.61.2", ] @@ -3529,9 +3563,21 @@ dependencies = [ "mio", "pin-project-lite", "socket2", + "tokio-macros", "windows-sys 0.61.2", ] +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "tokio-stream" version = "0.1.18" @@ -3639,6 +3685,12 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + [[package]] name = "unicode-bidi" version = "0.3.18" @@ -3713,6 +3765,12 @@ dependencies = [ "serde", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf8_iter" version = "1.0.4" diff --git a/Cargo.toml b/Cargo.toml index e428fe6..5d64e98 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,19 +35,28 @@ members = [ [features] default = ["cli", "tui"] -cli = ["dep:clap"] +cli = [] tui = ["cli", "dep:ratatui"] full = ["cli", "tui"] [dependencies] +asterdrive-entities = { path = "crates/asterdrive-entities" } argon2 = "0.5.3" +base64 = "0.22" chrono = { version = "0.4.45", default-features = false, features = ["clock", "serde"] } -clap = { version = "4", features = ["derive", "env"], optional = true } +clap = { version = "4", features = ["derive", "env"] } +cloudreve-entities = { path = "crates/cloudreve-entities" } color-eyre = "0.6.5" crossterm = "0.29.0" +hmac = "0.12" +mime_guess = "2" ratatui = { version = "0.30.2", optional = true, features = ["serde"] } sea-orm = { version = "2.0.0-rc.42", features = ["sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", "macros", "runtime-tokio-rustls"] } serde = { version = "1.0.228", features = ["derive"] } +serde_json = "1" +sha2 = "0.10" +tokio = { version = "1", features = ["macros", "rt-multi-thread"] } +urlencoding = "2" uuid = { version = "1", features = ["v4"] } [lib] diff --git a/README.md b/README.md index 0f04cbc..b4c8c16 100644 --- a/README.md +++ b/README.md @@ -1 +1,129 @@ # AsterDriveMigration + +Design documents: + +- [Cloudreve and AsterDrive field mapping](docs/cloudreve-to-asterdrive-field-mapping.md) +- [Migration architecture](docs/migration-architecture.md) + +Database migration tool for moving a Cloudreve v4 installation to AsterDrive (AD). + +## Supported data + +- Cloudreve groups to AD storage policy groups +- users and profiles, including quotas and disabled status +- local, S3, OSS, KS3, OBS and Tencent COS storage policies +- folders, files, physical entities/blobs and version history +- file/folder metadata, including Cloudreve v4 `tag:*` metadata as AD tags +- public shares and regenerated AD v2 direct-link mappings +- Cloudreve background tasks as non-executable terminal history + +Cloudreve password hashes are not compatible with AD. The migration assigns the supplied temporary password to every migrated user and sets `must_change_password = true`. Passkeys, WebDAV credentials, two-factor secrets, OAuth grants, sessions and filesystem events are not migrated. + +Cloudreve tasks are preserved only as terminal AD `system_runtime` history. Completed/error/canceled statuses are mapped to terminal AD statuses; queued/processing/suspending tasks are archived as canceled and are never resumed. + +The tool reuses existing storage objects. For local storage, pass the directory from which Cloudreve entity paths should be resolved. Object-storage policies keep their existing bucket, endpoint, credentials and object keys. + +## Prerequisites + +1. Stop writes to Cloudreve and back up both databases. +2. Create an empty AD database and run the AD database migrations against it. +3. Do not start the AD application against the target database until this migration completes. + +## Check compatibility + +```powershell +cargo run -- check ` + --source-url "sqlite://C:/cloudreve/cloudreve.db?mode=ro" ` + --target-url "sqlite://C:/asterdrive/asterdrive.db" ` + --report-path "C:/migration/cloudreve-preflight.json" +``` + +The URLs can also be provided through `CLOUDREVE_DATABASE_URL` and `ASTERDRIVE_DATABASE_URL`. SQLite, MySQL and PostgreSQL URLs supported by SeaORM can be used. + +## Run migration + +```powershell +$env:ASTER_MIGRATION_DEFAULT_PASSWORD = "replace-with-a-strong-temporary-password" +$env:ASTER_DIRECT_LINK_SECRET = "use-the-same-auth.direct_link_secret-as-ad" + +cargo run -- migrate ` + --source-url "sqlite://C:/cloudreve/cloudreve.db?mode=ro" ` + --target-url "sqlite://C:/asterdrive/asterdrive.db" ` + --local-base-path "C:/cloudreve" ` + --run-id "cloudreve-cutover-2026-07-13" ` + --report-path "C:/migration/cloudreve-to-ad.json" +``` + +`ASTER_DIRECT_LINK_SECRET` is optional only when there are no direct links to regenerate. When supplied, each active Cloudreve `direct_links` row is mapped to a newly signed AD `/d/v2...` URL and stored under the target file's `cloudreve.direct_links` properties. Existing Cloudreve `/f/...` URLs cannot remain valid because the URL path, ID and signing algorithms differ. + +Use `--dry-run` to perform preflight checks without writing the target. The target core tables must be empty by default. `--allow-non-empty-target` disables that guard but may still fail on unique values or conflicting data. + +Cloudreve Qiniu, Upyun, remote-node, OneDrive and encrypted storage policies cannot be reused safely by AD. The migration stops when they are present. `--skip-unsupported-policies` explicitly omits those policies and all dependent files. + +## Reuse local storage + +For compatible Cloudreve local storage, migration keeps each `entities.source` path unchanged and configures AD to read the same files through the target policy base path. This avoids copying object bytes and therefore does not require a second full data volume. + +`--local-base-path` is the fallback root for every local policy. When policies use different volumes, pass one or more explicit source-policy mappings: + +```powershell +cargo run -- migrate ` + --source-url "sqlite://C:/cloudreve/cloudreve.db?mode=ro" ` + --target-url "sqlite://C:/asterdrive/asterdrive.db" ` + --default-password "change-this-password" ` + --local-base-path "D:/cloudreve-default" ` + --local-policy-root "1=D:/cloudreve-default" ` + --local-policy-root "2=E:/cloudreve-archive" ` + --verify-local-storage +``` + +`--verify-local-storage` checks each compatible local policy root and every migrated local blob's resolved path, regular-file status, open permission and byte length. With `--dry-run`, it scans all eligible local blobs before any target writes. It does not verify S3-compatible buckets or prove that a running AD instance can access a Docker mount; validate those separately before cutover. + +## Limited resume + +Migration stages run in this order: policies, policy groups, users, folders, blobs, files, metadata, shares, direct links and tasks. Most stages are committed as one transaction. Blobs use keyset pages ordered by Cloudreve `entities.id`, while files use keyset pages ordered by Cloudreve `files.id`; each page writes target records, object mappings, its cursor and report progress in one target transaction. + +Use a stable run ID for an operational migration. If a stage fails, fix the external cause and rerun the same command with `--resume`: + +```powershell +cargo run -- migrate ` + --source-url "sqlite://C:/cloudreve/cloudreve.db?mode=ro" ` + --target-url "sqlite://C:/asterdrive/asterdrive.db" ` + --local-base-path "C:/cloudreve" ` + --run-id "cloudreve-cutover-2026-07-13" ` + --blob-batch-size 500 ` + --file-batch-size 500 ` + --resume ` + --report-path "C:/migration/cloudreve-to-ad.json" +``` + +Resume verifies the source URL/count fingerprint, target URL fingerprint and migration-plan fingerprint. Repeat the same password, direct-link secret and migration flags used by the original run. Completed stages are skipped and their source-to-target mappings are restored from the checkpoint. Blob batch size is operational only and may be adjusted when resuming. + +`--blob-batch-size` and `--file-batch-size` default to 500 and accept 1-10000. A failed blob or file page rolls back only that page; resume continues after the last committed entity or file ID. Blob and normal-file source rows are no longer loaded into memory as complete collections, and their mappings are stored row-by-row in `aster_external_migration_object_map`. + +Folders, metadata, shares, direct links and tasks are still stage-level only: a failure restarts that entire stage. A file page loads only its related entity/version rows into memory. Physical object bytes are still reused in place rather than copied or uploaded. + +## JSON migration report + +`--report-path` writes a pretty-printed JSON report for both `check` and `migrate`. A completed migration report contains: + +- source and migrated counts for every supported object type +- skipped counts grouped by type, plus source ID and reason for every skipped object +- sorted source-to-target ID mappings for policies, groups, users, folders, blobs, files, shares and archived tasks +- every Cloudreve tag metadata assignment and its AD tag/entity IDs +- every regenerated direct-link URL and its Cloudreve/AD file IDs +- post-commit database count checks, imported-task terminal-state checks, tag-binding checks and direct-link property checks +- run ID, whether the execution resumed, and the list of completed stages + +The CLI writes the JSON report before returning a validation error. A failed post-migration check therefore produces a usable report and exits with a non-zero status. Direct-link URLs are bearer-style public capabilities, so the report file must be stored with restricted access. + +## Verification before push + +Every migration feature must include focused tests plus end-to-end coverage where it writes database state. Before pushing, run and require all of these commands to pass: + +```powershell +cargo fmt -- --check +cargo check --offline +cargo clippy --offline --all-targets --all-features -- -D warnings +cargo test --offline --all-targets --all-features +``` diff --git a/crates/cloudreve-entities/src/dav_accounts.rs b/crates/cloudreve-entities/src/dav_accounts.rs index 53b248c..a9290da 100644 --- a/crates/cloudreve-entities/src/dav_accounts.rs +++ b/crates/cloudreve-entities/src/dav_accounts.rs @@ -4,7 +4,7 @@ use sea_orm::entity::prelude::*; #[sea_orm::model] #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] -#[sea_orm(schema_name = "public", table_name = "dav_accounts")] +#[sea_orm(table_name = "dav_accounts")] pub struct Model { #[sea_orm(primary_key)] pub id: i64, diff --git a/crates/cloudreve-entities/src/direct_links.rs b/crates/cloudreve-entities/src/direct_links.rs index 4b8c9be..a083a3c 100644 --- a/crates/cloudreve-entities/src/direct_links.rs +++ b/crates/cloudreve-entities/src/direct_links.rs @@ -4,7 +4,7 @@ use sea_orm::entity::prelude::*; #[sea_orm::model] #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] -#[sea_orm(schema_name = "public", table_name = "direct_links")] +#[sea_orm(table_name = "direct_links")] pub struct Model { #[sea_orm(primary_key)] pub id: i64, diff --git a/crates/cloudreve-entities/src/entities.rs b/crates/cloudreve-entities/src/entities.rs index 4ff3052..d1044c4 100644 --- a/crates/cloudreve-entities/src/entities.rs +++ b/crates/cloudreve-entities/src/entities.rs @@ -4,7 +4,7 @@ use sea_orm::entity::prelude::*; #[sea_orm::model] #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] -#[sea_orm(schema_name = "public", table_name = "entities")] +#[sea_orm(table_name = "entities")] pub struct Model { #[sea_orm(primary_key)] pub id: i64, diff --git a/crates/cloudreve-entities/src/file_entities.rs b/crates/cloudreve-entities/src/file_entities.rs index accde17..62b47ce 100644 --- a/crates/cloudreve-entities/src/file_entities.rs +++ b/crates/cloudreve-entities/src/file_entities.rs @@ -4,7 +4,7 @@ use sea_orm::entity::prelude::*; #[sea_orm::model] #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] -#[sea_orm(schema_name = "public", table_name = "file_entities")] +#[sea_orm(table_name = "file_entities")] pub struct Model { #[sea_orm(primary_key, auto_increment = false)] pub file_id: i64, diff --git a/crates/cloudreve-entities/src/files.rs b/crates/cloudreve-entities/src/files.rs index f4503a8..dca9b02 100644 --- a/crates/cloudreve-entities/src/files.rs +++ b/crates/cloudreve-entities/src/files.rs @@ -4,13 +4,12 @@ use sea_orm::entity::prelude::*; #[sea_orm::model] #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] -#[sea_orm(schema_name = "public", table_name = "files")] +#[sea_orm(table_name = "files")] pub struct Model { #[sea_orm(primary_key)] pub id: i64, pub created_at: DateTimeWithTimeZone, pub updated_at: DateTimeWithTimeZone, - pub deleted_at: Option, pub r#type: i64, #[sea_orm(unique_key = "file_file_children_name")] pub name: String, diff --git a/crates/cloudreve-entities/src/fs_events.rs b/crates/cloudreve-entities/src/fs_events.rs index 216db30..a71fbe7 100644 --- a/crates/cloudreve-entities/src/fs_events.rs +++ b/crates/cloudreve-entities/src/fs_events.rs @@ -4,7 +4,7 @@ use sea_orm::entity::prelude::*; #[sea_orm::model] #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] -#[sea_orm(schema_name = "public", table_name = "fs_events")] +#[sea_orm(table_name = "fs_events")] pub struct Model { #[sea_orm(primary_key)] pub id: i64, diff --git a/crates/cloudreve-entities/src/groups.rs b/crates/cloudreve-entities/src/groups.rs index 961c75a..29f0ec1 100644 --- a/crates/cloudreve-entities/src/groups.rs +++ b/crates/cloudreve-entities/src/groups.rs @@ -4,7 +4,7 @@ use sea_orm::entity::prelude::*; #[sea_orm::model] #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] -#[sea_orm(schema_name = "public", table_name = "groups")] +#[sea_orm(table_name = "groups")] pub struct Model { #[sea_orm(primary_key)] pub id: i64, diff --git a/crates/cloudreve-entities/src/metadata.rs b/crates/cloudreve-entities/src/metadata.rs index 085c420..6784f7b 100644 --- a/crates/cloudreve-entities/src/metadata.rs +++ b/crates/cloudreve-entities/src/metadata.rs @@ -4,7 +4,7 @@ use sea_orm::entity::prelude::*; #[sea_orm::model] #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] -#[sea_orm(schema_name = "public", table_name = "metadata")] +#[sea_orm(table_name = "metadata")] pub struct Model { #[sea_orm(primary_key)] pub id: i64, diff --git a/crates/cloudreve-entities/src/nodes.rs b/crates/cloudreve-entities/src/nodes.rs index 81178d9..fafef8f 100644 --- a/crates/cloudreve-entities/src/nodes.rs +++ b/crates/cloudreve-entities/src/nodes.rs @@ -4,7 +4,7 @@ use sea_orm::entity::prelude::*; #[sea_orm::model] #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] -#[sea_orm(schema_name = "public", table_name = "nodes")] +#[sea_orm(table_name = "nodes")] pub struct Model { #[sea_orm(primary_key)] pub id: i64, diff --git a/crates/cloudreve-entities/src/oauth_clients.rs b/crates/cloudreve-entities/src/oauth_clients.rs index 4d28025..2934c22 100644 --- a/crates/cloudreve-entities/src/oauth_clients.rs +++ b/crates/cloudreve-entities/src/oauth_clients.rs @@ -4,7 +4,7 @@ use sea_orm::entity::prelude::*; #[sea_orm::model] #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] -#[sea_orm(schema_name = "public", table_name = "oauth_clients")] +#[sea_orm(table_name = "oauth_clients")] pub struct Model { #[sea_orm(primary_key)] pub id: i64, diff --git a/crates/cloudreve-entities/src/oauth_grants.rs b/crates/cloudreve-entities/src/oauth_grants.rs index 5063856..0230643 100644 --- a/crates/cloudreve-entities/src/oauth_grants.rs +++ b/crates/cloudreve-entities/src/oauth_grants.rs @@ -4,7 +4,7 @@ use sea_orm::entity::prelude::*; #[sea_orm::model] #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] -#[sea_orm(schema_name = "public", table_name = "oauth_grants")] +#[sea_orm(table_name = "oauth_grants")] pub struct Model { #[sea_orm(primary_key)] pub id: i64, diff --git a/crates/cloudreve-entities/src/passkeys.rs b/crates/cloudreve-entities/src/passkeys.rs index 3e57935..09de791 100644 --- a/crates/cloudreve-entities/src/passkeys.rs +++ b/crates/cloudreve-entities/src/passkeys.rs @@ -4,7 +4,7 @@ use sea_orm::entity::prelude::*; #[sea_orm::model] #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] -#[sea_orm(schema_name = "public", table_name = "passkeys")] +#[sea_orm(table_name = "passkeys")] pub struct Model { #[sea_orm(primary_key)] pub id: i64, diff --git a/crates/cloudreve-entities/src/settings.rs b/crates/cloudreve-entities/src/settings.rs index 2ebcc4e..5093ab6 100644 --- a/crates/cloudreve-entities/src/settings.rs +++ b/crates/cloudreve-entities/src/settings.rs @@ -4,7 +4,7 @@ use sea_orm::entity::prelude::*; #[sea_orm::model] #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] -#[sea_orm(schema_name = "public", table_name = "settings")] +#[sea_orm(table_name = "settings")] pub struct Model { #[sea_orm(primary_key)] pub id: i64, diff --git a/crates/cloudreve-entities/src/shares.rs b/crates/cloudreve-entities/src/shares.rs index 5fd10f2..1b74540 100644 --- a/crates/cloudreve-entities/src/shares.rs +++ b/crates/cloudreve-entities/src/shares.rs @@ -4,7 +4,7 @@ use sea_orm::entity::prelude::*; #[sea_orm::model] #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] -#[sea_orm(schema_name = "public", table_name = "shares")] +#[sea_orm(table_name = "shares")] pub struct Model { #[sea_orm(primary_key)] pub id: i64, diff --git a/crates/cloudreve-entities/src/storage_policies.rs b/crates/cloudreve-entities/src/storage_policies.rs index 0fffcc3..b563933 100644 --- a/crates/cloudreve-entities/src/storage_policies.rs +++ b/crates/cloudreve-entities/src/storage_policies.rs @@ -4,7 +4,7 @@ use sea_orm::entity::prelude::*; #[sea_orm::model] #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] -#[sea_orm(schema_name = "public", table_name = "storage_policies")] +#[sea_orm(table_name = "storage_policies")] pub struct Model { #[sea_orm(primary_key)] pub id: i64, diff --git a/crates/cloudreve-entities/src/tasks.rs b/crates/cloudreve-entities/src/tasks.rs index 1c655f4..9b73397 100644 --- a/crates/cloudreve-entities/src/tasks.rs +++ b/crates/cloudreve-entities/src/tasks.rs @@ -4,7 +4,7 @@ use sea_orm::entity::prelude::*; #[sea_orm::model] #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] -#[sea_orm(schema_name = "public", table_name = "tasks")] +#[sea_orm(table_name = "tasks")] pub struct Model { #[sea_orm(primary_key)] pub id: i64, diff --git a/crates/cloudreve-entities/src/users.rs b/crates/cloudreve-entities/src/users.rs index 6eaad1a..b63daff 100644 --- a/crates/cloudreve-entities/src/users.rs +++ b/crates/cloudreve-entities/src/users.rs @@ -4,7 +4,7 @@ use sea_orm::entity::prelude::*; #[sea_orm::model] #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] -#[sea_orm(schema_name = "public", table_name = "users")] +#[sea_orm(table_name = "users")] pub struct Model { #[sea_orm(primary_key)] pub id: i64, diff --git a/docs/cloudreve-to-asterdrive-field-mapping.md b/docs/cloudreve-to-asterdrive-field-mapping.md new file mode 100644 index 0000000..a8c0a50 --- /dev/null +++ b/docs/cloudreve-to-asterdrive-field-mapping.md @@ -0,0 +1,559 @@ +# Cloudreve v4 到 AsterDrive 字段映射与差异 + +本文档基于以下代码快照整理： + +- Cloudreve v4：[`cloudreve/cloudreve`](https://github.com/cloudreve/cloudreve) ，Ent 实际表定义位于 `ent/migrate/schema.go` +- AsterDrive：[`AsterCommunity/AsterDrive`](https://github.com/AsterCommunity/AsterDrive)，SeaORM Entity 位于 `src/entities/` +- 迁移工具：当前仓库 `crates/cloudreve-entities` 和 `crates/asterdrive-entities` + +本文档的目标不是假设所有数据都应原样复制，而是明确： + +1. 两边分别有什么表和字段。 +2. 哪些字段可以直接映射。 +3. 哪些字段需要转换、拆分或生成。 +4. 哪些数据无法安全迁移，需要重新配置或由用户决策。 + +## 1. 总体结构差异 + +| 主题 | Cloudreve v4 | AsterDrive | 迁移影响 | +|---|---|---|---| +| 表数量 | 17 张业务表 | 45 张业务表 | AD 对认证、MFA、团队、任务、远端节点和存储凭据拆分更细 | +| 文件树 | `files` 同时保存文件和目录，通过 `type` 区分 | `files` 与 `folders` 分表 | 必须拆分 Cloudreve `files` | +| 物理对象 | `entities` 保存物理对象，`file_entities` 关联逻辑文件 | `file_blobs` 保存物理对象，`file_versions` 保存历史版本 | 需要重建当前 blob 和历史版本关系 | +| 用户组 | `groups` 同时承载权限、配额和默认存储策略 | `storage_policy_groups` 只负责存储路由；团队使用 `teams` | Cloudreve 用户组不应直接迁移为 AD 团队 | +| 用户密码 | Cloudreve SHA256/SHA1/旧 MD5 加盐格式 | AD Argon2 PHC 字符串 | 不能直接复用，必须设置临时密码或走重置流程 | +| 分享标识 | 分享 URL token 由 ID 和 Cloudreve HashID 密钥计算 | `shares.token` 为独立持久化 token | 必须生成新 token，旧分享 URL 不能保持不变 | +| 存储策略 | 多个 Cloudreve 专用驱动，配置集中在一张表 | 驱动类型更少，OAuth 凭据和应用配置拆表 | 仅部分驱动可直接复用 | +| 系统配置 | `settings(name, value)` 键值表 | `system_config` 包含类型、命名空间、可见性和敏感标记 | 不能盲目全量复制，需要配置键映射表 | +| 软删除 | 大部分 Cloudreve 表有 `deleted_at`，但当前 v4 `files` 没有 | AD 仅部分资源有 `deleted_at` | 需要决定是否包含已删除数据 | +| 认证数据 | OAuth Client/Grant、Passkey、TOTP secret 混合保存 | 外部认证、Passkey、MFA 流程分别建模 | 凭据结构和加密方式不兼容，不应直接复制 | + +## 2. 迁移状态说明 + +| 标记 | 含义 | +|---|---| +| 直接 | 类型和语义基本一致，只需替换外键 ID | +| 转换 | 可以迁移，但需要格式、枚举、类型或语义转换 | +| 生成 | AD 必填，但 Cloudreve 没有，需要迁移工具生成 | +| 拆分 | 一个 Cloudreve 记录需要写入多张 AD 表 | +| 合并 | 多张 Cloudreve 表共同生成一张 AD 表或一组记录 | +| 跳过 | 运行时或临时数据，不建议迁移 | +| 不兼容 | 凭据、加密或协议语义不同，不能安全直接迁移 | +| 决策 | 没有唯一正确映射，需要迁移前由管理员确定 | + +## 3. 用户组映射 + +Cloudreve `groups` 更接近“用户套餐/权限组”，而不是 AD 的协作团队。当前建议映射为： + +- `groups` -> `storage_policy_groups` +- `groups.storage_policy_id` -> `storage_policy_group_items.policy_id` +- `users.group_users` -> `users.policy_group_id` +- `groups.permissions` 中的管理员位 -> `users.role = admin` + +### `groups` -> `storage_policy_groups` + +| Cloudreve 字段 | AD 字段 | 状态 | 规则或差异 | +|---|---|---|---| +| `id` | 新生成 `storage_policy_groups.id` | 转换 | 保存旧 ID -> 新 ID 映射，不建议强行复用 ID | +| `name` | `name` | 直接 | 注意目标库可能已有同名组 | +| `created_at` | `created_at` | 直接 | 保留原时间 | +| `updated_at` | `updated_at` | 直接 | 保留原时间 | +| `deleted_at` | 无直接字段 | 转换 | 默认跳过已删除组；若包含则建议 `is_enabled = false` | +| `max_storage` | `users.storage_quota` | 转换 | AD 策略组自身没有配额，需要下放到组内每个用户 | +| `speed_limit` | 无直接字段 | 决策 | AD 当前用户/组模型没有等价限速列，可写入用户 `config` 或忽略 | +| `permissions` | `users.role`，部分可能进入 `config` | 转换 | 管理员位映射为 `admin`；其他 Cloudreve 权限没有一一对应字段 | +| `settings` | 无直接字段 | 决策 | 压缩、离线下载、回收站配置可选择写入 AD 配置扩展，不宜直接复制 | +| `storage_policy_id` | `storage_policy_group_items.policy_id` | 拆分 | 先迁移存储策略，再创建 group item | +| 无 | `description` | 生成 | 可写 `Migrated from Cloudreve group ` | +| 无 | `is_enabled` | 生成 | 活跃组为 `true`，已删除组为 `false` | +| 无 | `is_default` | 决策 | 选择 Cloudreve 默认用户组对应的 AD 默认策略组 | + +### `groups.storage_policy_id` -> `storage_policy_group_items` + +| AD 字段 | 来源 | 规则 | +|---|---|---| +| `id` | 生成 | AD 自增 ID | +| `group_id` | Cloudreve group ID 映射 | 指向新 `storage_policy_groups.id` | +| `policy_id` | `groups.storage_policy_id` 映射 | 指向新 `storage_policies.id` | +| `priority` | 生成 | 单策略组可设为 `0` | +| `min_file_size` | 生成 | `0` | +| `max_file_size` | 生成 | `0` 表示不额外限制 | +| `created_at` | `groups.created_at` | 保留原时间 | + +## 4. 用户与资料映射 + +Cloudreve 一条 `users` 记录需要拆成 AD 的 `users` 和 `user_profiles`。 + +### `users` -> AD `users` + +| Cloudreve 字段 | AD 字段 | 状态 | 规则或差异 | +|---|---|---|---| +| `id` | 新生成 `id` | 转换 | 保存用户 ID 映射，所有 owner/user 外键都要替换 | +| `email` | `email` | 直接 | AD 同样要求唯一；迁移前检查重复和空值 | +| `nick` | `username` | 转换 | AD `username` 唯一且最长 64；重名时追加 Cloudreve ID | +| `nick` | `user_profiles.display_name` | 拆分 | 原昵称同时作为展示名 | +| `password` | `password_hash` | 不兼容 | Cloudreve SHA/MD5 格式不能被 AD Argon2 验证 | +| `status=active` | `status=active` | 转换 | 直接映射 | +| `status=inactive/manual_banned/sys_banned` | `status=disabled` | 转换 | AD 当前只有 `active`/`disabled` | +| `storage` | `storage_used` | 直接 | 保留已用空间统计，迁移后应重新校验 blob 汇总 | +| `group_users -> groups.max_storage` | `storage_quota` | 转换 | 配额来自用户所属 Cloudreve 组 | +| `group_users` | `policy_group_id` | 转换 | 使用 group ID 映射 | +| `settings` | `config` | 转换 | 可保存原 JSON，但 AD 不会自动理解 Cloudreve 键 | +| `two_factor_secret` | 无安全直接映射 | 不兼容 | AD 需要加密后的 `mfa_factors.secret_ciphertext`，必须重新绑定 MFA | +| `avatar` | `user_profiles.avatar_source/avatar_key` | 转换 | 根据值判断 `none`、`gravatar` 或 `upload` | +| `created_at` | `created_at` | 直接 | 保留原时间 | +| `updated_at` | `updated_at` | 直接 | 保留原时间 | +| `deleted_at` | `status=disabled` 或跳过 | 决策 | 默认跳过软删除用户；包含时应禁用 | +| `groups.permissions` 管理员位 | `role` | 转换 | 管理员组用户 -> `admin`，其他 -> `user` | +| 无 | `session_version` | 生成 | 建议初始化为 `1` | +| 无 | `email_verified_at` | 生成/决策 | Cloudreve 没有等价字段；可对活跃用户设为 `created_at`，或要求重新验证 | +| 无 | `pending_email` | 生成 | `NULL` | +| 无 | `must_change_password` | 生成 | 设置为 `true` | + +### AD `user_profiles` 字段 + +| AD 字段 | Cloudreve 来源 | 规则 | +|---|---|---| +| `user_id` | 新 AD 用户 ID | 与 `users.id` 一对一 | +| `display_name` | `users.nick` | 保留原昵称 | +| `wopi_user_info` | 无 | `NULL` | +| `avatar_source` | `users.avatar` | `none` / `gravatar` / `upload` | +| `avatar_key` | `users.avatar` | 非空时保存原值；之后可能需要单独搬迁头像文件 | +| `avatar_version` | 无 | `0` | +| `created_at` | `users.created_at` | 保留 | +| `updated_at` | `users.updated_at` | 保留 | + +## 5. 存储策略映射 + +### 驱动类型 + +| Cloudreve `type` | AD `driver_type` | 状态 | 说明 | +|---|---|---|---| +| `local` | `local` | 可迁移 | `entity.source` 作为 `storage_path`，需指定正确 AD `base_path` | +| `s3` | `s3` | 可迁移 | 复用 endpoint、bucket、key 和 secret | +| `oss` | `s3` | 需验证 | 通过 S3 兼容接口接入；需验证 endpoint 和 path-style | +| `ks3` | `s3` | 需验证 | 通过 S3 兼容接口接入 | +| `obs` | `s3` | 需验证 | 通过 S3 兼容接口接入；签名/endpoint 兼容性需实测 | +| `cos` | `tencent_cos` | 可迁移 | 使用 AD 原生腾讯 COS 驱动 | +| `qiniu` | 无 | 不兼容 | AD 当前没有七牛驱动，也不能保证 S3 兼容 | +| `upyun` | 无 | 不兼容 | AD 当前没有又拍云驱动 | +| `onedrive` | `onedrive` 但凭据结构不同 | 不兼容 | AD 将 OAuth token 拆到 credential/config 表，不能只复制策略字段 | +| `remote` | `remote` 但节点协议不同 | 不兼容 | Cloudreve Slave 与 AD follower 协议完全不同 | + +启用了 Cloudreve `settings.encryption=true` 的策略不能只迁移数据库元数据。Cloudreve 对象内容仍是 Cloudreve 加密格式，AD 无法直接读取，必须先通过 Cloudreve 解密导出再上传到 AD。 + +### `storage_policies` 字段 + +| Cloudreve 字段 | AD 字段 | 状态 | 规则或差异 | +|---|---|---|---| +| `id` | 新生成 `id` | 转换 | 保存策略 ID 映射 | +| `name` | `name` | 直接 | 保留名称 | +| `type` | `driver_type` | 转换 | 按上表映射 | +| `server` | `endpoint` | 转换 | `NULL` 转为空字符串；不同驱动 endpoint 语义需核实 | +| `bucket_name` | `bucket` | 转换 | `NULL` 转为空字符串 | +| `access_key` | `access_key` | 直接/敏感 | 仅对兼容驱动复制；不得写日志 | +| `secret_key` | `secret_key` | 直接/敏感 | 仅对兼容驱动复制；不得写日志 | +| `max_size` | `max_file_size` | 转换 | Cloudreve `NULL` -> AD `0`，两边都表示无限制时才成立 | +| `settings.file_type` | `allowed_types` | 转换 | 保存为 JSON 数组；还需处理 deny-list 语义 | +| `settings.chunk_size` | `chunk_size` | 转换 | 保留字节数；`0` 表示单次上传 | +| `settings.s3_path_style` | `options.s3_path_style` | 转换 | 保留布尔值 | +| `settings.relay` | `options.object_storage_upload_strategy` | 转换 | `true` 倾向 `relay_stream`；否则可考虑 `presigned` | +| `is_private` | 无单独字段 | 合并 | AD 下载策略由 `options` 控制，不应机械复制 | +| `dir_name_rule` | 无直接字段 | 决策 | AD 对象 key 规则不同；旧对象迁移时保留现有 `entity.source` | +| `file_name_rule` | 无直接字段 | 决策 | 同上 | +| `settings` 其他键 | `options.cloudreve_source` 或忽略 | 决策 | 可存档原 JSON，但 AD 不会自动使用未知键 | +| `node_id` | `remote_node_id` | 不兼容 | Cloudreve node 不能直接变成 AD managed follower | +| `created_at` | `created_at` | 直接 | 保留 | +| `updated_at` | `updated_at` | 直接 | 保留 | +| `deleted_at` | 无 | 转换 | 默认跳过已删除策略 | +| 无 | `base_path` | 生成/配置 | 本地策略必须由管理员提供 Cloudreve 数据根目录 | +| 无 | `remote_storage_target_key` | 生成 | 普通策略为 `NULL` | +| 无 | `is_default` | 决策 | 选择一个迁移策略作为默认策略 | + +## 6. 文件树拆分 + +Cloudreve `files.type`： + +| 值 | 含义 | AD 目标 | +|---|---|---| +| `0` | 文件 | `files` | +| `1` | 目录 | `folders` | + +### Cloudreve 目录记录 -> AD `folders` + +| Cloudreve `files` 字段 | AD `folders` 字段 | 状态 | 规则或差异 | +|---|---|---|---| +| `id` | 新生成 `id` | 转换 | 保存 folder ID 映射 | +| `type=1` | 决定目标表 | 拆分 | 不写入 AD `files` | +| `name` | `name` | 直接 | 保留 | +| `file_children` | `parent_id` | 转换 | Cloudreve 字段名实际表示父目录 ID | +| `owner_id` | `owner_user_id` | 转换 | 使用用户 ID 映射 | +| `owner_id` | `created_by_user_id` | 生成/转换 | Cloudreve 没有独立创建者时使用 owner | +| `owner_id -> users.nick` | `created_by_username` | 生成 | 保存迁移后的 username | +| `storage_policy_files` | `policy_id` | 转换 | 使用策略 ID 映射 | +| `created_at` | `created_at` | 直接 | 保留 | +| `updated_at` | `updated_at` | 直接 | 保留 | +| `props` | 无直接字段 | 决策 | 可按需要拆到 `entity_properties` | +| `size` | 无 | 忽略 | 目录大小不写入 AD folder | +| `primary_entity` | 无 | 忽略 | 目录不应有主实体 | +| `is_symbolic` | 无 | 不兼容 | Cloudreve 占位/符号对象需要单独处理 | +| 无 | `team_id` | 生成 | 个人空间迁移为 `NULL`；若要转团队需另行规划 | +| 无 | `deleted_at` | 生成/决策 | 当前 Cloudreve v4 `files` 无此字段，通常为 `NULL` | +| 无 | `is_locked` | 生成 | `false` | + +### Cloudreve 文件记录 -> AD `files` + +| Cloudreve `files` 字段 | AD `files` 字段 | 状态 | 规则或差异 | +|---|---|---|---| +| `id` | 新生成 `id` | 转换 | 保存 file ID 映射 | +| `type=0` | 决定目标表 | 拆分 | 不写入 AD `folders` | +| `name` | `name` | 直接 | 保留 | +| `file_children` | `folder_id` | 转换 | 使用 folder ID 映射 | +| `owner_id` | `owner_user_id` | 转换 | 使用用户 ID 映射 | +| `owner_id` | `created_by_user_id` | 生成/转换 | 没有独立创建者时使用 owner | +| `owner_id -> username` | `created_by_username` | 生成 | 保存迁移后的 AD username | +| `primary_entity` | `blob_id` | 转换 | 先将主 entity 迁移为 `file_blobs` | +| `size` | `size` | 直接 | 建议同时与主 entity.size 校验 | +| `created_at` | `created_at` | 直接 | 保留 | +| `updated_at` | `updated_at` | 直接 | 保留 | +| `props` | 无直接字段 | 决策 | 可存入 `entity_properties` | +| `storage_policy_files` | 无直接字段 | 间接 | 文件实际策略由 `blob_id -> file_blobs.policy_id` 决定 | +| `is_symbolic=true` | 无直接模型 | 不兼容 | 默认跳过；若是占位文件需先在 Cloudreve 物化 | +| 无 | `mime_type` | 生成 | 根据文件名扩展名推断，建议迁移后重新扫描 | +| 无 | `extension` | 生成 | 小写末级扩展名 | +| 无 | `compound_extension` | 生成 | 如 `tar.gz` | +| 无 | `file_category` | 生成 | image/video/audio/document 等分类 | +| 无 | `team_id` | 生成 | 个人空间为 `NULL` | +| 无 | `deleted_at` | 生成 | 当前 Cloudreve v4 文件表没有该字段，通常为 `NULL` | +| 无 | `is_locked` | 生成 | `false`；Cloudreve 运行时锁不迁移 | + +## 7. 物理对象、当前版本与历史版本 + +Cloudreve 使用： + +- `entities`：物理对象记录 +- `file_entities`：逻辑文件与物理对象多对多关系 +- `files.primary_entity`：当前使用的物理对象 +- `entities.type=0`：文件版本 +- `entities.type=1`：缩略图 +- `entities.type=2`：Live Photo 关联对象 + +AD 使用： + +- `file_blobs`：物理对象 +- `files.blob_id`：当前 blob +- `file_versions`：历史 blob +- `file_blobs.thumbnail_path`：缩略图路径 + +### `entities` -> `file_blobs` + +| Cloudreve 字段 | AD 字段 | 状态 | 规则或差异 | +|---|---|---|---| +| `id` | 新生成 `id` | 转换 | 保存 entity ID -> blob ID 映射 | +| `type=0` | 创建 `file_blobs` | 转换 | 普通版本实体 | +| `type=1` | `thumbnail_path` | 合并 | 找到同一文件的缩略图 entity，将其 `source` 写入主 blob | +| `type=2` | 无直接字段 | 决策 | 可作为额外 blob + property，当前 AD 无 Live Photo 专用关系 | +| `source` | `storage_path` | 直接/关键 | 复用旧对象时必须原样保留 | +| `size` | `size` | 直接 | 保留 | +| `storage_policy_entities` | `policy_id` | 转换 | 使用策略 ID 映射 | +| `reference_count` | `ref_count` | 校验后转换 | 建议根据 `file_entities` 重新计算，不完全信任旧计数 | +| `created_at` | `created_at` | 直接 | 保留 | +| `updated_at` | `updated_at` | 直接 | 保留 | +| `deleted_at` | 无直接字段 | 决策 | 默认跳过已删除实体 | +| `upload_session_id` | 无 | 跳过 | 历史上传会话不应迁移 | +| `recycle_options` | 无直接字段 | 决策 | 回收/恢复信息可归档到 property，但 AD 不会自动使用 | +| `created_by` | 无 blob 创建者字段 | 间接 | 文件记录仍保存 owner/creator | +| 无 | `hash` | 生成 | Cloudreve 没有可靠内容 SHA256；应生成非 SHA256 形态的 opaque key，例如 `cloudreve-` | +| 无 | `thumbnail_processor` | 生成 | `NULL`，等待 AD 重新处理 | +| 无 | `thumbnail_version` | 生成 | `NULL` | + +不能把 `SHA256(policy_id + source + size)` 伪装成内容哈希。AD 会将 64 位十六进制值视为真实内容 SHA256，在校验或存储迁移时会产生错误语义。 + +### `file_entities` 和 `primary_entity` -> `file_versions` + +| Cloudreve 来源 | AD 字段 | 规则 | +|---|---|---| +| `file_entities.file_id` | `file_versions.file_id` | 使用逻辑文件 ID 映射 | +| `file_entities.entity_id` | `file_versions.blob_id` | 使用 entity -> blob 映射 | +| `files.primary_entity` | `files.blob_id` | 当前版本，不写入历史版本表 | +| 非 primary 的 `entity.type=0` | 一条 `file_versions` | 按创建时间升序编号 | +| `entities.size` | `file_versions.size` | 保留 | +| `entities.created_at` | `file_versions.created_at` | 保留 | +| 无明确版本号 | `file_versions.version` | 按历史 entity 创建时间生成 `1..N` | + +## 8. 元数据映射 + +### `metadata` -> `entity_properties` + +| Cloudreve 字段 | AD 字段 | 状态 | 规则 | +|---|---|---|---| +| `id` | 新生成 `id` | 转换 | 无需保留原 ID | +| `file_id` | `entity_id` | 转换 | 根据 Cloudreve `files.type` 使用 AD file 或 folder ID 映射 | +| `files.type` | `entity_type` | 转换 | `type=0 -> file`，`type=1 -> folder` | +| `is_public=true` | `namespace` | 转换 | `cloudreve.public` | +| `is_public=false` | `namespace` | 转换 | `cloudreve.private` | +| `name` | `name` | 直接 | 保留 | +| `value` | `value` | 直接 | 转为 `Some(value)` | +| `created_at` | 无 | 差异 | AD `entity_properties` 当前没有时间字段 | +| `updated_at` | 无 | 差异 | 无法保留 | +| `deleted_at` | 无 | 转换 | 默认跳过已删除元数据 | + +### Cloudreve v4 标签 -> AD `tags` + `entity_properties` + +Cloudreve v4 确实有标签功能，但没有独立 `tags` 表。标签保存在 `metadata.name = tag:<标签名>` 中，`metadata.value` 保存可选颜色。AD 使用 `tags` 保存标签定义，并使用 `entity_properties(namespace=system.tags, name=)` 关联文件或目录。 + +| Cloudreve 来源 | AD 目标 | 迁移规则 | +|---|---|---| +| `metadata.name = tag:` | `tags.name` | 去掉 `tag:`；去除首尾空白；最长保留 64 个字符 | +| 标签名小写值 | `tags.normalized_name` | 按 AD 当前规则 `trim + lowercase`，并按用户个人空间去重 | +| `metadata.value` | `tags.color` | `#RRGGBB` 直接规范化；`#RGB` 展开；空值或非法值使用 `#3b82f6` | +| `files.owner_id` | `tags.owner_user_id` | 使用用户 ID 映射，`scope_type=personal`，`team_id=NULL` | +| `metadata.created_at/updated_at` | `tags.created_at/updated_at` | 创建标签定义时保留首条标签元数据时间 | +| 目标 file/folder ID | `entity_properties.entity_id` | 根据源 `files.type` 关联文件或目录 | +| 无 | `entity_properties.namespace` | 固定为 AD 原生标签命名空间 `system.tags` | +| 新 AD tag ID | `entity_properties.name` | 十进制字符串；`value=NULL` | + +同一用户下大小写不同但规范化名称相同的 Cloudreve 标签会合并为一个 AD 标签定义。Cloudreve v3 曾有独立标签模型，但 Cloudreve 官方 v3 -> v4 迁移阶段没有对应标签迁移步骤；若数据仍停留在 v3 表中，需要直接读取 v3 数据库，不能从 v4 `metadata` 反推出已丢失记录。 + +Cloudreve `files.props` 是另一组 JSON 属性，不在 `metadata` 表中。是否拆成多个 `entity_properties` 需要先定义允许迁移的键，避免将 Cloudreve 内部状态直接暴露给 AD。 + +## 9. 分享映射 + +### `shares` -> AD `shares` + +| Cloudreve 字段 | AD 字段 | 状态 | 规则或差异 | +|---|---|---|---| +| `id` | 新生成 `id` | 转换 | 保存旧 ID仅用于报告 | +| 无持久化 token | `token` | 生成 | 生成新唯一 token，旧 URL 会失效 | +| `user_shares` | `user_id` | 转换 | 使用用户 ID 映射 | +| `file_shares` 指向文件 | `file_id` | 转换 | 根据源 `files.type=0` 判断 | +| `file_shares` 指向目录 | `folder_id` | 转换 | 根据源 `files.type=1` 判断 | +| `password` | `password` | 转换 | Cloudreve 保存明文分享密码；AD 必须保存 Argon2 hash | +| `expires` | `expires_at` | 直接 | 保留 | +| `downloads` | `download_count` | 直接 | 保留已下载次数 | +| `views` | `view_count` | 直接 | 保留浏览次数 | +| `remain_downloads` | `max_downloads` | 转换 | 有剩余次数时：`downloads + remain_downloads`；`NULL` 通常表示无限制，映射为 `0` | +| `props` | 无直接字段 | 决策 | Cloudreve 分享展示/权限属性需逐键分析 | +| `created_at` | `created_at` | 直接 | 保留 | +| `updated_at` | `updated_at` | 直接 | 保留 | +| `deleted_at` | 无 | 转换 | 默认跳过已删除分享 | +| 无 | `team_id` | 生成 | 个人分享为 `NULL` | + +### `direct_links` -> AD v2 直链 + 迁移映射属性 + +Cloudreve 直链并非“完全不能迁移”，但不能原样复制。Cloudreve 表只保存 `direct_link.id/name/downloads/speed/file_id`，旧 URL `/f/{hashid}/{name}` 中的 HashID 是运行时由 `[direct_link.id, SourceLinkID]` 和 Cloudreve salt 计算的。AD 没有 `direct_links` 表，而是使用 `auth.direct_link_secret` 对目标 file ID 和个人/团队 scope 做 HMAC-SHA256，生成 `/d/v2.{base62(file_id)}.{base64url_hmac}/{filename}`。 + +| Cloudreve 字段/语义 | AD 迁移结果 | 差异 | +|---|---|---| +| `file_id` | 映射后的 AD file ID | 文件必须已成功迁移 | +| `id` | `entity_properties.name` | 保存源 direct-link ID，便于查回映射 | +| `name` | 映射 JSON 的 `source_name` | AD URL 必须使用目标文件真实名称，不能继续使用任意旧 link name | +| `downloads` | 映射 JSON 的 `source_downloads` | AD 无持久化直链行，不能继续累计每条旧链接的计数 | +| `speed` | 映射 JSON 的 `source_speed_limit` | AD 没有每条直链限速字段，仅作历史存档 | +| Cloudreve HashID URL | 新 AD v2 URL | 通过 `--direct-link-secret` / `ASTER_DIRECT_LINK_SECRET` 重新签发 | +| 无 | `entity_properties.namespace` | `cloudreve.direct_links` | + +多个指向同一文件的 Cloudreve 直链会得到同一个确定性 AD token，但每个源 `direct_link.id` 都保留一条映射属性。旧 `/f/...` URL、逐条撤销语义、逐条下载计数和限速无法自动延续；若必须保留旧 URL，需要额外实现兼容重定向路由和持久化映射表。 + +即使使用 `--include-deleted`，已软删除的 Cloudreve direct-link 也不会重新签发，以免把已经撤销的公开入口重新激活。 + +### `tasks` -> AD 终态历史归档 + +Cloudreve 任务不能恢复到 AD 执行器中，但可以保存为不可执行的历史记录。迁移使用 AD `kind=system_runtime` 的合法 payload，并把 Cloudreve 原始字段保存在 `runtime_json`；所有记录均写成终态，`lease_expires_at=NULL`，不会被 worker 领取。 + +| Cloudreve `status` | AD `status` | 规则 | +|---|---|---| +| `completed` | `succeeded` | 保存为已完成历史 | +| `error` | `failed` | `failure_can_retry=false`，不允许在 AD 重试源任务 | +| `canceled` | `canceled` | 保存原终态 | +| `queued` | `canceled` | 仅归档，明确不恢复执行 | +| `processing` | `canceled` | 仅归档，明确不恢复执行 | +| `suspending` | `canceled` | 仅归档，明确不恢复执行 | + +`runtime_json` 保存源 task ID、type、status、public_state、private_state、correlation_id、deleted_at 和 `archived_without_resume=true`。这保留了排查历史，但不承诺 Cloudreve payload 能被 AD UI 或执行器理解。 + +## 10. 不能原样迁移的 Cloudreve 表 + +| Cloudreve 表 | 最接近的 AD 表 | 建议 | 原因 | +|---|---|---|---| +| `dav_accounts` | `webdav_accounts` | 不直接迁移 | URI/root 语义不同；Cloudreve 密码不是 AD Argon2 hash；options 位集需重建 | +| `direct_links` | AD stateless direct link + `entity_properties` | 重新签发 | 不能复用旧 HashID URL；提供 AD direct-link secret 后生成新 v2 URL | +| `fs_events` | 无持久化对应表 | 跳过 | 事件订阅属于运行时状态，subscriber UUID 对 AD 无意义 | +| `nodes` | `managed_followers` / `remote_storage_targets` | 不直接迁移 | Cloudreve master/slave 协议与 AD primary/follower 协议不同，密钥也不能复用 | +| `oauth_clients` | `external_auth_providers` 并不等价 | 跳过或导出报告 | Cloudreve 表是“第三方应用访问 Cloudreve”，AD external provider 是“用户通过外部 IdP 登录 AD” | +| `oauth_grants` | 无直接等价 | 跳过 | grant、scope 和 client 都是 Cloudreve 安全域数据 | +| `passkeys` | `passkeys` | 要求重新注册 | credential JSON、user_handle、签名计数和序列化格式不同，直接复制可能导致认证失败或安全风险 | +| `settings` | `system_config` | 只迁移白名单键 | AD 需要 value type、namespace、category、visibility、sensitive 等元数据 | +| `tasks` | `background_tasks` | 只归档终态历史 | 不复制成可执行任务；活动任务统一归档为 canceled | + +## 11. AD 新增但 Cloudreve 没有的数据域 + +以下 AD 表没有可直接对应的 Cloudreve 数据。迁移时通常保持为空，由 AD 运行时重新生成或由管理员配置。 + +| AD 数据域 | AD 表 | 迁移策略 | +|---|---|---| +| 登录会话 | `auth_sessions` | 不迁移，所有用户重新登录 | +| 审计 | `audit_logs` | 可选生成一条“从 Cloudreve 迁移”审计记录，不伪造历史审计 | +| 后台任务 | `background_tasks`、`storage_migration_checkpoints` | Cloudreve 旧任务仅写成 `system_runtime` 终态历史；不创建 checkpoint，不恢复执行 | +| 邮件 | `mail_outbox` | 不迁移 | +| 邮箱验证 | `contact_verification_tokens`、`external_auth_email_verification_flows` | 不迁移临时 token | +| 外部登录 | `external_auth_providers`、`external_auth_identities`、`external_auth_login_flows` | 由管理员重新配置 IdP，用户重新绑定 | +| MFA | `mfa_factors`、`mfa_recovery_codes`、`mfa_login_flows`、`mfa_email_codes`、`mfa_totp_setup_flows` | 不复制 Cloudreve secret，要求重新绑定 | +| 团队空间 | `teams`、`team_members` | Cloudreve group 不等于 team；需要单独的组织映射方案 | +| 标签 | `tags` + `entity_properties(system.tags)` | 从 Cloudreve v4 `metadata.name=tag:*` 创建标签定义和文件/目录关联 | +| 锁 | `resource_locks` | 不迁移运行时锁 | +| WOPI | `wopi_sessions` | 不迁移会话 | +| 上传 | `upload_sessions`、`upload_session_parts` | 不迁移未完成上传 | +| 用户邀请 | `user_invitations` | 不迁移 | +| 远端节点 | `managed_followers`、`follower_enrollment_sessions`、`master_bindings`、`remote_storage_targets` | 重新注册 AD follower | +| 存储 OAuth | `storage_policy_credentials`、`storage_policy_authorization_flows`、`storage_connector_application_configs` | OneDrive 等策略必须在 AD 中重新授权 | +| 媒体解析 | `blob_media_metadata` | 可在迁移后由 AD 重新扫描生成 | + +## 12. 完整 Cloudreve 表与字段清单 + +| 表 | 字段 | +|---|---| +| `dav_accounts` | `id`, `created_at`, `updated_at`, `deleted_at`, `name`, `uri`, `password`, `options`, `props`, `owner_id` | +| `direct_links` | `id`, `created_at`, `updated_at`, `deleted_at`, `name`, `downloads`, `speed`, `file_id` | +| `entities` | `id`, `created_at`, `updated_at`, `deleted_at`, `type`, `source`, `size`, `reference_count`, `upload_session_id`, `recycle_options`, `storage_policy_entities`, `created_by` | +| `file_entities` | `file_id`, `entity_id` | +| `files` | `id`, `created_at`, `updated_at`, `type`, `name`, `size`, `primary_entity`, `is_symbolic`, `props`, `file_children`, `storage_policy_files`, `owner_id` | +| `fs_events` | `id`, `created_at`, `updated_at`, `deleted_at`, `event`, `subscriber`, `user_fsevent` | +| `groups` | `id`, `created_at`, `updated_at`, `deleted_at`, `name`, `max_storage`, `speed_limit`, `permissions`, `settings`, `storage_policy_id` | +| `metadata` | `id`, `created_at`, `updated_at`, `deleted_at`, `name`, `value`, `is_public`, `file_id` | +| `nodes` | `id`, `created_at`, `updated_at`, `deleted_at`, `status`, `name`, `type`, `server`, `slave_key`, `capabilities`, `settings`, `weight` | +| `oauth_clients` | `id`, `created_at`, `updated_at`, `deleted_at`, `guid`, `secret`, `name`, `homepage_url`, `redirect_uris`, `scopes`, `props`, `is_enabled` | +| `oauth_grants` | `id`, `created_at`, `updated_at`, `deleted_at`, `scopes`, `last_used_at`, `client_id`, `user_id` | +| `passkeys` | `id`, `created_at`, `updated_at`, `deleted_at`, `credential_id`, `name`, `credential`, `used_at`, `user_id` | +| `settings` | `id`, `created_at`, `updated_at`, `deleted_at`, `name`, `value` | +| `shares` | `id`, `created_at`, `updated_at`, `deleted_at`, `password`, `views`, `downloads`, `expires`, `remain_downloads`, `file_shares`, `user_shares`, `props` | +| `storage_policies` | `id`, `created_at`, `updated_at`, `deleted_at`, `name`, `type`, `server`, `bucket_name`, `is_private`, `access_key`, `secret_key`, `max_size`, `dir_name_rule`, `file_name_rule`, `settings`, `node_id` | +| `tasks` | `id`, `created_at`, `updated_at`, `deleted_at`, `type`, `status`, `public_state`, `private_state`, `correlation_id`, `user_tasks` | +| `users` | `id`, `created_at`, `updated_at`, `deleted_at`, `email`, `nick`, `password`, `status`, `storage`, `two_factor_secret`, `avatar`, `settings`, `group_users` | + +## 13. 完整 AsterDrive 表与字段清单 + +| 表 | 字段 | +|---|---| +| `audit_logs` | `id`, `user_id`, `action`, `entity_type`, `entity_id`, `entity_name`, `details`, `ip_address`, `user_agent`, `created_at` | +| `auth_sessions` | `id`, `user_id`, `current_refresh_jti`, `previous_refresh_jti`, `refresh_expires_at`, `ip_address`, `user_agent`, `created_at`, `last_seen_at`, `revoked_at` | +| `background_tasks` | `id`, `kind`, `status`, `creator_user_id`, `team_id`, `share_id`, `display_name`, `payload_json`, `result_json`, `steps_json`, `progress_current`, `progress_total`, `status_text`, `attempt_count`, `max_attempts`, `next_run_at`, `processing_token`, `processing_started_at`, `last_heartbeat_at`, `lease_expires_at`, `started_at`, `finished_at`, `last_error`, `failure_can_retry`, `expires_at`, `created_at`, `updated_at`, `runtime_json` | +| `blob_media_metadata` | `id`, `blob_id`, `blob_hash`, `kind`, `status`, `metadata_json`, `error_message`, `parser`, `parser_version`, `created_at`, `updated_at` | +| `contact_verification_tokens` | `id`, `user_id`, `channel`, `purpose`, `target`, `token_hash`, `expires_at`, `consumed_at`, `created_at` | +| `entity_properties` | `id`, `entity_type`, `entity_id`, `namespace`, `name`, `value` | +| `external_auth_email_verification_flows` | `id`, `provider_id`, `identity_namespace`, `subject`, `target_email`, `display_name_snapshot`, `preferred_username_snapshot`, `return_path`, `flow_token_hash`, `verification_token_hash`, `email_requested_at`, `created_at`, `expires_at`, `consumed_at` | +| `external_auth_identities` | `id`, `user_id`, `provider_id`, `identity_namespace`, `subject`, `email_snapshot`, `display_name_snapshot`, `created_at`, `updated_at`, `last_login_at` | +| `external_auth_login_flows` | `id`, `provider_id`, `state_hash`, `nonce`, `pkce_verifier`, `redirect_uri`, `return_path`, `created_at`, `expires_at`, `consumed_at` | +| `external_auth_providers` | `id`, `key`, `display_name`, `icon_url`, `provider_kind`, `protocol`, `issuer_url`, `authorization_url`, `token_url`, `userinfo_url`, `client_id`, `client_secret`, `scopes`, `enabled`, `auto_provision_enabled`, `auto_link_verified_email_enabled`, `require_email_verified`, `subject_claim`, `username_claim`, `display_name_claim`, `email_claim`, `email_verified_claim`, `groups_claim`, `avatar_url_claim`, `allowed_domains`, `created_at`, `updated_at`, `options` | +| `file_blobs` | `id`, `hash`, `size`, `policy_id`, `storage_path`, `thumbnail_path`, `thumbnail_processor`, `thumbnail_version`, `ref_count`, `created_at`, `updated_at` | +| `file_versions` | `id`, `file_id`, `blob_id`, `version`, `size`, `created_at` | +| `files` | `id`, `name`, `folder_id`, `team_id`, `blob_id`, `size`, `owner_user_id`, `created_by_user_id`, `created_by_username`, `mime_type`, `created_at`, `updated_at`, `deleted_at`, `is_locked`, `extension`, `compound_extension`, `file_category` | +| `folders` | `id`, `name`, `parent_id`, `team_id`, `owner_user_id`, `created_by_user_id`, `created_by_username`, `policy_id`, `created_at`, `updated_at`, `deleted_at`, `is_locked` | +| `follower_enrollment_sessions` | `id`, `managed_follower_id`, `token_hash`, `ack_token_hash`, `expires_at`, `redeemed_at`, `acked_at`, `invalidated_at`, `created_at` | +| `mail_outbox` | `id`, `template_code`, `to_address`, `to_name`, `payload_json`, `status`, `attempt_count`, `next_attempt_at`, `processing_started_at`, `sent_at`, `last_error`, `created_at`, `updated_at` | +| `managed_followers` | `id`, `name`, `base_url`, `access_key`, `secret_key`, `is_enabled`, `last_capabilities`, `last_error`, `last_checked_at`, `created_at`, `updated_at`, `transport_mode`, `tunnel_last_error`, `tunnel_last_seen_at` | +| `master_bindings` | `id`, `name`, `master_url`, `access_key`, `secret_key`, `storage_namespace`, `is_enabled`, `created_at`, `updated_at` | +| `mfa_email_codes` | `id`, `flow_id`, `user_id`, `code_hash`, `expires_at`, `consumed_at`, `created_at` | +| `mfa_factors` | `id`, `user_id`, `method`, `name`, `secret_ciphertext`, `secret_version`, `enabled_at`, `last_used_at`, `created_at`, `updated_at` | +| `mfa_login_flows` | `id`, `flow_token_hash`, `user_id`, `user_session_version`, `first_factor`, `return_path`, `ip_address`, `user_agent`, `attempt_count`, `expires_at`, `consumed_at`, `created_at` | +| `mfa_recovery_codes` | `id`, `user_id`, `code_hash`, `used_at`, `created_at` | +| `mfa_totp_setup_flows` | `id`, `flow_token_hash`, `user_id`, `secret_ciphertext`, `secret_version`, `expires_at`, `consumed_at`, `created_at` | +| `passkeys` | `id`, `user_id`, `credential_id`, `user_handle`, `credential`, `name`, `transports`, `backup_eligible`, `backed_up`, `sign_count`, `created_at`, `updated_at`, `last_used_at` | +| `remote_storage_targets` | `id`, `master_binding_id`, `target_key`, `name`, `driver_type`, `endpoint`, `bucket`, `access_key`, `secret_key`, `base_path`, `is_default`, `desired_revision`, `applied_revision`, `last_error`, `created_at`, `updated_at` | +| `resource_locks` | `id`, `token`, `entity_type`, `entity_id`, `path`, `owner_id`, `owner_info`, `timeout_at`, `shared`, `deep`, `created_at` | +| `shares` | `id`, `token`, `user_id`, `team_id`, `file_id`, `folder_id`, `password`, `expires_at`, `max_downloads`, `download_count`, `view_count`, `created_at`, `updated_at` | +| `storage_connector_application_configs` | `id`, `policy_id`, `provider`, `tenant_id`, `scopes`, `client_id`, `client_secret_ciphertext`, `metadata`, `created_at`, `updated_at` | +| `storage_migration_checkpoints` | `task_id`, `source_policy_id`, `target_policy_id`, `plan_hash`, `stage`, `last_processed_blob_id`, `scanned_blobs`, `migrated_blobs`, `merged_blobs`, `skipped_blobs`, `failed_blobs`, `migrated_bytes`, `last_error`, `created_at`, `updated_at`, `renamed_opaque_blobs` | +| `storage_policies` | `id`, `name`, `driver_type`, `endpoint`, `bucket`, `access_key`, `secret_key`, `base_path`, `remote_node_id`, `max_file_size`, `allowed_types`, `options`, `is_default`, `chunk_size`, `created_at`, `updated_at`, `remote_storage_target_key` | +| `storage_policy_authorization_flows` | `id`, `provider`, `policy_id`, `created_by_user_id`, `state_hash`, `pkce_verifier`, `redirect_uri`, `scopes`, `context`, `status`, `created_at`, `expires_at`, `consumed_at` | +| `storage_policy_credentials` | `id`, `policy_id`, `provider`, `credential_kind`, `account_label`, `subject`, `tenant_id`, `scopes`, `access_token_ciphertext`, `refresh_token_ciphertext`, `metadata`, `status`, `status_reason`, `expires_at`, `authorized_at`, `last_refreshed_at`, `last_validated_at`, `created_at`, `updated_at` | +| `storage_policy_group_items` | `id`, `group_id`, `policy_id`, `priority`, `min_file_size`, `max_file_size`, `created_at` | +| `storage_policy_groups` | `id`, `name`, `description`, `is_enabled`, `is_default`, `created_at`, `updated_at` | +| `system_config` | `id`, `key`, `value`, `value_type`, `requires_restart`, `is_sensitive`, `source`, `namespace`, `category`, `description`, `updated_at`, `updated_by`, `visibility` | +| `tags` | `id`, `scope_type`, `owner_user_id`, `team_id`, `name`, `normalized_name`, `color`, `sort_order`, `created_at`, `updated_at` | +| `team_members` | `id`, `team_id`, `user_id`, `role`, `created_at`, `updated_at` | +| `teams` | `id`, `name`, `description`, `created_by`, `storage_used`, `storage_quota`, `policy_group_id`, `created_at`, `updated_at`, `archived_at` | +| `upload_session_parts` | `id`, `upload_id`, `part_number`, `etag`, `size`, `created_at`, `updated_at` | +| `upload_sessions` | `id`, `user_id`, `team_id`, `filename`, `total_size`, `chunk_size`, `total_chunks`, `received_count`, `folder_id`, `policy_id`, `status`, `object_temp_key`, `object_multipart_id`, `file_id`, `created_at`, `expires_at`, `updated_at`, `frontend_client_id` | +| `user_invitations` | `id`, `email`, `token_hash`, `status`, `invited_by`, `accepted_user_id`, `expires_at`, `created_at`, `updated_at`, `accepted_at`, `revoked_at` | +| `user_profiles` | `user_id`, `display_name`, `wopi_user_info`, `avatar_source`, `avatar_key`, `avatar_version`, `created_at`, `updated_at` | +| `users` | `id`, `username`, `email`, `password_hash`, `role`, `status`, `session_version`, `email_verified_at`, `pending_email`, `storage_used`, `storage_quota`, `policy_group_id`, `created_at`, `updated_at`, `config`, `must_change_password` | +| `webdav_accounts` | `id`, `user_id`, `username`, `password_hash`, `root_folder_id`, `is_active`, `created_at`, `updated_at`, `team_id` | +| `wopi_sessions` | `id`, `token_hash`, `actor_user_id`, `session_version`, `team_id`, `file_id`, `app_key`, `expires_at`, `created_at` | + +## 14. 建议在执行迁移前确认的决策 + +| 决策项 | 可选方案 | 当前迁移实现默认值 | +|---|---|---| +| 已删除数据 | 跳过 / 迁移并标记删除或禁用 | 默认跳过；可显式包含部分软删除表 | +| 用户密码 | 统一临时密码 / 每人随机密码并导出 / 邮件重置 | 统一临时 Argon2 密码，`must_change_password=true` | +| 邮箱验证 | 视为已验证 / 全部重新验证 | 活跃用户按已验证处理 | +| Cloudreve group | AD 策略组 / AD team / 两者都建 | 映射为存储策略组，不自动创建 team | +| 本地存储根目录 | 复用 Cloudreve 工作目录 / 搬迁文件到 AD 新目录 | 通过 `--local-base-path` 指定并复用旧对象路径 | +| 多本地策略根目录 | 对所有策略使用同一根目录 / 按 source policy 分别指定 | `--local-policy-root =` 覆盖指定策略，未指定策略回退 `--local-base-path` | +| 本地对象预检 | 仅检查根目录 / 逐对象路径、权限和大小检查 | `--verify-local-storage`；配合 `--dry-run` 可在写入前分页扫描全部可复用 local blob | +| 对象存储 | 复用原 bucket / 复制到新 bucket | 复用原 endpoint、bucket 和 object key | +| 历史版本 | 全部迁移 / 只迁移当前版本 | 当前实现迁移全部 `type=0` entity | +| 分享 URL | 生成新 token / 尝试兼容旧 HashID | 生成新 token，旧 URL 失效 | +| Direct link | 重新签发 AD v2 URL / 跳过 / 建旧 URL 兼容层 | 提供 `--direct-link-secret` 时重新签发并存档 source ID -> URL 映射；旧 `/f/...` 失效 | +| Cloudreve metadata | 全量存档 / 白名单迁移 | `metadata` 表全量迁移到带命名空间的 property；`files.props` 暂不迁移 | +| Cloudreve 标签 | 从 `tag:*` 生成 AD 原生标签 / 仅按普通 property 存档 | 当前生成 `tags` 定义和 `system.tags` 关联 | +| Cloudreve 任务 | 跳过 / 历史归档 / 尝试恢复执行 | 当前全部历史归档；活动任务归档为 canceled，绝不恢复执行 | +| 不兼容策略 | 中止 / 跳过策略和依赖文件 / 解密导出后再迁移 | 默认中止；显式参数可跳过 | +| Passkey/MFA/WebDAV | 尝试转换 / 要求重新绑定 | 要求重新绑定 | +| 系统设置 | 全量复制 / 白名单映射 / 不迁移 | 当前不迁移，建议单独制作配置键白名单 | + +## 15. 建议的验证顺序 + +| 阶段 | 检查内容 | +|---|---| +| 迁移前预检 | 表是否存在、目标核心表是否为空、源外键是否完整、是否存在不兼容存储策略 | +| 用户迁移后 | 用户数、管理员数、禁用用户数、邮箱唯一性、用户名冲突、配额合计 | +| 文件树迁移后 | 根目录数、目录数、文件数、孤儿 parent、孤儿 owner、同目录重名 | +| blob 迁移后 | entity 数、blob 数、当前 blob 关联、历史版本数、ref_count 重算结果 | +| 存储验证 | 随机抽样每种策略执行 metadata、读取、范围读取，校验文件大小 | +| 分享验证 | 文件分享、目录分享、密码分享、过期分享、下载次数限制 | +| 标签验证 | 每个 `tag:*` 元数据有对应 AD tag；`system.tags` 关联指向正确 file/folder 和 scope | +| 直链验证 | 使用与 AD 相同的 `auth.direct_link_secret` 请求新 `/d/v2...` URL；核对映射属性数量 | +| 任务验证 | 所有导入任务均为 `succeeded/failed/canceled`，无 `pending/retry/processing`，`lease_expires_at` 为空 | +| AD 启动后 | 重新生成媒体元数据、缩略图和搜索索引；检查 storage_used 汇总 | +| 切换前 | 冻结 Cloudreve 写入，再跑最终迁移或增量校验，备份目标库 | + +## 16. JSON 报告字段 + +通过 `--report-path ` 可为 `check` 或 `migrate` 输出 `schema_version=1` 的 JSON 报告。 + +| 报告字段 | 内容 | +|---|---| +| `source_*` | Cloudreve 各类源对象数量 | +| `migrated_*` | 本次写入 AD 的各类对象数量 | +| `skipped_by_type` | 按 `file`、`blob`、`share`、`direct_link` 等类型聚合的跳过数量 | +| `skipped_objects` | 每条跳过记录的对象类型、Cloudreve source ID 和明确原因 | +| `mappings` | policy、policy group、user、folder、blob、file、share、task 的排序 source ID -> target ID | +| `tag_assignments` | Cloudreve metadata ID、源 file/folder ID、AD entity ID、AD tag ID 和标签名 | +| `direct_links` | Cloudreve direct-link/file ID、AD file ID、新 URL、原名称、下载次数和限速 | +| `validation` | 是否执行/通过，以及每项检查的 expected、actual 和失败信息 | +| `run_id` | checkpoint run ID；未指定时由迁移工具生成 UUID | +| `resumed` | 本次执行是否从已有 checkpoint 恢复 | +| `completed_stages` | 已原子提交完成的迁移阶段列表 | + +当前提交后校验覆盖核心表增量数量、导入任务是否全为终态且无 lease、`system.tags` 关联是否存在、`cloudreve.direct_links` 属性中的 URL 是否与报告一致。报告不会保存数据库密码、存储密钥或 Cloudreve task private state，但会包含新 direct-link URL，因此必须限制报告文件访问权限。 + +## 17. 有限断点续传语义 + +迁移工具会在 AD 数据库自动创建 `aster_external_migration_runs`。首次迁移建议显式指定 `--run-id`；失败后使用相同参数和 `--resume --run-id `。 + +| 行为 | 当前实现 | +|---|---| +| checkpoint 粒度 | blobs/files 为 batch 级；其他 stage 为 stage 级 | +| 原子性 | blob/file page 的目标记录、object mapping、cursor 和 report 同事务；其他 stage 的目标记录、context、report 和 last completed stage 同事务 | +| 已完成 stage | resume 时跳过，不重复插入 | +| 失败 stage | 整个 stage 回滚，resume 时从该 stage 开头重新执行 | +| ID mapping | blob/file 映射逐行保存在 `aster_external_migration_object_map`；其他映射仍在 `context_json` | +| 初始目标计数 | 保存在 `baseline_json`，恢复完成后仍可执行正确的增量数量校验 | +| 源校验 | URL 摘要 + 源表数量指纹 | +| 计划校验 | local path、迁移 flags、临时密码摘要和 direct-link secret 摘要 | +| stage 内分页 cursor | blobs 使用 `entities.id`、files 使用 `files.id` keyset cursor；其他 stage 尚未实现 | +| 对象上传断点 | 尚未实现 | + +由于源指纹不能检测数量不变的内容修改，断点恢复期间仍必须冻结 Cloudreve 写入。blobs 可从最后提交的 entity ID 继续，files 可从最后提交的 file ID 继续，批大小由 `--blob-batch-size` 和 `--file-batch-size` 控制，默认均为 500；folders、metadata、shares、direct links 和 tasks 等其他 stage 仍会从 stage 起点重跑。对象字节复制/上传仍未实现，因此也没有对象存储分片上传续传。 diff --git a/docs/migration-architecture.md b/docs/migration-architecture.md new file mode 100644 index 0000000..dfd8b06 --- /dev/null +++ b/docs/migration-architecture.md @@ -0,0 +1,825 @@ +# AsterDriveMigration 目标架构设计 + +本文档将现有架构图和文件/对象关系图转写为可实施的文字设计。它描述的是迁移工具的目标架构，同时明确当前代码已经覆盖的部分和后续仍需实现的部分。 + +配套字段文档：[`cloudreve-to-asterdrive-field-mapping.md`](cloudreve-to-asterdrive-field-mapping.md)。 + +## 1. 设计目标 + +AsterDriveMigration 不应成为一组把源表逐行复制到目标表的脚本，而应成为一个可恢复、可校验、可扩展的数据迁移引擎。 + +核心目标： + +| 目标 | 说明 | +|---|---| +| 无直接表复制 | 读取源系统对象，转换为统一领域模型，再通过 AD writer 写入目标 | +| 源系统隔离 | Cloudreve 特有字段、状态和存储规则只存在于 Cloudreve Adapter | +| 目标系统隔离 | AD schema、事务、去重、配额和 ref_count 规则只存在于 AsterDrive Writer | +| 可恢复 | 每个 stage 保存 cursor，进程中断后可以从 checkpoint 继续 | +| 幂等 | 同一个源对象重复处理时，通过 object mapping 找到已有目标对象，不重复创建 | +| 可校验 | 迁移后能够重新读取源和目标，验证数量、关系、大小、哈希、配额和引用计数 | +| 可审计 | 每次运行保留状态、统计、错误、警告和对象映射 | +| 可扩展 | 后续增加 Alist、FileBrowser、Nextcloud 等 source adapter 时不修改 AD writer 核心 | +| 大文件友好 | 文件内容使用流式读写，不将整个对象载入内存 | + +## 2. 核心原则 + +### 2.1 禁止直接复制表 + +不采用以下模式： + +```text +Cloudreve users row -> INSERT INTO AsterDrive users +Cloudreve files row -> INSERT INTO AsterDrive files +``` + +采用以下模式： + +```text +Cloudreve DB/Storage + -> Cloudreve Adapter + -> ExternalUser / ExternalFolder / ExternalFile / ExternalBlob + -> Migration Core + -> AsterDrive Writer + -> AsterDrive DB/Storage +``` + +原因是 Cloudreve 和 AD 的表语义不同。例如 Cloudreve `files` 同时表示文件和目录，而 AD 使用 `files`、`folders`、`file_blobs` 和 `file_versions` 分别建模。 + +### 2.2 映射驱动，而不是 ID 假设驱动 + +源 ID 和目标 ID 不能假设相等。所有跨对象关系必须通过持久化 object mapping 解析： + +```text +(source_type, source_id) -> (target_type, target_id) +``` + +典型映射： + +| 源对象 | 目标对象 | +|---|---| +| Cloudreve user ID | AD user ID | +| Cloudreve folder/file record ID | AD folder ID 或 file ID | +| Cloudreve entity ID/object key | AD file_blob ID | +| Cloudreve share ID | AD share ID | +| Cloudreve policy ID | AD storage_policy ID | + +### 2.3 文件记录与物理对象分离 + +迁移模型必须区分： + +- `ExternalFile`：用户看到的逻辑文件，包含名称、父目录、owner、大小、MIME。 +- `ExternalBlob`：存储系统中的物理对象，包含 object key、策略、大小、可选哈希和读取方式。 + +多个逻辑文件可以复用一个 AD `file_blob`。同一逻辑文件的旧 blob 应写入 `file_versions`，当前 blob 写入 `files.blob_id`。 + +## 3. 分层架构 + +## 3.1 Layer 1：Interface Layer + +负责用户交互、配置和输出，不承载源系统或 AD 业务规则。 + +### CLI Commands + +目标命令集合： + +| 命令 | 作用 | +|---|---| +| `migrate start` | 创建迁移运行，执行预检、规划和迁移 | +| `migrate list` | 列出历史运行及状态 | +| `migrate resume` | 从 checkpoint 恢复中断运行 | +| `migrate verify` | 对已完成或部分完成的运行执行一致性校验 | +| `migrate report` | 导出人类可读或 JSON 报告 | + +当前代码提供的是简化命令： + +| 当前命令 | 作用 | +|---|---| +| `check` | 连接源/目标数据库，检查 schema 和源数据概况 | +| `migrate` | 在一个目标数据库事务中执行一次性迁移 | + +### TUI Progress View（可选） + +目标能力： + +- 实时迁移进度 +- 当前 stage 和状态 +- 对象吞吐量、字节吞吐量 +- 错误与重试次数 +- 预计剩余时间 ETA + +TUI 只订阅 engine progress event，不直接访问数据库或实现迁移逻辑。 + +### Config Loader + +配置来源： + +- 配置文件，例如 `config.yaml` +- 环境变量 +- secret provider 或运行时安全输入 +- source 和 target 数据库连接 +- source 和 target 存储连接 +- 冲突处理、并发、重试和验证选项 + +密码、数据库凭据、OAuth token、存储 secret 和任务 private state 不得出现在普通日志、迁移报告或 TUI 中。JSON 报告按需求包含重新签发的 direct-link URL；该 URL 本身是公开访问能力，报告文件必须按敏感文件保护。 + +### Output Renderer + +至少支持： + +- Human Text：供管理员直接阅读 +- JSON：供自动化、CI 或外部控制器消费 +- Machine-readable exit code：区分成功、警告完成、校验失败和运行失败 + +## 3.2 Layer 2：Migration Core + +Migration Core 编排完整流程，但不直接理解 Cloudreve 表或 AD Entity。 + +### Engine + +职责： + +- 创建或恢复 migration run +- 控制完整生命周期 +- 管理取消信号和优雅停止 +- 协调并发任务 +- 汇总错误和最终状态 +- 确保阶段切换符合状态机 + +建议运行状态： + +```text +planned -> running -> verifying -> completed + -> completed_with_warnings + -> failed + -> cancelled +``` + +### Stage Runner + +职责： + +- 按依赖顺序执行 stages +- 控制每个 stage 的并行度 +- 实现可配置 retry 和 exponential backoff +- 在每批数据完成后保存 cursor +- 支持安全停止和 resume + +建议 stage 顺序： + +| 顺序 | Stage | 主要输出 | +|---:|---|---| +| 1 | `preflight` | schema、版本、连接、权限和存储能力检查 | +| 2 | `inventory` | 源对象数量、字节数和不兼容能力清单 | +| 3 | `policies` | AD storage policy 及 ID mapping | +| 4 | `policy_groups` | AD policy group、group item 及 mapping | +| 5 | `users` | AD user/profile 及 mapping | +| 6 | `folders` | 目录树及 parent mapping | +| 7 | `blobs` | 对象复制/复用、哈希和 blob mapping | +| 8 | `files` | 当前文件和历史版本 | +| 9 | `shares` | 分享和新 token | +| 10 | `metadata_tags` | properties、Cloudreve `tag:*` 到 AD 原生标签及关联 | +| 11 | `direct_links` | 使用 AD secret 重新签发 v2 URL，保存 source link -> URL 映射 | +| 12 | `task_history` | 将 Cloudreve 任务写成不可领取的终态历史记录 | +| 13 | `recalculate` | ref_count、storage_used、配额和统计重算 | +| 14 | `verify` | 数量、关系、哈希、直链签名、任务终态和可读性校验 | +| 15 | `finalize` | 最终报告和运行状态 | + +### Plan Builder + +职责： + +- 扫描源 inventory +- 识别源版本和 capabilities +- 判断哪些资源可迁移、跳过或必须中止 +- 生成 `MigrationPlan` +- 估算对象数、总字节数、阶段成本和风险 + +计划中应明确： + +- 用户、目录、文件、blob、版本、分享数量 +- 各存储策略的对象数量和字节数 +- 不兼容的存储驱动 +- Cloudreve 加密对象数量 +- symbolic/placeholder 文件数量 +- 重名冲突和孤儿关系数量 +- 预计需要的目标存储空间 + +### Checkpoint Manager + +职责： + +- 保存和加载 run state +- 保存每个 stage 的 cursor/offset +- 保存 object mapping +- 提供 resume 支持 +- 判断重复执行是否安全 + +checkpoint 必须在目标记录成功提交后更新，不能先推进 cursor 再写目标数据。 + +### Object Mapping Registry + +职责： + +- 根据 source type + ID 查询目标对象 +- 创建或更新映射 +- 保存源对象哈希、元数据摘要和状态 +- 支持 lookup、upsert 和验证 +- 为断点续传、去重和冲突处理提供依据 + +映射不只保存整数 ID。Cloudreve blob 还可能使用 object key 作为稳定源标识。 + +### Conflict Resolver + +需要检测： + +- 用户名或邮箱冲突 +- 同目录同名文件/目录冲突 +- 目标对象已经存在但来源不同 +- mapping 指向的目标对象已被删除或修改 +- 相同 hash+policy 已有 blob +- 源文件在迁移过程中发生变化 + +`ImportDecision` 至少支持： + +| 决策 | 行为 | +|---|---| +| `fail` | 遇到冲突立即停止，适合严格迁移 | +| `rename` | 生成唯一目标名，例如 `name (1).ext` | +| `skip` | 保留目标数据，记录源对象被跳过 | +| `reuse` | 已验证为同一对象时复用目标记录或 blob | +| `update` | 明确允许时更新已映射的目标对象 | + +所有自动决策必须写入报告，不能静默改名或跳过。 + +### Verification Orchestrator + +职责： + +- 重新读取源 inventory +- 读取 object mapping +- 读取目标状态 +- 执行一致性检查 +- 将校验结果交给 Report Builder + +### Report Builder + +当前实现通过 `--report-path` 为 `check` 和 `migrate` 输出 `schema_version=1` 的 pretty JSON，同时保留终端摘要。报告内容： + +- 源和目标对象统计 +- 各对象类型迁移数量 +- 按对象类型聚合的跳过数量 +- 每个跳过对象的 source ID 和原因 +- policy、policy group、user、folder、blob、file、share、task 的 source ID -> target ID +- Cloudreve tag metadata -> AD tag/entity 关联 +- Cloudreve direct-link ID -> AD file ID/new URL +- 不兼容能力和人工处理项 +- 提交后重新查询 AD 得到的数量校验 +- 导入任务终态/lease 校验 +- `system.tags` 和 `cloudreve.direct_links` property 校验 +- 错误和警告摘要 + +校验失败时 CLI 必须先写报告和终端摘要，再以非零状态退出。当前报告只覆盖成功提交后的校验；事务提交前发生的致命错误仍直接返回错误，后续可引入 migration run 持久化来保存失败中间态。 + +## 3.3 Layer 3：Source Adapter Layer + +Source Adapter 读取源系统并产出统一领域对象。所有源系统专有逻辑必须留在 adapter 内。 + +建议 trait： + +```rust +trait SourceAdapter { + async fn capabilities(&self) -> Result; + async fn inventory(&self) -> Result; + async fn list_users(&self, cursor: Cursor) -> Result>; + async fn list_folders(&self, cursor: Cursor) -> Result>; + async fn list_files(&self, cursor: Cursor) -> Result>; + async fn list_shares(&self, cursor: Cursor) -> Result>; + async fn get_blob(&self, blob: &ExternalBlobRef) -> Result; +} +``` + +### Cloudreve Adapter + +内部组件： + +| 组件 | 职责 | +|---|---| +| Cloudreve DB Reader | 读取 users、groups、files、entities、file_entities、shares、metadata、direct_links、tasks 和 policies | +| Cloudreve Storage Resolver | 解析 policy、driver、physical location 和对象是否存在 | +| Cloudreve Model Decoder | 解码 JSON、boolset、版本关系、文件类型和软删除状态 | +| Cloudreve -> Domain Converter | 转成 ExternalUser/Folder/File/Blob/Share，并规范化字段 | + +Cloudreve 特有逻辑示例： + +- `files.type=0/1` 的文件/目录拆分 +- `file_children` 实际是 parent ID +- `primary_entity` 与 `file_entities` 的当前/历史版本关系 +- `entities.type=0/1/2` 的版本、缩略图和 Live Photo 语义 +- Cloudreve policy type 到通用 storage capability 的转换 +- Cloudreve password、MFA、Passkey 和 share password 格式识别 +- 从 `metadata.name=tag:*` 提取标签，在用户个人 scope 内去重，并生成 AD `system.tags` 关联 +- 将 Cloudreve `/f/{hashid}/{name}` 语义转换为 AD `/d/v2...` 重新签发映射，而不是复制旧 token +- 将 Cloudreve 任务作为终态历史归档；queued/processing/suspending 不进入 AD 可执行队列 + +### Future Adapters + +Alist、FileBrowser、Nextcloud 等 adapter 只需实现 SourceAdapter 并产出相同领域模型，不应修改 AsterDrive Writer。 + +## 3.4 Layer 4：Domain Model Layer + +领域模型必须与 Cloudreve 和 AD 的 ORM Entity 解耦。 + +### ExternalUser + +建议字段： + +```text +source_user_id +source_username +source_email +display_name +status +role_hint +storage_used +storage_quota +source_group_id +created_at +updated_at +metadata +``` + +### ExternalFolder + +```text +source_folder_id +source_name +source_parent_id +source_owner_id +source_policy_id +created_at +updated_at +metadata +``` + +### ExternalFile + +```text +source_file_id +source_name +source_folder_id +source_owner_id +source_current_blob +source_historical_blobs +source_size +mime_type +created_at +updated_at +metadata +``` + +### ExternalBlob + +```text +source_blob_id +source_object_key +source_policy_id +source_size +source_hash (optional) +source_thumbnail +created_at +updated_at +``` + +### ExternalShare + +```text +source_share_id +source_owner_id +source_target_type +source_target_id +source_password +view_count +download_count +remaining_downloads +expires_at +metadata +``` + +### WorkspaceScope + +描述目标写入个人空间还是团队空间： + +```text +Personal { owner_user_id } +Team { team_id, actor_user_id } +``` + +Cloudreve group 默认不自动转换成 WorkspaceScope::Team。只有管理员提供明确组织映射时才创建 AD team。 + +### MigrationPlan + +保存： + +- source/target 标识 +- enabled stages +- storage policy decisions +- conflict policy +- include_deleted +- copy/reuse storage mode +- concurrency/retry 参数 +- verification level +- inventory estimate + +### ImportDecision + +保存 conflict resolver 对某个对象的最终决策，以及决策原因和是否需要人工确认。 + +## 3.5 Layer 5：Target Layer / AsterDrive Writer + +Target Writer 是唯一允许理解 AD schema 和 AD 存储规则的层。 + +建议 trait： + +```rust +trait TargetWriter { + async fn write_user(&self, user: &ExternalUser) -> Result; + async fn write_folder(&self, folder: &ExternalFolder) -> Result; + async fn write_blob(&self, blob: &ExternalBlob) -> Result; + async fn write_file(&self, file: &ExternalFile) -> Result; + async fn write_share(&self, share: &ExternalShare) -> Result; + async fn finalize(&self, run: &MigrationRun) -> Result<()>; +} +``` + +### User Writer + +- Create/update user +- 创建 user profile +- 映射 role/status +- 设置临时 Argon2 密码 +- 设置 `must_change_password` +- 写入 quota 和 policy group + +### Folder Writer + +- Create/update folder +- 通过 object mapping 解析 parent +- 保证父目录先于子目录创建 +- 处理同目录重名 + +### Blob Writer + +- 根据 hash + policy 查找可复用 blob +- 流式上传对象 +- 生成 AD storage path +- 校验 size 和 checksum +- 插入或复用 `file_blobs` + +### File Writer + +- Create/update `files` +- 写入 metadata/classification +- 指向当前 blob +- 将历史 blob 写入 `file_versions` +- 处理 rename/skip/fail 决策 + +### Share Writer（可选） + +- 创建或更新分享 +- 生成新的 AD token/link code +- 将 Cloudreve 明文分享密码转为 Argon2 hash +- 保留过期时间、访问和下载计数 + +### Quota Recalculator + +- 按用户或 workspace 重新计算 `storage_used` +- 不只信任 Cloudreve 旧统计 +- 明确去重 blob 是否按逻辑文件大小还是物理占用计入配额 + +### RefCount Recalculator + +- 根据 `files.blob_id` 和 `file_versions.blob_id` 重算 ref_count +- 找出孤儿 blob +- 找出 ref_count 与实际引用不一致的 blob + +### AsterDrive Storage Writer + +- Stream/chunk 上传 +- 计算 SHA-256 +- 写入 AD 规范 storage path +- 支持本地和对象存储 +- 完成后重新读取 metadata/size 验证 + +### AsterDrive DB Writer + +- 事务边界 +- 批量插入 +- index-aware update +- 与 checkpoint 更新保持提交顺序一致 + +## 3.6 Layer 6：Persistence Layer + +### Source System + +- Cloudreve Database：用户、目录、文件、分享、设置、自定义字段 +- Cloudreve Storage：文件 blob、缩略图、附件和其他物理对象 + +### Target System + +- AsterDrive Database：用户、workspace、目录、文件、blob、分享、配额和 metadata +- AsterDrive Storage：规范化 blob、去重对象和备份/归档对象 + +### Checkpoint Tables + +checkpoint 表保存在 AD 数据库中，以便迁移进程重启后恢复。当前实现先提供阶段级有限恢复。 + +#### `aster_external_migration_runs` + +| 字段 | 当前类型 | 说明 | +|---|---|---| +| `id` | UUID/String PK | Run ID | +| `source_fingerprint` | String | 源数据库/实例指纹，避免连错源 | +| `target_fingerprint` | String | 目标 AD 实例指纹 | +| `plan_fingerprint` | String | local path、flags 和 secret/password 摘要组成的计划指纹 | +| `status` | String | running/failed/completed/validation_failed | +| `last_completed_stage` | String nullable | 最后一个与数据一起原子提交的阶段 | +| `context_json` | JSON/JSONB | 当前内存 ID mapping 和 username mapping 的持久化快照 | +| `report_json` | JSON/JSONB | 已完成阶段的完整结构化报告 | +| `baseline_json` | JSON/JSONB | 新 run 开始前的目标表计数，用于恢复后的最终增量校验 | +| `last_error` | Text nullable | 最近一次阶段失败摘要，不包含 migration secret | +| `created_at` | DateTime | 创建时间 | +| `updated_at` | DateTime | 最后更新时间 | + +当前阶段顺序为 policies、policy_groups、users、folders、blobs、files、metadata、shares、direct_links、tasks。除 blobs/files 外，每个阶段使用独立目标事务；目标数据和 `last_completed_stage/context_json/report_json` 在同一事务中提交。blobs 按 Cloudreve `entities.id`、files 按 Cloudreve `files.id` keyset 分页；每页独立事务提交目标记录、逐行 mapping、cursor 和报告进度。 + +当前 source fingerprint 使用数据库 URL 摘要和各源表记录数量，只能防止明显连错源或数量变化，不能检测“数量不变但内容变化”。生产迁移仍应冻结 Cloudreve 写入。 + +对于 `reuse_source_storage` 的 local policy，当前实现支持用全局 `local_base_path` 或按 Cloudreve source policy ID 覆盖的 local root 配置 AD `base_path`。启用 `verify_local_storage` 后，预检会检查 local root；正式 blob batch 会检查每个复用对象解析后的路径、常规文件状态、打开权限和大小。`dry-run` 会在写入前分页扫描所有兼容 local blob。S3/COS 等远程复用策略仍需要后续的 provider-level `HeadObject`/range-read 验证。 + +#### `aster_external_migration_stage_cursors` + +已用于 blobs 和 files；其他 stage 尚未接入。 + +| 字段 | 当前类型 | 说明 | +|---|---|---| +| `run_id` | String PK | 所属 run | +| `stage` | String PK | 当前为 `blobs` 或 `files` | +| `cursor_value` | BigInt | 最后提交的 Cloudreve entity ID 或 file ID | +| `processed_count` | BigInt | 已扫描并提交的 blob entity 或 file 数量，包含明确跳过项 | +| `updated_at` | DateTime | checkpoint 时间 | + +建议 `(run_id, stage)` 为联合主键或唯一键。 + +#### `aster_external_migration_object_map` + +已用于 blob 和 file 映射；其他对象类型仍使用 run `context_json` 中的 HashMap。 + +| 字段 | 当前类型 | 说明 | +|---|---|---| +| `run_id` | String PK | 所属 run | +| `object_type` | String PK | 当前为 `blob` 或 `file` | +| `source_id` | BigInt PK | Cloudreve entity ID 或 file ID | +| `target_id` | BigInt | AD file_blob 或 file ID | +| `created_at` | DateTime | 映射创建时间 | +| `state` | String | created/reused/skipped/renamed/failed | +| `created_at` | DateTime | 创建时间 | +| `updated_at` | DateTime | 更新时间 | + +建议唯一键： + +```text +(run_id, source_type, source_id) +``` + +## 4. 文件与对象关系 + +## 4.1 源模型到领域模型 + +| Cloudreve 对象 | 领域对象 | 关键字段 | +|---|---|---| +| User | ExternalUser | source_user_id、username、email、used_storage | +| Folder/Directory | ExternalFolder | source_folder_id、name、parent_id、owner_id | +| File | ExternalFile | source_file_id、name、folder_id、object/entity、size、MIME | +| Policy | Storage capability/plan decision | policy ID、driver、rules | +| Physical Entity | ExternalBlob | object key、size、policy、storage path | +| Share | ExternalShare | source_share_id、target、password、权限、expires_at | + +## 4.2 领域模型到 AD + +| 领域对象 | AD 表 | 关键关系 | +|---|---|---| +| ExternalUser | `users` + `user_profiles` | 一个 source user 对应一个 target user | +| ExternalFolder | `folders` | parent_id 通过 mapping 解析 | +| ExternalBlob | `file_blobs` | hash+policy 去重或创建 | +| ExternalFile | `files` | `blob_id` 指向当前 blob | +| Historical ExternalBlob | `file_versions` | `file_id` + `blob_id` + version | +| ExternalShare | `shares` | 指向 file_id 或 folder_id | + +## 4.3 关键不变量 + +| 不变量 | 说明 | +|---|---| +| 文件与 blob 分离 | `files` 只表示逻辑文件，`file_blobs` 表示物理对象 | +| blob 可复用 | 多个 files 可以指向同一 blob | +| 同策略内容去重 | 只有内容哈希可信且 policy 相同才复用 blob | +| parent 先创建 | folder parent mapping 必须在 child 写入前存在 | +| file 依赖 blob | 当前 blob 写入成功后才能插入 files | +| mapping 持久化 | source->target 映射是 resume、验证和幂等的核心 | +| ref_count 可重算 | ref_count 必须能从 files + file_versions 反向计算 | +| storage_used 可重算 | 用户/workspace 使用量必须能从目标文件关系反向校验 | + +## 5. 物理对象复制流程 + +目标流程不是只复用 Cloudreve `entity.source`，而是支持将对象实际复制到 AD 管理的存储空间。 + +| 步骤 | 动作 | 关键要求 | +|---:|---|---| +| 1 | Read source physical object | SourceAdapter 根据 policy 和 object key 打开流 | +| 2 | Stream bytes | 固定大小 buffer，支持 backpressure，不整文件载入内存 | +| 3 | Calculate SHA-256 | 在流式传输过程中计算真实内容哈希 | +| 4 | Write to AD storage | 使用 AD storage writer 上传到目标策略 | +| 5 | Generate storage path | 使用 AD 规范，例如 `ab/cd/` | +| 6 | Insert or reuse blob | 按可信 hash + target policy 查找或插入 `file_blobs` | +| 7 | Insert file | 创建 `files` 并设置 `blob_id` | +| 8 | Increment/recalculate ref_count | 避免并发下计数漂移，最终统一重算 | +| 9 | Update storage_used | 按 AD 配额语义更新或最终统一重算 | + +### 对象复制模式 + +建议支持两种模式： + +| 模式 | 行为 | 适用场景 | +|---|---|---| +| `reuse_source_storage` | AD policy 指向原 bucket/path，不复制字节 | 快速切换、源存储长期保留、驱动完全兼容 | +| `copy_to_target_storage` | 流式读取源对象并写入 AD 新存储 | 真正脱离 Cloudreve、统一对象布局、计算真实 SHA-256 | + +当前实现属于 `reuse_source_storage`：保留 Cloudreve object key，并使用 opaque blob key，避免把非内容摘要伪装成 SHA-256。 + +## 6. 冲突处理 + +### 文件和目录重名 + +| 策略 | 行为 | +|---|---| +| fail | 停止运行，要求管理员清理目标或选择策略 | +| rename | 生成 `name (1).ext`、`name (2).ext` 等唯一名称 | +| skip | 不修改目标，mapping 标记 skipped | + +### 用户冲突 + +需要分别处理： + +- username 相同但 email 不同 +- email 相同但 username 不同 +- 已存在 mapping，但目标用户被修改 +- 多个源用户规范化后得到同一 username + +自动 rename username 时必须保存原 nick 到 `user_profiles.display_name` 和 mapping metadata。 + +### Blob 冲突 + +只有以下条件同时成立才可以自动复用： + +```text +可信内容 SHA-256 相同 +目标 policy 相同 +目标 size 相同 +目标对象可读且校验通过 +``` + +Cloudreve entity ID、object key、路径或 `policy+path+size` 摘要都不能当作真实内容哈希。 + +## 7. 断点续传与幂等 + +每个 batch 的安全顺序： + +```text +1. 读取 source page +2. 查询 object mapping +3. 转换并写入 AD +4. 提交目标事务 +5. upsert object mapping +6. 更新 stage cursor +``` + +如果目标写入、mapping 和 cursor 使用同一个 AD 数据库，建议放入同一事务。对象存储上传无法纳入数据库事务，因此需要： + +- 上传使用确定性临时 key 或 upload ID +- DB 提交失败时记录待清理对象 +- resume 时检测目标对象是否已存在并校验 +- finalize 或后台任务清理孤儿对象 + +幂等判断不能只依赖“目标表已有同名记录”，必须依赖 object mapping 和对象摘要。 + +## 8. 验证设计 + +### 数量校验 + +- 用户、策略组、目录、文件、分享数量 +- blob、历史版本、metadata 数量 +- skipped/renamed/reused 数量与计划一致 + +### 关系校验 + +- 每个 folder parent 存在 +- 每个 file folder/owner/blob 存在 +- 每个 file_version file/blob 存在 +- 每个 share 的 file/folder 和 owner 存在 +- 每个 object mapping 的目标对象存在 + +### 存储校验 + +- 对象存在 +- metadata size 与 DB size 一致 +- 随机抽样完整读取 +- 支持 range read 的策略执行范围读取 +- copy 模式校验 SHA-256 + +### 可重算校验 + +- `file_blobs.ref_count` 与实际引用数一致 +- `users.storage_used` 与 AD 配额算法计算结果一致 +- 没有未映射的目标对象或孤儿 blob + +### 迁移期间源变化检测 + +迁移前后重新读取 source inventory 或关键对象摘要。如果源仍在写入： + +- 标记 run 为 verification failed 或 source_changed +- 不宣称迁移完成 +- 要求冻结 Cloudreve 写入后执行最终同步 + +## 9. 当前实现与目标架构差距 + +| 能力 | 当前状态 | 目标状态 | +|---|---|---| +| Cloudreve/AD schema 预检 | 已实现 | 扩展版本、权限、capability 和存储连通性检查 | +| 用户、策略组、目录、文件、blob、版本、分享、metadata | 已实现基础迁移 | 改为 adapter/domain/writer 分层和分页 stage | +| Cloudreve v4 标签 | 已从 `tag:*` metadata 生成 AD `tags` 和 `system.tags` 文件/目录关联 | 增加跨批次持久化去重和冲突报告 | +| Direct links | 提供 AD `direct_link_secret` 时生成 v2 URL，并以 `cloudreve.direct_links` property 保存源 ID 映射 | 增加独立 JSON/CSV 报告和可选旧 URL 重定向层 | +| Cloudreve 任务 | 已全部写成 AD `system_runtime` 终态历史；活动任务归档为 canceled | 后续可增加独立 legacy task archive 表，避免占用运维任务列表 | +| 目标事务 | blobs/files 已按 page 提交；其他 stage 独立事务 | folders、metadata、shares 等 stage 继续分页，增加对象上传事务补偿 | +| ID mapping | blob/file 使用独立 object mapping 表；其他映射仍在 `context_json` | 将 folders、shares 等映射也迁入独立表 | +| 断点续传 | blobs/files 已支持 source ID cursor；其他 stage 为阶段级恢复 | folders 等 cursor、对象存储上传恢复和并行 worker checkpoint | +| 幂等重复运行 | 同一 run 会跳过已完成 stage；失败 stage 因事务回滚可安全重跑 | 不依赖 run checkpoint 的对象级 upsert/reuse 和完整重复运行 | +| 冲突策略 | 仅 username 自动后缀，其他冲突依赖 DB 报错 | fail/rename/skip/reuse/update 可配置 | +| 物理对象复制 | 未实现，当前复用源 storage path | 流式 copy、SHA-256、目标路径和校验 | +| 真正内容去重 | 未实现 | hash + policy 去重 | +| ref_count 重算 | 基于源关联计算初值 | 迁移后从 AD 关系统一重算 | +| storage_used 重算 | 当前复制 Cloudreve 用户统计 | 迁移后按 AD 语义统一重算 | +| 验证编排 | 提交后重查核心表数量、导入任务终态、标签绑定和直链 property；有 SQLite 端到端测试 | 增加 parent/owner/blob/ref_count/storage_used 和真实对象可读性验证 | +| 报告 | 已支持终端摘要和 `--report-path` JSON，包含分类跳过原因、核心 ID 映射、标签/直链记录和校验结果 | 增加失败中间态、stage 进度、吞吐量、字节数和独立 CSV 导出 | +| TUI | 未接入新迁移核心 | 可选进度视图 | +| 多源系统插件 | 未实现 | SourceAdapter 插件化 | + +## 10. 推荐实施阶段 + +### Phase 1：稳定当前一次性迁移 + +- 增加真实 Cloudreve 样本库测试 +- 覆盖 SQLite、MySQL、PostgreSQL +- 增加 orphan、cycle、重名和不兼容策略预检 +- 增加迁移后 ref_count/storage_used 验证 + +### Phase 2：领域模型和 Adapter/Writer 重构 + +- 定义 SourceAdapter、TargetWriter +- 引入 ExternalUser/Folder/File/Blob/Share +- 将 Cloudreve ORM 查询移动到 Cloudreve Adapter +- 将 AD ActiveModel 写入移动到 AsterDrive Writer + +### Phase 3：持久化运行状态 + +- 新增 migration runs、stage cursors 和 object map 表 +- 分页 stage runner +- resume 和幂等 upsert +- `migrate list/resume/report` + +### Phase 4：物理对象复制 + +- StorageResolver 和 BlobReader +- AsterDrive Storage Writer +- 流式 SHA-256 +- copy/reuse 两种模式 +- 孤儿对象清理 + +### Phase 5：验证和运维体验 + +- Verification Orchestrator +- Human/JSON report +- TUI progress +- 吞吐量、ETA、失败重试和人工冲突处理 + +## 11. 最终完成标准 + +一次迁移只有同时满足以下条件，才能标记为 completed： + +1. 所有必需 stages 完成并持久化 cursor。 +2. 所有迁移对象都有 created/reused/skipped/renamed 等明确 mapping 状态。 +3. 目标关系完整，没有孤儿 owner、parent、blob、version 或 share。 +4. copy 模式下对象大小和 SHA-256 校验通过。 +5. ref_count 和 storage_used 重算通过。 +6. 源在最终校验窗口内没有发生未处理变化。 +7. 所有 skipped 和 warning 都出现在最终报告中。 +8. 不存在包含明文密码、token、secret 或 access key 的日志和报告。 diff --git a/src/lib.rs b/src/lib.rs index ab5cfeb..a571eaa 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,2 +1 @@ -mod entities; -mod interfaces; +pub mod migration; diff --git a/src/main.rs b/src/main.rs index fcd38bd..1add820 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,8 +1,157 @@ -/// AsterDriveMigration is a tool for migrating data from other storage solution to AsterDrive. +use std::path::{Path, PathBuf}; -mod entities; -mod interfaces; +use aster_drive_migration::migration::{ + MigrationOptions, MigrationReport, inspect, migrate, write_json_report, +}; +use clap::{Parser, Subcommand}; +use color_eyre::eyre::{Result, bail}; -fn main() { - println!("Hello, world!"); +#[derive(Debug, Parser)] +#[command(name = "aster-drive-migration", version, about)] +struct Cli { + #[command(subcommand)] + command: Command, +} + +#[derive(Debug, Subcommand)] +enum Command { + Check(ConnectionArgs), + Migrate(MigrateArgs), +} + +#[derive(Debug, clap::Args)] +struct ConnectionArgs { + #[arg(long, env = "CLOUDREVE_DATABASE_URL")] + source_url: String, + #[arg(long, env = "ASTERDRIVE_DATABASE_URL")] + target_url: String, + #[arg(long)] + include_deleted: bool, + #[arg(long, value_name = "PATH")] + report_path: Option, +} + +#[derive(Debug, clap::Args)] +struct MigrateArgs { + #[command(flatten)] + connection: ConnectionArgs, + #[arg(long, env = "ASTER_MIGRATION_DEFAULT_PASSWORD")] + default_password: String, + #[arg(long, default_value = ".")] + local_base_path: String, + #[arg(long = "local-policy-root", value_name = "SOURCE_POLICY_ID=PATH")] + local_policy_roots: Vec, + #[arg(long)] + verify_local_storage: bool, + #[arg(long, env = "ASTER_DIRECT_LINK_SECRET", hide_env_values = true)] + direct_link_secret: Option, + #[arg(long)] + allow_non_empty_target: bool, + #[arg(long)] + skip_unsupported_policies: bool, + #[arg(long)] + dry_run: bool, + #[arg(long, value_name = "ID")] + run_id: Option, + #[arg(long, requires = "run_id")] + resume: bool, + #[arg(long, default_value_t = 500, value_name = "COUNT")] + blob_batch_size: usize, + #[arg(long, default_value_t = 500, value_name = "COUNT")] + file_batch_size: usize, +} + +#[tokio::main] +async fn main() -> Result<()> { + color_eyre::install()?; + match Cli::parse().command { + Command::Check(args) => { + let report = inspect(&args.source_url, &args.target_url, args.include_deleted).await?; + emit_report(args.report_path.as_deref(), &report)?; + } + Command::Migrate(args) => { + let report_path = args.connection.report_path.clone(); + let report = migrate(MigrationOptions { + source_url: args.connection.source_url, + target_url: args.connection.target_url, + default_password: args.default_password, + local_base_path: args.local_base_path, + local_policy_roots: parse_local_policy_roots(args.local_policy_roots)?, + verify_local_storage: args.verify_local_storage, + direct_link_secret: args.direct_link_secret, + include_deleted: args.connection.include_deleted, + allow_non_empty_target: args.allow_non_empty_target, + skip_unsupported_policies: args.skip_unsupported_policies, + dry_run: args.dry_run, + run_id: args.run_id, + resume: args.resume, + blob_batch_size: args.blob_batch_size, + file_batch_size: args.file_batch_size, + }) + .await?; + emit_report(report_path.as_deref(), &report)?; + } + } + Ok(()) +} + +fn parse_local_policy_roots( + values: Vec, +) -> Result> { + let mut roots = std::collections::BTreeMap::new(); + for value in values { + let Some((source_policy_id, path)) = value.split_once('=') else { + bail!("--local-policy-root must use SOURCE_POLICY_ID=PATH"); + }; + let source_policy_id = source_policy_id.parse::().map_err(|_| { + color_eyre::eyre::eyre!("invalid local storage policy ID {source_policy_id}") + })?; + if source_policy_id <= 0 || path.trim().is_empty() { + bail!("--local-policy-root must use a positive policy ID and non-empty path"); + } + if roots.insert(source_policy_id, path.to_string()).is_some() { + bail!("local storage policy {source_policy_id} was configured more than once"); + } + } + Ok(roots) +} + +fn emit_report(report_path: Option<&Path>, report: &MigrationReport) -> Result<()> { + if let Some(path) = report_path { + write_json_report(path, report)?; + } + println!("{report}"); + if report.validation.performed && !report.validation.passed { + bail!("migration committed but post-migration validation failed; inspect the JSON report"); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_local_policy_roots() -> Result<()> { + let roots = parse_local_policy_roots(vec![ + "1=D:/cloudreve-data".to_string(), + "2=E:/cloudreve-archive".to_string(), + ])?; + assert_eq!(roots.get(&1).map(String::as_str), Some("D:/cloudreve-data")); + assert_eq!( + roots.get(&2).map(String::as_str), + Some("E:/cloudreve-archive") + ); + Ok(()) + } + + #[test] + fn rejects_invalid_local_policy_roots() { + assert!(parse_local_policy_roots(vec!["missing-separator".to_string()]).is_err()); + assert!(parse_local_policy_roots(vec!["0=C:/data".to_string()]).is_err()); + assert!( + parse_local_policy_roots(vec!["1=C:/data".to_string(), "1=D:/data".to_string()]) + .is_err() + ); + } } diff --git a/src/migration/checkpoint.rs b/src/migration/checkpoint.rs new file mode 100644 index 0000000..e3abd3a --- /dev/null +++ b/src/migration/checkpoint.rs @@ -0,0 +1,350 @@ +use color_eyre::eyre::{Result, WrapErr, bail}; +use sea_orm::entity::prelude::*; +use sea_orm::{ActiveModelTrait, ConnectionTrait, EntityTrait, Schema, Set, Unchanged}; + +use super::{MigrationContext, MigrationReport, TargetCounts}; + +pub(super) mod stage_cursor { + use sea_orm::entity::prelude::*; + + #[sea_orm::model] + #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] + #[sea_orm(table_name = "aster_external_migration_stage_cursors")] + pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub run_id: String, + #[sea_orm(primary_key, auto_increment = false)] + pub stage: String, + pub cursor_value: i64, + pub processed_count: i64, + pub updated_at: DateTimeWithTimeZone, + } + + impl ActiveModelBehavior for ActiveModel {} +} + +pub(super) mod object_map { + use sea_orm::entity::prelude::*; + + #[sea_orm::model] + #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] + #[sea_orm(table_name = "aster_external_migration_object_map")] + pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub run_id: String, + #[sea_orm(primary_key, auto_increment = false)] + pub object_type: String, + #[sea_orm(primary_key, auto_increment = false)] + pub source_id: i64, + pub target_id: i64, + pub created_at: DateTimeWithTimeZone, + } + + impl ActiveModelBehavior for ActiveModel {} +} + +#[sea_orm::model] +#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] +#[sea_orm(table_name = "aster_external_migration_runs")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub id: String, + pub source_fingerprint: String, + pub target_fingerprint: String, + pub plan_fingerprint: String, + pub status: String, + pub last_completed_stage: Option, + #[sea_orm(column_type = "JsonBinary")] + pub context_json: Json, + #[sea_orm(column_type = "JsonBinary")] + pub report_json: Json, + #[sea_orm(column_type = "JsonBinary")] + pub baseline_json: Json, + #[sea_orm(column_type = "Text", nullable)] + pub last_error: Option, + pub created_at: DateTimeWithTimeZone, + pub updated_at: DateTimeWithTimeZone, +} + +impl ActiveModelBehavior for ActiveModel {} + +pub(super) struct LoadedCheckpoint { + pub status: String, + pub last_completed_stage: Option, + pub context: MigrationContext, + pub report: MigrationReport, + pub baseline: TargetCounts, +} + +pub(super) struct NewCheckpoint<'a> { + pub run_id: &'a str, + pub source_fingerprint: &'a str, + pub target_fingerprint: &'a str, + pub plan_fingerprint: &'a str, + pub context: &'a MigrationContext, + pub report: &'a MigrationReport, + pub baseline: &'a TargetCounts, +} + +pub(super) async fn ensure_table(db: &C) -> Result<()> { + let schema = Schema::new(db.get_database_backend()); + let mut statement = schema.create_table_from_entity(Entity); + statement.if_not_exists(); + db.execute(&statement) + .await + .wrap_err("create external migration checkpoint table")?; + let mut statement = schema.create_table_from_entity(stage_cursor::Entity); + statement.if_not_exists(); + db.execute(&statement) + .await + .wrap_err("create external migration stage cursor table")?; + let mut statement = schema.create_table_from_entity(object_map::Entity); + statement.if_not_exists(); + db.execute(&statement) + .await + .wrap_err("create external migration object map table")?; + Ok(()) +} + +pub(super) async fn load_stage_cursor( + db: &C, + run_id: &str, + stage: &str, +) -> Result> { + Ok( + stage_cursor::Entity::find_by_id((run_id.to_string(), stage.to_string())) + .one(db) + .await?, + ) +} + +pub(super) async fn save_stage_cursor( + db: &C, + run_id: &str, + stage: &str, + cursor_value: i64, + processed_count: i64, +) -> Result<()> { + use sea_orm::sea_query::OnConflict; + + stage_cursor::Entity::insert(stage_cursor::ActiveModel { + run_id: Set(run_id.to_string()), + stage: Set(stage.to_string()), + cursor_value: Set(cursor_value), + processed_count: Set(processed_count), + updated_at: Set(chrono::Utc::now().fixed_offset()), + }) + .on_conflict( + OnConflict::columns([stage_cursor::Column::RunId, stage_cursor::Column::Stage]) + .update_columns([ + stage_cursor::Column::CursorValue, + stage_cursor::Column::ProcessedCount, + stage_cursor::Column::UpdatedAt, + ]) + .to_owned(), + ) + .exec(db) + .await + .wrap_err_with(|| format!("save migration run {run_id} stage {stage} cursor"))?; + Ok(()) +} + +pub(super) async fn save_object_mappings( + db: &C, + run_id: &str, + object_type: &str, + mappings: &[(i64, i64)], +) -> Result<()> { + if mappings.is_empty() { + return Ok(()); + } + let now = chrono::Utc::now().fixed_offset(); + object_map::Entity::insert_many(mappings.iter().map(|(source_id, target_id)| { + object_map::ActiveModel { + run_id: Set(run_id.to_string()), + object_type: Set(object_type.to_string()), + source_id: Set(*source_id), + target_id: Set(*target_id), + created_at: Set(now), + } + })) + .exec(db) + .await + .wrap_err_with(|| format!("save migration run {run_id} {object_type} mappings"))?; + Ok(()) +} + +pub(super) async fn load_object_mappings( + db: &C, + run_id: &str, + object_type: &str, +) -> Result> { + use sea_orm::{ColumnTrait, QueryFilter}; + + Ok(object_map::Entity::find() + .filter(object_map::Column::RunId.eq(run_id)) + .filter(object_map::Column::ObjectType.eq(object_type)) + .all(db) + .await? + .into_iter() + .map(|mapping| (mapping.source_id, mapping.target_id)) + .collect()) +} + +pub(super) async fn create( + db: &C, + checkpoint: NewCheckpoint<'_>, +) -> Result<()> { + let NewCheckpoint { + run_id, + source_fingerprint, + target_fingerprint, + plan_fingerprint, + context, + report, + baseline, + } = checkpoint; + if Entity::find_by_id(run_id.to_string()) + .one(db) + .await? + .is_some() + { + bail!("migration run {run_id} already exists; use --resume or choose another --run-id"); + } + let now = chrono::Utc::now().fixed_offset(); + ActiveModel { + id: Set(run_id.to_string()), + source_fingerprint: Set(source_fingerprint.to_string()), + target_fingerprint: Set(target_fingerprint.to_string()), + plan_fingerprint: Set(plan_fingerprint.to_string()), + status: Set("running".to_string()), + last_completed_stage: Set(None), + context_json: Set(serde_json::to_value(context).wrap_err("serialize migration context")?), + report_json: Set(serde_json::to_value(report).wrap_err("serialize migration report")?), + baseline_json: Set(serde_json::to_value(baseline).wrap_err("serialize target baseline")?), + last_error: Set(None), + created_at: Set(now), + updated_at: Set(now), + } + .insert(db) + .await + .wrap_err_with(|| format!("create migration run {run_id}"))?; + Ok(()) +} + +pub(super) async fn load( + db: &C, + run_id: &str, + source_fingerprint: &str, + target_fingerprint: &str, + plan_fingerprint: &str, +) -> Result { + let model = Entity::find_by_id(run_id.to_string()) + .one(db) + .await? + .ok_or_else(|| color_eyre::eyre::eyre!("migration run {run_id} does not exist"))?; + if model.source_fingerprint != source_fingerprint { + bail!("migration run {run_id} belongs to a different Cloudreve source"); + } + if model.target_fingerprint != target_fingerprint { + bail!("migration run {run_id} belongs to a different AD target"); + } + if model.plan_fingerprint != plan_fingerprint { + bail!("migration run {run_id} options differ from the original migration plan"); + } + Ok(LoadedCheckpoint { + status: model.status, + last_completed_stage: model.last_completed_stage, + context: serde_json::from_value(model.context_json) + .wrap_err("decode checkpoint migration context")?, + report: serde_json::from_value(model.report_json) + .wrap_err("decode checkpoint migration report")?, + baseline: serde_json::from_value(model.baseline_json) + .wrap_err("decode checkpoint target baseline")?, + }) +} + +pub(super) async fn save_stage( + db: &C, + run_id: &str, + stage: &str, + context: &MigrationContext, + report: &MigrationReport, +) -> Result<()> { + ActiveModel { + id: Unchanged(run_id.to_string()), + status: Set("running".to_string()), + last_completed_stage: Set(Some(stage.to_string())), + context_json: Set(serde_json::to_value(context).wrap_err("serialize migration context")?), + report_json: Set(serde_json::to_value(report).wrap_err("serialize migration report")?), + last_error: Set(None), + updated_at: Set(chrono::Utc::now().fixed_offset()), + ..Default::default() + } + .update(db) + .await + .wrap_err_with(|| format!("save migration run {run_id} stage {stage}"))?; + Ok(()) +} + +pub(super) async fn save_progress( + db: &C, + run_id: &str, + context: &MigrationContext, + report: &MigrationReport, +) -> Result<()> { + ActiveModel { + id: Unchanged(run_id.to_string()), + status: Set("running".to_string()), + context_json: Set(serde_json::to_value(context).wrap_err("serialize migration context")?), + report_json: Set(serde_json::to_value(report).wrap_err("serialize migration report")?), + last_error: Set(None), + updated_at: Set(chrono::Utc::now().fixed_offset()), + ..Default::default() + } + .update(db) + .await + .wrap_err_with(|| format!("save migration run {run_id} progress"))?; + Ok(()) +} + +pub(super) async fn finish( + db: &C, + run_id: &str, + status: &str, + context: &MigrationContext, + report: &MigrationReport, +) -> Result<()> { + ActiveModel { + id: Unchanged(run_id.to_string()), + status: Set(status.to_string()), + context_json: Set(serde_json::to_value(context).wrap_err("serialize migration context")?), + report_json: Set(serde_json::to_value(report).wrap_err("serialize migration report")?), + last_error: Set(None), + updated_at: Set(chrono::Utc::now().fixed_offset()), + ..Default::default() + } + .update(db) + .await + .wrap_err_with(|| format!("finish migration run {run_id}"))?; + Ok(()) +} + +pub(super) async fn mark_failed( + db: &C, + run_id: &str, + error: &str, +) -> Result<()> { + let error = error.chars().take(2048).collect::(); + ActiveModel { + id: Unchanged(run_id.to_string()), + status: Set("failed".to_string()), + last_error: Set(Some(error)), + updated_at: Set(chrono::Utc::now().fixed_offset()), + ..Default::default() + } + .update(db) + .await + .wrap_err_with(|| format!("mark migration run {run_id} failed"))?; + Ok(()) +} diff --git a/src/migration/mod.rs b/src/migration/mod.rs new file mode 100644 index 0000000..89d2ea2 --- /dev/null +++ b/src/migration/mod.rs @@ -0,0 +1,3131 @@ +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::fmt::{self, Write}; +use std::path::Path; + +use argon2::Argon2; +use argon2::password_hash::{PasswordHasher, SaltString}; +use base64::Engine; +use color_eyre::eyre::{Result, WrapErr, bail}; +use hmac::{Hmac, Mac}; +use sea_orm::{ + ActiveModelTrait, ColumnTrait, Database, DatabaseConnection, EntityTrait, PaginatorTrait, + QueryFilter, QueryOrder, QuerySelect, Set, TransactionTrait, +}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; + +use asterdrive_entities as ad; +use cloudreve_entities as cr; + +#[derive(Debug, Clone)] +pub struct MigrationOptions { + pub source_url: String, + pub target_url: String, + pub default_password: String, + pub local_base_path: String, + pub local_policy_roots: BTreeMap, + pub verify_local_storage: bool, + pub direct_link_secret: Option, + pub include_deleted: bool, + pub allow_non_empty_target: bool, + pub skip_unsupported_policies: bool, + pub dry_run: bool, + pub run_id: Option, + pub resume: bool, + pub blob_batch_size: usize, + pub file_batch_size: usize, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MigrationReport { + pub schema_version: u32, + pub generated_at: chrono::DateTime, + pub source_users: u64, + pub source_groups: u64, + pub source_policies: u64, + pub source_folders: u64, + pub source_files: u64, + pub source_entities: u64, + pub source_shares: u64, + pub source_direct_links: u64, + pub source_tag_assignments: u64, + pub source_tasks: u64, + pub migrated_users: usize, + pub migrated_policy_groups: usize, + pub migrated_policies: usize, + pub migrated_folders: usize, + pub migrated_files: usize, + pub migrated_blobs: usize, + pub migrated_versions: usize, + pub migrated_shares: usize, + pub migrated_properties: usize, + pub migrated_tags: usize, + pub migrated_tag_assignments: usize, + pub migrated_direct_links: usize, + pub migrated_tasks: usize, + pub skipped: usize, + pub dry_run: bool, + pub warnings: Vec, + pub skipped_by_type: BTreeMap, + pub skipped_objects: Vec, + pub mappings: MigrationMappings, + pub direct_links: Vec, + pub tag_assignments: Vec, + pub validation: MigrationValidation, + pub run_id: Option, + pub resumed: bool, + pub completed_stages: Vec, +} + +impl Default for MigrationReport { + fn default() -> Self { + Self { + schema_version: 1, + generated_at: chrono::Utc::now(), + source_users: 0, + source_groups: 0, + source_policies: 0, + source_folders: 0, + source_files: 0, + source_entities: 0, + source_shares: 0, + source_direct_links: 0, + source_tag_assignments: 0, + source_tasks: 0, + migrated_users: 0, + migrated_policy_groups: 0, + migrated_policies: 0, + migrated_folders: 0, + migrated_files: 0, + migrated_blobs: 0, + migrated_versions: 0, + migrated_shares: 0, + migrated_properties: 0, + migrated_tags: 0, + migrated_tag_assignments: 0, + migrated_direct_links: 0, + migrated_tasks: 0, + skipped: 0, + dry_run: true, + warnings: Vec::new(), + skipped_by_type: BTreeMap::new(), + skipped_objects: Vec::new(), + mappings: MigrationMappings::default(), + direct_links: Vec::new(), + tag_assignments: Vec::new(), + validation: MigrationValidation::default(), + run_id: None, + resumed: false, + completed_stages: Vec::new(), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SkippedObject { + pub object_type: String, + pub source_id: Option, + pub reason: String, +} + +#[derive(Debug, Default, Clone, Serialize, Deserialize)] +pub struct MigrationMappings { + pub policies: Vec, + pub policy_groups: Vec, + pub users: Vec, + pub folders: Vec, + pub blobs: Vec, + pub files: Vec, + pub shares: Vec, + pub tasks: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct IdMapping { + pub source_id: i64, + pub target_id: i64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DirectLinkReport { + pub source_direct_link_id: i64, + pub source_file_id: i64, + pub target_file_id: i64, + pub source_name: String, + pub source_downloads: i64, + pub source_speed_limit: i64, + pub url: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TagAssignmentReport { + pub source_metadata_id: i64, + pub source_entity_id: i64, + pub target_entity_type: String, + pub target_entity_id: i64, + pub target_tag_id: i64, + pub tag_name: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MigrationValidation { + pub performed: bool, + pub passed: bool, + pub checks: Vec, +} + +impl Default for MigrationValidation { + fn default() -> Self { + Self { + performed: false, + passed: true, + checks: Vec::new(), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ValidationCheck { + pub name: String, + pub passed: bool, + pub expected: String, + pub actual: String, + pub message: Option, +} + +impl MigrationReport { + fn record_skip( + &mut self, + object_type: &str, + source_id: Option, + reason: impl Into, + ) { + self.skipped += 1; + *self + .skipped_by_type + .entry(object_type.to_string()) + .or_default() += 1; + self.skipped_objects.push(SkippedObject { + object_type: object_type.to_string(), + source_id, + reason: reason.into(), + }); + } + + fn set_mappings( + &mut self, + context: &MigrationContext, + blobs: &HashMap, + files: &HashMap, + ) { + self.mappings = MigrationMappings { + policies: sorted_id_mappings(&context.policies), + policy_groups: sorted_id_mappings(&context.policy_groups), + users: sorted_id_mappings(&context.users), + folders: sorted_id_mappings(&context.folders), + blobs: sorted_id_mappings(blobs), + files: sorted_id_mappings(files), + shares: sorted_id_mappings(&context.shares), + tasks: sorted_id_mappings(&context.tasks), + }; + } +} + +impl fmt::Display for MigrationReport { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut output = String::new(); + writeln!(output, "Cloudreve -> AsterDrive migration report")?; + if let Some(run_id) = &self.run_id { + writeln!( + output, + "run: {run_id}{}", + if self.resumed { " (resumed)" } else { "" } + )?; + } + writeln!( + output, + "source: users={}, groups={}, policies={}, folders={}, files={}, entities={}, shares={}, direct_links={}, tag_assignments={}, tasks={}", + self.source_users, + self.source_groups, + self.source_policies, + self.source_folders, + self.source_files, + self.source_entities, + self.source_shares, + self.source_direct_links, + self.source_tag_assignments, + self.source_tasks + )?; + if self.dry_run { + writeln!(output, "mode: dry-run (target was not modified)")?; + } else { + writeln!( + output, + "migrated: users={}, policy_groups={}, policies={}, folders={}, files={}, blobs={}, versions={}, shares={}, properties={}, tags={}, tag_assignments={}, direct_links={}, archived_tasks={}", + self.migrated_users, + self.migrated_policy_groups, + self.migrated_policies, + self.migrated_folders, + self.migrated_files, + self.migrated_blobs, + self.migrated_versions, + self.migrated_shares, + self.migrated_properties, + self.migrated_tags, + self.migrated_tag_assignments, + self.migrated_direct_links, + self.migrated_tasks + )?; + writeln!(output, "skipped: {}", self.skipped)?; + if !self.skipped_by_type.is_empty() { + let categories = self + .skipped_by_type + .iter() + .map(|(object_type, count)| format!("{object_type}={count}")) + .collect::>() + .join(", "); + writeln!(output, "skipped_by_type: {categories}")?; + } + if self.validation.performed { + writeln!( + output, + "validation: {} ({} checks)", + if self.validation.passed { + "passed" + } else { + "failed" + }, + self.validation.checks.len() + )?; + } + if !self.completed_stages.is_empty() { + writeln!( + output, + "completed_stages: {}", + self.completed_stages.join(",") + )?; + } + } + if !self.warnings.is_empty() { + writeln!(output, "warnings:")?; + for warning in &self.warnings { + writeln!(output, "- {warning}")?; + } + } + formatter.write_str(output.trim_end()) + } +} + +pub fn write_json_report(path: impl AsRef, report: &MigrationReport) -> Result<()> { + let path = path.as_ref(); + if let Some(parent) = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + std::fs::create_dir_all(parent) + .wrap_err_with(|| format!("create report directory {}", parent.display()))?; + } + let contents = serde_json::to_vec_pretty(report).wrap_err("serialize migration report")?; + std::fs::write(path, contents) + .wrap_err_with(|| format!("write migration report {}", path.display())) +} + +pub async fn inspect( + source_url: &str, + target_url: &str, + include_deleted: bool, +) -> Result { + let source = connect(source_url, "Cloudreve").await?; + let target = connect(target_url, "AsterDrive").await?; + let source_data = SourceData::load(&source, include_deleted).await?; + validate_target_schema(&target).await?; + let mut report = source_data.report(); + report.dry_run = true; + report.warnings.extend(source_data.compatibility_warnings()); + Ok(report) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum MigrationStage { + Policies, + PolicyGroups, + Users, + Folders, + Blobs, + Files, + Metadata, + Shares, + DirectLinks, + Tasks, +} + +impl MigrationStage { + const ALL: [Self; 10] = [ + Self::Policies, + Self::PolicyGroups, + Self::Users, + Self::Folders, + Self::Blobs, + Self::Files, + Self::Metadata, + Self::Shares, + Self::DirectLinks, + Self::Tasks, + ]; + + const fn as_str(self) -> &'static str { + match self { + Self::Policies => "policies", + Self::PolicyGroups => "policy_groups", + Self::Users => "users", + Self::Folders => "folders", + Self::Blobs => "blobs", + Self::Files => "files", + Self::Metadata => "metadata", + Self::Shares => "shares", + Self::DirectLinks => "direct_links", + Self::Tasks => "tasks", + } + } + + fn should_run_after(self, last_completed_stage: Option<&str>) -> Result { + let Some(last_completed_stage) = last_completed_stage else { + return Ok(true); + }; + let last_index = Self::ALL + .iter() + .position(|stage| stage.as_str() == last_completed_stage) + .ok_or_else(|| { + color_eyre::eyre::eyre!("checkpoint contains unknown stage {last_completed_stage}") + })?; + let current_index = Self::ALL + .iter() + .position(|stage| *stage == self) + .expect("migration stage must exist in ALL"); + Ok(current_index > last_index) + } +} + +pub async fn migrate(options: MigrationOptions) -> Result { + if options.default_password.chars().count() < 8 { + bail!("--default-password must contain at least 8 characters"); + } + if options + .direct_link_secret + .as_deref() + .is_some_and(|secret| secret.chars().count() < 16) + { + bail!("--direct-link-secret must contain at least 16 characters"); + } + if options.resume && options.run_id.is_none() { + bail!("--resume requires --run-id"); + } + if options.resume && options.dry_run { + bail!("--resume cannot be combined with --dry-run"); + } + if let Some(run_id) = options.run_id.as_deref() { + validate_run_id(run_id)?; + } + if !(1..=10_000).contains(&options.blob_batch_size) { + bail!("--blob-batch-size must be between 1 and 10000"); + } + if !(1..=10_000).contains(&options.file_batch_size) { + bail!("--file-batch-size must be between 1 and 10000"); + } + + let source = connect(&options.source_url, "Cloudreve").await?; + let target = connect(&options.target_url, "AsterDrive").await?; + let source_data = SourceData::load(&source, options.include_deleted).await?; + validate_target_schema(&target).await?; + + let unsupported = source_data.unsupported_policy_types(); + if !unsupported.is_empty() && !options.skip_unsupported_policies { + bail!( + "unsupported Cloudreve storage policy types: {}; rerun with --skip-unsupported-policies to omit their files", + unsupported.join(", ") + ); + } + validate_local_policy_roots(&source_data, &options)?; + if options.verify_local_storage { + verify_local_storage_roots(&source_data, &options)?; + if options.dry_run { + verify_all_local_source_objects(&source, &source_data, &options).await?; + } + } + + let mut report = source_data.report(); + report.dry_run = options.dry_run; + report.warnings.extend(source_data.compatibility_warnings()); + if options.dry_run { + report.run_id = options.run_id.clone(); + return Ok(report); + } + + checkpoint::ensure_table(&target).await?; + let run_id = options + .run_id + .clone() + .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()); + let source_fingerprint = source_fingerprint(&options.source_url, &source_data); + let target_fingerprint = hash_fingerprint(&options.target_url); + let plan_fingerprint = plan_fingerprint(&options); + + let (mut context, target_before, mut last_completed_stage) = if options.resume { + let loaded = checkpoint::load( + &target, + &run_id, + &source_fingerprint, + &target_fingerprint, + &plan_fingerprint, + ) + .await?; + if !matches!( + loaded.status.as_str(), + "running" | "failed" | "validation_failed" | "completed" + ) { + bail!( + "migration run {run_id} has unsupported status {}", + loaded.status + ); + } + report = loaded.report; + report.resumed = true; + report.run_id = Some(run_id.clone()); + (loaded.context, loaded.baseline, loaded.last_completed_stage) + } else { + ensure_target_safe(&target, options.allow_non_empty_target).await?; + let baseline = TargetCounts::load(&target).await?; + report.run_id = Some(run_id.clone()); + let context = MigrationContext::default(); + checkpoint::create( + &target, + checkpoint::NewCheckpoint { + run_id: &run_id, + source_fingerprint: &source_fingerprint, + target_fingerprint: &target_fingerprint, + plan_fingerprint: &plan_fingerprint, + context: &context, + report: &report, + baseline: &baseline, + }, + ) + .await?; + (context, baseline, None) + }; + let mut blob_mappings = checkpoint::load_object_mappings(&target, &run_id, "blob").await?; + if blob_mappings.is_empty() && !context.blobs.is_empty() { + let legacy_mappings = context + .blobs + .iter() + .map(|(source_id, target_id)| (*source_id, *target_id)) + .collect::>(); + checkpoint::save_object_mappings(&target, &run_id, "blob", &legacy_mappings).await?; + blob_mappings.extend(legacy_mappings); + context.blobs.clear(); + } + if report.migrated_blobs != blob_mappings.len() { + bail!( + "migration run {run_id} blob mapping count {} does not match migrated blob count {}; restore the checkpoint tables before resuming", + blob_mappings.len(), + report.migrated_blobs + ); + } + let mut file_mappings = checkpoint::load_object_mappings(&target, &run_id, "file").await?; + if file_mappings.is_empty() && !context.files.is_empty() { + let legacy_mappings = context + .files + .iter() + .map(|(source_id, target_id)| (*source_id, *target_id)) + .collect::>(); + checkpoint::save_object_mappings(&target, &run_id, "file", &legacy_mappings).await?; + file_mappings.extend(legacy_mappings); + context.files.clear(); + } + if report.migrated_files != file_mappings.len() { + bail!( + "migration run {run_id} file mapping count {} does not match migrated file count {}; restore the checkpoint tables before resuming", + file_mappings.len(), + report.migrated_files + ); + } + + let password_hash = hash_password(&options.default_password)?; + + for stage in MigrationStage::ALL { + if !stage.should_run_after(last_completed_stage.as_deref())? { + continue; + } + if stage == MigrationStage::Blobs { + let inputs = BlobBatchInputs { + source: &source, + target: &target, + run_id: &run_id, + source_data: &source_data, + options: &options, + context: &context, + file_mappings: &file_mappings, + }; + if let Err(error) = + migrate_blobs_batched(&inputs, &mut blob_mappings, &mut report).await + { + let _ = checkpoint::mark_failed(&target, &run_id, &error.to_string()).await; + return Err(error).wrap_err_with(|| { + format!( + "migration run {run_id} failed at stage {}; rerun with --resume --run-id {run_id}", + stage.as_str() + ) + }); + } + last_completed_stage = Some(stage.as_str().to_string()); + continue; + } + if stage == MigrationStage::Files { + let inputs = FileBatchInputs { + source: &source, + target: &target, + run_id: &run_id, + source_data: &source_data, + options: &options, + context: &context, + blob_mappings: &blob_mappings, + }; + if let Err(error) = + migrate_files_batched(&inputs, &mut file_mappings, &mut report).await + { + let _ = checkpoint::mark_failed(&target, &run_id, &error.to_string()).await; + return Err(error).wrap_err_with(|| { + format!( + "migration run {run_id} failed at stage {}; rerun with --resume --run-id {run_id}", + stage.as_str() + ) + }); + } + last_completed_stage = Some(stage.as_str().to_string()); + continue; + } + let transaction = target + .begin() + .await + .wrap_err_with(|| format!("begin migration stage {}", stage.as_str()))?; + let stage_result: Result<()> = async { + let inputs = StageInputs { + source_db: &source, + source_data: &source_data, + options: &options, + password_hash: &password_hash, + file_mappings: &file_mappings, + }; + execute_stage(stage, &transaction, &inputs, &mut context, &mut report).await?; + report.set_mappings(&context, &blob_mappings, &file_mappings); + if !report + .completed_stages + .iter() + .any(|completed| completed == stage.as_str()) + { + report.completed_stages.push(stage.as_str().to_string()); + } + checkpoint::save_stage(&transaction, &run_id, stage.as_str(), &context, &report).await + } + .await; + if let Err(error) = stage_result { + drop(transaction); + let _ = checkpoint::mark_failed(&target, &run_id, &error.to_string()).await; + return Err(error).wrap_err_with(|| { + format!( + "migration run {run_id} failed at stage {}; rerun with --resume --run-id {run_id}", + stage.as_str() + ) + }); + } + if let Err(error) = transaction.commit().await { + let _ = checkpoint::mark_failed(&target, &run_id, &error.to_string()).await; + return Err(error).wrap_err_with(|| { + format!( + "commit migration run {run_id} stage {}; resume with the same run ID", + stage.as_str() + ) + }); + } + last_completed_stage = Some(stage.as_str().to_string()); + } + + report.set_mappings(&context, &blob_mappings, &file_mappings); + report.validation = match validate_migration_result(&target, &target_before, &report).await { + Ok(validation) => validation, + Err(error) => { + let _ = checkpoint::mark_failed(&target, &run_id, &error.to_string()).await; + return Err(error) + .wrap_err_with(|| format!("validate completed migration run {run_id}")); + } + }; + report.generated_at = chrono::Utc::now(); + checkpoint::finish( + &target, + &run_id, + if report.validation.passed { + "completed" + } else { + "validation_failed" + }, + &context, + &report, + ) + .await?; + Ok(report) +} + +struct StageInputs<'a> { + source_db: &'a DatabaseConnection, + source_data: &'a SourceData, + options: &'a MigrationOptions, + password_hash: &'a str, + file_mappings: &'a HashMap, +} + +async fn execute_stage( + stage: MigrationStage, + transaction: &sea_orm::DatabaseTransaction, + inputs: &StageInputs<'_>, + context: &mut MigrationContext, + report: &mut MigrationReport, +) -> Result<()> { + match stage { + MigrationStage::Policies => { + migrate_policies( + transaction, + inputs.source_data, + inputs.options, + context, + report, + ) + .await + } + MigrationStage::PolicyGroups => { + migrate_policy_groups(transaction, inputs.source_data, context, report).await + } + MigrationStage::Users => { + migrate_users( + transaction, + inputs.source_data, + inputs.password_hash, + context, + report, + ) + .await + } + MigrationStage::Folders => { + migrate_folders(transaction, inputs.source_data, context, report).await + } + MigrationStage::Blobs => bail!("blobs stage must use the batched runner"), + MigrationStage::Files => bail!("files stage must use the batched runner"), + MigrationStage::Metadata => { + migrate_metadata( + transaction, + inputs.source_db, + inputs.source_data, + inputs.file_mappings, + context, + report, + ) + .await + } + MigrationStage::Shares => { + migrate_shares( + transaction, + inputs.source_db, + inputs.source_data, + inputs.file_mappings, + context, + report, + ) + .await + } + MigrationStage::DirectLinks => { + migrate_direct_links( + transaction, + inputs.source_db, + inputs.source_data, + inputs.file_mappings, + context, + inputs.options.direct_link_secret.as_deref(), + report, + ) + .await + } + MigrationStage::Tasks => { + migrate_tasks(transaction, inputs.source_data, context, report).await + } + } +} + +struct BlobBatchInputs<'a> { + source: &'a DatabaseConnection, + target: &'a DatabaseConnection, + run_id: &'a str, + source_data: &'a SourceData, + options: &'a MigrationOptions, + context: &'a MigrationContext, + file_mappings: &'a HashMap, +} + +async fn migrate_blobs_batched( + inputs: &BlobBatchInputs<'_>, + blob_mappings: &mut HashMap, + report: &mut MigrationReport, +) -> Result<()> { + let cursor = + checkpoint::load_stage_cursor(inputs.target, inputs.run_id, MigrationStage::Blobs.as_str()) + .await?; + let mut cursor_value = cursor.as_ref().map_or(0, |cursor| cursor.cursor_value); + let mut processed_count = cursor.as_ref().map_or(0, |cursor| cursor.processed_count); + + loop { + let mut query = cr::entities::Entity::find() + .filter(cr::entities::Column::Type.eq(0)) + .filter(cr::entities::Column::Id.gt(cursor_value)) + .order_by_asc(cr::entities::Column::Id) + .limit(inputs.options.blob_batch_size as u64); + if !inputs.source_data.include_deleted { + query = query.filter(cr::entities::Column::DeletedAt.is_null()); + } + let entities = query.all(inputs.source).await?; + if entities.is_empty() { + let transaction = inputs + .target + .begin() + .await + .wrap_err("begin blobs completion")?; + report.set_mappings(inputs.context, blob_mappings, inputs.file_mappings); + if !report + .completed_stages + .iter() + .any(|completed| completed == MigrationStage::Blobs.as_str()) + { + report + .completed_stages + .push(MigrationStage::Blobs.as_str().to_string()); + } + checkpoint::save_stage( + &transaction, + inputs.run_id, + MigrationStage::Blobs.as_str(), + inputs.context, + report, + ) + .await?; + transaction + .commit() + .await + .wrap_err("commit blobs completion")?; + return Ok(()); + } + + if inputs.options.verify_local_storage { + verify_local_blob_batch( + &entities, + inputs.source_data, + inputs.options, + inputs.context, + )?; + } + + let entity_ids = entities.iter().map(|entity| entity.id).collect::>(); + let association_info = load_blob_association_info(inputs.source, &entity_ids).await?; + let last_entity_id = entities.last().expect("non-empty blob batch").id; + let transaction = inputs.target.begin().await.wrap_err("begin blobs batch")?; + let mappings = migrate_blob_batch( + &transaction, + &entities, + &association_info.reference_counts, + &association_info.thumbnail_paths, + inputs.context, + report, + ) + .await?; + checkpoint::save_object_mappings(&transaction, inputs.run_id, "blob", &mappings).await?; + processed_count += entities.len() as i64; + checkpoint::save_stage_cursor( + &transaction, + inputs.run_id, + MigrationStage::Blobs.as_str(), + last_entity_id, + processed_count, + ) + .await?; + checkpoint::save_progress(&transaction, inputs.run_id, inputs.context, report).await?; + transaction.commit().await.wrap_err("commit blobs batch")?; + blob_mappings.extend(mappings); + cursor_value = last_entity_id; + } +} + +struct FileBatchInputs<'a> { + source: &'a DatabaseConnection, + target: &'a DatabaseConnection, + run_id: &'a str, + source_data: &'a SourceData, + options: &'a MigrationOptions, + context: &'a MigrationContext, + blob_mappings: &'a HashMap, +} + +async fn migrate_files_batched( + inputs: &FileBatchInputs<'_>, + file_mappings: &mut HashMap, + report: &mut MigrationReport, +) -> Result<()> { + let cursor = + checkpoint::load_stage_cursor(inputs.target, inputs.run_id, MigrationStage::Files.as_str()) + .await?; + let mut cursor_value = cursor.as_ref().map_or(0, |cursor| cursor.cursor_value); + let mut processed_count = cursor.as_ref().map_or(0, |cursor| cursor.processed_count); + + loop { + let files = cr::files::Entity::find() + .filter(cr::files::Column::Type.eq(0)) + .filter(cr::files::Column::Id.gt(cursor_value)) + .order_by_asc(cr::files::Column::Id) + .limit(inputs.options.file_batch_size as u64) + .all(inputs.source) + .await?; + if files.is_empty() { + let transaction = inputs + .target + .begin() + .await + .wrap_err("begin files completion")?; + report.set_mappings(inputs.context, inputs.blob_mappings, file_mappings); + if !report + .completed_stages + .iter() + .any(|completed| completed == MigrationStage::Files.as_str()) + { + report + .completed_stages + .push(MigrationStage::Files.as_str().to_string()); + } + checkpoint::save_stage( + &transaction, + inputs.run_id, + MigrationStage::Files.as_str(), + inputs.context, + report, + ) + .await?; + transaction + .commit() + .await + .wrap_err("commit files completion")?; + return Ok(()); + } + + let (associations, entities) = + load_file_batch_data(inputs.source, inputs.source_data.include_deleted, &files).await?; + let last_file_id = files.last().expect("non-empty file batch").id; + let transaction = inputs.target.begin().await.wrap_err("begin files batch")?; + let mappings = migrate_file_batch( + &transaction, + &files, + &associations, + &entities, + inputs.blob_mappings, + inputs.context, + report, + ) + .await?; + checkpoint::save_object_mappings(&transaction, inputs.run_id, "file", &mappings).await?; + processed_count += files.len() as i64; + checkpoint::save_stage_cursor( + &transaction, + inputs.run_id, + MigrationStage::Files.as_str(), + last_file_id, + processed_count, + ) + .await?; + checkpoint::save_progress(&transaction, inputs.run_id, inputs.context, report).await?; + transaction.commit().await.wrap_err("commit files batch")?; + file_mappings.extend(mappings); + cursor_value = last_file_id; + } +} + +async fn load_file_batch_data( + source: &DatabaseConnection, + include_deleted: bool, + files: &[cr::files::Model], +) -> Result<(HashMap>, HashMap)> { + const QUERY_ID_BATCH_SIZE: usize = 500; + + let file_ids = files.iter().map(|file| file.id).collect::>(); + let mut file_entities = Vec::new(); + for file_ids in file_ids.chunks(QUERY_ID_BATCH_SIZE) { + file_entities.extend( + cr::file_entities::Entity::find() + .filter(cr::file_entities::Column::FileId.is_in(file_ids.iter().copied())) + .all(source) + .await?, + ); + } + let mut entity_ids = file_entities + .iter() + .map(|relation| relation.entity_id) + .collect::>(); + entity_ids.extend(files.iter().filter_map(|file| file.primary_entity)); + let entity_ids = entity_ids.into_iter().collect::>(); + let mut entities = HashMap::with_capacity(entity_ids.len()); + for entity_ids in entity_ids.chunks(QUERY_ID_BATCH_SIZE) { + let mut query = cr::entities::Entity::find() + .filter(cr::entities::Column::Id.is_in(entity_ids.iter().copied())) + .filter(cr::entities::Column::Type.eq(0)); + if !include_deleted { + query = query.filter(cr::entities::Column::DeletedAt.is_null()); + } + for entity in query.all(source).await? { + entities.insert(entity.id, entity); + } + } + Ok((associations(files, &file_entities), entities)) +} + +struct BlobAssociationInfo { + reference_counts: HashMap, + thumbnail_paths: HashMap, +} + +async fn load_blob_association_info( + source: &DatabaseConnection, + blob_ids: &[i64], +) -> Result { + if blob_ids.is_empty() { + return Ok(BlobAssociationInfo { + reference_counts: HashMap::new(), + thumbnail_paths: HashMap::new(), + }); + } + let relations = cr::file_entities::Entity::find() + .filter(cr::file_entities::Column::EntityId.is_in(blob_ids.iter().copied())) + .all(source) + .await?; + let primary_files = cr::files::Entity::find() + .filter(cr::files::Column::PrimaryEntity.is_in(blob_ids.iter().copied())) + .all(source) + .await?; + let file_ids = relations + .iter() + .map(|relation| relation.file_id) + .chain(primary_files.iter().map(|file| file.id)) + .collect::>(); + if file_ids.is_empty() { + return Ok(BlobAssociationInfo { + reference_counts: HashMap::new(), + thumbnail_paths: HashMap::new(), + }); + } + let all_relations = cr::file_entities::Entity::find() + .filter(cr::file_entities::Column::FileId.is_in(file_ids.iter().copied())) + .all(source) + .await?; + let thumbnail_ids = all_relations + .iter() + .map(|relation| relation.entity_id) + .collect::>(); + let thumbnails = cr::entities::Entity::find() + .filter(cr::entities::Column::Id.is_in(thumbnail_ids.iter().copied())) + .filter(cr::entities::Column::Type.eq(1)) + .all(source) + .await? + .into_iter() + .map(|entity| (entity.id, entity.source)) + .collect::>(); + let relations_by_file = + all_relations + .into_iter() + .fold(HashMap::>::new(), |mut values, relation| { + values + .entry(relation.file_id) + .or_default() + .push(relation.entity_id); + values + }); + let blob_id_set = blob_ids.iter().copied().collect::>(); + let mut file_blob_pairs = HashSet::new(); + let mut thumbnail_paths = HashMap::new(); + for (blob_id, file_id) in relations + .into_iter() + .map(|relation| (relation.entity_id, relation.file_id)) + .chain( + primary_files + .into_iter() + .filter_map(|file| file.primary_entity.map(|entity_id| (entity_id, file.id))), + ) + { + if blob_id_set.contains(&blob_id) { + file_blob_pairs.insert((file_id, blob_id)); + } + if let Some(path) = relations_by_file + .get(&file_id) + .into_iter() + .flatten() + .find_map(|entity_id| thumbnails.get(entity_id)) + { + thumbnail_paths + .entry(blob_id) + .or_insert_with(|| path.clone()); + } + } + let mut reference_counts = HashMap::new(); + for (_, blob_id) in file_blob_pairs { + *reference_counts.entry(blob_id).or_insert(0) += 1; + } + Ok(BlobAssociationInfo { + reference_counts, + thumbnail_paths, + }) +} + +fn validate_run_id(run_id: &str) -> Result<()> { + let run_id = run_id.trim(); + if run_id.is_empty() || run_id.chars().count() > 128 || run_id.chars().any(char::is_control) { + bail!("--run-id must contain 1-128 non-control characters"); + } + Ok(()) +} + +fn hash_fingerprint(value: &str) -> String { + format!("{:x}", Sha256::digest(value.as_bytes())) +} + +fn source_fingerprint(source_url: &str, source: &SourceData) -> String { + hash_fingerprint(&format!( + "{source_url}|{}|{}|{}|{}|{}|{}|{}|{}|{}|{}", + source.users.len(), + source.groups.len(), + source.policies.len(), + source.source_file_records, + source.source_entities, + source.source_file_entities, + source.shares.len(), + source.metadata.len(), + source.direct_links.len(), + source.tasks.len(), + )) +} + +fn plan_fingerprint(options: &MigrationOptions) -> String { + hash_fingerprint(&format!( + "{}|{}|{}|{}|{}|{}|{}|{}", + options.local_base_path, + options + .local_policy_roots + .iter() + .map(|(policy_id, path)| format!("{policy_id}={path}")) + .collect::>() + .join("|"), + options.verify_local_storage, + options.include_deleted, + options.allow_non_empty_target, + options.skip_unsupported_policies, + hash_fingerprint(&options.default_password), + options + .direct_link_secret + .as_deref() + .map(hash_fingerprint) + .unwrap_or_default(), + )) +} + +fn local_policy_root(options: &MigrationOptions, source_policy_id: i64) -> &str { + options + .local_policy_roots + .get(&source_policy_id) + .map(String::as_str) + .unwrap_or(&options.local_base_path) +} + +fn local_storage_path(root: &str, storage_path: &str) -> std::path::PathBuf { + let storage_path = std::path::Path::new(storage_path); + if storage_path.is_absolute() { + storage_path.to_path_buf() + } else { + std::path::Path::new(root).join(storage_path) + } +} + +fn validate_local_policy_roots(source: &SourceData, options: &MigrationOptions) -> Result<()> { + let policies = source + .policies + .iter() + .map(|policy| (policy.id, policy)) + .collect::>(); + for source_policy_id in options.local_policy_roots.keys() { + let Some(policy) = policies.get(source_policy_id) else { + bail!( + "--local-policy-root references Cloudreve policy {source_policy_id}, which was not found" + ); + }; + if policy.r#type != "local" { + bail!( + "--local-policy-root references Cloudreve policy {source_policy_id}, which is not a local policy" + ); + } + } + Ok(()) +} + +fn verify_local_storage_roots(source: &SourceData, options: &MigrationOptions) -> Result<()> { + for policy in source.policies.iter().filter(|policy| { + policy.r#type == "local" + && map_driver_type(&policy.r#type).is_some() + && !source_settings(&policy.settings) + .get("encryption") + .and_then(Value::as_bool) + .unwrap_or(false) + }) { + let root = local_policy_root(options, policy.id); + let metadata = std::fs::metadata(root).wrap_err_with(|| { + format!( + "read local storage root for Cloudreve policy {}: {root}", + policy.id + ) + })?; + if !metadata.is_dir() { + bail!( + "local storage root for Cloudreve policy {} is not a directory: {root}", + policy.id + ); + } + } + Ok(()) +} + +async fn verify_all_local_source_objects( + source_db: &DatabaseConnection, + source: &SourceData, + options: &MigrationOptions, +) -> Result<()> { + const VERIFY_BATCH_SIZE: u64 = 500; + + let policies = source + .policies + .iter() + .map(|policy| (policy.id, policy)) + .collect::>(); + let mut cursor_value = 0; + loop { + let mut query = cr::entities::Entity::find() + .filter(cr::entities::Column::Type.eq(0)) + .filter(cr::entities::Column::Id.gt(cursor_value)) + .order_by_asc(cr::entities::Column::Id) + .limit(VERIFY_BATCH_SIZE); + if !source.include_deleted { + query = query.filter(cr::entities::Column::DeletedAt.is_null()); + } + let entities = query.all(source_db).await?; + let Some(last_entity) = entities.last() else { + return Ok(()); + }; + for entity in &entities { + let Some(policy) = policies.get(&entity.storage_policy_entities) else { + continue; + }; + if policy.r#type == "local" + && map_driver_type(&policy.r#type).is_some() + && !source_settings(&policy.settings) + .get("encryption") + .and_then(Value::as_bool) + .unwrap_or(false) + { + verify_local_entity(entity, policy, options)?; + } + } + cursor_value = last_entity.id; + } +} + +fn verify_local_blob_batch( + entities: &[cr::entities::Model], + source: &SourceData, + options: &MigrationOptions, + context: &MigrationContext, +) -> Result<()> { + let policies = source + .policies + .iter() + .map(|policy| (policy.id, policy)) + .collect::>(); + for entity in entities { + if !context + .policies + .contains_key(&entity.storage_policy_entities) + { + continue; + } + let Some(policy) = policies.get(&entity.storage_policy_entities) else { + continue; + }; + if policy.r#type != "local" { + continue; + } + verify_local_entity(entity, policy, options)?; + } + Ok(()) +} + +fn verify_local_entity( + entity: &cr::entities::Model, + policy: &cr::storage_policies::Model, + options: &MigrationOptions, +) -> Result<()> { + let root = local_policy_root(options, policy.id); + let path = local_storage_path(root, &entity.source); + let metadata = std::fs::metadata(&path).wrap_err_with(|| { + format!( + "read Cloudreve local entity {} at {}", + entity.id, + path.display() + ) + })?; + if !metadata.is_file() { + bail!( + "Cloudreve local entity {} is not a regular file: {}", + entity.id, + path.display() + ); + } + std::fs::File::open(&path).wrap_err_with(|| { + format!( + "open Cloudreve local entity {} at {}", + entity.id, + path.display() + ) + })?; + let expected_size = u64::try_from(entity.size).map_err(|_| { + color_eyre::eyre::eyre!( + "Cloudreve local entity {} has negative size {}", + entity.id, + entity.size + ) + })?; + if metadata.len() != expected_size { + bail!( + "Cloudreve local entity {} size mismatch at {}: database={}, filesystem={}", + entity.id, + path.display(), + expected_size, + metadata.len() + ); + } + Ok(()) +} + +async fn connect(url: &str, label: &str) -> Result { + Database::connect(url) + .await + .wrap_err_with(|| format!("connect to {label} database")) +} + +async fn validate_target_schema(db: &DatabaseConnection) -> Result<()> { + ad::users::Entity::find() + .count(db) + .await + .wrap_err("AsterDrive schema is unavailable; run AD database migrations first")?; + ad::storage_policies::Entity::find() + .count(db) + .await + .wrap_err("AsterDrive storage_policies table is unavailable")?; + ad::files::Entity::find() + .count(db) + .await + .wrap_err("AsterDrive files table is unavailable")?; + ad::entity_properties::Entity::find() + .count(db) + .await + .wrap_err("AsterDrive entity_properties table is unavailable")?; + ad::tags::Entity::find() + .count(db) + .await + .wrap_err("AsterDrive tags table is unavailable")?; + ad::background_tasks::Entity::find() + .count(db) + .await + .wrap_err("AsterDrive background_tasks table is unavailable")?; + Ok(()) +} + +async fn ensure_target_safe(db: &DatabaseConnection, allow_non_empty: bool) -> Result<()> { + if allow_non_empty { + return Ok(()); + } + let counts = [ + ( + "storage_policies", + ad::storage_policies::Entity::find().count(db).await?, + ), + ( + "storage_policy_groups", + ad::storage_policy_groups::Entity::find().count(db).await?, + ), + ("users", ad::users::Entity::find().count(db).await?), + ( + "user_profiles", + ad::user_profiles::Entity::find().count(db).await?, + ), + ("folders", ad::folders::Entity::find().count(db).await?), + ("files", ad::files::Entity::find().count(db).await?), + ( + "file_blobs", + ad::file_blobs::Entity::find().count(db).await?, + ), + ( + "file_versions", + ad::file_versions::Entity::find().count(db).await?, + ), + ("shares", ad::shares::Entity::find().count(db).await?), + ( + "entity_properties", + ad::entity_properties::Entity::find().count(db).await?, + ), + ("tags", ad::tags::Entity::find().count(db).await?), + ( + "background_tasks", + ad::background_tasks::Entity::find().count(db).await?, + ), + ]; + let occupied: Vec = counts + .into_iter() + .filter(|(_, count)| *count > 0) + .map(|(table, count)| format!("{table}={count}")) + .collect(); + if !occupied.is_empty() { + bail!( + "target AD database is not empty ({}); use a freshly migrated database or pass --allow-non-empty-target", + occupied.join(", ") + ); + } + Ok(()) +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct TargetCounts { + policies: u64, + policy_groups: u64, + users: u64, + user_profiles: u64, + folders: u64, + blobs: u64, + files: u64, + versions: u64, + shares: u64, + properties: u64, + tags: u64, + tasks: u64, +} + +impl TargetCounts { + async fn load(db: &DatabaseConnection) -> Result { + Ok(Self { + policies: ad::storage_policies::Entity::find().count(db).await?, + policy_groups: ad::storage_policy_groups::Entity::find().count(db).await?, + users: ad::users::Entity::find().count(db).await?, + user_profiles: ad::user_profiles::Entity::find().count(db).await?, + folders: ad::folders::Entity::find().count(db).await?, + blobs: ad::file_blobs::Entity::find().count(db).await?, + files: ad::files::Entity::find().count(db).await?, + versions: ad::file_versions::Entity::find().count(db).await?, + shares: ad::shares::Entity::find().count(db).await?, + properties: ad::entity_properties::Entity::find().count(db).await?, + tags: ad::tags::Entity::find().count(db).await?, + tasks: ad::background_tasks::Entity::find().count(db).await?, + }) + } +} + +fn count_check(name: &str, before: u64, migrated: usize, actual: u64) -> ValidationCheck { + let migrated = u64::try_from(migrated).unwrap_or(u64::MAX); + let expected = before.saturating_add(migrated); + ValidationCheck { + name: name.to_string(), + passed: actual == expected, + expected: expected.to_string(), + actual: actual.to_string(), + message: (actual != expected) + .then(|| format!("expected baseline {before} plus {migrated} migrated records")), + } +} + +fn invariant_check(name: &str, expected: usize, actual: usize, message: &str) -> ValidationCheck { + ValidationCheck { + name: name.to_string(), + passed: actual == expected, + expected: expected.to_string(), + actual: actual.to_string(), + message: (actual != expected).then(|| message.to_string()), + } +} + +async fn validate_migration_result( + db: &DatabaseConnection, + before: &TargetCounts, + report: &MigrationReport, +) -> Result { + let after = TargetCounts::load(db).await?; + let mut checks = vec![ + count_check( + "storage_policies_count", + before.policies, + report.migrated_policies, + after.policies, + ), + count_check( + "storage_policy_groups_count", + before.policy_groups, + report.migrated_policy_groups, + after.policy_groups, + ), + count_check( + "users_count", + before.users, + report.migrated_users, + after.users, + ), + count_check( + "user_profiles_count", + before.user_profiles, + report.migrated_users, + after.user_profiles, + ), + count_check( + "folders_count", + before.folders, + report.migrated_folders, + after.folders, + ), + count_check( + "file_blobs_count", + before.blobs, + report.migrated_blobs, + after.blobs, + ), + count_check( + "files_count", + before.files, + report.migrated_files, + after.files, + ), + count_check( + "file_versions_count", + before.versions, + report.migrated_versions, + after.versions, + ), + count_check( + "shares_count", + before.shares, + report.migrated_shares, + after.shares, + ), + count_check( + "entity_properties_count", + before.properties, + report.migrated_properties, + after.properties, + ), + count_check("tags_count", before.tags, report.migrated_tags, after.tags), + count_check( + "background_tasks_count", + before.tasks, + report.migrated_tasks, + after.tasks, + ), + invariant_check( + "policy_mappings_count", + report.migrated_policies, + report.mappings.policies.len(), + "storage policy source-to-target mappings are incomplete", + ), + invariant_check( + "policy_group_mappings_count", + report.migrated_policy_groups, + report.mappings.policy_groups.len(), + "policy group source-to-target mappings are incomplete", + ), + invariant_check( + "user_mappings_count", + report.migrated_users, + report.mappings.users.len(), + "user source-to-target mappings are incomplete", + ), + invariant_check( + "folder_mappings_count", + report.migrated_folders, + report.mappings.folders.len(), + "folder source-to-target mappings are incomplete", + ), + invariant_check( + "blob_mappings_count", + report.migrated_blobs, + report.mappings.blobs.len(), + "blob source-to-target mappings are incomplete", + ), + invariant_check( + "file_mappings_count", + report.migrated_files, + report.mappings.files.len(), + "file source-to-target mappings are incomplete", + ), + invariant_check( + "share_mappings_count", + report.migrated_shares, + report.mappings.shares.len(), + "share source-to-target mappings are incomplete", + ), + invariant_check( + "task_mappings_count", + report.migrated_tasks, + report.mappings.tasks.len(), + "task source-to-target mappings are incomplete", + ), + ]; + + let task_ids = report + .mappings + .tasks + .iter() + .map(|mapping| mapping.target_id) + .collect::>(); + let mut imported_tasks = Vec::new(); + for chunk in task_ids.chunks(500) { + imported_tasks.extend( + ad::background_tasks::Entity::find() + .filter(ad::background_tasks::Column::Id.is_in(chunk.iter().copied())) + .all(db) + .await?, + ); + } + checks.push(invariant_check( + "imported_tasks_exist", + task_ids.len(), + imported_tasks.len(), + "one or more imported task IDs are missing", + )); + let terminal_tasks = imported_tasks + .iter() + .filter(|task| { + matches!(task.status.as_str(), "succeeded" | "failed" | "canceled") + && task.lease_expires_at.is_none() + }) + .count(); + checks.push(invariant_check( + "imported_tasks_are_terminal", + imported_tasks.len(), + terminal_tasks, + "imported tasks must be terminal and have no active lease", + )); + + let tag_properties = ad::entity_properties::Entity::find() + .filter(ad::entity_properties::Column::Namespace.eq("system.tags")) + .all(db) + .await?; + let tag_binding_keys = tag_properties + .into_iter() + .map(|property| (property.entity_type, property.entity_id, property.name)) + .collect::>(); + let valid_tag_assignments = report + .tag_assignments + .iter() + .filter(|assignment| { + tag_binding_keys.contains(&( + assignment.target_entity_type.clone(), + assignment.target_entity_id, + assignment.target_tag_id.to_string(), + )) + }) + .count(); + checks.push(invariant_check( + "tag_assignments_exist", + report.tag_assignments.len(), + valid_tag_assignments, + "one or more system.tags bindings are missing", + )); + + let direct_link_properties = ad::entity_properties::Entity::find() + .filter(ad::entity_properties::Column::Namespace.eq("cloudreve.direct_links")) + .all(db) + .await?; + let direct_link_values = direct_link_properties + .into_iter() + .map(|property| ((property.entity_id, property.name), property.value)) + .collect::>(); + let valid_direct_links = report + .direct_links + .iter() + .filter(|link| { + direct_link_values + .get(&(link.target_file_id, link.source_direct_link_id.to_string())) + .and_then(|value| value.as_deref()) + .and_then(|value| serde_json::from_str::(value).ok()) + .and_then(|value| value.get("url").and_then(Value::as_str).map(str::to_string)) + .is_some_and(|url| url == link.url) + }) + .count(); + checks.push(invariant_check( + "direct_link_mappings_exist", + report.direct_links.len(), + valid_direct_links, + "one or more cloudreve.direct_links properties are missing or changed", + )); + + Ok(MigrationValidation { + performed: true, + passed: checks.iter().all(|check| check.passed), + checks, + }) +} + +fn hash_password(password: &str) -> Result { + let salt = SaltString::encode_b64(uuid::Uuid::new_v4().as_bytes()) + .map_err(|error| color_eyre::eyre::eyre!("create password salt: {error}"))?; + Argon2::default() + .hash_password(password.as_bytes(), &salt) + .map(|hash| hash.to_string()) + .map_err(|error| color_eyre::eyre::eyre!("hash temporary AD password: {error}")) +} + +#[derive(Debug, Default, Clone, Serialize, Deserialize)] +struct MigrationContext { + policies: HashMap, + policy_groups: HashMap, + users: HashMap, + usernames: HashMap, + folders: HashMap, + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + blobs: HashMap, + files: HashMap, + shares: HashMap, + tasks: HashMap, +} + +fn sorted_id_mappings(values: &HashMap) -> Vec { + let mut mappings = values + .iter() + .map(|(source_id, target_id)| IdMapping { + source_id: *source_id, + target_id: *target_id, + }) + .collect::>(); + mappings.sort_by_key(|mapping| mapping.source_id); + mappings +} + +struct SourceData { + groups: Vec, + users: Vec, + policies: Vec, + folders: Vec, + source_file_records: u64, + source_files: u64, + symbolic_files: u64, + source_entities: u64, + source_file_entities: u64, + include_deleted: bool, + shares: Vec, + metadata: Vec, + direct_links: Vec, + tasks: Vec, +} + +impl SourceData { + async fn load(db: &DatabaseConnection, include_deleted: bool) -> Result { + let groups = cr::groups::Entity::find().all(db).await?; + let users = cr::users::Entity::find().all(db).await?; + let policies = cr::storage_policies::Entity::find().all(db).await?; + let folders = cr::files::Entity::find() + .filter(cr::files::Column::Type.eq(1)) + .all(db) + .await?; + let source_file_records = cr::files::Entity::find().count(db).await?; + let source_files = cr::files::Entity::find() + .filter(cr::files::Column::Type.eq(0)) + .count(db) + .await?; + let symbolic_files = cr::files::Entity::find() + .filter(cr::files::Column::Type.eq(0)) + .filter(cr::files::Column::IsSymbolic.eq(true)) + .count(db) + .await?; + let entity_query = if include_deleted { + cr::entities::Entity::find() + } else { + cr::entities::Entity::find().filter(cr::entities::Column::DeletedAt.is_null()) + }; + let source_entities = entity_query.count(db).await?; + let source_file_entities = cr::file_entities::Entity::find().count(db).await?; + let shares = cr::shares::Entity::find().all(db).await?; + let metadata = cr::metadata::Entity::find().all(db).await?; + let direct_links = cr::direct_links::Entity::find().all(db).await?; + let tasks = cr::tasks::Entity::find().all(db).await?; + + Ok(Self { + groups: filter_deleted(groups, include_deleted, |model| model.deleted_at.is_some()), + users: filter_deleted(users, include_deleted, |model| model.deleted_at.is_some()), + policies: filter_deleted(policies, include_deleted, |model| { + model.deleted_at.is_some() + }), + folders, + source_file_records, + source_files, + symbolic_files, + source_entities, + source_file_entities, + include_deleted, + shares: filter_deleted(shares, include_deleted, |model| model.deleted_at.is_some()), + metadata: filter_deleted(metadata, include_deleted, |model| { + model.deleted_at.is_some() + }), + direct_links: filter_deleted(direct_links, include_deleted, |model| { + model.deleted_at.is_some() + }), + tasks: filter_deleted(tasks, include_deleted, |model| model.deleted_at.is_some()), + }) + } + + fn report(&self) -> MigrationReport { + MigrationReport { + source_users: self.users.len() as u64, + source_groups: self.groups.len() as u64, + source_policies: self.policies.len() as u64, + source_folders: self.folders.len() as u64, + source_files: self.source_files, + source_entities: self.source_entities, + source_shares: self.shares.len() as u64, + source_direct_links: self.direct_links.len() as u64, + source_tag_assignments: self + .metadata + .iter() + .filter(|metadata| tag_name(&metadata.name).is_some()) + .count() as u64, + source_tasks: self.tasks.len() as u64, + ..Default::default() + } + } + + fn unsupported_policy_types(&self) -> Vec { + let mut values: Vec = self + .policies + .iter() + .filter_map(unsupported_policy_reason) + .collect(); + values.sort(); + values.dedup(); + values + } + + fn compatibility_warnings(&self) -> Vec { + let mut warnings = vec![ + "Cloudreve user passwords use SHA/legacy MD5 formats and are replaced by the supplied temporary Argon2 password; every migrated user is marked must_change_password".to_string(), + "OAuth grants, login sessions and Cloudreve filesystem events are intentionally not migrated".to_string(), + "Cloudreve Passkeys, WebDAV credentials and two-factor secrets are not portable to AD and must be enrolled again".to_string(), + "file objects are reused in their existing local/object-storage locations; the migration does not duplicate object bytes".to_string(), + "Cloudreve tasks are archived as terminal AD system_runtime records; queued, processing and suspending tasks are canceled instead of resumed".to_string(), + ]; + let symbolic = self.symbolic_files; + if symbolic > 0 { + warnings.push(format!( + "{symbolic} symbolic/placeholder Cloudreve files cannot be represented in AD and will be skipped" + )); + } + let unsupported = self.unsupported_policy_types(); + if !unsupported.is_empty() { + warnings.push(format!( + "unsupported storage policy types detected: {}", + unsupported.join(", ") + )); + } + if !self.direct_links.is_empty() { + warnings.push("Cloudreve direct links require --direct-link-secret to regenerate AD v2 URLs; old /f/... URLs, per-link counters, speed limits and revocation semantics cannot be preserved".to_string()); + } + warnings + } +} + +fn filter_deleted(items: Vec, include_deleted: bool, deleted: F) -> Vec +where + F: Fn(&T) -> bool, +{ + if include_deleted { + items + } else { + items.into_iter().filter(|item| !deleted(item)).collect() + } +} + +fn map_driver_type(source: &str) -> Option<&'static str> { + match source { + "local" => Some("local"), + "s3" | "oss" | "ks3" | "obs" => Some("s3"), + "cos" => Some("tencent_cos"), + _ => None, + } +} + +fn unsupported_policy_reason(policy: &cr::storage_policies::Model) -> Option { + if map_driver_type(&policy.r#type).is_none() { + return Some(policy.r#type.clone()); + } + if source_settings(&policy.settings) + .get("encryption") + .and_then(Value::as_bool) + .unwrap_or(false) + { + return Some(format!("{} (Cloudreve encryption enabled)", policy.r#type)); + } + None +} + +fn source_settings(value: &Option) -> Value { + value.clone().unwrap_or_else(|| json!({})) +} + +fn policy_options(policy: &cr::storage_policies::Model) -> String { + let settings = source_settings(&policy.settings); + let path_style = settings + .get("s3_path_style") + .and_then(Value::as_bool) + .unwrap_or(true); + json!({ + "s3_path_style": path_style, + "object_storage_upload_strategy": "relay_stream", + "object_storage_download_strategy": "relay_stream", + "cloudreve_source": settings, + "cloudreve_policy_type": policy.r#type, + }) + .to_string() +} + +fn allowed_types(policy: &cr::storage_policies::Model) -> String { + source_settings(&policy.settings) + .get("file_type") + .cloned() + .unwrap_or_else(|| json!([])) + .to_string() +} + +fn chunk_size(policy: &cr::storage_policies::Model) -> i64 { + source_settings(&policy.settings) + .get("chunk_size") + .and_then(Value::as_i64) + .unwrap_or(0) +} + +fn group_is_admin(group: &cr::groups::Model) -> bool { + group + .permissions + .first() + .is_some_and(|permissions| permissions & 1 == 1) +} + +fn opaque_blob_key(entity_id: i64) -> String { + format!("cloudreve-{entity_id:016x}") +} + +fn share_token(share_id: i64) -> String { + let digest = Sha256::digest(format!("cloudreve-share-{share_id}").as_bytes()); + format!("cr-{share_id}-{}", &format!("{digest:x}")[..16]) +} + +fn tag_name(metadata_name: &str) -> Option<&str> { + metadata_name + .strip_prefix("tag:") + .map(str::trim) + .filter(|name| !name.is_empty()) +} + +fn normalize_tag_name(name: &str) -> String { + name.trim().to_lowercase() +} + +fn target_tag_name(name: &str) -> String { + name.trim().chars().take(64).collect() +} + +fn target_tag_color(color: &str) -> String { + let color = color.trim().to_ascii_lowercase(); + if color.len() == 7 + && color.starts_with('#') + && color[1..].bytes().all(|byte| byte.is_ascii_hexdigit()) + { + return color; + } + if color.len() == 4 + && color.starts_with('#') + && color[1..].bytes().all(|byte| byte.is_ascii_hexdigit()) + { + let mut expanded = String::with_capacity(7); + expanded.push('#'); + for character in color[1..].chars() { + expanded.push(character); + expanded.push(character); + } + return expanded; + } + "#3b82f6".to_string() +} + +fn encode_base62(mut value: u64) -> String { + const BASE62: &[u8] = b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + if value == 0 { + return "a".to_string(); + } + let mut encoded = Vec::new(); + while value > 0 { + encoded.push(char::from(BASE62[(value % 62) as usize])); + value /= 62; + } + encoded.iter().rev().collect() +} + +fn direct_link_url( + file_id: i64, + owner_user_id: i64, + file_name: &str, + secret: &str, +) -> Result { + let file_id = u64::try_from(file_id).wrap_err("AD direct link file ID must be non-negative")?; + let mut mac = Hmac::::new_from_slice(secret.as_bytes()) + .map_err(|error| color_eyre::eyre::eyre!("initialize direct link HMAC: {error}"))?; + mac.update(b"direct_link:v2:"); + mac.update(format!("user:{owner_user_id}").as_bytes()); + mac.update(b":"); + mac.update(file_id.to_string().as_bytes()); + let signature = + base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(mac.finalize().into_bytes()); + Ok(format!( + "/d/v2.{}.{}/{}", + encode_base62(file_id), + signature, + urlencoding::encode(file_name) + )) +} + +fn archived_task_status(source_status: &str) -> &'static str { + match source_status { + "completed" => "succeeded", + "error" => "failed", + "canceled" | "queued" | "processing" | "suspending" => "canceled", + _ => "canceled", + } +} + +fn source_task_was_active(source_status: &str) -> bool { + matches!(source_status, "queued" | "processing" | "suspending") +} + +fn unique_username(source: &str, source_id: i64, used: &mut HashSet) -> String { + let mut base: String = source + .trim() + .chars() + .filter(|character| !character.is_control()) + .take(64) + .collect(); + if base.is_empty() { + base = format!("cloudreve-user-{source_id}"); + } + if used.insert(base.clone()) { + return base; + } + let suffix = format!("-{source_id}"); + let keep = 64usize.saturating_sub(suffix.chars().count()); + let mut candidate: String = base.chars().take(keep).collect(); + candidate.push_str(&suffix); + let mut discriminator = 2; + while !used.insert(candidate.clone()) { + let suffix = format!("-{source_id}-{discriminator}"); + let keep = 64usize.saturating_sub(suffix.chars().count()); + candidate = base.chars().take(keep).collect(); + candidate.push_str(&suffix); + discriminator += 1; + } + candidate +} + +fn file_classification(name: &str) -> (String, Option, String, String) { + let mime = mime_guess::from_path(name) + .first_or_octet_stream() + .essence_str() + .to_string(); + let lowercase = name.to_ascii_lowercase(); + let extension = lowercase + .rsplit_once('.') + .map(|(_, extension)| extension.to_string()) + .unwrap_or_default(); + let compound_extension = ["tar.gz", "tar.bz2", "tar.xz", "user.js"] + .into_iter() + .find(|candidate| lowercase.ends_with(candidate)) + .map(str::to_string); + let category = if mime.starts_with("image/") { + "image" + } else if mime.starts_with("video/") { + "video" + } else if mime.starts_with("audio/") { + "audio" + } else if ["zip", "rar", "7z", "gz", "bz2", "xz", "tar"].contains(&extension.as_str()) { + "archive" + } else if ["xls", "xlsx", "csv", "ods"].contains(&extension.as_str()) { + "spreadsheet" + } else if ["ppt", "pptx", "odp"].contains(&extension.as_str()) { + "presentation" + } else if [ + "rs", "go", "js", "ts", "py", "java", "c", "cpp", "html", "css", "json", "yaml", "yml", + "toml", + ] + .contains(&extension.as_str()) + { + "code" + } else if mime.starts_with("text/") + || ["pdf", "doc", "docx", "odt", "md"].contains(&extension.as_str()) + { + "document" + } else { + "other" + }; + (mime, compound_extension, extension, category.to_string()) +} + +mod checkpoint; +mod phases; +use phases::*; + +#[cfg(test)] +mod tests { + use super::*; + use sea_orm::{ConnectionTrait, DbBackend, Schema}; + + #[test] + fn classifies_common_file_types() { + assert_eq!(file_classification("photo.JPG").3, "image"); + assert_eq!( + file_classification("backup.tar.gz").1.as_deref(), + Some("tar.gz") + ); + assert_eq!(file_classification("main.rs").3, "code"); + } + + #[test] + fn maps_supported_storage_drivers_conservatively() { + assert_eq!(map_driver_type("local"), Some("local")); + assert_eq!(map_driver_type("oss"), Some("s3")); + assert_eq!(map_driver_type("cos"), Some("tencent_cos")); + assert_eq!(map_driver_type("onedrive"), None); + assert_eq!(map_driver_type("qiniu"), None); + } + + #[test] + fn parses_and_normalizes_cloudreve_tags_for_ad() { + assert_eq!(tag_name("tag:Important"), Some("Important")); + assert_eq!(tag_name("tag: Project A "), Some("Project A")); + assert_eq!(tag_name("author"), None); + assert_eq!(normalize_tag_name(" Important "), "important"); + assert_eq!(target_tag_color("#AbC"), "#aabbcc"); + assert_eq!(target_tag_color("#3B82F6"), "#3b82f6"); + assert_eq!(target_tag_color(""), "#3b82f6"); + assert_eq!(target_tag_name(&"x".repeat(80)).chars().count(), 64); + } + + #[test] + fn builds_asterdrive_v2_direct_link_urls() -> Result<()> { + let url = direct_link_url(1, 7, "hello world.txt", "test-direct-link-secret")?; + assert!(url.starts_with("/d/v2.b.")); + assert!(url.ends_with("/hello%20world.txt")); + assert_eq!( + url, + direct_link_url(1, 7, "hello world.txt", "test-direct-link-secret")? + ); + assert_ne!( + url, + direct_link_url(1, 8, "hello world.txt", "test-direct-link-secret")? + ); + Ok(()) + } + + #[test] + fn maps_cloudreve_tasks_to_non_executable_terminal_statuses() { + assert_eq!(archived_task_status("completed"), "succeeded"); + assert_eq!(archived_task_status("error"), "failed"); + assert_eq!(archived_task_status("canceled"), "canceled"); + for status in ["queued", "processing", "suspending"] { + assert!(source_task_was_active(status)); + assert_eq!(archived_task_status(status), "canceled"); + } + assert_eq!(archived_task_status("unknown"), "canceled"); + } + + #[test] + fn records_skipped_objects_by_type() { + let mut report = MigrationReport::default(); + report.record_skip("file", Some(42), "missing blob"); + report.record_skip("file", Some(43), "symbolic file"); + report.record_skip("share", None, "missing target"); + + assert_eq!(report.skipped, 3); + assert_eq!(report.skipped_by_type.get("file"), Some(&2)); + assert_eq!(report.skipped_by_type.get("share"), Some(&1)); + assert_eq!(report.skipped_objects[0].source_id, Some(42)); + assert_eq!(report.skipped_objects[0].reason, "missing blob"); + } + + #[test] + fn writes_structured_json_report() -> Result<()> { + let report_path = std::env::temp_dir().join(format!( + "asterdrive-migration-report-{}.json", + uuid::Uuid::new_v4() + )); + let mut report = MigrationReport { + migrated_users: 1, + validation: MigrationValidation { + performed: true, + passed: true, + checks: vec![ValidationCheck { + name: "users_count".to_string(), + passed: true, + expected: "1".to_string(), + actual: "1".to_string(), + message: None, + }], + }, + ..Default::default() + }; + report.mappings.users.push(IdMapping { + source_id: 7, + target_id: 11, + }); + + write_json_report(&report_path, &report)?; + let value: Value = serde_json::from_slice(&std::fs::read(&report_path)?)?; + assert_eq!(value["schema_version"], 1); + assert_eq!(value["migrated_users"], 1); + assert_eq!(value["mappings"]["users"][0]["source_id"], 7); + assert_eq!(value["mappings"]["users"][0]["target_id"], 11); + assert_eq!(value["validation"]["passed"], true); + + let _ = std::fs::remove_file(report_path); + Ok(()) + } + + #[tokio::test] + async fn migrates_minimal_cloudreve_database() -> Result<()> { + let suffix = uuid::Uuid::new_v4(); + let source_path = std::env::temp_dir().join(format!("cloudreve-{suffix}.db")); + let target_path = std::env::temp_dir().join(format!("asterdrive-{suffix}.db")); + let source_url = sqlite_url(&source_path); + let target_url = sqlite_url(&target_path); + + let source = Database::connect(&source_url).await?; + create_source_schema(&source).await?; + seed_source(&source).await?; + source.close().await?; + + let target = Database::connect(&target_url).await?; + create_target_schema(&target).await?; + target.close().await?; + + let report = migrate(MigrationOptions { + source_url: source_url.clone(), + target_url: target_url.clone(), + default_password: "temporary-password".to_string(), + local_base_path: "C:/cloudreve".to_string(), + local_policy_roots: std::collections::BTreeMap::new(), + verify_local_storage: false, + direct_link_secret: Some("test-direct-link-secret".to_string()), + include_deleted: false, + allow_non_empty_target: false, + skip_unsupported_policies: false, + dry_run: false, + run_id: None, + resume: false, + blob_batch_size: 500, + file_batch_size: 500, + }) + .await?; + + assert_eq!(report.migrated_users, 1); + assert_eq!(report.migrated_folders, 1); + assert_eq!(report.migrated_files, 1); + assert_eq!(report.migrated_blobs, 1); + assert_eq!(report.migrated_shares, 1); + assert_eq!(report.migrated_properties, 3); + assert_eq!(report.migrated_tags, 1); + assert_eq!(report.migrated_tag_assignments, 1); + assert_eq!(report.migrated_direct_links, 1); + assert_eq!(report.migrated_tasks, 2); + assert!(report.validation.performed); + assert!(report.validation.passed); + assert!(report.validation.checks.iter().all(|check| check.passed)); + assert_eq!(report.mappings.users.len(), 1); + assert_eq!(report.mappings.folders.len(), 1); + assert_eq!(report.mappings.files.len(), 1); + assert_eq!(report.mappings.blobs.len(), 1); + assert_eq!(report.mappings.shares.len(), 1); + assert_eq!(report.mappings.tasks.len(), 2); + assert_eq!(report.direct_links.len(), 1); + assert!(report.direct_links[0].url.starts_with("/d/v2.")); + assert_eq!(report.tag_assignments.len(), 1); + assert_eq!(report.tag_assignments[0].target_entity_type, "file"); + + let target = Database::connect(&target_url).await?; + assert_eq!(ad::users::Entity::find().count(&target).await?, 1); + assert_eq!(ad::folders::Entity::find().count(&target).await?, 1); + assert_eq!(ad::files::Entity::find().count(&target).await?, 1); + assert_eq!(ad::file_blobs::Entity::find().count(&target).await?, 1); + assert_eq!(ad::shares::Entity::find().count(&target).await?, 1); + assert_eq!(ad::tags::Entity::find().count(&target).await?, 1); + assert_eq!( + ad::background_tasks::Entity::find().count(&target).await?, + 2 + ); + assert_eq!( + ad::entity_properties::Entity::find().count(&target).await?, + 3 + ); + let properties = ad::entity_properties::Entity::find().all(&target).await?; + assert!( + properties.iter().any(|property| { + property.namespace == "system.tags" && property.value.is_none() + }) + ); + let direct_link = properties + .iter() + .find(|property| property.namespace == "cloudreve.direct_links") + .and_then(|property| property.value.as_deref()) + .expect("migrated direct link mapping"); + assert!(direct_link.contains("/d/v2.")); + let tasks = ad::background_tasks::Entity::find().all(&target).await?; + assert!(tasks.iter().all(|task| { + matches!(task.status.as_str(), "succeeded" | "failed" | "canceled") + && task.kind == "system_runtime" + && task.lease_expires_at.is_none() + })); + assert!(tasks.iter().any(|task| task.status == "succeeded")); + assert!(tasks.iter().any(|task| task.status == "canceled")); + let blob = ad::file_blobs::Entity::find().one(&target).await?.unwrap(); + assert_eq!(blob.storage_path, "uploads/object.bin"); + let user = ad::users::Entity::find().one(&target).await?.unwrap(); + assert!(user.must_change_password); + assert_eq!(user.role, "admin"); + target.close().await?; + + let _ = std::fs::remove_file(source_path); + let _ = std::fs::remove_file(target_path); + Ok(()) + } + + #[tokio::test] + async fn reuses_and_verifies_local_storage_per_policy_root() -> Result<()> { + let suffix = uuid::Uuid::new_v4(); + let source_path = std::env::temp_dir().join(format!("cloudreve-local-reuse-{suffix}.db")); + let target_path = std::env::temp_dir().join(format!("asterdrive-local-reuse-{suffix}.db")); + let storage_root = std::env::temp_dir().join(format!("cloudreve-local-storage-{suffix}")); + let source_url = sqlite_url(&source_path); + let target_url = sqlite_url(&target_path); + + let source = Database::connect(&source_url).await?; + create_source_schema(&source).await?; + seed_source(&source).await?; + let source_policy_id = cr::storage_policies::Entity::find() + .one(&source) + .await? + .expect("seeded storage policy") + .id; + source.close().await?; + std::fs::create_dir_all(storage_root.join("uploads"))?; + std::fs::write(storage_root.join("uploads/object.bin"), vec![0_u8; 128])?; + + let target = Database::connect(&target_url).await?; + create_target_schema(&target).await?; + target.close().await?; + + let report = migrate(MigrationOptions { + source_url: source_url.clone(), + target_url: target_url.clone(), + default_password: "temporary-password".to_string(), + local_base_path: "unused-local-root".to_string(), + local_policy_roots: std::collections::BTreeMap::from([( + source_policy_id, + storage_root.to_string_lossy().to_string(), + )]), + verify_local_storage: true, + direct_link_secret: Some("test-direct-link-secret".to_string()), + include_deleted: false, + allow_non_empty_target: false, + skip_unsupported_policies: false, + dry_run: false, + run_id: None, + resume: false, + blob_batch_size: 500, + file_batch_size: 500, + }) + .await?; + assert!(report.validation.passed); + + let target = Database::connect(&target_url).await?; + let policy = ad::storage_policies::Entity::find() + .one(&target) + .await? + .expect("migrated storage policy"); + assert_eq!(policy.base_path, storage_root.to_string_lossy()); + target.close().await?; + + let _ = std::fs::remove_file(source_path); + let _ = std::fs::remove_file(target_path); + let _ = std::fs::remove_dir_all(storage_root); + Ok(()) + } + + #[tokio::test] + async fn rejects_local_storage_size_mismatch() -> Result<()> { + let suffix = uuid::Uuid::new_v4(); + let source_path = + std::env::temp_dir().join(format!("cloudreve-local-mismatch-{suffix}.db")); + let target_path = + std::env::temp_dir().join(format!("asterdrive-local-mismatch-{suffix}.db")); + let storage_root = + std::env::temp_dir().join(format!("cloudreve-local-mismatch-storage-{suffix}")); + let source_url = sqlite_url(&source_path); + let target_url = sqlite_url(&target_path); + + let source = Database::connect(&source_url).await?; + create_source_schema(&source).await?; + seed_source(&source).await?; + let source_policy_id = cr::storage_policies::Entity::find() + .one(&source) + .await? + .expect("seeded storage policy") + .id; + source.close().await?; + std::fs::create_dir_all(storage_root.join("uploads"))?; + std::fs::write(storage_root.join("uploads/object.bin"), vec![0_u8; 127])?; + + let target = Database::connect(&target_url).await?; + create_target_schema(&target).await?; + target.close().await?; + + let error = migrate(MigrationOptions { + source_url: source_url.clone(), + target_url: target_url.clone(), + default_password: "temporary-password".to_string(), + local_base_path: "unused-local-root".to_string(), + local_policy_roots: std::collections::BTreeMap::from([( + source_policy_id, + storage_root.to_string_lossy().to_string(), + )]), + verify_local_storage: true, + direct_link_secret: Some("test-direct-link-secret".to_string()), + include_deleted: false, + allow_non_empty_target: false, + skip_unsupported_policies: false, + dry_run: true, + run_id: None, + resume: false, + blob_batch_size: 500, + file_batch_size: 500, + }) + .await + .unwrap_err(); + assert!(format!("{error:?}").contains("size mismatch")); + + let target = Database::connect(&target_url).await?; + assert_eq!( + ad::storage_policies::Entity::find().count(&target).await?, + 0 + ); + target.close().await?; + + let _ = std::fs::remove_file(source_path); + let _ = std::fs::remove_file(target_path); + let _ = std::fs::remove_dir_all(storage_root); + Ok(()) + } + + #[tokio::test] + async fn rejects_unknown_local_policy_root() -> Result<()> { + let suffix = uuid::Uuid::new_v4(); + let source_path = std::env::temp_dir().join(format!("cloudreve-local-policy-{suffix}.db")); + let target_path = std::env::temp_dir().join(format!("asterdrive-local-policy-{suffix}.db")); + let source_url = sqlite_url(&source_path); + let target_url = sqlite_url(&target_path); + + let source = Database::connect(&source_url).await?; + create_source_schema(&source).await?; + seed_source(&source).await?; + source.close().await?; + + let target = Database::connect(&target_url).await?; + create_target_schema(&target).await?; + target.close().await?; + + let error = migrate(MigrationOptions { + source_url: source_url.clone(), + target_url: target_url.clone(), + default_password: "temporary-password".to_string(), + local_base_path: "unused-local-root".to_string(), + local_policy_roots: std::collections::BTreeMap::from([(999, "C:/missing".to_string())]), + verify_local_storage: false, + direct_link_secret: Some("test-direct-link-secret".to_string()), + include_deleted: false, + allow_non_empty_target: false, + skip_unsupported_policies: false, + dry_run: true, + run_id: None, + resume: false, + blob_batch_size: 500, + file_batch_size: 500, + }) + .await + .unwrap_err(); + assert!(error.to_string().contains("policy 999")); + + let _ = std::fs::remove_file(source_path); + let _ = std::fs::remove_file(target_path); + Ok(()) + } + + #[tokio::test] + async fn resumes_from_last_completed_stage() -> Result<()> { + let suffix = uuid::Uuid::new_v4(); + let run_id = format!("resume-test-{suffix}"); + let source_path = std::env::temp_dir().join(format!("cloudreve-resume-{suffix}.db")); + let target_path = std::env::temp_dir().join(format!("asterdrive-resume-{suffix}.db")); + let source_url = sqlite_url(&source_path); + let target_url = sqlite_url(&target_path); + + let source = Database::connect(&source_url).await?; + create_source_schema(&source).await?; + seed_source(&source).await?; + source.close().await?; + + let target = Database::connect(&target_url).await?; + create_target_schema(&target).await?; + target + .execute_unprepared( + "CREATE TRIGGER fail_folder_insert BEFORE INSERT ON folders \ + BEGIN SELECT RAISE(ABORT, 'forced folder stage failure'); END", + ) + .await?; + target.close().await?; + + let options = MigrationOptions { + source_url: source_url.clone(), + target_url: target_url.clone(), + default_password: "temporary-password".to_string(), + local_base_path: "C:/cloudreve".to_string(), + local_policy_roots: std::collections::BTreeMap::new(), + verify_local_storage: false, + direct_link_secret: Some("test-direct-link-secret".to_string()), + include_deleted: false, + allow_non_empty_target: false, + skip_unsupported_policies: false, + dry_run: false, + run_id: Some(run_id.clone()), + resume: false, + blob_batch_size: 500, + file_batch_size: 500, + }; + + let error = migrate(options.clone()).await.unwrap_err(); + assert!(error.to_string().contains(&run_id)); + + let target = Database::connect(&target_url).await?; + assert_eq!( + ad::storage_policies::Entity::find().count(&target).await?, + 1 + ); + assert_eq!(ad::users::Entity::find().count(&target).await?, 1); + assert_eq!(ad::folders::Entity::find().count(&target).await?, 0); + let failed_checkpoint = checkpoint::Entity::find_by_id(run_id.clone()) + .one(&target) + .await? + .expect("failed migration checkpoint"); + assert_eq!(failed_checkpoint.status, "failed"); + assert_eq!( + failed_checkpoint.last_completed_stage.as_deref(), + Some("users") + ); + target + .execute_unprepared("DROP TRIGGER fail_folder_insert") + .await?; + target.close().await?; + + let report = migrate(MigrationOptions { + resume: true, + ..options + }) + .await?; + assert!(report.resumed); + assert_eq!(report.run_id.as_deref(), Some(run_id.as_str())); + assert!(report.validation.passed); + assert_eq!(report.migrated_users, 1); + assert_eq!(report.migrated_folders, 1); + + let target = Database::connect(&target_url).await?; + assert_eq!(ad::users::Entity::find().count(&target).await?, 1); + assert_eq!(ad::folders::Entity::find().count(&target).await?, 1); + assert_eq!(ad::files::Entity::find().count(&target).await?, 1); + let completed_checkpoint = checkpoint::Entity::find_by_id(run_id) + .one(&target) + .await? + .expect("completed migration checkpoint"); + assert_eq!(completed_checkpoint.status, "completed"); + assert_eq!( + completed_checkpoint.last_completed_stage.as_deref(), + Some("tasks") + ); + target.close().await?; + + let _ = std::fs::remove_file(source_path); + let _ = std::fs::remove_file(target_path); + Ok(()) + } + + #[tokio::test] + async fn resumes_blobs_from_last_committed_batch() -> Result<()> { + let suffix = uuid::Uuid::new_v4(); + let run_id = format!("blob-resume-test-{suffix}"); + let source_path = std::env::temp_dir().join(format!("cloudreve-blob-resume-{suffix}.db")); + let target_path = std::env::temp_dir().join(format!("asterdrive-blob-resume-{suffix}.db")); + let source_url = sqlite_url(&source_path); + let target_url = sqlite_url(&target_path); + + let source = Database::connect(&source_url).await?; + create_source_schema(&source).await?; + seed_source(&source).await?; + let extra_blob_ids = seed_extra_blob_entities(&source, 3).await?; + source.close().await?; + + let failing_blob_id = extra_blob_ids[1]; + let target = Database::connect(&target_url).await?; + create_target_schema(&target).await?; + target + .execute_unprepared(&format!( + "CREATE TRIGGER fail_blob_batch BEFORE INSERT ON file_blobs \ + WHEN NEW.hash = '{}' \ + BEGIN SELECT RAISE(ABORT, 'forced blob batch failure'); END", + opaque_blob_key(failing_blob_id) + )) + .await?; + target.close().await?; + + let options = MigrationOptions { + source_url: source_url.clone(), + target_url: target_url.clone(), + default_password: "temporary-password".to_string(), + local_base_path: "C:/cloudreve".to_string(), + local_policy_roots: std::collections::BTreeMap::new(), + verify_local_storage: false, + direct_link_secret: Some("test-direct-link-secret".to_string()), + include_deleted: false, + allow_non_empty_target: false, + skip_unsupported_policies: false, + dry_run: false, + run_id: Some(run_id.clone()), + resume: false, + blob_batch_size: 2, + file_batch_size: 500, + }; + + let error = migrate(options.clone()).await.unwrap_err(); + assert!(error.to_string().contains("blobs")); + + let target = Database::connect(&target_url).await?; + assert_eq!(ad::file_blobs::Entity::find().count(&target).await?, 2); + let cursor = checkpoint::load_stage_cursor(&target, &run_id, "blobs") + .await? + .expect("committed blob cursor"); + assert_eq!(cursor.cursor_value, extra_blob_ids[0]); + assert_eq!(cursor.processed_count, 2); + assert_eq!( + checkpoint::object_map::Entity::find() + .filter(checkpoint::object_map::Column::RunId.eq(&run_id)) + .filter(checkpoint::object_map::Column::ObjectType.eq("blob")) + .count(&target) + .await?, + 2 + ); + let failed_checkpoint = checkpoint::Entity::find_by_id(run_id.clone()) + .one(&target) + .await? + .expect("failed blob migration checkpoint"); + assert_eq!(failed_checkpoint.status, "failed"); + assert_eq!( + failed_checkpoint.last_completed_stage.as_deref(), + Some("folders") + ); + let failed_report: MigrationReport = serde_json::from_value(failed_checkpoint.report_json)?; + assert_eq!(failed_report.migrated_blobs, 2); + target + .execute_unprepared("DROP TRIGGER fail_blob_batch") + .await?; + target.close().await?; + + let report = migrate(MigrationOptions { + resume: true, + ..options + }) + .await?; + assert!(report.resumed); + assert_eq!(report.migrated_blobs, 4); + assert_eq!(report.mappings.blobs.len(), 4); + assert!(report.validation.passed); + + let target = Database::connect(&target_url).await?; + assert_eq!(ad::file_blobs::Entity::find().count(&target).await?, 4); + assert_eq!( + checkpoint::object_map::Entity::find() + .filter(checkpoint::object_map::Column::RunId.eq(&run_id)) + .filter(checkpoint::object_map::Column::ObjectType.eq("blob")) + .count(&target) + .await?, + 4 + ); + let completed_cursor = checkpoint::load_stage_cursor(&target, &run_id, "blobs") + .await? + .expect("completed blob cursor"); + assert_eq!(completed_cursor.cursor_value, extra_blob_ids[2]); + assert_eq!(completed_cursor.processed_count, 4); + target.close().await?; + + let _ = std::fs::remove_file(source_path); + let _ = std::fs::remove_file(target_path); + Ok(()) + } + + #[tokio::test] + async fn resumes_files_from_last_committed_batch() -> Result<()> { + let suffix = uuid::Uuid::new_v4(); + let run_id = format!("file-resume-test-{suffix}"); + let source_path = std::env::temp_dir().join(format!("cloudreve-file-resume-{suffix}.db")); + let target_path = std::env::temp_dir().join(format!("asterdrive-file-resume-{suffix}.db")); + let source_url = sqlite_url(&source_path); + let target_url = sqlite_url(&target_path); + + let source = Database::connect(&source_url).await?; + create_source_schema(&source).await?; + seed_source(&source).await?; + let extra_blob_ids = seed_extra_blob_entities(&source, 3).await?; + let extra_file_ids = seed_extra_files(&source, &extra_blob_ids).await?; + source.close().await?; + + let target = Database::connect(&target_url).await?; + create_target_schema(&target).await?; + target + .execute_unprepared( + "CREATE TRIGGER fail_file_batch BEFORE INSERT ON files \ + WHEN NEW.name = 'extra-1.txt' \ + BEGIN SELECT RAISE(ABORT, 'forced file batch failure'); END", + ) + .await?; + target.close().await?; + + let options = MigrationOptions { + source_url: source_url.clone(), + target_url: target_url.clone(), + default_password: "temporary-password".to_string(), + local_base_path: "C:/cloudreve".to_string(), + local_policy_roots: std::collections::BTreeMap::new(), + verify_local_storage: false, + direct_link_secret: Some("test-direct-link-secret".to_string()), + include_deleted: false, + allow_non_empty_target: false, + skip_unsupported_policies: false, + dry_run: false, + run_id: Some(run_id.clone()), + resume: false, + blob_batch_size: 500, + file_batch_size: 2, + }; + + let error = migrate(options.clone()).await.unwrap_err(); + assert!(error.to_string().contains("files")); + + let target = Database::connect(&target_url).await?; + assert_eq!(ad::files::Entity::find().count(&target).await?, 2); + assert_eq!(ad::file_versions::Entity::find().count(&target).await?, 1); + let cursor = checkpoint::load_stage_cursor(&target, &run_id, "files") + .await? + .expect("committed file cursor"); + assert_eq!(cursor.cursor_value, extra_file_ids[0]); + assert_eq!(cursor.processed_count, 2); + assert_eq!( + checkpoint::object_map::Entity::find() + .filter(checkpoint::object_map::Column::RunId.eq(&run_id)) + .filter(checkpoint::object_map::Column::ObjectType.eq("file")) + .count(&target) + .await?, + 2 + ); + let failed_checkpoint = checkpoint::Entity::find_by_id(run_id.clone()) + .one(&target) + .await? + .expect("failed file migration checkpoint"); + assert_eq!(failed_checkpoint.status, "failed"); + assert_eq!( + failed_checkpoint.last_completed_stage.as_deref(), + Some("blobs") + ); + let failed_report: MigrationReport = serde_json::from_value(failed_checkpoint.report_json)?; + assert_eq!(failed_report.migrated_files, 2); + assert_eq!(failed_report.migrated_versions, 1); + target + .execute_unprepared("DROP TRIGGER fail_file_batch") + .await?; + target.close().await?; + + let report = migrate(MigrationOptions { + resume: true, + ..options + }) + .await?; + assert!(report.resumed); + assert_eq!(report.migrated_files, 4); + assert_eq!(report.migrated_versions, 1); + assert_eq!(report.mappings.files.len(), 4); + assert!(report.validation.passed); + + let target = Database::connect(&target_url).await?; + assert_eq!(ad::files::Entity::find().count(&target).await?, 4); + assert_eq!(ad::file_versions::Entity::find().count(&target).await?, 1); + assert_eq!( + checkpoint::object_map::Entity::find() + .filter(checkpoint::object_map::Column::RunId.eq(&run_id)) + .filter(checkpoint::object_map::Column::ObjectType.eq("file")) + .count(&target) + .await?, + 4 + ); + let completed_cursor = checkpoint::load_stage_cursor(&target, &run_id, "files") + .await? + .expect("completed file cursor"); + assert_eq!(completed_cursor.cursor_value, extra_file_ids[2]); + assert_eq!(completed_cursor.processed_count, 4); + target.close().await?; + + let _ = std::fs::remove_file(source_path); + let _ = std::fs::remove_file(target_path); + Ok(()) + } + + fn sqlite_url(path: &std::path::Path) -> String { + format!( + "sqlite://{}?mode=rwc", + path.to_string_lossy().replace('\\', "/") + ) + } + + async fn create_table(db: &DatabaseConnection, entity: E) -> Result<()> { + let schema = Schema::new(DbBackend::Sqlite); + db.execute(&schema.create_table_from_entity(entity)).await?; + Ok(()) + } + + async fn create_source_schema(db: &DatabaseConnection) -> Result<()> { + create_table(db, cr::nodes::Entity).await?; + create_table(db, cr::groups::Entity).await?; + create_table(db, cr::users::Entity).await?; + create_table(db, cr::storage_policies::Entity).await?; + create_table(db, cr::files::Entity).await?; + create_table(db, cr::entities::Entity).await?; + create_table(db, cr::file_entities::Entity).await?; + create_table(db, cr::shares::Entity).await?; + create_table(db, cr::metadata::Entity).await?; + create_table(db, cr::direct_links::Entity).await?; + create_table(db, cr::tasks::Entity).await?; + Ok(()) + } + + async fn create_target_schema(db: &DatabaseConnection) -> Result<()> { + create_table(db, ad::managed_followers::Entity).await?; + create_table(db, ad::storage_policy_groups::Entity).await?; + create_table(db, ad::storage_policies::Entity).await?; + create_table(db, ad::storage_policy_group_items::Entity).await?; + create_table(db, ad::users::Entity).await?; + create_table(db, ad::user_profiles::Entity).await?; + create_table(db, ad::folders::Entity).await?; + create_table(db, ad::file_blobs::Entity).await?; + create_table(db, ad::files::Entity).await?; + create_table(db, ad::file_versions::Entity).await?; + create_table(db, ad::entity_properties::Entity).await?; + create_table(db, ad::shares::Entity).await?; + create_table(db, ad::teams::Entity).await?; + create_table(db, ad::tags::Entity).await?; + create_table(db, ad::background_tasks::Entity).await?; + Ok(()) + } + + async fn seed_source(db: &DatabaseConnection) -> Result<()> { + let now = chrono::Utc::now().fixed_offset(); + let policy = cr::storage_policies::ActiveModel { + created_at: Set(now), + updated_at: Set(now), + deleted_at: Set(None), + name: Set("Local".to_string()), + r#type: Set("local".to_string()), + server: Set(None), + bucket_name: Set(None), + is_private: Set(Some(true)), + access_key: Set(None), + secret_key: Set(None), + max_size: Set(None), + dir_name_rule: Set(None), + file_name_rule: Set(None), + settings: Set(Some(json!({"chunk_size": 0}))), + node_id: Set(None), + ..Default::default() + } + .insert(db) + .await?; + let group = cr::groups::ActiveModel { + created_at: Set(now), + updated_at: Set(now), + deleted_at: Set(None), + name: Set("Administrators".to_string()), + max_storage: Set(Some(1024 * 1024)), + speed_limit: Set(None), + permissions: Set(vec![1]), + settings: Set(None), + storage_policy_id: Set(Some(policy.id)), + ..Default::default() + } + .insert(db) + .await?; + let user = cr::users::ActiveModel { + created_at: Set(now), + updated_at: Set(now), + deleted_at: Set(None), + email: Set("admin@example.test".to_string()), + nick: Set("admin".to_string()), + password: Set(Some("legacy:hash".to_string())), + status: Set("active".to_string()), + storage: Set(128), + two_factor_secret: Set(None), + avatar: Set(None), + settings: Set(None), + group_users: Set(group.id), + ..Default::default() + } + .insert(db) + .await?; + let folder = cr::files::ActiveModel { + created_at: Set(now), + updated_at: Set(now), + r#type: Set(1), + name: Set("Documents".to_string()), + size: Set(0), + primary_entity: Set(None), + is_symbolic: Set(false), + props: Set(None), + file_children: Set(None), + storage_policy_files: Set(Some(policy.id)), + owner_id: Set(user.id), + ..Default::default() + } + .insert(db) + .await?; + let entity = cr::entities::ActiveModel { + created_at: Set(now), + updated_at: Set(now), + deleted_at: Set(None), + r#type: Set(0), + source: Set("uploads/object.bin".to_string()), + size: Set(128), + reference_count: Set(1), + upload_session_id: Set(None), + recycle_options: Set(None), + storage_policy_entities: Set(policy.id), + created_by: Set(Some(user.id)), + ..Default::default() + } + .insert(db) + .await?; + let file = cr::files::ActiveModel { + created_at: Set(now), + updated_at: Set(now), + r#type: Set(0), + name: Set("hello.txt".to_string()), + size: Set(128), + primary_entity: Set(Some(entity.id)), + is_symbolic: Set(false), + props: Set(None), + file_children: Set(Some(folder.id)), + storage_policy_files: Set(Some(policy.id)), + owner_id: Set(user.id), + ..Default::default() + } + .insert(db) + .await?; + cr::file_entities::ActiveModel { + file_id: Set(file.id), + entity_id: Set(entity.id), + } + .insert(db) + .await?; + cr::metadata::ActiveModel { + created_at: Set(now), + updated_at: Set(now), + deleted_at: Set(None), + name: Set("author".to_string()), + value: Set("Cloudreve".to_string()), + is_public: Set(true), + file_id: Set(file.id), + ..Default::default() + } + .insert(db) + .await?; + cr::metadata::ActiveModel { + created_at: Set(now), + updated_at: Set(now), + deleted_at: Set(None), + name: Set("tag:Important".to_string()), + value: Set("#abc".to_string()), + is_public: Set(true), + file_id: Set(file.id), + ..Default::default() + } + .insert(db) + .await?; + cr::shares::ActiveModel { + created_at: Set(now), + updated_at: Set(now), + deleted_at: Set(None), + password: Set(Some("share-password".to_string())), + views: Set(4), + downloads: Set(2), + expires: Set(None), + remain_downloads: Set(Some(3)), + file_shares: Set(Some(file.id)), + user_shares: Set(Some(user.id)), + props: Set(None), + ..Default::default() + } + .insert(db) + .await?; + cr::direct_links::ActiveModel { + created_at: Set(now), + updated_at: Set(now), + deleted_at: Set(None), + name: Set("legacy-name.txt".to_string()), + downloads: Set(7), + speed: Set(1024), + file_id: Set(file.id), + ..Default::default() + } + .insert(db) + .await?; + for status in ["completed", "processing"] { + cr::tasks::ActiveModel { + created_at: Set(now), + updated_at: Set(now), + deleted_at: Set(None), + r#type: Set("remote_download".to_string()), + status: Set(status.to_string()), + public_state: Set(json!({"progress": 50})), + private_state: Set(Some("legacy-private-state".to_string())), + correlation_id: Set(None), + user_tasks: Set(Some(user.id)), + ..Default::default() + } + .insert(db) + .await?; + } + Ok(()) + } + + async fn seed_extra_blob_entities(db: &DatabaseConnection, count: usize) -> Result> { + let now = chrono::Utc::now().fixed_offset(); + let policy_id = cr::storage_policies::Entity::find() + .one(db) + .await? + .expect("seeded storage policy") + .id; + let user_id = cr::users::Entity::find() + .one(db) + .await? + .expect("seeded user") + .id; + let mut ids = Vec::with_capacity(count); + for index in 0..count { + let entity = cr::entities::ActiveModel { + created_at: Set(now), + updated_at: Set(now), + deleted_at: Set(None), + r#type: Set(0), + source: Set(format!("uploads/extra-{index}.bin")), + size: Set(256 + index as i64), + reference_count: Set(1), + upload_session_id: Set(None), + recycle_options: Set(None), + storage_policy_entities: Set(policy_id), + created_by: Set(Some(user_id)), + ..Default::default() + } + .insert(db) + .await?; + ids.push(entity.id); + } + Ok(ids) + } + + async fn seed_extra_files(db: &DatabaseConnection, entity_ids: &[i64]) -> Result> { + let now = chrono::Utc::now().fixed_offset(); + let policy_id = cr::storage_policies::Entity::find() + .one(db) + .await? + .expect("seeded storage policy") + .id; + let user_id = cr::users::Entity::find() + .one(db) + .await? + .expect("seeded user") + .id; + let folder_id = cr::files::Entity::find() + .filter(cr::files::Column::Type.eq(1)) + .one(db) + .await? + .expect("seeded folder") + .id; + let mut ids = Vec::with_capacity(entity_ids.len()); + for (index, entity_id) in entity_ids.iter().copied().enumerate() { + let file = cr::files::ActiveModel { + created_at: Set(now), + updated_at: Set(now), + r#type: Set(0), + name: Set(format!("extra-{index}.txt")), + size: Set(256 + index as i64), + primary_entity: Set(Some(entity_id)), + is_symbolic: Set(false), + props: Set(None), + file_children: Set(Some(folder_id)), + storage_policy_files: Set(Some(policy_id)), + owner_id: Set(user_id), + ..Default::default() + } + .insert(db) + .await?; + cr::file_entities::ActiveModel { + file_id: Set(file.id), + entity_id: Set(entity_id), + } + .insert(db) + .await?; + ids.push(file.id); + } + if entity_ids.len() >= 2 { + cr::file_entities::ActiveModel { + file_id: Set(ids[0]), + entity_id: Set(entity_ids[1]), + } + .insert(db) + .await?; + } + Ok(ids) + } +} diff --git a/src/migration/phases.rs b/src/migration/phases.rs new file mode 100644 index 0000000..2abbbe1 --- /dev/null +++ b/src/migration/phases.rs @@ -0,0 +1,855 @@ +use super::*; +use sea_orm::DatabaseTransaction; + +pub(super) async fn migrate_policies( + transaction: &DatabaseTransaction, + source: &SourceData, + options: &MigrationOptions, + context: &mut MigrationContext, + report: &mut MigrationReport, +) -> Result<()> { + let mut default_assigned = false; + for policy in &source.policies { + let Some(driver_type) = map_driver_type(&policy.r#type).filter(|_| { + !source_settings(&policy.settings) + .get("encryption") + .and_then(Value::as_bool) + .unwrap_or(false) + }) else { + report.record_skip( + "storage_policy", + Some(policy.id), + unsupported_policy_reason(policy) + .unwrap_or_else(|| "storage policy is not compatible with AD".to_string()), + ); + continue; + }; + let base_path = if policy.r#type == "local" { + local_policy_root(options, policy.id).to_string() + } else { + String::new() + }; + let model = ad::storage_policies::ActiveModel { + name: Set(policy.name.clone()), + driver_type: Set(driver_type.to_string()), + endpoint: Set(policy.server.clone().unwrap_or_default()), + bucket: Set(policy.bucket_name.clone().unwrap_or_default()), + access_key: Set(policy.access_key.clone().unwrap_or_default()), + secret_key: Set(policy.secret_key.clone().unwrap_or_default()), + base_path: Set(base_path), + remote_node_id: Set(None), + max_file_size: Set(policy.max_size.unwrap_or(0)), + allowed_types: Set(allowed_types(policy)), + options: Set(policy_options(policy)), + is_default: Set(!default_assigned), + chunk_size: Set(chunk_size(policy)), + created_at: Set(policy.created_at), + updated_at: Set(policy.updated_at), + remote_storage_target_key: Set(None), + ..Default::default() + } + .insert(transaction) + .await + .wrap_err_with(|| format!("migrate storage policy {}", policy.id))?; + default_assigned = true; + context.policies.insert(policy.id, model.id); + report.migrated_policies += 1; + } + Ok(()) +} + +pub(super) async fn migrate_policy_groups( + transaction: &DatabaseTransaction, + source: &SourceData, + context: &mut MigrationContext, + report: &mut MigrationReport, +) -> Result<()> { + for group in &source.groups { + let target_group = ad::storage_policy_groups::ActiveModel { + name: Set(group.name.clone()), + description: Set(format!("Migrated from Cloudreve group {}", group.id)), + is_enabled: Set(true), + is_default: Set(false), + created_at: Set(group.created_at), + updated_at: Set(group.updated_at), + ..Default::default() + } + .insert(transaction) + .await + .wrap_err_with(|| format!("migrate Cloudreve group {}", group.id))?; + context.policy_groups.insert(group.id, target_group.id); + report.migrated_policy_groups += 1; + + if let Some(source_policy_id) = group.storage_policy_id + && let Some(target_policy_id) = context.policies.get(&source_policy_id) + { + ad::storage_policy_group_items::ActiveModel { + group_id: Set(target_group.id), + policy_id: Set(*target_policy_id), + priority: Set(0), + min_file_size: Set(0), + max_file_size: Set(0), + created_at: Set(group.created_at), + ..Default::default() + } + .insert(transaction) + .await + .wrap_err_with(|| format!("link Cloudreve group {} storage policy", group.id))?; + } + } + Ok(()) +} + +pub(super) async fn migrate_users( + transaction: &DatabaseTransaction, + source: &SourceData, + password_hash: &str, + context: &mut MigrationContext, + report: &mut MigrationReport, +) -> Result<()> { + let groups: HashMap = source + .groups + .iter() + .map(|group| (group.id, group)) + .collect(); + let mut used_usernames = HashSet::new(); + for user in &source.users { + let username = unique_username(&user.nick, user.id, &mut used_usernames); + let group = groups.get(&user.group_users).copied(); + let role = if group.is_some_and(group_is_admin) { + "admin" + } else { + "user" + }; + let status = if user.status == "active" && user.deleted_at.is_none() { + "active" + } else { + "disabled" + }; + let target = ad::users::ActiveModel { + username: Set(username.clone()), + email: Set(user.email.clone()), + password_hash: Set(password_hash.to_string()), + role: Set(role.to_string()), + status: Set(status.to_string()), + session_version: Set(1), + email_verified_at: Set((status == "active").then_some(user.created_at)), + pending_email: Set(None), + storage_used: Set(user.storage), + storage_quota: Set(group.and_then(|group| group.max_storage).unwrap_or(0)), + policy_group_id: Set(context.policy_groups.get(&user.group_users).copied()), + created_at: Set(user.created_at), + updated_at: Set(user.updated_at), + config: Set(user.settings.as_ref().map(Value::to_string)), + must_change_password: Set(true), + ..Default::default() + } + .insert(transaction) + .await + .wrap_err_with(|| format!("migrate Cloudreve user {}", user.id))?; + + let avatar = user.avatar.clone().unwrap_or_default(); + let avatar_source = if avatar.is_empty() { + "none" + } else if avatar.to_ascii_lowercase().contains("gravatar") { + "gravatar" + } else { + "upload" + }; + ad::user_profiles::ActiveModel { + user_id: Set(target.id), + display_name: Set(Some(user.nick.clone())), + wopi_user_info: Set(None), + avatar_source: Set(avatar_source.to_string()), + avatar_key: Set((!avatar.is_empty()).then_some(avatar)), + avatar_version: Set(0), + created_at: Set(user.created_at), + updated_at: Set(user.updated_at), + } + .insert(transaction) + .await + .wrap_err_with(|| format!("create profile for Cloudreve user {}", user.id))?; + + context.users.insert(user.id, target.id); + context.usernames.insert(user.id, username); + report.migrated_users += 1; + } + Ok(()) +} + +pub(super) async fn migrate_folders( + transaction: &DatabaseTransaction, + source: &SourceData, + context: &mut MigrationContext, + report: &mut MigrationReport, +) -> Result<()> { + let mut pending: Vec<&cr::files::Model> = source + .folders + .iter() + .filter(|file| file.r#type == 1) + .collect(); + pending.sort_by_key(|folder| folder.id); + + while !pending.is_empty() { + let mut progress = false; + let mut next = Vec::new(); + for folder in pending { + let parent = folder.file_children; + if parent.is_some_and(|parent_id| { + source + .folders + .iter() + .any(|file| file.id == parent_id && file.r#type == 1) + && !context.folders.contains_key(&parent_id) + }) { + next.push(folder); + continue; + } + let Some(owner_id) = context.users.get(&folder.owner_id).copied() else { + report.record_skip( + "folder", + Some(folder.id), + format!("owner user {} was not migrated", folder.owner_id), + ); + continue; + }; + let target = ad::folders::ActiveModel { + name: Set(folder.name.clone()), + parent_id: Set(parent.and_then(|id| context.folders.get(&id).copied())), + team_id: Set(None), + owner_user_id: Set(Some(owner_id)), + created_by_user_id: Set(Some(owner_id)), + created_by_username: Set(context + .usernames + .get(&folder.owner_id) + .cloned() + .unwrap_or_default()), + policy_id: Set(folder + .storage_policy_files + .and_then(|id| context.policies.get(&id).copied())), + created_at: Set(folder.created_at), + updated_at: Set(folder.updated_at), + deleted_at: Set(None), + is_locked: Set(false), + ..Default::default() + } + .insert(transaction) + .await + .wrap_err_with(|| format!("migrate Cloudreve folder {}", folder.id))?; + context.folders.insert(folder.id, target.id); + report.migrated_folders += 1; + progress = true; + } + if !progress && !next.is_empty() { + bail!("Cloudreve folder hierarchy contains a cycle or a missing parent"); + } + pending = next; + } + Ok(()) +} + +pub(super) async fn migrate_blob_batch( + transaction: &DatabaseTransaction, + entities: &[cr::entities::Model], + reference_counts: &HashMap, + thumbnail_paths: &HashMap, + context: &MigrationContext, + report: &mut MigrationReport, +) -> Result> { + let mut mappings = Vec::with_capacity(entities.len()); + for entity in entities { + let Some(policy_id) = context + .policies + .get(&entity.storage_policy_entities) + .copied() + else { + report.record_skip( + "blob", + Some(entity.id), + format!( + "storage policy {} was not migrated", + entity.storage_policy_entities + ), + ); + continue; + }; + let reference_count = reference_counts.get(&entity.id).copied().unwrap_or(1); + let thumbnail_path = thumbnail_paths.get(&entity.id).cloned(); + let target = ad::file_blobs::ActiveModel { + hash: Set(opaque_blob_key(entity.id)), + size: Set(entity.size), + policy_id: Set(policy_id), + storage_path: Set(entity.source.clone()), + thumbnail_path: Set(thumbnail_path), + thumbnail_processor: Set(None), + thumbnail_version: Set(None), + ref_count: Set(reference_count), + created_at: Set(entity.created_at), + updated_at: Set(entity.updated_at), + ..Default::default() + } + .insert(transaction) + .await + .wrap_err_with(|| format!("migrate Cloudreve entity {}", entity.id))?; + mappings.push((entity.id, target.id)); + report.migrated_blobs += 1; + } + Ok(mappings) +} + +pub(super) async fn migrate_file_batch( + transaction: &DatabaseTransaction, + files: &[cr::files::Model], + associations: &HashMap>, + entities: &HashMap, + blob_mappings: &HashMap, + context: &MigrationContext, + report: &mut MigrationReport, +) -> Result> { + let mut mappings = Vec::with_capacity(files.len()); + for file in files { + if file.is_symbolic { + report.record_skip( + "file", + Some(file.id), + "symbolic/placeholder files are not representable in AD", + ); + continue; + } + let Some(owner_id) = context.users.get(&file.owner_id).copied() else { + report.record_skip( + "file", + Some(file.id), + format!("owner user {} was not migrated", file.owner_id), + ); + continue; + }; + let mut version_entities: Vec<&cr::entities::Model> = associations + .get(&file.id) + .into_iter() + .flatten() + .filter_map(|entity_id| entities.get(entity_id)) + .filter(|entity| entity.r#type == 0 && blob_mappings.contains_key(&entity.id)) + .collect(); + version_entities.sort_by_key(|entity| entity.created_at); + let primary_entity_id = file + .primary_entity + .filter(|id| blob_mappings.contains_key(id)) + .or_else(|| version_entities.last().map(|entity| entity.id)); + let Some(primary_entity_id) = primary_entity_id else { + report.record_skip( + "file", + Some(file.id), + "file has no migratable version entity", + ); + report + .warnings + .push(format!("file {} has no migratable version entity", file.id)); + continue; + }; + let blob_id = blob_mappings[&primary_entity_id]; + let (mime_type, compound_extension, extension, category) = file_classification(&file.name); + let target = ad::files::ActiveModel { + name: Set(file.name.clone()), + folder_id: Set(file + .file_children + .and_then(|folder_id| context.folders.get(&folder_id).copied())), + team_id: Set(None), + blob_id: Set(blob_id), + size: Set(file.size), + owner_user_id: Set(Some(owner_id)), + created_by_user_id: Set(Some(owner_id)), + created_by_username: Set(context + .usernames + .get(&file.owner_id) + .cloned() + .unwrap_or_default()), + mime_type: Set(mime_type), + created_at: Set(file.created_at), + updated_at: Set(file.updated_at), + deleted_at: Set(None), + is_locked: Set(false), + extension: Set(extension), + compound_extension: Set(compound_extension), + file_category: Set(category), + ..Default::default() + } + .insert(transaction) + .await + .wrap_err_with(|| format!("migrate Cloudreve file {}", file.id))?; + mappings.push((file.id, target.id)); + report.migrated_files += 1; + + let historical: Vec<&cr::entities::Model> = version_entities + .into_iter() + .filter(|entity| entity.id != primary_entity_id) + .collect(); + for (index, entity) in historical.into_iter().enumerate() { + ad::file_versions::ActiveModel { + file_id: Set(target.id), + blob_id: Set(blob_mappings[&entity.id]), + version: Set((index + 1) as i64), + size: Set(entity.size), + created_at: Set(entity.created_at), + ..Default::default() + } + .insert(transaction) + .await + .wrap_err_with(|| format!("migrate version {} for file {}", entity.id, file.id))?; + report.migrated_versions += 1; + } + } + Ok(mappings) +} + +pub(super) async fn migrate_metadata( + transaction: &DatabaseTransaction, + source_db: &DatabaseConnection, + source: &SourceData, + file_mappings: &HashMap, + context: &MigrationContext, + report: &mut MigrationReport, +) -> Result<()> { + let source_file_ids = source + .metadata + .iter() + .map(|metadata| metadata.file_id) + .collect::>(); + let source_files = load_source_files(source_db, &source_file_ids).await?; + let mut tags: HashMap<(i64, String), i64> = HashMap::new(); + for metadata in &source.metadata { + let Some(source_file) = source_files.get(&metadata.file_id) else { + report.record_skip( + "metadata", + Some(metadata.id), + format!("source file {} does not exist", metadata.file_id), + ); + continue; + }; + let target_entity = if source_file.r#type == 0 { + file_mappings + .get(&metadata.file_id) + .copied() + .map(|id| ("file", id)) + } else { + context + .folders + .get(&metadata.file_id) + .copied() + .map(|id| ("folder", id)) + }; + let Some((entity_type, entity_id)) = target_entity else { + report.record_skip( + "metadata", + Some(metadata.id), + format!("source entity {} was not migrated", metadata.file_id), + ); + continue; + }; + + if let Some(source_tag_name) = tag_name(&metadata.name) { + let Some(owner_user_id) = context.users.get(&source_file.owner_id).copied() else { + report.record_skip( + "tag_assignment", + Some(metadata.id), + format!("owner user {} was not migrated", source_file.owner_id), + ); + continue; + }; + let name = target_tag_name(source_tag_name); + if name.is_empty() { + report.record_skip( + "tag_assignment", + Some(metadata.id), + "tag name is empty after normalization", + ); + continue; + } + let normalized_name = normalize_tag_name(&name); + let tag_id = match tags.get(&(owner_user_id, normalized_name.clone())) { + Some(tag_id) => *tag_id, + None => { + let tag = ad::tags::ActiveModel { + scope_type: Set("personal".to_string()), + owner_user_id: Set(Some(owner_user_id)), + team_id: Set(None), + name: Set(name), + normalized_name: Set(normalized_name.clone()), + color: Set(target_tag_color(&metadata.value)), + sort_order: Set(0), + created_at: Set(metadata.created_at), + updated_at: Set(metadata.updated_at), + ..Default::default() + } + .insert(transaction) + .await + .wrap_err_with(|| format!("migrate tag metadata {}", metadata.id))?; + tags.insert((owner_user_id, normalized_name), tag.id); + report.migrated_tags += 1; + tag.id + } + }; + ad::entity_properties::ActiveModel { + entity_type: Set(entity_type.to_string()), + entity_id: Set(entity_id), + namespace: Set("system.tags".to_string()), + name: Set(tag_id.to_string()), + value: Set(None), + ..Default::default() + } + .insert(transaction) + .await + .wrap_err_with(|| format!("attach migrated tag for metadata {}", metadata.id))?; + report.migrated_properties += 1; + report.migrated_tag_assignments += 1; + report.tag_assignments.push(TagAssignmentReport { + source_metadata_id: metadata.id, + source_entity_id: metadata.file_id, + target_entity_type: entity_type.to_string(), + target_entity_id: entity_id, + target_tag_id: tag_id, + tag_name: source_tag_name.to_string(), + }); + continue; + } + + let namespace = if metadata.is_public { + "cloudreve.public" + } else { + "cloudreve.private" + }; + ad::entity_properties::ActiveModel { + entity_type: Set(entity_type.to_string()), + entity_id: Set(entity_id), + namespace: Set(namespace.to_string()), + name: Set(metadata.name.clone()), + value: Set(Some(metadata.value.clone())), + ..Default::default() + } + .insert(transaction) + .await + .wrap_err_with(|| format!("migrate metadata {}", metadata.id))?; + report.migrated_properties += 1; + } + Ok(()) +} + +pub(super) async fn migrate_direct_links( + transaction: &DatabaseTransaction, + source_db: &DatabaseConnection, + source: &SourceData, + file_mappings: &HashMap, + context: &MigrationContext, + direct_link_secret: Option<&str>, + report: &mut MigrationReport, +) -> Result<()> { + if source.direct_links.is_empty() { + return Ok(()); + } + let Some(secret) = direct_link_secret else { + for link in &source.direct_links { + report.record_skip( + "direct_link", + Some(link.id), + if link.deleted_at.is_some() { + "source direct link is deleted and must not be reactivated" + } else { + "AD direct_link_secret was not supplied" + }, + ); + } + report.warnings.push(format!( + "{} Cloudreve direct links were not regenerated because --direct-link-secret was not supplied", + source.direct_links.len() + )); + return Ok(()); + }; + let source_file_ids = source + .direct_links + .iter() + .map(|link| link.file_id) + .collect::>(); + let source_files = load_source_files(source_db, &source_file_ids).await?; + for link in &source.direct_links { + if link.deleted_at.is_some() { + report.record_skip( + "direct_link", + Some(link.id), + "source direct link is deleted and must not be reactivated", + ); + continue; + } + let Some(source_file) = source_files.get(&link.file_id) else { + report.record_skip( + "direct_link", + Some(link.id), + format!("source file {} does not exist", link.file_id), + ); + continue; + }; + let Some(file_id) = file_mappings.get(&link.file_id).copied() else { + report.record_skip( + "direct_link", + Some(link.id), + format!("source file {} was not migrated", link.file_id), + ); + continue; + }; + let Some(owner_user_id) = context.users.get(&source_file.owner_id).copied() else { + report.record_skip( + "direct_link", + Some(link.id), + format!("owner user {} was not migrated", source_file.owner_id), + ); + continue; + }; + let url = direct_link_url(file_id, owner_user_id, &source_file.name, secret)?; + ad::entity_properties::ActiveModel { + entity_type: Set("file".to_string()), + entity_id: Set(file_id), + namespace: Set("cloudreve.direct_links".to_string()), + name: Set(link.id.to_string()), + value: Set(Some( + json!({ + "url": url.clone(), + "source_direct_link_id": link.id, + "source_file_id": link.file_id, + "source_name": link.name, + "source_downloads": link.downloads, + "source_speed_limit": link.speed, + }) + .to_string(), + )), + ..Default::default() + } + .insert(transaction) + .await + .wrap_err_with(|| format!("archive direct link {} mapping", link.id))?; + report.migrated_properties += 1; + report.migrated_direct_links += 1; + report.direct_links.push(DirectLinkReport { + source_direct_link_id: link.id, + source_file_id: link.file_id, + target_file_id: file_id, + source_name: link.name.clone(), + source_downloads: link.downloads, + source_speed_limit: link.speed, + url, + }); + } + Ok(()) +} + +pub(super) async fn migrate_tasks( + transaction: &DatabaseTransaction, + source: &SourceData, + context: &mut MigrationContext, + report: &mut MigrationReport, +) -> Result<()> { + let active_count = source + .tasks + .iter() + .filter(|task| source_task_was_active(&task.status)) + .count(); + if active_count > 0 { + report.warnings.push(format!( + "{active_count} active Cloudreve tasks were archived as canceled terminal records and were not resumed" + )); + } + + for task in &source.tasks { + let status = archived_task_status(&task.status); + let duration_ms = (task.updated_at - task.created_at) + .num_milliseconds() + .max(0); + let task_name = format!( + "cloudreve-legacy-{}", + task.r#type + .replace(|character: char| !character.is_ascii_alphanumeric(), "-") + ); + let result = json!({ + "duration_ms": duration_ms, + "summary": format!("Archived Cloudreve {} task with source status {}", task.r#type, task.status), + }); + let runtime = json!({ + "source": "cloudreve", + "source_task_id": task.id, + "source_type": task.r#type, + "source_status": task.status, + "source_public_state": task.public_state, + "source_private_state": task.private_state, + "source_correlation_id": task.correlation_id, + "source_deleted_at": task.deleted_at, + "archived_without_resume": true, + }); + let started_at = (!matches!(task.status.as_str(), "queued")).then_some(task.created_at); + let expires_at = task + .updated_at + .checked_add_signed(chrono::Duration::days(36_500)) + .unwrap_or(task.updated_at); + let target = ad::background_tasks::ActiveModel { + kind: Set("system_runtime".to_string()), + status: Set(status.to_string()), + creator_user_id: Set(task + .user_tasks + .and_then(|user_id| context.users.get(&user_id).copied())), + team_id: Set(None), + share_id: Set(None), + display_name: Set(format!("Cloudreve task: {}", task.r#type)), + payload_json: Set(json!({"task_name": task_name}).to_string()), + result_json: Set(Some(result.to_string())), + steps_json: Set(Some("[]".to_string())), + progress_current: Set(i64::from(status == "succeeded")), + progress_total: Set(1), + status_text: Set(Some(format!( + "Archived from Cloudreve with source status {}; execution was not resumed", + task.status + ))), + attempt_count: Set(0), + max_attempts: Set(1), + next_run_at: Set(task.updated_at), + processing_token: Set(0), + processing_started_at: Set(None), + last_heartbeat_at: Set(None), + lease_expires_at: Set(None), + started_at: Set(started_at), + finished_at: Set(Some(task.updated_at)), + last_error: Set( + (status == "failed").then(|| "Cloudreve task ended with status error".to_string()) + ), + failure_can_retry: Set(Some(false)), + expires_at: Set(expires_at), + created_at: Set(task.created_at), + updated_at: Set(task.updated_at), + runtime_json: Set(Some(runtime.to_string())), + ..Default::default() + } + .insert(transaction) + .await + .wrap_err_with(|| format!("archive Cloudreve task {}", task.id))?; + context.tasks.insert(task.id, target.id); + report.migrated_tasks += 1; + } + Ok(()) +} + +pub(super) async fn migrate_shares( + transaction: &DatabaseTransaction, + source_db: &DatabaseConnection, + source: &SourceData, + file_mappings: &HashMap, + context: &mut MigrationContext, + report: &mut MigrationReport, +) -> Result<()> { + let source_file_ids = source + .shares + .iter() + .filter_map(|share| share.file_shares) + .collect::>(); + let source_files = load_source_files(source_db, &source_file_ids).await?; + for share in &source.shares { + let Some(source_user_id) = share.user_shares else { + report.record_skip("share", Some(share.id), "share has no owner user"); + continue; + }; + let Some(user_id) = context.users.get(&source_user_id).copied() else { + report.record_skip( + "share", + Some(share.id), + format!("owner user {source_user_id} was not migrated"), + ); + continue; + }; + let Some(source_file_id) = share.file_shares else { + report.record_skip("share", Some(share.id), "share has no file/folder target"); + continue; + }; + let source_file = source_files.get(&source_file_id); + let file_id = source_file + .filter(|file| file.r#type == 0) + .and_then(|_| file_mappings.get(&source_file_id).copied()); + let folder_id = source_file + .filter(|file| file.r#type == 1) + .and_then(|_| context.folders.get(&source_file_id).copied()); + if file_id.is_none() && folder_id.is_none() { + report.record_skip( + "share", + Some(share.id), + format!("source target {source_file_id} was not migrated"), + ); + continue; + } + let password = match share.password.as_deref().filter(|value| !value.is_empty()) { + Some(password) => Some(hash_password(password)?), + None => None, + }; + let max_downloads = share + .remain_downloads + .map(|remaining| share.downloads.saturating_add(remaining)) + .unwrap_or(0); + let target = ad::shares::ActiveModel { + token: Set(share_token(share.id)), + user_id: Set(user_id), + team_id: Set(None), + file_id: Set(file_id), + folder_id: Set(folder_id), + password: Set(password), + expires_at: Set(share.expires), + max_downloads: Set(max_downloads), + download_count: Set(share.downloads), + view_count: Set(share.views), + created_at: Set(share.created_at), + updated_at: Set(share.updated_at), + ..Default::default() + } + .insert(transaction) + .await + .wrap_err_with(|| format!("migrate share {}", share.id))?; + context.shares.insert(share.id, target.id); + report.migrated_shares += 1; + } + Ok(()) +} + +pub(super) async fn load_source_files( + source_db: &DatabaseConnection, + source_ids: &[i64], +) -> Result> { + const QUERY_ID_BATCH_SIZE: usize = 500; + + let source_ids = source_ids.iter().copied().collect::>(); + let mut files = HashMap::with_capacity(source_ids.len()); + let source_ids = source_ids.into_iter().collect::>(); + for source_ids in source_ids.chunks(QUERY_ID_BATCH_SIZE) { + for file in cr::files::Entity::find() + .filter(cr::files::Column::Id.is_in(source_ids.iter().copied())) + .all(source_db) + .await? + { + files.insert(file.id, file); + } + } + Ok(files) +} + +pub(super) fn associations( + files: &[cr::files::Model], + file_entities: &[cr::file_entities::Model], +) -> HashMap> { + let mut result: HashMap> = HashMap::new(); + for relation in file_entities { + result + .entry(relation.file_id) + .or_default() + .push(relation.entity_id); + } + for file in files { + if let Some(primary_entity) = file.primary_entity { + let values = result.entry(file.id).or_default(); + if !values.contains(&primary_entity) { + values.push(primary_entity); + } + } + } + result +}