diff --git a/apps/cli/commands/pull-reprint.ts b/apps/cli/commands/pull-reprint.ts index 9805c2261b..9f781d3c1d 100644 --- a/apps/cli/commands/pull-reprint.ts +++ b/apps/cli/commands/pull-reprint.ts @@ -21,6 +21,11 @@ import { sortSites } from '@studio/common/lib/sort-sites'; import { PullReprintCommandLoggerAction as LoggerAction } from '@studio/common/logger-actions'; import { __, sprintf } from '@wordpress/i18n'; import chalk from 'chalk'; +import { + getSetAdminCredentialsRequestBody, + shouldSetAdminCredentials, + toUrlSearchParams, +} from 'cli/lib/admin-credentials'; import { enableReprintExporter, rotateReprintSecret } from 'cli/lib/api'; import { lockCliConfig, @@ -30,7 +35,12 @@ import { unlockCliConfig, } from 'cli/lib/cli-config/core'; import { getSiteUrl, updateSiteAutoStart, updateSiteLatestCliPid } from 'cli/lib/cli-config/sites'; -import { connectToDaemon, disconnectFromDaemon, emitCliEvent } from 'cli/lib/daemon-client'; +import { + connectToDaemon, + disconnectFromDaemon, + emitCliEvent, + isProcessRunning, +} from 'cli/lib/daemon-client'; import { type ReprintProcessResult, runReprintCommandUntilComplete, @@ -40,7 +50,6 @@ import { getReprintStatePath, hasSkippedFiles, readReprintState, - shouldRestartFilesSyncIndex, } from 'cli/lib/pull/reprint-state'; import { ensureImportedSiteSqliteReady, @@ -51,7 +60,7 @@ import { buildAutoLoginUrl } from 'cli/lib/site-utils'; import { fetchSyncableSites } from 'cli/lib/sync-api'; import { pickSyncSite } from 'cli/lib/sync-site-picker'; import { getPrettyPath } from 'cli/lib/utils'; -import { startWordPressServer } from 'cli/lib/wordpress-server-manager'; +import { getProcessName, startWordPressServer } from 'cli/lib/wordpress-server-manager'; import { Logger, LoggerError } from 'cli/logger'; import { StudioArgv } from 'cli/types'; import type { SyncSite } from '@studio/common/types/sync'; @@ -132,11 +141,7 @@ const PULLS_ROOT = path.join( os.homedir(), '.studio', 'pulls' ); const pullStageOrder = [ 'initialized', - 'essential-files-complete', - 'flattened', - 'db-downloaded', - 'db-applied', - 'runtime-generated', + 'pulled', 'site-registered', 'site-started', 'completed', @@ -182,6 +187,14 @@ interface PullSessionMetadata { runtimeDirectory: string; runtimeBlueprintPath: string; stage: PullStage; + /** + * True once this pull has reached the 'completed' stage at least once. + * A re-run after that point is a delta re-pull: the stage machine is + * reset so every phase re-executes, and the non-empty site-directory + * guard is skipped (the directory legitimately holds the previous + * pull's output). + */ + hasCompletedOnce?: boolean; siteId?: string; port?: number; localUrl?: string; @@ -220,20 +233,24 @@ class PullError extends LoggerError { * resolvePullSource → * resolvePullMetadata → * runPreflight (with secret-rotate retry on WP.com) → - * downloadEssentialSiteFiles → - * refreshFlattenedSiteDirectory → - * downloadRemoteDatabase → * ensurePort → - * applyDownloadedDatabase → - * generateRuntimeConfiguration → + * runFullPull (one `reprint pull`: files-pull → db-pull → db-apply → + * flat-docroot → apply-runtime) → * registerSite → * startWordPressServer → * downloadSkippedFiles. * - * Every stage persists its completion to `pull.json` before moving - * on (see {@link recordCompletedStage}), so a crash anywhere in the - * pipeline resumes at the next stage on re-run. `--abort` detours to - * {@link abortPull} instead. + * Each Studio stage persists to `pull.json` (see {@link + * recordCompletedStage}), so a crash resumes at the next stage; within + * the pull, reprint resumes its own pipeline from its last completed + * sub-stage. `--abort` detours to {@link abortPull} instead. + * + * Re-running after a pull reached 'completed' performs a delta + * re-pull: Studio's stage machine resets to 'initialized' and reprint + * resets its own sub-command state via prepare_repull(). Each phase is + * incremental — files re-sync as a delta (re-index + diff), the + * database is fully re-downloaded and re-applied (the dump is + * idempotent, so edits, inserts, and deletes all propagate). */ export async function runCommand( userProvidedUrl?: string, @@ -271,10 +288,30 @@ export async function runCommand( ); const apiUrl = getReprintApiUrlForSite( studioMetadata.normalizedUrl ); + // A previously completed pull re-runs as a delta sync: reset the + // stage machine so every phase executes again. Reprint does the + // incremental work against the preserved state directory — files + // re-sync as a delta (re-index + diff), the database is fully + // re-downloaded and re-applied (the dump is idempotent, so remote + // edits, inserts, and deletes all land locally). + const isRepull = studioMetadata.stage === 'completed'; + if ( isRepull ) { + studioMetadata.stage = 'initialized'; + studioMetadata.hasCompletedOnce = true; + savePullMetadata( studioMetadata ); + + // Re-verify connectivity (and give the secret-rotation retry path + // a chance to run) instead of trusting the cached preflight from + // the original pull, which may be days old. + fs.rmSync( path.join( studioMetadata.stateDirectory, 'preflight.json' ), { force: true } ); + } + // Refuse to clobber an existing non-empty site directory before the // flatten stage. Once flattened, the directory legitimately holds - // reprint's output; before that, anything there is user data. - if ( ! hasPullCompletedStage( studioMetadata, 'flattened' ) ) { + // reprint's output; before that, anything there is user data. On a + // re-pull (hasCompletedOnce) the directory holds the previous pull's + // output, so the guard doesn't apply. + if ( ! studioMetadata.hasCompletedOnce && ! hasPullCompletedStage( studioMetadata, 'pulled' ) ) { if ( ( await fsUtils.pathExists( studioMetadata.sitePath ) ) && ! ( await fsUtils.isEmptyDir( studioMetadata.sitePath ) ) @@ -317,13 +354,13 @@ export async function runCommand( fs.mkdirSync( studioMetadata.runtimeDirectory, { recursive: true } ); fs.mkdirSync( studioMetadata.sitePath, { recursive: true } ); - if ( studioMetadata.stage === 'completed' ) { - printCompletionMessage( studioMetadata ); - process.exit( 0 ); - } - const isResume = ! created || fs.readdirSync( studioMetadata.stateDirectory ).length > 0; - if ( isResume ) { + if ( isRepull ) { + console.log( + `Updating "${ studioMetadata.siteName }" from ${ studioMetadata.normalizedUrl } (delta sync)` + ); + console.log( '' ); + } else if ( isResume ) { console.log( `Resuming previous pull of "${ studioMetadata.siteName }" from ${ studioMetadata.normalizedUrl }` ); @@ -354,9 +391,14 @@ export async function runCommand( try { preflight = await runPreflight( studioMetadata, apiUrl, secret, verbose ); } catch ( preflightError ) { - // The stored secret may have expired. Resolve the WP.com site - // (loading the site list only now, if we haven't already) and - // rotate the secret before retrying the preflight. + // Preflight against ?reprint-api can fail for two reasons we can + // recover from on WP.com: the stored secret expired, or the + // wpcomsh exporter gate (`reprint_exporter_enabled`, a 60-minute + // sliding window) closed since the last run. A cached-secret + // resume skips the happy-path enable above, so this is the common + // case on a delta re-pull. Resolve the WP.com site (loading the + // site list only now, if we haven't already), then both rotate the + // secret AND re-enable the exporter before retrying. if ( sourceSite.wpComSite && sourceSite.wpComToken ) { secret = await rotateReprintSecret( sourceSite.wpComSite.id, @@ -383,6 +425,14 @@ export async function runCommand( sourceSite.wpComToken = token; secret = await rotateReprintSecret( matched.id, token.accessToken ); } + // Rotating the secret does not bump `reprint_exporter_enabled`, so + // re-open the gate explicitly; otherwise the retry hits the same + // closed window and ?reprint-api falls through to an HTML page. + await enableReprintExporter( + sourceSite.wpComSite.id, + sourceSite.wpComToken.accessToken, + verbose + ); preflight = await runPreflight( studioMetadata, apiUrl, secret, verbose ); } studioMetadata.remoteSiteUrl = preflight.siteurl || studioMetadata.normalizedUrl; @@ -390,29 +440,18 @@ export async function runCommand( studioMetadata.secret = secret; savePullMetadata( studioMetadata ); - if ( ! hasPullCompletedStage( studioMetadata, 'essential-files-complete' ) ) { - await downloadEssentialSiteFiles( studioMetadata, apiUrl, secret, verbose ); - } - - if ( ! hasPullCompletedStage( studioMetadata, 'flattened' ) ) { - logger.reportStart( LoggerAction.CREATE_SITE, __( 'Preparing site directory…' ) ); - await refreshFlattenedSiteDirectory( studioMetadata, verbose ); - logger.reportSuccess( __( 'Site directory prepared' ) ); - recordCompletedStage( studioMetadata, 'flattened' ); - } - - if ( ! hasPullCompletedStage( studioMetadata, 'db-downloaded' ) ) { - await downloadRemoteDatabase( studioMetadata, apiUrl, secret, verbose ); - } - + // Allocate the local port before the pull so db-apply (run inside + // the composite `pull`) can rewrite the remote site URL to the + // local one the Studio server will serve. await ensurePort( studioMetadata ); - if ( ! hasPullCompletedStage( studioMetadata, 'db-applied' ) ) { - await applyDownloadedDatabase( studioMetadata, secret, verbose ); - } - - if ( ! hasPullCompletedStage( studioMetadata, 'runtime-generated' ) ) { - await generateRuntimeConfiguration( studioMetadata, verbose ); + // A single `reprint pull` runs the whole pipeline in one PHP-WASM + // fork: files-pull → db-pull → db-apply → flat-docroot → + // apply-runtime. reprint owns the stage ordering internally and, on + // a delta re-pull, resets its own sub-command state via + // prepare_repull(). + if ( ! hasPullCompletedStage( studioMetadata, 'pulled' ) ) { + await runFullPull( studioMetadata, apiUrl, secret, verbose ); } let createdSiteRecord = false; @@ -469,16 +508,47 @@ export async function runCommand( try { await connectToDaemon(); - const processDesc = await startWordPressServer( site, logger, runtimeStartOptions ); - logger.reportSuccess( __( 'WordPress server started' ) ); - if ( processDesc.status === 'online' ) { - await updateSiteLatestCliPid( site.id, processDesc.pid ); + // On a re-pull, the site's server is often already running. + // The synced files and database are picked up live (PHP + // opens them per request), so there's nothing to restart — + // but db-apply rebuilt the database from the remote dump, + // wiping the local admin user and the studio_admin_username + // option that /studio-auto-login depends on. A server start + // re-applies the credentials; when we skip the restart we + // must re-apply them over the running site's admin API. + // A connection failure means the daemon's view is stale and + // the server is actually down, so fall through to a start + // (which re-applies the credentials itself). + const runningProcess = await isProcessRunning( getProcessName( site.id ) ); + const credentialsResult = runningProcess + ? await reapplyAdminCredentials( site ) + : 'unreachable'; + if ( runningProcess && credentialsResult !== 'unreachable' ) { + logger.reportSuccess( __( 'WordPress server already running' ) ); + // Mirror the start branch (and `studio site start`'s + // already-running path): refresh latestCliPid so + // running-status checks match the live process, and keep + // autoStart enabled as every pull has. + if ( runningProcess.status === 'online' ) { + await updateSiteLatestCliPid( site.id, runningProcess.pid ); + } + await updateSiteAutoStart( site.id, true ); + studioMetadata.localUrl = getSiteUrl( site ); + savePullMetadata( studioMetadata ); + recordCompletedStage( studioMetadata, 'site-started' ); + } else { + const processDesc = await startWordPressServer( site, logger, runtimeStartOptions ); + logger.reportSuccess( __( 'WordPress server started' ) ); + + if ( processDesc.status === 'online' ) { + await updateSiteLatestCliPid( site.id, processDesc.pid ); + } + await updateSiteAutoStart( site.id, true ); + studioMetadata.localUrl = getSiteUrl( site ); + savePullMetadata( studioMetadata ); + recordCompletedStage( studioMetadata, 'site-started' ); } - await updateSiteAutoStart( site.id, true ); - studioMetadata.localUrl = getSiteUrl( site ); - savePullMetadata( studioMetadata ); - recordCompletedStage( studioMetadata, 'site-started' ); } catch ( serverError ) { throw new LoggerError( __( 'Failed to start the WordPress server for the pulled site.' ), @@ -740,177 +810,61 @@ function readPullMetadata( metadataPath: string ): PullSessionMetadata | null { } /** - * Apply the downloaded SQL dump into an SQLite database that Studio - * will mount as the imported site's `wp-content/database/.ht.sqlite`. + * Run reprint's composite `pull` command: the whole site-clone + * pipeline (preflight → files-pull → db-pull → db-apply → + * flat-docroot → apply-runtime) in a single PHP-WASM fork, with + * reprint owning the stage ordering and, when the prior pull already + * completed, resetting its own sub-command state for a delta re-pull + * via prepare_repull(). + * + * The SQLite target geometry: + * - If preflight exposed the remote `wp-content` (contentDir set), + * the database lands under `rawDirectory + contentDir`, an + * already-mounted host path that flat-docroot later symlinks into + * the flattened site. + * - Otherwise it falls back to `sitePath/wp-content`. * - * Mount strategy depends on whether preflight told us where the - * remote site kept its `wp-content`: - * - If it did (contentDir set), reprint writes the SQLite file under - * `rawDirectory + contentDir`, which is already a mounted host path, - * so no extra mount is needed. - * - If not, we write into `sitePath/wp-content` directly and mount - * that host path into the PHP WASM runtime so reprint can see it. + * The flattened site (`--flatten-to`) and runtime output + * (`--output-dir`) directories are mounted up front so the single + * fork can write them onto the host filesystem. `ensurePort` must + * run first so `--new-site-url` points at the local server. * - * Advances the pull stage to 'db-applied' on success; thrown errors - * propagate up to the pull orchestrator for a user-facing abort. + * Advances the pull stage to 'pulled'. */ -export async function applyDownloadedDatabase( +export async function runFullPull( metadata: PullSessionMetadata, + apiUrl: string, secret: string, verbose: boolean ): Promise< void > { - logger.reportStart( LoggerAction.IMPORT_SQL, __( 'Applying database…' ) ); const contentDir = getContentDirFromState( metadata.stateDirectory ); const sqlitePath = contentDir ? `${ metadata.rawDirectory }${ contentDir }/database/.ht.sqlite` : `${ metadata.sitePath }/wp-content/database/.ht.sqlite`; - const dbApplyMounts = contentDir - ? [] - : [ { hostPath: metadata.sitePath, vfsPath: metadata.sitePath } ]; - await runReprintCommandUntilComplete( - metadata.stateDirectory, - metadata.rawDirectory, - [ - 'db-apply', - getReprintApiUrlForSite( metadata.normalizedUrl ), - `--state-dir=${ metadata.stateDirectory }`, - `--fs-root=${ metadata.rawDirectory }`, - '--target-engine=sqlite', - `--target-sqlite-path=${ sqlitePath }`, - `--new-site-url=${ metadata.localUrl! }`, - `--secret=${ secret }`, - '--no-adaptive', - ], - ( progress ) => logger.reportProgress( progress ), - { - progressLabel: __( 'Applying database' ), - mounts: dbApplyMounts, - verboseCommands: verbose, - } - ); - logger.reportSuccess( __( 'Database applied' ) ); - recordCompletedStage( metadata, 'db-applied' ); -} - -/** - * Fetch the minimum set of files needed to produce a usable flattened - * site directory: wp-config.php, wp-includes, active plugins/themes, - * uploads. Heavier wp-content payload (unused plugins, dev caches) - * is deferred to {@link downloadSkippedFiles} so the site becomes - * runnable sooner. - * - * Advances the pull stage to 'essential-files-complete'. - */ -export async function downloadEssentialSiteFiles( - metadata: PullSessionMetadata, - apiUrl: string, - secret: string, - verbose: boolean -): Promise< void > { - // When reprint crashed mid-indexing on a previous run without - // persisting a cursor, a plain files-sync would try to resume from - // a cursor that doesn't exist. Clear that state via - // `files-sync --abort` so the real sync below starts fresh. - if ( shouldRestartFilesSyncIndex( metadata.stateDirectory ) ) { - logger.reportWarning( - __( - 'Restarting remote file indexing before resume because the previous run did not save a resumable cursor.' - ) - ); - await runReprintCommandUntilComplete( - metadata.stateDirectory, - metadata.rawDirectory, - buildFilesSyncArgs( metadata, apiUrl, secret, [ '--abort' ] ), - undefined, - { - verboseCommands: verbose, - } - ); - logger.reportSuccess( __( 'Interrupted file indexing state cleared' ) ); - } - - logger.reportStart( LoggerAction.DOWNLOAD_FILES, __( 'Downloading site files…' ) ); - await runReprintCommandUntilComplete( - metadata.stateDirectory, - metadata.rawDirectory, - buildFilesSyncArgs( metadata, apiUrl, secret, [ - '--filter=essential-files', - '--follow-symlinks', - ] ), - ( progress ) => logger.reportProgress( progress ), - { - progressLabel: 'Downloading files', - verboseCommands: verbose, - } - ); - logger.reportSuccess( __( 'Files downloaded' ) ); - recordCompletedStage( metadata, 'essential-files-complete' ); -} -/** - * Stream the remote database into `stateDirectory/db.sql` via reprint's - * `db-sync` command. We download-first-then-apply (rather than piping - * straight into sqlite) so a crash during apply can resume without - * re-fetching the dump. - * - * Advances the pull stage to 'db-downloaded'. - */ -export async function downloadRemoteDatabase( - metadata: PullSessionMetadata, - apiUrl: string, - secret: string, - verbose: boolean -): Promise< void > { - logger.reportStart( LoggerAction.DOWNLOAD_SQL, __( 'Downloading database…' ) ); + logger.reportStart( LoggerAction.DOWNLOAD_FILES, __( 'Pulling site…' ) ); await runReprintCommandUntilComplete( metadata.stateDirectory, metadata.rawDirectory, [ - 'db-sync', + 'pull', apiUrl, `--secret=${ secret }`, - '--sql-output=file', + '--filter=essential-files', + '--target-engine=sqlite', + `--target-sqlite-path=${ sqlitePath }`, + `--new-site-url=${ metadata.localUrl! }`, + `--flatten-to=${ metadata.sitePath }`, + '--runtime=playground-cli', + '--start-runtime=none', + `--output-dir=${ metadata.runtimeDirectory }`, '--no-adaptive', `--state-dir=${ metadata.stateDirectory }`, `--fs-root=${ metadata.rawDirectory }`, ], ( progress ) => logger.reportProgress( progress ), { - progressLabel: __( 'Downloading database' ), - verboseCommands: verbose, - } - ); - logger.reportSuccess( __( 'Database downloaded' ) ); - recordCompletedStage( metadata, 'db-downloaded' ); -} - -/** - * Produce the Playground CLI start script + runtime blueprint that - * Studio uses to boot the imported site. reprint reads the flattened - * site layout and preflight output, and emits a ready-to-run runtime - * under `runtimeDirectory`. Both directories are mounted into WASM - * so the runtime output lands on the host filesystem. - * - * Advances the pull stage to 'runtime-generated'. - */ -export async function generateRuntimeConfiguration( - metadata: PullSessionMetadata, - verbose: boolean -): Promise< void > { - logger.reportStart( LoggerAction.URL_REWRITE, __( 'Generating runtime configuration…' ) ); - await runReprintCommandUntilComplete( - metadata.stateDirectory, - metadata.rawDirectory, - [ - 'apply-runtime', - '--no-adaptive', - `--state-dir=${ metadata.stateDirectory }`, - `--flat-document-root=${ metadata.sitePath }`, - `--output-dir=${ metadata.runtimeDirectory }`, - '--runtime=playground-cli', - ], - undefined, - { + progressLabel: __( 'Pulling site' ), mounts: [ { hostPath: metadata.sitePath, vfsPath: metadata.sitePath }, { hostPath: metadata.runtimeDirectory, vfsPath: metadata.runtimeDirectory }, @@ -918,8 +872,8 @@ export async function generateRuntimeConfiguration( verboseCommands: verbose, } ); - logger.reportSuccess( __( 'Runtime configuration generated' ) ); - recordCompletedStage( metadata, 'runtime-generated' ); + logger.reportSuccess( __( 'Site pulled' ) ); + recordCompletedStage( metadata, 'pulled' ); } /** @@ -1311,58 +1265,48 @@ function recordCompletedStage( metadata: PullSessionMetadata, stage: PullStage ) * rewritten URLs) as the interrupted run. */ async function ensurePort( metadata: PullSessionMetadata ): Promise< void > { - if ( metadata.port && metadata.localUrl ) { - return; - } - const cliConfig = await readCliConfig(); - for ( const site of cliConfig.sites ) { - portFinder.addUnavailablePort( site.port ); - } + // When a Studio site record already exists for this pull, adopt its + // identity even if the metadata already carries a port — the record + // can change between runs (e.g. the site was deleted and re-created + // and got a different id/port). db-apply rewrites the database URLs + // to metadata.localUrl, so a stale port here would rewrite the site + // to a URL nothing serves. const existingSite = cliConfig.sites.find( ( site ) => ( metadata.siteId && site.id === metadata.siteId ) || fsUtils.arePathsEqual( site.path, metadata.sitePath ) || site.technicalSiteDirectory === metadata.technicalSiteDirectory ); + if ( existingSite ) { + if ( + metadata.siteId !== existingSite.id || + metadata.port !== existingSite.port || + metadata.localUrl !== getSiteUrl( existingSite ) + ) { + metadata.siteId = existingSite.id; + metadata.port = existingSite.port; + metadata.localUrl = getSiteUrl( existingSite ); + savePullMetadata( metadata ); + } + return; + } - const port = existingSite?.port ?? ( await portFinder.getOpenPort() ); + if ( metadata.port && metadata.localUrl ) { + return; + } + + for ( const site of cliConfig.sites ) { + portFinder.addUnavailablePort( site.port ); + } + + const port = await portFinder.getOpenPort(); metadata.port = port; - metadata.localUrl = existingSite ? getSiteUrl( existingSite ) : `http://localhost:${ port }`; + metadata.localUrl = `http://localhost:${ port }`; savePullMetadata( metadata ); } -/** - * Runs reprint's `flat-document-root` to produce the flattened site - * directory (symlinks + merged wp-content layout) from the raw tree. - */ -async function refreshFlattenedSiteDirectory( - metadata: Pick< - PullSessionMetadata, - 'stateDirectory' | 'rawDirectory' | 'sitePath' | 'runtimeBlueprintPath' | 'normalizedUrl' - >, - verbose: boolean -): Promise< void > { - await runReprintCommandUntilComplete( - metadata.stateDirectory, - metadata.rawDirectory, - [ - 'flat-document-root', - getReprintApiUrlForSite( metadata.normalizedUrl ), - '--no-adaptive', - `--state-dir=${ metadata.stateDirectory }`, - `--fs-root=${ metadata.rawDirectory }`, - `--flatten-to=${ metadata.sitePath }`, - ], - undefined, - { - mounts: [ { hostPath: metadata.sitePath, vfsPath: metadata.sitePath } ], - verboseCommands: verbose, - } - ); -} - async function findExistingSite( metadata: PullSessionMetadata ): Promise< SiteData | undefined > { const cliConfig = await readCliConfig(); return cliConfig.sites.find( @@ -1373,6 +1317,58 @@ async function findExistingSite( metadata: PullSessionMetadata ): Promise< SiteD ); } +/** + * Re-applies the site's stored admin credentials over the running + * site's admin API (`POST /?studio-admin-api`) — the same endpoint + * both server runtimes hit on startup. + * + * Needed after a re-pull's db-apply: the remote dump contains neither + * the local admin user nor the `studio_admin_username` option, so + * rebuilding the database from it breaks `/studio-auto-login` until + * the credentials are applied again. + * + * Returns: + * - 'applied' credentials re-applied on the running server + * - 'skipped' the site record has no credentials to apply + * - 'unreachable' the server didn't answer — the caller should + * treat the site as not running and start it + */ +export async function reapplyAdminCredentials( + site: SiteData +): Promise< 'applied' | 'skipped' | 'unreachable' > { + const credentials = { + adminUsername: site.adminUsername, + adminPassword: site.adminPassword, + adminEmail: site.adminEmail, + }; + if ( ! shouldSetAdminCredentials( credentials ) ) { + return 'skipped'; + } + + let response: Response; + try { + response = await fetch( new URL( '/?studio-admin-api', getSiteUrl( site ) ), { + method: 'POST', + body: toUrlSearchParams( getSetAdminCredentialsRequestBody( credentials ) ), + signal: AbortSignal.timeout( 15_000 ), + } ); + } catch { + return 'unreachable'; + } + + if ( ! response.ok ) { + throw new LoggerError( + sprintf( + // translators: %d: HTTP status code. + __( 'Failed to re-apply the admin credentials after the database refresh (HTTP %d).' ), + response.status + ) + ); + } + + return 'applied'; +} + function printSiteUrls( localUrl: string ): void { console.log( __( 'Site URL: ' ), buildAutoLoginUrl( localUrl ) ); console.log( diff --git a/apps/cli/commands/tests/pull-reprint.test.ts b/apps/cli/commands/tests/pull-reprint.test.ts index ee17efc955..09eba3f6f5 100644 --- a/apps/cli/commands/tests/pull-reprint.test.ts +++ b/apps/cli/commands/tests/pull-reprint.test.ts @@ -3,13 +3,13 @@ import os from 'node:os'; import path from 'node:path'; import { readAuthToken } from '@studio/common/lib/shared-config'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { rotateReprintSecret } from 'cli/lib/api'; +import { enableReprintExporter, rotateReprintSecret } from 'cli/lib/api'; import * as migrationClient from 'cli/lib/pull/migration-client'; import { shouldRestartFilesSyncIndex } from 'cli/lib/pull/reprint-state'; import { fetchSyncableSites } from 'cli/lib/sync-api'; import { pickSyncSite } from 'cli/lib/sync-site-picker'; import { - applyDownloadedDatabase, + runFullPull, downloadSkippedFiles, findMatchingWpComSite, getReprintApiUrlForSite, @@ -27,6 +27,7 @@ vi.mock( '@studio/common/lib/shared-config', async ( importOriginal ) => ( { vi.mock( 'cli/lib/api', async ( importOriginal ) => ( { ...( await importOriginal< typeof import('cli/lib/api') >() ), rotateReprintSecret: vi.fn(), + enableReprintExporter: vi.fn(), } ) ); vi.mock( 'cli/lib/sync-api', async ( importOriginal ) => ( { ...( await importOriginal< typeof import('cli/lib/sync-api') >() ), @@ -177,25 +178,26 @@ describe( 'CLI: studio pull-reprint helpers', () => { } ); } ); -describe( 'CLI: studio pull-reprint db-apply phase', () => { +describe( 'CLI: studio pull-reprint single pull phase', () => { afterEach( () => { vi.restoreAllMocks(); } ); - it( 'runs reprint db-apply against the content dir from preflight, mounts nothing extra, and advances the stage', async () => { + it( 'runs one reprint pull with sqlite under the content dir, mounts the site + runtime, and advances the stage', async () => { const technicalSiteDirectory = fs.mkdtempSync( - path.join( os.tmpdir(), 'studio-import-db-apply-' ) + path.join( os.tmpdir(), 'studio-import-pull-' ) ); const stateDirectory = path.join( technicalSiteDirectory, 'state' ); const rawDirectory = path.join( technicalSiteDirectory, 'raw' ); const sitePath = path.join( technicalSiteDirectory, 'site' ); + const runtimeDirectory = path.join( technicalSiteDirectory, 'runtime' ); fs.mkdirSync( stateDirectory, { recursive: true } ); fs.mkdirSync( rawDirectory, { recursive: true } ); // Preflight reported the remote site's wp-content path at - // database.wp.paths_urls.content_dir; db-apply should target an - // sqlite file under rawDirectory + that path — no extra mount - // needed because rawDirectory is already mounted. + // database.wp.paths_urls.content_dir; the pull's db-apply stage targets + // an sqlite file under rawDirectory + that path so flat-docroot can + // symlink it into the flattened site. fs.writeFileSync( path.join( stateDirectory, '.import-state.json' ), JSON.stringify( { @@ -226,50 +228,59 @@ describe( 'CLI: studio pull-reprint db-apply phase', () => { technicalSiteDirectory, rawDirectory, stateDirectory, - runtimeDirectory: path.join( technicalSiteDirectory, 'runtime' ), - runtimeBlueprintPath: path.join( technicalSiteDirectory, 'runtime', 'blueprint.json' ), - stage: 'db-downloaded', + runtimeDirectory, + runtimeBlueprintPath: path.join( runtimeDirectory, 'blueprint.json' ), + stage: 'initialized', localUrl: 'http://localhost:8881', remoteSiteUrl: 'https://example.com', } as never; - await applyDownloadedDatabase( metadata, 'hmac-secret', false ); + await runFullPull( metadata, 'https://example.com/?reprint-api', 'hmac-secret', false ); expect( reprint ).toHaveBeenCalledTimes( 1 ); const [ passedState, passedRaw, passedArgs, , passedOptions ] = reprint.mock.calls[ 0 ]; expect( passedState ).toBe( stateDirectory ); expect( passedRaw ).toBe( rawDirectory ); expect( passedArgs ).toEqual( [ - 'db-apply', + 'pull', 'https://example.com/?reprint-api', - `--state-dir=${ stateDirectory }`, - `--fs-root=${ rawDirectory }`, + '--secret=hmac-secret', + '--filter=essential-files', '--target-engine=sqlite', `--target-sqlite-path=${ rawDirectory }/srv/htdocs/wp-content/database/.ht.sqlite`, '--new-site-url=http://localhost:8881', - '--secret=hmac-secret', + `--flatten-to=${ sitePath }`, + '--runtime=playground-cli', + '--start-runtime=none', + `--output-dir=${ runtimeDirectory }`, '--no-adaptive', + `--state-dir=${ stateDirectory }`, + `--fs-root=${ rawDirectory }`, + ] ); + // The flattened site and runtime output dirs are mounted up front so + // the single fork can write them to the host filesystem. + expect( passedOptions?.mounts ).toEqual( [ + { hostPath: sitePath, vfsPath: sitePath }, + { hostPath: runtimeDirectory, vfsPath: runtimeDirectory }, ] ); - // No extra mount needed — reprint can already see the sqlite target - // through the rawDirectory mount. - expect( passedOptions?.mounts ).toEqual( [] ); - // Stage is bumped + persisted so a resumed run skips db-apply. + // Stage is bumped + persisted so a resumed run skips the pull. const persisted = JSON.parse( fs.readFileSync( path.join( technicalSiteDirectory, 'pull.json' ), 'utf-8' ) ); - expect( persisted.stage ).toBe( 'db-applied' ); + expect( persisted.stage ).toBe( 'pulled' ); fs.rmSync( technicalSiteDirectory, { recursive: true, force: true } ); } ); - it( 'mounts the flattened site path when preflight did not expose a content dir', async () => { + it( 'falls back to the flattened wp-content sqlite path when preflight exposes no content dir', async () => { const technicalSiteDirectory = fs.mkdtempSync( - path.join( os.tmpdir(), 'studio-import-db-apply-fallback-' ) + path.join( os.tmpdir(), 'studio-import-pull-fallback-' ) ); const stateDirectory = path.join( technicalSiteDirectory, 'state' ); const rawDirectory = path.join( technicalSiteDirectory, 'raw' ); const sitePath = path.join( technicalSiteDirectory, 'site' ); + const runtimeDirectory = path.join( technicalSiteDirectory, 'runtime' ); fs.mkdirSync( stateDirectory, { recursive: true } ); fs.mkdirSync( rawDirectory, { recursive: true } ); @@ -291,33 +302,38 @@ describe( 'CLI: studio pull-reprint db-apply phase', () => { technicalSiteDirectory, rawDirectory, stateDirectory, - runtimeDirectory: path.join( technicalSiteDirectory, 'runtime' ), - runtimeBlueprintPath: path.join( technicalSiteDirectory, 'runtime', 'blueprint.json' ), - stage: 'db-downloaded', + runtimeDirectory, + runtimeBlueprintPath: path.join( runtimeDirectory, 'blueprint.json' ), + stage: 'initialized', localUrl: 'http://localhost:8881', remoteSiteUrl: 'https://example.com', } as never; - await applyDownloadedDatabase( metadata, 'hmac-secret', false ); + await runFullPull( metadata, 'https://example.com/?reprint-api', 'hmac-secret', false ); const [ , , passedArgs, , passedOptions ] = reprint.mock.calls[ 0 ]; - // Sqlite now lands under the flattened site directly, and we mount - // that host path so reprint can reach it inside PHP WASM. + // With no content dir from preflight, the sqlite target falls back to + // the flattened site's wp-content. expect( passedArgs ).toContain( `--target-sqlite-path=${ sitePath }/wp-content/database/.ht.sqlite` ); - expect( passedOptions?.mounts ).toEqual( [ { hostPath: sitePath, vfsPath: sitePath } ] ); + // The site + runtime dirs are always mounted for the single fork. + expect( passedOptions?.mounts ).toEqual( [ + { hostPath: sitePath, vfsPath: sitePath }, + { hostPath: runtimeDirectory, vfsPath: runtimeDirectory }, + ] ); fs.rmSync( technicalSiteDirectory, { recursive: true, force: true } ); } ); - it( 'propagates the reprint error and leaves stage at db-downloaded for a safe resume', async () => { + it( 'propagates the reprint error and leaves the stage before "pulled" for a safe resume', async () => { const technicalSiteDirectory = fs.mkdtempSync( - path.join( os.tmpdir(), 'studio-import-db-apply-fail-' ) + path.join( os.tmpdir(), 'studio-import-pull-fail-' ) ); const stateDirectory = path.join( technicalSiteDirectory, 'state' ); const rawDirectory = path.join( technicalSiteDirectory, 'raw' ); const sitePath = path.join( technicalSiteDirectory, 'site' ); + const runtimeDirectory = path.join( technicalSiteDirectory, 'runtime' ); fs.mkdirSync( stateDirectory, { recursive: true } ); fs.mkdirSync( rawDirectory, { recursive: true } ); fs.writeFileSync( @@ -338,20 +354,20 @@ describe( 'CLI: studio pull-reprint db-apply phase', () => { technicalSiteDirectory, rawDirectory, stateDirectory, - runtimeDirectory: path.join( technicalSiteDirectory, 'runtime' ), - runtimeBlueprintPath: path.join( technicalSiteDirectory, 'runtime', 'blueprint.json' ), - stage: 'db-downloaded' as const, + runtimeDirectory, + runtimeBlueprintPath: path.join( runtimeDirectory, 'blueprint.json' ), + stage: 'initialized' as const, localUrl: 'http://localhost:8881', remoteSiteUrl: 'https://example.com', }; await expect( - applyDownloadedDatabase( metadata as never, 'hmac-secret', false ) + runFullPull( metadata as never, 'https://example.com/?reprint-api', 'hmac-secret', false ) ).rejects.toThrow( 'reprint exited with code 1' ); - // Stage must NOT advance — otherwise a resume would skip db-apply - // even though the database never made it into sqlite. - expect( metadata.stage ).toBe( 'db-downloaded' ); + // Stage must NOT advance to 'pulled' — otherwise a resume would skip + // the pull even though the site never finished importing. + expect( metadata.stage ).toBe( 'initialized' ); expect( fs.existsSync( path.join( technicalSiteDirectory, 'pull.json' ) ) ).toBe( false ); fs.rmSync( technicalSiteDirectory, { recursive: true, force: true } ); @@ -713,3 +729,295 @@ describe( 'CLI: studio pull-reprint confirmation before creating a site', () => expect( fs.existsSync( technicalSiteDirectory ) ).toBe( true ); } ); } ); + +describe( 'CLI: studio pull-reprint delta re-pull of a completed pull', () => { + let fakeHome: string; + + afterEach( () => { + vi.restoreAllMocks(); + vi.resetModules(); + if ( fakeHome ) { + fs.rmSync( fakeHome, { recursive: true, force: true } ); + } + } ); + + /** + * Same throwaway-home harness as the confirmation tests: anchors + * PULLS_ROOT and the Studio sites root to a temp directory so the + * real runCommand never touches the developer's machine. + */ + async function loadRunCommandWithFakeHome() { + fakeHome = fs.mkdtempSync( path.join( os.tmpdir(), 'studio-pull-repull-home-' ) ); + + vi.resetModules(); + vi.doMock( 'os', async () => { + const actual = await vi.importActual< typeof import('os') >( 'os' ); + return { + ...actual, + default: { ...actual, homedir: () => fakeHome }, + homedir: () => fakeHome, + }; + } ); + + const mod = await import( '../pull-reprint' ); + return mod; + } + + it( 'resets a completed pull for a delta re-run instead of exiting early', async () => { + const { runCommand, getPrivateDirNameForImportSession, normalizeSiteUrl } = + await loadRunCommandWithFakeHome(); + + const normalizedUrl = normalizeSiteUrl( 'https://example.com' ); + const pullKey = getPrivateDirNameForImportSession( normalizedUrl, 'My Completed Site' ); + const pullsRoot = path.join( fakeHome, '.studio', 'pulls' ); + const technicalSiteDirectory = path.join( pullsRoot, pullKey ); + const stateDirectory = path.join( technicalSiteDirectory, 'state' ); + const sitePath = path.join( fakeHome, 'Studio', 'My-Completed-Site' ); + + // Seed a completed pull whose site directory is non-empty (it holds + // the previous pull's output) and whose preflight response is cached. + fs.mkdirSync( stateDirectory, { recursive: true } ); + fs.mkdirSync( sitePath, { recursive: true } ); + fs.writeFileSync( path.join( sitePath, 'wp-config.php' ), ' undefined ); + vi.spyOn( console, 'error' ).mockImplementation( () => undefined ); + + await expect( + runCommand( 'https://example.com', 'hmac-secret', 'My Completed Site', false, false, false ) + ).rejects.toThrow(); + + // The old behavior exited early without ever invoking reprint; the + // re-pull must re-enter the pipeline (preflight is its first call). + expect( reprintSpy ).toHaveBeenCalled(); + expect( reprintSpy.mock.calls[ 0 ][ 2 ][ 0 ] ).toBe( 'preflight' ); + + // The stage machine was reset and the re-pull marker persisted. + const metadata = JSON.parse( + fs.readFileSync( path.join( technicalSiteDirectory, 'pull.json' ), 'utf-8' ) + ); + expect( metadata.stage ).toBe( 'initialized' ); + expect( metadata.hasCompletedOnce ).toBe( true ); + + // The cached preflight was dropped so connectivity is re-verified. + expect( fs.existsSync( path.join( stateDirectory, 'preflight.json' ) ) ).toBe( false ); + + // The non-empty site directory did not trip the clobber guard. + expect( fs.existsSync( path.join( sitePath, 'wp-config.php' ) ) ).toBe( true ); + + // The user sees update messaging, not a no-op success. + expect( logSpy.mock.calls.flat().join( '\n' ) ).toContain( 'Updating "My Completed Site"' ); + } ); +} ); + +describe( 'CLI: studio pull-reprint preflight retry re-enables the exporter', () => { + let fakeHome: string; + + const token = { + accessToken: 'access-token', + id: 1, + email: 'user@example.com', + displayName: 'User', + expiresIn: 1209600, + expirationTime: Date.now() + 1209600000, + }; + + const sites: SyncSite[] = [ + { + id: 22, + name: 'Example', + url: 'https://example.com', + localSiteId: '', + isStaging: false, + isPressable: false, + syncSupport: 'syncable', + lastPullTimestamp: null, + lastPushTimestamp: null, + }, + ]; + + afterEach( () => { + vi.restoreAllMocks(); + vi.resetModules(); + if ( fakeHome ) { + fs.rmSync( fakeHome, { recursive: true, force: true } ); + } + } ); + + async function loadRunCommandWithFakeHome() { + fakeHome = fs.mkdtempSync( path.join( os.tmpdir(), 'studio-pull-retry-home-' ) ); + + vi.resetModules(); + vi.doMock( 'os', async () => { + const actual = await vi.importActual< typeof import('os') >( 'os' ); + return { + ...actual, + default: { ...actual, homedir: () => fakeHome }, + homedir: () => fakeHome, + }; + } ); + + const mod = await import( '../pull-reprint' ); + return mod; + } + + it( 're-enables the exporter (not just rotates the secret) before retrying a failed preflight', async () => { + const { runCommand, getPrivateDirNameForImportSession, normalizeSiteUrl } = + await loadRunCommandWithFakeHome(); + + // Seed a previously-completed pull with a cached secret but no + // wpComSite/wpComToken — exactly the delta-re-pull shape that makes + // resolveSourceSite short-circuit on the cached secret and skip the + // happy-path exporter enable. + const normalizedUrl = normalizeSiteUrl( 'https://example.com' ); + const pullKey = getPrivateDirNameForImportSession( normalizedUrl, 'My Retry Site' ); + const pullsRoot = path.join( fakeHome, '.studio', 'pulls' ); + const technicalSiteDirectory = path.join( pullsRoot, pullKey ); + const stateDirectory = path.join( technicalSiteDirectory, 'state' ); + const sitePath = path.join( fakeHome, 'Studio', 'My-Retry-Site' ); + + fs.mkdirSync( stateDirectory, { recursive: true } ); + fs.mkdirSync( sitePath, { recursive: true } ); + fs.writeFileSync( path.join( sitePath, 'wp-config.php' ), ' undefined ); + vi.spyOn( console, 'error' ).mockImplementation( () => undefined ); + + await expect( + runCommand( 'https://example.com', undefined, 'My Retry Site', false, false, true ) + ).rejects.toThrow(); + + // The fix: the retry resolves the WP.com site, rotates the secret, AND + // re-opens the exporter gate. Without the enable call the retry would + // hit the same closed window and fail identically. + expect( rotateReprintSecret ).toHaveBeenCalledWith( 22, token.accessToken ); + expect( enableReprintExporter ).toHaveBeenCalledWith( 22, token.accessToken, false ); + } ); +} ); + +describe( 'CLI: studio pull-reprint admin credentials re-apply', () => { + afterEach( () => { + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + } ); + + function makeSite( overrides: Record< string, unknown > = {} ) { + return { + id: 'site-1', + name: 'Test Site', + path: '/tmp/test-site', + port: 8901, + phpVersion: '8.2', + running: true, + ...overrides, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any; + } + + it( 'skips when the site record has no admin credentials', async () => { + const { reapplyAdminCredentials } = await import( '../pull-reprint' ); + const fetchSpy = vi.fn(); + vi.stubGlobal( 'fetch', fetchSpy ); + + await expect( reapplyAdminCredentials( makeSite() ) ).resolves.toBe( 'skipped' ); + expect( fetchSpy ).not.toHaveBeenCalled(); + } ); + + it( 'posts the stored credentials to the running site admin API', async () => { + const { reapplyAdminCredentials } = await import( '../pull-reprint' ); + const { encodePassword } = await import( '@studio/common/lib/passwords' ); + const fetchSpy = vi.fn().mockResolvedValue( { ok: true, status: 200 } ); + vi.stubGlobal( 'fetch', fetchSpy ); + + const site = makeSite( { adminPassword: encodePassword( 'secret-pw' ) } ); + await expect( reapplyAdminCredentials( site ) ).resolves.toBe( 'applied' ); + + expect( fetchSpy ).toHaveBeenCalledTimes( 1 ); + const [ url, init ] = fetchSpy.mock.calls[ 0 ]; + expect( String( url ) ).toContain( 'studio-admin-api' ); + expect( init.method ).toBe( 'POST' ); + const params = init.body as URLSearchParams; + expect( params.get( 'action' ) ).toBe( 'set_admin_password' ); + expect( params.get( 'password' ) ).toBe( 'secret-pw' ); + } ); + + it( 'reports an unreachable server instead of throwing on connection failure', async () => { + const { reapplyAdminCredentials } = await import( '../pull-reprint' ); + const { encodePassword } = await import( '@studio/common/lib/passwords' ); + vi.stubGlobal( 'fetch', vi.fn().mockRejectedValue( new Error( 'ECONNREFUSED' ) ) ); + + const site = makeSite( { adminPassword: encodePassword( 'secret-pw' ) } ); + await expect( reapplyAdminCredentials( site ) ).resolves.toBe( 'unreachable' ); + } ); + + it( 'throws when the admin API answers with an error status', async () => { + const { reapplyAdminCredentials } = await import( '../pull-reprint' ); + const { encodePassword } = await import( '@studio/common/lib/passwords' ); + vi.stubGlobal( 'fetch', vi.fn().mockResolvedValue( { ok: false, status: 400 } ) ); + + const site = makeSite( { adminPassword: encodePassword( 'secret-pw' ) } ); + await expect( reapplyAdminCredentials( site ) ).rejects.toThrow( + 'Failed to re-apply the admin credentials' + ); + } ); +} ); diff --git a/apps/cli/lib/pull/reprint-state.ts b/apps/cli/lib/pull/reprint-state.ts index 274f6c6bc4..62de7afb38 100644 --- a/apps/cli/lib/pull/reprint-state.ts +++ b/apps/cli/lib/pull/reprint-state.ts @@ -92,7 +92,13 @@ export function shouldRestartFilesSyncIndex( stateDirectory: string ): boolean { return false; } - if ( state.command !== 'files-sync' || state.status === 'complete' ) { + // reprint canonicalizes the legacy 'files-sync' command name to + // 'files-pull' when it saves state; accept both so this check keeps + // working across reprint versions. + if ( + ( state.command !== 'files-sync' && state.command !== 'files-pull' ) || + state.status === 'complete' + ) { return false; } diff --git a/apps/cli/lib/wordpress-server-manager.ts b/apps/cli/lib/wordpress-server-manager.ts index fffe123f40..68fde2bce7 100644 --- a/apps/cli/lib/wordpress-server-manager.ts +++ b/apps/cli/lib/wordpress-server-manager.ts @@ -202,6 +202,32 @@ async function ensurePhpBinaryAvailableIfNeeded( } } +/** + * Drops mounts of reprint state files whose host paths no longer exist. + * + * reprint's apply-runtime mounts importer state files (under /tmp/reprint + * in the VFS) for the temporary remote-uploads proxy. Those files are + * transient — a later sync can empty or remove them — so a persisted + * start-options.json can reference paths that are gone, and mounting a + * missing path crashes the server start with ENOENT. Critical site mounts + * (core, wp-content, wp-config.php) are intentionally NOT filtered: if + * those are missing, failing loudly is correct. + */ +function dropStaleReprintStateMounts( options: StartServerOptions ): StartServerOptions { + const isStale = ( mount: { hostPath: string; vfsPath: string } ) => + mount.vfsPath.startsWith( '/tmp/reprint/' ) && ! fs.existsSync( mount.hostPath ); + + return { + ...options, + ...( options.mountsBeforeInstall && { + mountsBeforeInstall: options.mountsBeforeInstall.filter( ( m ) => ! isStale( m ) ), + } ), + ...( options.mounts && { + mounts: options.mounts.filter( ( m ) => ! isStale( m ) ), + } ), + }; +} + export async function startWordPressServer( site: SiteData, logger: Logger< string >, @@ -218,6 +244,7 @@ export async function startWordPressServer( ); if ( fs.existsSync( optionsPath ) ) { options = JSON.parse( fs.readFileSync( optionsPath, 'utf-8' ) ) as StartServerOptions; + options = dropStaleReprintStateMounts( options ); } } diff --git a/apps/cli/reprint-child.ts b/apps/cli/reprint-child.ts index 7addcdbb00..cf407c7d51 100644 --- a/apps/cli/reprint-child.ts +++ b/apps/cli/reprint-child.ts @@ -167,7 +167,11 @@ async function runReprint( msg: RunMessage ) { 'openssl.cafile': '/tmp/ca-bundle.crt', 'curl.cainfo': '/tmp/ca-bundle.crt', allow_url_fopen: 1, - memory_limit: '512M', + // The composite `pull` runs the whole pipeline in one long-lived + // fork (no per-sub-command teardown to free the heap), so the + // WASM high-water-mark from the file index carries across phases. + // 1024M gives headroom over the ~510M peak seen on large sites. + memory_limit: '1024M', error_reporting: String( 32767 & ~8192 ), display_errors: 'stderr', log_errors: 0,