Skip to content

#15 - D.T Migration Plugin - Too Much Data #20

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kodinkat wants to merge 2 commits into
base: master
Choose a base branch
from
Open

#15 - D.T Migration Plugin - Too Much Data #20

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c881858
Implement preflight checks for downloadable JSON exports and enhance …
kodinkat Apr 28, 2026
ae4bf2b
Fix formatting inconsistencies in migration export file class for imp…
kodinkat Apr 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 18 additions & 1 deletion admin/admin-menu-and-tabs.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,14 +95,31 @@ public static function get_settings(): array {
'file' => [
'compression' => 'zip',
],
'file_export_memory' => [
'budget_ratio' => null,
'bytes_per_record' => null,
'bytes_per_user' => null,
'settings_overhead_bytes' => null,
],
];

$current = get_option( 'dt_migration_settings', [] );
if ( ! is_array( $current ) ) {
$current = [];
}

return wp_parse_args( $current, $defaults );
$merged = wp_parse_args( $current, $defaults );

// Shallow wp_parse_args replaces the whole file_export_memory key; merge inner keys so stale
// partial saves (e.g. only bytes_per_record) still fill missing keys from defaults.
if ( isset( $merged['file_export_memory'] ) && is_array( $merged['file_export_memory'] ) ) {
$merged['file_export_memory'] = wp_parse_args(
$merged['file_export_memory'],
$defaults['file_export_memory']
);
}

return $merged;
}

/**
Expand Down
40 changes: 14 additions & 26 deletions admin/class-dt-migration-export-download.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,38 +38,26 @@ public function handle_download() : void {
wp_die( esc_html__( 'Migration is not enabled.', 'disciple-tools-migration' ) );
}

$record_options = [];
$export_by = $this->sanitize_post_type_assoc_array( 'dt_migration_export_by', 'sanitize_key' );
$limits = $this->sanitize_post_type_assoc_array( 'dt_migration_export_limit', 'absint' );
$min_ids = $this->sanitize_post_type_assoc_array( 'dt_migration_export_min_id', 'absint' );
$max_ids = $this->sanitize_post_type_assoc_array( 'dt_migration_export_max_id', 'absint' );

$allowed_records = $settings['allowed_items']['records'] ?? [];
foreach ( $allowed_records as $post_type => $enabled ) {
if ( ! $enabled ) {
continue;
}
$raw_mode = isset( $export_by[ $post_type ] ) ? sanitize_key( (string) $export_by[ $post_type ] ) : 'all';
if ( $raw_mode === 'limit' ) {
$mode = 'limit';
} elseif ( $raw_mode === 'range' ) {
$mode = 'range';
} else {
$mode = 'all';
}
if ( $mode === 'all' ) {
continue;
}
$limit = $mode === 'limit' ? absint( $limits[ $post_type ] ?? 0 ) : 0;
$min_id = $mode === 'range' ? absint( $min_ids[ $post_type ] ?? 0 ) : 0;
$max_id = $mode === 'range' ? absint( $max_ids[ $post_type ] ?? 0 ) : 0;
if ( $limit > 0 || $min_id > 0 || $max_id > 0 ) {
$record_options[ $post_type ] = [
'limit' => $limit,
'min_id' => $min_id,
'max_id' => $max_id,
];
}

$record_options = Disciple_Tools_Migration_Export_File::parse_download_record_options(
is_array( $allowed_records ) ? $allowed_records : [],
$export_by,
$limits,
$min_ids,
$max_ids
);

$memory_check = Disciple_Tools_Migration_Export_File::evaluate_file_export_memory( $record_options );
if ( empty( $memory_check['allowed'] ) ) {
set_transient( 'dt_migration_export_flash_notice_' . get_current_user_id(), 'file_export_memory', MINUTE_IN_SECONDS );
wp_safe_redirect( admin_url( 'admin.php?page=disciple_tools_migration&tab=export' ) );
exit;
}

$payload = Disciple_Tools_Migration_Export_File::build_export( $record_options );
Expand Down
24 changes: 24 additions & 0 deletions admin/class-dt-migration-import-ajax.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,30 @@ public function enqueue_scripts( string $hook ) : void {
}

$plugin_url = plugin_dir_url( dirname( __FILE__ ) );
$tab = isset( $_GET['tab'] ) ? sanitize_key( wp_unslash( $_GET['tab'] ) ) : 'settings';

if ( $tab === 'export' ) {
wp_enqueue_script(
'dt-migration-export',
$plugin_url . 'admin/js/export.js',
[ 'jquery' ],
'1.0.0',
true
);
wp_localize_script(
'dt-migration-export',
'dtMigrationExport',
[
'preflightUrl' => esc_url_raw( rest_url( 'dt-migration/v1/export-file-preflight' ) ),
'nonce' => wp_create_nonce( 'wp_rest' ),
'strings' => [
'memoryBlocked' => __( 'This downloadable export is estimated to exceed the server memory limit. Use the Import tab to connect to the target site and migrate over the API instead, or reduce what is enabled for export on the Settings tab.', 'disciple-tools-migration' ),
'preflightFailed' => __( 'Could not verify export safety. Try again or check your connection.', 'disciple-tools-migration' ),
],
]
);
}

wp_enqueue_script(
'dt-migration-import',
$plugin_url . 'admin/js/import.js',
Expand Down
31 changes: 30 additions & 1 deletion admin/class-dt-migration-tab-export.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,39 @@
* Placeholder for the Migration Export tab. Will be wired to settings in later phases.
*/
class Disciple_Tools_Migration_Tab_Export {

/**
* Shows a short-lived admin notice after redirect (e.g. file export blocked by memory heuristic).
*
* @return void
*/
private function maybe_render_file_export_flash_notice() : void {
$key = 'dt_migration_export_flash_notice_' . get_current_user_id();
$flag = get_transient( $key );
if ( 'file_export_memory' !== $flag ) {
return;
}

delete_transient( $key );
?>
<div class="notice notice-error">
<p>
<?php
esc_html_e(
'This downloadable export is estimated to exceed the server memory limit. Use the Import tab to connect to the target site and migrate over the API instead, or reduce what is enabled for export on the Settings tab.',
'disciple-tools-migration'
);
?>
</p>
</div>
<?php
}

public function content() {
$settings = Disciple_Tools_Migration_Menu::get_settings();
?>
<div class="wrap">
<?php $this->maybe_render_file_export_flash_notice(); ?>
<div id="poststuff">
<div id="post-body" class="metabox-holder columns-2">
<div id="post-body-content">
Expand Down Expand Up @@ -151,7 +180,7 @@ public function main_column( array $settings ) {
$show_export_record_filters = apply_filters( 'dt_migration_show_export_record_filters', false );
if ( ! empty( $record_stats ) ) :
?>
<form method="post" action="<?php echo esc_url( admin_url( 'admin-post.php' ) ); ?>">
<form id="dt-migration-download-export-form" method="post" action="<?php echo esc_url( admin_url( 'admin-post.php' ) ); ?>">
<input type="hidden" name="action" value="dt_migration_download_export">
<?php wp_nonce_field( 'dt_migration_download_export', 'dt_migration_download_export_nonce' ); ?>
<?php if ( $show_export_record_filters ) : ?>
Expand Down
93 changes: 93 additions & 0 deletions admin/class-dt-migration-tab-settings.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,55 @@ public function main_column( array $settings ) {
</p>
</fieldset>
</td>
</tr>
<tr>
<td>
<?php esc_html_e( 'Downloadable export (JSON) memory guard', 'disciple-tools-migration' ); ?>
</td>
<td>
<?php
$mem = $settings['file_export_memory'] ?? [];
if ( ! is_array( $mem ) ) {
$mem = [];
}
$def_ratio = Disciple_Tools_Migration_Export_File::DEFAULT_FILE_EXPORT_MEMORY_BUDGET_RATIO;
$def_bpr = Disciple_Tools_Migration_Export_File::DEFAULT_FILE_EXPORT_BYTES_PER_RECORD;
$def_bpu = Disciple_Tools_Migration_Export_File::DEFAULT_FILE_EXPORT_BYTES_PER_USER;
$def_overhead = Disciple_Tools_Migration_Export_File::DEFAULT_FILE_EXPORT_SETTINGS_OVERHEAD_BYTES;
$ratio_val = isset( $mem['budget_ratio'] ) && $mem['budget_ratio'] !== '' && null !== $mem['budget_ratio'] ? (string) $mem['budget_ratio'] : '';
$bpr_val = isset( $mem['bytes_per_record'] ) && $mem['bytes_per_record'] !== '' && null !== $mem['bytes_per_record'] ? (string) (int) $mem['bytes_per_record'] : '';
$bpu_val = isset( $mem['bytes_per_user'] ) && $mem['bytes_per_user'] !== '' && null !== $mem['bytes_per_user'] ? (string) (int) $mem['bytes_per_user'] : '';
$overhead_val = isset( $mem['settings_overhead_bytes'] ) && $mem['settings_overhead_bytes'] !== '' && null !== $mem['settings_overhead_bytes'] ? (string) (int) $mem['settings_overhead_bytes'] : '';
?>
<p class="description" style="margin-top:0;">
<?php esc_html_e( 'When you download a JSON export, the plugin estimates whether the payload may exceed PHP’s memory limit. Leave a field blank to use the built-in default shown as the placeholder. Lower the budget ratio or raise per-record estimates to block large exports earlier (or the opposite to allow larger packages on well-provisioned servers).', 'disciple-tools-migration' ); ?>
</p>
<p>
<label for="dt_migration_export_memory_budget_ratio">
<?php esc_html_e( 'Memory budget ratio', 'disciple-tools-migration' ); ?>
</label><br>
<input type="number" name="dt_migration_export_memory_budget_ratio" id="dt_migration_export_memory_budget_ratio" value="<?php echo esc_attr( $ratio_val ); ?>" min="0.05" max="0.95" step="0.01" style="max-width:7em;" placeholder="<?php echo esc_attr( (string) $def_ratio ); ?>" />
<span class="description"><?php esc_html_e( 'Fraction of PHP memory_limit used as the safe budget (heuristic).', 'disciple-tools-migration' ); ?></span>
</p>
<p>
<label for="dt_migration_export_memory_bytes_per_record">
<?php esc_html_e( 'Estimated bytes per record', 'disciple-tools-migration' ); ?>
</label><br>
<input type="number" name="dt_migration_export_memory_bytes_per_record" id="dt_migration_export_memory_bytes_per_record" value="<?php echo esc_attr( $bpr_val ); ?>" min="1024" step="1" style="max-width:10em;" placeholder="<?php echo esc_attr( (string) $def_bpr ); ?>" />
</p>
<p>
<label for="dt_migration_export_memory_bytes_per_user">
<?php esc_html_e( 'Estimated bytes per WordPress user', 'disciple-tools-migration' ); ?>
</label><br>
<input type="number" name="dt_migration_export_memory_bytes_per_user" id="dt_migration_export_memory_bytes_per_user" value="<?php echo esc_attr( $bpu_val ); ?>" min="128" step="1" style="max-width:10em;" placeholder="<?php echo esc_attr( (string) $def_bpu ); ?>" />
</p>
<p>
<label for="dt_migration_export_memory_settings_overhead_bytes">
<?php esc_html_e( 'Settings / metadata overhead (bytes)', 'disciple-tools-migration' ); ?>
</label><br>
<input type="number" name="dt_migration_export_memory_settings_overhead_bytes" id="dt_migration_export_memory_settings_overhead_bytes" value="<?php echo esc_attr( $overhead_val ); ?>" min="0" step="1" style="max-width:10em;" placeholder="<?php echo esc_attr( (string) $def_overhead ); ?>" />
</p>
</td>
</tr>
<tr>
<td>
Expand Down Expand Up @@ -180,6 +229,50 @@ public function process_form_fields(): void {
$settings['allowed_items']['records']['contacts'] = ! empty( $allowed['records']['contacts'] );
$settings['allowed_items']['records']['groups'] = ! empty( $allowed['records']['groups'] );

if ( ! isset( $settings['file_export_memory'] ) || ! is_array( $settings['file_export_memory'] ) ) {
$settings['file_export_memory'] = [];
}

$ratio_raw = isset( $post_vars['dt_migration_export_memory_budget_ratio'] ) ? trim( (string) $post_vars['dt_migration_export_memory_budget_ratio'] ) : '';
if ( '' === $ratio_raw ) {
$settings['file_export_memory']['budget_ratio'] = null;
} else {
$r = (float) str_replace( ',', '.', $ratio_raw );
$settings['file_export_memory']['budget_ratio'] = max( 0.05, min( 0.95, $r ) );
}

$bpr_raw = isset( $post_vars['dt_migration_export_memory_bytes_per_record'] ) ? trim( (string) $post_vars['dt_migration_export_memory_bytes_per_record'] ) : '';
if ( '' === $bpr_raw ) {
$settings['file_export_memory']['bytes_per_record'] = null;
} else {
$settings['file_export_memory']['bytes_per_record'] = max( 1024, absint( $bpr_raw ) );
}

$bpu_raw = isset( $post_vars['dt_migration_export_memory_bytes_per_user'] ) ? trim( (string) $post_vars['dt_migration_export_memory_bytes_per_user'] ) : '';
if ( '' === $bpu_raw ) {
$settings['file_export_memory']['bytes_per_user'] = null;
} else {
$settings['file_export_memory']['bytes_per_user'] = max( 128, absint( $bpu_raw ) );
}

$ov_raw = isset( $post_vars['dt_migration_export_memory_settings_overhead_bytes'] ) ? trim( (string) $post_vars['dt_migration_export_memory_settings_overhead_bytes'] ) : '';
if ( '' === $ov_raw ) {
$settings['file_export_memory']['settings_overhead_bytes'] = null;
} else {
$settings['file_export_memory']['settings_overhead_bytes'] = absint( $ov_raw );
}

// Do not persist file_export_memory when every field means "use defaults". update_option() can
// drop null leaf values, which used to leave stale keys (e.g. a large bytes_per_record alone).
if (
null === $settings['file_export_memory']['budget_ratio']
&& null === $settings['file_export_memory']['bytes_per_record']
&& null === $settings['file_export_memory']['bytes_per_user']
&& null === $settings['file_export_memory']['settings_overhead_bytes']
) {
unset( $settings['file_export_memory'] );
}

Disciple_Tools_Migration_Menu::update_settings( $settings );
}

Expand Down
58 changes: 58 additions & 0 deletions admin/js/export.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
/**
* Disciple.Tools Migration - Downloadable JSON export preflight (memory heuristic).
*/
( function( $ ) {
'use strict';

const strings = ( typeof dtMigrationExport !== 'undefined' && dtMigrationExport.strings ) ? dtMigrationExport.strings : {};

function t( key, fallback ) {
return strings[ key ] || fallback;
}

$( function() {
const $form = $( '#dt-migration-download-export-form' );
if ( ! $form.length || typeof dtMigrationExport === 'undefined' ) {
return;
}

const endpoint = dtMigrationExport.preflightUrl || '';
const nonce = dtMigrationExport.nonce || '';

$form.on( 'submit', function( e ) {
e.preventDefault();

const bodyStr = $form.serialize();

fetch( endpoint, {
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
'X-WP-Nonce': nonce
},
credentials: 'same-origin',
body: bodyStr
} )
.then( function( resp ) {
return resp.json().then( function( data ) {
return { ok_http: resp.ok, status: resp.status, payload: data };
} );
} )
.then( function( wrapped ) {
const data = wrapped.payload;
if ( ! wrapped.ok_http ) {
throw new Error( 'http' );
}
if ( data && data.ok ) {
$form.off( 'submit' );
$form.get( 0 ).submit();
return;
}
window.alert( data && data.message ? data.message : t( 'memoryBlocked', t( 'preflightFailed', 'Request failed.' ) ) );
} )
.catch( function() {
window.alert( t( 'preflightFailed', 'Could not verify export safety. Try again or check your connection.' ) );
} );
} );
} );
}( jQuery ) );
5 changes: 4 additions & 1 deletion disciple-tools-migration.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,8 +106,11 @@ private function __construct() {
require_once 'includes/class-dt-migration-preflight.php';
}

if ( is_admin() ) {
if ( is_admin() || $is_rest ) {
require_once 'includes/class-dt-migration-export-file.php';
}

if ( is_admin() ) {
require_once 'admin/class-dt-migration-export-download.php';
new Disciple_Tools_Migration_Export_Download();
}
Expand Down
Loading
Loading