diff --git a/Service/ChangeEncryptionKey.php b/Service/ChangeEncryptionKey.php index cc78b1b..245b551 100644 --- a/Service/ChangeEncryptionKey.php +++ b/Service/ChangeEncryptionKey.php @@ -32,13 +32,14 @@ public function setSkipSavedCreditCards($skipSavedCreditCards) } /** - * @param string $text + * @param $text + * @param int $type * @return void */ - private function writeOutput($text) + private function writeOutput($text, $type = OutputInterface::OUTPUT_NORMAL) { if ($this->output instanceof OutputInterface) { - $this->output->writeln($text); + $this->output->writeln($text, $type); } } @@ -69,22 +70,133 @@ protected function _reEncryptCreditCardNumbers() } $this->writeOutput('_reEncryptCreditCardNumbers - start'); $table = $this->getTable('sales_order_payment'); - $select = $this->getConnection()->select()->from($table, ['entity_id', 'cc_number_enc']); - $attributeValues = $this->getConnection()->fetchPairs($select); - // save new values - foreach ($attributeValues as $valueId => $value) { - // GENE CHANGE START - if (!$value) { + $batchSize = 10000; // TODO worth making configurable? + + $minId = (int) $this->getConnection()->fetchOne( + $this->getConnection()->select() + ->from($table, ['min(entity_id) AS min_id']) + ); + $maxId = (int) $this->getConnection()->fetchOne( + $this->getConnection()->select() + ->from($table, ['max(entity_id) AS max_id']) + ); + $totalCount = ($maxId - $minId) + 1; // the numbers are inclusive so add 1 + + $numberOfBatches = ceil($totalCount / $batchSize); + $this->writeOutput("_reEncryptCreditCardNumbers - total possible records: $totalCount"); + $this->writeOutput("_reEncryptCreditCardNumbers - batch size: $batchSize"); + $this->writeOutput("_reEncryptCreditCardNumbers - batch count: $numberOfBatches"); + + $updatedCount = 0; + $currentMin = $minId; + for ($i = 0; $i < $numberOfBatches; $i++) { + $currentMax = $currentMin + $batchSize; + $select = $this->getConnection()->select() + ->from($table, ['entity_id', 'cc_number_enc']) + ->where("entity_id >= ?", $currentMin) + ->where("entity_id < ?", $currentMax); + + $this->writeOutput((string)$select, OutputInterface::VERBOSITY_VERBOSE); + + /** + * @see https://github.com/magento/inventory/blob/750be5b07053331bc0bf3cb0f4d19366a67694f4/Inventory/Model/ResourceModel/SourceItem/SaveMultiple.php#L165-L188 + */ + $pairsToUpdate = []; + $attributeValues = $this->getConnection()->fetchPairs($select); + foreach ($attributeValues as $valueId => $value) { + if (!$value) { + continue; + } + // TODO i think the encryption is the limiting factor here + // TODO i can insert 1 million rows as asdfasdfsdfasdf in ~8 seconds locally but ~2 mins for the full process +// $pairsToUpdate[(int)$valueId] = 'asdfasdfsdfasdf'; + $pairsToUpdate[(int)$valueId] = $this->encryptor->encrypt($this->encryptor->decrypt($value)); + $updatedCount++; + } + + $currentMin = $currentMax; + if (empty($pairsToUpdate)) { continue; } - // GENE CHANGE END - $this->getConnection()->update( + + $columnsSql = $this->buildColumnsSqlPart(['entity_id', 'cc_number_enc']); + $valuesSql = $this->buildValuesSqlPart($pairsToUpdate); + $onDuplicateSql = $this->buildOnDuplicateSqlPart(['cc_number_enc']); + $bind = $this->getSqlBindData($pairsToUpdate); + + // todo worth checking this isnt artifically inflating the auto increment id + $insertSql = sprintf( + 'INSERT INTO `%s` (%s) VALUES %s %s', $table, - ['cc_number_enc' => $this->encryptor->encrypt($this->encryptor->decrypt($value))], - ['entity_id = ?' => (int)$valueId] + $columnsSql, + $valuesSql, + $onDuplicateSql ); + $this->getConnection()->query($insertSql, $bind); + $this->writeOutput("running total records updated: $updatedCount", OutputInterface::VERBOSITY_VERBOSE); } + + $this->writeOutput("_reEncryptCreditCardNumbers - total records updated: $updatedCount"); $this->writeOutput('_reEncryptCreditCardNumbers - end'); } + + /** + * Build sql query for on duplicate event + * + * @param array $fields + * @return string + */ + private function buildOnDuplicateSqlPart(array $fields): string + { + $connection = $this->getConnection(); + $processedFields = []; + foreach ($fields as $field) { + $processedFields[] = sprintf('%1$s = VALUES(%1$s)', $connection->quoteIdentifier($field)); + } + $sql = 'ON DUPLICATE KEY UPDATE ' . implode(', ', $processedFields); + return $sql; + } + + /** + * Build column sql part + * + * @param array $columns + * @return string + */ + private function buildColumnsSqlPart(array $columns): string + { + $connection = $this->getConnection(); + $processedColumns = array_map([$connection, 'quoteIdentifier'], $columns); + $sql = implode(', ', $processedColumns); + return $sql; + } + + /** + * Build sql query for values + * + * @param array $rows + * @return string + */ + private function buildValuesSqlPart(array $rows): string + { + $sql = rtrim(str_repeat('(?, ?), ', count($rows)), ', '); + return $sql; + } + + /** + * Get Sql bind data + * + * @param array $rows + * @return array + */ + private function getSqlBindData(array $rows): array + { + $bind = []; + foreach ($rows as $id => $encryptedValue) { + $bind[] = $id; + $bind[] = $encryptedValue; + } + return $bind; + } } diff --git a/dev/stub_sales_order_payment.php b/dev/stub_sales_order_payment.php new file mode 100644 index 0000000..8ac3584 --- /dev/null +++ b/dev/stub_sales_order_payment.php @@ -0,0 +1,31 @@ +getObjectManager(); + +/** @var \Magento\Framework\App\ResourceConnection $connection */ +$connection = $obj->get(\Magento\Framework\App\ResourceConnection::class); +$connection->getConnection()->query('SET FOREIGN_KEY_CHECKS = 0;'); +$connection->getConnection()->query('delete from sales_order_payment where parent_id=1;'); + +/** @var \Magento\Framework\Encryption\EncryptorInterface $encryptor */ +$encryptor = $obj->get(\Magento\Framework\Encryption\EncryptorInterface::class); +$ccNumberEnc = $encryptor->encrypt('cc_number_enc_abc123'); + +$rowData = "(1, '$ccNumberEnc'),"; + +$insertQueryNull = trim('INSERT INTO sales_order_payment (parent_id, cc_number_enc) VALUES ' . str_repeat($rowData, 10000), ", "); +for ($i = 0; $i < 250; $i++) { + $connection->getConnection()->query($insertQueryNull); +} +$connection->getConnection()->query('SET FOREIGN_KEY_CHECKS = 1;'); + +// Get the total count of records +$countSelect = $connection->getConnection()->select() + ->from('sales_order_payment', ['COUNT(*) AS total_count']); +$totalCount = $connection->getConnection()->fetchOne($countSelect); + +echo "There are $totalCount items in sales_order_payment" . PHP_EOL; +echo "DONE stub_sales_order_payment.php". PHP_EOL; diff --git a/dev/test.sh b/dev/test.sh index 8a80b3c..c108ef5 100755 --- a/dev/test.sh +++ b/dev/test.sh @@ -47,11 +47,15 @@ vendor/bin/n98-magerun2 db:query 'DROP TABLE IF EXISTS fake_json_table; CREATE T vendor/bin/n98-magerun2 db:query "insert into fake_json_table(text_column) values ('$FAKE_JSON_PAYLOAD');" vendor/bin/n98-magerun2 db:query "select * from fake_json_table"; +echo "Stubbing in a large volume of data to sales_order_payment" +php vendor/gene/module-encryption-key-manager/dev/stub_sales_order_payment.php +vendor/bin/n98-magerun2 db:query "select cc_number_enc from sales_order_payment where parent_id=1 limit 5"; + echo "";echo ""; echo "Verifying commands need to use --force" -php bin/magento gene:encryption-key-manager:generate > test.txt || true; +php bin/magento gene:encryption-key-manager:generate -vvv > test.txt || true; if grep -q 'Run with --force' test.txt; then echo "PASS: generate needs to run with force" else @@ -98,7 +102,7 @@ echo "";echo ""; echo "Generating a new encryption key" grep -q "$ENCRYPTED_ENV_VALUE" app/etc/env.php -php bin/magento gene:encryption-key-manager:generate --force > test.txt +time php bin/magento gene:encryption-key-manager:generate --force > test.txt if grep -q "$ENCRYPTED_ENV_VALUE" app/etc/env.php; then echo "FAIL: The old encrypted value in env.php was not updated" && false fi @@ -108,6 +112,8 @@ grep -q '_reEncryptSystemConfigurationValues - end' test.txt grep -q '_reEncryptCreditCardNumbers - start' test.txt grep -q '_reEncryptCreditCardNumbers - end' test.txt echo "PASS" +cat test.txt +vendor/bin/n98-magerun2 db:query "select cc_number_enc from sales_order_payment where parent_id=1 limit 5"; echo "";echo ""; echo "Generating a new encryption key - skipping _reEncryptCreditCardNumbers"