Improve performance with large data sets

Service/ChangeEncryptionKey.php

@@ -32,13 +32,14 @@ public function setSkipSavedCreditCards($skipSavedCreditCards)
     }
 
     /**
-     * @param string $text
+     * @param $text
+     * @param int $type
      * @return void
      */
-    private function writeOutput($text)
+    private function writeOutput($text, $type = OutputInterface::OUTPUT_NORMAL)
     {
         if ($this->output instanceof OutputInterface) {
-            $this->output->writeln($text);
+            $this->output->writeln($text, $type);
         }
     }
 
@@ -69,22 +70,133 @@ protected function _reEncryptCreditCardNumbers()
         }
         $this->writeOutput('_reEncryptCreditCardNumbers - start');
         $table = $this->getTable('sales_order_payment');
-        $select = $this->getConnection()->select()->from($table, ['entity_id', 'cc_number_enc']);
 
-        $attributeValues = $this->getConnection()->fetchPairs($select);
-        // save new values
-        foreach ($attributeValues as $valueId => $value) {
-            // GENE CHANGE START
-            if (!$value) {
+        $batchSize = 10000; // TODO worth making configurable?
+
+        $minId = (int) $this->getConnection()->fetchOne(
+            $this->getConnection()->select()
+                ->from($table, ['min(entity_id) AS min_id'])
+        );
+        $maxId = (int) $this->getConnection()->fetchOne(
+            $this->getConnection()->select()
+                ->from($table, ['max(entity_id) AS max_id'])
+        );
+        $totalCount = ($maxId - $minId) + 1; // the numbers are inclusive so add 1
+
+        $numberOfBatches = ceil($totalCount / $batchSize);
+        $this->writeOutput("_reEncryptCreditCardNumbers - total possible records: $totalCount");
+        $this->writeOutput("_reEncryptCreditCardNumbers - batch size:             $batchSize");
+        $this->writeOutput("_reEncryptCreditCardNumbers - batch count:            $numberOfBatches");
+
+        $updatedCount = 0;
+        $currentMin = $minId;
+        for ($i = 0; $i < $numberOfBatches; $i++) {
+            $currentMax = $currentMin + $batchSize;
+            $select = $this->getConnection()->select()
+                ->from($table, ['entity_id', 'cc_number_enc'])
+                ->where("entity_id >= ?", $currentMin)
+                ->where("entity_id < ?", $currentMax);
+
+            $this->writeOutput((string)$select, OutputInterface::VERBOSITY_VERBOSE);
+
+            /**
+             * @see https://github.com/magento/inventory/blob/750be5b07053331bc0bf3cb0f4d19366a67694f4/Inventory/Model/ResourceModel/SourceItem/SaveMultiple.php#L165-L188
+             */
+            $pairsToUpdate = [];
+            $attributeValues = $this->getConnection()->fetchPairs($select);
+            foreach ($attributeValues as $valueId => $value) {
+                if (!$value) {
+                    continue;
+                }
+                // TODO i think the encryption is the limiting factor here
+                // TODO i can insert 1 million rows as asdfasdfsdfasdf in ~8 seconds locally but ~2 mins for the full process
+//                $pairsToUpdate[(int)$valueId] = 'asdfasdfsdfasdf';
+                $pairsToUpdate[(int)$valueId] = $this->encryptor->encrypt($this->encryptor->decrypt($value));
+                $updatedCount++;
+            }
+
+            $currentMin = $currentMax;
+            if (empty($pairsToUpdate)) {
                 continue;
             }
-            // GENE CHANGE END
-            $this->getConnection()->update(
+
+            $columnsSql = $this->buildColumnsSqlPart(['entity_id', 'cc_number_enc']);
+            $valuesSql = $this->buildValuesSqlPart($pairsToUpdate);
+            $onDuplicateSql = $this->buildOnDuplicateSqlPart(['cc_number_enc']);
+            $bind = $this->getSqlBindData($pairsToUpdate);
+
+            // todo worth checking this isnt artifically inflating the auto increment id
+            $insertSql = sprintf(
+                'INSERT INTO `%s` (%s) VALUES %s %s',
                 $table,
-                ['cc_number_enc' => $this->encryptor->encrypt($this->encryptor->decrypt($value))],
-                ['entity_id = ?' => (int)$valueId]
+                $columnsSql,
+                $valuesSql,
+                $onDuplicateSql
             );
+            $this->getConnection()->query($insertSql, $bind);
+            $this->writeOutput("running total records updated: $updatedCount", OutputInterface::VERBOSITY_VERBOSE);
         }
+
+        $this->writeOutput("_reEncryptCreditCardNumbers - total records updated:   $updatedCount");
         $this->writeOutput('_reEncryptCreditCardNumbers - end');
     }
+
+    /**
+     * Build sql query for on duplicate event
+     *
+     * @param array $fields
+     * @return string
+     */
+    private function buildOnDuplicateSqlPart(array $fields): string
+    {
+        $connection = $this->getConnection();
+        $processedFields = [];
+        foreach ($fields as $field) {
+            $processedFields[] = sprintf('%1$s = VALUES(%1$s)', $connection->quoteIdentifier($field));
+        }
+        $sql = 'ON DUPLICATE KEY UPDATE ' . implode(', ', $processedFields);
+        return $sql;
+    }
+
+    /**
+     * Build column sql part
+     *
+     * @param array $columns
+     * @return string
+     */
+    private function buildColumnsSqlPart(array $columns): string
+    {
+        $connection = $this->getConnection();
+        $processedColumns = array_map([$connection, 'quoteIdentifier'], $columns);
+        $sql = implode(', ', $processedColumns);
+        return $sql;
+    }
+
+    /**
+     * Build sql query for values
+     *
+     * @param array $rows
+     * @return string
+     */
+    private function buildValuesSqlPart(array $rows): string
+    {
+        $sql = rtrim(str_repeat('(?, ?), ', count($rows)), ', ');
+        return $sql;
+    }
+
+    /**
+     * Get Sql bind data
+     *
+     * @param array $rows
+     * @return array
+     */
+    private function getSqlBindData(array $rows): array
+    {
+        $bind = [];
+        foreach ($rows as $id => $encryptedValue) {
+            $bind[] = $id;
+            $bind[] = $encryptedValue;
+        }
+        return $bind;
+    }
 }

dev/stub_sales_order_payment.php

@@ -0,0 +1,31 @@
+<?php
+use Magento\Framework\App\Bootstrap;
+require_once '/var/www/html/app/bootstrap.php';
+
+$bootstrap = Bootstrap::create(BP, $_SERVER);
+$obj = $bootstrap->getObjectManager();
+
+/** @var \Magento\Framework\App\ResourceConnection $connection */
+$connection = $obj->get(\Magento\Framework\App\ResourceConnection::class);
+$connection->getConnection()->query('SET FOREIGN_KEY_CHECKS = 0;');
+$connection->getConnection()->query('delete from sales_order_payment where parent_id=1;');
+
+/** @var \Magento\Framework\Encryption\EncryptorInterface $encryptor */
+$encryptor = $obj->get(\Magento\Framework\Encryption\EncryptorInterface::class);
+$ccNumberEnc = $encryptor->encrypt('cc_number_enc_abc123');
+
+$rowData = "(1, '$ccNumberEnc'),";
+
+$insertQueryNull = trim('INSERT INTO sales_order_payment (parent_id, cc_number_enc) VALUES ' . str_repeat($rowData, 10000), ", ");
+for ($i = 0; $i < 250; $i++) {
+    $connection->getConnection()->query($insertQueryNull);
+}
+$connection->getConnection()->query('SET FOREIGN_KEY_CHECKS = 1;');
+
+// Get the total count of records
+$countSelect = $connection->getConnection()->select()
+    ->from('sales_order_payment', ['COUNT(*) AS total_count']);
+$totalCount = $connection->getConnection()->fetchOne($countSelect);
+
+echo "There are $totalCount items in sales_order_payment" . PHP_EOL;
+echo "DONE stub_sales_order_payment.php". PHP_EOL;

dev/test.sh

@@ -47,11 +47,15 @@ vendor/bin/n98-magerun2 db:query 'DROP TABLE IF EXISTS fake_json_table; CREATE T
 vendor/bin/n98-magerun2 db:query "insert into fake_json_table(text_column) values ('$FAKE_JSON_PAYLOAD');"
 vendor/bin/n98-magerun2 db:query "select * from fake_json_table";
 
+echo "Stubbing in a large volume of data to sales_order_payment"
+php vendor/gene/module-encryption-key-manager/dev/stub_sales_order_payment.php
+vendor/bin/n98-magerun2 db:query "select cc_number_enc from sales_order_payment where parent_id=1 limit 5";
+
 echo "";echo "";
 
 echo "Verifying commands need to use --force"
 
-php bin/magento gene:encryption-key-manager:generate > test.txt || true;
+php bin/magento gene:encryption-key-manager:generate -vvv > test.txt || true;
 if grep -q 'Run with --force' test.txt; then
     echo "PASS: generate needs to run with force"
 else
@@ -98,7 +102,7 @@ echo "";echo "";
 
 echo "Generating a new encryption key"
 grep -q "$ENCRYPTED_ENV_VALUE" app/etc/env.php
-php bin/magento gene:encryption-key-manager:generate --force  > test.txt
+time php bin/magento gene:encryption-key-manager:generate --force  > test.txt
 if grep -q "$ENCRYPTED_ENV_VALUE" app/etc/env.php; then
     echo "FAIL: The old encrypted value in env.php was not updated" && false
 fi
@@ -108,6 +112,8 @@ grep -q '_reEncryptSystemConfigurationValues - end'   test.txt
 grep -q '_reEncryptCreditCardNumbers - start' test.txt
 grep -q '_reEncryptCreditCardNumbers - end'   test.txt
 echo "PASS"
+cat test.txt
+vendor/bin/n98-magerun2 db:query "select cc_number_enc from sales_order_payment where parent_id=1 limit 5";
 echo "";echo "";
 
 echo "Generating a new encryption key - skipping _reEncryptCreditCardNumbers"