Travis tests for dump and load data from ES

Author: Spamercz

composer.json

@@ -29,7 +29,7 @@
 		"nette/di": "^2.4.10",
 		"nette/security": "^2.4.3",
 		"nette/utils": "^2.5.3",
-		"spameri/elastic-query": "^0.4.0",
+		"spameri/elastic-query": "^0.5.0",
 		"elasticsearch/elasticsearch": "^6.0",
 		"kdyby/console": "^2.7.1",
 		"kdyby/datetime-provider": "v1.0.0",

makefile

@@ -5,10 +5,10 @@
 
 
 tests:
-	vendor/bin/tester tests
+	vendor/bin/tester -s -c ./tests/php.ini-unix ./tests
 
 tests-local:
-	vendor/bin/tester -c tests/SpameriTests/php.ini tests
+	vendor/bin/tester -c ./tests/SpameriTests/php.ini tests
 
 phpstan:
 	vendor/bin/phpstan analyse -l 7 -c phpstan.neon src tests

src/Commands/RestoreIndex.php src/Commands/LoadDump.phpsrc/Commands/RestoreIndex.php renamed to src/Commands/LoadDump.php

@@ -3,7 +3,7 @@
 namespace Spameri\Elastic\Commands;
 
 
-class RestoreIndex extends \Symfony\Component\Console\Command\Command
+class LoadDump extends \Symfony\Component\Console\Command\Command
 {
 
 	/**
@@ -22,13 +22,13 @@ public function __construct(
 
 
 	/**
-	 * @example spameri:elastic:restore-index
+	 * @example spameri:elastic:load-dump
 	 */
 	protected function configure() : void
 	{
 		$this
-			->setName('spameri:elastic:restore-index')
-			->setDescription('Restores data from provided dump file.')
+			->setName('spameri:elastic:load-dump')
+			->setDescription('Loads data from provided dump file.')
 			->addArgument('filename', \Symfony\Component\Console\Input\InputArgument::REQUIRED)
 			->addArgument(
 				'step',

src/Commands/TypeToNewIndex.php

@@ -33,6 +33,7 @@ protected function configure() : void
 			->addArgument('typeFrom', \Symfony\Component\Console\Input\InputArgument::REQUIRED)
 			->addArgument('indexTo', \Symfony\Component\Console\Input\InputArgument::REQUIRED)
 			->addArgument('aliasTo', \Symfony\Component\Console\Input\InputArgument::REQUIRED)
+			->addArgument('typeTo', \Symfony\Component\Console\Input\InputArgument::OPTIONAL, 'Use only on old ElasticSearch', NULL)
 			->addOption('allowClose', 'c', NULL,
 				'Allows command to close index for data transfer. After data is transferred index is opened and resumes normal operations. When open it needs to check changed files after move and sync remaining.',
 				TRUE
@@ -51,10 +52,11 @@ protected function execute(
 		$typeFrom = $input->getArgument('typeFrom');
 		$indexTo = $input->getArgument('indexTo');
 		$aliasTo = $input->getArgument('aliasTo');
+		$typeTo = $input->getOption('typeTo');
 		$allowClose = $input->getOption('allowClose');
 
 		$this->migrate->setOutput($output);
-		$this->migrate->execute($indexFrom, $typeFrom, $indexTo, $aliasTo, $allowClose);
+		$this->migrate->execute($indexFrom, $typeFrom, $indexTo, $aliasTo, $typeTo, $allowClose);
 
 		$output->writeln('Done');
 	}

src/Config/Elastic.neon

@@ -50,6 +50,9 @@ services:
 	indicePutMapping:
 		class: Spameri\Elastic\Model\Indices\PutMapping
 
+	indicePutSettings:
+		class: Spameri\Elastic\Model\Indices\PutSettings
+
 	indicesCreate:
 		class: Spameri\Elastic\Model\Indices\Create
 
@@ -92,7 +95,7 @@ services:
 	modelDumpIndex:
 		class: \Spameri\Elastic\Model\DumpIndex
 
-	modelRestoreIndex:
+	restoreIndex:
 		class: \Spameri\Elastic\Model\RestoreIndex
 
 #	###
@@ -147,8 +150,8 @@ services:
 			- kdyby.console.command
 			- console.command
 
-	restoreIndex:
-		class: Spameri\Elastic\Commands\RestoreIndex
+	loadDump:
+		class: Spameri\Elastic\Commands\LoadDump
 		tags:
 			- kdyby.console.command
 			- console.command
@@ -174,6 +177,3 @@ services:
 
 	dateTime:
 		class: \DateTimeImmutable
-
-	resultMapper:
-		class: Spameri\ElasticQuery\Response\ResultMapper

src/DI/ElasticSearch30Extension.php

@@ -106,7 +106,7 @@ public function loadConfiguration() : void
 
 		$services = $this->toggleDebugBar($config, $services);
 
-		if ( ! class_exists(\Symfony\Component\Console\Command\Command::class)) {
+		if ( ! \class_exists(\Symfony\Component\Console\Command\Command::class)) {
 			$services = $this->removeCommandDefinitions($services);
 		}
 

src/DI/ElasticSearchExtension.php

@@ -9,7 +9,7 @@ class DocumentMigrateStatus
 	/**
 	 * @var array<string=>int>
 	 */
-	private $storage;
+	private $storage = [];
 
 
 	public function add(

src/Model/TypeToNewIndex/DocumentMigrateStatus.php

@@ -125,6 +125,7 @@ public function execute(
 		, string $typeFrom
 		, string $indexTo
 		, string $aliasTo
+		, ?string $typeTo
 		, bool $allowClose
 	) : void
 	{
@@ -143,10 +144,9 @@ public function execute(
 		$settings = \reset($oldIndexSettings);
 		$this->create->execute($indexTo, [
 			'settings' => [
-				'index' => $settings['settings']['index']['analysis'],
+				'index' => $settings['settings']['index']['analysis'] ?? [],
 			]
 		]);
-		// TODO otestovat
 
 		// 2b. Set mapping in new index
 		$this->output->writeln('Transferring mapping from index: ' . $indexFrom . ' and type: ' . $typeFrom . ' to index: ' . $indexTo);
@@ -169,17 +169,18 @@ public function execute(
 			// 4. Input data to new index
 			// 4a. if closed delete data
 			// 4b. if open store migrated version
-			/** @var \Spameri\ElasticQuery\Response\Result\Hit $hit */
-			foreach ($result as $hit) {
-				$this->processHit($indexTo, $indexFrom, $hit, $allowClose);
-
-				/** @noinspection DisconnectedForeachInstructionInspection */
-				$progressBar->advance();
+			/** @var \Spameri\ElasticQuery\Response\Result\Hit $response */
+			foreach ($result->hits() as $response) {
+				$this->processHit($indexTo, $typeTo, $indexFrom, $response, $allowClose);
 			}
-			$from += 5000;
-			$elasticQuery->options()->changeFrom($from);
-			if ($result->stats()->total() === 0) {
+
+			if (\count($result->hits()->getIterator()) === 0) {
 				$continue = FALSE;
+
+			} else {
+				$progressBar->advance(5000);
+				$from += 5000;
+				$elasticQuery->options()->changeFrom($from);
 			}
 		}
 		$progressBar->finish();
@@ -197,15 +198,15 @@ public function execute(
 			while ($canContinue) {
 				$changed = 0;
 				foreach ($this->documentMigrateStatus->storage() as $documentId => $documentVersion) {
-					$hit = $this->get->execute(
-						new \Spameri\Elastic\Entity\Property\ElasticId($documentId),
+					$response = $this->get->execute(
+						new \Spameri\Elastic\Entity\Property\ElasticId((string) $documentId),
 						$indexFrom,
 						$typeFrom
 					);
 
-					if ($this->documentMigrateStatus->isChanged($documentId, $hit->version())) {
+					if ($this->documentMigrateStatus->isChanged((string) $documentId, $response->hit()->version())) {
 						// Reindex this document
-						$this->processHit($indexTo, $indexFrom, $hit, $allowClose);
+						$this->processHit($indexTo, $typeTo, $indexFrom, $response, $allowClose);
 						$changed++;
 
 						/** @noinspection DisconnectedForeachInstructionInspection */
@@ -246,6 +247,7 @@ public function execute(
 	 */
 	public function processHit(
 		string $indexTo
+		, ?string $typeTo
 		, string $indexFrom
 		, \Spameri\ElasticQuery\Response\Result\Hit $hit
 		, bool $allowClose
@@ -254,7 +256,7 @@ public function processHit(
 		$document = new \Spameri\ElasticQuery\Document(
 			$indexTo,
 			new \Spameri\ElasticQuery\Document\Body\Plain($hit->source()),
-			$indexTo,
+			$typeTo,
 			$hit->id()
 		);
 

src/Model/TypeToNewIndex/Migrate.php

@@ -0,0 +1,17 @@
+<?php declare(strict_types = 1);
+
+namespace SpameriTests\Elastic;
+
+class Config
+{
+
+	public const INDEX = 'spameri_product';
+	public const TYPE  = 'product';
+	public const HOST  = 'http://127.0.0.1';
+
+	public const INDEX_DUMP = 'spameri_product_dump';
+	public const INDEX_RESTORE = 'spameri_product_restore';
+	public const INDEX_MIGRATE = 'spameri_product_migrate';
+	public const INDEX_MIGRATE_NEW = 'spameri_product_migrate_new';
+
+}