diff --git a/.travis.yml b/.travis.yml index f6ae2db..6a2a384 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: php php: - - '7.4' + - '8.1' env: global: @@ -19,7 +19,7 @@ before_script: - git clone https://github.com/nextcloud/server.git - mv server/ nextcloud - cd nextcloud - - git checkout origin/stable19 -b stable19 + - git checkout origin/stable25 -b stable25 - mkdir custom_apps - cp -pi ${TRAVIS_BUILD_DIR}/travis/apps.config.php config - cd 3rdparty @@ -28,6 +28,10 @@ before_script: - php occ maintenance:install --database ${NEXTCLOUD_DB} --database-name ${NEXTCLOUD_DB_NAME} --database-user ${NEXTCLOUD_DB_USER} --database-pass ${NEXTCLOUD_DB_PASSWD} --admin-user ${NEXTCLOUD_ADMIN_NAME} --admin-pass ${NEXTCLOUD_ADMIN_PASSWD} - mkdir custom_apps/${NEXTCLOUD_APP} - cp -r ${TRAVIS_BUILD_DIR}/{appinfo,lib,tests} custom_apps/${NEXTCLOUD_APP} + - cp ${TRAVIS_BUILD_DIR}/composer.json custom_apps/${NEXTCLOUD_APP} + - cd custom_apps/${NEXTCLOUD_APP} + - composer install + - cd ../../ - php occ app:enable ${NEXTCLOUD_APP} script: diff --git a/README.md b/README.md index 2f5a622..672884e 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,11 @@ query "hash" tag,sets MD5/SHA256/SHA512. # Usage +- cd apps/checksum_api +- Composer update +- Composer dump-autoload + + ## Method GET diff --git a/appinfo/database.xml b/appinfo/database.xml deleted file mode 100644 index 5c0942b..0000000 --- a/appinfo/database.xml +++ /dev/null @@ -1,50 +0,0 @@ - - *dbname* - true - false - utf8 - - *dbprefix*checksum_api - - - id - integer - true - true - true - true - 8 - - - fileid - integer - true - 0 - true - 8 - - - revision - integer - true - 0 - true - 8 - - - type - text - true - - 30 - - - hash - text - true - - 64 - - -
-
diff --git a/appinfo/info.xml b/appinfo/info.xml index aded7ab..75a15be 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -16,6 +16,6 @@ tools https://github.com/RCOSDP/nextcloud-checksum_api/issues - + diff --git a/composer.json b/composer.json new file mode 100644 index 0000000..a5e04f5 --- /dev/null +++ b/composer.json @@ -0,0 +1,16 @@ +{ + "autoload-dev": { + "psr-4": { + "OCA\\ChecksumAPI\\": "lib/" + } + }, + "config": { + "optimize-autoloader": true + }, + "require-dev": { + "phpunit/phpunit": "^9" + }, + "require": { + "amphp/parallel": "^1.4" + } +} diff --git a/lib/Controller/ChecksumAPIController.php b/lib/Controller/ChecksumAPIController.php index 67e6558..f88d719 100644 --- a/lib/Controller/ChecksumAPIController.php +++ b/lib/Controller/ChecksumAPIController.php @@ -1,38 +1,47 @@ rootFolder = $rootFolder; $this->userSession = $userSession; @@ -41,7 +50,7 @@ public function __construct($appName, } private function isValidHash(string $hash) { - foreach($this->hashTypes as $hashType) { + foreach ($this->hashTypes as $hashType) { if ($hashType === $hash) { return true; } @@ -62,7 +71,7 @@ private function getMatchedVersion(string $path, string $revision) { return null; } - private function saveRecord(int $fileid, int $revision, string $hashType, string $hash) { + function saveRecord(int $fileid, int $revision, string $hashType, string $hash) { $entity = new Hash(); $entity->setFileid($fileid); $entity->setRevision($revision); @@ -94,7 +103,7 @@ public function checksum($hash, $path, $revision) { } $hashTypes = explode(',', $hash); - foreach($hashTypes as $hashType) { + foreach ($hashTypes as $hashType) { if (!$this->isValidHash($hashType)) { $this->logger->error('query parameter hash is invalid.'); return new DataResponse( @@ -139,8 +148,7 @@ public function checksum($hash, $path, $revision) { if ($revision === strval($latestRevision)) { $this->logger->info('latest version matches'); } else { - // check if version function is enabled - if (!\OCP\App::isEnabled($this->versionAppId)) { + if (!\OC::$server->getAppManager()->isEnabledForUser($this->versionAppId)) { $this->logger->error('version function is not enabled'); return new DataResponse( 'version function is not enabled', @@ -148,7 +156,6 @@ public function checksum($hash, $path, $revision) { ); } $this->logger->info($this->versionAppId . ' is enabled'); - $version = $this->getMatchedVersion($path, $revision); if (is_null($version)) { $this->logger->error('specified revision is not found'); @@ -163,6 +170,7 @@ public function checksum($hash, $path, $revision) { } $entities = []; + $tasks = []; foreach ($hashTypes as $hashType) { $entity = $this->mapper->find($fileid, $targetRevision, $hashType); if (is_null($entity)) { @@ -175,13 +183,48 @@ public function checksum($hash, $path, $revision) { $view = new \OC\Files\View('/'); $info = $view->getLocalFile($targetFile); } - $hash = hash_file($hashType, $info); - $this->logger->debug('hash: ' . $hash); - $entity = $this->saveRecord($fileid, $targetRevision, $hashType, $hash); + // check file size 20MB + if (fileSize($info) <= $this->minFileSizeToExcuteParallel || count($hashTypes) === 1) { + $hash = hash_file($hashType, $info); + $this->logger->debug('hash: ' . $hash); + $entity = $this->saveRecord($fileid, $targetRevision, $hashType, $hash); + array_push($entities, $entity); + } else { + array_push( + $tasks, + new HashableTask( + 'hashCalculator', + [ + "hashType" => $hashType, + "fileid" => $fileid, + "revision" => $targetRevision, + "info" => $info + ], + ), + ); + } + } else { + array_push($entities, $entity); } - array_push($entities, $entity); } - + if (!empty($tasks)) { + Amp\Loop::run(function () use (&$entities, $tasks) { + $pool = new DefaultPool; + $coroutines = []; + foreach ($tasks as $task) { + $coroutines[] = Amp\call(function () use ($pool, $task) { + $entity = yield $pool->enqueue($task); + $this->mapper->insert($entity); + return $entity; + }); + } + $completedTask = yield Amp\Promise\all($coroutines); + foreach ($completedTask as $task) { + array_push($entities, $task); + } + return yield $pool->shutdown(); + }); + } $res = []; $hashes = []; foreach ($entities as $entity) { diff --git a/lib/Db/Hash.php b/lib/Db/Hash.php index 963fb12..6b73675 100644 --- a/lib/Db/Hash.php +++ b/lib/Db/Hash.php @@ -4,6 +4,8 @@ namespace OCA\ChecksumAPI\Db; +require __DIR__ . '../../../../../lib/composer/autoload.php'; + use OCP\AppFramework\Db\Entity; class Hash extends Entity { diff --git a/lib/Db/HashMapper.php b/lib/Db/HashMapper.php index 792da44..6c4b782 100644 --- a/lib/Db/HashMapper.php +++ b/lib/Db/HashMapper.php @@ -4,6 +4,8 @@ namespace OCA\ChecksumAPI\Db; +require __DIR__ . '../../../vendor/autoload.php'; + use OCP\AppFramework\Db\DoesNotExistException; use OCP\AppFramework\Db\QBMapper; use OCP\DB\QueryBuilder\IQueryBuilder; diff --git a/lib/Jobs/HashableTask.php b/lib/Jobs/HashableTask.php new file mode 100644 index 0000000..365d958 --- /dev/null +++ b/lib/Jobs/HashableTask.php @@ -0,0 +1,43 @@ +function = $function; + $this->args = $args; + } + + /** + * {@inheritdoc} + */ + public function run(Environment $environment) { + $function = $this->function; + return $this->$function($this->args); + } + + public function hashCalculator($params) { + $hash = hash_file($params["hashType"], $params["info"]); + $entity = new Hash(); + $entity->setFileid($params["fileid"]); + $entity->setRevision($params["revision"]); + $entity->setType($params["hashType"]); + $entity->setHash($hash); + return $entity; + } +} \ No newline at end of file diff --git a/lib/Migration/Version00001Date20201016095257.php b/lib/Migration/Version00001Date20201016095257.php index d0a7dd8..5998449 100644 --- a/lib/Migration/Version00001Date20201016095257.php +++ b/lib/Migration/Version00001Date20201016095257.php @@ -59,7 +59,7 @@ public function changeSchema(IOutput $output, Closure $schemaClosure, array $opt ]); $table->addColumn('hash', 'string', [ 'notnull' => true, - 'length' => 64, + 'length' => 128, 'default' => '', ]); $table->setPrimaryKey(['id']); diff --git a/tests/Controller/ChecksumAPIControllerTest.php b/tests/Controller/ChecksumAPIControllerTest.php index 3651760..f7b66f5 100644 --- a/tests/Controller/ChecksumAPIControllerTest.php +++ b/tests/Controller/ChecksumAPIControllerTest.php @@ -8,20 +8,19 @@ use OCP\AppFramework\Http; use OCP\AppFramework\Http\DataResponse; use OCP\Files\NotFoundException; -use OCP\ILogger; use OCP\IRequest; use OCP\IUser; -use OCP\IUserManager; use OCP\IUserSession; -use PHPUnit\Framework\TestCase; use OCA\ChecksumAPI\Controller\ChecksumAPIController; use OCA\ChecksumAPI\Db\HashMapper; +use Psr\Log\LoggerInterface; +use PHPUnit\Framework\TestCase as PHPUnitTestCase; /** * @group DB */ -class ChecksumAPIControllerTest extends \Test\TestCase { +class ChecksumAPIControllerTest extends PHPUnitTestCase { private $request; private $user; @@ -29,6 +28,7 @@ class ChecksumAPIControllerTest extends \Test\TestCase { private $mapper; private $logger; private $file; + private $parallelFile; private $versionAppId = 'files_versions'; private $versionAppIdStatus; @@ -45,25 +45,31 @@ protected function setUp() :void { $this->mapper = $this->getMockBuilder(HashMapper::class)->disableOriginalConstructor()->getMock(); - $this->logger = $this->getMockBuilder(ILogger::class)->getMock(); + $this->logger = $this->getMockBuilder(LoggerInterface::class)->getMock(); $this->file = $this->getMockBuilder('OCP\Files\File')->disableOriginalConstructor()->getMock(); $this->file->method('getId')->willReturn(1); $this->file->method('getMTime')->willReturn(222222222); $this->file->method('getInternalPath')->willReturn('/data/test.txt'); + + $this->parallelFile = $this->getMockBuilder('OCP\Files\File')->disableOriginalConstructor()->getMock(); + $this->parallelFile->method('getId')->willReturn(1); + $this->parallelFile->method('getMTime')->willReturn(222222222); + $this->parallelFile->method('getInternalPath')->willReturn('/data/parallelTest.zip'); } protected function tearDown() :void { parent::tearDown(); } - public function testChecksumArgumentHashIsNull() :void { - $expected_stauts = Http::STATUS_BAD_REQUEST; + public function testChecksumArgumentHashIsNull(): void { + $expected_stauts = Http::STATUS_BAD_REQUEST; $expected_data = 'query parameter hash is missing'; $rootFolder = $this->getMockBuilder('OCP\Files\IRootFolder')->getMock(); - $controller = new ChecksumAPIController('checksum_api', + $controller = new ChecksumAPIController( + 'checksum_api', $this->request, $rootFolder, $this->userSession, @@ -76,14 +82,15 @@ public function testChecksumArgumentHashIsNull() :void { $this->assertEquals($expected_data, $response->getData()); } - public function testChecksumArgumentPathIsNull() :void { + public function testChecksumArgumentPathIsNull(): void { $hashType = 'sha512'; - $expected_stauts = Http::STATUS_BAD_REQUEST; + $expected_stauts = Http::STATUS_BAD_REQUEST; $expected_data = 'query parameter path is missing'; $rootFolder = $this->getMockBuilder('OCP\Files\IRootFolder')->getMock(); - $controller = new ChecksumAPIController('checksum_api', + $controller = new ChecksumAPIController( + 'checksum_api', $this->request, $rootFolder, $this->userSession, @@ -96,10 +103,10 @@ public function testChecksumArgumentPathIsNull() :void { $this->assertEquals($expected_data, $response->getData()); } - public function testChecksumArgumentPathIsInvalid() :void { + public function testChecksumArgumentPathIsInvalid(): void { $hashType = 'sha512'; $path = '/aaa'; - $expected_stauts = Http::STATUS_NOT_FOUND; + $expected_stauts = Http::STATUS_NOT_FOUND; $expected_data = 'file not found at specified path: ' . $path; $userFolder = $this->getMockBuilder('OCP\Files\Folder')->getMock(); @@ -108,7 +115,8 @@ public function testChecksumArgumentPathIsInvalid() :void { $rootFolder = $this->getMockBuilder('OCP\Files\IRootFolder')->getMock(); $rootFolder->method('getUserFolder')->with('userid')->willReturn($userFolder); - $controller = new ChecksumAPIController('checksum_api', + $controller = new ChecksumAPIController( + 'checksum_api', $this->request, $rootFolder, $this->userSession, @@ -121,11 +129,11 @@ public function testChecksumArgumentPathIsInvalid() :void { $this->assertEquals($expected_data, $response->getData()); } - public function testChecksumArgumentRevisionIsInvalid() :void { + public function testChecksumArgumentRevisionIsInvalid(): void { $hashType = 'sha512'; $path = '/test'; $revision = 'aaaa'; - $expected_stauts = Http::STATUS_BAD_REQUEST; + $expected_stauts = Http::STATUS_BAD_REQUEST; $expected_data = 'invalid revision is specified'; $userFolder = $this->getMockBuilder('OCP\Files\Folder')->getMock(); @@ -134,7 +142,35 @@ public function testChecksumArgumentRevisionIsInvalid() :void { $rootFolder = $this->getMockBuilder('OCP\Files\IRootFolder')->getMock(); $rootFolder->method('getUserFolder')->with('userid')->willReturn($userFolder); - $controller = new ChecksumAPIController('checksum_api', + $controller = new ChecksumAPIController( + 'checksum_api', + $this->request, + $rootFolder, + $this->userSession, + $this->mapper, + $this->logger + ); + + $response = $controller->checksum($hashType, $path, $revision); + $this->assertEquals($expected_stauts, $response->getStatus()); + $this->assertEquals($expected_data, $response->getData()); + } + + public function testChecksumArgumentRevisionIsInvalidWithParallel(): void { + $hashType = 'sha512'; + $path = '/parallelTest'; + $revision = 'aaaa'; + $expected_stauts = Http::STATUS_BAD_REQUEST; + $expected_data = 'invalid revision is specified'; + + $userFolder = $this->getMockBuilder('OCP\Files\Folder')->getMock(); + $userFolder->method('get')->willReturn($this->parallelFile); + + $rootFolder = $this->getMockBuilder('OCP\Files\IRootFolder')->getMock(); + $rootFolder->method('getUserFolder')->with('userid')->willReturn($userFolder); + + $controller = new ChecksumAPIController( + 'checksum_api', $this->request, $rootFolder, $this->userSession, @@ -147,11 +183,11 @@ public function testChecksumArgumentRevisionIsInvalid() :void { $this->assertEquals($expected_data, $response->getData()); } - public function testChecksumVersionAppIsDisabled() :void { + public function testChecksumVersionAppIsDisabled(): void { $hashType = 'sha512'; $path = '/test'; $revision = '1000000000'; - $expected_stauts = Http::STATUS_NOT_IMPLEMENTED; + $expected_stauts = Http::STATUS_NOT_IMPLEMENTED; $expected_data = 'version function is not enabled'; $userFolder = $this->getMockBuilder('OCP\Files\Folder')->getMock(); @@ -160,7 +196,8 @@ public function testChecksumVersionAppIsDisabled() :void { $rootFolder = $this->getMockBuilder('OCP\Files\IRootFolder')->getMock(); $rootFolder->method('getUserFolder')->with('userid')->willReturn($userFolder); - $controller = new ChecksumAPIController('checksum_api', + $controller = new ChecksumAPIController( + 'checksum_api', $this->request, $rootFolder, $this->userSession, @@ -168,7 +205,7 @@ public function testChecksumVersionAppIsDisabled() :void { $this->logger ); - $status = \OCP\App::isEnabled($this->versionAppId); + $status = \OC::$server->getAppManager()->isEnabledForUser($this->versionAppId); if ($status) { \OC::$server->getAppManager()->disableApp($this->versionAppId); } @@ -180,11 +217,45 @@ public function testChecksumVersionAppIsDisabled() :void { } } - public function testChecksumMatchesNoVersion() :void { + public function testChecksumVersionAppIsDisabledWithParallel(): void { + $hashType = 'sha512'; + $path = '/parallelTest'; + $revision = '1000000000'; + $expected_stauts = Http::STATUS_NOT_IMPLEMENTED; + $expected_data = 'version function is not enabled'; + + $userFolder = $this->getMockBuilder('OCP\Files\Folder')->getMock(); + $userFolder->method('get')->willReturn($this->parallelFile); + + $rootFolder = $this->getMockBuilder('OCP\Files\IRootFolder')->getMock(); + $rootFolder->method('getUserFolder')->with('userid')->willReturn($userFolder); + + $controller = new ChecksumAPIController( + 'checksum_api', + $this->request, + $rootFolder, + $this->userSession, + $this->mapper, + $this->logger + ); + + $status = \OC::$server->getAppManager()->isEnabledForUser($this->versionAppId); + if ($status) { + \OC::$server->getAppManager()->disableApp($this->versionAppId); + } + $response = $controller->checksum($hashType, $path, $revision); + $this->assertEquals($expected_stauts, $response->getStatus()); + $this->assertEquals($expected_data, $response->getData()); + if ($status) { + \OC::$server->getAppManager()->enableApp($this->versionAppId); + } + } + + public function testChecksumMatchesNoVersion(): void { $hashType = 'sha512'; $path = '/test'; $revision = '1000000000'; - $expected_stauts = Http::STATUS_NOT_FOUND; + $expected_stauts = Http::STATUS_NOT_FOUND; $expected_data = 'specified revision is not found'; $userFolder = $this->getMockBuilder('OCP\Files\Folder')->getMock(); @@ -193,7 +264,8 @@ public function testChecksumMatchesNoVersion() :void { $rootFolder = $this->getMockBuilder('OCP\Files\IRootFolder')->getMock(); $rootFolder->method('getUserFolder')->with('userid')->willReturn($userFolder); - $controller = new ChecksumAPIController('checksum_api', + $controller = new ChecksumAPIController( + 'checksum_api', $this->request, $rootFolder, $this->userSession, @@ -201,7 +273,7 @@ public function testChecksumMatchesNoVersion() :void { $this->logger ); - $status = \OCP\App::isEnabled($this->versionAppId); + $status = \OC::$server->getAppManager()->isEnabledForUser($this->versionAppId); if (!$status) { \OC::$server->getAppManager()->disableApp($this->versionAppId); } @@ -213,12 +285,47 @@ public function testChecksumMatchesNoVersion() :void { } } - public function testChecksumSucceedWithoutRevision() :void { + public function testChecksumMatchesNoVersionWithParallel(): void { + $hashType = 'sha512'; + $path = '/parallelTest'; + $revision = '1000000000'; + $expected_stauts = Http::STATUS_NOT_FOUND; + $expected_data = 'specified revision is not found'; + + $userFolder = $this->getMockBuilder('OCP\Files\Folder')->getMock(); + $userFolder->method('get')->willReturn($this->parallelFile); + + $rootFolder = $this->getMockBuilder('OCP\Files\IRootFolder')->getMock(); + $rootFolder->method('getUserFolder')->with('userid')->willReturn($userFolder); + + $controller = new ChecksumAPIController( + 'checksum_api', + $this->request, + $rootFolder, + $this->userSession, + $this->mapper, + $this->logger + ); + + $status = \OC::$server->getAppManager()->isEnabledForUser($this->versionAppId); + if (!$status) { + \OC::$server->getAppManager()->disableApp($this->versionAppId); + } + $response = $controller->checksum($hashType, $path, $revision); + $this->assertEquals($expected_stauts, $response->getStatus()); + $this->assertEquals($expected_data, $response->getData()); + if (!$status) { + \OC::$server->getAppManager()->enableApp($this->versionAppId); + } + } + + public function testChecksumSucceedWithoutRevision(): void { $hashType = 'sha512,sha256,md5'; $path = '/test.txt'; $revision = null; - $expected_stauts = Http::STATUS_OK; - $expected_data = ['hash' => + $expected_stauts = Http::STATUS_OK; + $expected_data = [ + 'hash' => [ 'sha512' => '44bd27c4fe929be2c4749aadb803c1103eb5b693571d6d73dbc4056d8e18309f88c617c4f5b0f625bfd1d91929cac19bab90c0afbe4042c81132afec6d8b5fa8', 'sha256' => '6a14d590372b7708dfbd52d813068f09f48f8e4c759b3dfb9265f674c10decf2', @@ -238,7 +345,55 @@ public function testChecksumSucceedWithoutRevision() :void { $queryBuilder = $this->getMockBuilder(IQueryBuilder::class)->getMock(); - $controller = new ChecksumAPIController('checksum_api', + $controller = new ChecksumAPIController( + 'checksum_api', + $this->request, + $rootFolder, + $this->userSession, + $this->mapper, + $this->logger + ); + + $status = \OC::$server->getAppManager()->isEnabledForUser($this->versionAppId); + if (!$status) { + \OC::$server->getAppManager()->disableApp($this->versionAppId); + } + $response = $controller->checksum($hashType, $path, $revision); + $this->assertEquals($expected_stauts, $response->getStatus()); + $this->assertEquals($expected_data, $response->getData()); + if (!$status) { + \OC::$server->getAppManager()->enableApp($this->versionAppId); + } + } + + public function testChecksumSucceedWithoutRevisionWithParallel(): void { + $hashType = 'md5,sha256,sha512'; + $path = '/parallelTest.zip'; + $revision = null; + $expected_stauts = Http::STATUS_OK; + $expected_data = [ + 'hash' => + [ + 'sha512' => '8f052607d695f6691edd26733e1b98dbb603363393cdb39e03d35cd96ea4c516e6cea3c319fbf0ed0a98b7ef1295bd32a4e3d822d3b99a9e1499702d1d13b47f', + 'sha256' => '2729740db307b1c88988a64cddedbb4fa0cce068f197431dfbe1b8a900ff25c5', + 'md5' => 'e6cff4641de65a8afd9501cf2e4a36d5' + ] + ]; + + $storage = $this->getMockBuilder('OCP\Files\Storage')->disableOriginalConstructor()->getMock(); + $storage->method('getLocalFile')->willReturn(__DIR__ . '/../data/parallelTest.zip'); + + $userFolder = $this->getMockBuilder('OCP\Files\Folder')->getMock(); + $userFolder->method('get')->willReturn($this->parallelFile); + $userFolder->method('getStorage')->willReturn($storage); + + $rootFolder = $this->getMockBuilder('OCP\Files\IRootFolder')->getMock(); + $rootFolder->method('getUserFolder')->with('userid')->willReturn($userFolder); + + $queryBuilder = $this->getMockBuilder(IQueryBuilder::class)->getMock(); + + $controller = new ChecksumAPIController( + 'checksum_api', $this->request, $rootFolder, $this->userSession, @@ -246,7 +401,7 @@ public function testChecksumSucceedWithoutRevision() :void { $this->logger ); - $status = \OCP\App::isEnabled($this->versionAppId); + $status = \OC::$server->getAppManager()->isEnabledForUser($this->versionAppId); if (!$status) { \OC::$server->getAppManager()->disableApp($this->versionAppId); } diff --git a/tests/data/parallelTest.zip b/tests/data/parallelTest.zip new file mode 100644 index 0000000..b3923a2 Binary files /dev/null and b/tests/data/parallelTest.zip differ