diff --git a/README.md b/README.md index 6a9e765..99bf9bd 100644 --- a/README.md +++ b/README.md @@ -87,10 +87,12 @@ the internal handling of Composer v1.x. The first database population may easily take hours. Be patient. -0. `bin/console doctrine:migrations:migrate`: Ensure the database schema is up to date with the code. -1. `bin/console refresh`: Query the WordPress.org SVN in order to find new and updated packages. -2. `bin/console update`: Update the version information for packages identified in `2`. Uses the WordPress.org API. -3. `bin/console build`: Rebuild all `PackageStore` data. +1. `bin/console doctrine:migrations:migrate`: Ensure the database schema is up to date with the code. +2. `bin/console refresh`: Query the WordPress.org SVN in order to find new and updated packages. +3. `bin/console update`: Update the version information for packages identified in `(2)`. Uses the WordPress.org API. +4. `bin/console build`: Rebuild all `PackageStore` data. + +Each of these can be run with the `-vvv` verbosity flag, to give useful progress updates ## Running locally with Docker diff --git a/src/Entity/PackageRepository.php b/src/Entity/PackageRepository.php index c7030d9..a1829e7 100644 --- a/src/Entity/PackageRepository.php +++ b/src/Entity/PackageRepository.php @@ -51,7 +51,7 @@ public function updateProviderGroups(): void * being fetched or that are inactive but have been updated in the past 90 days * and haven't been fetched in the past 7 days. * - * @return Package[] + * @return array consisting of count and iterable */ public function findDueUpdate(): array { @@ -63,14 +63,18 @@ public function findDueUpdate(): array OR (p.lastCommitted - p.lastFetched) > :twoHours OR (p.isActive = false AND p.lastCommitted > :threeMonthsAgo AND p.lastFetched < :oneWeekAgo) EOT; + $countDql = str_replace('SELECT p', 'SELECT COUNT(1)', $dql); $dateFormat = $this->getEntityManager()->getConnection()->getDatabasePlatform()->getDateTimeFormatString(); - $query = $entityManager->createQuery($dql) - // This seems to be how Doctrine wants its 'interval' type bound – not a DateInterval - ->setParameter('twoHours', '2 hour') - ->setParameter('threeMonthsAgo', (new \DateTime())->sub(new \DateInterval('P3M'))->format($dateFormat)) - ->setParameter('oneWeekAgo', (new \DateTime())->sub(new \DateInterval('P1W'))->format($dateFormat)); + // This seems to be how Doctrine wants its 'interval' type bound – not a DateInterval + $parameters = [ + 'twoHours' => '2 hour', + 'threeMonthsAgo' => (new \DateTime())->sub(new \DateInterval('P3M'))->format($dateFormat), + 'oneWeekAgo' => (new \DateTime())->sub(new \DateInterval('P1W'))->format($dateFormat) + ]; + $query = $entityManager->createQuery($dql)->setParameters($parameters); + $countQuery = $entityManager->createQuery($countDql)->setParameters($parameters); - return $query->getResult(); + return [$countQuery->getSingleScalarResult(), $query->toIterable()]; } /** diff --git a/src/Service/Update.php b/src/Service/Update.php index 8eee97d..3e371fa 100644 --- a/src/Service/Update.php +++ b/src/Service/Update.php @@ -32,7 +32,7 @@ public function __construct(EntityManagerInterface $entityManager) public function updateAll(LoggerInterface $logger): void { $packages = $this->repo->findDueUpdate(); - $this->update($logger, $packages); + $this->update($logger, $packages[1], $packages[0]); } /** @@ -62,7 +62,7 @@ public function updateOne(LoggerInterface $logger, string $name, int $allowMoreT if ($package) { try { - $this->update($logger, [$package]); + $this->update($logger, [$package], 1); } catch (UniqueConstraintViolationException $exception) { if ($allowMoreTries > 0) { return $this->updateOne($logger, $name, $allowMoreTries - 1); @@ -78,24 +78,28 @@ public function updateOne(LoggerInterface $logger, string $name, int $allowMoreT /** * @param LoggerInterface $logger - * @param Package[] $packages + * @param iterable|Package[] $packages + * @param int $count */ - protected function update(LoggerInterface $logger, array $packages): void + protected function update(LoggerInterface $logger, mixed $packages, int $count): void { - $count = count($packages); $versionParser = new VersionParser(); $wporgClient = WporgClient::getClient(); $logger->info("Updating {$count} packages"); - foreach ($packages as $index => $package) { - $percent = $index / $count * 100; + $batchSize = 100; + + $i = 0; + foreach ($packages as $package) { + $percent = ++$i / $count * 100; $name = $package->getName(); $info = null; $fields = ['versions']; + $deactivateReason = null; try { if ($package instanceof Plugin) { $info = $wporgClient->getPlugin($name, $fields); @@ -104,82 +108,99 @@ protected function update(LoggerInterface $logger, array $packages): void } $logger->info(sprintf("%04.1f%% Fetched %s %s", $percent, $package->getType(), $name)); + if (empty($info)) { + $deactivateReason = 'not active'; + } } catch (CommandClientException $exception) { $res = $exception->getResponse(); - $this->deactivate($package, $res->getStatusCode() . ': ' . $res->getReasonPhrase(), $logger); - continue; + $deactivateReason = $res->getStatusCode() . ': ' . $res->getReasonPhrase(); } catch (GuzzleException $exception) { $logger->warning("Skipped {$package->getType()} '{$name}' due to error: '{$exception->getMessage()}'"); } - if (empty($info)) { - // Plugin is not active - $this->deactivate($package, 'not active', $logger); - - continue; - } - - //get versions as [version => url] - $versions = $info['versions'] ?: []; - - //current version of plugin not present in tags so add it - if (empty($versions[$info['version']])) { - $logger->info('Adding trunk psuedo-version for ' . $name); - - //add to front of array - $versions = array_reverse($versions, true); - $versions[$info['version']] = 'trunk'; - $versions = array_reverse($versions, true); - } - - //all plugins have a dev-trunk version - if ($package instanceof Plugin) { - unset($versions['trunk']); - $versions['dev-trunk'] = 'trunk'; - } + if ($info && !$deactivateReason) { + $versions = $this->extractVersions($package, $info, $versionParser, $logger); - foreach ($versions as $version => $url) { - try { - //make sure versions are parseable by Composer - $versionParser->normalize($version); - if ($package instanceof Theme) { - //themes have different SVN folder structure - $versions[$version] = $version; - } elseif ($url !== 'trunk') { - //add ref to SVN tag - $versions[$version] = 'tags/' . $version; - } // else do nothing, for 'trunk'. - } catch (\UnexpectedValueException $e) { - // Version is invalid – we've seen this e.g. with 5 numeric parts. - $logger->info(sprintf( - 'Skipping invalid version %s for %s %s', - $version, - $package->getType(), - $name - )); - unset($versions[$version]); + if ($versions) { + $package->setLastFetched(new \DateTime()); + $package->setVersions($versions); + $package->setIsActive(true); + $package->setDisplayName($info['name']); + $this->entityManager->persist($package); + } else { + $deactivateReason = 'no versions found'; } } - if ($versions) { + if ($deactivateReason) { $package->setLastFetched(new \DateTime()); - $package->setVersions($versions); - $package->setIsActive(true); - $package->setDisplayName($info['name']); + $package->setIsActive(false); $this->entityManager->persist($package); - } else { - // Package is not active - $this->deactivate($package, 'no versions found', $logger); + $logger->info(sprintf("Deactivated %s %s because %s", $package->getType(), $package->getName(), $deactivateReason)); + } + + if (($i % $batchSize) === 0) { + $logger->info('---Persisting updated packages---'); + $this->entityManager->flush(); + $this->entityManager->clear(); } } $this->entityManager->flush(); } - private function deactivate(Package $package, string $reason, LoggerInterface $logger): void + /** + * @param Package $package + * @param array $info + * @param VersionParser $versionParser + * @param LoggerInterface $logger + * @return array|mixed + */ + protected function extractVersions($package, $info, $versionParser, $logger) { - $package->setLastFetched(new \DateTime()); - $package->setIsActive(false); - $this->entityManager->persist($package); - $logger->info(sprintf("Deactivated %s %s because %s", $package->getType(), $package->getName(), $reason)); + $name = $package->getName(); + + //get versions as [version => url] + $versions = $info['versions'] ?: []; + + //current version of plugin not present in tags so add it + if (empty($versions[$info['version']])) { + $logger->info('Adding trunk pseudo-version for ' . $name); + + //add to front of array + $versions = array_reverse($versions, true); + $versions[$info['version']] = 'trunk'; + $versions = array_reverse($versions, true); + } + + //all plugins have a dev-trunk version + if ($package instanceof Plugin) { + unset($versions['trunk']); + $versions['dev-trunk'] = 'trunk'; + } + + foreach ($versions as $version => $url) { + try { + //make sure versions are parseable by Composer + $versionParser->normalize($version); + if ($package instanceof Theme) { + //themes have different SVN folder structure + $versions[$version] = $version; + } elseif ($url !== 'trunk') { + //add ref to SVN tag + $versions[$version] = 'tags/' . $version; + } // else do nothing, for 'trunk'. + } catch (\UnexpectedValueException $e) { + // Version is invalid – we've seen this e.g. with 5 numeric parts. + $logger->info(sprintf( + 'Skipping invalid version %s for %s %s', + $version, + $package->getType(), + $name + )); + unset($versions[$version]); + } + } + + return $versions; } }