diff --git a/README.md b/README.md
index 6a9e765..99bf9bd 100644
--- a/README.md
+++ b/README.md
@@ -87,10 +87,12 @@ the internal handling of Composer v1.x.
The first database population may easily take hours. Be patient.
-0. `bin/console doctrine:migrations:migrate`: Ensure the database schema is up to date with the code.
-1. `bin/console refresh`: Query the WordPress.org SVN in order to find new and updated packages.
-2. `bin/console update`: Update the version information for packages identified in `2`. Uses the WordPress.org API.
-3. `bin/console build`: Rebuild all `PackageStore` data.
+1. `bin/console doctrine:migrations:migrate`: Ensure the database schema is up to date with the code.
+2. `bin/console refresh`: Query the WordPress.org SVN in order to find new and updated packages.
+3. `bin/console update`: Update the version information for packages identified in `(2)`. Uses the WordPress.org API.
+4. `bin/console build`: Rebuild all `PackageStore` data.
+
+Each of these can be run with the `-vvv` verbosity flag, to give useful progress updates
## Running locally with Docker
diff --git a/src/Entity/PackageRepository.php b/src/Entity/PackageRepository.php
index c7030d9..a1829e7 100644
--- a/src/Entity/PackageRepository.php
+++ b/src/Entity/PackageRepository.php
@@ -51,7 +51,7 @@ public function updateProviderGroups(): void
* being fetched or that are inactive but have been updated in the past 90 days
* and haven't been fetched in the past 7 days.
*
- * @return Package[]
+ * @return array consisting of count and iterable
*/
public function findDueUpdate(): array
{
@@ -63,14 +63,18 @@ public function findDueUpdate(): array
OR (p.lastCommitted - p.lastFetched) > :twoHours
OR (p.isActive = false AND p.lastCommitted > :threeMonthsAgo AND p.lastFetched < :oneWeekAgo)
EOT;
+ $countDql = str_replace('SELECT p', 'SELECT COUNT(1)', $dql);
$dateFormat = $this->getEntityManager()->getConnection()->getDatabasePlatform()->getDateTimeFormatString();
- $query = $entityManager->createQuery($dql)
- // This seems to be how Doctrine wants its 'interval' type bound – not a DateInterval
- ->setParameter('twoHours', '2 hour')
- ->setParameter('threeMonthsAgo', (new \DateTime())->sub(new \DateInterval('P3M'))->format($dateFormat))
- ->setParameter('oneWeekAgo', (new \DateTime())->sub(new \DateInterval('P1W'))->format($dateFormat));
+ // This seems to be how Doctrine wants its 'interval' type bound – not a DateInterval
+ $parameters = [
+ 'twoHours' => '2 hour',
+ 'threeMonthsAgo' => (new \DateTime())->sub(new \DateInterval('P3M'))->format($dateFormat),
+ 'oneWeekAgo' => (new \DateTime())->sub(new \DateInterval('P1W'))->format($dateFormat)
+ ];
+ $query = $entityManager->createQuery($dql)->setParameters($parameters);
+ $countQuery = $entityManager->createQuery($countDql)->setParameters($parameters);
- return $query->getResult();
+ return [$countQuery->getSingleScalarResult(), $query->toIterable()];
}
/**
diff --git a/src/Service/Update.php b/src/Service/Update.php
index 8eee97d..3e371fa 100644
--- a/src/Service/Update.php
+++ b/src/Service/Update.php
@@ -32,7 +32,7 @@ public function __construct(EntityManagerInterface $entityManager)
public function updateAll(LoggerInterface $logger): void
{
$packages = $this->repo->findDueUpdate();
- $this->update($logger, $packages);
+ $this->update($logger, $packages[1], $packages[0]);
}
/**
@@ -62,7 +62,7 @@ public function updateOne(LoggerInterface $logger, string $name, int $allowMoreT
if ($package) {
try {
- $this->update($logger, [$package]);
+ $this->update($logger, [$package], 1);
} catch (UniqueConstraintViolationException $exception) {
if ($allowMoreTries > 0) {
return $this->updateOne($logger, $name, $allowMoreTries - 1);
@@ -78,24 +78,28 @@ public function updateOne(LoggerInterface $logger, string $name, int $allowMoreT
/**
* @param LoggerInterface $logger
- * @param Package[] $packages
+ * @param iterable|Package[] $packages
+ * @param int $count
*/
- protected function update(LoggerInterface $logger, array $packages): void
+ protected function update(LoggerInterface $logger, mixed $packages, int $count): void
{
- $count = count($packages);
$versionParser = new VersionParser();
$wporgClient = WporgClient::getClient();
$logger->info("Updating {$count} packages");
- foreach ($packages as $index => $package) {
- $percent = $index / $count * 100;
+ $batchSize = 100;
+
+ $i = 0;
+ foreach ($packages as $package) {
+ $percent = ++$i / $count * 100;
$name = $package->getName();
$info = null;
$fields = ['versions'];
+ $deactivateReason = null;
try {
if ($package instanceof Plugin) {
$info = $wporgClient->getPlugin($name, $fields);
@@ -104,82 +108,99 @@ protected function update(LoggerInterface $logger, array $packages): void
}
$logger->info(sprintf("%04.1f%% Fetched %s %s", $percent, $package->getType(), $name));
+ if (empty($info)) {
+ $deactivateReason = 'not active';
+ }
} catch (CommandClientException $exception) {
$res = $exception->getResponse();
- $this->deactivate($package, $res->getStatusCode() . ': ' . $res->getReasonPhrase(), $logger);
- continue;
+ $deactivateReason = $res->getStatusCode() . ': ' . $res->getReasonPhrase();
} catch (GuzzleException $exception) {
$logger->warning("Skipped {$package->getType()} '{$name}' due to error: '{$exception->getMessage()}'");
}
- if (empty($info)) {
- // Plugin is not active
- $this->deactivate($package, 'not active', $logger);
-
- continue;
- }
-
- //get versions as [version => url]
- $versions = $info['versions'] ?: [];
-
- //current version of plugin not present in tags so add it
- if (empty($versions[$info['version']])) {
- $logger->info('Adding trunk psuedo-version for ' . $name);
-
- //add to front of array
- $versions = array_reverse($versions, true);
- $versions[$info['version']] = 'trunk';
- $versions = array_reverse($versions, true);
- }
-
- //all plugins have a dev-trunk version
- if ($package instanceof Plugin) {
- unset($versions['trunk']);
- $versions['dev-trunk'] = 'trunk';
- }
+ if ($info && !$deactivateReason) {
+ $versions = $this->extractVersions($package, $info, $versionParser, $logger);
- foreach ($versions as $version => $url) {
- try {
- //make sure versions are parseable by Composer
- $versionParser->normalize($version);
- if ($package instanceof Theme) {
- //themes have different SVN folder structure
- $versions[$version] = $version;
- } elseif ($url !== 'trunk') {
- //add ref to SVN tag
- $versions[$version] = 'tags/' . $version;
- } // else do nothing, for 'trunk'.
- } catch (\UnexpectedValueException $e) {
- // Version is invalid – we've seen this e.g. with 5 numeric parts.
- $logger->info(sprintf(
- 'Skipping invalid version %s for %s %s',
- $version,
- $package->getType(),
- $name
- ));
- unset($versions[$version]);
+ if ($versions) {
+ $package->setLastFetched(new \DateTime());
+ $package->setVersions($versions);
+ $package->setIsActive(true);
+ $package->setDisplayName($info['name']);
+ $this->entityManager->persist($package);
+ } else {
+ $deactivateReason = 'no versions found';
}
}
- if ($versions) {
+ if ($deactivateReason) {
$package->setLastFetched(new \DateTime());
- $package->setVersions($versions);
- $package->setIsActive(true);
- $package->setDisplayName($info['name']);
+ $package->setIsActive(false);
$this->entityManager->persist($package);
- } else {
- // Package is not active
- $this->deactivate($package, 'no versions found', $logger);
+ $logger->info(sprintf("Deactivated %s %s because %s", $package->getType(), $package->getName(), $deactivateReason));
+ }
+
+ if (($i % $batchSize) === 0) {
+ $logger->info('---Persisting updated packages---');
+ $this->entityManager->flush();
+ $this->entityManager->clear();
}
}
$this->entityManager->flush();
}
- private function deactivate(Package $package, string $reason, LoggerInterface $logger): void
+ /**
+ * @param Package $package
+ * @param array $info
+ * @param VersionParser $versionParser
+ * @param LoggerInterface $logger
+ * @return array|mixed
+ */
+ protected function extractVersions($package, $info, $versionParser, $logger)
{
- $package->setLastFetched(new \DateTime());
- $package->setIsActive(false);
- $this->entityManager->persist($package);
- $logger->info(sprintf("Deactivated %s %s because %s", $package->getType(), $package->getName(), $reason));
+ $name = $package->getName();
+
+ //get versions as [version => url]
+ $versions = $info['versions'] ?: [];
+
+ //current version of plugin not present in tags so add it
+ if (empty($versions[$info['version']])) {
+ $logger->info('Adding trunk pseudo-version for ' . $name);
+
+ //add to front of array
+ $versions = array_reverse($versions, true);
+ $versions[$info['version']] = 'trunk';
+ $versions = array_reverse($versions, true);
+ }
+
+ //all plugins have a dev-trunk version
+ if ($package instanceof Plugin) {
+ unset($versions['trunk']);
+ $versions['dev-trunk'] = 'trunk';
+ }
+
+ foreach ($versions as $version => $url) {
+ try {
+ //make sure versions are parseable by Composer
+ $versionParser->normalize($version);
+ if ($package instanceof Theme) {
+ //themes have different SVN folder structure
+ $versions[$version] = $version;
+ } elseif ($url !== 'trunk') {
+ //add ref to SVN tag
+ $versions[$version] = 'tags/' . $version;
+ } // else do nothing, for 'trunk'.
+ } catch (\UnexpectedValueException $e) {
+ // Version is invalid – we've seen this e.g. with 5 numeric parts.
+ $logger->info(sprintf(
+ 'Skipping invalid version %s for %s %s',
+ $version,
+ $package->getType(),
+ $name
+ ));
+ unset($versions[$version]);
+ }
+ }
+
+ return $versions;
}
}