Skip to content

Commit de1b933

Browse files
ArtificialOwlArtificialOwl
committed
new sync service
Signed-off-by: Maxence Lange <maxence@artificial-owl.com>
1 parent 246bfd3 commit de1b933

18 files changed

Lines changed: 677 additions & 11 deletions

‎appinfo/info.xml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,12 @@ Core App of the full-text search framework for your Nextcloud.
3434
<job>OCA\FullTextSearch\Cron\Index</job>
3535
</background-jobs>
3636

37+
<repair-steps>
38+
<install>
39+
<step>OCA\FullTextSearch\RepairStep\AppEnabled</step>
40+
</install>
41+
</repair-steps>
42+
3743
<commands>
3844
<command>OCA\FullTextSearch\Command\Check</command>
3945
<command>OCA\FullTextSearch\Command\CollectionInit</command>
@@ -50,6 +56,7 @@ Core App of the full-text search framework for your Nextcloud.
5056
<command>OCA\FullTextSearch\Command\Reset</command>
5157
<command>OCA\FullTextSearch\Command\Search</command>
5258
<command>OCA\FullTextSearch\Command\Stop</command>
59+
<command>OCA\FullTextSearch\Command\Sync</command>
5360
<command>OCA\FullTextSearch\Command\Test</command>
5461
</commands>
5562

‎lib/AppInfo/Application.php‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,14 @@
1111

1212

1313
use Closure;
14+
use NCU\FullTextSearch\IManager;
1415
use OC;
16+
use OC\FullTextSearch\FullTextSearchManager;
1517
use OCA\FullTextSearch\Capabilities;
1618
use OCA\FullTextSearch\ConfigLexicon;
19+
use OCA\FullTextSearch\Provider\FilesContentProvider;
1720
use OCA\FullTextSearch\Search\UnifiedSearchProvider;
21+
use OCA\FullTextSearch\Service\FullTextSearchService;
1822
use OCA\FullTextSearch\Service\IndexService;
1923
use OCA\FullTextSearch\Service\ProviderService;
2024
use OCA\FullTextSearch\Service\SearchService;
@@ -41,6 +45,9 @@ public function register(IRegistrationContext $context): void {
4145
$context->registerCapability(Capabilities::class);
4246
$context->registerSearchProvider(UnifiedSearchProvider::class);
4347
$context->registerConfigLexicon(ConfigLexicon::class);
48+
49+
$context->registerFullTextSearchService(FullTextSearchService::class);
50+
4451
$this->registerServices($this->getContainer());
4552
}
4653

@@ -53,11 +60,8 @@ public function boot(IBootContext $context): void {
5360
$context->injectFn(Closure::fromCallable([$this, 'registerNavigation']));
5461
}
5562

56-
5763
/**
58-
* Register Navigation Tab
59-
*
60-
* @param ContainerInterface $container
64+
* @deprecated
6165
*/
6266
protected function registerServices(ContainerInterface $container) {
6367
/** @var IFullTextSearchManager $fullTextSearchManager */

‎lib/Command/Stop.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ protected function configure() {
4141
protected function execute(InputInterface $input, OutputInterface $output) {
4242
$output->writeln('stopping all running indexes');
4343

44-
$this->runningService->forceStop();
44+
$this->runningService->stop();
4545

4646
return 0;
4747
}

‎lib/Command/Sync.php‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2016 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-or-later
8+
*/
9+
10+
namespace OCA\FullTextSearch\Command;
11+
12+
use OC\Core\Command\Base;
13+
use OCA\FullTextSearch\Exceptions\PlatformTemporaryException;
14+
use OCA\FullTextSearch\Service\FullTextSearchService;
15+
use OCA\FullTextSearch\Service\LockService;
16+
use OCA\FullTextSearch\Service\LoggerService;
17+
use OCA\FullTextSearch\Service\SyncService;
18+
use OCA\FullTextSearch\Tools\Traits\TArrayTools;
19+
use Exception;
20+
use OC\Core\Command\InterruptedException;
21+
use OCA\FullTextSearch\ACommandBase;
22+
use OCA\FullTextSearch\Exceptions\TickDoesNotExistException;
23+
use OCA\FullTextSearch\Model\Index as ModelIndex;
24+
use OCA\FullTextSearch\Model\Runner;
25+
use OCA\FullTextSearch\Service\CliService;
26+
use OCA\FullTextSearch\Service\ConfigService;
27+
use OCA\FullTextSearch\Service\IndexService;
28+
use OCA\FullTextSearch\Service\PlatformService;
29+
use OCA\FullTextSearch\Service\ProviderService;
30+
use OCA\FullTextSearch\Service\RunningService;
31+
use OCP\IUserManager;
32+
use OutOfBoundsException;
33+
use Psr\Log\LoggerInterface;
34+
use Symfony\Component\Console\Formatter\OutputFormatterStyle;
35+
use Symfony\Component\Console\Input\InputInterface;
36+
use Symfony\Component\Console\Input\InputOption;
37+
use Symfony\Component\Console\Output\OutputInterface;
38+
use Symfony\Component\Console\Terminal;
39+
use Throwable;
40+
41+
class Sync extends Base {
42+
public function __construct(
43+
private readonly LockService $lockService,
44+
private readonly SyncService $syncService,
45+
private readonly FullTextSearchService $fullTextSearchService,
46+
private readonly LoggerService $loggerService,
47+
) {
48+
parent::__construct();
49+
}
50+
51+
protected function configure() {
52+
parent::configure();
53+
$this->setName('fulltextsearch:sync')
54+
->setDescription('Index files')
55+
->addOption('info', '', InputOption::VALUE_NONE, 'display info entries')
56+
->addOption('no-output', '', InputOption::VALUE_NONE, 'no output, use nextcloud logs');
57+
}
58+
59+
protected function execute(InputInterface $input, OutputInterface $output) {
60+
if (!$input->getOption('no-output')) {
61+
$this->loggerService->setOutputInterface($output, $input->getOption('info'));
62+
}
63+
// $this->fullTextSearchService->requestIndex('files', '123');
64+
$this->lockService->lock();
65+
66+
while (true) {
67+
try {
68+
$this->lockService->update();
69+
$this->syncProcess();
70+
} catch (Exception $e) {
71+
$this->loggerService->error('Exception while running fulltextsearch:sync', ['exception' => $e]);
72+
}
73+
sleep(10);
74+
}
75+
76+
return 0;
77+
}
78+
79+
private function syncProcess() {
80+
$this->loggerService->info('initiating a new sync session');
81+
$this->syncService->smartSync();
82+
$this->loggerService->info('sync session closed');
83+
}
84+
}
85+

‎lib/ConfigLexicon.php‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ class ConfigLexicon implements ILexicon {
2828
public const COLLECTION_LINKS = 'collection_links';
2929
public const LOCK_ID = 'lock_id';
3030
public const LOCK_PING = 'lock_ping';
31+
public const ENABLED_SINCE = 'enabled_since';
3132

3233
public function getStrictness(): Strictness {
3334
return Strictness::NOTICE;
@@ -45,6 +46,7 @@ public function getAppConfigs(): array {
4546
// IAppConfig::FLAG_INTERNAL)
4647
new Entry(key: self::LOCK_ID, type: ValueType::STRING, defaultRaw: '', definition: 'internal lock id', lazy: true),
4748
new Entry(key: self::LOCK_PING, type: ValueType::INT, defaultRaw: 0, definition: 'internal lock time', lazy: true),
49+
new Entry(key: self::ENABLED_SINCE, type: ValueType::INT, defaultRaw: 0, definition: 'the time since the fulltextsearch app is enabled', lazy: true),
4850
];
4951
}
5052

‎lib/Db/SyncMapper.php‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
/**
5+
* SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors
6+
* SPDX-License-Identifier: AGPL-3.0-or-later
7+
*/
8+
9+
namespace OCA\FullTextSearch\Db;
10+
11+
use OCA\FullTextSearch\Model\DocumentSync;
12+
use OCP\AppFramework\Db\Entity;
13+
use OCP\AppFramework\Db\QBMapper;
14+
use OCP\DB\QueryBuilder\IQueryBuilder;
15+
use OCP\IDBConnection;
16+
17+
/**
18+
* @template-extends QBMapper<DocumentSync>
19+
*/
20+
class SyncMapper extends QBMapper {
21+
public const TABLE = 'fulltextsearch_sync';
22+
23+
public function __construct(
24+
IDBConnection $db,
25+
) {
26+
parent::__construct($db, self::TABLE, DocumentSync::class);
27+
}
28+
29+
/**
30+
* @return DocumentSync[]
31+
*/
32+
public function getForcedSyncs(int $limit = 100): array {
33+
$qb = $this->db->getQueryBuilder();
34+
$qb->select('*')
35+
->from($this->getTableName())
36+
->where($qb->expr()->eq('indexed', $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT)))
37+
->setMaxResults($limit);
38+
39+
return $this->findEntities($qb);
40+
}
41+
42+
public function update(Entity $entity): Entity {
43+
$qb = $this->db->getQueryBuilder();
44+
$qb->update($this->getTableName())
45+
->set('indexed', $qb->createNamedParameter($entity->getIndexed(), IQueryBuilder::PARAM_INT))
46+
->where(
47+
$qb->expr()->eq('provider_id', $qb->createNamedParameter($entity->getProviderId())),
48+
$qb->expr()->eq('document_id', $qb->createNamedParameter($entity->getDocumentId())),
49+
);
50+
$qb->executeStatement();
51+
return $entity;
52+
}
53+
54+
// public function reset(string $providerId, string $documentId): void {
55+
// $qb = $this->db->getQueryBuilder();
56+
// $qb->update($this->getTableName())
57+
// ->set('indexed', $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT))
58+
// ->where(
59+
// $qb->expr()->eq('provider_id', $qb->createNamedParameter($providerId())),
60+
// $qb->expr()->eq('document_id', $qb->createNamedParameter($documentId())),
61+
// );
62+
// $qb->executeStatement();
63+
// }
64+
}

‎lib/Enum/SessionType.php‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-or-later
8+
*/
9+
10+
namespace OCA\FullTextSearch\Enum;
11+
12+
enum SessionType: string {
13+
case UNKNOWN = '';
14+
case FORCED = 'forced';
15+
case SYNC = 'sync';
16+
case RESYNC = 'sync_recent';
17+
}

‎lib/Migration/Version23001Date20220408140253.php‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,11 @@ public function changeSchema(IOutput $output, Closure $schemaClosure, array $opt
2828
/** @var ISchemaWrapper $schema */
2929
$schema = $schemaClosure();
3030

31-
if (!$schema->hasTable('fulltextsearch_indexes')) {
31+
if (!$schema->hasTable('fulltextsearch_sync')) {
3232
return null;
3333
}
3434

35-
$table = $schema->getTable('fulltextsearch_indexes');
35+
$table = $schema->getTable('fulltextsearch_sync');
3636
$column = $table->getColumn('message');
3737

3838
if ($column->getType()->getName() === Types::TEXT) {

‎lib/Migration/Version33001Date202511271645.php‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-or-later
8+
*/
9+
10+
namespace OCA\FullTextSearch\Migration;
11+
12+
use Closure;
13+
use OCP\DB\ISchemaWrapper;
14+
use OCP\DB\Types;
15+
use OCP\Migration\Attributes\CreateTable;
16+
use OCP\Migration\IOutput;
17+
use OCP\Migration\SimpleMigrationStep;
18+
19+
#[CreateTable('fulltextsearch_sync', ['provider_id', 'document_id', 'indexed'], description: 'table to manage indexed documents and new index request')]
20+
class Version33001Date202511271645 extends SimpleMigrationStep {
21+
public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
22+
/** @var ISchemaWrapper $schema */
23+
$schema = $schemaClosure();
24+
if ($schema->hasTable('fulltextsearch_sync')) {
25+
return null;
26+
}
27+
28+
$table = $schema->createTable('fulltextsearch_sync');
29+
$table->addColumn('provider_id', Types::STRING, [
30+
'length' => 31,
31+
'notnull' => true,
32+
]);
33+
$table->addColumn('document_id', Types::STRING, [
34+
'length' => 31,
35+
'notnull' => true,
36+
]);
37+
$table->addColumn('flags', Types::INTEGER, [
38+
'length' => 7,
39+
'default' => 0,
40+
]);
41+
$table->addColumn('indexed', Types::BIGINT, [
42+
'length' => 11,
43+
'default' => 0,
44+
]);
45+
46+
$table->setPrimaryKey(['provider_id', 'document_id'], 'fts_i_pd');
47+
$table->addIndex(['indexed'], 'fts_i_i');
48+
49+
return $schema;
50+
}
51+
}

‎lib/Model/DocumentSync.php‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-or-later
8+
*/
9+
namespace OCA\FullTextSearch\Model;
10+
11+
use OCP\AppFramework\Db\Entity;
12+
13+
/**
14+
* @method void setProviderId(string $providerId)
15+
* @method string getProviderId()
16+
* @method void setDocumentId(string $documentId)
17+
* @method string getDocumentId()
18+
* @method void setFlags(int $indexed)
19+
* @method int getFlags()
20+
* @method void setIndexed(int $indexed)
21+
* @method int getIndexed()
22+
* @psalm-suppress PropertyNotSetInConstructor
23+
*/
24+
class DocumentSync extends Entity {
25+
protected string $providerId = '';
26+
protected string $documentId = '';
27+
protected int $flags = 0;
28+
protected int $indexed = 0;
29+
30+
public function __construct(
31+
) {
32+
$this->addType('providerId', 'string');
33+
$this->addType('documentId', 'string');
34+
$this->addType('flags', 'integer');
35+
$this->addType('indexed', 'integer');
36+
}
37+
38+
public function definition(): string {
39+
return $this->getProviderId() . '/' . $this->getDocumentId();
40+
}
41+
}

0 commit comments

Comments
 (0)