From 42538a61ed9131dae050eea301f131accecabca8 Mon Sep 17 00:00:00 2001 From: RafaUC Date: Sat, 22 Nov 2025 03:11:45 -0600 Subject: [PATCH 1/4] - feat(watcher): query missed watch events on startup - feat(watcher): auto-save watcher snapshots on app and watcher close - Fix the console popup bug when initializing the watcher by ensuring the watcher backend to use. --- src/frontend/entities/Location.ts | 18 +++++- src/frontend/stores/LocationStore.ts | 34 +++++++++++ src/frontend/stores/RootStore.ts | 1 + src/frontend/workers/folderWatcher.worker.ts | 62 ++++++++++++++++---- src/ipc/renderer.ts | 11 +++- 5 files changed, 113 insertions(+), 13 deletions(-) diff --git a/src/frontend/entities/Location.ts b/src/frontend/entities/Location.ts index f5b1f861..61a96a82 100644 --- a/src/frontend/entities/Location.ts +++ b/src/frontend/entities/Location.ts @@ -517,7 +517,17 @@ export class ClientLocation { this._worker?.terminate(); this._worker = worker; // Make a list of all files in this directory, which will be returned when all subdirs have been traversed - const initialFiles = await this.worker.watch(directory, this.extensions); + await fse.ensureDir(this.store.watcherSnapshotDirectory); + const snapshotFilePath = SysPath.join( + this.store.watcherSnapshotDirectory, + `${this.id}.snapshot.json`, + ); + const initialFiles = await this.worker.watch( + directory, + this.extensions, + snapshotFilePath, + this.store.PARCEL_WATCHER_BACKEND, + ); this.setSettingWatcher(false); // Filter out images from excluded sub-locations @@ -527,6 +537,12 @@ export class ClientLocation { !this.excludedPaths.some((subLoc) => absolutePath.startsWith(subLoc.path)), ); } + + // close and save snapshots of the watcher worker + @action async close(): Promise { + await this.worker?.cancel(); + await this.worker?.close(); + } } interface IDirectoryTreeItem { name: string; diff --git a/src/frontend/stores/LocationStore.ts b/src/frontend/stores/LocationStore.ts index 8aabf803..51e4451e 100644 --- a/src/frontend/stores/LocationStore.ts +++ b/src/frontend/stores/LocationStore.ts @@ -17,6 +17,9 @@ import { ClientStringSearchCriteria } from '../entities/SearchCriteria'; import ImageLoader from '../image/ImageLoader'; import RootStore from './RootStore'; import { ClientTag } from '../entities/Tag'; +import { IS_MAC, IS_WIN } from 'common/process'; +import { BackendType } from '@parcel/watcher'; +import { execSync } from 'child_process'; const PREFERENCES_STORAGE_KEY = 'location-store-preferences'; type Preferences = { extensions: IMG_EXTENSIONS_TYPE[] }; @@ -35,9 +38,20 @@ function areFilesIdenticalBesidesName(a: FileDTO, b: FileDTO): boolean { ); } +function isWatchmanInstalled(): boolean { + try { + execSync('watchman --version', { stdio: 'ignore' }); + return true; + } catch (err) { + return false; + } +} + class LocationStore { private readonly backend: DataStorage; private readonly rootStore: RootStore; + watcherSnapshotDirectory!: string; + PARCEL_WATCHER_BACKEND!: BackendType; readonly locationList = observable([]); @@ -63,6 +77,19 @@ class LocationStore { // By default, disable EXR for now (experimental) this.enabledFileExtensions.delete('exr'); } + this.watcherSnapshotDirectory = await RendererMessenger.getWatcherSnapshotsDirectory(); + + // Fix the console popup bug when initializing the watcher by ensuring the watcher backend to use. + // look at https://github.com/eclipse-theia/theia/pull/16335 + const isWatchman = isWatchmanInstalled(); + this.PARCEL_WATCHER_BACKEND = isWatchman + ? 'watchman' + : IS_WIN + ? 'windows' + : IS_MAC + ? 'fs-events' + : 'inotify'; + console.debug('Watcher backend:', this.PARCEL_WATCHER_BACKEND); // Get dirs from backend const dirs = await this.backend.fetchLocations(); @@ -677,6 +704,13 @@ class LocationStore { this.save(this.locationList[i].serialize()); } } + + // Close and save snapshots for all watcher workers + @action async close(): Promise { + for (const location of this.locationList) { + await location.close(); + } + } } export type FileStats = { diff --git a/src/frontend/stores/RootStore.ts b/src/frontend/stores/RootStore.ts index efecf4a4..453184ea 100644 --- a/src/frontend/stores/RootStore.ts +++ b/src/frontend/stores/RootStore.ts @@ -188,6 +188,7 @@ class RootStore { } async close(): Promise { + await this.locationStore.close(); // TODO: should be able to be done more reliably by running exiftool as a child process await this.exifTool.close(); } diff --git a/src/frontend/workers/folderWatcher.worker.ts b/src/frontend/workers/folderWatcher.worker.ts index ba8c8fac..e565a4f7 100644 --- a/src/frontend/workers/folderWatcher.worker.ts +++ b/src/frontend/workers/folderWatcher.worker.ts @@ -12,36 +12,57 @@ export class FolderWatcherWorker { // Whether the initial scan has been completed, and new/removed files are being watched private isReady = false; private isCancelled = false; + private directory?: string; + private snapshotFilePath?: string; + private backend?: parcelWatcher.BackendType; cancel() { this.isCancelled = true; } async close() { - this.watcher?.unsubscribe(); + if (this.watcher) { + this.watcher.unsubscribe(); + this.watcher = undefined; + } + // Save watcher snapshot on close + if (this.snapshotFilePath && this.directory) { + console.debug(`Creating watcher snapshot for ${this.directory}: ${this.snapshotFilePath}`); + try { + await parcelWatcher.writeSnapshot(this.directory, this.snapshotFilePath, { + backend: this.backend, + }); + } catch (err) { + console.error(`${this.snapshotFilePath} - Failed writing snapshot on close:`, err); + } + } } /** Returns all supported image files in the given directly, and callbacks for new or removed files */ - async watch(directory: string, extensions: IMG_EXTENSIONS_TYPE[]) { + async watch( + directory: string, + extensions: IMG_EXTENSIONS_TYPE[], + snapshotFilePath: string, + backend: parcelWatcher.BackendType, + ) { this.isCancelled = false; + this.backend = backend; // Replace backslash with forward slash, recommended by chokidar // See docs for the .watch method: https://github.com/paulmillr/chokidar#api directory = directory.replace(/\\/g, '/'); + snapshotFilePath = snapshotFilePath.replace(/\\/g, '/'); + this.directory = directory; + this.snapshotFilePath = snapshotFilePath; // Watch for files being added/changed/removed: // Usually you'd include a glob in the watch argument, e.g. `directory/**/.{jpg|png|...}`, but we cannot use globs unfortunately (see disableGlobbing) // watch for this https://github.com/parcel-bundler/watcher/pull/207 this.isReady = true; - this.watcher = await parcelWatcher.subscribe( - directory, - (err, events) => { + const handleEvents = // Small indentation hack to avoid affecting git blame + (events: parcelWatcher.Event[], extensions: IMG_EXTENSIONS_TYPE[]) => { for (const event of events) { - if (err) { - console.error('Error fired in watcher', directory, err); - ctx.postMessage({ type: 'error', value: err }); - } // Ignore Files that aren't our extension type const ext = SysPath.extname(event.path).toLowerCase().split('.')[1]; if (!extensions.includes(ext as IMG_EXTENSIONS_TYPE)) { @@ -99,8 +120,29 @@ export class FolderWatcherWorker { ctx.postMessage({ type: 'remove', value: event.path }); } } + }; + + //Query for changes made while the watcher was down. + try { + console.debug('Reading watcher snapshot...', directory); + const historical = await parcelWatcher.getEventsSince(directory, this.snapshotFilePath, { + backend: this.backend, + }); + handleEvents(historical, extensions); + } catch (err) { + console.warn('No snapshot available, skipping historical events.', err); + } + + this.watcher = await parcelWatcher.subscribe( + directory, + (err, events) => { + if (err) { + console.error('Error fired in watcher', directory, err); + ctx.postMessage({ type: 'error', value: err }); + } + handleEvents(events, extensions); }, - { ignore: [] }, + { ignore: [], backend: backend }, ); // Make a list of all files in this directory, which will be returned when all subdirs have been traversed diff --git a/src/ipc/renderer.ts b/src/ipc/renderer.ts index 4ae68db9..427faa77 100644 --- a/src/ipc/renderer.ts +++ b/src/ipc/renderer.ts @@ -194,6 +194,13 @@ export class RendererMessenger { return path.join(userDataPath, 'themes'); }; - static sendConsoleMessage = (type: 'log' | 'info' | 'error' | 'warn' | 'debug', message: string) => - ipcRenderer.send(CONSOLE_MESSAGE, { type, message }); + static getWatcherSnapshotsDirectory = async () => { + const userDataPath = await RendererMessenger.getPath('userData'); + return path.join(userDataPath, 'watcher_snapshots'); + }; + + static sendConsoleMessage = ( + type: 'log' | 'info' | 'error' | 'warn' | 'debug', + message: string, + ) => ipcRenderer.send(CONSOLE_MESSAGE, { type, message }); } From 92add69ba07017632dea34b9c16e69df45fc2809 Mon Sep 17 00:00:00 2001 From: RafaUC Date: Sun, 23 Nov 2025 23:27:34 -0600 Subject: [PATCH 2/4] =?UTF-8?q?-=C2=A0Remove=20initial=20scanning=20and=20?= =?UTF-8?q?file=20fetching=20during=20location=20initialization.=C2=A0Parc?= =?UTF-8?q?el/watcher=20is=20more=20efficient=20and=20does=20not=20return?= =?UTF-8?q?=20a=20full=20file=20list,=20so=20the=C2=A0disk=20files=20and?= =?UTF-8?q?=20DB=20files=C2=A0comparison=20caused=20missing-file=20bugs=20?= =?UTF-8?q?when=20the=C2=A0=20watcher=20returned=20an=20empty=20list.?= =?UTF-8?q?=C2=A0Plus=20now=20the=20app=20starts=20a=20lot=20quicker!?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Refactorize some related methods to avoid code duplication. - Improve the location state representation in the locations panel. Unwatched locations now appear muted, and the loading icon logic has been updated. --- resources/style/outliner.scss | 7 + .../Outliner/LocationsPanel/index.tsx | 1 + .../Outliner/TagsPanel/TagsTree.tsx | 4 +- src/frontend/entities/Location.ts | 17 +- src/frontend/stores/FileStore.ts | 10 +- src/frontend/stores/LocationStore.ts | 183 +++++++++++------- src/frontend/stores/RootStore.ts | 11 +- src/frontend/stores/TagStore.ts | 2 +- src/frontend/workers/folderWatcher.worker.ts | 20 +- 9 files changed, 142 insertions(+), 113 deletions(-) diff --git a/resources/style/outliner.scss b/resources/style/outliner.scss index c40be778..3e10d314 100644 --- a/resources/style/outliner.scss +++ b/resources/style/outliner.scss @@ -188,6 +188,13 @@ input { width: 100%; } + + &[data-watching='false'] { + SVG { + //color: var(--text-color-muted); + opacity: 0.63; + } + } } // ACTIONBAR diff --git a/src/frontend/containers/Outliner/LocationsPanel/index.tsx b/src/frontend/containers/Outliner/LocationsPanel/index.tsx index ee92589f..98ae3a4d 100644 --- a/src/frontend/containers/Outliner/LocationsPanel/index.tsx +++ b/src/frontend/containers/Outliner/LocationsPanel/index.tsx @@ -394,6 +394,7 @@ const Location = observer( return (
{ const { nodeData, dispatch, expansion, isEditing, submit, pos, select } = props; - const { uiStore } = useStore(); + const { uiStore, tagStore } = useStore(); const dndData = useTagDnD(); const show = useContextMenu(); @@ -327,7 +327,7 @@ const TagItem = observer((props: ITagItemProps) => { onSubmit={submit} tooltip={`${nodeData.path .map((v) => (v.startsWith('#') ? ' ' + v.slice(1) + ' ' : v)) - .join(' › ')} (${nodeData.fileCount})`} + .join(' › ')}${tagStore.fileCountsInitialized ? ` (${nodeData.fileCount})` : ''}`} /> {!isEditing && ( { + @action async watch(): Promise { if (this.isBroken) { console.error( 'Location watch error:', 'Cannot watch a location because it is broken or not initialized.', ); - return undefined; + return false; } this.setSettingWatcher(true); const directory = this.path; @@ -522,7 +522,7 @@ export class ClientLocation { this.store.watcherSnapshotDirectory, `${this.id}.snapshot.json`, ); - const initialFiles = await this.worker.watch( + await this.worker.watch( directory, this.extensions, snapshotFilePath, @@ -530,12 +530,7 @@ export class ClientLocation { ); this.setSettingWatcher(false); - // Filter out images from excluded sub-locations - // TODO: Could also put them in the chokidar ignore property - return initialFiles?.filter( - ({ absolutePath }) => - !this.excludedPaths.some((subLoc) => absolutePath.startsWith(subLoc.path)), - ); + return true; } // close and save snapshots of the watcher worker diff --git a/src/frontend/stores/FileStore.ts b/src/frontend/stores/FileStore.ts index fc2daa39..d1216db6 100644 --- a/src/frontend/stores/FileStore.ts +++ b/src/frontend/stores/FileStore.ts @@ -631,7 +631,7 @@ class FileStore { file.setBroken(true); this.rootStore.uiStore.deselectFile(file); this.incrementNumMissingFiles(); - if (file.tags.size === 0) { + if (file.tags.size === 0 && this.numUntaggedFiles > 0) { this.decrementNumUntaggedFiles(); } } @@ -731,6 +731,7 @@ class FileStore { } } + firstFetchAll = true; @action.bound async fetchAllFiles(): Promise { try { this.setContentAll(); @@ -748,10 +749,15 @@ class FileStore { // continue if the current taskId is the same else abort the fetch const currentFetchId = runInAction(() => this.fetchTaskIdPair[0]); if (start === currentFetchId) { - return this.updateFromBackend(fetchedFiles); + this.updateFromBackend(fetchedFiles); } else { console.debug('FETCH All ABORTED'); } + if (this.firstFetchAll) { + this.firstFetchAll = false; + this.rootStore.tagStore.initializeFileCounts(fetchedFiles); + this.rootStore.locationStore.updateLocations(undefined, fetchedFiles); + } } catch (err) { console.error('Could not load all files', err); } diff --git a/src/frontend/stores/LocationStore.ts b/src/frontend/stores/LocationStore.ts index 51e4451e..85c9aa88 100644 --- a/src/frontend/stores/LocationStore.ts +++ b/src/frontend/stores/LocationStore.ts @@ -114,62 +114,140 @@ class LocationStore { ), ); runInAction(() => this.locationList.replace(locations)); + runInAction(() => { + for (const location of this.locationList) { + location.init(); + } + }); } save(loc: LocationDTO): void { this.backend.saveLocation(loc); } - // chokidar.watch is significantly slower in windows. This negatively affects UX at - // Startup when dealing with locations containing a large number of files. - // Performing the initial scan for new or removed images directly with fse greatly - // improves the initial sync on Windows. - @action async updateLocations(locations?: ClientLocation | ClientLocation[]): Promise { + retryToastTimeout: NodeJS.Timeout | undefined; + private showLocationProcessToast( + action: 'show' | 'cancel-retry' | 'show-missing' | 'hide', + msgMode: 'update' | 'watch', + location: ClientLocation | { name: 'None'; id: 'None' } = { name: 'None', id: 'None' }, + total: number = 0, + progress: number = 0, + ) { + const watch = msgMode === 'watch'; + const progressToastKey = 'progress'; + switch (action) { + case 'show': + const toastsMsg = watch ? 'Syncing locations' : 'Looking for new images'; + + AppToaster.show( + { + message: `${toastsMsg}... [${progress + 1} / ${total}]`, + timeout: 6000, + }, + progressToastKey, + ); + + // TODO: Add a maximum timeout for init: sometimes it's hanging for me. Could also be some of the following steps though + // added a retry toast for now, can't figure out the cause, and it's hard to reproduce + // FIXME: Toasts should not be abused for error handling. Create some error messaging mechanism. + this.retryToastTimeout = setTimeout(() => { + AppToaster.show( + { + message: `${toastsMsg}... [${progress + 1} / ${total}]`, + timeout: 6000, + }, + progressToastKey, + ); + AppToaster.show( + { + message: 'This appears to be taking longer than usual.', + timeout: 10000, + clickAction: { + onClick: RendererMessenger.reload, + label: 'Retry?', + }, + }, + 'retry-init', + ); + }, 20000); + break; + case 'cancel-retry': + clearTimeout(this.retryToastTimeout); + AppToaster.dismiss('retry-init'); + break; + case 'show-missing': + AppToaster.show( + { + message: `Cannot ${watch ? 'watch' : 'find'} Location "${location.name}"`, + timeout: 6000, + }, + // a key such that the toast can be dismissed automatically on recovery + `missing-loc-${location.id}`, + ); + break; + case 'hide': + default: + AppToaster.dismiss(progressToastKey); + break; + } + } + + @action async watchLocations(locations?: ClientLocation | ClientLocation[]): Promise { let locs: ClientLocation[]; if (locations === undefined) { - locs = this.locationList.slice(); + locs = this.locationList; } else if (!Array.isArray(locations)) { locs = [locations]; } else { locs = locations; } - const foundNewFiles = await this.compareLocations(false, locs); - if (foundNewFiles) { - this.rootStore.fileStore.refetch(); + locs = locs.filter((l) => l.isWatchingFiles); + const len = locs.length; + for (let i = 0; i < len; i++) { + const location = locs[i]; + this.showLocationProcessToast('show', 'watch', location, len, i); + const success = await location.watch(); + this.showLocationProcessToast('cancel-retry', 'watch', location); + if (!success) { + this.showLocationProcessToast('show-missing', 'watch', location); + } } - return foundNewFiles; + this.showLocationProcessToast('hide', 'watch'); } - // We still need to initialize the watching and compare disk and db files like before, - // since changes may occur on disk during watcher initialization. - @action async watchLocations(locations?: ClientLocation | ClientLocation[]): Promise { + /** Manually synchronizes the database files and locations with the current file system state using a brute-force scan. */ + @action async updateLocations( + locations?: ClientLocation | ClientLocation[], + allDbFiles?: FileDTO[], + ): Promise { let locs: ClientLocation[]; if (locations === undefined) { - locs = this.locationList; + locs = this.locationList.slice(); } else if (!Array.isArray(locations)) { locs = [locations]; } else { locs = locations; } - locs = locs.filter((l) => l.isWatchingFiles); - - const foundNewFiles = await this.compareLocations(true, locs); + locs.forEach((loc) => (loc.isRefreshing = true)); + const foundNewFiles = await this.compareLocations(locs, allDbFiles); if (foundNewFiles) { this.rootStore.fileStore.refetch(); } return foundNewFiles; } - // E.g. in preview window, it's not needed to watch the locations // Returns whether files have been added, changed or removed - @action async compareLocations(watch = true, locations: ClientLocation[]): Promise { - const progressToastKey = 'progress'; + @action async compareLocations( + locations: ClientLocation[], + allDbFiles?: FileDTO[], + ): Promise { let foundNewFiles = false; const len = locations.length; // Get all files in the DB, set up data structures for quick lookups // Doing it for all locations, so files moved to another Location on disk, it's properly re-assigned in Allusion too - const dbFiles: FileDTO[] = await this.backend.fetchFiles('id', OrderDirection.Asc, false); + const dbFiles: FileDTO[] = + allDbFiles ?? (await this.backend.fetchFiles('id', OrderDirection.Asc, false)); // Taking advantage of the fact that we're doing a full fetch here, try to initialize file counts if they haven't been initialized yet. this.rootStore.tagStore.initializeFileCounts(dbFiles); const dbFilesPathSet = new Set(dbFiles.map((f) => f.absolutePath)); @@ -184,74 +262,31 @@ class LocationStore { } } - const toastsMsg = watch ? 'Syncing locations' : 'Looking for new images'; - // For every location, find created/moved/deleted files, and update the database accordingly. // TODO: Do this in a web worker, not in the renderer thread! for (let i = 0; i < len; i++) { const location = locations[i]; - AppToaster.show( - { - message: `${toastsMsg}... [${i + 1} / ${len}]`, - timeout: 6000, - }, - progressToastKey, - ); - - // TODO: Add a maximum timeout for init: sometimes it's hanging for me. Could also be some of the following steps though - // added a retry toast for now, can't figure out the cause, and it's hard to reproduce - // FIXME: Toasts should not be abused for error handling. Create some error messaging mechanism. - const readyTimeout = setTimeout(() => { - AppToaster.show( - { - message: `${toastsMsg}... [${i + 1} / ${len}]`, - timeout: 6000, - }, - progressToastKey, - ); - AppToaster.show( - { - message: 'This appears to be taking longer than usual.', - timeout: 10000, - clickAction: { - onClick: RendererMessenger.reload, - label: 'Retry?', - }, - }, - 'retry-init', - ); - }, 20000); + this.showLocationProcessToast('show', 'update', location, len, i); const wasInitialized = runInAction(() => location.isInitialized); if (!wasInitialized) { console.group(`Initializing location ${location.name}`); await location.init(); } - const diskFiles = - watch && runInAction(() => location.isWatchingFiles) - ? await location.watch() - : await (async () => { - const [files, rootDirectoryItem] = await location.getDiskFilesAndDirectories(); - location.refreshSublocations(rootDirectoryItem); - return files; - })(); + const diskFiles = await (async () => { + const [files, rootDirectoryItem] = await location.getDiskFilesAndDirectories(); + location.refreshSublocations(rootDirectoryItem); + return files; + })(); const diskFileMap = new Map( diskFiles?.map((f) => [f.absolutePath, f]) ?? [], ); - clearTimeout(readyTimeout); - AppToaster.dismiss('retry-init'); + this.showLocationProcessToast('cancel-retry', 'update', location); if (diskFiles === undefined) { - AppToaster.show( - { - message: `Cannot ${watch ? 'watch' : 'find'} Location "${location.name}"`, - timeout: 6000, - }, - // a key such that the toast can be dismissed automatically on recovery - `missing-loc-${location.id}`, - ); + this.showLocationProcessToast('show-missing', 'update', location); continue; } @@ -374,9 +409,9 @@ class LocationStore { } if (foundNewFiles) { - AppToaster.show({ message: 'New images detected.', timeout: 5000 }, progressToastKey); + AppToaster.show({ message: 'New images detected.', timeout: 5000 }, 'new-images'); } else { - AppToaster.dismiss(progressToastKey); + this.showLocationProcessToast('hide', 'update'); } return foundNewFiles; } diff --git a/src/frontend/stores/RootStore.ts b/src/frontend/stores/RootStore.ts index 453184ea..cafe41e5 100644 --- a/src/frontend/stores/RootStore.ts +++ b/src/frontend/stores/RootStore.ts @@ -13,6 +13,7 @@ import ImageLoader from '../image/ImageLoader'; import { RendererMessenger } from 'src/ipc/renderer'; import SearchStore from './SearchStore'; import ExtraPropertyStore from './ExtraPropertyStore'; +import { AppToaster } from '../components/Toaster'; // This will throw exceptions whenever we try to modify the state directly without an action // Actions will batch state modifications -> better for performance @@ -123,11 +124,8 @@ class RootStore { ); } - // Quick look for any new or removed images, and refetch if necessary - rootStore.locationStore.updateLocations().then(() => { - // Then, watch the locations - rootStore.locationStore.watchLocations(); - }); + // look for any new or removed images, handled by parcel/watcher + rootStore.locationStore.watchLocations(); return rootStore; } @@ -188,9 +186,12 @@ class RootStore { } async close(): Promise { + AppToaster.show({ message: 'Closing Allusion...', type: 'info', timeout: 0 }, 'closing'); await this.locationStore.close(); // TODO: should be able to be done more reliably by running exiftool as a child process await this.exifTool.close(); + AppToaster.show({ message: 'Closing Allusion...', type: 'success', timeout: 0 }, 'closing'); + await new Promise((resolve) => setTimeout(resolve, 200)); } } diff --git a/src/frontend/stores/TagStore.ts b/src/frontend/stores/TagStore.ts index 8b21ffe8..5e0ae946 100644 --- a/src/frontend/stores/TagStore.ts +++ b/src/frontend/stores/TagStore.ts @@ -19,6 +19,7 @@ class TagStore { /** A lookup map to speedup finding entities */ private readonly tagGraph = observable(new Map()); + @observable fileCountsInitialized = false; constructor(backend: DataStorage, rootStore: RootStore) { this.backend = backend; @@ -37,7 +38,6 @@ class TagStore { } } - fileCountsInitialized = false; @action.bound async initializeFileCounts(files: FileDTO[]): Promise { if (this.fileCountsInitialized) { return; diff --git a/src/frontend/workers/folderWatcher.worker.ts b/src/frontend/workers/folderWatcher.worker.ts index e565a4f7..17ebc2ce 100644 --- a/src/frontend/workers/folderWatcher.worker.ts +++ b/src/frontend/workers/folderWatcher.worker.ts @@ -9,8 +9,6 @@ const ctx: Worker = self as any; export class FolderWatcherWorker { private watcher?: parcelWatcher.AsyncSubscription; - // Whether the initial scan has been completed, and new/removed files are being watched - private isReady = false; private isCancelled = false; private directory?: string; private snapshotFilePath?: string; @@ -44,7 +42,7 @@ export class FolderWatcherWorker { extensions: IMG_EXTENSIONS_TYPE[], snapshotFilePath: string, backend: parcelWatcher.BackendType, - ) { + ): Promise { this.isCancelled = false; this.backend = backend; @@ -58,7 +56,6 @@ export class FolderWatcherWorker { // Watch for files being added/changed/removed: // Usually you'd include a glob in the watch argument, e.g. `directory/**/.{jpg|png|...}`, but we cannot use globs unfortunately (see disableGlobbing) // watch for this https://github.com/parcel-bundler/watcher/pull/207 - this.isReady = true; const handleEvents = // Small indentation hack to avoid affecting git blame (events: parcelWatcher.Event[], extensions: IMG_EXTENSIONS_TYPE[]) => { @@ -93,11 +90,7 @@ export class FolderWatcherWorker { ino: stats.ino.toString(), }; - if (this.isReady) { - ctx.postMessage({ type: 'add', value: fileStats }); - } else { - initialFiles.push(fileStats); - } + ctx.postMessage({ type: 'add', value: fileStats }); } else if (event.type === 'update') { const stats = statSync(event.path); if (this.isCancelled) { @@ -144,15 +137,6 @@ export class FolderWatcherWorker { }, { ignore: [], backend: backend }, ); - - // Make a list of all files in this directory, which will be returned when all subdirs have been traversed - const initialFiles: FileStats[] = []; - - // This is stubbed out as @parcel/watcher doesn't have a ready event like chokidar - // Because @parcel/watcher has the ability to have snapshots and historical changes, we can use it to reduce startup time - return new Promise((resolve) => { - resolve([]); - }); } } From 1298f955551355d2a7e41df333f24f0715e29253 Mon Sep 17 00:00:00 2001 From: RafaUC Date: Mon, 24 Nov 2025 20:03:22 -0600 Subject: [PATCH 3/4] - Add button to manually load file counts. --- src/frontend/containers/Outliner/TagsPanel/TagsTree.tsx | 8 ++++++++ src/frontend/stores/LocationStore.ts | 2 +- src/frontend/stores/TagStore.ts | 6 ++++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/frontend/containers/Outliner/TagsPanel/TagsTree.tsx b/src/frontend/containers/Outliner/TagsPanel/TagsTree.tsx index 4f7c227b..478aebb0 100644 --- a/src/frontend/containers/Outliner/TagsPanel/TagsTree.tsx +++ b/src/frontend/containers/Outliner/TagsPanel/TagsTree.tsx @@ -800,6 +800,14 @@ const TagsTree = observer((props: Partial) => { }} headerToolbar={ + {!tagStore.fileCountsInitialized && ( + tagStore.initializeFileCounts()} + tooltip={'Load Tag File Counts'} + /> + )} {uiStore.tagSelection.size > 0 ? ( { - for (const location of this.locationList) { + for (const location of this.locationList.slice()) { await location.close(); } } diff --git a/src/frontend/stores/TagStore.ts b/src/frontend/stores/TagStore.ts index 5e0ae946..2a65e6b8 100644 --- a/src/frontend/stores/TagStore.ts +++ b/src/frontend/stores/TagStore.ts @@ -9,6 +9,7 @@ import RootStore from './RootStore'; import { AppToaster, IToastProps } from '../components/Toaster'; import { FileDTO } from 'src/api/file'; import { normalizeBase } from 'common/core'; +import { OrderDirection } from 'src/api/data-storage-search'; /** * Based on https://mobx.js.org/best/store.html @@ -38,17 +39,18 @@ class TagStore { } } - @action.bound async initializeFileCounts(files: FileDTO[]): Promise { + @action.bound async initializeFileCounts(allDbFiles?: FileDTO[]): Promise { if (this.fileCountsInitialized) { return; } + this.fileCountsInitialized = true; + const files = allDbFiles ?? (await this.backend.fetchFiles('id', OrderDirection.Asc, false)); for (const file of files) { for (const tagID of file.tags) { const tag = this.get(tagID); tag?.incrementFileCount(file.id); } } - this.fileCountsInitialized = true; } @action get(tag: ID): ClientTag | undefined { From efe9c3d8ee9d27729c63608f62eb9b3b1674c0d0 Mon Sep 17 00:00:00 2001 From: RafaUC Date: Tue, 25 Nov 2025 01:31:10 -0600 Subject: [PATCH 4/4] - Feature: Configure whether to refresh non auto-synced locations and load tag file counts at startup, enabling users to optimize startup performance. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add “Load Tag File Counts” and “Refresh Non Auto-Synced Locations and Detect File Changes” toggles to the startup behavior settings. --- .../Masonry/VirtualizedRenderer.tsx | 4 +-- .../Outliner/TagsPanel/TagsTree.tsx | 2 +- .../containers/Settings/StartupBehavior.tsx | 13 ++++++++++ src/frontend/stores/FileStore.ts | 11 +++----- src/frontend/stores/LocationStore.ts | 2 -- src/frontend/stores/RootStore.ts | 26 +++++++++++++++++-- src/frontend/stores/TagStore.ts | 2 +- src/frontend/stores/UiStore.ts | 21 ++++++++++++++- 8 files changed, 65 insertions(+), 16 deletions(-) diff --git a/src/frontend/containers/ContentView/Masonry/VirtualizedRenderer.tsx b/src/frontend/containers/ContentView/Masonry/VirtualizedRenderer.tsx index 5a60a22d..b8969889 100644 --- a/src/frontend/containers/ContentView/Masonry/VirtualizedRenderer.tsx +++ b/src/frontend/containers/ContentView/Masonry/VirtualizedRenderer.tsx @@ -42,7 +42,7 @@ const VirtualizedRenderer = observer( padding, }: IRendererProps) => { const { uiStore, fileStore } = useStore(); - const [, isMountedRef] = useMountState(); + const [isMounted, isMountedRef] = useMountState(); const wrapperRef = useRef(null); const scrollAnchor = useRef(null); const [startRenderIndex, setStartRenderIndex] = useState(0); @@ -149,7 +149,7 @@ const VirtualizedRenderer = observer( // Call throttledRedetermine in case no scroll has been applied. throttledRedetermine.current(numImages, overscan, false); // eslint-disable-next-line react-hooks/exhaustive-deps - }, [layoutUpdateDate]); + }, [layoutUpdateDate, isMounted]); // When selection changes, scroll to last selected image. Nice when using cursor keys for navigation const fileSelectionSize = uiStore.fileSelection.size; diff --git a/src/frontend/containers/Outliner/TagsPanel/TagsTree.tsx b/src/frontend/containers/Outliner/TagsPanel/TagsTree.tsx index 478aebb0..8fb240ae 100644 --- a/src/frontend/containers/Outliner/TagsPanel/TagsTree.tsx +++ b/src/frontend/containers/Outliner/TagsPanel/TagsTree.tsx @@ -804,7 +804,7 @@ const TagsTree = observer((props: Partial) => { tagStore.initializeFileCounts()} + onClick={() => tagStore.initializeTagFileCounts()} tooltip={'Load Tag File Counts'} /> )} diff --git a/src/frontend/containers/Settings/StartupBehavior.tsx b/src/frontend/containers/Settings/StartupBehavior.tsx index 35891fbc..74ab680a 100644 --- a/src/frontend/containers/Settings/StartupBehavior.tsx +++ b/src/frontend/containers/Settings/StartupBehavior.tsx @@ -24,6 +24,19 @@ export const StartupBehavior = observer(() => { > Restore and query last submitted search query + + Load Tag File Counts + + + Refresh Non Auto-Synced Locations and Detect File Changes + +
Check for updates diff --git a/src/frontend/stores/FileStore.ts b/src/frontend/stores/FileStore.ts index d1216db6..b9bd89bb 100644 --- a/src/frontend/stores/FileStore.ts +++ b/src/frontend/stores/FileStore.ts @@ -731,7 +731,6 @@ class FileStore { } } - firstFetchAll = true; @action.bound async fetchAllFiles(): Promise { try { this.setContentAll(); @@ -748,16 +747,14 @@ class FileStore { this.setAverageFetchTime(end - start); // continue if the current taskId is the same else abort the fetch const currentFetchId = runInAction(() => this.fetchTaskIdPair[0]); + let promise = undefined; if (start === currentFetchId) { - this.updateFromBackend(fetchedFiles); + promise = this.updateFromBackend(fetchedFiles); } else { console.debug('FETCH All ABORTED'); } - if (this.firstFetchAll) { - this.firstFetchAll = false; - this.rootStore.tagStore.initializeFileCounts(fetchedFiles); - this.rootStore.locationStore.updateLocations(undefined, fetchedFiles); - } + this.rootStore.initStartupLoads(fetchedFiles); + return promise; } catch (err) { console.error('Could not load all files', err); } diff --git a/src/frontend/stores/LocationStore.ts b/src/frontend/stores/LocationStore.ts index 4aa7a691..10e1e68f 100644 --- a/src/frontend/stores/LocationStore.ts +++ b/src/frontend/stores/LocationStore.ts @@ -248,8 +248,6 @@ class LocationStore { // Doing it for all locations, so files moved to another Location on disk, it's properly re-assigned in Allusion too const dbFiles: FileDTO[] = allDbFiles ?? (await this.backend.fetchFiles('id', OrderDirection.Asc, false)); - // Taking advantage of the fact that we're doing a full fetch here, try to initialize file counts if they haven't been initialized yet. - this.rootStore.tagStore.initializeFileCounts(dbFiles); const dbFilesPathSet = new Set(dbFiles.map((f) => f.absolutePath)); const dbFilesByCreatedDate = new Map(); for (const file of dbFiles) { diff --git a/src/frontend/stores/RootStore.ts b/src/frontend/stores/RootStore.ts index cafe41e5..cc5c124c 100644 --- a/src/frontend/stores/RootStore.ts +++ b/src/frontend/stores/RootStore.ts @@ -14,6 +14,8 @@ import { RendererMessenger } from 'src/ipc/renderer'; import SearchStore from './SearchStore'; import ExtraPropertyStore from './ExtraPropertyStore'; import { AppToaster } from '../components/Toaster'; +import { FileDTO } from 'src/api/file'; +import { OrderDirection } from 'src/api/data-storage-search'; // This will throw exceptions whenever we try to modify the state directly without an action // Actions will batch state modifications -> better for performance @@ -100,10 +102,11 @@ class RootStore { numCriterias === 0 ? rootStore.fileStore.fetchAllFiles : async () => { - // When searching by criteria, the file counts won't be set (only when fetching all files), + // When searching by criteria, the file counts and startup Loads won't be set (only when fetching all files), // so fetch them manually await rootStore.fileStore.refetchFileCounts().catch(console.error); - return rootStore.fileStore.fetchFilesByQuery(); + await rootStore.fileStore.fetchFilesByQuery(); + return rootStore.initStartupLoads().catch(console.error); }; // Load the files already in the database so user instantly sees their images @@ -166,6 +169,25 @@ class RootStore { return rootStore; } + isStartupLoadsInitialized = false; + async initStartupLoads(allDbFiles?: FileDTO[]): Promise { + if (this.isStartupLoadsInitialized) { + return; + } + this.isStartupLoadsInitialized = true; + runInAction(async () => { + const doInitFileCounts = this.uiStore.isLoadFileCountsStartupEnabled; + const doRefreshLocations = this.uiStore.isRefreshLocationsStartupEnabled; + const files = allDbFiles ?? (await this.#backend.fetchFiles('id', OrderDirection.Asc, false)); + if (doInitFileCounts) { + await this.tagStore.initializeTagFileCounts(files); + } + if (doRefreshLocations) { + await this.locationStore.updateLocations(undefined, files); + } + }); + } + async backupDatabaseToFile(path: string): Promise { return this.#backup.backupToFile(path); } diff --git a/src/frontend/stores/TagStore.ts b/src/frontend/stores/TagStore.ts index 2a65e6b8..57be8bf6 100644 --- a/src/frontend/stores/TagStore.ts +++ b/src/frontend/stores/TagStore.ts @@ -39,7 +39,7 @@ class TagStore { } } - @action.bound async initializeFileCounts(allDbFiles?: FileDTO[]): Promise { + @action.bound async initializeTagFileCounts(allDbFiles?: FileDTO[]): Promise { if (this.fileCountsInitialized) { return; } diff --git a/src/frontend/stores/UiStore.ts b/src/frontend/stores/UiStore.ts index cd70cdcd..aed54aea 100644 --- a/src/frontend/stores/UiStore.ts +++ b/src/frontend/stores/UiStore.ts @@ -156,10 +156,13 @@ type PersistentPreferenceFields = | 'outlinerExpansion' | 'outlinerHeights' | 'inspectorWidth' - | 'isRememberSearchEnabled' | 'recentlyUsedTagsMaxLength' | 'recentlyUsedTags' | 'isClearTagSelectorsOnSelectEnabled' + // startup options + | 'isLoadFileCountsStartupEnabled' + | 'isRefreshLocationsStartupEnabled' + | 'isRememberSearchEnabled' // the following are only restored when isRememberSearchEnabled is enabled | 'isSlideMode' | 'firstItem' @@ -202,6 +205,10 @@ class UiStore { 'visible-when-inherited'; @observable isThumbnailFilenameOverlayEnabled: boolean = false; @observable isThumbnailResolutionOverlayEnabled: boolean = false; + /** Load File Counts at startup */ + @observable isLoadFileCountsStartupEnabled: boolean = true; + /** Refresh locations and detect file changes at startup */ + @observable isRefreshLocationsStartupEnabled: boolean = false; /** Whether to restore the last search query on start-up */ @observable isRememberSearchEnabled: boolean = true; /** Index of the first item in the viewport. Also acts as the current item shown in slide mode */ @@ -468,6 +475,14 @@ class UiStore { this.largeThumbFullResThreshold = value; } + @action.bound toggleRefreshLocationStartup(): void { + this.isRefreshLocationsStartupEnabled = !this.isRefreshLocationsStartupEnabled; + } + + @action.bound toggleLoadFileCountsStartup(): void { + this.isLoadFileCountsStartupEnabled = !this.isLoadFileCountsStartupEnabled; + } + @action.bound toggleRememberSearchQuery(): void { this.isRememberSearchEnabled = !this.isRememberSearchEnabled; } @@ -1418,6 +1433,8 @@ class UiStore { ([k, v]) => k in defaultHotkeyMap && (this.hotkeyMap[k as keyof IHotkeyMap] = v), ); + this.isLoadFileCountsStartupEnabled = Boolean(prefs.isLoadFileCountsStartupEnabled ?? true); + this.isRefreshLocationsStartupEnabled = Boolean(prefs.isRefreshLocationsStartupEnabled ?? false); // eslint-disable-line prettier/prettier this.isRememberSearchEnabled = Boolean(prefs.isRememberSearchEnabled); if (this.isRememberSearchEnabled) { // If remember search criteria, restore the search criteria list... @@ -1484,6 +1501,8 @@ class UiStore { outlinerHeights: this.outlinerHeights.slice(), outlinerWidth: this.outlinerWidth, inspectorWidth: this.inspectorWidth, + isLoadFileCountsStartupEnabled: this.isLoadFileCountsStartupEnabled, + isRefreshLocationsStartupEnabled: this.isRefreshLocationsStartupEnabled, isRememberSearchEnabled: this.isRememberSearchEnabled, isSlideMode: this.isSlideMode, firstItem: this.firstItem,