Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 13 additions & 4 deletions package.json
Original file line number Diff line number Diff line change
@@ -1,9 +1,7 @@
{
"name": "allusion",
"productName": "Allusion",

"version": "1.0.0-ruc.8.0",

"description": "A tool for managing your visual library",
"main": "build/main.bundle.js",
"scripts": {
Expand Down Expand Up @@ -68,6 +66,17 @@
},
"files": [
"!node_modules",
"node_modules/@parcel/**/*",
"node_modules/micromatch/**/*",
"node_modules/braces/**/*",
"node_modules/fill-range/**/*",
"node_modules/to-regex-range/**/*",
"node_modules/is-number/**/*",
"node_modules/is-glob/**/*",
"node_modules/detect-libc/**/*",
"node_modules/picomatch/**/*",
"node_modules/node-addon-api/**/*",
"node_modules/is-extglob/**/*",
"build/**/*",
"package.json"
],
Expand Down Expand Up @@ -124,8 +133,8 @@
"dependencies": {
"@floating-ui/core": "^1.2.1",
"@floating-ui/react-dom": "^1.3.0",
"@parcel/watcher": "^2.5.1",
"ag-psd": "^15.0.0",
"chokidar": "^3.5.3",
"comlink": "^4.4.1",
"dexie": "^3.2.3",
"dexie-export-import": "^1.0.3",
Expand All @@ -144,4 +153,4 @@
"utif": "^3.1.0",
"wasm-feature-detect": "^1.2.11"
}
}
}
159 changes: 59 additions & 100 deletions src/frontend/workers/folderWatcher.worker.ts
Original file line number Diff line number Diff line change
@@ -1,16 +1,14 @@
import chokidar, { FSWatcher } from 'chokidar';
import { expose } from 'comlink';
import { Stats } from 'fs';
import { BigIntStats } from 'original-fs';
import { statSync } from 'fs';
import SysPath from 'path';
import { RECURSIVE_DIR_WATCH_DEPTH } from 'common/config';
import { IMG_EXTENSIONS_TYPE } from 'src/api/file';
import { FileStats } from '../stores/LocationStore';
import * as parcelWatcher from '@parcel/watcher';

const ctx: Worker = self as any;

export class FolderWatcherWorker {
private watcher?: FSWatcher;
private watcher?: parcelWatcher.AsyncSubscription;
// Whether the initial scan has been completed, and new/removed files are being watched
private isReady = false;
private isCancelled = false;
Expand All @@ -20,7 +18,7 @@ export class FolderWatcherWorker {
}

async close() {
await this.watcher?.close();
this.watcher?.unsubscribe();
}

/** Returns all supported image files in the given directly, and callbacks for new or removed files */
Expand All @@ -33,68 +31,31 @@ export class FolderWatcherWorker {

// Watch for files being added/changed/removed:
// Usually you'd include a glob in the watch argument, e.g. `directory/**/.{jpg|png|...}`, but we cannot use globs unfortunately (see disableGlobbing)
// Instead, we ignore everything but image files in the `ignored` option
this.watcher = chokidar.watch(directory, {
disableGlobbing: true, // needed in order to support directories with brackets, quotes, asterisks, etc.
alwaysStat: true, // we need stats anyways during importing
depth: RECURSIVE_DIR_WATCH_DEPTH, // not really needed: added as a safety measure for infinite recursion between symbolic links
ignored: (path: string, stats?: Stats) => {
// We used to set `ignored` with regex patterns, but ran into problem with directories that
// contain dots in their file name.
// We ignore everything except image files but chokidar also matches entire directories. If
// those contain a dot, they will be ignored since they don't end with an image extension.
// So now we have to use a callback function that also provides `stats` through which we can
// use to detect whether the path is a file or a directory.

const basename = SysPath.basename(path);

// Ignore .dot files and folders.
if (basename.startsWith('.')) {
return true;
}
// If the path doesn't have an extension (likely a directory), don't ignore it.
// In the unlikely situation it is a file, we'll filter it out later in the .on('add', ...)
const ext = SysPath.extname(path).toLowerCase().split('.')[1];
if (!ext) {
return false;
}
// If the path (file or directory) ends with an image extension, don't ignore it.
if (extensions.includes(ext as IMG_EXTENSIONS_TYPE)) {
return false;
}
// Otherwise, we need to know whether it is a file or a directory before making a decision.
// If we don't return anything, this callback will be called a second time, with the stats
// variable as second argument
if (stats) {
// Ignore if
// * dot directory like `/home/.hidden-directory/` but not `/home/directory.with.dots/` and
// * not a directory, and not an image file either.
return !stats.isDirectory() || SysPath.basename(path).startsWith('.');
}
return false;
},
});

const watcher = this.watcher;

// Make a list of all files in this directory, which will be returned when all subdirs have been traversed
const initialFiles: FileStats[] = [];

return new Promise<FileStats[] | undefined>((resolve) => {
watcher
// we can assume stats exist since we passed alwaysStat: true to chokidar
.on('add', async (path, stats: Stats | BigIntStats) => {
if (this.isCancelled) {
console.log('Cancelling file watching');
await watcher.close();
resolve(undefined);
this.isCancelled = false;
// watch for this https://github.com/parcel-bundler/watcher/pull/207
this.isReady = true;

this.watcher = await parcelWatcher.subscribe(
directory,
(err, events) => {
for (const event of events) {
if (err) {
console.error('Error fired in watcher', directory, err);
ctx.postMessage({ type: 'error', value: err });
}

const ext = SysPath.extname(path).toLowerCase().split('.')[1];
if (extensions.includes(ext as IMG_EXTENSIONS_TYPE)) {
// Ignore Files that aren't our extension type
const ext = SysPath.extname(event.path).toLowerCase().split('.')[1];
if (!extensions.includes(ext as IMG_EXTENSIONS_TYPE)) {
continue;
}
if (event.type === 'create') {
const stats = statSync(event.path);
if (this.isCancelled) {
console.log('Cancelling file watching');
this.watcher?.unsubscribe();
this.isCancelled = false;
}
/**
* Chokidar doesn't detect renames as a unique event, it detects a "remove" and "add" event.
* Chokidar and @parcel/watcher doesn't detect renames as a unique event, it detects a "remove" and "add" event.
* We use the "ino" field of file stats to detect whether a new file is a previously detected file that was moved/renamed
* Relevant issue https://github.com/paulmillr/chokidar/issues/303#issuecomment-127039892
* Inspiration for using "ino" from https://github.com/chrismaltby/gb-studio/pull/576
Expand All @@ -104,7 +65,7 @@ export class FolderWatcherWorker {
*/

const fileStats: FileStats = {
absolutePath: path,
absolutePath: event.path,
dateCreated: stats.birthtime,
dateModified: stats.mtime,
size: Number(stats.size),
Expand All @@ -116,41 +77,39 @@ export class FolderWatcherWorker {
} else {
initialFiles.push(fileStats);
}
} else if (event.type === 'update') {
const stats = statSync(event.path);
if (this.isCancelled) {
console.log('Cancelling file watching');
this.watcher?.unsubscribe();
this.isCancelled = false;
}
const ext = SysPath.extname(event.path).toLowerCase().split('.')[1];
if (extensions.includes(ext as IMG_EXTENSIONS_TYPE)) {
const fileStats: FileStats = {
absolutePath: event.path,
dateCreated: stats.birthtime,
dateModified: stats.mtime,
size: Number(stats.size),
ino: stats.ino.toString(),
};
ctx.postMessage({ type: 'update', value: fileStats });
}
} else if (event.type === 'delete') {
ctx.postMessage({ type: 'remove', value: event.path });
}
})
.on('change', async (path, stats: Stats | BigIntStats) => {
if (this.isCancelled) {
console.log('Cancelling file watching');
await watcher.close();
this.isCancelled = false;
}
const ext = SysPath.extname(path).toLowerCase().split('.')[1];
if (extensions.includes(ext as IMG_EXTENSIONS_TYPE)) {
const fileStats: FileStats = {
absolutePath: path,
dateCreated: stats.birthtime,
dateModified: stats.mtime,
size: Number(stats.size),
ino: stats.ino.toString(),
};
ctx.postMessage({ type: 'update', value: fileStats });
}
})
// TODO: on directory change: update location hierarchy list
.on('unlink', (path: string) => ctx.postMessage({ type: 'remove', value: path }))
.on('ready', () => {
this.isReady = true;
resolve(initialFiles);
}
},
{ ignore: [] },
);

// Make a list of all files in this directory, which will be returned when all subdirs have been traversed
const initialFiles: FileStats[] = [];

// Clear memory: initialFiles no longer needed
// Doing this immediately after resolving will resolve with an empty list for some reason
// So, do it with a timeout. Would be nicer to do it after an acknowledgement from the main thread
setTimeout(() => initialFiles.splice(0, initialFiles.length), 5000);
})
.on('error', (error) => {
console.error('Error fired in watcher', directory, error);
ctx.postMessage({ type: 'error', value: error });
});
// This is stubbed out as @parcel/watcher doesn't have a ready event like chokidar
// Because @parcel/watcher has the ability to have snapshots and historical changes, we can use it to reduce startup time
return new Promise<FileStats[]>((resolve) => {
resolve([]);
});
}
}
Expand Down
6 changes: 4 additions & 2 deletions webpack.dev.js
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,8 @@ let mainConfig = {
],
},
externals: {
fsevents: "require('fsevents')"
fsevents: "require('fsevents')",
"@parcel/watcher": "require('@parcel/watcher')"
}
};

Expand Down Expand Up @@ -123,7 +124,8 @@ let rendererConfig = {
}),
],
externals: {
fsevents: "require('fsevents')"
fsevents: "require('fsevents')",
"@parcel/watcher": "require('@parcel/watcher')"
}
};

Expand Down
6 changes: 6 additions & 0 deletions webpack.prod.js
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,9 @@ let mainConfig = {
},
],
},
externals: {
"@parcel/watcher": "require('@parcel/watcher')"
}
};

let rendererConfig = {
Expand Down Expand Up @@ -132,6 +135,9 @@ let rendererConfig = {
chunkFilename: '[id].[contenthash].css',
}),
],
externals: {
"@parcel/watcher": "require('@parcel/watcher')"
}
};

module.exports = [mainConfig, rendererConfig];
Loading
Loading