Migration/db indexeddb to sqlite

.github/workflows/release.yml

@@ -22,7 +22,7 @@ jobs:
       - name: Install Node.js, NPM and Yarn
         uses: actions/setup-node@v1
         with:
-          node-version: 16
+          node-version: 22
 
       - name: Build/release Electron app
         uses: samuelmeuli/action-electron-builder@v1

.github/workflows/test.yml

@@ -23,7 +23,7 @@ jobs:
       - name: Install Node.js, NPM and Yarn
         uses: actions/setup-node@v1
         with:
-          node-version: 16
+          node-version: 22
 
       - name: Get yarn cache directory path
         id: yarn-cache-dir-path
@@ -57,7 +57,7 @@ jobs:
       - name: Install Node.js, NPM and Yarn
         uses: actions/setup-node@v1
         with:
-          node-version: 16
+          node-version: 22
           github_token: ${{ secrets.github_token }}
 
       - name: Get yarn cache directory path

common/core.ts

@@ -70,3 +70,20 @@ export function normalizeBase(str: string): string {
     .replace(/[\u0300-\u036f]/g, '')
     .toLowerCase();
 }
+
+/** Returns the date at 00:00 today */
+export function getToday(): Date {
+  const today = new Date();
+  today.setHours(0);
+  today.setMinutes(0);
+  today.setSeconds(0, 0);
+  return today;
+}
+
+/** Returns the date at the start of the current week (Sunday at 00:00) */
+export function getWeekStart(): Date {
+  const date = getToday();
+  const dayOfWeek = date.getDay();
+  date.setDate(date.getDate() - dayOfWeek);
+  return date;
+}

src/frontend/TimingManager.ts → common/debug.ts

@@ -66,3 +66,25 @@ export class TimingManager {
     this.timings.clear();
   }
 }
+
+export const debugShallowCompare = <T extends Record<string, any>>(
+  componentName = 'MemoComponent',
+) => {
+  return (prev: T, next: T): boolean => {
+    let areEqual = true;
+
+    const keys = new Set([...Object.keys(prev), ...Object.keys(next)]);
+
+    for (const key of keys) {
+      if (!Object.is(prev[key], next[key])) {
+        areEqual = false;
+        console.log(`[${componentName}] prop changed → "${key}"`, {
+          from: prev[key],
+          to: next[key],
+        });
+      }
+    }
+
+    return areEqual;
+  };
+};

common/promise.ts

@@ -87,3 +87,52 @@ export function promiseAllLimit<T>(
 
   return pendingPromise;
 }
+
+export type BatchFetcher<T, NextOpts> = (opts?: NextOpts) => Promise<{
+  items: T[];
+  nextOpts?: NextOpts;
+}>;
+
+export type BatchProcessor<T, Acc> = (batch: T[], acc: Acc) => Promise<Acc> | Acc;
+
+/**
+ * Iterates over paginated resources and applies a processing function to each batch.
+ * @template T - The type of items retrieved.
+ * @template NextOpts - The type of the cursor for pagination or any options to compute the next batch.
+ * @template Acc - The type of the result accumulator.
+ * @param fetchBatch - Function to retrieve the next batch of data.
+ * @param processBatch - Function to apply logic to the current batch and update the accumulator.
+ * @param initialAcc - The starting value for the accumulation.
+ * @param cancel - Optional callback to abort the process prematurely.
+ * @returns A promise that resolves to the final accumulated value.
+ */
+export async function batchReducer<T, NextOpts, Acc>(
+  fetchBatch: BatchFetcher<T, NextOpts>,
+  processBatch: BatchProcessor<T, Acc>,
+  initialAcc: Acc,
+  cancel?: () => boolean,
+): Promise<Acc> {
+  let opts: NextOpts | undefined;
+  let acc = initialAcc;
+
+  while (true) {
+    if (cancel?.()) {
+      console.log('CANCELLING!');
+      break;
+    }
+    const { items, nextOpts } = await fetchBatch(opts);
+
+    if (!items.length) {
+      break;
+    }
+
+    acc = await processBatch(items, acc);
+
+    if (!nextOpts) {
+      break;
+    }
+    opts = nextOpts;
+  }
+
+  return acc;
+}

package.json

@@ -13,6 +13,7 @@
     "package": "yarn build && electron-builder",
     "logo": "ncp ./resources/logo/icns/allusion-logomark-fc.icns ./build/icon.icns && ncp ./resources/logo/ico/allusion-logomark-fc-256x256.ico ./build/icon.ico",
     "build": "rimraf dist && yarn production && yarn logo",
+    "postinstall": "electron-rebuild -f -w better-sqlite3",
     "pack": "electron-builder --dir",
     "dist": "electron-builder",
     "build:masonry": "cd wasm/wasm-build && cargo run masonry masonry/masonry-scalar && cargo run masonry masonry/masonry-simd -- -C target-feature=+simd128",
@@ -77,6 +78,11 @@
       "node_modules/picomatch/**/*",
       "node_modules/node-addon-api/**/*",
       "node_modules/is-extglob/**/*",
+
+      "node_modules/better-sqlite3/**/*",
+      "node_modules/bindings/**/*",
+      "node_modules/file-uri-to-path/**/*",
+
       "build/**/*",
       "package.json"
     ],
@@ -94,6 +100,7 @@
   "homepage": "https://github.com/RafaUC/Allusion/Allusion#readme",
   "devDependencies": {
     "@svgr/webpack": "^6.5.1",
+    "@types/better-sqlite3": "^7.6.13",
     "@types/chrome": "^0.0.195",
     "@types/fs-extra": "^11.0.1",
     "@types/jest": "^29.5.1",
@@ -107,6 +114,7 @@
     "css-loader": "^6.7.3",
     "electron": "21.3.0",
     "electron-builder": "23.6.0",
+    "electron-rebuild": "^3.2.9",
     "eslint": "^8.34.0",
     "eslint-config-prettier": "^8.6.0",
     "eslint-plugin-prettier": "^4.2.1",
@@ -135,11 +143,12 @@
     "@floating-ui/react-dom": "^1.3.0",
     "@parcel/watcher": "^2.5.1",
     "ag-psd": "^15.0.0",
+    "better-sqlite3": "9.6.0",
+    "chokidar": "^3.5.3",
     "comlink": "^4.4.1",
-    "dexie": "^3.2.3",
-    "dexie-export-import": "^1.0.3",
     "electron-updater": "^5.3.0",
     "fs-extra": "^11.1.0",
+    "kysely": "^0.28.7",
     "mobx": "^6.8.0",
     "mobx-react-lite": "^3.4.0",
     "node-exiftool": "^2.3.0",

resources/style/advanced-search.scss

@@ -2,7 +2,7 @@
   display: flex;
   flex-direction: column;
   // changed from min-width since large tag names expand the dialog width which looks bad
-  width: min(85ch, 80vw);
+  width: min(100ch, 80vw);
   margin-inline: auto;
 
   .criteria-input,
@@ -19,46 +19,114 @@
   }
 }
 
+#criteria-builder-label {
+  display: flex;
+  padding-bottom: 0.4rem;
+}
+
 #criteria-builder {
   display: grid;
-  grid-template-columns: 1fr 1fr 1fr min-content;
+  grid-template-columns: 3.5rem 3fr 3fr 4fr min-content;
   gap: 0.25rem 0.5rem;
   align-items: center;
 
   label {
     text-transform: uppercase;
     font-size: smaller;
+    padding-left: 1ch;
+    align-self: end;
   }
 }
 
 #query-editor-container {
   overflow: hidden auto;
-  padding: 2px 0;
+  padding: 2px;
   // dialog height - height of basically everything except the container
   max-height: calc(80vh - 17.25rem);
 }
 
-#query-editor {
+#query-editor-container-label {
+  display: flex;
+  padding-block: 0.4rem;
+}
+
+.query-editor {
+  display: grid;
+  grid-template-columns: 3.5rem 3fr 3fr 4fr min-content;
+  gap: 0.25rem 0.5rem;
+  align-items: center;
   width: 100%;
   border-spacing: 0;
 
-  tr {
-    margin-bottom: 0.25rem;
+  .criteria-input,
+  [role='combobox'] {
+    //min-width: 175px;
+    width: 100%;
+  }
+
+  .group-containter {
+    grid-column: 1 / -1;
+    padding: 0.5rem;
+    border: 1px dashed var(--background-color-selected);
+    border-radius: 0.25rem;
+  }
+
+  .separator-line,
+  .criteria-separator,
+  .separator {
+    grid-column: 1 / -1;
+    display: flex;
+    align-items: center;
+    text-align: center;
+    color: var(--text-color-muted);
+    opacity: 0.5;
+
+    &::before,
+    &::after {
+      content: "";
+      flex: 1;
+      height: 1px;
+      margin: 0 8px;
+    }
+
+    .conjunction-input {
+      width: fit-content;
+      background: var(--background-color);
+    }
   }
 
-  td,
-  th {
-    padding: 0;
-    padding-bottom: 0.25rem;
+  .separator-line {
+    &::before,
+    &::after {
+      background: var(--text-color-muted);
+    }
   }
 
-  td {
-    padding-inline-start: 0.5rem;
+  .criteria-separator {
+    margin: -2.5px;
+    font-size: 0.55rem;
+
+    .conjunction-input {
+      height: 0.78rem;
+      min-height: unset;
+    }
   }
 
-  .criteria-input,
-  [role='combobox'] {
-    min-width: 175px;
-    width: 100%;
+  .group-controls {
+    display: contents;
+
+    .btn-icon,
+    .criteria-input {
+      height: 1.1rem !important;
+      min-height: unset !important;
+      margin-bottom: 2px;
+      overflow: visible;
+      color: var(--accent-color-orange);
+    }
+
+    // shiulw be 3 cell width
+    .group-name-input {
+      grid-column: 2 / 5;
+    }
   }
 }

resources/style/app-toolbar.scss

@@ -2,6 +2,8 @@
   justify-content: stretch;
 
   .toolbar-button {
+    flex-shrink: 0;
+
     &[aria-pressed='true'],
     &[aria-checked='true'] {
       color: var(--text-color-strong);

resources/style/content.scss

@@ -83,11 +83,14 @@
     background-color: var(--accent-color-yellow);
 
     :is(img, video) {
-      clip-path: inset(0.175rem round calc(var(--thumbnail-radius) - 0.175rem));
+      clip-path: inset(0.2rem round calc(var(--thumbnail-radius) - 0.2rem));
     }
   }
   [aria-selected=true] > & > :is(img, video, .image-error){
-    clip-path: inset(0.175rem round calc(var(--thumbnail-radius) - 0.175rem));
+    clip-path: inset(0.2rem round calc(var(--thumbnail-radius) - 0.2rem));
+  }
+  :not([aria-selected=true]) > & > :is(img, video, .image-error){
+    transition: all 0ms;
   }
 }
 

src/api/data-backup.ts

@@ -5,5 +5,6 @@ export interface DataBackup {
   schedule(): void;
   backupToFile(path: string): Promise<void>;
   restoreFromFile(path: string): Promise<void>;
+  restoreEmpty(): Promise<void>;
   peekFile(path: string): Promise<[numTags: number, numFiles: number]>;
 }

src/api/data-storage-search.ts

@@ -1,3 +1,5 @@
+import { ID } from './id';
+
 export type PropertyKeys<T> = {
   [K in keyof T]: K extends string ? K : never;
 }[keyof T];
@@ -13,8 +15,18 @@ export const enum OrderDirection {
   Desc,
 }
 
+export type IndexableType = number | string | Date | Array<number | string | Date> | Uint8Array;
+
+export type PaginationDirection = 'after' | 'before';
+
+export type Cursor = { id: ID; orderValue: number | string | bigint | null };
+
 // General search criteria for a database entity
 // FFR: Boolean keys are not supported in IndexedDB/Dexie - must store booleans as 0/1
+export type ConditionGroupDTO<T> = {
+  conjunction: SearchConjunction;
+  children: Array<ConditionGroupDTO<T> | ConditionDTO<T>>;
+};
 
 export type ConditionDTO<T> =
   | ArrayConditionDTO<T, any>
@@ -53,6 +65,9 @@ export type BaseIndexSignature = { [key: string]: any };
 
 // Trick for converting array to type https://stackoverflow.com/a/49529930/2350481
 
+export const SearchConjunctions = ['and', 'or'] as const;
+export type SearchConjunction = (typeof SearchConjunctions)[number];
+
 export const NumberOperators = [
   'equals',
   'notEqual',