From 2bce47f5ffd67f09ca1f499a90251ae459d22804 Mon Sep 17 00:00:00 2001 From: RafaUC Date: Sat, 27 Sep 2025 00:31:20 -0600 Subject: [PATCH 01/19] Add dependecies --- package.json | 6 +- yarn.lock | 206 +++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 204 insertions(+), 8 deletions(-) diff --git a/package.json b/package.json index b0afc68d..fd702523 100644 --- a/package.json +++ b/package.json @@ -1,9 +1,7 @@ { "name": "allusion", "productName": "Allusion", - "version": "1.0.0-ruc.8.0", - "description": "A tool for managing your visual library", "main": "build/main.bundle.js", "scripts": { @@ -125,12 +123,14 @@ "@floating-ui/core": "^1.2.1", "@floating-ui/react-dom": "^1.3.0", "ag-psd": "^15.0.0", + "better-sqlite3": "^12.4.1", "chokidar": "^3.5.3", "comlink": "^4.4.1", "dexie": "^3.2.3", "dexie-export-import": "^1.0.3", "electron-updater": "^5.3.0", "fs-extra": "^11.1.0", + "kysely": "^0.28.7", "mobx": "^6.8.0", "mobx-react-lite": "^3.4.0", "node-exiftool": "^2.3.0", @@ -144,4 +144,4 @@ "utif": "^3.1.0", "wasm-feature-detect": "^1.2.11" } -} \ No newline at end of file +} diff --git a/yarn.lock b/yarn.lock index e2f3e68d..6cb9b30a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2418,6 +2418,14 @@ base64-js@^1.3.1, base64-js@^1.5.1: resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== +better-sqlite3@^12.4.1: + version "12.4.1" + resolved "https://registry.yarnpkg.com/better-sqlite3/-/better-sqlite3-12.4.1.tgz#f78df6c80530d1a0b750b538033e6199b7d30d26" + integrity sha512-3yVdyZhklTiNrtg+4WqHpJpFDd+WHTg2oM7UcR80GqL05AOV0xEJzc6qNvFYoEtE+hRp1n9MpN6/+4yhlGkDXQ== + dependencies: + bindings "^1.5.0" + prebuild-install "^7.1.1" + big.js@^5.2.2: version "5.2.2" resolved "https://registry.yarnpkg.com/big.js/-/big.js-5.2.2.tgz#65f0af382f578bcdc742bd9c281e9cb2d7768328" @@ -2428,6 +2436,22 @@ binary-extensions@^2.0.0: resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.2.0.tgz#75f502eeaf9ffde42fc98829645be4ea76bd9e2d" integrity sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA== +bindings@^1.5.0: + version "1.5.0" + resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.5.0.tgz#10353c9e945334bc0511a6d90b38fbc7c9c504df" + integrity sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ== + dependencies: + file-uri-to-path "1.0.0" + +bl@^4.0.3: + version "4.1.0" + resolved "https://registry.yarnpkg.com/bl/-/bl-4.1.0.tgz#451535264182bec2fbbc83a62ab98cf11d9f7b3a" + integrity sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w== + dependencies: + buffer "^5.5.0" + inherits "^2.0.4" + readable-stream "^3.4.0" + bluebird-lst@^1.0.9: version "1.0.9" resolved "https://registry.yarnpkg.com/bluebird-lst/-/bluebird-lst-1.0.9.tgz#a64a0e4365658b9ab5fe875eb9dfb694189bb41c" @@ -2529,7 +2553,7 @@ buffer-from@^1.0.0: resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.2.tgz#2b146a6fd72e80b4f55d255f35ed59a3a9a41bd5" integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ== -buffer@^5.1.0: +buffer@^5.1.0, buffer@^5.5.0: version "5.7.1" resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.7.1.tgz#ba62e7c13133053582197160851a8f648e99eed0" integrity sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ== @@ -2659,6 +2683,11 @@ char-regex@^1.0.2: optionalDependencies: fsevents "~2.3.2" +chownr@^1.1.1: + version "1.1.4" + resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b" + integrity sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg== + chownr@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/chownr/-/chownr-2.0.0.tgz#15bfbe53d2eab4cf70f18a8cd68ebe5b3cb1dece" @@ -2955,11 +2984,23 @@ decompress-response@^3.3.0: dependencies: mimic-response "^1.0.0" +decompress-response@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/decompress-response/-/decompress-response-6.0.0.tgz#ca387612ddb7e104bd16d85aab00d5ecf09c66fc" + integrity sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ== + dependencies: + mimic-response "^3.1.0" + dedent@^0.7.0: version "0.7.0" resolved "https://registry.yarnpkg.com/dedent/-/dedent-0.7.0.tgz#2495ddbaf6eb874abb0e1be9df22d2e5a544326c" integrity sha512-Q6fKUPqnAHAyhiUgFU7BUzLiv0kd8saH9al7tnu5Q/okj6dnupxyTgFIBjVzJATdfIAm9NAsvXNzjaKa+bxVyA== +deep-extend@^0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.6.0.tgz#c4fa7c95404a17a9c3e8ca7e1537312b736330ac" + integrity sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA== + deep-is@^0.1.3: version "0.1.4" resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.4.tgz#a6f2dce612fadd2ef1f519b73551f17e85199831" @@ -2988,6 +3029,11 @@ delayed-stream@~1.0.0: resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" integrity sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ== +detect-libc@^2.0.0: + version "2.1.1" + resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-2.1.1.tgz#9f1e511ace6bb525efea4651345beac424dac7b9" + integrity sha512-ecqj/sy1jcK1uWrwpR67UhYrIFQ+5WlGxth34WquCbamhFA6hkkwiu37o6J5xCHdo1oixJRfVRw+ywV+Hq/0Aw== + detect-newline@^3.0.0: version "3.1.0" resolved "https://registry.yarnpkg.com/detect-newline/-/detect-newline-3.1.0.tgz#576f5dfc63ae1a192ff192d8ad3af6308991b651" @@ -3245,6 +3291,13 @@ end-of-stream@^1.1.0: dependencies: once "^1.4.0" +end-of-stream@^1.4.1: + version "1.4.5" + resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.5.tgz#7344d711dea40e0b74abc2ed49778743ccedb08c" + integrity sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg== + dependencies: + once "^1.4.0" + enhanced-resolve@^5.0.0, enhanced-resolve@^5.10.0: version "5.12.0" resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-5.12.0.tgz#300e1c90228f5b570c4d35babf263f6da7155634" @@ -3557,6 +3610,11 @@ exit@^0.1.2: resolved "https://registry.yarnpkg.com/exit/-/exit-0.1.2.tgz#0632638f8d877cc82107d30a0fff1a17cba1cd0c" integrity sha512-Zk/eNKV2zbjpKzrsQ+n1G6poVbErQxJ0LBOJXaKZ1EViLzH+hrLu9cdXI4zw9dBQJslwBEpbQ2P1oS7nDxs6jQ== +expand-template@^2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/expand-template/-/expand-template-2.0.3.tgz#6e14b3fcee0f3a6340ecb57d2e8918692052a47c" + integrity sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg== + expect@^29.0.0, expect@^29.5.0: version "29.5.0" resolved "https://registry.yarnpkg.com/expect/-/expect-29.5.0.tgz#68c0509156cb2a0adb8865d413b137eeaae682f7" @@ -3655,6 +3713,11 @@ file-entry-cache@^6.0.1: dependencies: flat-cache "^3.0.4" +file-uri-to-path@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz#553a7b8446ff6f684359c445f1e37a05dacc33dd" + integrity sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw== + filelist@^1.0.1: version "1.0.3" resolved "https://registry.yarnpkg.com/filelist/-/filelist-1.0.3.tgz#448607750376484932f67ef1b9ff07386b036c83" @@ -3714,6 +3777,11 @@ form-data@^4.0.0: combined-stream "^1.0.8" mime-types "^2.1.12" +fs-constants@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad" + integrity sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow== + fs-extra@^10.0.0, fs-extra@^10.1.0: version "10.1.0" resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-10.1.0.tgz#02873cfbc4084dde127eaa5f9905eef2325d1abf" @@ -3839,6 +3907,11 @@ get-symbol-description@^1.0.0: call-bind "^1.0.2" get-intrinsic "^1.1.1" +github-from-package@0.0.0: + version "0.0.0" + resolved "https://registry.yarnpkg.com/github-from-package/-/github-from-package-0.0.0.tgz#97fb5d96bfde8973313f20e8288ef9a167fa64ce" + integrity sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw== + glob-parent@^5.1.2, glob-parent@~5.1.2: version "5.1.2" resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.2.tgz#869832c58034fe68a4093c17dc15e8340d8401c4" @@ -4157,12 +4230,12 @@ inflight@^1.0.4: once "^1.3.0" wrappy "1" -inherits@2: +inherits@2, inherits@^2.0.3, inherits@^2.0.4: version "2.0.4" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== -ini@^1.3.4: +ini@^1.3.4, ini@~1.3.0: version "1.3.8" resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.8.tgz#a29da425b48806f34767a4efce397269af28432c" integrity sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew== @@ -4911,6 +4984,11 @@ klona@^2.0.4: resolved "https://registry.yarnpkg.com/klona/-/klona-2.0.5.tgz#d166574d90076395d9963aa7a928fabb8d76afbc" integrity sha512-pJiBpiXMbt7dkzXe8Ghj/u4FfXOOa98fPW+bihOJ4SjnoijweJrNThJfd3ifXpXhREjpoF2mZVH1GfS9LV3kHQ== +kysely@^0.28.7: + version "0.28.7" + resolved "https://registry.yarnpkg.com/kysely/-/kysely-0.28.7.tgz#461d160865825f89173a7f814489e92a91b13864" + integrity sha512-u/cAuTL4DRIiO2/g4vNGRgklEKNIj5Q3CG7RoUB5DV5SfEC2hMvPxKi0GWPmnzwL2ryIeud2VTcEEmqzTzEPNw== + lazy-val@^1.0.4, lazy-val@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/lazy-val/-/lazy-val-1.0.5.tgz#6cf3b9f5bc31cee7ee3e369c0832b7583dcd923d" @@ -5116,6 +5194,11 @@ mimic-response@^1.0.0, mimic-response@^1.0.1: resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-1.0.1.tgz#4923538878eef42063cb8a3e3b0798781487ab1b" integrity sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ== +mimic-response@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-3.1.0.tgz#2d1d59af9c1b129815accc2c46a022a5ce1fa3c9" + integrity sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ== + mini-css-extract-plugin@^2.7.2: version "2.7.2" resolved "https://registry.yarnpkg.com/mini-css-extract-plugin/-/mini-css-extract-plugin-2.7.2.tgz#e049d3ea7d3e4e773aad585c6cb329ce0c7b72d7" @@ -5149,6 +5232,11 @@ minimist@^1.2.0: resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.6.tgz#8637a5b759ea0d6e98702cfb3a9283323c93af44" integrity sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q== +minimist@^1.2.3: + version "1.2.8" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c" + integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA== + minipass@^3.0.0: version "3.3.4" resolved "https://registry.yarnpkg.com/minipass/-/minipass-3.3.4.tgz#ca99f95dd77c43c7a76bf51e6d200025eee0ffae" @@ -5164,6 +5252,11 @@ minizlib@^2.1.1: minipass "^3.0.0" yallist "^4.0.0" +mkdirp-classic@^0.5.2, mkdirp-classic@^0.5.3: + version "0.5.3" + resolved "https://registry.yarnpkg.com/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz#fa10c9115cc6d8865be221ba47ee9bed78601113" + integrity sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A== + mkdirp@^1.0.3: version "1.0.4" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e" @@ -5194,6 +5287,11 @@ nanoid@^3.3.4: resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.4.tgz#730b67e3cd09e2deacf03c027c81c9d9dbc5e8ab" integrity sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw== +napi-build-utils@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/napi-build-utils/-/napi-build-utils-2.0.0.tgz#13c22c0187fcfccce1461844136372a47ddc027e" + integrity sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA== + natural-compare-lite@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/natural-compare-lite/-/natural-compare-lite-1.4.0.tgz#17b09581988979fddafe0201e931ba933c96cbb4" @@ -5222,6 +5320,13 @@ no-case@^3.0.4: lower-case "^2.0.2" tslib "^2.0.3" +node-abi@^3.3.0: + version "3.77.0" + resolved "https://registry.yarnpkg.com/node-abi/-/node-abi-3.77.0.tgz#3ad90d5c9d45663420e5aa4ff58dbf4e3625419a" + integrity sha512-DSmt0OEcLoK4i3NuscSbGjOf3bqiDEutejqENSplMSFA/gmB8mkED9G4pKWnPl7MDU4rSHebKPHeitpDfyH0cQ== + dependencies: + semver "^7.3.5" + node-addon-api@^1.6.3: version "1.7.2" resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-1.7.2.tgz#3df30b95720b53c24e59948b49532b662444f54d" @@ -5577,6 +5682,24 @@ postcss@^8.4.19: picocolors "^1.0.0" source-map-js "^1.0.2" +prebuild-install@^7.1.1: + version "7.1.3" + resolved "https://registry.yarnpkg.com/prebuild-install/-/prebuild-install-7.1.3.tgz#d630abad2b147443f20a212917beae68b8092eec" + integrity sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug== + dependencies: + detect-libc "^2.0.0" + expand-template "^2.0.3" + github-from-package "0.0.0" + minimist "^1.2.3" + mkdirp-classic "^0.5.3" + napi-build-utils "^2.0.0" + node-abi "^3.3.0" + pump "^3.0.0" + rc "^1.2.7" + simple-get "^4.0.0" + tar-fs "^2.0.0" + tunnel-agent "^0.6.0" + prelude-ls@^1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.2.1.tgz#debc6489d7a6e6b0e7611888cec880337d316396" @@ -5673,6 +5796,16 @@ randombytes@^2.1.0: dependencies: safe-buffer "^5.1.0" +rc@^1.2.7: + version "1.2.8" + resolved "https://registry.yarnpkg.com/rc/-/rc-1.2.8.tgz#cd924bf5200a075b83c188cd6b9e211b7fc0d3ed" + integrity sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw== + dependencies: + deep-extend "^0.6.0" + ini "~1.3.0" + minimist "^1.2.0" + strip-json-comments "~2.0.1" + react-colorful@^5.6.1: version "5.6.1" resolved "https://registry.yarnpkg.com/react-colorful/-/react-colorful-5.6.1.tgz#7dc2aed2d7c72fac89694e834d179e32f3da563b" @@ -5722,6 +5855,15 @@ read-config-file@6.2.0: json5 "^2.2.0" lazy-val "^1.0.4" +readable-stream@^3.1.1, readable-stream@^3.4.0: + version "3.6.2" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.2.tgz#56a9b36ea965c00c5a93ef31eb111a0f11056967" + integrity sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA== + dependencies: + inherits "^2.0.3" + string_decoder "^1.1.1" + util-deprecate "^1.0.1" + readdirp@~3.6.0: version "3.6.0" resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-3.6.0.tgz#74a370bd857116e245b29cc97340cd431a02a6c7" @@ -5933,7 +6075,7 @@ rxjs@^7.5.2: dependencies: tslib "^2.1.0" -safe-buffer@^5.1.0: +safe-buffer@^5.0.1, safe-buffer@^5.1.0, safe-buffer@~5.2.0: version "5.2.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== @@ -6076,6 +6218,20 @@ signal-exit@^3.0.3, signal-exit@^3.0.7: resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.7.tgz#a9a1767f8af84155114eaabd73f99273c8f59ad9" integrity sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ== +simple-concat@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/simple-concat/-/simple-concat-1.0.1.tgz#f46976082ba35c2263f1c8ab5edfe26c41c9552f" + integrity sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q== + +simple-get@^4.0.0: + version "4.0.1" + resolved "https://registry.yarnpkg.com/simple-get/-/simple-get-4.0.1.tgz#4a39db549287c979d352112fa03fd99fd6bc3543" + integrity sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA== + dependencies: + decompress-response "^6.0.0" + once "^1.3.1" + simple-concat "^1.0.0" + simple-update-notifier@^1.0.7: version "1.0.7" resolved "https://registry.yarnpkg.com/simple-update-notifier/-/simple-update-notifier-1.0.7.tgz#7edf75c5bdd04f88828d632f762b2bc32996a9cc" @@ -6221,6 +6377,13 @@ string.prototype.trimstart@^1.0.6: define-properties "^1.1.4" es-abstract "^1.20.4" +string_decoder@^1.1.1: + version "1.3.0" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e" + integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA== + dependencies: + safe-buffer "~5.2.0" + strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" @@ -6243,6 +6406,11 @@ strip-json-comments@^3.1.0, strip-json-comments@^3.1.1: resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-3.1.1.tgz#31f1281b3832630434831c310c01cccda8cbe006" integrity sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig== +strip-json-comments@~2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a" + integrity sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ== + style-loader@^3.3.1: version "3.3.1" resolved "https://registry.yarnpkg.com/style-loader/-/style-loader-3.3.1.tgz#057dfa6b3d4d7c7064462830f9113ed417d38575" @@ -6304,6 +6472,27 @@ tapable@^2.0.0, tapable@^2.1.1, tapable@^2.2.0: resolved "https://registry.yarnpkg.com/tapable/-/tapable-2.2.1.tgz#1967a73ef4060a82f12ab96af86d52fdb76eeca0" integrity sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ== +tar-fs@^2.0.0: + version "2.1.4" + resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-2.1.4.tgz#800824dbf4ef06ded9afea4acafe71c67c76b930" + integrity sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ== + dependencies: + chownr "^1.1.1" + mkdirp-classic "^0.5.2" + pump "^3.0.0" + tar-stream "^2.1.4" + +tar-stream@^2.1.4: + version "2.2.0" + resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-2.2.0.tgz#acad84c284136b060dc3faa64474aa9aebd77287" + integrity sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ== + dependencies: + bl "^4.0.3" + end-of-stream "^1.4.1" + fs-constants "^1.0.0" + inherits "^2.0.3" + readable-stream "^3.1.1" + tar@^6.1.11: version "6.1.11" resolved "https://registry.yarnpkg.com/tar/-/tar-6.1.11.tgz#6760a38f003afa1b2ffd0ffe9e9abbd0eab3d621" @@ -6460,6 +6649,13 @@ tsutils@^3.21.0: dependencies: tslib "^1.8.1" +tunnel-agent@^0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd" + integrity sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w== + dependencies: + safe-buffer "^5.0.1" + tunnel@^0.0.6: version "0.0.6" resolved "https://registry.yarnpkg.com/tunnel/-/tunnel-0.0.6.tgz#72f1314b34a5b192db012324df2cc587ca47f92c" @@ -6604,7 +6800,7 @@ utif@^3.1.0: dependencies: pako "^1.0.5" -util-deprecate@^1.0.2: +util-deprecate@^1.0.1, util-deprecate@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" integrity sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw== From 7253a8659a140d72d02d6df69f72b63397249688 Mon Sep 17 00:00:00 2001 From: RafaUC Date: Sat, 27 Sep 2025 15:14:58 -0600 Subject: [PATCH 02/19] Mak has deprecated the previous backend. --- common/core.ts | 17 ++++++++++++++ src/backend/README.md | 4 ++-- src/backend/{ => _deprecated}/backend.ts | 19 +++++++-------- .../{ => _deprecated}/backup-scheduler.ts | 23 ++++--------------- src/backend/{ => _deprecated}/config.ts | 5 ++-- src/renderer.tsx | 6 ++--- tests/backend.test.ts | 4 ++-- tests/tag-store.test.ts | 4 ++-- 8 files changed, 43 insertions(+), 39 deletions(-) rename src/backend/{ => _deprecated}/backend.ts (98%) rename src/backend/{ => _deprecated}/backup-scheduler.ts (88%) rename src/backend/{ => _deprecated}/config.ts (98%) diff --git a/common/core.ts b/common/core.ts index 05233924..84103406 100644 --- a/common/core.ts +++ b/common/core.ts @@ -70,3 +70,20 @@ export function normalizeBase(str: string): string { .replace(/[\u0300-\u036f]/g, '') .toLowerCase(); } + +/** Returns the date at 00:00 today */ +export function getToday(): Date { + const today = new Date(); + today.setHours(0); + today.setMinutes(0); + today.setSeconds(0, 0); + return today; +} + +/** Returns the date at the start of the current week (Sunday at 00:00) */ +export function getWeekStart(): Date { + const date = getToday(); + const dayOfWeek = date.getDay(); + date.setDate(date.getDate() - dayOfWeek); + return date; +} diff --git a/src/backend/README.md b/src/backend/README.md index 98fc7402..046ec752 100644 --- a/src/backend/README.md +++ b/src/backend/README.md @@ -1,10 +1,10 @@ # Backend Although we call this our "backend", this section of code all runs in the Electron's renderer process. -This is where image and tag data is persisted to a database (IndexedDB). +This is where image and tag data is persisted to a database ~~(IndexedDB)~~ SQLite. The database is exposed to the web application through the `Backend.ts`, which acts as an API to the database. The idea behind this backend was to create a separation between the back- and frontend you usually find in web applications. If we ever want to change the type of database we use, this would make it relatively straightforward. -This set-up is not optimal for performant fetching of large amounts of items from the database. +~~This set-up is not optimal for performant fetching of large amounts of items from the database.~~ diff --git a/src/backend/backend.ts b/src/backend/_deprecated/backend.ts similarity index 98% rename from src/backend/backend.ts rename to src/backend/_deprecated/backend.ts index 1218d271..663852ab 100644 --- a/src/backend/backend.ts +++ b/src/backend/_deprecated/backend.ts @@ -1,7 +1,7 @@ import Dexie, { Collection, IndexableType, Table, WhereClause } from 'dexie'; -import { retainArray, shuffleArray } from '../../common/core'; -import { DataStorage } from '../api/data-storage'; +import { retainArray, shuffleArray } from '../../../common/core'; +import { DataStorage } from '../../api/data-storage'; import { ArrayConditionDTO, BaseIndexSignature, @@ -17,17 +17,18 @@ import { isExtraPropertyOperatorType, isNumberOperator, isStringOperator, -} from '../api/data-storage-search'; -import { FileDTO } from '../api/file'; -import { FileSearchDTO } from '../api/file-search'; -import { ID } from '../api/id'; -import { LocationDTO } from '../api/location'; -import { ROOT_TAG_ID, TagDTO } from '../api/tag'; -import { ExtraPropertyDTO, ExtraPropertyType } from '../api/extraProperty'; +} from '../../api/data-storage-search'; +import { FileDTO } from '../../api/file'; +import { FileSearchDTO } from '../../api/file-search'; +import { ID } from '../../api/id'; +import { LocationDTO } from '../../api/location'; +import { ROOT_TAG_ID, TagDTO } from '../../api/tag'; +import { ExtraPropertyDTO, ExtraPropertyType } from '../../api/extraProperty'; const USE_TIMING_PROXY = false; /** + * @deprecated * The backend of the application serves as an API, even though it runs on the same machine. * This helps code organization by enforcing a clear separation between backend/frontend logic. * Whenever we want to change things in the backend, this should have no consequences in the frontend. diff --git a/src/backend/backup-scheduler.ts b/src/backend/_deprecated/backup-scheduler.ts similarity index 88% rename from src/backend/backup-scheduler.ts rename to src/backend/_deprecated/backup-scheduler.ts index 6e4d9983..b5592cbd 100644 --- a/src/backend/backup-scheduler.ts +++ b/src/backend/_deprecated/backup-scheduler.ts @@ -3,27 +3,12 @@ import { exportDB, importDB, peakImportFile } from 'dexie-export-import'; import fse from 'fs-extra'; import path from 'path'; -import { debounce } from '../../common/timeout'; -import { DataBackup } from '../api/data-backup'; +import { debounce } from '../../../common/timeout'; +import { DataBackup } from '../../api/data-backup'; import { AUTO_BACKUP_TIMEOUT, NUM_AUTO_BACKUPS } from './config'; +import { getToday, getWeekStart } from 'common/core'; -/** Returns the date at 00:00 today */ -function getToday(): Date { - const today = new Date(); - today.setHours(0); - today.setMinutes(0); - today.setSeconds(0, 0); - return today; -} - -/** Returns the date at the start of the current week (Sunday at 00:00) */ -function getWeekStart(): Date { - const date = getToday(); - const dayOfWeek = date.getDay(); - date.setDate(date.getDate() - dayOfWeek); - return date; -} - +/** @deprecated */ export default class BackupScheduler implements DataBackup { #db: Dexie; #backupDirectory: string = ''; diff --git a/src/backend/config.ts b/src/backend/_deprecated/config.ts similarity index 98% rename from src/backend/config.ts rename to src/backend/_deprecated/config.ts index 268f7a4c..a88e76ca 100644 --- a/src/backend/config.ts +++ b/src/backend/_deprecated/config.ts @@ -1,9 +1,9 @@ import Dexie, { Transaction } from 'dexie'; import fse from 'fs-extra'; -import { FileDTO } from '../api/file'; +import { FileDTO } from '../../api/file'; import { TagDTO } from 'src/api/tag'; -import { ID } from '../api/id'; +import { ID } from '../../api/id'; import { ExtraProperties, ExtraPropertyType } from 'src/api/extraProperty'; import { LocationDTO, SubLocationDTO } from 'src/api/location'; @@ -273,6 +273,7 @@ type DBVersioningConfig = { }; /** + * @deprecated * A function that should be called before using the database. * It initializes the object stores */ diff --git a/src/renderer.tsx b/src/renderer.tsx index bb90d64e..d89e095d 100644 --- a/src/renderer.tsx +++ b/src/renderer.tsx @@ -16,7 +16,7 @@ import { IS_DEV } from 'common/process'; import { promiseRetry } from 'common/timeout'; import { IS_PREVIEW_WINDOW, WINDOW_STORAGE_KEY } from 'common/window'; import { RendererMessenger } from 'src/ipc/renderer'; -import Backend from './backend/backend'; +import Backend from './backend/_deprecated/backend'; import App from './frontend/App'; import SplashScreen from './frontend/containers/SplashScreen'; import StoreProvider from './frontend/contexts/StoreContext'; @@ -25,8 +25,8 @@ import PreviewApp from './frontend/Preview'; import { FILE_STORAGE_KEY } from './frontend/stores/FileStore'; import RootStore from './frontend/stores/RootStore'; import { PREFERENCES_STORAGE_KEY } from './frontend/stores/UiStore'; -import BackupScheduler from './backend/backup-scheduler'; -import { DB_NAME, dbInit } from './backend/config'; +import BackupScheduler from './backend/_deprecated/backup-scheduler'; +import { DB_NAME, dbInit } from './backend/_deprecated/config'; async function main(): Promise { // Render our react components in the div with id 'app' in the html file diff --git a/tests/backend.test.ts b/tests/backend.test.ts index 70657e28..70c80862 100644 --- a/tests/backend.test.ts +++ b/tests/backend.test.ts @@ -1,8 +1,8 @@ import { OrderDirection } from '../src/api/data-storage-search'; import { FileDTO } from '../src/api/file'; import { ROOT_TAG_ID, TagDTO } from '../src/api/tag'; -import Backend from '../src/backend/backend'; -import { dbInit } from '../src/backend/config'; +import Backend from '../src/backend/_deprecated/backend'; +import { dbInit } from '../src/backend/_deprecated/config'; describe('Backend', () => { let TEST_DATABASE_ID_COUNTER = 0; diff --git a/tests/tag-store.test.ts b/tests/tag-store.test.ts index 71663d52..fc41c826 100644 --- a/tests/tag-store.test.ts +++ b/tests/tag-store.test.ts @@ -1,5 +1,5 @@ -import Backend from '../src/backend/backend'; -import { dbInit } from '../src/backend/config'; +import Backend from '../src/backend/_deprecated/backend'; +import { dbInit } from '../src/backend/_deprecated/config'; import TagStore from '../src/frontend/stores/TagStore'; describe('TagStore', () => { From 0e6edcb0f1f6806bb962fe7d18eb2b966dfb62a3 Mon Sep 17 00:00:00 2001 From: RafaUC Date: Sun, 28 Sep 2025 01:59:25 -0600 Subject: [PATCH 03/19] Set up compatible better-sqlite3 version and configure Kysely manual migrations Create Tag table schemas to test Kysely. --- package.json | 5 +- src/backend/backend.ts | 29 ++ src/backend/config.ts | 41 ++ src/backend/migrations/000_initial.ts | 44 ++ src/backend/schemaTypes.ts | 45 ++ src/main.ts | 10 +- webpack.dev.js | 6 +- yarn.lock | 697 +++++++++++++++++++++++++- 8 files changed, 856 insertions(+), 21 deletions(-) create mode 100644 src/backend/backend.ts create mode 100644 src/backend/config.ts create mode 100644 src/backend/migrations/000_initial.ts create mode 100644 src/backend/schemaTypes.ts diff --git a/package.json b/package.json index fd702523..f7b1cc66 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "package": "yarn build && electron-builder", "logo": "ncp ./resources/logo/icns/allusion-logomark-fc.icns ./build/icon.icns && ncp ./resources/logo/ico/allusion-logomark-fc-256x256.ico ./build/icon.ico", "build": "rimraf dist && yarn production && yarn logo", + "postinstall": "electron-rebuild -f -w better-sqlite3", "pack": "electron-builder --dir", "dist": "electron-builder", "build:masonry": "cd wasm/wasm-build && cargo run masonry masonry/masonry-scalar && cargo run masonry masonry/masonry-simd -- -C target-feature=+simd128", @@ -83,6 +84,7 @@ "homepage": "https://github.com/RafaUC/Allusion/Allusion#readme", "devDependencies": { "@svgr/webpack": "^6.5.1", + "@types/better-sqlite3": "^7.6.13", "@types/chrome": "^0.0.195", "@types/fs-extra": "^11.0.1", "@types/jest": "^29.5.1", @@ -96,6 +98,7 @@ "css-loader": "^6.7.3", "electron": "21.3.0", "electron-builder": "23.6.0", + "electron-rebuild": "^3.2.9", "eslint": "^8.34.0", "eslint-config-prettier": "^8.6.0", "eslint-plugin-prettier": "^4.2.1", @@ -123,7 +126,7 @@ "@floating-ui/core": "^1.2.1", "@floating-ui/react-dom": "^1.3.0", "ag-psd": "^15.0.0", - "better-sqlite3": "^12.4.1", + "better-sqlite3": "9.6.0", "chokidar": "^3.5.3", "comlink": "^4.4.1", "dexie": "^3.2.3", diff --git a/src/backend/backend.ts b/src/backend/backend.ts new file mode 100644 index 00000000..58f1b3f4 --- /dev/null +++ b/src/backend/backend.ts @@ -0,0 +1,29 @@ +import { Database } from './schemaTypes'; +import SQLite from 'better-sqlite3'; +import { Kysely, SqliteDialect, CamelCasePlugin } from 'kysely'; +import { migrateToLatest } from './config'; + +export default class Backend { + #db: Kysely; + #notifyChange: () => void; + + constructor(db: Kysely, notifyChange: () => void) { + this.#db = db; + + this.#notifyChange = notifyChange; + } + + static async init(dbPath: string, notifyChange: () => void): Promise { + console.info(`SQLite3: Initializing database "${dbPath}"...`); + const dialect = new SqliteDialect({ + database: new SQLite(dbPath), + }); + const db = new Kysely({ + dialect: dialect, + plugins: [new CamelCasePlugin()], + }); + migrateToLatest(db); + const backend = new Backend(db, notifyChange); + return backend; + } +} diff --git a/src/backend/config.ts b/src/backend/config.ts new file mode 100644 index 00000000..6d8cdf4c --- /dev/null +++ b/src/backend/config.ts @@ -0,0 +1,41 @@ +import * as path from 'path'; +import { promises as fs } from 'fs'; +import { Kysely, Migrator, FileMigrationProvider, Migration, MigrationProvider } from 'kysely'; +import { Database } from './schemaTypes'; + +export const DB_NAME = 'Allusion'; + +export const NUM_AUTO_BACKUPS = 6; + +export const AUTO_BACKUP_TIMEOUT = 1000 * 60 * 10; // 10 minutes + +//Register the migrations here. +class InlineMigrationProvider implements MigrationProvider { + async getMigrations(): Promise> { + return { + '001_initial': await import('./migrations/000_initial'), + }; + } +} + +export async function migrateToLatest(db: Kysely): Promise { + const migrator = new Migrator({ + db, + provider: new InlineMigrationProvider(), + }); + + const { error, results } = await migrator.migrateToLatest(); + + results?.forEach((it) => { + if (it.status === 'Success') { + console.log(`migration "${it.migrationName}" was executed successfully`); + } else if (it.status === 'Error') { + console.error(`failed to execute migration "${it.migrationName}"`); + } + }); + + if (error) { + console.error('failed to migrate'); + console.error(error); + } +} diff --git a/src/backend/migrations/000_initial.ts b/src/backend/migrations/000_initial.ts new file mode 100644 index 00000000..2d9d4419 --- /dev/null +++ b/src/backend/migrations/000_initial.ts @@ -0,0 +1,44 @@ +import { Kysely } from 'kysely'; + +export async function up(db: Kysely) { + // tags + await db.schema + .createTable('tags') + .addColumn('id', 'text', (col) => col.primaryKey()) + .addColumn('name', 'text', (col) => col.notNull()) + .addColumn('date_added', 'integer', (col) => col.notNull()) + .addColumn('color', 'text', (col) => col.notNull()) + .addColumn('is_hidden', 'boolean', (col) => col.notNull().defaultTo(false)) + .addColumn('is_visible_inherited', 'boolean', (col) => col.notNull().defaultTo(false)) + .addColumn('is_header', 'boolean', (col) => col.notNull().defaultTo(false)) + .addColumn('description', 'text') + .execute(); + + // N:N subTags + await db.schema + .createTable('tag_subtags') + .addColumn('tag_id', 'text', (col) => col.notNull()) + .addColumn('subtag_id', 'text', (col) => col.notNull()) + .addForeignKeyConstraint('fk_tag_subtags_tag', ['tag_id'], 'tags', ['id']) + .addForeignKeyConstraint('fk_tag_subtags_subtag', ['subtag_id'], 'tags', ['id']) + .execute(); + + // N:N impliedTags + await db.schema + .createTable('tag_implied') + .addColumn('tag_id', 'text', (col) => col.notNull()) + .addColumn('implied_tag_id', 'text', (col) => col.notNull()) + .addForeignKeyConstraint('fk_tag_implied_tag', ['tag_id'], 'tags', ['id']) + .addForeignKeyConstraint('fk_tag_implied_implied', ['implied_tag_id'], 'tags', ['id']) + .execute(); + + // Aliases + await db.schema + .createTable('tag_aliases') + .addColumn('tag_id', 'text', (col) => col.notNull()) + .addColumn('alias', 'text', (col) => col.notNull()) + .addForeignKeyConstraint('fk_tag_aliases_tag', ['tag_id'], 'tags', ['id']) + .execute(); +} + +export async function down(db: Kysely) {} diff --git a/src/backend/schemaTypes.ts b/src/backend/schemaTypes.ts new file mode 100644 index 00000000..a6adbde0 --- /dev/null +++ b/src/backend/schemaTypes.ts @@ -0,0 +1,45 @@ +/** + * In this file we define the interfaces that Kisely will use to have types defined and build the sql queris, + * these are an equivalent representation to the actual SQLite database schema. + * + * These are just interfaces, and updating here the structures will not actualizze the database, to do taht you will need + * to manually write Kysely migrations taking care that the migration will update the database shema to be the same. + */ + +import { ColumnType } from 'kysely'; +import { ID } from '../api/id'; + +export interface Database { + tags: Tags; + tagSubTags: TagSubTags; + tagImplied: TagImplied; + tagAliases: TagAliases; +} + +///// TAGS ///// + +export interface Tags { + id: ID; + name: string; + dateAdded: ColumnType; + color: string; + isHidden: boolean; + isVisibleInherited: boolean; + isHeader: boolean; + description: string | null; +} + +export interface TagSubTags { + tagId: ID; + subtagId: string; +} + +export interface TagImplied { + tagId: ID; + impliedTagId: ID; +} + +export interface TagAliases { + tagId: ID; + alias: string; +} diff --git a/src/main.ts b/src/main.ts index 04f0c3c4..8382c708 100644 --- a/src/main.ts +++ b/src/main.ts @@ -24,13 +24,15 @@ import { IS_DEV, IS_MAC } from '../common/process'; import { TagDTO, ROOT_TAG_ID } from './api/tag'; import { MainMessenger } from './ipc/main'; import { WindowSystemButtonPress } from './ipc/messages'; +import { DB_NAME } from './backend/config'; +import Backend from './backend/backend'; // TODO: change this when running in portable mode, see portable-improvements branch const basePath = app.getPath('userData'); const preferencesFilePath = path.join(basePath, 'preferences.json'); const windowStateFilePath = path.join(basePath, 'windowState.json'); -const logFilePath = path.join(basePath, 'app.log'); +const databaseFilePath = path.join(basePath, 'databases', `${DB_NAME}.sqlite`); type PreferencesFile = { checkForUpdatesOnStartup?: boolean; @@ -45,6 +47,7 @@ let mainWindow: BrowserWindow | null = null; let previewWindow: BrowserWindow | null = null; let tray: Tray | null = null; let clipServer: ClipServer | null = null; +let backend: Backend | null = null; function initialize() { console.log('Initializing Allusion...'); @@ -75,6 +78,7 @@ function initialize() { } }); + createBackend(); createWindow(); createPreviewWindow(); @@ -97,6 +101,10 @@ function initialize() { } } +async function createBackend(): Promise { + backend = await Backend.init(databaseFilePath, () => {}); +} + function createWindow() { // Remember window size and position const previousWindowState = getPreviousWindowState(); diff --git a/webpack.dev.js b/webpack.dev.js index efcabf70..f1108320 100644 --- a/webpack.dev.js +++ b/webpack.dev.js @@ -43,7 +43,8 @@ let mainConfig = { ], }, externals: { - fsevents: "require('fsevents')" + fsevents: "require('fsevents')", + 'better-sqlite3': 'commonjs better-sqlite3', } }; @@ -123,7 +124,8 @@ let rendererConfig = { }), ], externals: { - fsevents: "require('fsevents')" + fsevents: "require('fsevents')", + 'better-sqlite3': 'commonjs better-sqlite3', } }; diff --git a/yarn.lock b/yarn.lock index 6cb9b30a..a07be6d2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1096,6 +1096,11 @@ dependencies: "@floating-ui/dom" "^1.2.1" +"@gar/promisify@^1.1.3": + version "1.1.3" + resolved "https://registry.yarnpkg.com/@gar/promisify/-/promisify-1.1.3.tgz#555193ab2e3bb3b6adc3d551c9c030d9e860daf6" + integrity sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw== + "@humanwhocodes/config-array@^0.11.8": version "0.11.8" resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.11.8.tgz#03595ac2075a4dc0f191cc2131de14fbd7d410b9" @@ -1370,6 +1375,13 @@ dependencies: cross-spawn "^7.0.1" +"@malept/cross-spawn-promise@^2.0.0": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@malept/cross-spawn-promise/-/cross-spawn-promise-2.0.0.tgz#d0772de1aa680a0bfb9ba2f32b4c828c7857cb9d" + integrity sha512-1DpKU0Z5ThltBwjNySMC14g0CkbyhCaz9FkhxqNsZI6uAPJXFS8cMXlBKo26FJ8ZuW6S9GCMcR9IO5k2X5/9Fg== + dependencies: + cross-spawn "^7.0.1" + "@malept/flatpak-bundler@^0.4.0": version "0.4.0" resolved "https://registry.yarnpkg.com/@malept/flatpak-bundler/-/flatpak-bundler-0.4.0.tgz#e8a32c30a95d20c2b1bb635cc580981a06389858" @@ -1401,6 +1413,22 @@ "@nodelib/fs.scandir" "2.1.5" fastq "^1.6.0" +"@npmcli/fs@^2.1.0": + version "2.1.2" + resolved "https://registry.yarnpkg.com/@npmcli/fs/-/fs-2.1.2.tgz#a9e2541a4a2fec2e69c29b35e6060973da79b865" + integrity sha512-yOJKRvohFOaLqipNtwYB9WugyZKhC/DZC4VYPmpaCzDBrA8YpK3qHZ8/HGscMnE4GqbkLNuVcCnxkeQEdGt6LQ== + dependencies: + "@gar/promisify" "^1.1.3" + semver "^7.3.5" + +"@npmcli/move-file@^2.0.0": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@npmcli/move-file/-/move-file-2.0.1.tgz#26f6bdc379d87f75e55739bab89db525b06100e4" + integrity sha512-mJd2Z5TjYWq/ttPLLGqArdtnC74J6bOzg4rMDnN+p1xTacZ2yPRCk2y0oSWQtygLR9YVQXgOcONrwtnk3JupxQ== + dependencies: + mkdirp "^1.0.4" + rimraf "^3.0.2" + "@sinclair/typebox@^0.25.16": version "0.25.24" resolved "https://registry.yarnpkg.com/@sinclair/typebox/-/typebox-0.25.24.tgz#8c7688559979f7079aacaf31aa881c3aa410b718" @@ -1411,6 +1439,11 @@ resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-0.14.0.tgz#9fb3a3cf3132328151f353de4632e01e52102bea" integrity sha512-9NET910DNaIPngYnLLPeg+Ogzqsi9uM4mSboU5y6p8S5DzMTVEsJZrawi+BoDNUVBa2DhJqQYUFvMDfgU062LQ== +"@sindresorhus/is@^4.0.0": + version "4.6.0" + resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-4.6.0.tgz#3c7c9c46e678feefe7a2e5bb609d3dbd665ffb3f" + integrity sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw== + "@sinonjs/commons@^3.0.0": version "3.0.0" resolved "https://registry.yarnpkg.com/@sinonjs/commons/-/commons-3.0.0.tgz#beb434fe875d965265e04722ccfc21df7f755d72" @@ -1538,6 +1571,13 @@ dependencies: defer-to-connect "^1.0.1" +"@szmarczak/http-timer@^4.0.5": + version "4.0.6" + resolved "https://registry.yarnpkg.com/@szmarczak/http-timer/-/http-timer-4.0.6.tgz#b4a914bb62e7c272d4e5989fe4440f812ab1d807" + integrity sha512-4BAffykYOgO+5nzBWYwE3W90sBgLJoUPRWWcL8wlyiM8IB8ipJz3UMJ9KXQd1RKQXpKp8Tutn80HZtWsu2u76w== + dependencies: + defer-to-connect "^2.0.0" + "@tootallnate/once@2": version "2.0.0" resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-2.0.0.tgz#f544a148d3ab35801c1f633a7441fd87c2e484bf" @@ -1586,6 +1626,23 @@ resolved "https://registry.yarnpkg.com/@types/base64-js/-/base64-js-1.3.0.tgz#c939fdba49846861caf5a246b165dbf5698a317c" integrity sha512-ZmI0sZGAUNXUfMWboWwi4LcfpoVUYldyN6Oe0oJ5cCsHDU/LlRq8nQKPXhYLOx36QYSW9bNIb1vvRrD6K7Llgw== +"@types/better-sqlite3@^7.6.13": + version "7.6.13" + resolved "https://registry.yarnpkg.com/@types/better-sqlite3/-/better-sqlite3-7.6.13.tgz#a72387f00d2f53cab699e63f2e2c05453cf953f0" + integrity sha512-NMv9ASNARoKksWtsq/SHakpYAYnhBrQgGD8zkLYk/jaK8jUGn08CfEdTRgYhMypUQAfzSP8W6gNLe0q19/t4VA== + dependencies: + "@types/node" "*" + +"@types/cacheable-request@^6.0.1": + version "6.0.3" + resolved "https://registry.yarnpkg.com/@types/cacheable-request/-/cacheable-request-6.0.3.tgz#a430b3260466ca7b5ca5bfd735693b36e7a9d183" + integrity sha512-IQ3EbTzGxIigb1I3qPZc1rWJnH0BmSKv5QYTalEwweFvyBDLSAe24zP0le/hyi7ecGfZVlIVAg4BZqb8WBwKqw== + dependencies: + "@types/http-cache-semantics" "*" + "@types/keyv" "^3.1.4" + "@types/node" "*" + "@types/responselike" "^1.0.0" + "@types/chrome@^0.0.195": version "0.0.195" resolved "https://registry.yarnpkg.com/@types/chrome/-/chrome-0.0.195.tgz#c570f72857e11caa0a453a444828f75bca592cff" @@ -1677,6 +1734,11 @@ resolved "https://registry.yarnpkg.com/@types/html-minifier-terser/-/html-minifier-terser-6.1.0.tgz#4fc33a00c1d0c16987b1a20cf92d20614c55ac35" integrity sha512-oh/6byDPnL1zeNXFrDXFLyZjkr1MsBG667IM792caf1L2UPOOMf65NFzjUH/ltyfwjAGfs1rsX1eftK0jC/KIg== +"@types/http-cache-semantics@*": + version "4.0.4" + resolved "https://registry.yarnpkg.com/@types/http-cache-semantics/-/http-cache-semantics-4.0.4.tgz#b979ebad3919799c979b17c72621c0bc0a31c6c4" + integrity sha512-1m0bIFVc7eJWyve9S0RnuRgcQqF/Xd5QsUZAZeQFr1Q3/p9JWoQQEqmVy+DPTNpGXwhgIetAoYF8JSc33q29QA== + "@types/istanbul-lib-coverage@*", "@types/istanbul-lib-coverage@^2.0.0", "@types/istanbul-lib-coverage@^2.0.1": version "2.0.4" resolved "https://registry.yarnpkg.com/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.4.tgz#8467d4b3c087805d63580480890791277ce35c44" @@ -1716,6 +1778,13 @@ dependencies: "@types/node" "*" +"@types/keyv@^3.1.4": + version "3.1.4" + resolved "https://registry.yarnpkg.com/@types/keyv/-/keyv-3.1.4.tgz#3ccdb1c6751b0c7e52300bcdacd5bcbf8faa75b6" + integrity sha512-BQ5aZNSCpj7D6K2ksrRCTmKRLEpnPvWDiLPfoGyhZ++8YtiK9d/3DBKPJgry359X/P1PfruyYwvnvwFjuEiEIg== + dependencies: + "@types/node" "*" + "@types/minimatch@*": version "3.0.4" resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.4.tgz#f0ec25dbf2f0e4b18647313ac031134ca5b24b21" @@ -1787,6 +1856,13 @@ "@types/scheduler" "*" csstype "^3.0.2" +"@types/responselike@^1.0.0": + version "1.0.3" + resolved "https://registry.yarnpkg.com/@types/responselike/-/responselike-1.0.3.tgz#cc29706f0a397cfe6df89debfe4bf5cea159db50" + integrity sha512-H/+L+UkTV33uf49PH5pCAUBVPNj2nDBXTN+qS1dOwyyg24l3CcicicCA7ca+HMvJBZcFgl5r8e+RR6elsb4Lyw== + dependencies: + "@types/node" "*" + "@types/scheduler@*": version "0.16.2" resolved "https://registry.yarnpkg.com/@types/scheduler/-/scheduler-0.16.2.tgz#1a62f89525723dde24ba1b01b092bf5df8ad4d39" @@ -2068,6 +2144,11 @@ resolved "https://registry.yarnpkg.com/@xtuc/long/-/long-4.2.2.tgz#d291c6a4e97989b5c61d9acf396ae4fe133a718d" integrity sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ== +abbrev@^1.0.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8" + integrity sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q== + acorn-import-assertions@^1.7.6: version "1.8.0" resolved "https://registry.yarnpkg.com/acorn-import-assertions/-/acorn-import-assertions-1.8.0.tgz#ba2b5939ce62c238db6d93d81c9b111b29b855e9" @@ -2093,13 +2174,28 @@ ag-psd@^15.0.0: base64-js "^1.5.1" pako "^2.0.4" -agent-base@6: +agent-base@6, agent-base@^6.0.2: version "6.0.2" resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-6.0.2.tgz#49fff58577cfee3f37176feab4c22e00f86d7f77" integrity sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ== dependencies: debug "4" +agentkeepalive@^4.2.1: + version "4.6.0" + resolved "https://registry.yarnpkg.com/agentkeepalive/-/agentkeepalive-4.6.0.tgz#35f73e94b3f40bf65f105219c623ad19c136ea6a" + integrity sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ== + dependencies: + humanize-ms "^1.2.1" + +aggregate-error@^3.0.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/aggregate-error/-/aggregate-error-3.1.0.tgz#92670ff50f5359bdb7a3e0d40d0ec30c5737687a" + integrity sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA== + dependencies: + clean-stack "^2.0.0" + indent-string "^4.0.0" + ajv-formats@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/ajv-formats/-/ajv-formats-2.1.1.tgz#6e669400659eb74973bbf2e33327180a0996b520" @@ -2223,6 +2319,19 @@ app-builder-lib@23.6.0: tar "^6.1.11" temp-file "^3.4.0" +"aproba@^1.0.3 || ^2.0.0": + version "2.1.0" + resolved "https://registry.yarnpkg.com/aproba/-/aproba-2.1.0.tgz#75500a190313d95c64e871e7e4284c6ac219f0b1" + integrity sha512-tLIEcj5GuR2RSTnxNKdkK0dJ/GrC7P38sUkiDmDuHfsHmbagTFAxDVIBltoklXEVIQ/f14IL8IMJ5pn9Hez1Ew== + +are-we-there-yet@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/are-we-there-yet/-/are-we-there-yet-3.0.1.tgz#679df222b278c64f2cdba1175cdc00b0d96164bd" + integrity sha512-QZW4EDmGwlYur0Yyf/b2uGucHQMa8aFUP7eu9ddR73vvhFyt4V0Vl3QHPcTNJ8l6qYOBdxgXdnBXQrHilfRQBg== + dependencies: + delegates "^1.0.0" + readable-stream "^3.6.0" + argparse@^1.0.7: version "1.0.10" resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911" @@ -2418,10 +2527,10 @@ base64-js@^1.3.1, base64-js@^1.5.1: resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== -better-sqlite3@^12.4.1: - version "12.4.1" - resolved "https://registry.yarnpkg.com/better-sqlite3/-/better-sqlite3-12.4.1.tgz#f78df6c80530d1a0b750b538033e6199b7d30d26" - integrity sha512-3yVdyZhklTiNrtg+4WqHpJpFDd+WHTg2oM7UcR80GqL05AOV0xEJzc6qNvFYoEtE+hRp1n9MpN6/+4yhlGkDXQ== +better-sqlite3@9.6.0: + version "9.6.0" + resolved "https://registry.yarnpkg.com/better-sqlite3/-/better-sqlite3-9.6.0.tgz#b01e58ba7c48abcdc0383b8301206ee2ab81d271" + integrity sha512-yR5HATnqeYNVnkaUTf4bOP2dJSnyhP4puJN/QPRyx4YkBEEUxib422n2XzPqDEHjQQqazoYoADdAm5vE15+dAQ== dependencies: bindings "^1.5.0" prebuild-install "^7.1.1" @@ -2443,7 +2552,7 @@ bindings@^1.5.0: dependencies: file-uri-to-path "1.0.0" -bl@^4.0.3: +bl@^4.0.3, bl@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/bl/-/bl-4.1.0.tgz#451535264182bec2fbbc83a62ab98cf11d9f7b3a" integrity sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w== @@ -2592,6 +2701,35 @@ builder-util@23.6.0: stat-mode "^1.0.0" temp-file "^3.4.0" +cacache@^16.1.0: + version "16.1.3" + resolved "https://registry.yarnpkg.com/cacache/-/cacache-16.1.3.tgz#a02b9f34ecfaf9a78c9f4bc16fceb94d5d67a38e" + integrity sha512-/+Emcj9DAXxX4cwlLmRI9c166RuL3w30zp4R7Joiv2cQTtTtA+jeuCAjH3ZlGnYS3tKENSrKhAzVVP9GVyzeYQ== + dependencies: + "@npmcli/fs" "^2.1.0" + "@npmcli/move-file" "^2.0.0" + chownr "^2.0.0" + fs-minipass "^2.1.0" + glob "^8.0.1" + infer-owner "^1.0.4" + lru-cache "^7.7.1" + minipass "^3.1.6" + minipass-collect "^1.0.2" + minipass-flush "^1.0.5" + minipass-pipeline "^1.2.4" + mkdirp "^1.0.4" + p-map "^4.0.0" + promise-inflight "^1.0.1" + rimraf "^3.0.2" + ssri "^9.0.0" + tar "^6.1.11" + unique-filename "^2.0.0" + +cacheable-lookup@^5.0.3: + version "5.0.4" + resolved "https://registry.yarnpkg.com/cacheable-lookup/-/cacheable-lookup-5.0.4.tgz#5a6b865b2c44357be3d5ebc2a467b032719a7005" + integrity sha512-2/kNscPhpcxrOigMZzbiWF7dz8ilhb/nIHU3EyZiXWXpeq/au8qJ8VhdftMkty3n7Gj6HIGalQG8oiBNB3AJgA== + cacheable-request@^6.0.0: version "6.1.0" resolved "https://registry.yarnpkg.com/cacheable-request/-/cacheable-request-6.1.0.tgz#20ffb8bd162ba4be11e9567d823db651052ca912" @@ -2605,6 +2743,19 @@ cacheable-request@^6.0.0: normalize-url "^4.1.0" responselike "^1.0.2" +cacheable-request@^7.0.2: + version "7.0.4" + resolved "https://registry.yarnpkg.com/cacheable-request/-/cacheable-request-7.0.4.tgz#7a33ebf08613178b403635be7b899d3e69bbe817" + integrity sha512-v+p6ongsrp0yTGbJXjgxPow2+DL93DASP4kXCDKb8/bwRtt9OEF3whggkkDkGNzgcWy2XaF4a8nZglC7uElscg== + dependencies: + clone-response "^1.0.2" + get-stream "^5.1.0" + http-cache-semantics "^4.0.0" + keyv "^4.0.0" + lowercase-keys "^2.0.0" + normalize-url "^6.0.1" + responselike "^2.0.0" + call-bind@^1.0.0, call-bind@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.2.tgz#b1d4e89e688119c3c9a903ad30abb2f6a919be3c" @@ -2720,6 +2871,23 @@ clean-css@^5.2.2: dependencies: source-map "~0.6.0" +clean-stack@^2.0.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/clean-stack/-/clean-stack-2.2.0.tgz#ee8472dbb129e727b31e8a10a427dee9dfe4008b" + integrity sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A== + +cli-cursor@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/cli-cursor/-/cli-cursor-3.1.0.tgz#264305a7ae490d1d03bf0c9ba7c925d1753af307" + integrity sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw== + dependencies: + restore-cursor "^3.1.0" + +cli-spinners@^2.5.0: + version "2.9.2" + resolved "https://registry.yarnpkg.com/cli-spinners/-/cli-spinners-2.9.2.tgz#1773a8f4b9c4d6ac31563df53b3fc1d79462fe41" + integrity sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg== + cli-truncate@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/cli-truncate/-/cli-truncate-2.1.0.tgz#c39e28bf05edcde5be3b98992a22deed5a2b93c7" @@ -2753,6 +2921,11 @@ clone-response@^1.0.2: dependencies: mimic-response "^1.0.0" +clone@^1.0.2: + version "1.0.4" + resolved "https://registry.yarnpkg.com/clone/-/clone-1.0.4.tgz#da309cc263df15994c688ca902179ca3c7cd7c7e" + integrity sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg== + co@^4.6.0: version "4.6.0" resolved "https://registry.yarnpkg.com/co/-/co-4.6.0.tgz#6ea6bdf3d853ae54ccb8e47bfa0bf3f9031fb184" @@ -2787,6 +2960,11 @@ color-name@~1.1.4: resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== +color-support@^1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/color-support/-/color-support-1.1.3.tgz#93834379a1cc9a0c61f82f52f0d04322251bd5a2" + integrity sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg== + colorette@^2.0.14: version "2.0.19" resolved "https://registry.yarnpkg.com/colorette/-/colorette-2.0.19.tgz#cdf044f47ad41a0f4b56b3a0d5b4e6e1a2d5a798" @@ -2859,6 +3037,11 @@ config-chain@^1.1.11: ini "^1.3.4" proto-list "~1.2.1" +console-control-strings@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/console-control-strings/-/console-control-strings-1.1.0.tgz#3d7cf4464db6446ea644bf4b39507f9851008e8e" + integrity sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ== + convert-source-map@^1.6.0, convert-source-map@^1.7.0: version "1.9.0" resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-1.9.0.tgz#7faae62353fb4213366d0ca98358d22e8368b05f" @@ -2977,6 +3160,13 @@ debug@^2.6.8: dependencies: ms "2.0.0" +debug@^4.3.3: + version "4.4.3" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.4.3.tgz#c6ae432d9bd9662582fce08709b038c58e9e3d6a" + integrity sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA== + dependencies: + ms "^2.1.3" + decompress-response@^3.3.0: version "3.3.0" resolved "https://registry.yarnpkg.com/decompress-response/-/decompress-response-3.3.0.tgz#80a4dd323748384bfa248083622aedec982adff3" @@ -3011,11 +3201,23 @@ deepmerge@^4.2.2: resolved "https://registry.yarnpkg.com/deepmerge/-/deepmerge-4.2.2.tgz#44d2ea3679b8f4d4ffba33f03d865fc1e7bf4955" integrity sha512-FJ3UgI4gIl+PHZm53knsuSFpE+nESMr7M4v9QcgB7S63Kj/6WqMiFQJpBBYz1Pt+66bZpP3Q7Lye0Oo9MPKEdg== +defaults@^1.0.3: + version "1.0.4" + resolved "https://registry.yarnpkg.com/defaults/-/defaults-1.0.4.tgz#b0b02062c1e2aa62ff5d9528f0f98baa90978d7a" + integrity sha512-eFuaLoy/Rxalv2kr+lqMlUnrDWV+3j4pljOIJgLIhI058IQfWJ7vXhyEIHu+HtC738klGALYxOKDO0bQP3tg8A== + dependencies: + clone "^1.0.2" + defer-to-connect@^1.0.1: version "1.1.3" resolved "https://registry.yarnpkg.com/defer-to-connect/-/defer-to-connect-1.1.3.tgz#331ae050c08dcf789f8c83a7b81f0ed94f4ac591" integrity sha512-0ISdNousHvZT2EiFlZeZAHBUvSxmKswVCEf8hW7KWgG4a8MVEu/3Vb6uWYozkjylyCxe0JBIiRB1jV45S70WVQ== +defer-to-connect@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/defer-to-connect/-/defer-to-connect-2.0.1.tgz#8016bdb4143e4632b77a3449c6236277de520587" + integrity sha512-4tvttepXG1VaYGrRibk5EwJd1t4udunSOVMdLSAL6mId1ix438oPwPZMALY41FCijukO1L0twNcGsdzS7dHgDg== + define-properties@^1.1.3, define-properties@^1.1.4: version "1.1.4" resolved "https://registry.yarnpkg.com/define-properties/-/define-properties-1.1.4.tgz#0b14d7bd7fbeb2f3572c3a7eda80ea5d57fb05b1" @@ -3029,7 +3231,12 @@ delayed-stream@~1.0.0: resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" integrity sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ== -detect-libc@^2.0.0: +delegates@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a" + integrity sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ== + +detect-libc@^2.0.0, detect-libc@^2.0.1: version "2.1.1" resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-2.1.1.tgz#9f1e511ace6bb525efea4651345beac424dac7b9" integrity sha512-ecqj/sy1jcK1uWrwpR67UhYrIFQ+5WlGxth34WquCbamhFA6hkkwiu37o6J5xCHdo1oixJRfVRw+ywV+Hq/0Aw== @@ -3235,6 +3442,26 @@ electron-publish@23.6.0: lazy-val "^1.0.5" mime "^2.5.2" +electron-rebuild@^3.2.9: + version "3.2.9" + resolved "https://registry.yarnpkg.com/electron-rebuild/-/electron-rebuild-3.2.9.tgz#ea372be15f591f8d6d978ee9bca6526dadbcf20f" + integrity sha512-FkEZNFViUem3P0RLYbZkUjC8LUFIK+wKq09GHoOITSJjfDAVQv964hwaNseTTWt58sITQX3/5fHNYcTefqaCWw== + dependencies: + "@malept/cross-spawn-promise" "^2.0.0" + chalk "^4.0.0" + debug "^4.1.1" + detect-libc "^2.0.1" + fs-extra "^10.0.0" + got "^11.7.0" + lzma-native "^8.0.5" + node-abi "^3.0.0" + node-api-version "^0.1.4" + node-gyp "^9.0.0" + ora "^5.1.0" + semver "^7.3.5" + tar "^6.0.5" + yargs "^17.0.1" + electron-to-chromium@^1.4.251: version "1.4.284" resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.284.tgz#61046d1e4cab3a25238f6bf7413795270f125592" @@ -3284,6 +3511,13 @@ encodeurl@^1.0.2: resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59" integrity sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k= +encoding@^0.1.13: + version "0.1.13" + resolved "https://registry.yarnpkg.com/encoding/-/encoding-0.1.13.tgz#56574afdd791f54a8e9b2785c0582a2d26210fa9" + integrity sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A== + dependencies: + iconv-lite "^0.6.2" + end-of-stream@^1.1.0: version "1.4.4" resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0" @@ -3326,6 +3560,11 @@ envinfo@^7.7.3: resolved "https://registry.yarnpkg.com/envinfo/-/envinfo-7.8.1.tgz#06377e3e5f4d379fea7ac592d5ad8927e0c4d475" integrity sha512-/o+BXHmB7ocbHEAs6F2EnG0ogybVVUdkRunTT2glZU9XAaGmhqskrvKwqXuDfNjEO0LZKWdejEEpnq8aM0tOaw== +err-code@^2.0.2: + version "2.0.3" + resolved "https://registry.yarnpkg.com/err-code/-/err-code-2.0.3.tgz#23c2f3b756ffdfc608d30e27c9a941024807e7f9" + integrity sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA== + error-ex@^1.3.1: version "1.3.2" resolved "https://registry.yarnpkg.com/error-ex/-/error-ex-1.3.2.tgz#b4ac40648107fdcdcfae242f428bea8a14d4f1bf" @@ -3626,6 +3865,11 @@ expect@^29.0.0, expect@^29.5.0: jest-message-util "^29.5.0" jest-util "^29.5.0" +exponential-backoff@^3.1.1: + version "3.1.2" + resolved "https://registry.yarnpkg.com/exponential-backoff/-/exponential-backoff-3.1.2.tgz#a8f26adb96bf78e8cd8ad1037928d5e5c0679d91" + integrity sha512-8QxYTVXUkuy7fIIoitQkPwGonB8F3Zj8eEO8Sqg9Zv/bkI7RJAzowee4gr81Hak/dUTpA2Z7VfQgoijjPNlUZA== + extract-zip@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/extract-zip/-/extract-zip-2.0.1.tgz#663dca56fe46df890d5f131ef4a06d22bb8ba13a" @@ -3819,7 +4063,7 @@ fs-extra@^9.0.0, fs-extra@^9.0.1: jsonfile "^6.0.1" universalify "^2.0.0" -fs-minipass@^2.0.0: +fs-minipass@^2.0.0, fs-minipass@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/fs-minipass/-/fs-minipass-2.1.0.tgz#7f5036fdbf12c63c169190cbe4199c852271f9fb" integrity sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg== @@ -3856,6 +4100,20 @@ functions-have-names@^1.2.2: resolved "https://registry.yarnpkg.com/functions-have-names/-/functions-have-names-1.2.3.tgz#0404fe4ee2ba2f607f0e0ec3c80bae994133b834" integrity sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ== +gauge@^4.0.3: + version "4.0.4" + resolved "https://registry.yarnpkg.com/gauge/-/gauge-4.0.4.tgz#52ff0652f2bbf607a989793d53b751bef2328dce" + integrity sha512-f9m+BEN5jkg6a0fZjleidjN51VE1X+mPFQ2DJ0uv1V39oCLCbsGe6yjbBnp7eK7z/+GAon99a3nHuqbuuthyPg== + dependencies: + aproba "^1.0.3 || ^2.0.0" + color-support "^1.1.3" + console-control-strings "^1.1.0" + has-unicode "^2.0.1" + signal-exit "^3.0.7" + string-width "^4.2.3" + strip-ansi "^6.0.1" + wide-align "^1.1.5" + gensync@^1.0.0-beta.2: version "1.0.0-beta.2" resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0" @@ -3943,6 +4201,17 @@ glob@^7.1.3, glob@^7.1.4, glob@^7.1.6: once "^1.3.0" path-is-absolute "^1.0.0" +glob@^8.0.1: + version "8.1.0" + resolved "https://registry.yarnpkg.com/glob/-/glob-8.1.0.tgz#d388f656593ef708ee3e34640fdfb99a9fd1c33e" + integrity sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ== + dependencies: + fs.realpath "^1.0.0" + inflight "^1.0.4" + inherits "2" + minimatch "^5.0.1" + once "^1.3.0" + global-agent@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/global-agent/-/global-agent-3.0.0.tgz#ae7cd31bd3583b93c5a16437a1afe27cc33a1ab6" @@ -4010,6 +4279,23 @@ gopd@^1.0.1: dependencies: get-intrinsic "^1.1.3" +got@^11.7.0: + version "11.8.6" + resolved "https://registry.yarnpkg.com/got/-/got-11.8.6.tgz#276e827ead8772eddbcfc97170590b841823233a" + integrity sha512-6tfZ91bOr7bOXnK7PRDCGBLa1H4U080YHNaAQ2KsMGlLEzRbk44nsZF2E1IeRc3vtJHPVbKCYgdFbaGO2ljd8g== + dependencies: + "@sindresorhus/is" "^4.0.0" + "@szmarczak/http-timer" "^4.0.5" + "@types/cacheable-request" "^6.0.1" + "@types/responselike" "^1.0.0" + cacheable-lookup "^5.0.3" + cacheable-request "^7.0.2" + decompress-response "^6.0.0" + http2-wrapper "^1.0.0-beta.5.2" + lowercase-keys "^2.0.0" + p-cancelable "^2.0.0" + responselike "^2.0.0" + got@^9.6.0: version "9.6.0" resolved "https://registry.yarnpkg.com/got/-/got-9.6.0.tgz#edf45e7d67f99545705de1f7bbeeeb121765ed85" @@ -4032,6 +4318,11 @@ graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0, graceful-fs@^4.2.4, resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.10.tgz#147d3a006da4ca3ce14728c7aefc287c367d7a6c" integrity sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA== +graceful-fs@^4.2.6: + version "4.2.11" + resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.11.tgz#4183e4e8bf08bb6e05bbb2f7d2e0c8f712ca40e3" + integrity sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ== + "graceful-readlink@>= 1.0.0": version "1.0.1" resolved "https://registry.yarnpkg.com/graceful-readlink/-/graceful-readlink-1.0.1.tgz#4cafad76bc62f02fa039b2f94e9a3dd3a391a725" @@ -4081,6 +4372,11 @@ has-tostringtag@^1.0.0: dependencies: has-symbols "^1.0.2" +has-unicode@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/has-unicode/-/has-unicode-2.0.1.tgz#e0e6fe6a28cf51138855e086d1691e771de2a8b9" + integrity sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ== + has@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/has/-/has-1.0.3.tgz#722d7cbfc1f6aa8241f16dd814e011e1f41e8796" @@ -4144,6 +4440,11 @@ http-cache-semantics@^4.0.0: resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz#abe02fcb2985460bf0323be664436ec3476a6d5a" integrity sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ== +http-cache-semantics@^4.1.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.2.0.tgz#205f4db64f8562b76a4ff9235aa5279839a09dd5" + integrity sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ== + http-proxy-agent@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz#5129800203520d434f142bc78ff3c170800f2b43" @@ -4153,6 +4454,14 @@ http-proxy-agent@^5.0.0: agent-base "6" debug "4" +http2-wrapper@^1.0.0-beta.5.2: + version "1.0.3" + resolved "https://registry.yarnpkg.com/http2-wrapper/-/http2-wrapper-1.0.3.tgz#b8f55e0c1f25d4ebd08b3b0c2c079f9590800b3d" + integrity sha512-V+23sDMr12Wnz7iTcDeJr3O6AIxlnvT/bmaAAAP/Xda35C90p9599p0F1eHR/N1KILWSoWVAiOMFjBBXaXSMxg== + dependencies: + quick-lru "^5.1.1" + resolve-alpn "^1.0.0" + https-proxy-agent@^5.0.0: version "5.0.1" resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz#c59ef224a04fe8b754f3db0063a25ea30d0005d6" @@ -4166,6 +4475,13 @@ human-signals@^2.1.0: resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-2.1.0.tgz#dc91fcba42e4d06e4abaed33b3e7a3c02f514ea0" integrity sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw== +humanize-ms@^1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/humanize-ms/-/humanize-ms-1.2.1.tgz#c46e3159a293f6b896da29316d8b6fe8bb79bbed" + integrity sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ== + dependencies: + ms "^2.0.0" + iconv-corefoundation@^1.1.7: version "1.1.7" resolved "https://registry.yarnpkg.com/iconv-corefoundation/-/iconv-corefoundation-1.1.7.tgz#31065e6ab2c9272154c8b0821151e2c88f1b002a" @@ -4222,6 +4538,16 @@ imurmurhash@^0.1.4: resolved "https://registry.yarnpkg.com/imurmurhash/-/imurmurhash-0.1.4.tgz#9218b9b2b928a238b13dc4fb6b6d576f231453ea" integrity sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA== +indent-string@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/indent-string/-/indent-string-4.0.0.tgz#624f8f4497d619b2d9768531d58f4122854d7251" + integrity sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg== + +infer-owner@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/infer-owner/-/infer-owner-1.0.4.tgz#c4cefcaa8e51051c2a40ba2ce8a3d27295af9467" + integrity sha512-IClj+Xz94+d7irH5qRyfJonOdfTzuDaifE6ZPWfx0N0+/ATZCbuTPq2prFl526urkQd90WyUKIh1DfBQ2hMz9A== + inflight@^1.0.4: version "1.0.6" resolved "https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9" @@ -4254,6 +4580,11 @@ interpret@^3.1.1: resolved "https://registry.yarnpkg.com/interpret/-/interpret-3.1.1.tgz#5be0ceed67ca79c6c4bc5cf0d7ee843dcea110c4" integrity sha512-6xwYfHbajpoF0xLW+iwLkhwgvLoZDfjYfoFNu8ftMoXINzwuymNLd9u/KmwtdT2GbR+/Cz66otEGEVVUHX9QLQ== +ip-address@^10.0.1: + version "10.0.1" + resolved "https://registry.yarnpkg.com/ip-address/-/ip-address-10.0.1.tgz#a8180b783ce7788777d796286d61bce4276818ed" + integrity sha512-NWv9YLW4PoW2B7xtzaS3NCot75m6nK7Icdv0o3lfMceJVRfSoQwqD4wEH5rLwoKJwUiZ/rfpiVBhnaF0FK4HoA== + is-array-buffer@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/is-array-buffer/-/is-array-buffer-3.0.1.tgz#deb1db4fcae48308d54ef2442706c0393997052a" @@ -4338,6 +4669,16 @@ is-glob@^4.0.0, is-glob@^4.0.1, is-glob@^4.0.3, is-glob@~4.0.1: dependencies: is-extglob "^2.1.1" +is-interactive@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-interactive/-/is-interactive-1.0.0.tgz#cea6e6ae5c870a7b0a0004070b7b587e0252912e" + integrity sha512-2HvIEKRoqS62guEC+qBjpvRubdX910WCMuJTZ+I9yvqKU2/12eSL549HMwtabb4oupdj2sMP50k+XJfB/8JE6w== + +is-lambda@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/is-lambda/-/is-lambda-1.0.1.tgz#3d9877899e6a53efc0160504cde15f82e6f061d5" + integrity sha512-z7CMFGNrENq5iFB9Bqo64Xk6Y9sg+epq1myIcdHaGnbMTYOxvzsEtdYqQUylB7LxfkvgrrjP32T6Ywciio9UIQ== + is-negative-zero@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/is-negative-zero/-/is-negative-zero-2.0.2.tgz#7bf6f03a28003b8b3965de3ac26f664d765f3150" @@ -4417,6 +4758,11 @@ is-typed-array@^1.1.10, is-typed-array@^1.1.9: gopd "^1.0.1" has-tostringtag "^1.0.0" +is-unicode-supported@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz#3f26c76a809593b52bfa2ecb5710ed2779b522a7" + integrity sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw== + is-weakref@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/is-weakref/-/is-weakref-1.0.2.tgz#9529f383a9338205e89765e0392efc2f100f06f2" @@ -4908,6 +5254,11 @@ json-buffer@3.0.0: resolved "https://registry.yarnpkg.com/json-buffer/-/json-buffer-3.0.0.tgz#5b1f397afc75d677bde8bcfc0e47e1f9a3d9a898" integrity sha1-Wx85evx11ne96Lz8Dkfh+aPZqJg= +json-buffer@3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/json-buffer/-/json-buffer-3.0.1.tgz#9338802a30d3b6605fbe0613e094008ca8c05a13" + integrity sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ== + json-parse-even-better-errors@^2.3.0, json-parse-even-better-errors@^2.3.1: version "2.3.1" resolved "https://registry.yarnpkg.com/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz#7c47805a94319928e05777405dc12e1f7a4ee02d" @@ -4969,6 +5320,13 @@ keyv@^3.0.0: dependencies: json-buffer "3.0.0" +keyv@^4.0.0: + version "4.5.4" + resolved "https://registry.yarnpkg.com/keyv/-/keyv-4.5.4.tgz#a879a99e29452f942439f2a405e3af8b31d4de93" + integrity sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw== + dependencies: + json-buffer "3.0.1" + kind-of@^6.0.2: version "6.0.3" resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-6.0.3.tgz#07c05034a6c349fa06e24fa35aa76db4580ce4dd" @@ -5070,6 +5428,14 @@ lodash@^4.17.10, lodash@^4.17.15, lodash@^4.17.20, lodash@^4.17.21, lodash@^4.7. resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== +log-symbols@^4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/log-symbols/-/log-symbols-4.1.0.tgz#3fbdbb95b4683ac9fc785111e792e558d4abd503" + integrity sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg== + dependencies: + chalk "^4.1.0" + is-unicode-supported "^0.1.0" + loose-envify@^1.1.0, loose-envify@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/loose-envify/-/loose-envify-1.4.0.tgz#71ee51fa7be4caec1a63839f7e682d8132d30caf" @@ -5108,6 +5474,20 @@ lru-cache@^6.0.0: dependencies: yallist "^4.0.0" +lru-cache@^7.7.1: + version "7.18.3" + resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-7.18.3.tgz#f793896e0fd0e954a59dfdd82f0773808df6aa89" + integrity sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA== + +lzma-native@^8.0.5: + version "8.0.6" + resolved "https://registry.yarnpkg.com/lzma-native/-/lzma-native-8.0.6.tgz#3ea456209d643bafd9b5d911781bdf0b396b2665" + integrity sha512-09xfg67mkL2Lz20PrrDeNYZxzeW7ADtpYFbwSQh9U8+76RIzx5QsJBMy8qikv3hbUPfpy6hqwxt6FcGK81g9AA== + dependencies: + node-addon-api "^3.1.0" + node-gyp-build "^4.2.1" + readable-stream "^3.6.0" + make-dir@^3.0.0: version "3.1.0" resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-3.1.0.tgz#415e967046b3a7f1d185277d84aa58203726a13f" @@ -5120,6 +5500,28 @@ make-error@1.x: resolved "https://registry.yarnpkg.com/make-error/-/make-error-1.3.6.tgz#2eb2e37ea9b67c4891f684a1394799af484cf7a2" integrity sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw== +make-fetch-happen@^10.0.3: + version "10.2.1" + resolved "https://registry.yarnpkg.com/make-fetch-happen/-/make-fetch-happen-10.2.1.tgz#f5e3835c5e9817b617f2770870d9492d28678164" + integrity sha512-NgOPbRiaQM10DYXvN3/hhGVI2M5MtITFryzBGxHM5p4wnFxsVCbxkrBrDsk+EZ5OB4jEOT7AjDxtdF+KVEFT7w== + dependencies: + agentkeepalive "^4.2.1" + cacache "^16.1.0" + http-cache-semantics "^4.1.0" + http-proxy-agent "^5.0.0" + https-proxy-agent "^5.0.0" + is-lambda "^1.0.1" + lru-cache "^7.7.1" + minipass "^3.1.6" + minipass-collect "^1.0.2" + minipass-fetch "^2.0.3" + minipass-flush "^1.0.5" + minipass-pipeline "^1.2.4" + negotiator "^0.6.3" + promise-retry "^2.0.1" + socks-proxy-agent "^7.0.0" + ssri "^9.0.0" + makeerror@1.0.12: version "1.0.12" resolved "https://registry.yarnpkg.com/makeerror/-/makeerror-1.0.12.tgz#3e5dd2079a82e812e983cc6610c4a2cb0eaa801a" @@ -5237,6 +5639,45 @@ minimist@^1.2.3: resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c" integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA== +minipass-collect@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/minipass-collect/-/minipass-collect-1.0.2.tgz#22b813bf745dc6edba2576b940022ad6edc8c617" + integrity sha512-6T6lH0H8OG9kITm/Jm6tdooIbogG9e0tLgpY6mphXSm/A9u8Nq1ryBG+Qspiub9LjWlBPsPS3tWQ/Botq4FdxA== + dependencies: + minipass "^3.0.0" + +minipass-fetch@^2.0.3: + version "2.1.2" + resolved "https://registry.yarnpkg.com/minipass-fetch/-/minipass-fetch-2.1.2.tgz#95560b50c472d81a3bc76f20ede80eaed76d8add" + integrity sha512-LT49Zi2/WMROHYoqGgdlQIZh8mLPZmOrN2NdJjMXxYe4nkN6FUyuPuOAOedNJDrx0IRGg9+4guZewtp8hE6TxA== + dependencies: + minipass "^3.1.6" + minipass-sized "^1.0.3" + minizlib "^2.1.2" + optionalDependencies: + encoding "^0.1.13" + +minipass-flush@^1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/minipass-flush/-/minipass-flush-1.0.5.tgz#82e7135d7e89a50ffe64610a787953c4c4cbb373" + integrity sha512-JmQSYYpPUqX5Jyn1mXaRwOda1uQ8HP5KAT/oDSLCzt1BYRhQU0/hDtsB1ufZfEEzMZ9aAVmsBw8+FWsIXlClWw== + dependencies: + minipass "^3.0.0" + +minipass-pipeline@^1.2.4: + version "1.2.4" + resolved "https://registry.yarnpkg.com/minipass-pipeline/-/minipass-pipeline-1.2.4.tgz#68472f79711c084657c067c5c6ad93cddea8214c" + integrity sha512-xuIq7cIOt09RPRJ19gdi4b+RiNvDFYe5JH+ggNvBqGqpQXcru3PcRmOZuHBKWK1Txf9+cQ+HMVN4d6z46LZP7A== + dependencies: + minipass "^3.0.0" + +minipass-sized@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/minipass-sized/-/minipass-sized-1.0.3.tgz#70ee5a7c5052070afacfbc22977ea79def353b70" + integrity sha512-MbkQQ2CTiBMlA2Dm/5cY+9SWFEN8pzzOXi6rlM5Xxq0Yqbda5ZQy9sU75a673FE9ZK0Zsbr6Y5iP6u9nktfg2g== + dependencies: + minipass "^3.0.0" + minipass@^3.0.0: version "3.3.4" resolved "https://registry.yarnpkg.com/minipass/-/minipass-3.3.4.tgz#ca99f95dd77c43c7a76bf51e6d200025eee0ffae" @@ -5244,7 +5685,19 @@ minipass@^3.0.0: dependencies: yallist "^4.0.0" -minizlib@^2.1.1: +minipass@^3.1.1, minipass@^3.1.6: + version "3.3.6" + resolved "https://registry.yarnpkg.com/minipass/-/minipass-3.3.6.tgz#7bba384db3a1520d18c9c0e5251c3444e95dd94a" + integrity sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw== + dependencies: + yallist "^4.0.0" + +minipass@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/minipass/-/minipass-5.0.0.tgz#3e9788ffb90b694a5d0ec94479a45b5d8738133d" + integrity sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ== + +minizlib@^2.1.1, minizlib@^2.1.2: version "2.1.2" resolved "https://registry.yarnpkg.com/minizlib/-/minizlib-2.1.2.tgz#e90d3466ba209b932451508a11ce3d3632145931" integrity sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg== @@ -5257,7 +5710,7 @@ mkdirp-classic@^0.5.2, mkdirp-classic@^0.5.3: resolved "https://registry.yarnpkg.com/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz#fa10c9115cc6d8865be221ba47ee9bed78601113" integrity sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A== -mkdirp@^1.0.3: +mkdirp@^1.0.3, mkdirp@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e" integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw== @@ -5282,6 +5735,11 @@ ms@2.1.2: resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== +ms@^2.0.0, ms@^2.1.3: + version "2.1.3" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" + integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== + nanoid@^3.3.4: version "3.3.4" resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.4.tgz#730b67e3cd09e2deacf03c027c81c9d9dbc5e8ab" @@ -5307,6 +5765,11 @@ ncp@^2.0.0: resolved "https://registry.yarnpkg.com/ncp/-/ncp-2.0.0.tgz#195a21d6c46e361d2fb1281ba38b91e9df7bdbb3" integrity sha1-GVoh1sRuNh0vsSgbo4uR6d9727M= +negotiator@^0.6.3: + version "0.6.4" + resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.4.tgz#777948e2452651c570b712dd01c23e262713fff7" + integrity sha512-myRT3DiWPHqho5PrJaIRyaMv2kgYf0mUVgBNOYMuCH5Ki1yEiQaf/ZJuQ62nvpc44wL5WDbTX7yGJi1Neevw8w== + neo-async@^2.6.2: version "2.6.2" resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.6.2.tgz#b4aafb93e3aeb2d8174ca53cf163ab7d7308305f" @@ -5320,7 +5783,7 @@ no-case@^3.0.4: lower-case "^2.0.2" tslib "^2.0.3" -node-abi@^3.3.0: +node-abi@^3.0.0, node-abi@^3.3.0: version "3.77.0" resolved "https://registry.yarnpkg.com/node-abi/-/node-abi-3.77.0.tgz#3ad90d5c9d45663420e5aa4ff58dbf4e3625419a" integrity sha512-DSmt0OEcLoK4i3NuscSbGjOf3bqiDEutejqENSplMSFA/gmB8mkED9G4pKWnPl7MDU4rSHebKPHeitpDfyH0cQ== @@ -5332,6 +5795,18 @@ node-addon-api@^1.6.3: resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-1.7.2.tgz#3df30b95720b53c24e59948b49532b662444f54d" integrity sha512-ibPK3iA+vaY1eEjESkQkM0BbCqFOaZMiXRTtdB0u7b4djtY6JnsjvPdUHVMg6xQt3B8fpTTWHI9A+ADjM9frzg== +node-addon-api@^3.1.0: + version "3.2.1" + resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-3.2.1.tgz#81325e0a2117789c0128dab65e7e38f07ceba161" + integrity sha512-mmcei9JghVNDYydghQmeDX8KoAm0FAiYyIcUt/N4nhyAipB17pllZQDOJD2fotxABnt4Mdz+dKTO7eftLg4d0A== + +node-api-version@^0.1.4: + version "0.1.4" + resolved "https://registry.yarnpkg.com/node-api-version/-/node-api-version-0.1.4.tgz#1ed46a485e462d55d66b5aa1fe2821720dedf080" + integrity sha512-KGXihXdUChwJAOHO53bv9/vXcLmdUsZ6jIptbvYvkpKfth+r7jw44JkVxQFA3kX5nQjzjmGu1uAu/xNNLNlI5g== + dependencies: + semver "^7.3.5" + node-exiftool@^2.3.0: version "2.3.0" resolved "https://registry.yarnpkg.com/node-exiftool/-/node-exiftool-2.3.0.tgz#d5142d34de6f1683b4655198b648e7e3ee6e80ac" @@ -5341,6 +5816,28 @@ node-exiftool@^2.3.0: restream "1.2.0" wrote "0.6.1" +node-gyp-build@^4.2.1: + version "4.8.4" + resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.8.4.tgz#8a70ee85464ae52327772a90d66c6077a900cfc8" + integrity sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ== + +node-gyp@^9.0.0: + version "9.4.1" + resolved "https://registry.yarnpkg.com/node-gyp/-/node-gyp-9.4.1.tgz#8a1023e0d6766ecb52764cc3a734b36ff275e185" + integrity sha512-OQkWKbjQKbGkMf/xqI1jjy3oCTgMKJac58G2+bjZb3fza6gW2YrCSdMQYaoTb70crvE//Gngr4f0AgVHmqHvBQ== + dependencies: + env-paths "^2.2.0" + exponential-backoff "^3.1.1" + glob "^7.1.4" + graceful-fs "^4.2.6" + make-fetch-happen "^10.0.3" + nopt "^6.0.0" + npmlog "^6.0.0" + rimraf "^3.0.2" + semver "^7.3.5" + tar "^6.1.2" + which "^2.0.2" + node-int64@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/node-int64/-/node-int64-0.4.0.tgz#87a9065cdb355d3182d8f94ce11188b825c68a3b" @@ -5363,6 +5860,13 @@ node-stream-zip@^1.15.0: resolved "https://registry.yarnpkg.com/node-stream-zip/-/node-stream-zip-1.15.0.tgz#158adb88ed8004c6c49a396b50a6a5de3bca33ea" integrity sha512-LN4fydt9TqhZhThkZIVQnF9cwjU3qmUH9h78Mx/K7d3VvfRqqwthLwJEUOEL0QPZ0XQmNN7be5Ggit5+4dq3Bw== +nopt@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/nopt/-/nopt-6.0.0.tgz#245801d8ebf409c6df22ab9d95b65e1309cdb16d" + integrity sha512-ZwLpbTgdhuZUnZzjd7nb1ZV+4DoiC6/sfiVKok72ym/4Tlf+DFdlHYmT2JPmcNNWV6Pi3SDf1kT+A4r9RTuT9g== + dependencies: + abbrev "^1.0.0" + normalize-path@^3.0.0, normalize-path@~3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-3.0.0.tgz#0dcd69ff23a1c9b11fd0978316644a0388216a65" @@ -5373,6 +5877,11 @@ normalize-url@^4.1.0: resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-4.5.1.tgz#0dd90cf1288ee1d1313b87081c9a5932ee48518a" integrity sha512-9UZCFRHQdNrfTpGg8+1INIg93B6zE0aXMVFkw1WFwvO4SlZywU6aLg5Of0Ap/PgcbSw4LNxvMWXMeugwMCX0AA== +normalize-url@^6.0.1: + version "6.1.0" + resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-6.1.0.tgz#40d0885b535deffe3f3147bec877d05fe4c5668a" + integrity sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A== + normalize.css@^8.0.1: version "8.0.1" resolved "https://registry.yarnpkg.com/normalize.css/-/normalize.css-8.0.1.tgz#9b98a208738b9cc2634caacbc42d131c97487bf3" @@ -5393,6 +5902,16 @@ npm-run-path@^4.0.1: dependencies: path-key "^3.0.0" +npmlog@^6.0.0: + version "6.0.2" + resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-6.0.2.tgz#c8166017a42f2dea92d6453168dd865186a70830" + integrity sha512-/vBvz5Jfr9dT/aFWd0FIRf+T/Q2WBsLENygUaFUqstqsycmZAP/t5BvFJTK0viFmSUxiUKTUplWy5vt+rvKIxg== + dependencies: + are-we-there-yet "^3.0.0" + console-control-strings "^1.1.0" + gauge "^4.0.3" + set-blocking "^2.0.0" + nth-check@^2.0.1: version "2.1.1" resolved "https://registry.yarnpkg.com/nth-check/-/nth-check-2.1.1.tgz#c9eab428effce36cd6b92c924bdb000ef1f1ed1d" @@ -5467,7 +5986,7 @@ once@^1.3.0, once@^1.3.1, once@^1.4.0: dependencies: wrappy "1" -onetime@^5.1.2: +onetime@^5.1.0, onetime@^5.1.2: version "5.1.2" resolved "https://registry.yarnpkg.com/onetime/-/onetime-5.1.2.tgz#d0e96ebb56b07476df1dd9c4806e5237985ca45e" integrity sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg== @@ -5486,11 +6005,31 @@ optionator@^0.9.1: type-check "^0.4.0" word-wrap "^1.2.3" +ora@^5.1.0: + version "5.4.1" + resolved "https://registry.yarnpkg.com/ora/-/ora-5.4.1.tgz#1b2678426af4ac4a509008e5e4ac9e9959db9e18" + integrity sha512-5b6Y85tPxZZ7QytO+BQzysW31HJku27cRIlkbAXaNx+BdcVi+LlRFmVXzeF6a7JCwJpyw5c4b+YSVImQIrBpuQ== + dependencies: + bl "^4.1.0" + chalk "^4.1.0" + cli-cursor "^3.1.0" + cli-spinners "^2.5.0" + is-interactive "^1.0.0" + is-unicode-supported "^0.1.0" + log-symbols "^4.1.0" + strip-ansi "^6.0.0" + wcwidth "^1.0.1" + p-cancelable@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/p-cancelable/-/p-cancelable-1.1.0.tgz#d078d15a3af409220c886f1d9a0ca2e441ab26cc" integrity sha512-s73XxOZ4zpt1edZYZzvhqFa6uvQc1vwUa0K0BdtIZgQMAJj9IbebH+JkgKZc9h+B05PKHLOTl4ajG1BmNrVZlw== +p-cancelable@^2.0.0: + version "2.1.1" + resolved "https://registry.yarnpkg.com/p-cancelable/-/p-cancelable-2.1.1.tgz#aab7fbd416582fa32a3db49859c122487c5ed2cf" + integrity sha512-BZOr3nRQHOntUjTrH8+Lh54smKHoHyur8We1V8DSMVrl5A2malOOwuJRnKRDjSnkoeBh4at6BwEnb5I7Jl31wg== + p-limit@^2.2.0: version "2.3.0" resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-2.3.0.tgz#3dd33c647a214fdfffd835933eb086da0dc21db1" @@ -5519,6 +6058,13 @@ p-locate@^5.0.0: dependencies: p-limit "^3.0.2" +p-map@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/p-map/-/p-map-4.0.0.tgz#bb2f95a5eda2ec168ec9274e06a747c3e2904d2b" + integrity sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ== + dependencies: + aggregate-error "^3.0.0" + p-try@^2.0.0: version "2.2.0" resolved "https://registry.yarnpkg.com/p-try/-/p-try-2.2.0.tgz#cb2868540e313d61de58fafbe35ce9004d5540e6" @@ -5744,6 +6290,19 @@ progress@^2.0.3: resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8" integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA== +promise-inflight@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/promise-inflight/-/promise-inflight-1.0.1.tgz#98472870bf228132fcbdd868129bad12c3c029e3" + integrity sha512-6zWPyEOFaQBJYcGMHBKTKJ3u6TBsnMFOIZSa6ce1e/ZrrsOlnHRHbabMjLiBYKp+n44X9eUI6VUPaukCXHuG4g== + +promise-retry@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/promise-retry/-/promise-retry-2.0.1.tgz#ff747a13620ab57ba688f5fc67855410c370da22" + integrity sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g== + dependencies: + err-code "^2.0.2" + retry "^0.12.0" + prompts@^2.0.1: version "2.4.2" resolved "https://registry.yarnpkg.com/prompts/-/prompts-2.4.2.tgz#7b57e73b3a48029ad10ebd44f74b01722a4cb069" @@ -5789,6 +6348,11 @@ queue-microtask@^1.2.2: resolved "https://registry.yarnpkg.com/queue-microtask/-/queue-microtask-1.2.3.tgz#4929228bbc724dfac43e0efb058caf7b6cfb6243" integrity sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A== +quick-lru@^5.1.1: + version "5.1.1" + resolved "https://registry.yarnpkg.com/quick-lru/-/quick-lru-5.1.1.tgz#366493e6b3e42a3a6885e2e99d18f80fb7a8c932" + integrity sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA== + randombytes@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/randombytes/-/randombytes-2.1.0.tgz#df6f84372f0270dc65cdf6291349ab7a473d4f2a" @@ -5855,7 +6419,7 @@ read-config-file@6.2.0: json5 "^2.2.0" lazy-val "^1.0.4" -readable-stream@^3.1.1, readable-stream@^3.4.0: +readable-stream@^3.1.1, readable-stream@^3.4.0, readable-stream@^3.6.0: version "3.6.2" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.2.tgz#56a9b36ea965c00c5a93ef31eb111a0f11056967" integrity sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA== @@ -5980,6 +6544,11 @@ require-from-string@^2.0.2: resolved "https://registry.yarnpkg.com/require-from-string/-/require-from-string-2.0.2.tgz#89a7fdd938261267318eafe14f9c32e598c36909" integrity sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw== +resolve-alpn@^1.0.0: + version "1.2.1" + resolved "https://registry.yarnpkg.com/resolve-alpn/-/resolve-alpn-1.2.1.tgz#b7adbdac3546aaaec20b45e7d8265927072726f9" + integrity sha512-0a1F4l73/ZFZOakJnQ3FvkJ2+gSTQWz/r2KE5OdDY0TxPm5h4GkqkWWfM47T7HsbnOtcJVEF4epCVy6u7Q3K+g== + resolve-cwd@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/resolve-cwd/-/resolve-cwd-3.0.0.tgz#0f0075f1bb2544766cf73ba6a6e2adfebcb13f2d" @@ -6027,11 +6596,31 @@ responselike@^1.0.2: dependencies: lowercase-keys "^1.0.0" +responselike@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/responselike/-/responselike-2.0.1.tgz#9a0bc8fdc252f3fb1cca68b016591059ba1422bc" + integrity sha512-4gl03wn3hj1HP3yzgdI7d3lCkF95F21Pz4BPGvKHinyQzALR5CapwC8yIi0Rh58DEMQ/SguC03wFj2k0M/mHhw== + dependencies: + lowercase-keys "^2.0.0" + +restore-cursor@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/restore-cursor/-/restore-cursor-3.1.0.tgz#39f67c54b3a7a58cea5236d95cf0034239631f7e" + integrity sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA== + dependencies: + onetime "^5.1.0" + signal-exit "^3.0.2" + restream@1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/restream/-/restream-1.2.0.tgz#fd13b031a54e80cc65d1878c43149f1b3a40efbb" integrity sha1-/ROwMaVOgMxl0YeMQxSfGzpA77s= +retry@^0.12.0: + version "0.12.0" + resolved "https://registry.yarnpkg.com/retry/-/retry-0.12.0.tgz#1b42a6266a21f07421d1b0b54b7dc167b01c013b" + integrity sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow== + reusify@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/reusify/-/reusify-1.0.4.tgz#90da382b1e126efc02146e90845a88db12925d76" @@ -6185,6 +6774,11 @@ serialize-javascript@^6.0.0: dependencies: randombytes "^2.1.0" +set-blocking@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7" + integrity sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw== + shallow-clone@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/shallow-clone/-/shallow-clone-3.0.1.tgz#8f2981ad92531f55035b01fb230769a40e02efa3" @@ -6213,7 +6807,7 @@ side-channel@^1.0.4: get-intrinsic "^1.0.2" object-inspect "^1.9.0" -signal-exit@^3.0.3, signal-exit@^3.0.7: +signal-exit@^3.0.2, signal-exit@^3.0.3, signal-exit@^3.0.7: version "3.0.7" resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.7.tgz#a9a1767f8af84155114eaabd73f99273c8f59ad9" integrity sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ== @@ -6263,6 +6857,28 @@ smart-buffer@^4.0.2: resolved "https://registry.yarnpkg.com/smart-buffer/-/smart-buffer-4.1.0.tgz#91605c25d91652f4661ea69ccf45f1b331ca21ba" integrity sha512-iVICrxOzCynf/SNaBQCw34eM9jROU/s5rzIhpOvzhzuYHfJR/DhZfDkXiZSgKXfgv26HT3Yni3AV/DGw0cGnnw== +smart-buffer@^4.2.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/smart-buffer/-/smart-buffer-4.2.0.tgz#6e1d71fa4f18c05f7d0ff216dd16a481d0e8d9ae" + integrity sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg== + +socks-proxy-agent@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/socks-proxy-agent/-/socks-proxy-agent-7.0.0.tgz#dc069ecf34436621acb41e3efa66ca1b5fed15b6" + integrity sha512-Fgl0YPZ902wEsAyiQ+idGd1A7rSFx/ayC1CQVMw5P+EQx2V0SgpGtf6OKFhVjPflPUl9YMmEOnmfjCdMUsygww== + dependencies: + agent-base "^6.0.2" + debug "^4.3.3" + socks "^2.6.2" + +socks@^2.6.2: + version "2.8.7" + resolved "https://registry.yarnpkg.com/socks/-/socks-2.8.7.tgz#e2fb1d9a603add75050a2067db8c381a0b5669ea" + integrity sha512-HLpt+uLy/pxB+bum/9DzAgiKS8CX1EvbWxI4zlmgGCExImLdiad2iCwXT5Z4c9c3Eq8rP2318mPW2c+QbtjK8A== + dependencies: + ip-address "^10.0.1" + smart-buffer "^4.2.0" + "source-map-js@>=0.6.2 <2.0.0", source-map-js@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/source-map-js/-/source-map-js-1.0.2.tgz#adbc361d9c62df380125e7f161f71c826f1e490c" @@ -6311,6 +6927,13 @@ sprintf-js@~1.0.2: resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" integrity sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g== +ssri@^9.0.0: + version "9.0.1" + resolved "https://registry.yarnpkg.com/ssri/-/ssri-9.0.1.tgz#544d4c357a8d7b71a19700074b6883fcb4eae057" + integrity sha512-o57Wcn66jMQvfHG1FlYbWeZWW/dHZhJXjpIcTfXldXEk5nz5lStPo3mK0OJQfGR3RbZUlbISexbljkJzuEj/8Q== + dependencies: + minipass "^3.1.1" + stable@^0.1.8: version "0.1.8" resolved "https://registry.yarnpkg.com/stable/-/stable-0.1.8.tgz#836eb3c8382fe2936feaf544631017ce7d47a3cf" @@ -6336,7 +6959,7 @@ string-length@^4.0.1: char-regex "^1.0.2" strip-ansi "^6.0.0" -string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: +"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -6493,6 +7116,18 @@ tar-stream@^2.1.4: inherits "^2.0.3" readable-stream "^3.1.1" +tar@^6.0.5, tar@^6.1.2: + version "6.2.1" + resolved "https://registry.yarnpkg.com/tar/-/tar-6.2.1.tgz#717549c541bc3c2af15751bea94b1dd068d4b03a" + integrity sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A== + dependencies: + chownr "^2.0.0" + fs-minipass "^2.0.0" + minipass "^5.0.0" + minizlib "^2.1.1" + mkdirp "^1.0.3" + yallist "^4.0.0" + tar@^6.1.11: version "6.1.11" resolved "https://registry.yarnpkg.com/tar/-/tar-6.1.11.tgz#6760a38f003afa1b2ffd0ffe9e9abbd0eab3d621" @@ -6756,6 +7391,20 @@ unicode-property-aliases-ecmascript@^2.0.0: resolved "https://registry.yarnpkg.com/unicode-property-aliases-ecmascript/-/unicode-property-aliases-ecmascript-2.1.0.tgz#43d41e3be698bd493ef911077c9b131f827e8ccd" integrity sha512-6t3foTQI9qne+OZoVQB/8x8rk2k1eVy1gRXhV3oFQ5T6R1dqQ1xtin3XqSlx3+ATBkliTaR/hHyJBm+LVPNM8w== +unique-filename@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/unique-filename/-/unique-filename-2.0.1.tgz#e785f8675a9a7589e0ac77e0b5c34d2eaeac6da2" + integrity sha512-ODWHtkkdx3IAR+veKxFV+VBkUMcN+FaqzUUd7IZzt+0zhDZFPFxhlqwPF3YQvMHx1TD0tdgYl+kuPnJ8E6ql7A== + dependencies: + unique-slug "^3.0.0" + +unique-slug@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/unique-slug/-/unique-slug-3.0.0.tgz#6d347cf57c8a7a7a6044aabd0e2d74e4d76dc7c9" + integrity sha512-8EyMynh679x/0gqE9fT9oilG+qEt+ibFyqjuVTsZn1+CMxH+XLlpvr2UZx4nVcCwTpx81nICr2JQFkM+HPLq4w== + dependencies: + imurmurhash "^0.1.4" + universalify@^0.1.0: version "0.1.2" resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.2.tgz#b646f69be3942dabcecc9d6639c80dc105efaa66" @@ -6848,6 +7497,13 @@ watchpack@^2.4.0: glob-to-regexp "^0.4.1" graceful-fs "^4.1.2" +wcwidth@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/wcwidth/-/wcwidth-1.0.1.tgz#f0b0dcf915bc5ff1528afadb2c0e17b532da2fe8" + integrity sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg== + dependencies: + defaults "^1.0.3" + webidl-conversions@^4.0.2: version "4.0.2" resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-4.0.2.tgz#a855980b1f0b6b359ba1d5d9fb39ae941faa63ad" @@ -6952,13 +7608,20 @@ which-typed-array@^1.1.9: has-tostringtag "^1.0.0" is-typed-array "^1.1.10" -which@^2.0.1: +which@^2.0.1, which@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/which/-/which-2.0.2.tgz#7c6a8dd0a636a0327e10b59c9286eee93f3f51b1" integrity sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA== dependencies: isexe "^2.0.0" +wide-align@^1.1.5: + version "1.1.5" + resolved "https://registry.yarnpkg.com/wide-align/-/wide-align-1.1.5.tgz#df1d4c206854369ecf3c9a4898f1b23fbd9d15d3" + integrity sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg== + dependencies: + string-width "^1.0.2 || 2 || 3 || 4" + wildcard@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/wildcard/-/wildcard-2.0.0.tgz#a77d20e5200c6faaac979e4b3aadc7b3dd7f8fec" @@ -7034,7 +7697,7 @@ yargs-parser@^21.0.1, yargs-parser@^21.1.1: resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-21.1.1.tgz#9096bceebf990d21bb31fa9516e0ede294a77d35" integrity sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw== -yargs@^17.3.1, yargs@^17.5.1: +yargs@^17.0.1, yargs@^17.3.1, yargs@^17.5.1: version "17.7.2" resolved "https://registry.yarnpkg.com/yargs/-/yargs-17.7.2.tgz#991df39aca675a192b816e1e0363f9d75d2aa269" integrity sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w== From f6bf5e2d371931deb67d23160aa19b957874b150 Mon Sep 17 00:00:00 2001 From: RafaUC Date: Fri, 3 Oct 2025 00:37:37 -0600 Subject: [PATCH 04/19] Create SQL schema, table migrations, and kysely typeScript types --- src/api/file.ts | 8 +- src/api/search-criteria.ts | 4 +- src/backend/_deprecated/backend.ts | 3 +- src/backend/_deprecated/config.ts | 2 +- src/backend/backend.ts | 8 +- src/backend/config.ts | 6 +- src/backend/migrations/000_initial.ts | 241 +++++++++++++++++++++--- src/backend/schemaTypes.ts | 162 +++++++++++++--- src/frontend/entities/File.ts | 11 +- src/frontend/entities/SearchCriteria.ts | 9 +- src/frontend/stores/LocationStore.ts | 11 +- tests/backend.test.ts | 3 +- 12 files changed, 389 insertions(+), 79 deletions(-) diff --git a/src/api/file.ts b/src/api/file.ts index b5119107..0c68b8b2 100644 --- a/src/api/file.ts +++ b/src/api/file.ts @@ -10,6 +10,7 @@ export type FileDTO = { relativePath: string; absolutePath: string; tags: ID[]; + tagsSorting: FILE_TAGS_SORTING_TYPE; /** used only for index on dexie */ extraPropertyIDs: ID[]; extraProperties: ExtraProperties; @@ -17,11 +18,11 @@ export type FileDTO = { dateAdded: Date; /** When the file was modified in Allusion, not related to OS modified date */ dateModified: Date; - /** Original dateModified for checking when searching for overwritten files + /** Original OS dateModified for checking when searching for overwritten files * If the system's modified date is not the same, it means the file has been overwritten or another file with the same name * overwritten in place of the previous one, and the thumbnail and metadata needs to be updated. */ - OrigDateModified: Date; + dateModifiedOS: Date; /** * When the file was last indexed in Allusion: concerning the metadata and thumbnail. * If the system's modified date of the file exceeds this date, those properties shoudld be re-initialized @@ -67,3 +68,6 @@ export const IMG_EXTENSIONS = [ 'ogg', ] as const; export type IMG_EXTENSIONS_TYPE = (typeof IMG_EXTENSIONS)[number]; + +export const FILE_TAGS_SORTING = ['insertion', 'hierarchy'] as const; +export type FILE_TAGS_SORTING_TYPE = (typeof FILE_TAGS_SORTING)[number]; diff --git a/src/api/search-criteria.ts b/src/api/search-criteria.ts index 46d2d9c9..c7682fcc 100644 --- a/src/api/search-criteria.ts +++ b/src/api/search-criteria.ts @@ -25,10 +25,12 @@ export type OperatorType = | StringOperatorType | BinaryOperatorType; +export type CriteriaValueType = 'number' | 'date' | 'string' | 'array' | 'indexSignature'; + // FFR: Boolean keys are not supported in IndexedDB/Dexie - must store booleans as 0/1 export interface IBaseSearchCriteria { key: keyof FileDTO; - valueType: 'number' | 'date' | 'string' | 'array' | 'indexSignature'; + valueType: CriteriaValueType; readonly operator: OperatorType; } diff --git a/src/backend/_deprecated/backend.ts b/src/backend/_deprecated/backend.ts index 663852ab..fe2feb04 100644 --- a/src/backend/_deprecated/backend.ts +++ b/src/backend/_deprecated/backend.ts @@ -434,6 +434,7 @@ const exampleFileDTO: FileDTO = { absolutePath: '', locationId: '', extension: 'jpg', + tagsSorting: 'hierarchy', size: 0, width: 0, height: 0, @@ -441,7 +442,7 @@ const exampleFileDTO: FileDTO = { dateCreated: new Date(), dateLastIndexed: new Date(), dateModified: new Date(), - OrigDateModified: new Date(), + dateModifiedOS: new Date(), extraProperties: {}, extraPropertyIDs: [], tags: [], diff --git a/src/backend/_deprecated/config.ts b/src/backend/_deprecated/config.ts index a88e76ca..5d4341e2 100644 --- a/src/backend/_deprecated/config.ts +++ b/src/backend/_deprecated/config.ts @@ -148,7 +148,7 @@ const dbConfig: DBVersioningConfig[] = [ tx.table('files') .toCollection() .modify((file: FileDTO) => { - file.OrigDateModified = file.dateAdded; + file.dateModifiedOS = file.dateAdded; return file; }); }, diff --git a/src/backend/backend.ts b/src/backend/backend.ts index 58f1b3f4..188baae2 100644 --- a/src/backend/backend.ts +++ b/src/backend/backend.ts @@ -1,13 +1,13 @@ -import { Database } from './schemaTypes'; +import { AllusionDB_SQL } from './schemaTypes'; import SQLite from 'better-sqlite3'; import { Kysely, SqliteDialect, CamelCasePlugin } from 'kysely'; import { migrateToLatest } from './config'; export default class Backend { - #db: Kysely; + #db: Kysely; #notifyChange: () => void; - constructor(db: Kysely, notifyChange: () => void) { + constructor(db: Kysely, notifyChange: () => void) { this.#db = db; this.#notifyChange = notifyChange; @@ -18,7 +18,7 @@ export default class Backend { const dialect = new SqliteDialect({ database: new SQLite(dbPath), }); - const db = new Kysely({ + const db = new Kysely({ dialect: dialect, plugins: [new CamelCasePlugin()], }); diff --git a/src/backend/config.ts b/src/backend/config.ts index 6d8cdf4c..9fb213d6 100644 --- a/src/backend/config.ts +++ b/src/backend/config.ts @@ -1,7 +1,7 @@ import * as path from 'path'; import { promises as fs } from 'fs'; import { Kysely, Migrator, FileMigrationProvider, Migration, MigrationProvider } from 'kysely'; -import { Database } from './schemaTypes'; +import { AllusionDB_SQL } from './schemaTypes'; export const DB_NAME = 'Allusion'; @@ -13,12 +13,12 @@ export const AUTO_BACKUP_TIMEOUT = 1000 * 60 * 10; // 10 minutes class InlineMigrationProvider implements MigrationProvider { async getMigrations(): Promise> { return { - '001_initial': await import('./migrations/000_initial'), + '000_initial': await import('./migrations/000_initial'), }; } } -export async function migrateToLatest(db: Kysely): Promise { +export async function migrateToLatest(db: Kysely): Promise { const migrator = new Migrator({ db, provider: new InlineMigrationProvider(), diff --git a/src/backend/migrations/000_initial.ts b/src/backend/migrations/000_initial.ts index 2d9d4419..a3837548 100644 --- a/src/backend/migrations/000_initial.ts +++ b/src/backend/migrations/000_initial.ts @@ -1,44 +1,237 @@ -import { Kysely } from 'kysely'; +/* eslint-disable prettier/prettier */ +import { Kysely, sql } from 'kysely'; -export async function up(db: Kysely) { - // tags +/* +Migration to create the SQLite database. Note that SQL table and column names +are in snake_case, which will later be converted to camelCase +by the Kysely camel case plugin. +*/ + +export async function up(db: Kysely): Promise { + //// TAGS //// await db.schema .createTable('tags') - .addColumn('id', 'text', (col) => col.primaryKey()) + .addColumn('id', 'text', (col) => col.primaryKey().notNull()) + .addColumn('parent_id', 'text') + .addColumn('idx', 'integer', (col) => col.notNull()) .addColumn('name', 'text', (col) => col.notNull()) - .addColumn('date_added', 'integer', (col) => col.notNull()) - .addColumn('color', 'text', (col) => col.notNull()) - .addColumn('is_hidden', 'boolean', (col) => col.notNull().defaultTo(false)) - .addColumn('is_visible_inherited', 'boolean', (col) => col.notNull().defaultTo(false)) - .addColumn('is_header', 'boolean', (col) => col.notNull().defaultTo(false)) + .addColumn('date_added', 'timestamp', (col) => col.notNull().defaultTo(sql`CURRENT_TIMESTAMP`)) + .addColumn('color', 'text') + .addColumn('is_hidden', 'boolean', (col) => col.notNull().defaultTo(0)) + .addColumn('is_visible_inherited', 'boolean', (col) => col.notNull().defaultTo(1)) + .addColumn('is_header', 'boolean', (col) => col.notNull().defaultTo(0)) .addColumn('description', 'text') + .addForeignKeyConstraint('fk_tag_parent', ['parent_id'], 'tags', ['id'], (cb) => cb.onDelete('cascade')) .execute(); - // N:N subTags await db.schema - .createTable('tag_subtags') + .createTable('tag_implications') .addColumn('tag_id', 'text', (col) => col.notNull()) - .addColumn('subtag_id', 'text', (col) => col.notNull()) - .addForeignKeyConstraint('fk_tag_subtags_tag', ['tag_id'], 'tags', ['id']) - .addForeignKeyConstraint('fk_tag_subtags_subtag', ['subtag_id'], 'tags', ['id']) + .addColumn('implied_tag_id', 'text', (col) => col.notNull()) + .addPrimaryKeyConstraint('pk_tag_implications', ['tag_id', 'implied_tag_id']) + .addForeignKeyConstraint('fk_tag_implications_tag', ['tag_id'], 'tags', ['id'], (cb) => cb.onDelete('cascade')) + .addForeignKeyConstraint('fk_tag_implications_implied', ['implied_tag_id'], 'tags', ['id'], (cb) => cb.onDelete('cascade')) .execute(); - // N:N impliedTags await db.schema - .createTable('tag_implied') + .createTable('tag_aliases') .addColumn('tag_id', 'text', (col) => col.notNull()) - .addColumn('implied_tag_id', 'text', (col) => col.notNull()) - .addForeignKeyConstraint('fk_tag_implied_tag', ['tag_id'], 'tags', ['id']) - .addForeignKeyConstraint('fk_tag_implied_implied', ['implied_tag_id'], 'tags', ['id']) + .addColumn('alias', 'text', (col) => col.notNull()) + .addPrimaryKeyConstraint('pk_tag_aliases', ['tag_id', 'alias']) + .addForeignKeyConstraint('fk_tag_aliases_tag', ['tag_id'], 'tags', ['id'], (cb) => cb.onDelete('cascade')) .execute(); - // Aliases + //// LOCATIONS //// await db.schema - .createTable('tag_aliases') + .createTable('location_node') + .addColumn('id', 'text', (col) => col.primaryKey().notNull()) + .addColumn('parent_id', 'text') + .addColumn('path', 'text', (col) => col.notNull()) + .addForeignKeyConstraint('fk_location_node_parent', ['parent_id'], 'location_node', ['id'], (cb) => cb.onDelete('cascade')) + .addUniqueConstraint('uq_location_node_parent_path', ['parent_id', 'path']) + .execute(); + + await db.schema + .createTable('location') + .addColumn('node_id', 'text', (col) => col.primaryKey().notNull()) + .addColumn('date_added', 'timestamp', (col) => col.notNull().defaultTo(sql`CURRENT_TIMESTAMP`)) + .addColumn('idx', 'integer', (col) => col.notNull()) + .addColumn('is_watching_files', 'boolean', (col) => col.notNull().defaultTo(0)) + .addForeignKeyConstraint('fk_location_node', ['node_id'], 'location_node', ['id'], (cb) => cb.onDelete('cascade')) + .execute(); + + await db.schema + .createTable('sub_location') + .addColumn('node_id', 'text', (col) => col.primaryKey().notNull()) + .addColumn('is_excluded', 'boolean', (col) => col.notNull().defaultTo(0)) + .addForeignKeyConstraint('fk_sub_location_node', ['node_id'], 'location_node', ['id'], (cb) => cb.onDelete('cascade')) + .execute(); + + await db.schema + .createTable('location_tags') + .addColumn('node_id', 'text', (col) => col.notNull()) .addColumn('tag_id', 'text', (col) => col.notNull()) - .addColumn('alias', 'text', (col) => col.notNull()) - .addForeignKeyConstraint('fk_tag_aliases_tag', ['tag_id'], 'tags', ['id']) + .addPrimaryKeyConstraint('pk_location_tags', ['node_id', 'tag_id']) + .addForeignKeyConstraint('fk_location_tags_node', ['node_id'], 'location_node', ['id'], (cb) => cb.onDelete('cascade')) + .addForeignKeyConstraint('fk_location_tags_tag', ['tag_id'], 'tags', ['id'], (cb) => cb.onDelete('cascade')) + .execute(); + + //// FILES //// + await db.schema + .createTable('files') + .addColumn('id', 'text', (col) => col.primaryKey().notNull()) + .addColumn('ino', 'text', (col) => col.notNull()) + .addColumn('location_id', 'text', (col) => col.notNull()) + .addColumn('relative_path', 'text', (col) => col.notNull()) + .addColumn('absolute_path', 'text', (col) => col.notNull()) + .addColumn('tag_sorting', 'text', (col) => col.notNull()) + .addColumn('date_added', 'timestamp', (col) => col.notNull().defaultTo(sql`CURRENT_TIMESTAMP`)) + .addColumn('date_modified', 'timestamp') + .addColumn('date_modified_os', 'timestamp') + .addColumn('date_last_indexed', 'timestamp') + .addColumn('name', 'text', (col) => col.notNull()) + .addColumn('extension', 'text') + .addColumn('size', 'integer') + .addColumn('width', 'integer') + .addColumn('height', 'integer') + .addColumn('date_created', 'timestamp') + .addForeignKeyConstraint('fk_files_location', ['location_id'], 'location', ['node_id'], (cb) => cb.onDelete('cascade')) + .addUniqueConstraint('uq_location_node_parent_path', ['location_id', 'relative_path']) + .addUniqueConstraint('uq_absolute_path', ['relative_path']) + .execute(); + // await db.schema.createIndex('idx_files_name').on('files').column('name').execute(); + // await db.schema.createIndex('idx_files_extension').on('files').column('extension').execute(); + // await db.schema.createIndex('idx_files_size').on('files').column('size').execute(); + // await db.schema.createIndex('idx_files_width').on('files').column('width').execute(); + // await db.schema.createIndex('idx_files_height').on('files').column('height').execute(); + // await db.schema.createIndex('idx_files_date_added').on('files').column('date_added').execute(); + // await db.schema.createIndex('idx_files_date_modified').on('files').column('date_modified').execute(); + // await db.schema.createIndex('idx_files_date_created').on('files').column('date_created').execute(); + // await db.schema.createIndex('idx_files_relative_path').on('files').column('relative_path').unique().execute(); + // await db.schema.createIndex('idx_files_location').on('files').column('location_id').execute(); + + await db.schema + .createTable('file_tags') + .addColumn('file_id', 'text', (col) => col.notNull()) + .addColumn('tag_id', 'text', (col) => col.notNull()) + .addPrimaryKeyConstraint('pk_file_tags', ['file_id', 'tag_id']) + .addForeignKeyConstraint('fk_file_tags_file', ['file_id'], 'files', ['id'], (cb) => cb.onDelete('cascade')) + .addForeignKeyConstraint('fk_file_tags_tag', ['tag_id'], 'tags', ['id'], (cb) => cb.onDelete('cascade')) + .execute(); + await db.schema.createIndex('idx_file_tags_tag').on('file_tags').column('tag_id').execute(); + await db.schema.createIndex('idx_file_tags_file').on('file_tags').column('file_id').execute(); + + //// EXTRA PROPERTIES //// + await db.schema + .createTable('extra_properties') + .addColumn('id', 'text', (col) => col.primaryKey().notNull()) + .addColumn('type', 'text', (col) => col.notNull()) + .addColumn('name', 'text', (col) => col.notNull()) + .addColumn('date_added', 'timestamp', (col) => col.notNull().defaultTo(sql`CURRENT_TIMESTAMP`)) + .execute(); + + await db.schema + .createTable('ep_values_text') + .addColumn('file_id', 'text', (col) => col.notNull()) + .addColumn('ep_id', 'text', (col) => col.notNull()) + .addColumn('value', 'text', (col) => col.notNull()) + .addPrimaryKeyConstraint('pk_ep_values_text', ['file_id', 'ep_id']) + .addForeignKeyConstraint('fk_ep_values_text_file', ['file_id'], 'files', ['id'], (cb) => cb.onDelete('cascade')) + .addForeignKeyConstraint('fk_ep_values_text_ep', ['ep_id'], 'extra_properties', ['id'], (cb) => cb.onDelete('cascade')) + .execute(); + await db.schema.createIndex('idx_ep_values_text_file').on('ep_values_text').column('file_id').execute(); + await db.schema.createIndex('idx_ep_values_text_value').on('ep_values_text').column('value').execute(); + await db.schema.createIndex('idx_ep_values_text_ep').on('ep_values_text').column('ep_id').execute(); + + + await db.schema + .createTable('ep_values_number') + .addColumn('file_id', 'text', (col) => col.notNull()) + .addColumn('ep_id', 'text', (col) => col.notNull()) + .addColumn('value', 'integer', (col) => col.notNull()) + .addPrimaryKeyConstraint('pk_ep_values_number', ['file_id', 'ep_id']) + .addForeignKeyConstraint('fk_ep_values_number_file', ['file_id'], 'files', ['id'], (cb) => cb.onDelete('cascade')) + .addForeignKeyConstraint('fk_ep_values_number_ep', ['ep_id'], 'extra_properties', ['id'], (cb) => cb.onDelete('cascade')) + .execute(); + await db.schema.createIndex('idx_ep_values_number_file').on('ep_values_number').column('file_id').execute(); + await db.schema.createIndex('idx_ep_values_number_value').on('ep_values_number').column('value').execute(); + await db.schema.createIndex('idx_ep_values_number_ep').on('ep_values_number').column('ep_id').execute(); + + await db.schema + .createTable('ep_values_timestamp') + .addColumn('file_id', 'text', (col) => col.notNull()) + .addColumn('ep_id', 'text', (col) => col.notNull()) + .addColumn('value', 'timestamp', (col) => col.notNull()) + .addPrimaryKeyConstraint('pk_ep_values_timestamp', ['file_id', 'ep_id']) + .addForeignKeyConstraint('fk_ep_values_timestamp_file', ['file_id'], 'files', ['id'], (cb) => cb.onDelete('cascade')) + .addForeignKeyConstraint('fk_ep_values_timestamp_ep', ['ep_id'], 'extra_properties', ['id'], (cb) => cb.onDelete('cascade')) + .execute(); + await db.schema.createIndex('idx_ep_values_timestamp_file').on('ep_values_timestamp').column('file_id').execute(); + await db.schema.createIndex('idx_ep_values_timestamp_value').on('ep_values_timestamp').column('value').execute(); + await db.schema.createIndex('idx_ep_values_timestamp_ep').on('ep_values_timestamp').column('ep_id').execute(); + + //// SAVED SEARCHES //// + await db.schema + .createTable('saved_searches') + .addColumn('id', 'text', (col) => col.primaryKey().notNull()) + .addColumn('name', 'text', (col) => col.notNull()) + .addColumn('idx', 'integer', (col) => col.notNull()) + .execute(); + + await db.schema + .createTable('search_criterias') + .addColumn('id', 'text', (col) => col.primaryKey().notNull()) + .addColumn('saved_search_id', 'text', (col) => col.notNull()) + .addColumn('idx', 'integer', (col) => col.notNull()) + .addColumn('match_group', 'text', (col) => col.notNull()) // 'any' | 'all' + .addColumn('key', 'text', (col) => col.notNull()) + .addColumn('value_type', 'text', (col) => col.notNull()) + .addColumn('operator', 'text', (col) => col.notNull()) + .addColumn('json_value', 'text', (col) => col.notNull()) + .addForeignKeyConstraint('fk_search_criterias_saved_search', ['saved_search_id'], 'saved_searches', ['id'], (cb) => cb.onDelete('cascade')) .execute(); } -export async function down(db: Kysely) {} +export async function down(db: Kysely): Promise { + await db.schema.dropIndex('idx_ep_values_text_file').execute(); + await db.schema.dropIndex('idx_ep_values_text_value').execute(); + await db.schema.dropIndex('idx_ep_values_text_ep').execute(); + + await db.schema.dropIndex('idx_ep_values_number_file').execute(); + await db.schema.dropIndex('idx_ep_values_number_value').execute(); + await db.schema.dropIndex('idx_ep_values_number_ep').execute(); + + await db.schema.dropIndex('idx_ep_values_timestamp_file').execute(); + await db.schema.dropIndex('idx_ep_values_timestamp_value').execute(); + await db.schema.dropIndex('idx_ep_values_timestamp_ep').execute(); + + await db.schema.dropIndex('idx_file_tags_file').execute(); + await db.schema.dropIndex('idx_file_tags_tag').execute(); + + await db.schema.dropIndex('idx_files_location').execute(); + await db.schema.dropIndex('idx_files_relative_path').execute(); + await db.schema.dropIndex('idx_files_date_created').execute(); + await db.schema.dropIndex('idx_files_date_modified').execute(); + await db.schema.dropIndex('idx_files_date_added').execute(); + await db.schema.dropIndex('idx_files_height').execute(); + await db.schema.dropIndex('idx_files_width').execute(); + await db.schema.dropIndex('idx_files_size').execute(); + await db.schema.dropIndex('idx_files_extension').execute(); + await db.schema.dropIndex('idx_files_name').execute(); + + await db.schema.dropTable('search_criterias').execute(); + await db.schema.dropTable('saved_searches').execute(); + await db.schema.dropTable('ep_values_timestamp').execute(); + await db.schema.dropTable('ep_values_number').execute(); + await db.schema.dropTable('ep_values_text').execute(); + await db.schema.dropTable('extra_properties').execute(); + await db.schema.dropTable('file_tags').execute(); + await db.schema.dropTable('files').execute(); + await db.schema.dropTable('location_tags').execute(); + await db.schema.dropTable('sub_location').execute(); + await db.schema.dropTable('location').execute(); + await db.schema.dropTable('location_node').execute(); + await db.schema.dropTable('tag_aliases').execute(); + await db.schema.dropTable('tag_implications').execute(); + await db.schema.dropTable('tags').execute(); +} diff --git a/src/backend/schemaTypes.ts b/src/backend/schemaTypes.ts index a6adbde0..e1937b6c 100644 --- a/src/backend/schemaTypes.ts +++ b/src/backend/schemaTypes.ts @@ -1,45 +1,153 @@ /** - * In this file we define the interfaces that Kisely will use to have types defined and build the sql queris, - * these are an equivalent representation to the actual SQLite database schema. + * In this file we define the types that Kysely will use to provide typing and build SQL queries. + * These types are a type-level equivalent representation of the actual SQLite database schema. * - * These are just interfaces, and updating here the structures will not actualizze the database, to do taht you will need - * to manually write Kysely migrations taking care that the migration will update the database shema to be the same. + * Each exported interface represents a table in the SQLite database. Some schemas differ from + * Allusion's DTO API in favor of better normalization, avoiding nulls, and ensuring query-building compatibility. + * The serialization to and from the DTO API is handled by the data-storage implementation (backend) class. + * + * Note: These are only TypeScript types. Updating them will not update the database automatically. + * To apply changes to the actual schema you must manually write Kysely migrations, + * ensuring that the database schema is kept in sync with this definitions. */ import { ColumnType } from 'kysely'; import { ID } from '../api/id'; +import { CriteriaValueType, OperatorType } from 'src/api/search-criteria'; +import { FILE_TAGS_SORTING_TYPE, FileDTO } from 'src/api/file'; -export interface Database { +export type AllusionDB_SQL = { tags: Tags; - tagSubTags: TagSubTags; - tagImplied: TagImplied; + tagImplications: TagImplications; tagAliases: TagAliases; -} + locationNode: LocationNode; + location: Location; + subLocation: SubLocation; + locationTags: LocationTags; + files: Files; + fileTags: FileTags; + extraProperties: ExtraProperties; + epValuesText: EpValuesText; + epValuesNumber: EpValuesNumber; + epValuesTimestamp: EpValuesTimestamp; + savedSearches: SavedSearches; + searchCriterias: SearchCriterias; +}; ///// TAGS ///// -export interface Tags { - id: ID; +export type Tags = { + id: ColumnType; //pk + parentId: ID; //fk + idx: number; name: string; dateAdded: ColumnType; color: string; isHidden: boolean; isVisibleInherited: boolean; isHeader: boolean; - description: string | null; -} - -export interface TagSubTags { - tagId: ID; - subtagId: string; -} - -export interface TagImplied { - tagId: ID; - impliedTagId: ID; -} - -export interface TagAliases { - tagId: ID; - alias: string; -} + description: string; +}; + +export type TagImplications = { + tagId: ID; //pk fk + impliedTagId: ID; //pk fk +}; + +export type TagAliases = { + tagId: ID; //pk + alias: string; //pk +}; + +/// LOCATIONS /// + +export type LocationNode = { + id: ColumnType; //pk + parentId: ID; //fk + path: string; +}; + +export type Location = { + nodeId: ID; //pk fk + dateAdded: ColumnType; + idx: number; + isWatchingFiles: boolean; +}; + +export type SubLocation = { + nodeId: ID; //pk fk + isExcluded: boolean; +}; + +export type LocationTags = { + nodeId: ID; //pk fk + tagId: ID; //pk fk +}; + +/// FILES /// + +export type Files = { + id: ColumnType; //pk + ino: string; + locationId: ID; //fk - to Location, not node table + relativePath: string; + absolutePath: string; + tagSorting: FILE_TAGS_SORTING_TYPE; + dateAdded: ColumnType; + dateModified: Date; + DateModifiedOS: Date; + dateLastIndexed: Date; + name: string; + extension: string; + size: number; + width: number; + height: number; + dateCreated: Date; +}; + +export type FileTags = { + fileId: ID; //pk fk + tagId: ID; //pk fk +}; + +/// EXTRA PROPERTIES /// + +export type ExtraProperties = { + id: ColumnType; //pk + type: string; + name: string; + dateAdded: ColumnType; +}; + +type EpValues = { + fileId: ID; //pk fk + epId: ID; //pk fk + value: T; +}; + +export type EpValuesText = EpValues; +export type EpValuesNumber = EpValues; +export type EpValuesTimestamp = EpValues; + +/// SAVED SEARCHES /// + +export type SavedSearches = { + id: ColumnType; //pk + name: string; + idx: number; +}; + +export type SearchCriterias = { + id: ColumnType; //pk + savedSearchId: ID; //fk + idx: number; + matchGroup: 'any' | 'all'; + key: keyof FileDTO; + valueType: CriteriaValueType; + operator: OperatorType; + // Since we only need to filter by saved_search_id and not by individual value types, + // all values are stored as stringified JSON regardless of type. + // This simplifies the schema (single column) and querying. The type check is managed + // inside the app logic in the searchStore and thir related api types. + jsonValue: string; +}; diff --git a/src/frontend/entities/File.ts b/src/frontend/entities/File.ts index 1ab4f09e..348ee47c 100644 --- a/src/frontend/entities/File.ts +++ b/src/frontend/entities/File.ts @@ -10,7 +10,7 @@ import { } from 'mobx'; import Path from 'path'; -import { FileDTO, IMG_EXTENSIONS_TYPE } from '../../api/file'; +import { FILE_TAGS_SORTING_TYPE, FileDTO, IMG_EXTENSIONS_TYPE } from '../../api/file'; import { ID } from '../../api/id'; import ImageLoader from '../image/ImageLoader'; import FileStore from '../stores/FileStore'; @@ -65,7 +65,7 @@ export class ClientFile { readonly dateAdded: Date; readonly dateCreated: Date; readonly dateModified: Date; - readonly OrigDateModified: Date; + readonly dateModifiedOS: Date; readonly dateLastIndexed: Date; readonly name: string; readonly extension: IMG_EXTENSIONS_TYPE; @@ -73,6 +73,7 @@ export class ClientFile { readonly filename: string; @observable thumbnailPath: string = ''; + @observable tagsSorting: FILE_TAGS_SORTING_TYPE; // Is undefined until existence check has been completed @observable isBroken?: boolean; @@ -90,10 +91,11 @@ export class ClientFile { this.dateAdded = fileProps.dateAdded; this.dateCreated = fileProps.dateCreated; this.dateModified = fileProps.dateModified; - this.OrigDateModified = fileProps.OrigDateModified; + this.dateModifiedOS = fileProps.dateModifiedOS || new Date('2000-01-01T00:00:00Z'); this.dateLastIndexed = fileProps.dateLastIndexed; this.name = fileProps.name; this.extension = fileProps.extension; + this.tagsSorting = fileProps.tagsSorting || 'hierarchy'; const location = store.getLocation(this.locationId); this.absolutePath = Path.join(location.path, this.relativePath); @@ -266,6 +268,7 @@ export class ClientFile { relativePath: this.relativePath, absolutePath: this.absolutePath, tags: Array.from(this.tags, (t) => t.id), // removes observable properties from observable array + tagsSorting: this.tagsSorting, extraPropertyIDs: extraPropertyIDs, extraProperties: extraProperties, size: this.size, @@ -274,7 +277,7 @@ export class ClientFile { dateAdded: this.dateAdded, dateCreated: this.dateCreated, dateModified: this.dateModified, - OrigDateModified: this.OrigDateModified, + dateModifiedOS: this.dateModifiedOS, dateLastIndexed: this.dateLastIndexed, name: this.name, extension: this.extension, diff --git a/src/frontend/entities/SearchCriteria.ts b/src/frontend/entities/SearchCriteria.ts index 59dba813..b275074b 100644 --- a/src/frontend/entities/SearchCriteria.ts +++ b/src/frontend/entities/SearchCriteria.ts @@ -17,6 +17,7 @@ import { import { FileDTO } from '../../api/file'; import { ID } from '../../api/id'; import { + CriteriaValueType, IBaseSearchCriteria, IDateSearchCriteria, IExtraProperySearchCriteria, @@ -65,16 +66,12 @@ export const ExtraPropertyOperatorLabels: Record { relativePath: `test (${index}).jpg`, locationId: 'Default location', name: `test (${index}).jpg`, + tagsSorting: 'hierarchy', size: 42, width: 640, height: 480, dateAdded: new Date(), dateModified: new Date(), dateCreated: new Date(), - OrigDateModified: new Date(), + dateModifiedOS: new Date(), dateLastIndexed: new Date(), extension: 'jpg', ino: index.toString(), From 1cda3e522006354fdd44a344d111a79987a21380 Mon Sep 17 00:00:00 2001 From: RafaUC Date: Fri, 10 Oct 2025 20:20:53 -0600 Subject: [PATCH 05/19] Created import from old JSON format, migrated backup data, and fixed schema errors. --- src/backend/backup-scheduler.ts | 394 ++++++++++++++++++++++ src/backend/config.ts | 1 + src/backend/migrations/000_initial.ts | 34 +- src/backend/migrations/001_migrateJSON.ts | 31 ++ src/backend/schemaTypes.ts | 51 +-- src/main.ts | 7 - src/renderer.tsx | 7 + 7 files changed, 479 insertions(+), 46 deletions(-) create mode 100644 src/backend/backup-scheduler.ts create mode 100644 src/backend/migrations/001_migrateJSON.ts diff --git a/src/backend/backup-scheduler.ts b/src/backend/backup-scheduler.ts new file mode 100644 index 00000000..b0ca075d --- /dev/null +++ b/src/backend/backup-scheduler.ts @@ -0,0 +1,394 @@ +import { promises as fs } from 'fs'; +import { Insertable, InsertObject, Kysely, sql } from 'kysely'; +import { generateId, ID } from 'src/api/id'; +import { ROOT_TAG_ID } from 'src/api/tag'; +import { + AllusionDB_SQL, + EpValuesNumber, + EpValuesText, + EpValuesTimestamp, + ExtraProperties, + Files, + FileTags, + LocationNodes, + Locations, + LocationTags, + SavedSearches, + serializeBoolean, + serializeDate, + SubLocations, + SearchCriteria, + TagImplications, + TagAliases, +} from './schemaTypes'; +import { ExtraPropertyType } from 'src/api/extraProperty'; + +const fallbackIds = { + tag: 'fallback_tag', + location: 'fallback_location', + locationNode: 'fallback_location_node', + extraProperty: 'fallback_ep', +}; + +export async function restoreFromOldJsonFormat( + db: Kysely, + backupFilePath: string, +): Promise { + const content = await fs.readFile(backupFilePath, 'utf8'); + const json = JSON.parse(content); + console.log('===================================================='); + console.log('[] Importing Dexie backup from', backupFilePath, '[]'); + if (json.formatName !== 'dexie') { + throw new Error('Invalid backup format (expected dexie)'); + } + + const tables = Object.fromEntries( + json.data.data.map((table: any) => [table.tableName, table.rows]), + ); + + const saveEntries = async < + TableName extends keyof AllusionDB_SQL, // nombre de tabla (clave) + >( + entityName: TableName, + entries: InsertObject[], + ) => { + let errors = 0; + console.log(`Importing ${entries.length} ${entityName} from old format.`); + await db.transaction().execute(async (trx) => { + const batchSize = 2000; + for (let i = 0; i < entries.length; i += batchSize) { + try { + const batch = entries.slice(i, i + batchSize); + await trx + .insertInto(entityName) + .values(batch) + .onConflict((oc) => oc.doNothing()) + .execute(); + } catch (err) { + console.warn(`Insert ${entityName} error`, err); + errors += batchSize; + } + } + }); + console.log(`Finished importing ${entityName}: ${errors} errors.`); + }; + + // Disable foreign key constraints + await sql`PRAGMA foreign_keys = OFF;`.execute(db); + + // Create fallback references for missing foreign keys + // Ensure fallback base records exist + await db + .insertInto('tags') + .values({ + id: fallbackIds.tag, + parentId: ROOT_TAG_ID, + idx: 0, + name: 'Fallback Tag', + color: '', + description: '', + isHidden: serializeBoolean(false), + isVisibleInherited: serializeBoolean(true), + isHeader: serializeBoolean(false), + dateAdded: serializeDate(new Date()), + }) + .onConflict((oc) => oc.doNothing()) + .execute(); + + await db + .insertInto('locationNodes') + .values({ + id: fallbackIds.locationNode, + parentId: fallbackIds.locationNode, + path: 'fallback', + }) + .onConflict((oc) => oc.doNothing()) + .execute(); + + await db + .insertInto('locations') + .values({ + nodeId: fallbackIds.locationNode, + idx: 0, + isWatchingFiles: serializeBoolean(false), + dateAdded: serializeDate(new Date()), + }) + .onConflict((oc) => oc.doNothing()) + .execute(); + + await db + .insertInto('extraProperties') + .values({ + id: fallbackIds.extraProperty, + name: 'Fallback Property', + type: 'text', + dateAdded: serializeDate(new Date()), + }) + .onConflict((oc) => oc.doNothing()) + .execute(); + + /// IMPORTING DATA /// + + // Import tags + const { normalizedTags, tagImplications, tagAliases } = normalizeTags(tables.tags ?? []); + + await saveEntries('tags', normalizedTags); + await saveEntries('tagImplications', tagImplications); + await saveEntries('tagAliases', tagAliases); + + // Import locations + const { locationNodes, locations, subLocations } = normalizeLocations(tables.locations ?? []); + + await saveEntries('locationNodes', locationNodes); + await saveEntries('locations', locations); + await saveEntries('subLocations', subLocations); + + // Import extra properties definitions + const extraProperties: Insertable[] = ( + tables.extraProperties ? (tables.extraProperties as Array) : [] + ).map((ep) => ({ + id: ep.id ?? generateId(), + type: ep.type ?? ExtraPropertyType.text, + name: ep.name ?? '(unnamed)', + dateAdded: serializeDate(ep.dateAdded ? new Date(ep.dateAdded) : new Date()), + })); + + await saveEntries('extraProperties', extraProperties); + + // Import files + const { files, fileTags, epValText, epValNumber, epValTime } = normalizeFiles( + tables.files ?? [], + extraProperties, + ); + + await saveEntries('files', files); + await saveEntries('fileTags', fileTags); + await saveEntries('epValuesText', epValText); + await saveEntries('epValuesNumber', epValNumber); + await saveEntries('epValuesTimestamp', epValTime); + + // Import seved searches + const { savedSearches, searchCriteria } = normalizeSavedSearches(tables.searches ?? []); + await saveEntries('savedSearches', savedSearches); + await saveEntries('searchCriteria', searchCriteria); + + // Re-enable foreign keys + await sql`PRAGMA foreign_keys = ON;`.execute(db); + + // Validate foreign keys + const fkCheck = await sql`PRAGMA foreign_key_check;`.execute(db); + if (fkCheck.rows.length > 0) { + console.warn('Foreign key issues found:', fkCheck.rows); + // optional cleanup: remove invalid references + await sql`DELETE FROM files WHERE location_id NOT IN (SELECT node_id FROM locations);`.execute( + db, + ); + await sql`DELETE FROM file_tags WHERE tag_id NOT IN (SELECT id FROM tags);`.execute(db); + } else { + console.log('Complete succes! no foreign key issues found:', fkCheck.rows); + } + + console.log('Dexie backup import completed successfully.'); + console.log('===================================================='); +} + +export async function down(_: Kysely): Promise { + // No rollback for imports, maybe delete fallback and imported data + void _; +} + +function normalizeTags(tags: any[]) { + const parentMap = new Map(); + const tagImplications: Insertable[] = []; + const tagAliases: Insertable[] = []; + + for (const tag of tags) { + for (const [idx, childId] of (Array.isArray(tag.subTags) ? tag.subTags : []).entries()) { + parentMap.set(childId, [tag.id, idx]); + } + if (!parentMap.has(tag.id)) { + parentMap.set(tag.id, [ROOT_TAG_ID, 0]); + } + + for (const impliedTagId of Array.isArray(tag.impliedTags) ? tag.impliedTags : []) { + tagImplications.push({ tagId: tag.id, impliedTagId: impliedTagId }); + } + + // Convert to Set to get rid of duplicates. + const aliases = new Set(Array.isArray(tag.aliases) ? tag.aliases : []); + for (const alias of aliases) { + tagAliases.push({ tagId: tag.id, alias: alias }); + } + } + + const normalizedTags = tags.map((tag) => ({ + id: tag.id ?? generateId(), + parentId: (parentMap.get(tag.id)?.at(0) ?? fallbackIds.tag) as ID, + idx: (parentMap.get(tag.id)?.at(1) ?? 0) as number, + name: tag.name ?? '(untitled)', + color: tag.color ?? '', + isHidden: serializeBoolean(!!tag.isHidden), + isVisibleInherited: serializeBoolean(!!tag.isVisibleInherited), + isHeader: serializeBoolean(!!tag.isHeader), + description: tag.description ?? '', + dateAdded: serializeDate(tag.dateAdded ? new Date(tag.dateAdded) : new Date()), + })); + + return { normalizedTags, tagImplications, tagAliases }; +} + +function normalizeLocations(sourcelocations: any[]) { + const locationNodes: Insertable[] = []; + const locations: Insertable[] = []; + const subLocations: Insertable[] = []; + const locationTags: Insertable[] = []; + + function normalizeLocationNodeRecursive( + node: any, //LocationDTO | SubLocationDTO, + parentId: ID, + isRoot: boolean, + ) { + const nodeId = node.id ?? generateId(); + const parentIdvalue = isRoot ? nodeId : parentId; + const pathValue = isRoot ? node.path ?? '' : node.name ?? ''; + // Insert into locationNodes + locationNodes.push({ + id: nodeId, + parentId: parentIdvalue, + path: pathValue, + }); + if (isRoot) { + locations.push({ + nodeId: nodeId, + idx: node.index ?? 0, + isWatchingFiles: serializeBoolean(!!node.isWatchingFiles), + dateAdded: serializeDate(node.dateAdded ? new Date(node.dateAdded) : new Date()), + }); + } else { + // Insert into sub_location + subLocations.push({ + nodeId: nodeId, + isExcluded: serializeBoolean(!!node.isExcluded), + }); + } + // Insert tags + for (const tagId of Array.isArray(node.tags) ? node.tags : []) { + locationTags.push({ + nodeId: nodeId, + tagId: tagId, + }); + } + // Recurse for sublocations + for (const sub of Array.isArray(node.subLocations) ? node.subLocations : []) { + normalizeLocationNodeRecursive(sub, nodeId, false); + } + } + + for (const loc of sourcelocations) { + normalizeLocationNodeRecursive(loc, loc.id ?? generateId(), true); + } + return { locationNodes, locations, subLocations }; +} + +function normalizeFiles(sourceFiles: any[], extraProperties: Insertable[]) { + const files: Insertable[] = []; + const fileTags: Insertable[] = []; + const epValText: Insertable[] = []; + const epValNumber: Insertable[] = []; + const epValTime: Insertable[] = []; + + for (const file of sourceFiles) { + const fileId = file.id ?? generateId(); + files.push({ + id: fileId, + ino: file.ino ?? '', + locationId: file.locationId ?? fallbackIds.locationNode, + relativePath: file.relativePath ?? '', + absolutePath: file.absolutePath ?? '', + tagSorting: file.tagsSorting ?? 'none', + name: file.name ?? '(unnamed)', + extension: file.extension ?? '', + size: file.size ?? 10, + width: file.width ?? 10, + height: file.height ?? 10, + dateAdded: serializeDate(file.dateAdded ? new Date(file.dateAdded) : new Date()), + dateModified: serializeDate(file.dateModified ? new Date(file.dateModified) : new Date()), + dateModifiedOS: serializeDate( + file.dateModifiedOS ? new Date(file.dateModifiedOS) : new Date(), + ), + dateLastIndexed: serializeDate( + file.dateLastIndexed ? new Date(file.dateLastIndexed) : new Date(), + ), + dateCreated: serializeDate(file.dateCreated ? new Date(file.dateCreated) : new Date()), + }); + + // file_tags (tags relations) + for (const tagId of Array.isArray(file.tags) ? file.tags : []) { + fileTags.push({ + fileId: fileId, + tagId: tagId, + }); + } + + // ep_values (extra properties relations) + if (file.extraPropertyIDs) { + for (const epId of Array.isArray(file.extraPropertyIDs) ? file.extraPropertyIDs : []) { + const epRow = extraProperties.find((ep: any) => ep.id === epId); + + const value = file.extraProperties?.[epId]; + if (value !== undefined && value !== null) { + const epType = epRow?.type ?? typeof value; + if (epType === 'number') { + epValNumber.push({ + fileId, + epId, + value: value, + }); + } else if (epType === 'timestamp' || value instanceof Date) { + epValTime.push({ + fileId, + epId, + value: serializeDate(value), + }); + } else { + epValText.push({ + fileId, + epId, + value: value, + }); + } + } + } + } + } + return { files, fileTags, epValText, epValNumber, epValTime }; +} + +function normalizeSavedSearches(sourceSearches: any[]) { + const savedSearches: Insertable[] = []; + const searchCriteria: Insertable[] = []; + + for (const search of sourceSearches) { + const searchId = search.id ?? generateId(); + savedSearches.push({ + id: searchId, + name: search.name ?? '(unnamed search)', + idx: search.index ?? 0, + }); + + for (const [idx, crit] of (Array.isArray(search.criteria) ? search.criteria : []).entries()) { + const criteriaId = generateId(); + searchCriteria.push({ + id: criteriaId, + savedSearchId: searchId, + idx: idx, + matchGroup: search.matchAny ? 'any' : 'all', + key: crit.key ?? 'name', + valueType: crit.valueType ?? 'string', + operator: crit.operator ?? 'equals', + jsonValue: JSON.stringify(crit.value ?? 'error'), + }); + } + } + return { savedSearches, searchCriteria }; +} diff --git a/src/backend/config.ts b/src/backend/config.ts index 9fb213d6..124fdc44 100644 --- a/src/backend/config.ts +++ b/src/backend/config.ts @@ -14,6 +14,7 @@ class InlineMigrationProvider implements MigrationProvider { async getMigrations(): Promise> { return { '000_initial': await import('./migrations/000_initial'), + '001_migrateJSON': await import('./migrations/001_migrateJSON'), }; } } diff --git a/src/backend/migrations/000_initial.ts b/src/backend/migrations/000_initial.ts index a3837548..ee15a72f 100644 --- a/src/backend/migrations/000_initial.ts +++ b/src/backend/migrations/000_initial.ts @@ -12,7 +12,7 @@ export async function up(db: Kysely): Promise { await db.schema .createTable('tags') .addColumn('id', 'text', (col) => col.primaryKey().notNull()) - .addColumn('parent_id', 'text') + .addColumn('parent_id', 'text', (col) => col.notNull()) .addColumn('idx', 'integer', (col) => col.notNull()) .addColumn('name', 'text', (col) => col.notNull()) .addColumn('date_added', 'timestamp', (col) => col.notNull().defaultTo(sql`CURRENT_TIMESTAMP`)) @@ -43,28 +43,28 @@ export async function up(db: Kysely): Promise { //// LOCATIONS //// await db.schema - .createTable('location_node') + .createTable('location_nodes') .addColumn('id', 'text', (col) => col.primaryKey().notNull()) .addColumn('parent_id', 'text') .addColumn('path', 'text', (col) => col.notNull()) - .addForeignKeyConstraint('fk_location_node_parent', ['parent_id'], 'location_node', ['id'], (cb) => cb.onDelete('cascade')) + .addForeignKeyConstraint('fk_location_node_parent', ['parent_id'], 'location_nodes', ['id'], (cb) => cb.onDelete('cascade')) .addUniqueConstraint('uq_location_node_parent_path', ['parent_id', 'path']) .execute(); await db.schema - .createTable('location') + .createTable('locations') .addColumn('node_id', 'text', (col) => col.primaryKey().notNull()) .addColumn('date_added', 'timestamp', (col) => col.notNull().defaultTo(sql`CURRENT_TIMESTAMP`)) .addColumn('idx', 'integer', (col) => col.notNull()) .addColumn('is_watching_files', 'boolean', (col) => col.notNull().defaultTo(0)) - .addForeignKeyConstraint('fk_location_node', ['node_id'], 'location_node', ['id'], (cb) => cb.onDelete('cascade')) + .addForeignKeyConstraint('fk_location_node', ['node_id'], 'location_nodes', ['id'], (cb) => cb.onDelete('cascade')) .execute(); await db.schema - .createTable('sub_location') + .createTable('sub_locations') .addColumn('node_id', 'text', (col) => col.primaryKey().notNull()) .addColumn('is_excluded', 'boolean', (col) => col.notNull().defaultTo(0)) - .addForeignKeyConstraint('fk_sub_location_node', ['node_id'], 'location_node', ['id'], (cb) => cb.onDelete('cascade')) + .addForeignKeyConstraint('fk_sub_location_node', ['node_id'], 'location_nodes', ['id'], (cb) => cb.onDelete('cascade')) .execute(); await db.schema @@ -72,7 +72,7 @@ export async function up(db: Kysely): Promise { .addColumn('node_id', 'text', (col) => col.notNull()) .addColumn('tag_id', 'text', (col) => col.notNull()) .addPrimaryKeyConstraint('pk_location_tags', ['node_id', 'tag_id']) - .addForeignKeyConstraint('fk_location_tags_node', ['node_id'], 'location_node', ['id'], (cb) => cb.onDelete('cascade')) + .addForeignKeyConstraint('fk_location_tags_node', ['node_id'], 'location_nodes', ['id'], (cb) => cb.onDelete('cascade')) .addForeignKeyConstraint('fk_location_tags_tag', ['tag_id'], 'tags', ['id'], (cb) => cb.onDelete('cascade')) .execute(); @@ -95,7 +95,7 @@ export async function up(db: Kysely): Promise { .addColumn('width', 'integer') .addColumn('height', 'integer') .addColumn('date_created', 'timestamp') - .addForeignKeyConstraint('fk_files_location', ['location_id'], 'location', ['node_id'], (cb) => cb.onDelete('cascade')) + .addForeignKeyConstraint('fk_files_location', ['location_id'], 'locations', ['node_id'], (cb) => cb.onDelete('cascade')) .addUniqueConstraint('uq_location_node_parent_path', ['location_id', 'relative_path']) .addUniqueConstraint('uq_absolute_path', ['relative_path']) .execute(); @@ -118,8 +118,8 @@ export async function up(db: Kysely): Promise { .addForeignKeyConstraint('fk_file_tags_file', ['file_id'], 'files', ['id'], (cb) => cb.onDelete('cascade')) .addForeignKeyConstraint('fk_file_tags_tag', ['tag_id'], 'tags', ['id'], (cb) => cb.onDelete('cascade')) .execute(); - await db.schema.createIndex('idx_file_tags_tag').on('file_tags').column('tag_id').execute(); - await db.schema.createIndex('idx_file_tags_file').on('file_tags').column('file_id').execute(); + //await db.schema.createIndex('idx_file_tags_tag').on('file_tags').column('tag_id').execute(); + //await db.schema.createIndex('idx_file_tags_file').on('file_tags').column('file_id').execute(); //// EXTRA PROPERTIES //// await db.schema @@ -179,7 +179,7 @@ export async function up(db: Kysely): Promise { .execute(); await db.schema - .createTable('search_criterias') + .createTable('search_criteria') .addColumn('id', 'text', (col) => col.primaryKey().notNull()) .addColumn('saved_search_id', 'text', (col) => col.notNull()) .addColumn('idx', 'integer', (col) => col.notNull()) @@ -188,7 +188,7 @@ export async function up(db: Kysely): Promise { .addColumn('value_type', 'text', (col) => col.notNull()) .addColumn('operator', 'text', (col) => col.notNull()) .addColumn('json_value', 'text', (col) => col.notNull()) - .addForeignKeyConstraint('fk_search_criterias_saved_search', ['saved_search_id'], 'saved_searches', ['id'], (cb) => cb.onDelete('cascade')) + .addForeignKeyConstraint('fk_search_criteria_saved_search', ['saved_search_id'], 'saved_searches', ['id'], (cb) => cb.onDelete('cascade')) .execute(); } @@ -219,7 +219,7 @@ export async function down(db: Kysely): Promise { await db.schema.dropIndex('idx_files_extension').execute(); await db.schema.dropIndex('idx_files_name').execute(); - await db.schema.dropTable('search_criterias').execute(); + await db.schema.dropTable('search_criteria').execute(); await db.schema.dropTable('saved_searches').execute(); await db.schema.dropTable('ep_values_timestamp').execute(); await db.schema.dropTable('ep_values_number').execute(); @@ -228,9 +228,9 @@ export async function down(db: Kysely): Promise { await db.schema.dropTable('file_tags').execute(); await db.schema.dropTable('files').execute(); await db.schema.dropTable('location_tags').execute(); - await db.schema.dropTable('sub_location').execute(); - await db.schema.dropTable('location').execute(); - await db.schema.dropTable('location_node').execute(); + await db.schema.dropTable('sub_locations').execute(); + await db.schema.dropTable('locations').execute(); + await db.schema.dropTable('location_nodes').execute(); await db.schema.dropTable('tag_aliases').execute(); await db.schema.dropTable('tag_implications').execute(); await db.schema.dropTable('tags').execute(); diff --git a/src/backend/migrations/001_migrateJSON.ts b/src/backend/migrations/001_migrateJSON.ts new file mode 100644 index 00000000..382bb21a --- /dev/null +++ b/src/backend/migrations/001_migrateJSON.ts @@ -0,0 +1,31 @@ +import { Kysely } from 'kysely'; +import path from 'path'; +import { readdir, stat } from 'fs/promises'; +import { RendererMessenger } from 'src/ipc/renderer'; +import { restoreFromOldJsonFormat } from '../backup-scheduler'; + +export async function getLastJsonBackupPath(): Promise { + const dir = await RendererMessenger.getDefaultBackupDirectory(); + const files = await readdir(dir); + const jsonFiles = files.filter((f) => f.endsWith('.json')); + if (!jsonFiles.length) { + throw new Error(`No .json files found in ${dir}`); + } + const stats = await Promise.all( + jsonFiles.map(async (f) => ({ + path: path.join(dir, f), + mtime: (await stat(path.join(dir, f))).mtime, + })), + ); + return stats.reduce((a, b) => (a.mtime > b.mtime ? a : b)).path; +} + + +export async function up(db: Kysely): Promise { + restoreFromOldJsonFormat(db, await getLastJsonBackupPath()); +} + +export async function down(_: Kysely): Promise { + // No rollback for imports, maybe delete all the data + void _; +} diff --git a/src/backend/schemaTypes.ts b/src/backend/schemaTypes.ts index e1937b6c..1c0d48e9 100644 --- a/src/backend/schemaTypes.ts +++ b/src/backend/schemaTypes.ts @@ -16,13 +16,20 @@ import { ID } from '../api/id'; import { CriteriaValueType, OperatorType } from 'src/api/search-criteria'; import { FILE_TAGS_SORTING_TYPE, FileDTO } from 'src/api/file'; +export type BooleanAsNumber = number; +export const serializeBoolean = (value: boolean): number => (value ? 1 : 0); +export const deserializeBoolean = (value: number): boolean => value === 1; +export type DateAsNumber = number; +export const serializeDate = (value: Date): number => value.getTime(); +export const deserializeDate = (value: number): Date => new Date(value); + export type AllusionDB_SQL = { tags: Tags; tagImplications: TagImplications; tagAliases: TagAliases; - locationNode: LocationNode; - location: Location; - subLocation: SubLocation; + locationNodes: LocationNodes; + locations: Locations; + subLocations: SubLocations; locationTags: LocationTags; files: Files; fileTags: FileTags; @@ -31,7 +38,7 @@ export type AllusionDB_SQL = { epValuesNumber: EpValuesNumber; epValuesTimestamp: EpValuesTimestamp; savedSearches: SavedSearches; - searchCriterias: SearchCriterias; + searchCriteria: SearchCriteria; }; ///// TAGS ///// @@ -41,11 +48,11 @@ export type Tags = { parentId: ID; //fk idx: number; name: string; - dateAdded: ColumnType; + dateAdded: ColumnType; color: string; - isHidden: boolean; - isVisibleInherited: boolean; - isHeader: boolean; + isHidden: BooleanAsNumber; + isVisibleInherited: BooleanAsNumber; + isHeader: BooleanAsNumber; description: string; }; @@ -61,22 +68,22 @@ export type TagAliases = { /// LOCATIONS /// -export type LocationNode = { +export type LocationNodes = { id: ColumnType; //pk parentId: ID; //fk path: string; }; -export type Location = { +export type Locations = { nodeId: ID; //pk fk - dateAdded: ColumnType; + dateAdded: ColumnType; idx: number; - isWatchingFiles: boolean; + isWatchingFiles: BooleanAsNumber; }; -export type SubLocation = { +export type SubLocations = { nodeId: ID; //pk fk - isExcluded: boolean; + isExcluded: BooleanAsNumber; }; export type LocationTags = { @@ -93,16 +100,16 @@ export type Files = { relativePath: string; absolutePath: string; tagSorting: FILE_TAGS_SORTING_TYPE; - dateAdded: ColumnType; - dateModified: Date; - DateModifiedOS: Date; - dateLastIndexed: Date; + dateAdded: ColumnType; + dateModified: DateAsNumber; + dateModifiedOS: DateAsNumber; + dateLastIndexed: DateAsNumber; name: string; extension: string; size: number; width: number; height: number; - dateCreated: Date; + dateCreated: DateAsNumber; }; export type FileTags = { @@ -116,7 +123,7 @@ export type ExtraProperties = { id: ColumnType; //pk type: string; name: string; - dateAdded: ColumnType; + dateAdded: ColumnType; }; type EpValues = { @@ -127,7 +134,7 @@ type EpValues = { export type EpValuesText = EpValues; export type EpValuesNumber = EpValues; -export type EpValuesTimestamp = EpValues; +export type EpValuesTimestamp = EpValues; /// SAVED SEARCHES /// @@ -137,7 +144,7 @@ export type SavedSearches = { idx: number; }; -export type SearchCriterias = { +export type SearchCriteria = { id: ColumnType; //pk savedSearchId: ID; //fk idx: number; diff --git a/src/main.ts b/src/main.ts index 8382c708..6f83f296 100644 --- a/src/main.ts +++ b/src/main.ts @@ -32,7 +32,6 @@ const basePath = app.getPath('userData'); const preferencesFilePath = path.join(basePath, 'preferences.json'); const windowStateFilePath = path.join(basePath, 'windowState.json'); -const databaseFilePath = path.join(basePath, 'databases', `${DB_NAME}.sqlite`); type PreferencesFile = { checkForUpdatesOnStartup?: boolean; @@ -47,7 +46,6 @@ let mainWindow: BrowserWindow | null = null; let previewWindow: BrowserWindow | null = null; let tray: Tray | null = null; let clipServer: ClipServer | null = null; -let backend: Backend | null = null; function initialize() { console.log('Initializing Allusion...'); @@ -78,7 +76,6 @@ function initialize() { } }); - createBackend(); createWindow(); createPreviewWindow(); @@ -101,10 +98,6 @@ function initialize() { } } -async function createBackend(): Promise { - backend = await Backend.init(databaseFilePath, () => {}); -} - function createWindow() { // Remember window size and position const previousWindowState = getPreviousWindowState(); diff --git a/src/renderer.tsx b/src/renderer.tsx index d89e095d..d1dd7470 100644 --- a/src/renderer.tsx +++ b/src/renderer.tsx @@ -17,6 +17,7 @@ import { promiseRetry } from 'common/timeout'; import { IS_PREVIEW_WINDOW, WINDOW_STORAGE_KEY } from 'common/window'; import { RendererMessenger } from 'src/ipc/renderer'; import Backend from './backend/_deprecated/backend'; +import BackendTest from './backend/backend'; import App from './frontend/App'; import SplashScreen from './frontend/containers/SplashScreen'; import StoreProvider from './frontend/contexts/StoreContext'; @@ -27,6 +28,7 @@ import RootStore from './frontend/stores/RootStore'; import { PREFERENCES_STORAGE_KEY } from './frontend/stores/UiStore'; import BackupScheduler from './backend/_deprecated/backup-scheduler'; import { DB_NAME, dbInit } from './backend/_deprecated/config'; +import path from 'path'; async function main(): Promise { // Render our react components in the div with id 'app' in the html file @@ -52,6 +54,11 @@ async function main(): Promise { async function runMainApp(db: Dexie, root: Root): Promise { const defaultBackupDirectory = await RendererMessenger.getDefaultBackupDirectory(); const backup = new BackupScheduler(db, defaultBackupDirectory); + + const basePath = await RendererMessenger.getPath('userData'); + const databaseTestFilePath = path.join(basePath, 'databases', `${DB_NAME}.sqlite`); + const testbackend = BackendTest.init(databaseTestFilePath, () => {}); + const [backend] = await Promise.all([ Backend.init(db, () => backup.schedule()), fse.ensureDir(defaultBackupDirectory), From 161a987e14b7f3119f449a40c5823484b1ca1381 Mon Sep 17 00:00:00 2001 From: RafaUC Date: Mon, 20 Oct 2025 00:04:30 -0600 Subject: [PATCH 06/19] WIP Creating Backend Class: Fetch methods - Implement backend fetch methods. - Implement support for conjunction per criteria in search files. - Implement optional startup of the backend as a web worker. - Add a small delay before a heavy fetch in average to ensure loading animations start before the backend blocks the render thread, improving UX. --- src/api/extraProperty.ts | 1 + src/api/location.ts | 1 + src/backend/backend.ts | 960 +++++++++++++++++++++- src/backend/backup-scheduler.ts | 107 +-- src/backend/config.ts | 8 +- src/backend/migrations/000_initial.ts | 50 +- src/backend/migrations/001_migrateJSON.ts | 3 +- src/backend/schemaTypes.ts | 27 +- src/frontend/entities/Location.ts | 9 +- src/frontend/entities/Tag.ts | 4 +- src/frontend/stores/FileStore.ts | 65 +- src/renderer.tsx | 87 +- 12 files changed, 1151 insertions(+), 171 deletions(-) diff --git a/src/api/extraProperty.ts b/src/api/extraProperty.ts index 964cf37c..bc16ce79 100644 --- a/src/api/extraProperty.ts +++ b/src/api/extraProperty.ts @@ -3,6 +3,7 @@ import { ID } from './id'; export enum ExtraPropertyType { text = 'text', number = 'number', + //timestamp = 'timestamp', } //ToDo: Only support number and string for now, more types could be added in the future. diff --git a/src/api/location.ts b/src/api/location.ts index e80bb70b..fe1bd155 100644 --- a/src/api/location.ts +++ b/src/api/location.ts @@ -11,6 +11,7 @@ export type LocationDTO = { }; export type SubLocationDTO = { + id: ID; name: string; isExcluded: boolean; subLocations: SubLocationDTO[]; diff --git a/src/backend/backend.ts b/src/backend/backend.ts index 188baae2..d32aeec5 100644 --- a/src/backend/backend.ts +++ b/src/backend/backend.ts @@ -1,29 +1,959 @@ -import { AllusionDB_SQL } from './schemaTypes'; +import { + AllusionDB_SQL, + deserializeBoolean, + deserializeDate, + EpValues, + Files, + serializeBoolean, + serializeDate, +} from './schemaTypes'; +import { expose } from 'comlink'; import SQLite from 'better-sqlite3'; -import { Kysely, SqliteDialect, CamelCasePlugin } from 'kysely'; -import { migrateToLatest } from './config'; +import { + Kysely, + SqliteDialect, + ParseJSONResultsPlugin, + CamelCasePlugin, + sql, + SelectQueryBuilder, + SqlBool, + ExpressionBuilder, + OrderByDirection, + AnyColumn, +} from 'kysely'; +import { migrateToLatest, PAD_STRING_LENGTH } from './config'; +import { DataStorage } from 'src/api/data-storage'; +import { IndexableType } from 'dexie'; +import { + OrderBy, + OrderDirection, + ConditionDTO, + StringOperatorType, + NumberOperatorType, + ArrayOperatorType, + ExtraPropertyOperatorType, + isNumberOperator, + isStringOperator, + PropertyKeys, + StringProperties, +} from 'src/api/data-storage-search'; +import { ExtraProperties, ExtraPropertyDTO } from 'src/api/extraProperty'; +import { FileDTO } from 'src/api/file'; +import { FileSearchDTO } from 'src/api/file-search'; +import { ID } from 'src/api/id'; +import { LocationDTO, SubLocationDTO } from 'src/api/location'; +import { ROOT_TAG_ID, TagDTO } from 'src/api/tag'; +// eslint-disable-next-line @typescript-eslint/no-unused-vars +import { jsonArrayFrom, jsonObjectFrom, jsonBuildObject } from 'kysely/helpers/sqlite'; +import { IS_DEV } from 'common/process'; -export default class Backend { - #db: Kysely; - #notifyChange: () => void; +// Use to debug perfomance. +const USE_TIMING_PROXY = IS_DEV; - constructor(db: Kysely, notifyChange: () => void) { - this.#db = db; +export default class Backend implements DataStorage { + readonly MAX_VARS!: number; + #db!: Kysely; + #notifyChange!: () => void; + /** State variable that indicates if we need to recompute preAggregateJSON */ + #isQueryDirty: boolean = true; - this.#notifyChange = notifyChange; + constructor() { + // Must call init() before using to init the properties. + return USE_TIMING_PROXY ? createTimingProxy(this) : this; } - static async init(dbPath: string, notifyChange: () => void): Promise { + async init(dbPath: string, notifyChange: () => void): Promise { console.info(`SQLite3: Initializing database "${dbPath}"...`); + const database = new SQLite(dbPath, { timeout: 50000 }); + // HACK + // Use a padded string to do natural sorting + database.function('pad_string', { deterministic: true }, (str) => { + return str.replace(/\d+/g, (num: string) => num.padStart(PAD_STRING_LENGTH, '0')); + }); const dialect = new SqliteDialect({ - database: new SQLite(dbPath), + database: database, }); const db = new Kysely({ dialect: dialect, - plugins: [new CamelCasePlugin()], + plugins: [new ParseJSONResultsPlugin(), new CamelCasePlugin()], + log: IS_DEV ? ['query', 'error'] : undefined, // Used only for debugging. + }); + // Instead of initializing this through the constructor, set the class properties here, + // this allows us to use the class as a worker having async await calls at init. + this.#db = db; + this.#notifyChange = notifyChange; + (this as any).MAX_VARS = await getSqliteMaxVariables(db); + + // check if any migration is needed before configure pragma + await migrateToLatest(db); + + // We enable case sensitive like for search queries + await sql`PRAGMA case_sensitive_like = ON;`.execute(db); + // Do not wait for writes + await sql`PRAGMA journal_mode = WAL;`.execute(db); + await sql`PRAGMA synchronous = NORMAL;`.execute(db); + await sql`PRAGMA temp_store = MEMORY;`.execute(db); + await sql`PRAGMA automatic_index = ON;`.execute(db); + await sql`PRAGMA cache_size = -64000;`.execute(db); + await sql`PRAGMA VACUUM;`.execute(db); + await sql`PRAGMA OPTIMIZE;`.execute(db); + + // Create Root Tag if not exists. + if ( + !(await db.selectFrom('tags').selectAll().where('id', '=', ROOT_TAG_ID).executeTakeFirst()) + ) { + await db + .insertInto('tags') + .values({ + id: ROOT_TAG_ID, + parentId: null, + idx: 0, + name: 'Root', + dateAdded: serializeDate(new Date()), + color: '', + isHidden: serializeBoolean(false), + isVisibleInherited: serializeBoolean(false), + description: '', + isHeader: serializeBoolean(false), + }) + .execute(); + } + await this.preAggregateJSON(); + } + + async fetchTags(): Promise { + console.info('SQLite: Fetching tags...'); + const tags = ( + await this.#db + .selectFrom('tags') + .selectAll('tags') + .select((eb) => [ + jsonArrayFrom( + eb + .selectFrom('tags as subTags') + .select('subTags.id') + .whereRef('subTags.parentId', '=', 'tags.id') + .orderBy('subTags.idx'), + ).as('subTags'), + jsonArrayFrom( + eb + .selectFrom('tagImplications') + .select('tagImplications.impliedTagId') + .whereRef('tagImplications.tagId', '=', 'tags.id'), + ).as('impliedTags'), + jsonArrayFrom( + eb + .selectFrom('tagAliases') + .select('tagAliases.alias') + .whereRef('tagAliases.tagId', '=', 'tags.id'), + ).as('aliases'), + ]) + .execute() + ) + // convert data into TagDTO format + .map((dbTag) => ({ + id: dbTag.id, + name: dbTag.name, + dateAdded: deserializeDate(dbTag.dateAdded), + color: dbTag.color, + subTags: dbTag.subTags.map((st) => st.id), + impliedTags: dbTag.impliedTags.map((it) => it.impliedTagId), + isHidden: deserializeBoolean(dbTag.isHidden), + isVisibleInherited: deserializeBoolean(dbTag.isVisibleInherited), + isHeader: deserializeBoolean(dbTag.isHeader), + aliases: dbTag.aliases.map((a) => a.alias), + description: dbTag.description, + })); + return tags; + } + + // Original implementation by Pianissi + // Because creating the jsons takes a lot of time, let's preaggregate them everytime we save our files. + async preAggregateJSON(): Promise { + console.info('SQLite: Updating temp aggregates...'); + this.#isQueryDirty = false; + await sql` + DROP TABLE IF EXISTS file_tag_aggregates_temp; + `.execute(this.#db); + await sql` + DROP TABLE IF EXISTS file_ep_aggregates_temp; + `.execute(this.#db); + + await sql` + CREATE TEMPORARY TABLE IF NOT EXISTS file_tag_aggregates_temp AS + SELECT + file_id, + json_group_array(tag_id) AS tags + FROM file_tags + GROUP BY file_id; + `.execute(this.#db); + await sql` + CREATE TEMPORARY TABLE IF NOT EXISTS file_ep_aggregates_temp AS + SELECT + file_id, + json_group_array(json_object( + 'file_id', file_id, + 'ep_id', ep_id, + 'text_value', text_value, + 'number_value', number_value, + 'timestamp_value', timestamp_value)) + as extra_properties + FROM ep_values; + `.execute(this.#db); + + await sql` + CREATE INDEX IF NOT EXISTS idx_file_tag_aggregates_temp_file ON file_tag_aggregates_temp(file_id); + `.execute(this.#db); + await sql` + CREATE INDEX IF NOT EXISTS idx_file_ep_aggregates_temp_file ON file_ep_aggregates_temp(file_id); + `.execute(this.#db); + } + + async queryFiles( + criteria: ConditionDTO | ConditionDTO[] = [], + sortOptions: SortOptions, + keyInListOptions?: KeyInListOptions, + ): Promise { + const criterias = (Array.isArray(criteria) ? criteria : [criteria]) as ConditionDTO[]; + + if (this.#isQueryDirty) { + await this.preAggregateJSON(); + } + const dbWithTemp = this.#db.withTables<{ + fileTagAggregatesTemp: { + fileId: ID; + tags: ID[]; + }; + fileEpAggregatesTemp: { + fileId: ID; + extraProperties: EpValues[]; + }; + }>(); + // Apply the filter criterias expressions to the files QueryBuilder and execute the query. + let query; + query = dbWithTemp + .selectFrom('files') + .leftJoin('fileTagAggregatesTemp as ft', 'ft.fileId', 'files.id') + .leftJoin('fileEpAggregatesTemp as fe', 'fe.fileId', 'files.id') + .selectAll('files') + .select(['ft.tags', 'fe.extraProperties']); + query = applyFileFilters(query, criterias); + query = await applySortOrder(this.#db, query, sortOptions); + if (keyInListOptions) { + query = query.where(keyInListOptions.key, 'in', keyInListOptions.values); + } + + const files = (await query.execute()).map((dbFile): FileDTO => { + // convert data into FileDTO format + const extraPropertyIDs: ID[] = []; + const extraProperties: ExtraProperties = {}; + for (const ep of dbFile.extraProperties ?? []) { + extraPropertyIDs.push(ep.epId); + const val = ep.textValue ?? ep.numberValue; // ?? ep.timestampValue; + if (val) { + extraProperties[ep.epId] = val; + } + } + return { + id: dbFile.id, + ino: dbFile.ino, + locationId: dbFile.locationId, + relativePath: dbFile.relativePath, + absolutePath: dbFile.absolutePath, + tagsSorting: dbFile.tagSorting, + dateAdded: deserializeDate(dbFile.dateAdded), + dateModified: deserializeDate(dbFile.dateModified), + dateModifiedOS: deserializeDate(dbFile.dateModifiedOS), + dateLastIndexed: deserializeDate(dbFile.dateLastIndexed), + dateCreated: deserializeDate(dbFile.dateCreated), + name: dbFile.name, + extension: dbFile.extension, + size: dbFile.size, + width: dbFile.width, + height: dbFile.height, + tags: dbFile.tags ?? [], + extraPropertyIDs: extraPropertyIDs, + extraProperties: extraProperties, + }; + }); + return files; + } + + async fetchFiles( + order: OrderBy, + fileOrder: OrderDirection, + useNaturalOrdering: boolean, + extraPropertyID?: ID, + ): Promise { + console.info('SQLite: Fetching all files...'); + return this.queryFiles(undefined, { + order, + direction: fileOrder, + useNaturalOrdering, + extraPropertyID, + }); + } + + async searchFiles( + criteria: ConditionDTO | [ConditionDTO, ...ConditionDTO[]], + order: OrderBy, + fileOrder: OrderDirection, + useNaturalOrdering: boolean, + extraPropertyID?: ID, + matchAny?: boolean, + ): Promise { + console.info('SQLite: Searching files...'); + return this.queryFiles(criteria, { + order, + direction: fileOrder, + useNaturalOrdering, + extraPropertyID, + }); + } + + async fetchFilesByID(ids: ID[]): Promise { + console.info('SQLite: Fetching files by ID...'); + return this.queryFiles(undefined, { order: 'dateAdded' }, { key: 'id', values: ids }); + } + + async fetchFilesByKey(key: keyof FileDTO, value: IndexableType): Promise { + console.info('SQLite: Fetching files by key...'); + if (!['tags', 'extraProperties', 'extraPropertyIDs'].includes(key) && Array.isArray(value)) { + return this.queryFiles( + undefined, + { order: 'dateAdded' }, + { key: key as keyof Files, values: value }, + ); + } + console.error('fetchFilesByKey error: Key or values not supported.'); + return []; + } + + async fetchLocations(): Promise { + console.info('SQLite: Fetching locations...'); + /** Map to quicly find a node and his parent */ + const locationNodesMap = new Map(); + const locations: LocationDTO[] = ( + await this.#db + .selectFrom('locations') + .innerJoin('locationNodes as node', 'node.id', 'locations.nodeId') + .selectAll() + .select((eb) => [ + jsonArrayFrom( + eb + .selectFrom('locationTags') + .select('locationTags.tagId') + .whereRef('locationTags.nodeId', '=', 'locations.nodeId'), + ).as('tags'), + ]) + .execute() + ).map((dbLoc) => { + // convert data into LocationDTO format + const lc: LocationDTO = { + id: dbLoc.id, + path: dbLoc.path, + dateAdded: deserializeDate(dbLoc.dateAdded), + subLocations: [], + tags: dbLoc.tags.map((t) => t.tagId), + index: dbLoc.idx, + isWatchingFiles: deserializeBoolean(dbLoc.isWatchingFiles), + }; + locationNodesMap.set(dbLoc.id, [lc, dbLoc.parentId]); + return lc; + }); + const subLocations: SubLocationDTO[] = ( + await this.#db + .selectFrom('subLocations') + .innerJoin('locationNodes as node', 'node.id', 'subLocations.nodeId') + .selectAll() + .select((eb) => [ + jsonArrayFrom( + eb + .selectFrom('locationTags') + .select('locationTags.tagId') + .whereRef('locationTags.nodeId', '=', 'subLocations.nodeId'), + ).as('tags'), + ]) + .execute() + ).map((dbLoc) => { + // convert data into SubLocationDTO format + const slc: SubLocationDTO = { + id: dbLoc.id, + name: dbLoc.path, + subLocations: [], + tags: dbLoc.tags.map((t) => t.tagId), + isExcluded: deserializeBoolean(dbLoc.isExcluded), + }; + locationNodesMap.set(dbLoc.id, [slc, dbLoc.parentId]); + return slc; }); - migrateToLatest(db); - const backend = new Backend(db, notifyChange); - return backend; + // Insert sublocations into their parents + for (const subLocation of subLocations) { + const parent = locationNodesMap.get(locationNodesMap.get(subLocation.id)?.[1] ?? '')?.[0]; + if (parent) { + parent.subLocations.push(subLocation); + } + } + return locations; + } + + async fetchSearches(): Promise { + console.info('SQLite: Fetching saved searches...'); + const searches = ( + await this.#db + .selectFrom('savedSearches') + .selectAll('savedSearches') + .select((eb) => [ + jsonArrayFrom( + eb + .selectFrom('searchCriteria as criteria') + .select([ + 'id', + 'savedSearchId', + 'idx', + 'matchGroup', + 'key', + 'valueType', + 'operator', + 'jsonValue', + ]) + .whereRef('criteria.savedSearchId', '=', 'savedSearches.id'), + ).as('criteria'), + ]) + .execute() + ).map( + // convert data into FileSearchDTO format + (dbSearch): FileSearchDTO => ({ + id: dbSearch.id, + name: dbSearch.name, + criteria: dbSearch.criteria.map((dbCrit) => ({ + key: dbCrit.key, + operator: dbCrit.operator, + valueType: dbCrit.valueType, + value: + // the ParseJSONResultsPlugin already parses the arrays but not strings + dbCrit.valueType === 'string' + ? JSON.parse(dbCrit.jsonValue as string) + : dbCrit.jsonValue, + })), + index: dbSearch.idx, + }), + ); + return searches; + } + + async fetchExtraProperties(): Promise { + console.info('SQLite: Fetching extra properties...'); + const eProperties = ( + await this.#db.selectFrom('extraProperties').selectAll().orderBy('name').execute() + ).map( + (dbEp): ExtraPropertyDTO => ({ + id: dbEp.id, + type: dbEp.type, + name: dbEp.name, + dateAdded: deserializeDate(dbEp.dateAdded), + }), + ); + return eProperties; + } + + async createTag(tag: TagDTO): Promise { + console.warn('Method not implemented.'); + } + async createFilesFromPath(path: string, files: FileDTO[]): Promise { + console.warn('Method not implemented.'); + } + async createLocation(location: LocationDTO): Promise { + console.warn('Method not implemented.'); + } + async createSearch(search: FileSearchDTO): Promise { + console.warn('Method not implemented.'); + } + async createExtraProperty(extraProperty: ExtraPropertyDTO): Promise { + console.warn('Method not implemented.'); + } + async saveTag(tag: TagDTO): Promise { + console.warn('Method not implemented.'); + } + async saveFiles(files: FileDTO[]): Promise { + console.warn('Method not implemented.'); + } + async saveLocation(location: LocationDTO): Promise { + console.warn('Method not implemented.'); + } + async saveSearch(search: FileSearchDTO): Promise { + console.warn('Method not implemented.'); + } + async saveExtraProperty(extraProperty: ExtraPropertyDTO): Promise { + console.warn('Method not implemented.'); + } + async removeTags(tags: ID[]): Promise { + console.warn('Method not implemented.'); + } + async mergeTags(tagToBeRemoved: ID, tagToMergeWith: ID): Promise { + console.warn('Method not implemented.'); + } + async removeFiles(files: ID[]): Promise { + console.warn('Method not implemented.'); + } + async removeLocation(location: ID): Promise { + console.warn('Method not implemented.'); + } + async removeSearch(search: ID): Promise { + console.warn('Method not implemented.'); + } + async removeExtraProperties(extraProperty: ID[]): Promise { + console.warn('Method not implemented.'); + } + async countFiles(): Promise<[fileCount: number, untaggedFileCount: number]> { + console.warn('Method not implemented.'); + return [0, 0]; + } + async clear(): Promise { + console.warn('Method not implemented.'); + } +} + +// https://lorefnon.tech/2019/03/24/using-comlink-with-typescript-and-worker-loader/ +expose(Backend, self); + +// Creates a proxy that wraps the Backend instance to log the execution time of its methods. +function createTimingProxy(obj: Backend): Backend { + console.log('Creating timing proxy for Backend'); + return new Proxy(obj, { + get(target, prop, receiver) { + const original = Reflect.get(target, prop, receiver); + if (typeof original === 'function') { + return (...args: any[]) => { + const startTime = performance.now(); + const result = original.apply(target, args); + // Ensure both synchronous and asynchronous results are handled uniformly + return Promise.resolve(result).then((res) => { + const endTime = performance.now(); + console.log(`[Timing] ${String(prop)} took ${(endTime - startTime).toFixed(2)}ms`); + return res; + }); + }; + } + return original; + }, + }); +} + +export async function getSqliteMaxVariables(db: Kysely): Promise { + const rows = (await sql`PRAGMA compile_options`.execute(db)).rows; + const opt: any = rows.find((r: any) => r.compileOptions?.includes('MAX_VARIABLE_NUMBER')); + if (!opt) { + console.warn('MAX_VARIABLE_NUMBER not found, using 22766'); + return 22766; + } + const maxVars = parseInt(opt.compileOptions.split('=')[1], 10); + return isNaN(maxVars) ? 22766 : maxVars; +} + +export function computeBatchSize(maxVars: number, sampleObject?: Record): number { + if (!sampleObject) { + return 501; + } + const numCols = Object.keys(sampleObject).length; + return Math.floor(maxVars / numCols); +} + +/////////////////// +///// SORTING ///// +/////////////////// + +const exampleFileDTO: FileDTO = { + id: '', + ino: '', + name: '', + relativePath: '', + absolutePath: '', + locationId: '', + extension: 'jpg', + tagsSorting: 'hierarchy', + size: 0, + width: 0, + height: 0, + dateAdded: new Date(), + dateCreated: new Date(), + dateLastIndexed: new Date(), + dateModified: new Date(), + dateModifiedOS: new Date(), + extraProperties: {}, + extraPropertyIDs: [], + tags: [], +}; + +function isFileDTOPropString(prop: PropertyKeys): prop is StringProperties { + return typeof exampleFileDTO[prop] === 'string'; +} + +type SortOptions = { + order: OrderBy; + direction?: OrderDirection; + useNaturalOrdering?: boolean; + extraPropertyID?: string; +}; + +// Original implementation by Pianissi +async function applySortOrder( + db: Kysely, + q: SelectQueryBuilder, + sortOptions: SortOptions, +): Promise> { + const { direction, useNaturalOrdering, extraPropertyID } = sortOptions; + let { order } = sortOptions; + + const sqlDirection: OrderByDirection = direction === OrderDirection.Asc ? 'asc' : 'desc'; + // because of how the joined table is returned as, we need to aggregate a sort value in the joined table which can be used as a key + if (order === 'extraProperty') { + q = q.orderBy('sortValue' as any, sqlDirection); + order = 'dateAdded'; + } + + if (order === 'random') { + q = q.orderBy(sql`RANDOM()`); + } else if (useNaturalOrdering && isFileDTOPropString(order)) { + q = q.orderBy(sql`PAD_STRING(files.${sql.ref(order)})`, sqlDirection); + } else { + // Default + q = q.orderBy(`files.${order}` as any, sqlDirection); + } + + /// + /// extraproperty optional value /// + + if (!extraPropertyID) { + return q.select(sql`NULL`.as('sortValue')); + } + const extraProp = await db + .selectFrom('extraProperties' as any) + .select('type') + .where('id' as any, '=', extraPropertyID) + .executeTakeFirst(); + if (!extraProp) { + return q.select(sql`NULL`.as('sortValue')); + } + // maping value type to column + // TODO: add timestamp mapping when implementing + const valueColumn = extraProp.type === 'text' ? 'textValue' : 'numberValue'; + // Left join the corresponding extraProperty value and select it as sortValue + return q + .leftJoin('epValues', (join) => + join.onRef('epValues.fileId', '=', 'files.id').on('epValues.epId', '=', extraPropertyID), + ) + .select(`epValues.${valueColumn} as sortValue` as any) as any; +} + +/////////////////////////// +///////// FILTERS ///////// +/////////////////////////// + +type KeyInListOptions = { key: AnyColumn; values: any[] }; + +type SearchConjunction = 'and' | 'or'; + +export type ConditionWithConjunction = ConditionDTO & { + conjunction?: SearchConjunction; +}; + +function applyFileFilters( + q: SelectQueryBuilder, + criterias: ConditionWithConjunction[], +): SelectQueryBuilder { + if (criterias.length === 0) { + return q; + } + + // group criterias by consecutive conjuntions + const groups: Array<{ + conjunction: SearchConjunction; + criterias: ConditionDTO[]; + }> = []; + + let currentGroup = { + conjunction: criterias[0].conjunction ?? 'and', + criterias: [criterias[0]], + }; + + for (let i = 1; i < criterias.length; i++) { + const crit = criterias[i]; + const conj = crit.conjunction ?? 'and'; + + // if same group + if (conj === currentGroup.conjunction) { + currentGroup.criterias.push(crit); + // else create new group + } else { + groups.push(currentGroup); + currentGroup = { + conjunction: conj, + criterias: [crit], + }; + } + } + groups.push(currentGroup); + + // create conjuction grouped expressions and concatenate them + for (const group of groups) { + const groupExpression = (eb: ExpressionBuilder) => { + const expressions = group.criterias.map((crit) => expressionFromCriteria(eb, crit)); + + return group.conjunction === 'or' ? eb.or(expressions) : eb.and(expressions); + }; + + q = q.where(groupExpression); + } + + return q; +} + +const expressionFromCriteria = ( + eb: ExpressionBuilder, + crit: ConditionDTO, +) => { + switch (crit.valueType) { + case 'string': + return applyStringCondition(eb, crit.key, crit.operator, crit.value); + case 'number': + return applyNumberCondition(eb, crit.key, crit.operator, crit.value); + case 'date': + return applyDateCondition(eb, crit.key, crit.operator, crit.value); + case 'array': + return applyTagArrayCondition(eb, crit.key, crit.operator, crit.value); + case 'indexSignature': + return applyExtraPropertyCondition(eb, crit.key, crit.operator, crit.value); + } +}; + +function applyStringCondition( + eb: ExpressionBuilder, + key: keyof Files, + operator: StringOperatorType, + value: string, +) { + switch (operator) { + case 'equals': + return eb(`files.${key}`, '=', value); + case 'equalsIgnoreCase': + return eb(sql`lower(${sql.ref(`files.${key}`)})`, '=', value.toLowerCase()); + case 'notEqual': + return eb(`files.${key}`, '!=', value); + case 'contains': + return eb(`files.${key}`, 'like', `%${value}%`); + case 'notContains': + // use NOT LIKE + return eb(`files.${key}`, 'not like', `%${value}%`); + case 'startsWith': + return eb(`files.${key}`, 'like', `${value}%`); + case 'startsWithIgnoreCase': + return eb(sql`lower(${sql.ref(`files.${key}`)})`, 'like', `${value.toLowerCase()}%`); + case 'notStartsWith': + return eb(`files.${key}`, 'not like', `${value}%`); + default: + const _exhaustiveCheck: never = operator; + return _exhaustiveCheck; + } +} + +function applyNumberCondition( + eb: ExpressionBuilder, + key: keyof Files, + operator: NumberOperatorType, + value: number, +) { + switch (operator) { + case 'equals': + return eb(`files.${key}`, '=', value); + case 'notEqual': + return eb(`files.${key}`, '!=', value); + case 'smallerThan': + return eb(`files.${key}`, '<', value); + case 'smallerThanOrEquals': + return eb(`files.${key}`, '<=', value); + case 'greaterThan': + return eb(`files.${key}`, '>', value); + case 'greaterThanOrEquals': + return eb(`files.${key}`, '>=', value); + default: + const _exhaustiveCheck: never = operator; + return _exhaustiveCheck; + } +} + +function applyDateCondition( + eb: ExpressionBuilder, + key: keyof Files, + operator: NumberOperatorType, + value: Date, +) { + // In DB dates are DateAsNumber, convert Date to number. + const startOfDay = new Date(value); + startOfDay.setHours(0, 0, 0, 0); + const endOfDay = new Date(value); + endOfDay.setHours(23, 59, 59, 999); + const s = serializeDate(startOfDay); + const e = serializeDate(endOfDay); + + switch (operator) { + case 'equals': + // equal to this day, so between 0:00 and 23:59 + return eb(`files.${key}`, '>=', s).and(`files.${key}`, '<=', e); + case 'notEqual': + // not equal to this day, so before 0:00 or after 23:59 + return eb.or([eb(`files.${key}`, '<', s), eb(`files.${key}`, '>', e)]); + case 'smallerThan': + return eb(`files.${key}`, '<', s); + case 'smallerThanOrEquals': + return eb(`files.${key}`, '<=', e); + case 'greaterThan': + return eb(`files.${key}`, '>', e); + case 'greaterThanOrEquals': + return eb(`files.${key}`, '>=', s); + default: + const _exhaustiveCheck: never = operator; + return _exhaustiveCheck; + } +} + +/** + * Note / TODO: + * Array and IndexSignature condition appliers would work the same way as the next two examples. + * They could be used for any array or index signature property, but since those properties + * only exist in the DTO objects (not in the raw fetched data from the database) and are instead + * represented through relation tables, a mapping between the DTO property key and the corresponding + * subquery table must be defined. + * + * Currently, since only the "tags" and "extraProperties" properties use these conditions, + * the mapping is hard-coded to those specific database tables in each case. + */ + +function applyTagArrayCondition( + eb: ExpressionBuilder, + key: keyof FileDTO, + operator: ArrayOperatorType, + values: any[], +) { + // If the key is not tags return a neutral condition (always true) to avoid breaking + // the WHERE clause when no filter is applied + if (key !== 'tags') { + return sql`TRUE`; + } + if (values.length === 0) { + const anyTagFiles = eb.selectFrom('fileTags').select('fileId').distinct(); + if (operator === 'contains') { + // files with 0 tags -> NOT EXISTS fileTags for this file + return eb.not(eb('files.id', 'in', anyTagFiles)); + } else { + // notContains empty -> files which have at least one tag + return eb('files.id', 'in', anyTagFiles); + } + } else { + const matchingFiles = eb + .selectFrom('fileTags') + .select('fileId') + .where('tagId', 'in', values) + .distinct(); + if (operator === 'contains') { + return eb('files.id', 'in', matchingFiles); + } else { + // notContains: ensure NOT EXISTS any tag in the list for that file + return eb.not(eb('files.id', 'in', matchingFiles)); + } + } +} + +function applyExtraPropertyCondition( + eb: ExpressionBuilder, + key: keyof FileDTO, + operator: NumberOperatorType | StringOperatorType | ExtraPropertyOperatorType, + valueTuple: [string, any], +) { + // If the key is not extraProperties return a neutral condition (always true) + // to avoid breaking the WHERE clause when no filter is applied + if (key !== 'extraProperties') { + return sql`TRUE`; + } + const [epID, innerValue] = valueTuple; + let subquery = eb + .selectFrom('extraProperties') + .innerJoin('epValues', 'extraProperties.id', 'epValues.epId') + .select('epValues.fileId') + .distinct() + .where('extraProperties.id', '=', epID); + //.whereRef('epValues.fileId', '=', sql.ref('files.id')); + + if (operator === 'existsInFile') { + return eb('files.id', 'in', subquery); + } + + if (operator === 'notExistsInFile') { + return eb.not(eb('files.id', 'in', subquery)); + } + + // For typed comparisons add an echtra filter to the subquery + if (typeof innerValue === 'number' && isNumberOperator(operator)) { + // prettier-ignore + // use epValues.numberValue + switch (operator) { + case 'equals': + subquery = subquery.where('epValues.numberValue', '=', innerValue); + break; + case 'notEqual': + subquery = subquery.where('epValues.numberValue', '!=', innerValue); + break; + case 'greaterThan': + subquery = subquery.where('epValues.numberValue', '>', innerValue); + break; + case 'greaterThanOrEquals': + subquery = subquery.where('epValues.numberValue', '>=', innerValue); + break; + case 'smallerThan': + subquery = subquery.where('epValues.numberValue', '<', innerValue); + break; + case 'smallerThanOrEquals': + subquery = subquery.where('epValues.numberValue', '<=', innerValue); + break; + default: + const _exhaustiveCheck: never = operator; + return _exhaustiveCheck; + } + } else if (typeof innerValue === 'string' && isStringOperator(operator)) { + // prettier-ignore + // use epValues.textValue + switch (operator) { + case 'equals': + subquery = subquery.where('epValues.textValue', '=', innerValue); + break; + case 'equalsIgnoreCase': + subquery = subquery.where(sql`LOWER(epValues.textValue)`, '=', innerValue.toLowerCase()); + break; + case 'notEqual': + subquery = subquery.where('epValues.textValue', '=', innerValue); + break; + case 'contains': + subquery = subquery.where('epValues.textValue', 'like', `%${innerValue}%`); + break; + case 'notContains': + subquery = subquery.where('epValues.textValue', 'not like', `%${innerValue}%`); + break; + case 'startsWith': + subquery = subquery.where('epValues.textValue', 'like', `${innerValue}%`); + break; + case 'notStartsWith': + subquery = subquery.where('epValues.textValue', 'not like', `${innerValue}%`); + break; + case 'startsWithIgnoreCase': + subquery = subquery.where(sql`LOWER(epValues.textValue)`, 'like', `${innerValue.toLowerCase()}%`); + break; + default: + const _exhaustiveCheck: never = operator; + return _exhaustiveCheck; + } + } else { + throw new Error('Unsupported indexSignature value type'); } + // Return the expression + return eb('files.id', 'in', subquery); } diff --git a/src/backend/backup-scheduler.ts b/src/backend/backup-scheduler.ts index b0ca075d..0c53618c 100644 --- a/src/backend/backup-scheduler.ts +++ b/src/backend/backup-scheduler.ts @@ -2,12 +2,11 @@ import { promises as fs } from 'fs'; import { Insertable, InsertObject, Kysely, sql } from 'kysely'; import { generateId, ID } from 'src/api/id'; import { ROOT_TAG_ID } from 'src/api/tag'; +import { setTimeout as delay } from 'node:timers/promises'; import { AllusionDB_SQL, - EpValuesNumber, - EpValuesText, - EpValuesTimestamp, ExtraProperties, + EpValues, Files, FileTags, LocationNodes, @@ -22,6 +21,7 @@ import { TagAliases, } from './schemaTypes'; import { ExtraPropertyType } from 'src/api/extraProperty'; +import { computeBatchSize, getSqliteMaxVariables } from './backend'; const fallbackIds = { tag: 'fallback_tag', @@ -46,27 +46,51 @@ export async function restoreFromOldJsonFormat( json.data.data.map((table: any) => [table.tableName, table.rows]), ); - const saveEntries = async < - TableName extends keyof AllusionDB_SQL, // nombre de tabla (clave) - >( + const MAX_VARS = await getSqliteMaxVariables(db); + console.log(`MAX_VARS: ${MAX_VARS}`); + + const saveEntries = async ( entityName: TableName, entries: InsertObject[], ) => { let errors = 0; - console.log(`Importing ${entries.length} ${entityName} from old format.`); + const batchSize = computeBatchSize(MAX_VARS, entries.find(Boolean)); + const MAX_RETRIES = 5; + const BASE_DELAY_MS = 100; + console.log( + `Importing ${entries.length} ${entityName} from old format. (Batch size: ${batchSize})`, + ); await db.transaction().execute(async (trx) => { - const batchSize = 2000; for (let i = 0; i < entries.length; i += batchSize) { - try { - const batch = entries.slice(i, i + batchSize); - await trx - .insertInto(entityName) - .values(batch) - .onConflict((oc) => oc.doNothing()) - .execute(); - } catch (err) { - console.warn(`Insert ${entityName} error`, err); - errors += batchSize; + const batch = entries.slice(i, i + batchSize); + + let attempt = 0; + while (true) { + try { + await trx + .insertInto(entityName) + .values(batch) + .onConflict((oc) => oc.doNothing()) + .execute(); + // If success breack the while + break; + } catch (err: any) { + if (err.code === 'SQLITE_BUSY' && attempt < MAX_RETRIES) { + const wait = BASE_DELAY_MS * Math.pow(2, attempt); + console.warn( + `SQLITE_BUSY on ${entityName} (batch ${ + i / batchSize + 1 + }). Retrying in ${wait} ms... (attempt ${attempt + 1}/${MAX_RETRIES})`, + ); + attempt++; + await delay(wait); + continue; // retry + } + + console.warn(`❌ Error al insertar ${entityName}`, err); + errors += batchSize; + break; // stop retry loop for this batch + } } } }); @@ -121,7 +145,7 @@ export async function restoreFromOldJsonFormat( .values({ id: fallbackIds.extraProperty, name: 'Fallback Property', - type: 'text', + type: ExtraPropertyType.text, dateAdded: serializeDate(new Date()), }) .onConflict((oc) => oc.doNothing()) @@ -156,16 +180,11 @@ export async function restoreFromOldJsonFormat( await saveEntries('extraProperties', extraProperties); // Import files - const { files, fileTags, epValText, epValNumber, epValTime } = normalizeFiles( - tables.files ?? [], - extraProperties, - ); + const { files, fileTags, epVal } = normalizeFiles(tables.files ?? [], extraProperties); await saveEntries('files', files); await saveEntries('fileTags', fileTags); - await saveEntries('epValuesText', epValText); - await saveEntries('epValuesNumber', epValNumber); - await saveEntries('epValuesTimestamp', epValTime); + await saveEntries('epValues', epVal); // Import seved searches const { savedSearches, searchCriteria } = normalizeSavedSearches(tables.searches ?? []); @@ -192,11 +211,6 @@ export async function restoreFromOldJsonFormat( console.log('===================================================='); } -export async function down(_: Kysely): Promise { - // No rollback for imports, maybe delete fallback and imported data - void _; -} - function normalizeTags(tags: any[]) { const parentMap = new Map(); const tagImplications: Insertable[] = []; @@ -223,7 +237,8 @@ function normalizeTags(tags: any[]) { const normalizedTags = tags.map((tag) => ({ id: tag.id ?? generateId(), - parentId: (parentMap.get(tag.id)?.at(0) ?? fallbackIds.tag) as ID, + parentId: + tag.id === ROOT_TAG_ID ? null : ((parentMap.get(tag.id)?.at(0) ?? fallbackIds.tag) as ID), idx: (parentMap.get(tag.id)?.at(1) ?? 0) as number, name: tag.name ?? '(untitled)', color: tag.color ?? '', @@ -249,7 +264,7 @@ function normalizeLocations(sourcelocations: any[]) { isRoot: boolean, ) { const nodeId = node.id ?? generateId(); - const parentIdvalue = isRoot ? nodeId : parentId; + const parentIdvalue = isRoot ? null : parentId; const pathValue = isRoot ? node.path ?? '' : node.name ?? ''; // Insert into locationNodes locationNodes.push({ @@ -293,9 +308,7 @@ function normalizeLocations(sourcelocations: any[]) { function normalizeFiles(sourceFiles: any[], extraProperties: Insertable[]) { const files: Insertable[] = []; const fileTags: Insertable[] = []; - const epValText: Insertable[] = []; - const epValNumber: Insertable[] = []; - const epValTime: Insertable[] = []; + const epVal: Insertable[] = []; for (const file of sourceFiles) { const fileId = file.id ?? generateId(); @@ -314,7 +327,11 @@ function normalizeFiles(sourceFiles: any[], extraProperties: Insertable> { diff --git a/src/backend/migrations/000_initial.ts b/src/backend/migrations/000_initial.ts index ee15a72f..06577b2a 100644 --- a/src/backend/migrations/000_initial.ts +++ b/src/backend/migrations/000_initial.ts @@ -12,7 +12,7 @@ export async function up(db: Kysely): Promise { await db.schema .createTable('tags') .addColumn('id', 'text', (col) => col.primaryKey().notNull()) - .addColumn('parent_id', 'text', (col) => col.notNull()) + .addColumn('parent_id', 'text') .addColumn('idx', 'integer', (col) => col.notNull()) .addColumn('name', 'text', (col) => col.notNull()) .addColumn('date_added', 'timestamp', (col) => col.notNull().defaultTo(sql`CURRENT_TIMESTAMP`)) @@ -23,6 +23,7 @@ export async function up(db: Kysely): Promise { .addColumn('description', 'text') .addForeignKeyConstraint('fk_tag_parent', ['parent_id'], 'tags', ['id'], (cb) => cb.onDelete('cascade')) .execute(); + await db.schema.createIndex('idx_tags_parent').on('tags').column('parent_id').execute(); await db.schema .createTable('tag_implications') @@ -96,7 +97,6 @@ export async function up(db: Kysely): Promise { .addColumn('height', 'integer') .addColumn('date_created', 'timestamp') .addForeignKeyConstraint('fk_files_location', ['location_id'], 'locations', ['node_id'], (cb) => cb.onDelete('cascade')) - .addUniqueConstraint('uq_location_node_parent_path', ['location_id', 'relative_path']) .addUniqueConstraint('uq_absolute_path', ['relative_path']) .execute(); // await db.schema.createIndex('idx_files_name').on('files').column('name').execute(); @@ -118,8 +118,8 @@ export async function up(db: Kysely): Promise { .addForeignKeyConstraint('fk_file_tags_file', ['file_id'], 'files', ['id'], (cb) => cb.onDelete('cascade')) .addForeignKeyConstraint('fk_file_tags_tag', ['tag_id'], 'tags', ['id'], (cb) => cb.onDelete('cascade')) .execute(); - //await db.schema.createIndex('idx_file_tags_tag').on('file_tags').column('tag_id').execute(); - //await db.schema.createIndex('idx_file_tags_file').on('file_tags').column('file_id').execute(); + await db.schema.createIndex('idx_file_tags_tag').on('file_tags').column('tag_id').execute(); + await db.schema.createIndex('idx_file_tags_file').on('file_tags').column('file_id').execute(); //// EXTRA PROPERTIES //// await db.schema @@ -131,44 +131,19 @@ export async function up(db: Kysely): Promise { .execute(); await db.schema - .createTable('ep_values_text') + .createTable('ep_values') .addColumn('file_id', 'text', (col) => col.notNull()) .addColumn('ep_id', 'text', (col) => col.notNull()) - .addColumn('value', 'text', (col) => col.notNull()) + .addColumn('text_value', 'text') + .addColumn('number_value', 'integer') + .addColumn('timestamp_value', 'timestamp') .addPrimaryKeyConstraint('pk_ep_values_text', ['file_id', 'ep_id']) .addForeignKeyConstraint('fk_ep_values_text_file', ['file_id'], 'files', ['id'], (cb) => cb.onDelete('cascade')) .addForeignKeyConstraint('fk_ep_values_text_ep', ['ep_id'], 'extra_properties', ['id'], (cb) => cb.onDelete('cascade')) .execute(); - await db.schema.createIndex('idx_ep_values_text_file').on('ep_values_text').column('file_id').execute(); - await db.schema.createIndex('idx_ep_values_text_value').on('ep_values_text').column('value').execute(); - await db.schema.createIndex('idx_ep_values_text_ep').on('ep_values_text').column('ep_id').execute(); - - - await db.schema - .createTable('ep_values_number') - .addColumn('file_id', 'text', (col) => col.notNull()) - .addColumn('ep_id', 'text', (col) => col.notNull()) - .addColumn('value', 'integer', (col) => col.notNull()) - .addPrimaryKeyConstraint('pk_ep_values_number', ['file_id', 'ep_id']) - .addForeignKeyConstraint('fk_ep_values_number_file', ['file_id'], 'files', ['id'], (cb) => cb.onDelete('cascade')) - .addForeignKeyConstraint('fk_ep_values_number_ep', ['ep_id'], 'extra_properties', ['id'], (cb) => cb.onDelete('cascade')) - .execute(); - await db.schema.createIndex('idx_ep_values_number_file').on('ep_values_number').column('file_id').execute(); - await db.schema.createIndex('idx_ep_values_number_value').on('ep_values_number').column('value').execute(); - await db.schema.createIndex('idx_ep_values_number_ep').on('ep_values_number').column('ep_id').execute(); - - await db.schema - .createTable('ep_values_timestamp') - .addColumn('file_id', 'text', (col) => col.notNull()) - .addColumn('ep_id', 'text', (col) => col.notNull()) - .addColumn('value', 'timestamp', (col) => col.notNull()) - .addPrimaryKeyConstraint('pk_ep_values_timestamp', ['file_id', 'ep_id']) - .addForeignKeyConstraint('fk_ep_values_timestamp_file', ['file_id'], 'files', ['id'], (cb) => cb.onDelete('cascade')) - .addForeignKeyConstraint('fk_ep_values_timestamp_ep', ['ep_id'], 'extra_properties', ['id'], (cb) => cb.onDelete('cascade')) - .execute(); - await db.schema.createIndex('idx_ep_values_timestamp_file').on('ep_values_timestamp').column('file_id').execute(); - await db.schema.createIndex('idx_ep_values_timestamp_value').on('ep_values_timestamp').column('value').execute(); - await db.schema.createIndex('idx_ep_values_timestamp_ep').on('ep_values_timestamp').column('ep_id').execute(); + await db.schema.createIndex('idx_ep_values_text_value').ifNotExists().on('ep_values').column('text_value').execute(); + await db.schema.createIndex('idx_ep_values_number_value').ifNotExists().on('ep_values').column('number_value').execute(); + await db.schema.createIndex('idx_ep_values_timestamp_value').ifNotExists().on('ep_values').column('timestamp_value').execute(); //// SAVED SEARCHES //// await db.schema @@ -193,6 +168,7 @@ export async function up(db: Kysely): Promise { } export async function down(db: Kysely): Promise { + /* await db.schema.dropIndex('idx_ep_values_text_file').execute(); await db.schema.dropIndex('idx_ep_values_text_value').execute(); await db.schema.dropIndex('idx_ep_values_text_ep').execute(); @@ -218,7 +194,7 @@ export async function down(db: Kysely): Promise { await db.schema.dropIndex('idx_files_size').execute(); await db.schema.dropIndex('idx_files_extension').execute(); await db.schema.dropIndex('idx_files_name').execute(); - +*/ await db.schema.dropTable('search_criteria').execute(); await db.schema.dropTable('saved_searches').execute(); await db.schema.dropTable('ep_values_timestamp').execute(); diff --git a/src/backend/migrations/001_migrateJSON.ts b/src/backend/migrations/001_migrateJSON.ts index 382bb21a..36266dcb 100644 --- a/src/backend/migrations/001_migrateJSON.ts +++ b/src/backend/migrations/001_migrateJSON.ts @@ -20,9 +20,8 @@ export async function getLastJsonBackupPath(): Promise { return stats.reduce((a, b) => (a.mtime > b.mtime ? a : b)).path; } - export async function up(db: Kysely): Promise { - restoreFromOldJsonFormat(db, await getLastJsonBackupPath()); + await restoreFromOldJsonFormat(db, await getLastJsonBackupPath()); } export async function down(_: Kysely): Promise { diff --git a/src/backend/schemaTypes.ts b/src/backend/schemaTypes.ts index 1c0d48e9..81716fc4 100644 --- a/src/backend/schemaTypes.ts +++ b/src/backend/schemaTypes.ts @@ -9,12 +9,15 @@ * Note: These are only TypeScript types. Updating them will not update the database automatically. * To apply changes to the actual schema you must manually write Kysely migrations, * ensuring that the database schema is kept in sync with this definitions. + * + * Note: All index properties are named idx because index is a reserved keyword in SQLite. */ import { ColumnType } from 'kysely'; import { ID } from '../api/id'; import { CriteriaValueType, OperatorType } from 'src/api/search-criteria'; -import { FILE_TAGS_SORTING_TYPE, FileDTO } from 'src/api/file'; +import { FILE_TAGS_SORTING_TYPE, FileDTO, IMG_EXTENSIONS_TYPE } from 'src/api/file'; +import { ExtraPropertyType } from 'src/api/extraProperty'; export type BooleanAsNumber = number; export const serializeBoolean = (value: boolean): number => (value ? 1 : 0); @@ -34,9 +37,7 @@ export type AllusionDB_SQL = { files: Files; fileTags: FileTags; extraProperties: ExtraProperties; - epValuesText: EpValuesText; - epValuesNumber: EpValuesNumber; - epValuesTimestamp: EpValuesTimestamp; + epValues: EpValues; savedSearches: SavedSearches; searchCriteria: SearchCriteria; }; @@ -45,7 +46,7 @@ export type AllusionDB_SQL = { export type Tags = { id: ColumnType; //pk - parentId: ID; //fk + parentId: ID | null; //fk idx: number; name: string; dateAdded: ColumnType; @@ -70,7 +71,7 @@ export type TagAliases = { export type LocationNodes = { id: ColumnType; //pk - parentId: ID; //fk + parentId: ID | null; //fk path: string; }; @@ -105,7 +106,7 @@ export type Files = { dateModifiedOS: DateAsNumber; dateLastIndexed: DateAsNumber; name: string; - extension: string; + extension: IMG_EXTENSIONS_TYPE; size: number; width: number; height: number; @@ -121,21 +122,19 @@ export type FileTags = { export type ExtraProperties = { id: ColumnType; //pk - type: string; + type: ExtraPropertyType; name: string; dateAdded: ColumnType; }; -type EpValues = { +export type EpValues = { fileId: ID; //pk fk epId: ID; //pk fk - value: T; + textValue: string | null; + numberValue: number | null; + timestampValue: DateAsNumber | null; }; -export type EpValuesText = EpValues; -export type EpValuesNumber = EpValues; -export type EpValuesTimestamp = EpValues; - /// SAVED SEARCHES /// export type SavedSearches = { diff --git a/src/frontend/entities/Location.ts b/src/frontend/entities/Location.ts index f5b1f861..800fd40d 100644 --- a/src/frontend/entities/Location.ts +++ b/src/frontend/entities/Location.ts @@ -12,7 +12,7 @@ import SysPath from 'path'; import { retainArray } from 'common/core'; import { IMG_EXTENSIONS_TYPE } from '../../api/file'; -import { ID } from '../../api/id'; +import { generateId, ID } from '../../api/id'; import { LocationDTO, SubLocationDTO } from '../../api/location'; import { RendererMessenger } from '../../ipc/renderer'; import { AppToaster } from '../components/Toaster'; @@ -25,6 +25,7 @@ const sort = (a: SubLocationDTO | ClientSubLocation, b: SubLocationDTO | ClientS a.name.localeCompare(b.name, undefined, { numeric: true }); export class ClientSubLocation { + id: ID; @observable name: string; @observable @@ -36,11 +37,13 @@ export class ClientSubLocation { store: LocationStore, public location: ClientLocation, public path: string, + id: ID, name: string, excluded: boolean, subLocations: SubLocationDTO[], tags: ID[], ) { + this.id = id; this.name = name; this.isExcluded = excluded; this.subLocations = observable( @@ -52,6 +55,7 @@ export class ClientSubLocation { store, this.location, SysPath.join(path, subLoc.name), + subLoc.id, subLoc.name, subLoc.isExcluded, subLoc.subLocations, @@ -73,6 +77,7 @@ export class ClientSubLocation { @action.bound serialize(): SubLocationDTO { return { + id: this.id, name: this.name.toString(), isExcluded: Boolean(this.isExcluded), subLocations: this.subLocations.map((subLoc) => subLoc.serialize()), @@ -140,6 +145,7 @@ export class ClientLocation { this.store, this, SysPath.join(this.path, subLoc.name), + subLoc.id, subLoc.name, subLoc.isExcluded, subLoc.subLocations, @@ -312,6 +318,7 @@ export class ClientLocation { this.store, this, item.fullPath, + generateId(), item.name, item.name.startsWith('.'), [], diff --git a/src/frontend/entities/Tag.ts b/src/frontend/entities/Tag.ts index 457d9ff3..d2046bd9 100644 --- a/src/frontend/entities/Tag.ts +++ b/src/frontend/entities/Tag.ts @@ -147,7 +147,7 @@ export class ClientTag { visited = new Set(), path = new Set(), ): Generator { - if (path.has(tag)) { + if (path.has(tag) && tag.id !== ROOT_TAG_ID) { tag.store.showTagToast( tag, 'has circular relations with other tags', @@ -179,7 +179,7 @@ export class ClientTag { visited = new Set(), path = new Set(), ): Generator { - if (path.has(tag)) { + if (path.has(tag) && tag.id !== ROOT_TAG_ID) { tag.store.showTagToast( tag, 'has circular implied relations with other tags', diff --git a/src/frontend/stores/FileStore.ts b/src/frontend/stores/FileStore.ts index fc2daa39..e57682cc 100644 --- a/src/frontend/stores/FileStore.ts +++ b/src/frontend/stores/FileStore.ts @@ -1,5 +1,6 @@ import fse from 'fs-extra'; import { action, computed, makeObservable, observable, runInAction } from 'mobx'; +import { setTimeout as delay } from 'node:timers/promises'; import { getThumbnailPath } from 'common/fs'; import { promiseAllLimit } from 'common/promise'; @@ -28,9 +29,12 @@ import { import { InheritedTagsVisibilityModeType } from './UiStore'; import { clamp } from 'common/core'; import { RendererMessenger } from 'src/ipc/renderer'; +import { USE_BACKEND_AS_WORKER } from 'src/backend/config'; export const FILE_STORAGE_KEY = 'Allusion_File'; +type FetchArgs = [OrderBy, OrderDirection, boolean, string]; + /** These fields are stored and recovered when the application opens up */ type PersistentPreferenceFields = | 'orderDirection' @@ -711,11 +715,18 @@ class FileStore { * of `fetchTaskIdPair` with the current timestamp to uniquely identify the task. * @returns The generated fetch ID (timestamp) */ - @action.bound newFetchTaskId(): number { + @action.bound async newFetchTaskId(): Promise { this.numLoadedFiles = 0; const now = performance.now(); this.fetchTaskIdPair[0] = now; this.fetchTaskIdPair[1] = 1; + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + if (!USE_BACKEND_AS_WORKER && this.activeAverageFetchTime > 2000) { + // If the backend is not in a worker and the fetch is heavy + // Apply a small delay to give time to the progressbar + // to start the animation before the backend blocks the tread. + await delay(600); + } return now; } @@ -731,18 +742,22 @@ class FileStore { } } + @action.bound getFetchArgs(): FetchArgs { + return [ + this.orderBy, + this.orderDirection, + this.isNaturalOrderingEnabled, + this.orderByExtraProperty, + ]; + } + @action.bound async fetchAllFiles(): Promise { try { this.setContentAll(); // Indicate a new fetch process - const start = this.newFetchTaskId(); + const start = await this.newFetchTaskId(); this.rootStore.uiStore.clearSearchCriteriaList(); - const fetchedFiles = await this.backend.fetchFiles( - this.orderBy, - this.orderDirection, - this.isNaturalOrderingEnabled, - this.orderByExtraProperty, - ); + const fetchedFiles = await this.backend.fetchFiles(...this.getFetchArgs()); const end = performance.now(); this.setAverageFetchTime(end - start); // continue if the current taskId is the same else abort the fetch @@ -761,18 +776,16 @@ class FileStore { try { this.setContentUntagged(); // Indicate a new fetch process - const start = this.newFetchTaskId(); + const start = await this.newFetchTaskId(); const { uiStore } = this.rootStore; + const { searchMatchAny } = uiStore; uiStore.clearSearchCriteriaList(); const criteria = new ClientTagSearchCriteria('tags'); uiStore.searchCriteriaList.push(criteria); const fetchedFiles = await this.backend.searchFiles( criteria.toCondition(this.rootStore), - this.orderBy, - this.orderDirection, - this.isNaturalOrderingEnabled, - this.orderByExtraProperty, - uiStore.searchMatchAny, + ...this.getFetchArgs(), + searchMatchAny, ); const end = performance.now(); this.setAverageFetchTime(end - start); @@ -791,26 +804,17 @@ class FileStore { @action.bound async fetchMissingFiles(): Promise { try { const { - orderBy, - orderDirection, - isNaturalOrderingEnabled, - orderByExtraProperty, rootStore: { uiStore }, } = this; this.setContentMissing(); // Indicate a new fetch process - const start = this.newFetchTaskId(); + const start = await this.newFetchTaskId(); uiStore.clearSearchCriteriaList(); // Fetch all files, then check their existence and only show the missing ones // Similar to {@link updateFromBackend}, but the existence check needs to be awaited before we can show the images - const backendFiles = await this.backend.fetchFiles( - orderBy, - orderDirection, - isNaturalOrderingEnabled, - orderByExtraProperty, - ); + const backendFiles = await this.backend.fetchFiles(...this.getFetchArgs()); const end = performance.now(); this.setAverageFetchTime(end - start); // continue if the current taskId is the same else abort the fetch @@ -879,7 +883,7 @@ class FileStore { @action.bound async fetchFilesByQuery(): Promise { const { uiStore } = this.rootStore; - + const { searchMatchAny } = uiStore; if (uiStore.searchCriteriaList.length === 0) { return this.fetchAllFiles(); } @@ -890,14 +894,11 @@ class FileStore { try { this.setContentQuery(); // Indicate a new fetch process - const start = this.newFetchTaskId(); + const start = await this.newFetchTaskId(); const fetchedFiles = await this.backend.searchFiles( criterias as [ConditionDTO, ...ConditionDTO[]], - this.orderBy, - this.orderDirection, - this.isNaturalOrderingEnabled, - this.orderByExtraProperty, - uiStore.searchMatchAny, + ...this.getFetchArgs(), + searchMatchAny, ); const end = performance.now(); this.setAverageFetchTime(end - start); diff --git a/src/renderer.tsx b/src/renderer.tsx index d1dd7470..bda71fe7 100644 --- a/src/renderer.tsx +++ b/src/renderer.tsx @@ -12,12 +12,13 @@ import { autorun, reaction, runInAction } from 'mobx'; import React from 'react'; import { Root, createRoot } from 'react-dom/client'; +import { wrap, Remote, proxy } from 'comlink'; +import Backend from './backend/backend'; + import { IS_DEV } from 'common/process'; import { promiseRetry } from 'common/timeout'; import { IS_PREVIEW_WINDOW, WINDOW_STORAGE_KEY } from 'common/window'; import { RendererMessenger } from 'src/ipc/renderer'; -import Backend from './backend/_deprecated/backend'; -import BackendTest from './backend/backend'; import App from './frontend/App'; import SplashScreen from './frontend/containers/SplashScreen'; import StoreProvider from './frontend/contexts/StoreContext'; @@ -27,8 +28,9 @@ import { FILE_STORAGE_KEY } from './frontend/stores/FileStore'; import RootStore from './frontend/stores/RootStore'; import { PREFERENCES_STORAGE_KEY } from './frontend/stores/UiStore'; import BackupScheduler from './backend/_deprecated/backup-scheduler'; -import { DB_NAME, dbInit } from './backend/_deprecated/config'; +import { dbInit } from './backend/_deprecated/config'; import path from 'path'; +import { DB_NAME, USE_BACKEND_AS_WORKER } from './backend/config'; async function main(): Promise { // Render our react components in the div with id 'app' in the html file @@ -43,26 +45,43 @@ async function main(): Promise { root.render(); const db = dbInit(DB_NAME); + const basePath = await RendererMessenger.getPath('userData'); + const databaseTestFilePath = path.join(basePath, 'databases', `${DB_NAME}.sqlite`); if (!IS_PREVIEW_WINDOW) { - await runMainApp(db, root); + await runMainApp(databaseTestFilePath, db, root); } else { - await runPreviewApp(db, root); + await runPreviewApp(databaseTestFilePath, db, root); } } -async function runMainApp(db: Dexie, root: Root): Promise { +async function runMainApp(dbPath: string, db: Dexie, root: Root): Promise { const defaultBackupDirectory = await RendererMessenger.getDefaultBackupDirectory(); const backup = new BackupScheduler(db, defaultBackupDirectory); - const basePath = await RendererMessenger.getPath('userData'); - const databaseTestFilePath = path.join(basePath, 'databases', `${DB_NAME}.sqlite`); - const testbackend = BackendTest.init(databaseTestFilePath, () => {}); - - const [backend] = await Promise.all([ - Backend.init(db, () => backup.schedule()), - fse.ensureDir(defaultBackupDirectory), - ]); + let backend: Backend; + + // Check if the database file already exists + const dbExists = await fse.pathExists(dbPath); + // If using worker mode and DB already exists, initialize backend in worker + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + if (USE_BACKEND_AS_WORKER && dbExists) { + const backendService = new BackendService(); + const [remoteBackend] = await Promise.all([ + backendService.init(dbPath, () => {}), + fse.ensureDir(defaultBackupDirectory), + ]); + backend = remoteBackend as unknown as Backend; + } else { + // If DB does not exist or worker mode is disabled, + // initialize backend in the main thread to safely run migrations + backend = new Backend(); + await Promise.all([ + backend.init(dbPath, () => {}), + fse.ensureDir(defaultBackupDirectory), + // + ]); + } const rootStore = await RootStore.main(backend, backup); @@ -132,7 +151,7 @@ async function runMainApp(db: Dexie, root: Root): Promise { await addTagsToFile(item.filePath, item.tagNames); }); - RendererMessenger.onGetTags(async () => ({ tags: await backend.fetchTags() })); + RendererMessenger.onGetTags(async () => ({ tags: (await backend?.fetchTags()) ?? [] })); RendererMessenger.onFullScreenChanged((val) => rootStore.uiStore.setFullScreen(val)); @@ -195,8 +214,11 @@ async function runMainApp(db: Dexie, root: Root): Promise { window.addEventListener('beforeunload', handleBeforeUnload); } -async function runPreviewApp(db: Dexie, root: Root): Promise { - const backend = new Backend(db, () => {}); +async function runPreviewApp(dbPath: string, db: Dexie, root: Root): Promise { + //const backend = await Backend.init(dbPath, () => {}); + // TODO: create an apropiated initPreview mode + const backend = new Backend(); + await backend.init(dbPath, () => {}); const rootStore = await RootStore.preview(backend, new BackupScheduler(db, '')); RendererMessenger.initialized(); @@ -266,6 +288,37 @@ async function runPreviewApp(db: Dexie, root: Root): Promise { }); } +class BackendService { + worker?: Remote; + workerInstance?: Worker; + private initialized = false; + + async init(dbPath: string, notifyChange: () => void): Promise | undefined> { + if (this.initialized) { + console.warn('BackendService: Already initialized'); + return this.worker; + } + + console.log('BackendService: Creating worker...'); + + const worker = new Worker(new URL('src/backend/backend', import.meta.url), { + type: 'module', + }); + const WorkerClass = wrap(worker); + + this.worker = await new WorkerClass(); + this.workerInstance = worker; + + console.log('BackendService: Initializing worker backend...'); + await this.worker.init(dbPath, proxy(notifyChange)); + + this.initialized = true; + console.log('BackendService: Ready!'); + + return this.worker; + } +} + main() .then(() => console.info('Successfully initialized Allusion!')) .catch((err) => { From c861a1114d7431b07e933e58dddf4c5052fcfaff Mon Sep 17 00:00:00 2001 From: RafaUC Date: Mon, 3 Nov 2025 22:57:26 -0600 Subject: [PATCH 07/19] Created Backend class implement create and save methods and all the other interface methods. --- src/api/data-storage-search.ts | 3 + src/api/file.ts | 2 - src/api/search-criteria.ts | 3 + src/backend/_deprecated/backend.ts | 4 +- src/backend/backend.ts | 643 ++++++++++++++++-- src/backend/backup-scheduler.ts | 30 +- src/backend/config.ts | 16 +- src/backend/migrations/000_initial.ts | 17 +- src/backend/schemaTypes.ts | 12 +- .../containers/AdvancedSearch/data.ts | 28 +- .../containers/AppToolbar/Searchbar.tsx | 9 +- .../containers/ContentView/menu-items.tsx | 21 +- .../Outliner/LocationsPanel/index.tsx | 2 +- .../Outliner/TagsPanel/ContextMenu.tsx | 4 +- .../Outliner/TagsPanel/TagsTree.tsx | 9 +- .../containers/Outliner/TagsPanel/index.tsx | 2 +- src/frontend/entities/File.ts | 2 - src/frontend/entities/SearchCriteria.ts | 90 +-- src/frontend/stores/FileStore.ts | 6 +- src/frontend/stores/LocationStore.ts | 12 +- src/frontend/stores/UiStore.ts | 7 +- tests/backend.test.ts | 1 - 22 files changed, 765 insertions(+), 158 deletions(-) diff --git a/src/api/data-storage-search.ts b/src/api/data-storage-search.ts index 0b22523f..1fd87c6a 100644 --- a/src/api/data-storage-search.ts +++ b/src/api/data-storage-search.ts @@ -13,6 +13,8 @@ export const enum OrderDirection { Desc, } +export type SearchConjunction = 'and' | 'or'; + // General search criteria for a database entity // FFR: Boolean keys are not supported in IndexedDB/Dexie - must store booleans as 0/1 @@ -40,6 +42,7 @@ export type IndexSignatureConditionDTO = BaseConditionDTO< type BaseConditionDTO = { key: ExtractKeyByValue; + conjunction?: SearchConjunction; operator: O; value: V; valueType: VT; diff --git a/src/api/file.ts b/src/api/file.ts index 0c68b8b2..a34f917b 100644 --- a/src/api/file.ts +++ b/src/api/file.ts @@ -11,8 +11,6 @@ export type FileDTO = { absolutePath: string; tags: ID[]; tagsSorting: FILE_TAGS_SORTING_TYPE; - /** used only for index on dexie */ - extraPropertyIDs: ID[]; extraProperties: ExtraProperties; /** When the file was imported into Allusion */ dateAdded: Date; diff --git a/src/api/search-criteria.ts b/src/api/search-criteria.ts index c7682fcc..98412bd3 100644 --- a/src/api/search-criteria.ts +++ b/src/api/search-criteria.ts @@ -3,6 +3,7 @@ import { FileDTO } from './file'; import { ExtraPropertyOperatorType, NumberOperatorType, + SearchConjunction, StringOperatorType, } from './data-storage-search'; import { ExtraPropertyValue } from './extraProperty'; @@ -29,8 +30,10 @@ export type CriteriaValueType = 'number' | 'date' | 'string' | 'array' | 'indexS // FFR: Boolean keys are not supported in IndexedDB/Dexie - must store booleans as 0/1 export interface IBaseSearchCriteria { + id: ID; key: keyof FileDTO; valueType: CriteriaValueType; + conjunction?: SearchConjunction; readonly operator: OperatorType; } diff --git a/src/backend/_deprecated/backend.ts b/src/backend/_deprecated/backend.ts index fe2feb04..fd0942ee 100644 --- a/src/backend/_deprecated/backend.ts +++ b/src/backend/_deprecated/backend.ts @@ -339,7 +339,7 @@ export default class Backend implements DataStorage { for (const id of extraPropertyIDs) { delete file.extraProperties[id]; } - retainArray(file.extraPropertyIDs, (id) => !extraPropertyIDs.includes(id)); + //retainArray(file.extraPropertyIDs, (id) => !extraPropertyIDs.includes(id)); }); await this.#extraProperties.bulkDelete(extraPropertyIDs); @@ -444,7 +444,7 @@ const exampleFileDTO: FileDTO = { dateModified: new Date(), dateModifiedOS: new Date(), extraProperties: {}, - extraPropertyIDs: [], + //extraPropertyIDs: [], tags: [], }; diff --git a/src/backend/backend.ts b/src/backend/backend.ts index d32aeec5..9df1a02d 100644 --- a/src/backend/backend.ts +++ b/src/backend/backend.ts @@ -4,8 +4,19 @@ import { deserializeDate, EpValues, Files, + LocationNodes, + Locations, + LocationTags, serializeBoolean, serializeDate, + SubLocations, + TagAliases, + TagImplications, + ExtraProperties as DbExtraProperties, + SavedSearches, + SearchCriteria, + FileTags, + SubTags, } from './schemaTypes'; import { expose } from 'comlink'; import SQLite from 'better-sqlite3'; @@ -20,8 +31,10 @@ import { ExpressionBuilder, OrderByDirection, AnyColumn, + Insertable, + Expression, } from 'kysely'; -import { migrateToLatest, PAD_STRING_LENGTH } from './config'; +import { kyselyLogger, migrateToLatest, PAD_STRING_LENGTH } from './config'; import { DataStorage } from 'src/api/data-storage'; import { IndexableType } from 'dexie'; import { @@ -36,19 +49,21 @@ import { isStringOperator, PropertyKeys, StringProperties, + SearchConjunction, } from 'src/api/data-storage-search'; import { ExtraProperties, ExtraPropertyDTO } from 'src/api/extraProperty'; import { FileDTO } from 'src/api/file'; import { FileSearchDTO } from 'src/api/file-search'; -import { ID } from 'src/api/id'; +import { generateId, ID } from 'src/api/id'; import { LocationDTO, SubLocationDTO } from 'src/api/location'; import { ROOT_TAG_ID, TagDTO } from 'src/api/tag'; -// eslint-disable-next-line @typescript-eslint/no-unused-vars -import { jsonArrayFrom, jsonObjectFrom, jsonBuildObject } from 'kysely/helpers/sqlite'; +import { jsonArrayFrom } from 'kysely/helpers/sqlite'; import { IS_DEV } from 'common/process'; +import { UpdateObject } from 'kysely/dist/cjs/parser/update-set-parser'; // Use to debug perfomance. const USE_TIMING_PROXY = IS_DEV; +const USE_QUERY_LOGGER = false ? IS_DEV : false; export default class Backend implements DataStorage { readonly MAX_VARS!: number; @@ -65,8 +80,7 @@ export default class Backend implements DataStorage { async init(dbPath: string, notifyChange: () => void): Promise { console.info(`SQLite3: Initializing database "${dbPath}"...`); const database = new SQLite(dbPath, { timeout: 50000 }); - // HACK - // Use a padded string to do natural sorting + // HACK Use a padded string to do natural sorting database.function('pad_string', { deterministic: true }, (str) => { return str.replace(/\d+/g, (num: string) => num.padStart(PAD_STRING_LENGTH, '0')); }); @@ -76,7 +90,7 @@ export default class Backend implements DataStorage { const db = new Kysely({ dialect: dialect, plugins: [new ParseJSONResultsPlugin(), new CamelCasePlugin()], - log: IS_DEV ? ['query', 'error'] : undefined, // Used only for debugging. + log: USE_QUERY_LOGGER ? kyselyLogger : undefined, // Used only for debugging. }); // Instead of initializing this through the constructor, set the class properties here, // this allows us to use the class as a worker having async await calls at init. @@ -106,8 +120,6 @@ export default class Backend implements DataStorage { .insertInto('tags') .values({ id: ROOT_TAG_ID, - parentId: null, - idx: 0, name: 'Root', dateAdded: serializeDate(new Date()), color: '', @@ -130,9 +142,9 @@ export default class Backend implements DataStorage { .select((eb) => [ jsonArrayFrom( eb - .selectFrom('tags as subTags') - .select('subTags.id') - .whereRef('subTags.parentId', '=', 'tags.id') + .selectFrom('subTags') + .select('subTags.subTagId') + .whereRef('subTags.tagId', '=', 'tags.id') .orderBy('subTags.idx'), ).as('subTags'), jsonArrayFrom( @@ -156,7 +168,7 @@ export default class Backend implements DataStorage { name: dbTag.name, dateAdded: deserializeDate(dbTag.dateAdded), color: dbTag.color, - subTags: dbTag.subTags.map((st) => st.id), + subTags: dbTag.subTags.map((st) => st.subTagId), impliedTags: dbTag.impliedTags.map((it) => it.impliedTagId), isHidden: deserializeBoolean(dbTag.isHidden), isVisibleInherited: deserializeBoolean(dbTag.isVisibleInherited), @@ -171,7 +183,6 @@ export default class Backend implements DataStorage { // Because creating the jsons takes a lot of time, let's preaggregate them everytime we save our files. async preAggregateJSON(): Promise { console.info('SQLite: Updating temp aggregates...'); - this.#isQueryDirty = false; await sql` DROP TABLE IF EXISTS file_tag_aggregates_temp; `.execute(this.#db); @@ -198,7 +209,8 @@ export default class Backend implements DataStorage { 'number_value', number_value, 'timestamp_value', timestamp_value)) as extra_properties - FROM ep_values; + FROM ep_values + GROUP BY file_id; `.execute(this.#db); await sql` @@ -207,6 +219,7 @@ export default class Backend implements DataStorage { await sql` CREATE INDEX IF NOT EXISTS idx_file_ep_aggregates_temp_file ON file_ep_aggregates_temp(file_id); `.execute(this.#db); + this.#isQueryDirty = false; } async queryFiles( @@ -272,7 +285,6 @@ export default class Backend implements DataStorage { width: dbFile.width, height: dbFile.height, tags: dbFile.tags ?? [], - extraPropertyIDs: extraPropertyIDs, extraProperties: extraProperties, }; }); @@ -302,7 +314,7 @@ export default class Backend implements DataStorage { extraPropertyID?: ID, matchAny?: boolean, ): Promise { - console.info('SQLite: Searching files...'); + console.info('SQLite: Searching files...', criteria); return this.queryFiles(criteria, { order, direction: fileOrder, @@ -312,7 +324,7 @@ export default class Backend implements DataStorage { } async fetchFilesByID(ids: ID[]): Promise { - console.info('SQLite: Fetching files by ID...'); + console.info('SQLite: Fetching files by ID...', ids); return this.queryFiles(undefined, { order: 'dateAdded' }, { key: 'id', values: ids }); } @@ -411,7 +423,7 @@ export default class Backend implements DataStorage { 'id', 'savedSearchId', 'idx', - 'matchGroup', + 'conjunction', 'key', 'valueType', 'operator', @@ -427,8 +439,10 @@ export default class Backend implements DataStorage { id: dbSearch.id, name: dbSearch.name, criteria: dbSearch.criteria.map((dbCrit) => ({ + id: dbCrit.id, key: dbCrit.key, operator: dbCrit.operator, + conjunction: dbCrit.conjunction, valueType: dbCrit.valueType, value: // the ParseJSONResultsPlugin already parses the arrays but not strings @@ -458,59 +472,362 @@ export default class Backend implements DataStorage { } async createTag(tag: TagDTO): Promise { - console.warn('Method not implemented.'); + console.info('SQLite: Creating tag...', tag); + return this.upsertTag(tag); } - async createFilesFromPath(path: string, files: FileDTO[]): Promise { - console.warn('Method not implemented.'); + + // Creates many files at once, and checks for duplicates in the path they are in + async createFilesFromPath(path: string, filesDTO: FileDTO[]): Promise { + console.info('SQLite: Creating files...', path, filesDTO.length); + + if (filesDTO.length === 0) { + return; + } + const { files } = normalizeFiles(filesDTO); + const FILES_BATCH_SIZE = computeBatchSize(this.MAX_VARS, files[0]); + await this.#db.transaction().execute(async (trx) => { + for (let i = 0; i < files.length; i += FILES_BATCH_SIZE) { + const batch = files.slice(i, i + FILES_BATCH_SIZE); + try { + await trx + .insertInto('files') + .values(batch) + .onConflict((oc) => oc.doNothing()) + .execute(); + } catch (error) { + console.error(`Failed to insert files batch at index ${i}:`, error); + } + } + }); + this.#isQueryDirty = true; + this.#notifyChange(); + console.info('SQLite: Files created successfully'); } + async createLocation(location: LocationDTO): Promise { - console.warn('Method not implemented.'); + console.info('SQLite: Creating location...', location); + return this.upsertLocation(location); } + async createSearch(search: FileSearchDTO): Promise { - console.warn('Method not implemented.'); + console.info('SQLite: Creating search...', search); + return this.upsertSearch(search); } + async createExtraProperty(extraProperty: ExtraPropertyDTO): Promise { - console.warn('Method not implemented.'); + console.info('SQLite: Creating extra property...', extraProperty); + return this.upsertExtraProperty(extraProperty); } + async saveTag(tag: TagDTO): Promise { - console.warn('Method not implemented.'); + console.info('SQLite: Saving tag...', tag); + return this.upsertTag(tag); + } + + async upsertTag(tag: TagDTO): Promise { + const { tagIds, tags, subTags, tagImplications, tagAliases } = normalizeTags([tag]); + if (tags.length === 0) { + return; + } + await this.#db.transaction().execute(async (trx) => { + await trx.deleteFrom('subTags').where('tagId', 'in', tagIds).execute(); + await trx.deleteFrom('tagImplications').where('tagId', 'in', tagIds).execute(); + await trx.deleteFrom('tagAliases').where('tagId', 'in', tagIds).execute(); + await upsertTable(trx, 'tags', tags, ['id'], ['dateAdded']); + if (subTags.length > 0) { + await upsertTable(trx, 'subTags', subTags, ['tagId', 'subTagId']); + } + if (tagImplications.length > 0) { + await upsertTable(trx, 'tagImplications', tagImplications, ['tagId', 'impliedTagId']); + } + if (tagAliases.length > 0) { + await upsertTable(trx, 'tagAliases', tagAliases, ['tagId', 'alias']); + } + }); + this.#notifyChange(); } - async saveFiles(files: FileDTO[]): Promise { - console.warn('Method not implemented.'); + + async saveFiles(filesDTO: FileDTO[]): Promise { + console.info('SQLite: Saving files...', filesDTO); + if (filesDTO.length === 0) { + return; + } + + const { fileIds, files, fileTags, epVal } = normalizeFiles(filesDTO); + + // Compute batch sizes. To use the maximum number of vars SQLite can handle per query. + const DELETE_BATCH_SIZE = this.MAX_VARS; + const FILES_BATCH_SIZE = computeBatchSize(this.MAX_VARS, files[0]); + const FILE_TAGS_BATCH_SIZE = computeBatchSize(this.MAX_VARS, fileTags[0]); + const EP_VALUES_BATCH_SIZE = computeBatchSize(this.MAX_VARS, epVal[0]); + + await this.#db.transaction().execute(async (trx) => { + // Create unique temp table names. + const tempSuffix = generateId(); + const tempFiles = `files_temp_${tempSuffix}`; + const tempFileTags = `file_tags_temp_${tempSuffix}`; + const tempEpValues = `ep_values_temp_${tempSuffix}`; + + try { + // Create temp tables form a copy of the actual tables. + await sql`CREATE TEMP TABLE ${sql.id(tempFiles)} AS SELECT * FROM files WHERE 0`.execute( + trx, + ); + await sql`CREATE TEMP TABLE ${sql.id( + tempFileTags, + )} AS SELECT * FROM file_tags WHERE 0`.execute(trx); + await sql`CREATE TEMP TABLE ${sql.id( + tempEpValues, + )} AS SELECT * FROM ep_values WHERE 0`.execute(trx); + // Insert files into temp files table + for (let i = 0; i < files.length; i += FILES_BATCH_SIZE) { + const batch = files.slice(i, i + FILES_BATCH_SIZE); + await trx + .insertInto(tempFiles as any) + .values(batch) + .execute(); + } + // Delete previous fileTags and epValues, it is quicker to delete all from related files and insert them in bulk. + if (fileIds.length > 0) { + for (let i = 0; i < fileIds.length; i += DELETE_BATCH_SIZE) { + const batchIds = fileIds.slice(i, i + DELETE_BATCH_SIZE); + await trx.deleteFrom('fileTags').where('fileId', 'in', batchIds).execute(); + await trx.deleteFrom('epValues').where('fileId', 'in', batchIds).execute(); + } + } + // Insert fileTags into temp table + if (fileTags.length > 0) { + for (let i = 0; i < fileTags.length; i += FILE_TAGS_BATCH_SIZE) { + const batch = fileTags.slice(i, i + FILE_TAGS_BATCH_SIZE); + await trx + .insertInto(tempFileTags as any) + .values(batch) + .execute(); + } + } + // Insert epValues into temp table + if (epVal.length > 0) { + for (let i = 0; i < epVal.length; i += EP_VALUES_BATCH_SIZE) { + const batch = epVal.slice(i, i + EP_VALUES_BATCH_SIZE); + await trx + .insertInto(tempEpValues as any) + .values(batch) + .execute(); + } + } + // Transfer from temp tables + // Upsert FILES + upsertTable( + trx, + 'files', + sql`SELECT * FROM ${sql.id(tempFiles)} WHERE true`, + ['id'], + ['dateAdded'], + files[0], + ); + // Insert FileTags + if (fileTags.length > 0) { + await sql` + INSERT INTO file_tags + SELECT * FROM ${sql.id(tempFileTags)} + `.execute(trx); + } + // Insert EpValues + if (epVal.length > 0) { + await sql` + INSERT INTO ep_values + SELECT * FROM ${sql.id(tempEpValues)} + `.execute(trx); + } + this.#isQueryDirty = true; + console.info('SQLite: Files saved successfully'); + } finally { + // Clean temp table. + await sql`DROP TABLE IF EXISTS ${sql.id(tempFiles)}`.execute(trx); + await sql`DROP TABLE IF EXISTS ${sql.id(tempFileTags)}`.execute(trx); + await sql`DROP TABLE IF EXISTS ${sql.id(tempEpValues)}`.execute(trx); + } + }); + this.#notifyChange(); } + async saveLocation(location: LocationDTO): Promise { - console.warn('Method not implemented.'); + console.info('SQLite: Saving location...', location); + return this.upsertLocation(location); } + + async upsertLocation(location: LocationDTO): Promise { + const { nodeIds, locationNodes, locations, subLocations, locationTags } = normalizeLocations([ + location, + ]); + if (locationNodes.length === 0) { + return; + } + await this.#db.transaction().execute(async (trx) => { + await trx.deleteFrom('locationTags').where('nodeId', 'in', nodeIds).execute(); + await trx.deleteFrom('locationNodes').where('parentId', 'in', nodeIds).execute(); + await upsertTable(trx, 'locationNodes', locationNodes, ['id']); + if (locations.length > 0) { + await upsertTable(trx, 'locations', locations, ['nodeId'], ['dateAdded']); + } + if (subLocations.length > 0) { + await upsertTable(trx, 'subLocations', subLocations, ['nodeId']); + } + if (locationTags.length > 0) { + await upsertTable(trx, 'locationTags', locationTags, ['nodeId', 'tagId']); + } + }); + this.#notifyChange(); + } + async saveSearch(search: FileSearchDTO): Promise { - console.warn('Method not implemented.'); + console.info('SQLite: Saving search...', search); + return this.upsertSearch(search); } + + async upsertSearch(search: FileSearchDTO): Promise { + const { savedSearchesIds, savedSearches, searchCriteria } = normalizeSavedSearches([search]); + if (savedSearches.length === 0) { + return; + } + await this.#db.transaction().execute(async (trx) => { + await trx + .deleteFrom('searchCriteria') + .where('savedSearchId', 'in', savedSearchesIds) + .execute(); + await upsertTable(trx, 'savedSearches', savedSearches, ['id']); + if (searchCriteria.length > 0) { + await upsertTable(trx, 'searchCriteria', searchCriteria, ['id']); + } + }); + this.#notifyChange(); + } + async saveExtraProperty(extraProperty: ExtraPropertyDTO): Promise { - console.warn('Method not implemented.'); + console.info('SQLite: Saving extra property...', extraProperty); + return this.upsertExtraProperty(extraProperty); } - async removeTags(tags: ID[]): Promise { - console.warn('Method not implemented.'); + + async upsertExtraProperty(extraProperty: ExtraPropertyDTO): Promise { + const extraProperties: Insertable[] = [extraProperty].map((ep) => ({ + id: ep.id, + type: ep.type, + name: ep.name, + dateAdded: serializeDate(ep.dateAdded), + })); + await this.#db.transaction().execute(async (trx) => { + await upsertTable(trx, 'extraProperties', extraProperties, ['id'], ['dateAdded']); + }); + this.#notifyChange(); } + async mergeTags(tagToBeRemoved: ID, tagToMergeWith: ID): Promise { - console.warn('Method not implemented.'); + console.info('SQLite: Merging tags...', tagToBeRemoved, tagToMergeWith); + + await this.#db.transaction().execute(async (trx) => { + // Merge in FileTags + // first delete the records that would make a duplicate + await trx + .deleteFrom('fileTags') + .where('tagId', '=', tagToBeRemoved) + .where('fileId', 'in', (eb) => + eb.selectFrom('fileTags').select('fileId').where('tagId', '=', tagToMergeWith), + ) + .execute(); + // Update the thag ids + await trx + .updateTable('fileTags') + .set({ tagId: tagToMergeWith }) + .where('tagId', '=', tagToBeRemoved) + .execute(); + // Merge in locationTags + await trx + .deleteFrom('locationTags') + .where('tagId', '=', tagToBeRemoved) + .where('nodeId', 'in', (eb) => + eb.selectFrom('locationTags').select('nodeId').where('tagId', '=', tagToMergeWith), + ) + .execute(); + await trx + .updateTable('locationTags') + .set({ tagId: tagToMergeWith }) + .where('tagId', '=', tagToBeRemoved) + .execute(); + + // delete the tag + await trx.deleteFrom('tags').where('id', '=', tagToBeRemoved).execute(); + }); + this.#notifyChange(); + } + + async removeTags(tags: ID[]): Promise { + console.info('SQLite: Removing tags...', tags); + // Cascade delte in other tables deleting from tags table. + await this.#db.deleteFrom('tags').where('id', 'in', tags).execute(); + this.#notifyChange(); } + async removeFiles(files: ID[]): Promise { - console.warn('Method not implemented.'); + console.info('SQLite: Removing files...', files); + // Cascade delte in other tables deleting from files table. + await this.#db.deleteFrom('files').where('id', 'in', files).execute(); + this.#notifyChange(); } + async removeLocation(location: ID): Promise { - console.warn('Method not implemented.'); + console.info('SQLite: Removing location...', location); + // Cascade delte in other tables deleting from locationNodes table. + await this.#db.deleteFrom('locationNodes').where('id', '=', location).execute(); + this.#notifyChange(); } + async removeSearch(search: ID): Promise { - console.warn('Method not implemented.'); + console.info('SQLite: Removing search...', search); + // Cascade delte in other tables deleting from savedSearches table. + await this.#db.deleteFrom('savedSearches').where('id', '=', search).execute(); + this.#notifyChange(); } - async removeExtraProperties(extraProperty: ID[]): Promise { - console.warn('Method not implemented.'); + + async removeExtraProperties(extraPropertyIDs: ID[]): Promise { + console.info('SQLite: Removing extra properties...', extraPropertyIDs); + // Cascade delte in other tables deleting from extraProperties table. + await this.#db.deleteFrom('extraProperties').where('id', 'in', extraPropertyIDs).execute(); + this.#notifyChange(); } + async countFiles(): Promise<[fileCount: number, untaggedFileCount: number]> { - console.warn('Method not implemented.'); - return [0, 0]; + console.info('SQLite: Counting files...'); + const totalResult = await this.#db + .selectFrom('files') + .select(({ fn }) => fn.count('id').as('count')) + .executeTakeFirst(); + const fileCount = totalResult?.count ?? 0; + + const untaggedResult = await this.#db + .selectFrom('files as f') + .leftJoin('fileTags as ft', 'ft.fileId', 'f.id') + .where('ft.fileId', 'is', null) + .select(({ fn }) => fn.count('f.id').as('count')) + .executeTakeFirst(); + const untaggedFileCount = untaggedResult?.count ?? 0; + return [fileCount, untaggedFileCount]; } + async clear(): Promise { - console.warn('Method not implemented.'); + console.info('SQLite: Clearing database...'); + const tables = await this.#db + .selectFrom('sqlite_master' as any) + .select('name') + .where('type', '=', 'table') + .where('name', 'not like', 'sqlite_%') + .execute(); + + for (const { name } of tables) { + if (name === 'kysely_migration' || name === 'kysely_migration_lock') { + continue; + } + await this.#db.deleteFrom(name as any).execute(); + } } } @@ -581,7 +898,6 @@ const exampleFileDTO: FileDTO = { dateModified: new Date(), dateModifiedOS: new Date(), extraProperties: {}, - extraPropertyIDs: [], tags: [], }; @@ -652,8 +968,6 @@ async function applySortOrder( type KeyInListOptions = { key: AnyColumn; values: any[] }; -type SearchConjunction = 'and' | 'or'; - export type ConditionWithConjunction = ConditionDTO & { conjunction?: SearchConjunction; }; @@ -957,3 +1271,242 @@ function applyExtraPropertyCondition( // Return the expression return eb('files.id', 'in', subquery); } + +/////////////////// +///// HELPERS ///// +/////////////////// + +async function upsertTable< + Table extends keyof AllusionDB_SQL, + Columns extends ReadonlyArray>, +>( + db: Kysely, + table: Table, + values: Insertable[] | Expression, + conflictColumns: Columns, + excludeFromUpdate?: (keyof Insertable)[], + sampleObject?: Insertable, +) { + const isExpression = !Array.isArray(values); + if (!isExpression && values.length === 0) { + return; + } + + // Infer Columns + let columnsToUpdate: string[]; + if (isExpression) { + if (!sampleObject) { + throw new Error( + `sampleObject is required when using SQL expressions for table ${String(table)}`, + ); + } + columnsToUpdate = Object.keys(sampleObject).filter( + (key) => + !conflictColumns.includes(key as any) && + (!excludeFromUpdate || !excludeFromUpdate.includes(key as any)), + ); + } else { + const firstRow = (sampleObject || values[0]) as Record; + columnsToUpdate = Object.keys(firstRow).filter( + (key) => + !conflictColumns.includes(key as any) && + (!excludeFromUpdate || !excludeFromUpdate.includes(key as any)), + ); + } + const updateSet = columnsToUpdate.reduce((acc, column) => { + acc[column] = (eb: any) => eb.ref(`excluded.${column}`); + return acc; + }, {} as Record) as UpdateObject; + + let query; + if (isExpression) { + query = db.insertInto(table as keyof AllusionDB_SQL & string).expression(values as any); + } else { + query = db.insertInto(table as keyof AllusionDB_SQL & string).values(values as any); + } + + if (columnsToUpdate.length === 0) { + query = query.onConflict((oc) => oc.columns(conflictColumns as any).doNothing()); + } else { + query = query.onConflict((oc) => + oc.columns(conflictColumns as any).doUpdateSet(updateSet as any), + ); + } + + return query.execute(); +} + +function normalizeTags(tags: TagDTO[]) { + const tagIds: ID[] = []; + const subTags: Insertable[] = []; + const tagImplications: Insertable[] = []; + const tagAliases: Insertable[] = []; + + for (const tag of tags) { + tagIds.push(tag.id); + for (const [index, subTagId] of (Array.isArray(tag.subTags) ? tag.subTags : []).entries()) { + subTags.push({ tagId: tag.id, subTagId: subTagId, idx: index }); + } + for (const impliedTagId of Array.isArray(tag.impliedTags) ? tag.impliedTags : []) { + tagImplications.push({ tagId: tag.id, impliedTagId: impliedTagId }); + } + // Convert to Set to get rid of duplicates. + const aliases = new Set(Array.isArray(tag.aliases) ? tag.aliases : []); + for (const alias of aliases) { + tagAliases.push({ tagId: tag.id, alias: alias }); + } + } + + const normalizedTags = tags.map((tag) => ({ + id: tag.id, + name: tag.name, + color: tag.color, + isHidden: serializeBoolean(tag.isHidden), + isVisibleInherited: serializeBoolean(tag.isVisibleInherited), + isHeader: serializeBoolean(tag.isHeader), + description: tag.description, + dateAdded: serializeDate(tag.dateAdded), + })); + + return { tagIds, tags: normalizedTags, subTags, tagImplications, tagAliases }; +} + +function normalizeLocations(sourcelocations: LocationDTO[]) { + const locationNodes: Insertable[] = []; + const locations: Insertable[] = []; + const subLocations: Insertable[] = []; + const locationTags: Insertable[] = []; + const nodeIds: ID[] = []; + + function normalizeLocationNodeRecursive( + node: LocationDTO | SubLocationDTO, + parentId: ID | null, + isRoot: boolean, + ) { + const parentIdvalue = isRoot ? null : parentId; + const pathValue = 'path' in node ? node.path : node.name; + nodeIds.push(node.id); + locationNodes.push({ + id: node.id, + parentId: parentIdvalue, + path: pathValue, + }); + if (isRoot) { + const location = node as LocationDTO; + locations.push({ + nodeId: node.id, + idx: location.index, + isWatchingFiles: serializeBoolean(!!location.isWatchingFiles), + dateAdded: serializeDate(new Date(location.dateAdded)), + }); + } else { + const subLocation = node as SubLocationDTO; + subLocations.push({ + nodeId: node.id, + isExcluded: serializeBoolean(subLocation.isExcluded), + }); + } + // Insert tags + for (const tagId of Array.isArray(node.tags) ? node.tags : []) { + locationTags.push({ + nodeId: node.id, + tagId: tagId, + }); + } + // Recurse for sublocations + for (const sub of Array.isArray(node.subLocations) ? node.subLocations : []) { + normalizeLocationNodeRecursive(sub, node.id, false); + } + } + + for (const loc of sourcelocations) { + normalizeLocationNodeRecursive(loc, null, true); + } + return { nodeIds, locationNodes, locations, subLocations, locationTags }; +} + +function normalizeSavedSearches(sourceSearches: FileSearchDTO[]) { + const savedSearchesIds: ID[] = []; + const savedSearches: Insertable[] = []; + const searchCriteria: Insertable[] = []; + + for (const search of sourceSearches) { + savedSearchesIds.push(search.id); + savedSearches.push({ + id: search.id, + name: search.name, + idx: search.index, + }); + + for (const [idx, crit] of (Array.isArray(search.criteria) ? search.criteria : []).entries()) { + searchCriteria.push({ + id: crit.id, + savedSearchId: search.id, + idx: idx, + conjunction: crit.conjunction ?? 'and', + key: crit.key, + valueType: crit.valueType, + operator: crit.operator, + jsonValue: JSON.stringify(crit.value), + }); + } + } + return { savedSearchesIds, savedSearches, searchCriteria }; +} + +function normalizeFiles(sourceFiles: FileDTO[]) { + const fileIds: ID[] = []; + const files: Insertable[] = []; + const fileTags: Insertable[] = []; + const epVal: Insertable[] = []; + + for (const file of sourceFiles) { + const fileId = file.id; + fileIds.push(fileId); + files.push({ + id: fileId, + ino: file.ino, + locationId: file.locationId, + relativePath: file.relativePath, + absolutePath: file.absolutePath, + tagSorting: file.tagsSorting, + name: file.name, + extension: file.extension, + size: file.size, + width: file.width, + height: file.height, + dateAdded: serializeDate(file.dateAdded), + dateModified: serializeDate(file.dateModified), + dateModifiedOS: serializeDate(file.dateModifiedOS), + dateLastIndexed: serializeDate(file.dateLastIndexed), + dateCreated: serializeDate(file.dateCreated), + }); + // file_tags (tags relations) + for (const tagId of Array.isArray(file.tags) ? file.tags : []) { + fileTags.push({ + fileId: fileId, + tagId: tagId, + }); + } + // ep_values (extra properties relations) + for (const [epId, value] of Object.entries(file.extraProperties)) { + // TODO: Maybe should fetch the ExtraProperties types to assign the type based on + // the extra property definition, but since the DTO types do not overlap for now, this + // is good enough. + if (typeof value === 'number') { + epVal.push({ + fileId, + epId, + numberValue: value, + }); + } else { + epVal.push({ + fileId, + epId, + textValue: value, + }); + } + } + } + return { fileIds, files, fileTags, epVal }; +} diff --git a/src/backend/backup-scheduler.ts b/src/backend/backup-scheduler.ts index 0c53618c..5cf85c0f 100644 --- a/src/backend/backup-scheduler.ts +++ b/src/backend/backup-scheduler.ts @@ -1,7 +1,6 @@ import { promises as fs } from 'fs'; import { Insertable, InsertObject, Kysely, sql } from 'kysely'; import { generateId, ID } from 'src/api/id'; -import { ROOT_TAG_ID } from 'src/api/tag'; import { setTimeout as delay } from 'node:timers/promises'; import { AllusionDB_SQL, @@ -19,6 +18,8 @@ import { SearchCriteria, TagImplications, TagAliases, + SubTags, + Tags, } from './schemaTypes'; import { ExtraPropertyType } from 'src/api/extraProperty'; import { computeBatchSize, getSqliteMaxVariables } from './backend'; @@ -87,7 +88,7 @@ export async function restoreFromOldJsonFormat( continue; // retry } - console.warn(`❌ Error al insertar ${entityName}`, err); + console.warn(`❌ Error while inserting ${entityName}`, err); errors += batchSize; break; // stop retry loop for this batch } @@ -106,8 +107,6 @@ export async function restoreFromOldJsonFormat( .insertInto('tags') .values({ id: fallbackIds.tag, - parentId: ROOT_TAG_ID, - idx: 0, name: 'Fallback Tag', color: '', description: '', @@ -154,9 +153,10 @@ export async function restoreFromOldJsonFormat( /// IMPORTING DATA /// // Import tags - const { normalizedTags, tagImplications, tagAliases } = normalizeTags(tables.tags ?? []); + const { tags, subTags, tagImplications, tagAliases } = normalizeTags(tables.tags ?? []); - await saveEntries('tags', normalizedTags); + await saveEntries('tags', tags); + await saveEntries('subTags', subTags); await saveEntries('tagImplications', tagImplications); await saveEntries('tagAliases', tagAliases); @@ -212,16 +212,13 @@ export async function restoreFromOldJsonFormat( } function normalizeTags(tags: any[]) { - const parentMap = new Map(); + const subTags: Insertable[] = []; const tagImplications: Insertable[] = []; const tagAliases: Insertable[] = []; for (const tag of tags) { - for (const [idx, childId] of (Array.isArray(tag.subTags) ? tag.subTags : []).entries()) { - parentMap.set(childId, [tag.id, idx]); - } - if (!parentMap.has(tag.id)) { - parentMap.set(tag.id, [ROOT_TAG_ID, 0]); + for (const [index, subTagId] of (Array.isArray(tag.subTags) ? tag.subTags : []).entries()) { + subTags.push({ tagId: tag.id, subTagId: subTagId, idx: index }); } for (const impliedTagId of Array.isArray(tag.impliedTags) ? tag.impliedTags : []) { @@ -235,11 +232,8 @@ function normalizeTags(tags: any[]) { } } - const normalizedTags = tags.map((tag) => ({ + const normalizedTags: Insertable[] = tags.map((tag) => ({ id: tag.id ?? generateId(), - parentId: - tag.id === ROOT_TAG_ID ? null : ((parentMap.get(tag.id)?.at(0) ?? fallbackIds.tag) as ID), - idx: (parentMap.get(tag.id)?.at(1) ?? 0) as number, name: tag.name ?? '(untitled)', color: tag.color ?? '', isHidden: serializeBoolean(!!tag.isHidden), @@ -249,7 +243,7 @@ function normalizeTags(tags: any[]) { dateAdded: serializeDate(tag.dateAdded ? new Date(tag.dateAdded) : new Date()), })); - return { normalizedTags, tagImplications, tagAliases }; + return { tags: normalizedTags, subTags, tagImplications, tagAliases }; } function normalizeLocations(sourcelocations: any[]) { @@ -393,7 +387,7 @@ function normalizeSavedSearches(sourceSearches: any[]) { id: criteriaId, savedSearchId: searchId, idx: idx, - matchGroup: search.matchAny ? 'any' : 'all', + conjunction: search.matchAny ? 'or' : 'and', key: crit.key ?? 'name', valueType: crit.valueType ?? 'string', operator: crit.operator ?? 'equals', diff --git a/src/backend/config.ts b/src/backend/config.ts index 216d8a44..b37077cd 100644 --- a/src/backend/config.ts +++ b/src/backend/config.ts @@ -1,4 +1,4 @@ -import { Kysely, Migrator, Migration, MigrationProvider } from 'kysely'; +import { Kysely, Migrator, Migration, MigrationProvider, Logger, LogEvent } from 'kysely'; import { AllusionDB_SQL } from './schemaTypes'; export const DB_NAME = 'Allusion'; @@ -42,3 +42,17 @@ export async function migrateToLatest(db: Kysely): Promise console.error(error); } } + +export const kyselyLogger: Logger = (event: LogEvent): void => { + if (event.level === 'query') { + console.log('SQL:', event.query.sql); + console.log('Parameters:', event.query.parameters); + console.log('Duration:', event.queryDurationMillis, 'ms'); + } + + if (event.level === 'error') { + console.error('SQL Error:', event.error); + console.error('Failed Query:', event.query.sql); + console.error('Parameters:', event.query.parameters); + } +}; diff --git a/src/backend/migrations/000_initial.ts b/src/backend/migrations/000_initial.ts index 06577b2a..c48356f6 100644 --- a/src/backend/migrations/000_initial.ts +++ b/src/backend/migrations/000_initial.ts @@ -12,8 +12,6 @@ export async function up(db: Kysely): Promise { await db.schema .createTable('tags') .addColumn('id', 'text', (col) => col.primaryKey().notNull()) - .addColumn('parent_id', 'text') - .addColumn('idx', 'integer', (col) => col.notNull()) .addColumn('name', 'text', (col) => col.notNull()) .addColumn('date_added', 'timestamp', (col) => col.notNull().defaultTo(sql`CURRENT_TIMESTAMP`)) .addColumn('color', 'text') @@ -21,9 +19,18 @@ export async function up(db: Kysely): Promise { .addColumn('is_visible_inherited', 'boolean', (col) => col.notNull().defaultTo(1)) .addColumn('is_header', 'boolean', (col) => col.notNull().defaultTo(0)) .addColumn('description', 'text') - .addForeignKeyConstraint('fk_tag_parent', ['parent_id'], 'tags', ['id'], (cb) => cb.onDelete('cascade')) .execute(); - await db.schema.createIndex('idx_tags_parent').on('tags').column('parent_id').execute(); + + await db.schema + .createTable('sub_tags') + .addColumn('tag_id', 'text', (col) => col.notNull()) + .addColumn('sub_tag_id', 'text', (col) => col.notNull()) + .addColumn('idx', 'integer', (col) => col.notNull()) + .addPrimaryKeyConstraint('pk_tag_implications', ['tag_id', 'sub_tag_id']) + .addForeignKeyConstraint('fk_tag_implications_tag', ['tag_id'], 'tags', ['id'], (cb) => cb.onDelete('cascade')) + .addForeignKeyConstraint('fk_tag_implications_implied', ['sub_tag_id'], 'tags', ['id'], (cb) => cb.onDelete('cascade')) + .addUniqueConstraint('uq_sub_tags_sub_tag', ['sub_tag_id']) + .execute(); await db.schema .createTable('tag_implications') @@ -158,7 +165,7 @@ export async function up(db: Kysely): Promise { .addColumn('id', 'text', (col) => col.primaryKey().notNull()) .addColumn('saved_search_id', 'text', (col) => col.notNull()) .addColumn('idx', 'integer', (col) => col.notNull()) - .addColumn('match_group', 'text', (col) => col.notNull()) // 'any' | 'all' + .addColumn('conjunction', 'text', (col) => col.notNull()) .addColumn('key', 'text', (col) => col.notNull()) .addColumn('value_type', 'text', (col) => col.notNull()) .addColumn('operator', 'text', (col) => col.notNull()) diff --git a/src/backend/schemaTypes.ts b/src/backend/schemaTypes.ts index 81716fc4..c99b034d 100644 --- a/src/backend/schemaTypes.ts +++ b/src/backend/schemaTypes.ts @@ -18,6 +18,7 @@ import { ID } from '../api/id'; import { CriteriaValueType, OperatorType } from 'src/api/search-criteria'; import { FILE_TAGS_SORTING_TYPE, FileDTO, IMG_EXTENSIONS_TYPE } from 'src/api/file'; import { ExtraPropertyType } from 'src/api/extraProperty'; +import { SearchConjunction } from 'src/api/data-storage-search'; export type BooleanAsNumber = number; export const serializeBoolean = (value: boolean): number => (value ? 1 : 0); @@ -28,6 +29,7 @@ export const deserializeDate = (value: number): Date => new Date(value); export type AllusionDB_SQL = { tags: Tags; + subTags: SubTags; tagImplications: TagImplications; tagAliases: TagAliases; locationNodes: LocationNodes; @@ -46,8 +48,6 @@ export type AllusionDB_SQL = { export type Tags = { id: ColumnType; //pk - parentId: ID | null; //fk - idx: number; name: string; dateAdded: ColumnType; color: string; @@ -57,6 +57,12 @@ export type Tags = { description: string; }; +export type SubTags = { + tagId: ID; //pk fk + subTagId: ID; //pk fk + idx: number; +}; + export type TagImplications = { tagId: ID; //pk fk impliedTagId: ID; //pk fk @@ -147,7 +153,7 @@ export type SearchCriteria = { id: ColumnType; //pk savedSearchId: ID; //fk idx: number; - matchGroup: 'any' | 'all'; + conjunction: SearchConjunction; key: keyof FileDTO; valueType: CriteriaValueType; operator: OperatorType; diff --git a/src/frontend/containers/AdvancedSearch/data.ts b/src/frontend/containers/AdvancedSearch/data.ts index 26ef348c..80adde64 100644 --- a/src/frontend/containers/AdvancedSearch/data.ts +++ b/src/frontend/containers/AdvancedSearch/data.ts @@ -5,7 +5,7 @@ import { StringOperatorType, } from '../../../api/data-storage-search'; import { FileDTO, IMG_EXTENSIONS } from '../../../api/file'; -import { ID } from '../../../api/id'; +import { generateId, ID } from '../../../api/id'; import { BinaryOperatorType, OperatorType, TagOperatorType } from '../../../api/search-criteria'; import { ClientDateSearchCriteria, @@ -34,6 +34,7 @@ export type Criteria = | ExtraPropertyField; interface Field { + id: ID; key: K; operator: O; value: V; @@ -66,17 +67,19 @@ export type ExtraPropertyID = ID | undefined; export function defaultQuery(key: Key, extraPropertyType?: ExtraPropertyType): Criteria { if (key === 'name' || key === 'absolutePath') { - return { key, operator: 'contains', value: '' }; + return { id: generateId(), key, operator: 'contains', value: '' }; } else if (key === 'tags') { - return { key, operator: 'contains', value: undefined }; + return { id: generateId(), key, operator: 'contains', value: undefined }; } else if (key === 'extension') { return { + id: generateId(), key, operator: 'equals', value: IMG_EXTENSIONS[0], }; } else if (key === 'dateAdded') { return { + id: generateId(), key, operator: 'equals', value: new Date(), @@ -85,6 +88,7 @@ export function defaultQuery(key: Key, extraPropertyType?: ExtraPropertyType): C if (extraPropertyType !== undefined) { if (extraPropertyType === ExtraPropertyType.number) { return { + id: generateId(), extraProperty: undefined, key: 'extraProperties', value: 0, @@ -92,6 +96,7 @@ export function defaultQuery(key: Key, extraPropertyType?: ExtraPropertyType): C }; } else if (extraPropertyType === ExtraPropertyType.text) { return { + id: generateId(), extraProperty: undefined, key: 'extraProperties', value: '', @@ -100,13 +105,14 @@ export function defaultQuery(key: Key, extraPropertyType?: ExtraPropertyType): C } } return { + id: generateId(), extraProperty: undefined, key: 'extraProperties', value: 0, operator: 'equals', }; } else { - return { key: key, operator: 'greaterThanOrEquals', value: 0 }; + return { id: generateId(), key: key, operator: 'greaterThanOrEquals', value: 0 }; } } @@ -152,25 +158,27 @@ export function fromCriteria(criteria: ClientFileSearchCriteria): [ID, Criteria] return [generateCriteriaId(), query]; } +//prettier-ignore export function intoCriteria(query: Criteria, tagStore: TagStore): ClientFileSearchCriteria { if (query.key === 'name' || query.key === 'absolutePath' || query.key === 'extension') { - return new ClientStringSearchCriteria(query.key, query.value, query.operator); + return new ClientStringSearchCriteria(query.id, query.key, query.value, query.operator); } else if (query.key === 'dateAdded') { - return new ClientDateSearchCriteria(query.key, query.value, query.operator); + return new ClientDateSearchCriteria(query.id, query.key, query.value, query.operator); } else if (query.key === 'size') { - return new ClientNumberSearchCriteria(query.key, query.value * BYTES_IN_MB, query.operator); + return new ClientNumberSearchCriteria(query.id, query.key, query.value * BYTES_IN_MB, query.operator); } else if (query.key === 'width' || query.key === 'height') { - return new ClientNumberSearchCriteria(query.key, query.value, query.operator); + return new ClientNumberSearchCriteria(query.id, query.key, query.value, query.operator); } else if (query.key === 'tags') { const tag = query.value !== undefined ? tagStore.get(query.value) : undefined; - return new ClientTagSearchCriteria('tags', tag?.id, query.operator); + return new ClientTagSearchCriteria(query.id, 'tags', tag?.id, query.operator); } else if (query.key === 'extraProperties') { return new ClientExtraPropertySearchCriteria( + query.id, query.key, [query.extraProperty ?? '', query.value], query.operator, ); } else { - return new ClientTagSearchCriteria('tags'); + return new ClientTagSearchCriteria(query.id, 'tags'); } } diff --git a/src/frontend/containers/AppToolbar/Searchbar.tsx b/src/frontend/containers/AppToolbar/Searchbar.tsx index a9c5842a..d6d0d2b7 100644 --- a/src/frontend/containers/AppToolbar/Searchbar.tsx +++ b/src/frontend/containers/AppToolbar/Searchbar.tsx @@ -67,7 +67,9 @@ const QuickSearchList = observer(() => { }); const handleSelect = useAction((item: Readonly) => - uiStore.addSearchCriteria(new ClientTagSearchCriteria('tags', item.id, 'containsRecursively')), + uiStore.addSearchCriteria( + new ClientTagSearchCriteria(undefined, 'tags', item.id, 'containsRecursively'), + ), ); const handleDeselect = useAction((item: Readonly) => { @@ -96,7 +98,9 @@ const QuickSearchList = observer(() => { value={`Search in file paths for "${query}"`} onClick={() => { resetTextBox(); - uiStore.addSearchCriteria(new ClientStringSearchCriteria('absolutePath', query)); + uiStore.addSearchCriteria( + new ClientStringSearchCriteria(undefined, 'absolutePath', query), + ); }} />, { uiStore.addSearchCriteria( new ClientExtraPropertySearchCriteria( + undefined, 'extraProperties', [eventExtraProperty.id, value], operator, diff --git a/src/frontend/containers/ContentView/menu-items.tsx b/src/frontend/containers/ContentView/menu-items.tsx index 76115c2f..d801d16e 100644 --- a/src/frontend/containers/ContentView/menu-items.tsx +++ b/src/frontend/containers/ContentView/menu-items.tsx @@ -131,7 +131,9 @@ export const FileViewerMenuItems = ({ file }: { file: ClientFile }) => { onClick={(e) => handleSearchSimilar( e, - file.tags.toJSON().map((t) => new ClientTagSearchCriteria('tags', t.id, 'contains')), + file.tags + .toJSON() + .map((t) => new ClientTagSearchCriteria(undefined, 'tags', t.id, 'contains')), ) } text="Same Tags" @@ -142,6 +144,7 @@ export const FileViewerMenuItems = ({ file }: { file: ClientFile }) => { handleSearchSimilar( e, new ClientStringSearchCriteria( + undefined, 'absolutePath', SysPath.dirname(file.absolutePath) + SysPath.sep, 'startsWith', @@ -156,6 +159,7 @@ export const FileViewerMenuItems = ({ file }: { file: ClientFile }) => { handleSearchSimilar( e, new ClientStringSearchCriteria( + undefined, 'absolutePath', locationStore.get(file.locationId)!.path + SysPath.sep, 'startsWith', @@ -168,7 +172,7 @@ export const FileViewerMenuItems = ({ file }: { file: ClientFile }) => { onClick={(e) => handleSearchSimilar( e, - new ClientStringSearchCriteria('extension', file.extension, 'equals'), + new ClientStringSearchCriteria(undefined, 'extension', file.extension, 'equals'), ) } text="Same File Type" @@ -177,8 +181,8 @@ export const FileViewerMenuItems = ({ file }: { file: ClientFile }) => { handleSearchSimilar(e, [ - new ClientNumberSearchCriteria('width', file.width, 'equals'), - new ClientNumberSearchCriteria('height', file.height, 'equals'), + new ClientNumberSearchCriteria(undefined, 'width', file.width, 'equals'), + new ClientNumberSearchCriteria(undefined, 'height', file.height, 'equals'), ]) } text="Same Resolution" @@ -186,7 +190,10 @@ export const FileViewerMenuItems = ({ file }: { file: ClientFile }) => { /> - handleSearchSimilar(e, new ClientNumberSearchCriteria('size', file.size, 'equals')) + handleSearchSimilar( + e, + new ClientNumberSearchCriteria(undefined, 'size', file.size, 'equals'), + ) } text="Same File Size" icon={IconSet.FILTER_FILTER_DOWN} @@ -195,7 +202,7 @@ export const FileViewerMenuItems = ({ file }: { file: ClientFile }) => { onClick={(e) => handleSearchSimilar( e, - new ClientDateSearchCriteria('dateCreated', file.dateCreated, 'equals'), + new ClientDateSearchCriteria(undefined, 'dateCreated', file.dateCreated, 'equals'), ) } text="Same Creation Date" @@ -205,7 +212,7 @@ export const FileViewerMenuItems = ({ file }: { file: ClientFile }) => { onClick={(e) => handleSearchSimilar( e, - new ClientDateSearchCriteria('dateModified', file.dateModified, 'equals'), + new ClientDateSearchCriteria(undefined, 'dateModified', file.dateModified, 'equals'), ) } text="Same Modification Date" diff --git a/src/frontend/containers/Outliner/LocationsPanel/index.tsx b/src/frontend/containers/Outliner/LocationsPanel/index.tsx index ee92589f..7db12f85 100644 --- a/src/frontend/containers/Outliner/LocationsPanel/index.tsx +++ b/src/frontend/containers/Outliner/LocationsPanel/index.tsx @@ -98,7 +98,7 @@ const isExpanded = (nodeData: ClientLocation | ClientSubLocation, treeData: ITre const pathAsSearchPath = (path: string) => `${path}${SysPath.sep}`; const pathCriteria = (path: string) => - new ClientStringSearchCriteria('absolutePath', pathAsSearchPath(path), 'startsWith'); + new ClientStringSearchCriteria(undefined, 'absolutePath', pathAsSearchPath(path), 'startsWith'); const customKeys = ( search: (path: string) => void, diff --git a/src/frontend/containers/Outliner/TagsPanel/ContextMenu.tsx b/src/frontend/containers/Outliner/TagsPanel/ContextMenu.tsx index f4448af6..6b25d225 100644 --- a/src/frontend/containers/Outliner/TagsPanel/ContextMenu.tsx +++ b/src/frontend/containers/Outliner/TagsPanel/ContextMenu.tsx @@ -158,7 +158,7 @@ export const TagItemContextMenu = observer((props: IContextMenuProps) => { onClick={() => tag.isSelected ? uiStore.addTagSelectionToCriteria() - : uiStore.addSearchCriteria(new ClientTagSearchCriteria('tags', tag.id)) + : uiStore.addSearchCriteria(new ClientTagSearchCriteria(undefined, 'tags', tag.id)) } text="Add to Search" icon={IconSet.SEARCH} @@ -167,7 +167,7 @@ export const TagItemContextMenu = observer((props: IContextMenuProps) => { onClick={() => tag.isSelected ? uiStore.replaceCriteriaWithTagSelection() - : uiStore.replaceSearchCriteria(new ClientTagSearchCriteria('tags', tag.id)) + : uiStore.replaceSearchCriteria(new ClientTagSearchCriteria(undefined, 'tags', tag.id)) } text="Replace Search" icon={IconSet.REPLACE} diff --git a/src/frontend/containers/Outliner/TagsPanel/TagsTree.tsx b/src/frontend/containers/Outliner/TagsPanel/TagsTree.tsx index 105d978b..f05342aa 100644 --- a/src/frontend/containers/Outliner/TagsPanel/TagsTree.tsx +++ b/src/frontend/containers/Outliner/TagsPanel/TagsTree.tsx @@ -128,7 +128,7 @@ const toggleQuery = (nodeData: ClientTag, uiStore: UiStore) => { ); } } else { - uiStore.addSearchCriteria(new ClientTagSearchCriteria('tags', nodeData.id)); + uiStore.addSearchCriteria(new ClientTagSearchCriteria(undefined, 'tags', nodeData.id)); } }; @@ -287,7 +287,12 @@ const TagItem = observer((props: ITagItemProps) => { } } else { // otherwise, search it - const query = new ClientTagSearchCriteria('tags', nodeData.id, 'containsRecursively'); + const query = new ClientTagSearchCriteria( + undefined, + 'tags', + nodeData.id, + 'containsRecursively', + ); if (event.ctrlKey || event.metaKey) { uiStore.addSearchCriteria(query); } else { diff --git a/src/frontend/containers/Outliner/TagsPanel/index.tsx b/src/frontend/containers/Outliner/TagsPanel/index.tsx index 06bf0a02..20650162 100644 --- a/src/frontend/containers/Outliner/TagsPanel/index.tsx +++ b/src/frontend/containers/Outliner/TagsPanel/index.tsx @@ -36,7 +36,7 @@ export const OutlinerActionBar = observer(() => { if (maybeUntaggedCrit) { uiStore.removeSearchCriteria(maybeUntaggedCrit); } else { - uiStore.addSearchCriteria(new ClientTagSearchCriteria('tags')); + uiStore.addSearchCriteria(new ClientTagSearchCriteria(undefined, 'tags')); } // eslint-disable-next-line react-hooks/exhaustive-deps }, []); diff --git a/src/frontend/entities/File.ts b/src/frontend/entities/File.ts index 348ee47c..d59d9b05 100644 --- a/src/frontend/entities/File.ts +++ b/src/frontend/entities/File.ts @@ -260,7 +260,6 @@ export class ClientFile { ([extraProperty, value]) => [extraProperty.id, value], ); const extraProperties: ExtraProperties = Object.fromEntries(entries); - const extraPropertyIDs = entries.map(([id]) => id); return { id: this.id, ino: this.ino, @@ -269,7 +268,6 @@ export class ClientFile { absolutePath: this.absolutePath, tags: Array.from(this.tags, (t) => t.id), // removes observable properties from observable array tagsSorting: this.tagsSorting, - extraPropertyIDs: extraPropertyIDs, extraProperties: extraProperties, size: this.size, width: this.width, diff --git a/src/frontend/entities/SearchCriteria.ts b/src/frontend/entities/SearchCriteria.ts index b275074b..a8901fba 100644 --- a/src/frontend/entities/SearchCriteria.ts +++ b/src/frontend/entities/SearchCriteria.ts @@ -3,7 +3,6 @@ import { action, Lambda, makeObservable, observable } from 'mobx'; import { camelCaseToSpaced } from '../../../common/fmt'; import { ArrayConditionDTO, - ArrayOperatorType, ConditionDTO, DateConditionDTO, ExtraPropertyOperatorType, @@ -11,11 +10,12 @@ import { isExtraPropertyOperatorType, NumberConditionDTO, NumberOperatorType, + SearchConjunction, StringConditionDTO, StringOperatorType, } from '../../api/data-storage-search'; import { FileDTO } from '../../api/file'; -import { ID } from '../../api/id'; +import { generateId, ID } from '../../api/id'; import { CriteriaValueType, IBaseSearchCriteria, @@ -65,13 +65,21 @@ export const ExtraPropertyOperatorLabels: Record - | [ArrayConditionDTO, IndexSignatureConditionDTO]; - export class ClientExtraPropertySearchCriteria extends ClientFileSearchCriteria { @observable public value: [string, ExtraPropertyValue]; constructor( + id: ID | undefined, key: keyof FileDTO, value: [string, ExtraPropertyValue] = ['', 0], operator: OperatorType = 'equals', ) { - super(key, 'indexSignature', operator); + super(id, key, 'indexSignature', operator); this.value = value; makeObservable(this); } @@ -226,6 +237,7 @@ export class ClientExtraPropertySearchCriteria extends ClientFileSearchCriteria serialize = (): IExtraProperySearchCriteria => { return { + id: this.id, key: this.key, valueType: this.valueType, operator: this.operator, @@ -233,31 +245,9 @@ export class ClientExtraPropertySearchCriteria extends ClientFileSearchCriteria }; }; - toCondition = (rootStore: RootStore): ExtraPropertySerializedCondition => { - const firstCiteria = rootStore.uiStore.searchCriteriaList[0] as - | ClientFileSearchCriteria - | undefined; - if (firstCiteria === this) { - const serialized = this.serialize(); - let ArrayOperator: ArrayOperatorType; - if (serialized.operator === 'notExistsInFile') { - ArrayOperator = 'notContains'; - } else { - ArrayOperator = 'contains'; - } - const whereSerialized: ArrayConditionDTO = { - key: 'extraPropertyIDs', - valueType: 'array', - operator: ArrayOperator, - value: [serialized.value[0]], - }; - return [ - whereSerialized as ArrayConditionDTO, - serialized as IndexSignatureConditionDTO, - ]; - } else { - return this.serialize() as IndexSignatureConditionDTO; - } + toCondition = (): IndexSignatureConditionDTO => { + // Hacky + return this.serialize() as unknown as IndexSignatureConditionDTO; }; @action.bound setOperator(op: StringOperatorType | NumberOperatorType): void { @@ -272,8 +262,13 @@ export class ClientExtraPropertySearchCriteria extends ClientFileSearchCriteria export class ClientStringSearchCriteria extends ClientFileSearchCriteria { @observable public value: string; - constructor(key: keyof FileDTO, value: string = '', operator: StringOperatorType = 'contains') { - super(key, 'string', operator); + constructor( + id: ID | undefined, + key: keyof FileDTO, + value: string = '', + operator: StringOperatorType = 'contains', + ) { + super(id, key, 'string', operator); this.value = value; makeObservable(this); } @@ -285,6 +280,7 @@ export class ClientStringSearchCriteria extends ClientFileSearchCriteria { serialize = (): IStringSearchCriteria => { return { + id: this.id, key: this.key, valueType: this.valueType, operator: this.operator as StringOperatorType, @@ -309,11 +305,12 @@ export class ClientNumberSearchCriteria extends ClientFileSearchCriteria { @observable public value: number; constructor( + id: ID | undefined, key: keyof FileDTO, value: number = 0, operator: NumberOperatorType = 'greaterThanOrEquals', ) { - super(key, 'number', operator); + super(id, key, 'number', operator); this.value = value; makeObservable(this); } @@ -324,6 +321,7 @@ export class ClientNumberSearchCriteria extends ClientFileSearchCriteria { serialize = (): INumberSearchCriteria => { return { + id: this.id, key: this.key, valueType: this.valueType, operator: this.operator as NumberOperatorType, @@ -348,11 +346,12 @@ export class ClientDateSearchCriteria extends ClientFileSearchCriteria { @observable public value: Date; constructor( + id: ID | undefined, key: keyof FileDTO, value: Date = new Date(), operator: NumberOperatorType = 'equals', ) { - super(key, 'date', operator); + super(id, key, 'date', operator); this.value = value; this.value.setHours(0, 0, 0, 0); makeObservable(this); @@ -365,6 +364,7 @@ export class ClientDateSearchCriteria extends ClientFileSearchCriteria { serialize = (): IDateSearchCriteria => { return { + id: this.id, key: this.key, valueType: this.valueType, operator: this.operator as NumberOperatorType, diff --git a/src/frontend/stores/FileStore.ts b/src/frontend/stores/FileStore.ts index e57682cc..db00dc61 100644 --- a/src/frontend/stores/FileStore.ts +++ b/src/frontend/stores/FileStore.ts @@ -692,7 +692,7 @@ class FileStore { @action async deleteFilesByExtension(ext: IMG_EXTENSIONS_TYPE): Promise { try { - const crit = new ClientStringSearchCriteria('extension', ext, 'equals'); + const crit = new ClientStringSearchCriteria(undefined, 'extension', ext, 'equals'); const files = await this.backend.searchFiles( crit.toCondition(), 'id', @@ -726,6 +726,8 @@ class FileStore { // Apply a small delay to give time to the progressbar // to start the animation before the backend blocks the tread. await delay(600); + } else { + await delay(10); } return now; } @@ -780,7 +782,7 @@ class FileStore { const { uiStore } = this.rootStore; const { searchMatchAny } = uiStore; uiStore.clearSearchCriteriaList(); - const criteria = new ClientTagSearchCriteria('tags'); + const criteria = new ClientTagSearchCriteria(undefined, 'tags'); uiStore.searchCriteriaList.push(criteria); const fetchedFiles = await this.backend.searchFiles( criteria.toCondition(this.rootStore), diff --git a/src/frontend/stores/LocationStore.ts b/src/frontend/stores/LocationStore.ts index 12181af7..2385789e 100644 --- a/src/frontend/stores/LocationStore.ts +++ b/src/frontend/stores/LocationStore.ts @@ -316,9 +316,6 @@ class LocationStore { ...match, tags: Array.from(new Set([...missingFiles[i].tags, ...match.tags])), extraProperties: { ...missingFiles[i].extraProperties, ...match.extraProperties }, - extraPropertyIDs: Array.from( - new Set([...missingFiles[i].extraPropertyIDs, ...match.extraPropertyIDs]), - ), }); } } @@ -645,12 +642,18 @@ class LocationStore { * Fetches the files belonging to a location */ @action async findLocationFiles(locationId: ID): Promise { - const crit = new ClientStringSearchCriteria('locationId', locationId, 'equals').toCondition(); + const crit = new ClientStringSearchCriteria( + undefined, + 'locationId', + locationId, + 'equals', + ).toCondition(); return this.backend.searchFiles(crit, 'id', OrderDirection.Asc, false); } @action async removeSublocationFiles(subLoc: ClientSubLocation): Promise { const crit = new ClientStringSearchCriteria( + undefined, 'absolutePath', subLoc.path, 'startsWith', @@ -705,7 +708,6 @@ export async function pathToIFile( locationId: loc.id, tags: [], tagsSorting: 'hierarchy', - extraPropertyIDs: [], extraProperties: {}, dateAdded: now, dateModified: now, diff --git a/src/frontend/stores/UiStore.ts b/src/frontend/stores/UiStore.ts index cd70cdcd..5bd4182a 100644 --- a/src/frontend/stores/UiStore.ts +++ b/src/frontend/stores/UiStore.ts @@ -1206,7 +1206,7 @@ class UiStore { @action.bound addTagSelectionToCriteria(): void { const newCrits = Array.from( this.tagSelection, - (tag) => new ClientTagSearchCriteria('tags', tag.id), + (tag) => new ClientTagSearchCriteria(undefined, 'tags', tag.id), ); this.addSearchCriterias(newCrits); for (const tag of this.tagSelection) { @@ -1217,7 +1217,10 @@ class UiStore { @action.bound replaceCriteriaWithTagSelection(): void { this.replaceSearchCriterias( - Array.from(this.tagSelection, (tag) => new ClientTagSearchCriteria('tags', tag.id)), + Array.from( + this.tagSelection, + (tag) => new ClientTagSearchCriteria(undefined, 'tags', tag.id), + ), ); for (const tag of this.tagSelection) { this.addRecentlyUsedTag(tag); diff --git a/tests/backend.test.ts b/tests/backend.test.ts index acf386f0..c5ea6dae 100644 --- a/tests/backend.test.ts +++ b/tests/backend.test.ts @@ -53,7 +53,6 @@ describe('Backend', () => { ino: index.toString(), id: index.toString(), tags: [], - extraPropertyIDs: [], extraProperties: {}, }); } From 62af34b43ea2a29f9993e350bd6373410071cc4c Mon Sep 17 00:00:00 2001 From: RafaUC Date: Mon, 10 Nov 2025 00:00:57 -0600 Subject: [PATCH 08/19] Implement the BackupScheduler class and refactor some DB import methods. --- src/api/data-backup.ts | 1 + src/backend/_deprecated/backup-scheduler.ts | 4 + src/backend/backend.ts | 64 ++++- src/backend/backup-scheduler.ts | 260 ++++++++++++++++-- src/backend/config.ts | 17 +- src/backend/migrations/001_migrateJSON.ts | 37 +-- .../containers/Settings/ImportExport.tsx | 8 +- src/main.ts | 20 +- src/renderer.tsx | 71 +++-- 9 files changed, 382 insertions(+), 100 deletions(-) diff --git a/src/api/data-backup.ts b/src/api/data-backup.ts index cf95b6d4..54e45c5d 100644 --- a/src/api/data-backup.ts +++ b/src/api/data-backup.ts @@ -5,5 +5,6 @@ export interface DataBackup { schedule(): void; backupToFile(path: string): Promise; restoreFromFile(path: string): Promise; + restoreEmpty(): Promise; peekFile(path: string): Promise<[numTags: number, numFiles: number]>; } diff --git a/src/backend/_deprecated/backup-scheduler.ts b/src/backend/_deprecated/backup-scheduler.ts index b5592cbd..2bbcc0a4 100644 --- a/src/backend/_deprecated/backup-scheduler.ts +++ b/src/backend/_deprecated/backup-scheduler.ts @@ -20,6 +20,10 @@ export default class BackupScheduler implements DataBackup { this.#backupDirectory = directory; } + restoreEmpty(): Promise { + throw new Error('Method not implemented.'); + } + static async init(db: Dexie, backupDirectory: string): Promise { await fse.ensureDir(backupDirectory); return new BackupScheduler(db, backupDirectory); diff --git a/src/backend/backend.ts b/src/backend/backend.ts index 9df1a02d..c5d55be9 100644 --- a/src/backend/backend.ts +++ b/src/backend/backend.ts @@ -68,7 +68,9 @@ const USE_QUERY_LOGGER = false ? IS_DEV : false; export default class Backend implements DataStorage { readonly MAX_VARS!: number; #db!: Kysely; + #dbPath!: string; #notifyChange!: () => void; + #restoreEmpty!: () => Promise; /** State variable that indicates if we need to recompute preAggregateJSON */ #isQueryDirty: boolean = true; @@ -77,45 +79,70 @@ export default class Backend implements DataStorage { return USE_TIMING_PROXY ? createTimingProxy(this) : this; } - async init(dbPath: string, notifyChange: () => void): Promise { + async init({ + dbPath, + jsonToImport, + notifyChange, + restoreEmpty, + mode = 'default', + }: { + dbPath: string; + jsonToImport: string | undefined; + notifyChange: () => void; + restoreEmpty: () => Promise; + mode?: 'default' | 'migrate' | 'readonly'; + }): Promise { console.info(`SQLite3: Initializing database "${dbPath}"...`); - const database = new SQLite(dbPath, { timeout: 50000 }); + // For some reason, if initializing the better-sqlite3 db with readonly true, later when disposing the instance, + // it does not remove the WAL files, which is bothersome to leave in the backup directory. + //const isReadOnly = mode === 'readonly'; + const database = new SQLite(dbPath, { timeout: 50000 }); //, readonly: isReadOnly }); + // HACK Use a padded string to do natural sorting database.function('pad_string', { deterministic: true }, (str) => { return str.replace(/\d+/g, (num: string) => num.padStart(PAD_STRING_LENGTH, '0')); }); - const dialect = new SqliteDialect({ - database: database, - }); + + const dialect = new SqliteDialect({ database }); const db = new Kysely({ dialect: dialect, plugins: [new ParseJSONResultsPlugin(), new CamelCasePlugin()], log: USE_QUERY_LOGGER ? kyselyLogger : undefined, // Used only for debugging. }); + // Instead of initializing this through the constructor, set the class properties here, // this allows us to use the class as a worker having async await calls at init. this.#db = db; + this.#dbPath = dbPath; this.#notifyChange = notifyChange; + this.#restoreEmpty = restoreEmpty; (this as any).MAX_VARS = await getSqliteMaxVariables(db); - // check if any migration is needed before configure pragma - await migrateToLatest(db); + // Run migrations if required + if (mode === 'default' || mode === 'migrate') { + await migrateToLatest(db, { jsonToImport }); + } - // We enable case sensitive like for search queries - await sql`PRAGMA case_sensitive_like = ON;`.execute(db); - // Do not wait for writes + if (mode === 'migrate' || mode === 'readonly') { + return; + } + // Configure PRAGMA settings (these can create WAL/SHM files) + // Enable WAL mode to not wait for writes and optimize database await sql`PRAGMA journal_mode = WAL;`.execute(db); + await sql`PRAGMA case_sensitive_like = ON;`.execute(db); await sql`PRAGMA synchronous = NORMAL;`.execute(db); await sql`PRAGMA temp_store = MEMORY;`.execute(db); await sql`PRAGMA automatic_index = ON;`.execute(db); await sql`PRAGMA cache_size = -64000;`.execute(db); - await sql`PRAGMA VACUUM;`.execute(db); await sql`PRAGMA OPTIMIZE;`.execute(db); // Create Root Tag if not exists. - if ( - !(await db.selectFrom('tags').selectAll().where('id', '=', ROOT_TAG_ID).executeTakeFirst()) - ) { + const rootTag = await db + .selectFrom('tags') + .selectAll() + .where('id', '=', ROOT_TAG_ID) + .executeTakeFirst(); + if (!rootTag) { await db .insertInto('tags') .values({ @@ -778,6 +805,8 @@ export default class Backend implements DataStorage { console.info('SQLite: Removing location...', location); // Cascade delte in other tables deleting from locationNodes table. await this.#db.deleteFrom('locationNodes').where('id', '=', location).execute(); + // Run VACUUM to free disk space after large deletions. + await sql`VACUUM;`.execute(this.#db); this.#notifyChange(); } @@ -815,6 +844,7 @@ export default class Backend implements DataStorage { async clear(): Promise { console.info('SQLite: Clearing database...'); + /* const tables = await this.#db .selectFrom('sqlite_master' as any) .select('name') @@ -827,7 +857,11 @@ export default class Backend implements DataStorage { continue; } await this.#db.deleteFrom(name as any).execute(); - } + } */ + + // Empy the tables with a large database takes too long, instead create an emprty DB, + // reinit and restore it at startup relying in the backup-scheduler checkAndRestoreDB behaviour. + await this.#restoreEmpty(); } } diff --git a/src/backend/backup-scheduler.ts b/src/backend/backup-scheduler.ts index 5cf85c0f..9ed4ccf0 100644 --- a/src/backend/backup-scheduler.ts +++ b/src/backend/backup-scheduler.ts @@ -21,24 +21,246 @@ import { SubTags, Tags, } from './schemaTypes'; +import fse from 'fs-extra'; +import path from 'path'; import { ExtraPropertyType } from 'src/api/extraProperty'; -import { computeBatchSize, getSqliteMaxVariables } from './backend'; +import Backend, { computeBatchSize, getSqliteMaxVariables } from './backend'; +import { AUTO_BACKUP_TIMEOUT, DB_TO_IMPORT_NAME, NUM_AUTO_BACKUPS } from './config'; +import { DataBackup } from 'src/api/data-backup'; +import SQLite from 'better-sqlite3'; +import { debounce } from 'common/timeout'; +import { getToday, getWeekStart } from 'common/core'; + +export default class BackupScheduler implements DataBackup { + #db: SQLite.Database; + #backupDirectory: string = ''; + #batabaseDirectory: string = ''; + #lastBackupIndex: number = 0; + #lastBackupDate: Date = new Date(0); + + constructor(databasePath: string, batabaseDirectory: string, backupDirectory: string) { + this.#db = new SQLite(databasePath, { readonly: true }); + this.#batabaseDirectory = batabaseDirectory; + this.#backupDirectory = backupDirectory; + } + + static async init( + databasePath: string, + batabaseDirectory: string, + backupDirectory: string, + ): Promise<{ backupScheduler: BackupScheduler; tempJsonToImport: string | undefined }> { + await fse.ensureDir(backupDirectory); + await fse.ensureDir(batabaseDirectory); + const tempJsonToImport = await this.checkAndRestoreDB( + databasePath, + batabaseDirectory, + backupDirectory, + ); + await delay(5000); + await fse.ensureFile(databasePath); + const backupScheduler = new BackupScheduler(databasePath, batabaseDirectory, backupDirectory); + return { backupScheduler, tempJsonToImport }; + } + + private static async getLastJsonBackupPath(backupDirectory: string): Promise { + const files = await fse.readdir(backupDirectory); + const jsonFiles = files.filter((f) => f.endsWith('.json')); + if (!jsonFiles.length) { + return undefined; + } + const stats = await Promise.all( + jsonFiles.map(async (f) => ({ + path: path.join(backupDirectory, f), + mtime: (await fse.stat(path.join(backupDirectory, f))).mtime, + })), + ); + return stats.reduce((a, b) => (a.mtime > b.mtime ? a : b)).path; + } + + // Check if the DB to import exists, + // if it does and its a json we delete the old DB and return the json path to import. + // if it is a sqlite file we replace the old DB with the new file without opening it. + private static async checkAndRestoreDB( + databasePath: string, + batabaseDirectory: string, + backupDirectory: string, + ): Promise { + const importJsonPath = path.join(batabaseDirectory, `${DB_TO_IMPORT_NAME}.json`); + const importDbPath = path.join(batabaseDirectory, `${DB_TO_IMPORT_NAME}.sqlite`); + try { + if ((await fse.pathExists(importJsonPath)) || (await fse.pathExists(importDbPath))) { + console.info('BackupScheduler: Remove previous DB', databasePath); + await fse.remove(databasePath); + await fse.remove(`${databasePath}-shm`); + await fse.remove(`${databasePath}-wal`); + } + if (await fse.pathExists(importJsonPath)) { + return importJsonPath; + } + if (await fse.pathExists(importDbPath)) { + await fse.move(importDbPath, databasePath, { overwrite: true }); + return undefined; + } + } catch (error) { + console.error(error); + } + return this.getLastJsonBackupPath(backupDirectory); + } + + schedule(): void { + if (new Date().getTime() > this.#lastBackupDate.getTime() + AUTO_BACKUP_TIMEOUT) { + this.#createPeriodicBackup(); + } + } + + /** Creates a copy of a backup file, when the target file creation date is less than the provided date */ + static async #copyFileIfCreatedBeforeDate( + srcPath: string, + targetPath: string, + dateToCheck: Date, + ): Promise { + let createBackup = false; + try { + // If file creation date is less than provided date, create a back-up + const stats = await fse.stat(targetPath); + createBackup = stats.ctime < dateToCheck; + } catch (e) { + // File not found + createBackup = true; + } + if (createBackup) { + try { + await fse.copyFile(srcPath, targetPath); + console.log('Created backup', targetPath); + return true; + } catch (e) { + console.error('Could not create backup', targetPath, e); + } + } + return false; + } + + // Wait 10 seconds after a change for any other changes before creating a backup. + #createPeriodicBackup = debounce(async (): Promise => { + const filePath = path.join( + this.#backupDirectory, + `auto-backup-${this.#lastBackupIndex}.sqlite`, + ); + + this.#lastBackupDate = new Date(); + this.#lastBackupIndex = (this.#lastBackupIndex + 1) % NUM_AUTO_BACKUPS; + + try { + await this.backupToFile(filePath); + + console.log('Created automatic backup', filePath); + + // Check for daily backup + await BackupScheduler.#copyFileIfCreatedBeforeDate( + filePath, + path.join(this.#backupDirectory, 'daily.sqlite'), + getToday(), + ); + + // Check for weekly backup + await BackupScheduler.#copyFileIfCreatedBeforeDate( + filePath, + path.join(this.#backupDirectory, 'weekly.sqlite'), + getWeekStart(), + ); + } catch (e) { + console.error('Could not create periodic backup', filePath, e); + } + }, 10000); + + async backupToFile(path: string): Promise { + console.info('SQLite: Exporting database backup...', path); + await this.#db.backup(path); + } + + async restoreFromFile(sourcePath: string): Promise { + console.info('SQLite: Importing database backup...', sourcePath); + + if (!(await fse.pathExists(sourcePath))) { + throw new Error(`Backup file not found: ${sourcePath}`); + } + const ext = path.extname(sourcePath); + const destPath = path.join(this.#batabaseDirectory, `${DB_TO_IMPORT_NAME}${ext}`); + // Replace file to import if exists. + await fse.remove(destPath); + await fse.copyFile(sourcePath, destPath); + console.info(`SQLite: Backup file copied to ${destPath}`); + } + + async restoreEmpty(): Promise { + const emptyDBPath = path.join(this.#batabaseDirectory, `${DB_TO_IMPORT_NAME}.sqlite`); + await fse.remove(emptyDBPath); + await fse.ensureFile(emptyDBPath); + const db = new Backend(); + // Init the DB to apply the migrations but passing an empty string to not import data brom backup folder. + await db.init({ + dbPath: emptyDBPath, + jsonToImport: '', + notifyChange: () => {}, + restoreEmpty: async () => {}, + mode: 'migrate', + }); + } + + async peekFile(sourcePath: string): Promise<[numTags: number, numFiles: number]> { + console.info('SQLite: Peeking database backup...', sourcePath); + const ext = path.extname(sourcePath); + if (ext === '.json') { + const content = await fs.readFile(sourcePath, 'utf8'); + const json = JSON.parse(content); + if (json.formatName !== 'dexie') { + throw new Error('Invalid backup format (expected dexie .json)'); + } + const tables = Object.fromEntries( + json.data.data.map((table: any) => [table.tableName, table.rows]), + ); + return [tables.tags.length, tables.files.length]; + } + if (ext === '.sqlite') { + let db = null; + db = new Backend(); + await db.init({ + dbPath: sourcePath, + jsonToImport: '', + notifyChange: () => {}, + restoreEmpty: async () => {}, + mode: 'readonly', + }); + const tags = (await db.fetchTags()).length; + const files = (await db.countFiles())[0]; + db = null; + if (global.gc) { + // Remove the backend instance to get rid of any WAL file. + console.log('Forcing Garbage Collection'); + global.gc(); + } + return [tags, files]; + } + throw new Error('Invalid backup format (expected dexie .json or .sqlite)'); + } +} const fallbackIds = { - tag: 'fallback_tag', - location: 'fallback_location', locationNode: 'fallback_location_node', extraProperty: 'fallback_ep', }; export async function restoreFromOldJsonFormat( db: Kysely, - backupFilePath: string, + backupFilePath: string | undefined, ): Promise { + if (backupFilePath === undefined) { + return; + } const content = await fs.readFile(backupFilePath, 'utf8'); const json = JSON.parse(content); - console.log('===================================================='); - console.log('[] Importing Dexie backup from', backupFilePath, '[]'); + console.info('===================================================='); + console.info('-> Importing Dexie backup from', backupFilePath); if (json.formatName !== 'dexie') { throw new Error('Invalid backup format (expected dexie)'); } @@ -48,7 +270,7 @@ export async function restoreFromOldJsonFormat( ); const MAX_VARS = await getSqliteMaxVariables(db); - console.log(`MAX_VARS: ${MAX_VARS}`); + console.info(`MAX_VARS: ${MAX_VARS}`); const saveEntries = async ( entityName: TableName, @@ -58,7 +280,7 @@ export async function restoreFromOldJsonFormat( const batchSize = computeBatchSize(MAX_VARS, entries.find(Boolean)); const MAX_RETRIES = 5; const BASE_DELAY_MS = 100; - console.log( + console.info( `Importing ${entries.length} ${entityName} from old format. (Batch size: ${batchSize})`, ); await db.transaction().execute(async (trx) => { @@ -95,7 +317,7 @@ export async function restoreFromOldJsonFormat( } } }); - console.log(`Finished importing ${entityName}: ${errors} errors.`); + console.info(`Finished importing ${entityName}: ${errors} errors.`); }; // Disable foreign key constraints @@ -103,20 +325,6 @@ export async function restoreFromOldJsonFormat( // Create fallback references for missing foreign keys // Ensure fallback base records exist - await db - .insertInto('tags') - .values({ - id: fallbackIds.tag, - name: 'Fallback Tag', - color: '', - description: '', - isHidden: serializeBoolean(false), - isVisibleInherited: serializeBoolean(true), - isHeader: serializeBoolean(false), - dateAdded: serializeDate(new Date()), - }) - .onConflict((oc) => oc.doNothing()) - .execute(); await db .insertInto('locationNodes') @@ -204,11 +412,11 @@ export async function restoreFromOldJsonFormat( ); await sql`DELETE FROM file_tags WHERE tag_id NOT IN (SELECT id FROM tags);`.execute(db); } else { - console.log('Complete succes! no foreign key issues found:', fkCheck.rows); + console.info('Complete succes! no foreign key issues found:', fkCheck.rows); } - console.log('Dexie backup import completed successfully.'); - console.log('===================================================='); + console.info('Dexie backup import completed successfully.'); + console.info('===================================================='); } function normalizeTags(tags: any[]) { diff --git a/src/backend/config.ts b/src/backend/config.ts index b37077cd..84de0229 100644 --- a/src/backend/config.ts +++ b/src/backend/config.ts @@ -3,6 +3,8 @@ import { AllusionDB_SQL } from './schemaTypes'; export const DB_NAME = 'Allusion'; +export const DB_TO_IMPORT_NAME = 'DB_TO_IMPORT'; + export const NUM_AUTO_BACKUPS = 6; export const AUTO_BACKUP_TIMEOUT = 1000 * 60 * 10; // 10 minutes @@ -13,18 +15,27 @@ export const PAD_STRING_LENGTH = 10; //Register the migrations here. class InlineMigrationProvider implements MigrationProvider { + #context: Record; + + constructor(context: Record = {}) { + this.#context = context; + } async getMigrations(): Promise> { + const context = this.#context; return { '000_initial': await import('./migrations/000_initial'), - '001_migrateJSON': await import('./migrations/001_migrateJSON'), + '001_migrateJSON': (await import('./migrations/001_migrateJSON')).default(context), }; } } -export async function migrateToLatest(db: Kysely): Promise { +export async function migrateToLatest( + db: Kysely, + context: { jsonToImport: string | undefined }, +): Promise { const migrator = new Migrator({ db, - provider: new InlineMigrationProvider(), + provider: new InlineMigrationProvider(context), }); const { error, results } = await migrator.migrateToLatest(); diff --git a/src/backend/migrations/001_migrateJSON.ts b/src/backend/migrations/001_migrateJSON.ts index 36266dcb..e9e9dce5 100644 --- a/src/backend/migrations/001_migrateJSON.ts +++ b/src/backend/migrations/001_migrateJSON.ts @@ -1,30 +1,13 @@ import { Kysely } from 'kysely'; -import path from 'path'; -import { readdir, stat } from 'fs/promises'; -import { RendererMessenger } from 'src/ipc/renderer'; import { restoreFromOldJsonFormat } from '../backup-scheduler'; -export async function getLastJsonBackupPath(): Promise { - const dir = await RendererMessenger.getDefaultBackupDirectory(); - const files = await readdir(dir); - const jsonFiles = files.filter((f) => f.endsWith('.json')); - if (!jsonFiles.length) { - throw new Error(`No .json files found in ${dir}`); - } - const stats = await Promise.all( - jsonFiles.map(async (f) => ({ - path: path.join(dir, f), - mtime: (await stat(path.join(dir, f))).mtime, - })), - ); - return stats.reduce((a, b) => (a.mtime > b.mtime ? a : b)).path; -} - -export async function up(db: Kysely): Promise { - await restoreFromOldJsonFormat(db, await getLastJsonBackupPath()); -} - -export async function down(_: Kysely): Promise { - // No rollback for imports, maybe delete all the data - void _; -} +export default (context: { jsonToImport?: string }) => ({ + async up(db: Kysely): Promise { + const jsonToImport = context.jsonToImport; + await restoreFromOldJsonFormat(db, jsonToImport); + }, + async down(_: Kysely): Promise { + // No rollback for imports, maybe delete all the data + void _; + }, +}); diff --git a/src/frontend/containers/Settings/ImportExport.tsx b/src/frontend/containers/Settings/ImportExport.tsx index 416fec31..e04e89e1 100644 --- a/src/frontend/containers/Settings/ImportExport.tsx +++ b/src/frontend/containers/Settings/ImportExport.tsx @@ -40,7 +40,7 @@ export const ImportExport = observer(() => { const handleCreateExport = async () => { const formattedDateTime = getFilenameFriendlyFormattedDateTime(new Date()); - const filename = `backup_${formattedDateTime}.json`.replaceAll(':', '-'); + const filename = `backup_${formattedDateTime}.sqlite`.replaceAll(':', '-'); const filepath = SysPath.join(backupDir, filename); try { await rootStore.backupDatabaseToFile(filepath); @@ -132,7 +132,11 @@ export const ImportExport = observer(() => { className="btn-outlined" options={{ properties: ['openFile'], - filters: [{ extensions: ['json'], name: 'JSON' }], + filters: [ + { extensions: ['sqlite', 'json'], name: 'Backup file' }, + { extensions: ['sqlite'], name: 'SQLite' }, + { extensions: ['json'], name: 'JSON' }, + ], defaultPath: backupDir, }} onChange={handleChooseImportDir} diff --git a/src/main.ts b/src/main.ts index 6f83f296..a0c9c600 100644 --- a/src/main.ts +++ b/src/main.ts @@ -77,7 +77,13 @@ function initialize() { }); createWindow(); - createPreviewWindow(); + // TODO: During DB backup import, initializing a second window at the same time + // will access the database and prevent the DB file from being removed when + // importing a backup at startup. In the future, we could implement a preview + // initialization that doesn't cause this conflict. + // The preview window can safely be initialized at any time after the main + // window initialization has completed. + //createPreviewWindow(); // Initialize preferences file and its consequences try { @@ -454,6 +460,8 @@ if (!HAS_INSTANCE_LOCK) { mainWindow.focus(); } }); + // Enable manual garbage collector + app.commandLine.appendSwitch('js-flags', '--expose-gc'); // Only initialize window if no other instance is already running: // This method will be called when Electron has finished @@ -505,8 +513,9 @@ autoUpdater.on('update-available', async (info: UpdateInfo) => { return; } - const message = `Update available: ${info.releaseName || info.version - }:\nDo you wish to update now?`; + const message = `Update available: ${ + info.releaseName || info.version + }:\nDo you wish to update now?`; // info.releaseNotes attribute is HTML, could show that in renderer at some point const dialogResult = await dialog.showMessageBox(mainWindow, { @@ -566,8 +575,9 @@ autoUpdater.on('download-progress', (progressObj: { percent: number }) => { process.on('uncaughtException', async (error) => { console.error('Uncaught exception', error); - const errorMessage = `An unexpected error occurred. Please file a bug report if you think this needs fixing!\n${error.stack?.includes(error.message) ? '' : `${error.name}: ${error.message.slice(0, 200)}\n` - }\n${error.stack?.slice(0, 300)}`; + const errorMessage = `An unexpected error occurred. Please file a bug report if you think this needs fixing!\n${ + error.stack?.includes(error.message) ? '' : `${error.name}: ${error.message.slice(0, 200)}\n` + }\n${error.stack?.slice(0, 300)}`; try { if (mainWindow != null && !mainWindow.isDestroyed()) { diff --git a/src/renderer.tsx b/src/renderer.tsx index bda71fe7..72142439 100644 --- a/src/renderer.tsx +++ b/src/renderer.tsx @@ -6,7 +6,6 @@ // in the HTML file import './style.scss'; -import Dexie from 'dexie'; import fse from 'fs-extra'; import { autorun, reaction, runInAction } from 'mobx'; import React from 'react'; @@ -27,8 +26,7 @@ import PreviewApp from './frontend/Preview'; import { FILE_STORAGE_KEY } from './frontend/stores/FileStore'; import RootStore from './frontend/stores/RootStore'; import { PREFERENCES_STORAGE_KEY } from './frontend/stores/UiStore'; -import BackupScheduler from './backend/_deprecated/backup-scheduler'; -import { dbInit } from './backend/_deprecated/config'; +import BackupScheduler from './backend/backup-scheduler'; import path from 'path'; import { DB_NAME, USE_BACKEND_AS_WORKER } from './backend/config'; @@ -44,31 +42,38 @@ async function main(): Promise { root.render(); - const db = dbInit(DB_NAME); const basePath = await RendererMessenger.getPath('userData'); - const databaseTestFilePath = path.join(basePath, 'databases', `${DB_NAME}.sqlite`); + const databaseDirectory = path.join(basePath, 'databases'); + const databaseFilePath = path.join(databaseDirectory, `${DB_NAME}.sqlite`); if (!IS_PREVIEW_WINDOW) { - await runMainApp(databaseTestFilePath, db, root); + await runMainApp(databaseFilePath, databaseDirectory, root); } else { - await runPreviewApp(databaseTestFilePath, db, root); + await runPreviewApp(databaseFilePath, root); } } -async function runMainApp(dbPath: string, db: Dexie, root: Root): Promise { +async function runMainApp(dbPath: string, dbDirectory: string, root: Root): Promise { + // Check if the database file already exists + const dbExists = await fse.pathExists(dbPath); const defaultBackupDirectory = await RendererMessenger.getDefaultBackupDirectory(); - const backup = new BackupScheduler(db, defaultBackupDirectory); + const { backupScheduler, tempJsonToImport } = await BackupScheduler.init( + dbPath, + dbDirectory, + defaultBackupDirectory, + ); let backend: Backend; - - // Check if the database file already exists - const dbExists = await fse.pathExists(dbPath); // If using worker mode and DB already exists, initialize backend in worker // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition - if (USE_BACKEND_AS_WORKER && dbExists) { + if (USE_BACKEND_AS_WORKER && dbExists && !tempJsonToImport) { const backendService = new BackendService(); const [remoteBackend] = await Promise.all([ - backendService.init(dbPath, () => {}), + backendService.init( + dbPath, + () => backupScheduler.schedule(), + () => backupScheduler.restoreEmpty(), + ), fse.ensureDir(defaultBackupDirectory), ]); backend = remoteBackend as unknown as Backend; @@ -77,13 +82,21 @@ async function runMainApp(dbPath: string, db: Dexie, root: Root): Promise // initialize backend in the main thread to safely run migrations backend = new Backend(); await Promise.all([ - backend.init(dbPath, () => {}), + backend.init({ + dbPath: dbPath, + jsonToImport: tempJsonToImport, + notifyChange: () => backupScheduler.schedule(), + restoreEmpty: () => backupScheduler.restoreEmpty(), + }), fse.ensureDir(defaultBackupDirectory), - // ]); + // remove temporal json to avoid infinite re import. + if (tempJsonToImport) { + await fse.remove(tempJsonToImport); + } } - const rootStore = await RootStore.main(backend, backup); + const rootStore = await RootStore.main(backend, backupScheduler); RendererMessenger.initialized(); @@ -214,12 +227,17 @@ async function runMainApp(dbPath: string, db: Dexie, root: Root): Promise window.addEventListener('beforeunload', handleBeforeUnload); } -async function runPreviewApp(dbPath: string, db: Dexie, root: Root): Promise { +async function runPreviewApp(dbPath: string, root: Root): Promise { //const backend = await Backend.init(dbPath, () => {}); // TODO: create an apropiated initPreview mode const backend = new Backend(); - await backend.init(dbPath, () => {}); - const rootStore = await RootStore.preview(backend, new BackupScheduler(db, '')); + await backend.init({ + dbPath: dbPath, + jsonToImport: undefined, + notifyChange: () => {}, + restoreEmpty: async () => {}, + }); + const rootStore = await RootStore.preview(backend, new BackupScheduler(dbPath, '', '')); RendererMessenger.initialized(); @@ -293,7 +311,11 @@ class BackendService { workerInstance?: Worker; private initialized = false; - async init(dbPath: string, notifyChange: () => void): Promise | undefined> { + async init( + dbPath: string, + notifyChange: () => void, + restoreEmpty: () => Promise, + ): Promise | undefined> { if (this.initialized) { console.warn('BackendService: Already initialized'); return this.worker; @@ -310,7 +332,12 @@ class BackendService { this.workerInstance = worker; console.log('BackendService: Initializing worker backend...'); - await this.worker.init(dbPath, proxy(notifyChange)); + await this.worker.init({ + dbPath: dbPath, + jsonToImport: undefined, + notifyChange: proxy(notifyChange), + restoreEmpty: proxy(restoreEmpty), + }); this.initialized = true; console.log('BackendService: Ready!'); From 1a9f520452a43ad243c39c2eb2f56208042bf272 Mon Sep 17 00:00:00 2001 From: RafaUC Date: Wed, 12 Nov 2025 21:31:33 -0600 Subject: [PATCH 09/19] Clean up commented lines, and remove Dexie dependencies. --- package.json | 2 -- src/api/data-storage.ts | 3 +- src/backend/_deprecated/backend.ts | 1 + src/backend/_deprecated/backup-scheduler.ts | 1 + src/backend/_deprecated/config.ts | 1 + src/backend/backend.ts | 2 +- src/backend/backup-scheduler.ts | 2 +- src/backend/migrations/000_initial.ts | 32 --------------------- yarn.lock | 10 ------- 9 files changed, 7 insertions(+), 47 deletions(-) diff --git a/package.json b/package.json index f7b1cc66..d7c84178 100644 --- a/package.json +++ b/package.json @@ -129,8 +129,6 @@ "better-sqlite3": "9.6.0", "chokidar": "^3.5.3", "comlink": "^4.4.1", - "dexie": "^3.2.3", - "dexie-export-import": "^1.0.3", "electron-updater": "^5.3.0", "fs-extra": "^11.1.0", "kysely": "^0.28.7", diff --git a/src/api/data-storage.ts b/src/api/data-storage.ts index bbb16717..862fbd10 100644 --- a/src/api/data-storage.ts +++ b/src/api/data-storage.ts @@ -1,4 +1,3 @@ -import { IndexableType } from 'dexie'; import { ConditionDTO, OrderBy, OrderDirection } from './data-storage-search'; import { FileDTO } from './file'; import { FileSearchDTO } from './file-search'; @@ -7,6 +6,8 @@ import { LocationDTO } from './location'; import { TagDTO } from './tag'; import { ExtraPropertyDTO } from './extraProperty'; +export type IndexableType = number | string | Date | Array | Uint8Array; + /** * The user generated persisted data edited or viewed by one or multiple actors (users, multiple devices etc.). * diff --git a/src/backend/_deprecated/backend.ts b/src/backend/_deprecated/backend.ts index fd0942ee..1d4f3011 100644 --- a/src/backend/_deprecated/backend.ts +++ b/src/backend/_deprecated/backend.ts @@ -1,3 +1,4 @@ +// @ts-nocheck import Dexie, { Collection, IndexableType, Table, WhereClause } from 'dexie'; import { retainArray, shuffleArray } from '../../../common/core'; diff --git a/src/backend/_deprecated/backup-scheduler.ts b/src/backend/_deprecated/backup-scheduler.ts index 2bbcc0a4..b5517b9a 100644 --- a/src/backend/_deprecated/backup-scheduler.ts +++ b/src/backend/_deprecated/backup-scheduler.ts @@ -1,3 +1,4 @@ +// @ts-nocheck import Dexie from 'dexie'; import { exportDB, importDB, peakImportFile } from 'dexie-export-import'; import fse from 'fs-extra'; diff --git a/src/backend/_deprecated/config.ts b/src/backend/_deprecated/config.ts index 5d4341e2..f5416573 100644 --- a/src/backend/_deprecated/config.ts +++ b/src/backend/_deprecated/config.ts @@ -1,3 +1,4 @@ +// @ts-nocheck import Dexie, { Transaction } from 'dexie'; import fse from 'fs-extra'; diff --git a/src/backend/backend.ts b/src/backend/backend.ts index c5d55be9..1ff3b5e6 100644 --- a/src/backend/backend.ts +++ b/src/backend/backend.ts @@ -36,7 +36,7 @@ import { } from 'kysely'; import { kyselyLogger, migrateToLatest, PAD_STRING_LENGTH } from './config'; import { DataStorage } from 'src/api/data-storage'; -import { IndexableType } from 'dexie'; +import { IndexableType } from '../api/data-storage'; import { OrderBy, OrderDirection, diff --git a/src/backend/backup-scheduler.ts b/src/backend/backup-scheduler.ts index 9ed4ccf0..4c62545e 100644 --- a/src/backend/backup-scheduler.ts +++ b/src/backend/backup-scheduler.ts @@ -295,7 +295,7 @@ export async function restoreFromOldJsonFormat( .values(batch) .onConflict((oc) => oc.doNothing()) .execute(); - // If success breack the while + // If success, break the while break; } catch (err: any) { if (err.code === 'SQLITE_BUSY' && attempt < MAX_RETRIES) { diff --git a/src/backend/migrations/000_initial.ts b/src/backend/migrations/000_initial.ts index c48356f6..9dfe0559 100644 --- a/src/backend/migrations/000_initial.ts +++ b/src/backend/migrations/000_initial.ts @@ -106,16 +106,6 @@ export async function up(db: Kysely): Promise { .addForeignKeyConstraint('fk_files_location', ['location_id'], 'locations', ['node_id'], (cb) => cb.onDelete('cascade')) .addUniqueConstraint('uq_absolute_path', ['relative_path']) .execute(); - // await db.schema.createIndex('idx_files_name').on('files').column('name').execute(); - // await db.schema.createIndex('idx_files_extension').on('files').column('extension').execute(); - // await db.schema.createIndex('idx_files_size').on('files').column('size').execute(); - // await db.schema.createIndex('idx_files_width').on('files').column('width').execute(); - // await db.schema.createIndex('idx_files_height').on('files').column('height').execute(); - // await db.schema.createIndex('idx_files_date_added').on('files').column('date_added').execute(); - // await db.schema.createIndex('idx_files_date_modified').on('files').column('date_modified').execute(); - // await db.schema.createIndex('idx_files_date_created').on('files').column('date_created').execute(); - // await db.schema.createIndex('idx_files_relative_path').on('files').column('relative_path').unique().execute(); - // await db.schema.createIndex('idx_files_location').on('files').column('location_id').execute(); await db.schema .createTable('file_tags') @@ -175,33 +165,11 @@ export async function up(db: Kysely): Promise { } export async function down(db: Kysely): Promise { - /* - await db.schema.dropIndex('idx_ep_values_text_file').execute(); await db.schema.dropIndex('idx_ep_values_text_value').execute(); - await db.schema.dropIndex('idx_ep_values_text_ep').execute(); - - await db.schema.dropIndex('idx_ep_values_number_file').execute(); await db.schema.dropIndex('idx_ep_values_number_value').execute(); - await db.schema.dropIndex('idx_ep_values_number_ep').execute(); - - await db.schema.dropIndex('idx_ep_values_timestamp_file').execute(); await db.schema.dropIndex('idx_ep_values_timestamp_value').execute(); - await db.schema.dropIndex('idx_ep_values_timestamp_ep').execute(); - await db.schema.dropIndex('idx_file_tags_file').execute(); await db.schema.dropIndex('idx_file_tags_tag').execute(); - - await db.schema.dropIndex('idx_files_location').execute(); - await db.schema.dropIndex('idx_files_relative_path').execute(); - await db.schema.dropIndex('idx_files_date_created').execute(); - await db.schema.dropIndex('idx_files_date_modified').execute(); - await db.schema.dropIndex('idx_files_date_added').execute(); - await db.schema.dropIndex('idx_files_height').execute(); - await db.schema.dropIndex('idx_files_width').execute(); - await db.schema.dropIndex('idx_files_size').execute(); - await db.schema.dropIndex('idx_files_extension').execute(); - await db.schema.dropIndex('idx_files_name').execute(); -*/ await db.schema.dropTable('search_criteria').execute(); await db.schema.dropTable('saved_searches').execute(); await db.schema.dropTable('ep_values_timestamp').execute(); diff --git a/yarn.lock b/yarn.lock index a07be6d2..d1d71e25 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3251,16 +3251,6 @@ detect-node@^2.0.4: resolved "https://registry.yarnpkg.com/detect-node/-/detect-node-2.0.4.tgz#014ee8f8f669c5c58023da64b8179c083a28c46c" integrity sha512-ZIzRpLJrOj7jjP2miAtgqIfmzbxa4ZOr5jJc601zklsfEx9oTzmmj2nVpIPRpNlRTIh8lc1kyViIY7BWSGNmKw== -dexie-export-import@^1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/dexie-export-import/-/dexie-export-import-1.0.3.tgz#e93926a0a3939c68f5e2b80b48517ea4c6d88fde" - integrity sha512-oun27bUUEaeOfSZ8Cv3Nvj5s0LeANYBYQ7ROpF/3Zg1X/IALUnrX0hk5ZUMlYC3s99kFHimXX57ac5AlOdMzWw== - -dexie@^3.2.3: - version "3.2.3" - resolved "https://registry.yarnpkg.com/dexie/-/dexie-3.2.3.tgz#f35c91ca797599df8e771b998e9ae9669c877f8c" - integrity sha512-iHayBd4UYryDCVUNa3PMsJMEnd8yjyh5p7a+RFeC8i8n476BC9wMhVvqiImq5zJZJf5Tuer+s4SSj+AA3x+ZbQ== - diff-sequences@^29.4.3: version "29.4.3" resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-29.4.3.tgz#9314bc1fabe09267ffeca9cbafc457d8499a13f2" From a122c0149ee77875a9b4c73073cfb976a79b414c Mon Sep 17 00:00:00 2001 From: RafaUC Date: Mon, 17 Nov 2025 23:24:49 -0600 Subject: [PATCH 10/19] Change all advanced search components to support criteria conjunctions. --- resources/style/advanced-search.scss | 56 +++-- src/api/data-storage-search.ts | 5 +- .../AdvancedSearch/CriteriaBuilder.tsx | 13 +- .../containers/AdvancedSearch/Inputs.tsx | 88 +++++++- .../containers/AdvancedSearch/QueryEditor.tsx | 199 +++++++++++------- .../containers/AdvancedSearch/data.ts | 19 +- .../containers/AdvancedSearch/index.tsx | 2 +- src/frontend/entities/SearchCriteria.ts | 5 + 8 files changed, 283 insertions(+), 104 deletions(-) diff --git a/resources/style/advanced-search.scss b/resources/style/advanced-search.scss index 4f95d03d..6b22694b 100644 --- a/resources/style/advanced-search.scss +++ b/resources/style/advanced-search.scss @@ -2,10 +2,11 @@ display: flex; flex-direction: column; // changed from min-width since large tag names expand the dialog width which looks bad - width: min(85ch, 80vw); + width: min(95ch, 80vw); margin-inline: auto; .criteria-input, + .conjuction-input, [role='combobox'] { height: 1.75rem; width: 100%; @@ -19,46 +20,65 @@ } } +#criteria-builder-label { + display: flex; + padding-bottom: 0.4rem; +} + #criteria-builder { display: grid; - grid-template-columns: 1fr 1fr 1fr min-content; + grid-template-columns: 1.6rem max-content 3fr 3fr 4fr min-content; gap: 0.25rem 0.5rem; align-items: center; label { text-transform: uppercase; font-size: smaller; + padding-left: 1ch; } } #query-editor-container { overflow: hidden auto; - padding: 2px 0; + padding: 2px; // dialog height - height of basically everything except the container max-height: calc(80vh - 17.25rem); } +#query-editor-container-label { + display: flex; + padding-block: 0.4rem; +} + #query-editor { + display: grid; + grid-template-columns: 1.6rem max-content 3fr 3fr 4fr min-content; + gap: 0.25rem 0.5rem; + align-items: center; width: 100%; border-spacing: 0; - tr { - margin-bottom: 0.25rem; - } - - td, - th { - padding: 0; - padding-bottom: 0.25rem; - } - - td { - padding-inline-start: 0.5rem; - } - .criteria-input, [role='combobox'] { - min-width: 175px; + //min-width: 175px; width: 100%; } + + .separator { + grid-column: 1 / -1; + display: flex; + align-items: center; + text-align: center; + color: var(--text-color-muted); + opacity: 0.5; + + &::before, + &::after { + content: ""; + flex: 1; + height: 1px; + background: var(--text-color-muted); + margin: 0 8px; + } + } } diff --git a/src/api/data-storage-search.ts b/src/api/data-storage-search.ts index 1fd87c6a..323ddd7e 100644 --- a/src/api/data-storage-search.ts +++ b/src/api/data-storage-search.ts @@ -13,8 +13,6 @@ export const enum OrderDirection { Desc, } -export type SearchConjunction = 'and' | 'or'; - // General search criteria for a database entity // FFR: Boolean keys are not supported in IndexedDB/Dexie - must store booleans as 0/1 @@ -56,6 +54,9 @@ export type BaseIndexSignature = { [key: string]: any }; // Trick for converting array to type https://stackoverflow.com/a/49529930/2350481 +export const SearchConjunctions = ['and', 'or'] as const; +export type SearchConjunction = (typeof SearchConjunctions)[number]; + export const NumberOperators = [ 'equals', 'notEqual', diff --git a/src/frontend/containers/AdvancedSearch/CriteriaBuilder.tsx b/src/frontend/containers/AdvancedSearch/CriteriaBuilder.tsx index 564ca13a..510a38d5 100644 --- a/src/frontend/containers/AdvancedSearch/CriteriaBuilder.tsx +++ b/src/frontend/containers/AdvancedSearch/CriteriaBuilder.tsx @@ -3,7 +3,7 @@ import React, { RefObject, memo, useMemo, useState } from 'react'; import { IconButton } from 'widgets/button'; import { IconSet } from 'widgets/icons'; import { InfoButton } from 'widgets/notifications'; -import { KeySelector, OperatorSelector, ValueInput } from './Inputs'; +import { ConjuctionSelector, KeySelector, OperatorSelector, ValueInput } from './Inputs'; import { QueryDispatch } from './QueryEditor'; import { defaultQuery, generateCriteriaId } from './data'; import { useStore } from 'src/frontend/contexts/StoreContext'; @@ -35,6 +35,9 @@ const CriteriaBuilder = memo(function QueryBuilder({ keySelector, dispatch }: Qu A criteria is made of three components:
    +
  • + conjunction (conj) defines how this criteria will be grouped with others, +
  • key (a property of the image file),
    • @@ -57,11 +60,19 @@ const CriteriaBuilder = memo(function QueryBuilder({ keySelector, dispatch }: Qu
    + + +
     
    + void; + labelledby?: string; + total: number; +} + +export const IndexInput = ({ value, setValue, labelledby, total }: IndexInputProps) => { + const [text, setText] = useState(String(value)); + useEffect(() => { + setText(`${value}`); + }, [value]); + const debouncesetValue = useMemo(() => debounce(setValue, 400), [setValue]); + const commit = useCallback( + (raw: string) => { + if (raw === '' || raw === '-') { + return; + } + let newIdx = parseInt(raw, 10); + if (isNaN(newIdx)) { + return; + } + newIdx = clamp(newIdx, 1, total); + setText(String(newIdx)); + debouncesetValue(newIdx); + }, + [debouncesetValue, total], + ); + const handleInput = useCallback( + (e: React.ChangeEvent) => { + const raw = e.target.value; + if (/^-?\d*$/.test(raw)) { + commit(raw); + } + }, + [commit], + ); + const handleBlur = () => { + commit(text); + }; + + return ( + + ); +}; + +export const ConjuctionSelector = ({ + labelledby, + value, + dispatch, +}: Omit, 'operator' | 'keyValue' | 'extraProperty'>) => { + const handleChange = (e: React.ChangeEvent) => { + const conjuction = e.target.value as SearchConjunction; + dispatch((criteria) => { + criteria.conjunction = conjuction; + return { ...criteria }; + }); + }; + + return ( + + ); +}; type SetCriteria = (fn: (criteria: Criteria) => Criteria) => void; @@ -444,7 +530,7 @@ function getOperatorOptions(key: Key, extraPropertyType?: ExtraPropertyType) { return []; } -const toOperatorOption = (o: T, labels?: Record) => ( +export const toOperatorOption = (o: T, labels?: Record) => ( diff --git a/src/frontend/containers/AdvancedSearch/QueryEditor.tsx b/src/frontend/containers/AdvancedSearch/QueryEditor.tsx index 484eb58b..9fbf6140 100644 --- a/src/frontend/containers/AdvancedSearch/QueryEditor.tsx +++ b/src/frontend/containers/AdvancedSearch/QueryEditor.tsx @@ -4,9 +4,17 @@ import { ID } from 'src/api/id'; import { IconSet } from 'widgets/icons'; import { Callout, InfoButton } from 'widgets/notifications'; import { Radio, RadioGroup } from 'widgets/radio'; -import { KeySelector, OperatorSelector, ValueInput } from './Inputs'; +import { + ConjuctionSelector, + IndexInput, + KeySelector, + OperatorSelector, + ValueInput, +} from './Inputs'; import { Criteria } from './data'; import { useStore } from 'src/frontend/contexts/StoreContext'; +import { clamp } from 'common/core'; +import { SearchConjunction } from 'src/api/data-storage-search'; export type Query = Map; export type QueryDispatch = React.Dispatch>; @@ -22,10 +30,11 @@ export const QueryEditor = memo(function QueryEditor({ setQuery, submissionButtonText = 'Search', }: QueryEditorProps) { + let lastconjuction: SearchConjunction = query.entries().next().value?.[1].conjunction ?? 'and'; return (
    -
    - Query Editor + + Query Editor A query is a list of criterias.
    @@ -38,10 +47,24 @@ export const QueryEditor = memo(function QueryEditor({ icon button next to the inputs.

    - Additionally, there is Match option that decides whether all criterias must match - or just one. +

    + When the search runs, criteria are automatically{' '} + grouped by consecutive conjunctions. In practice this means: +

    +
      +
    • + Adjacent criteria using the same conjunction (either AND{' '} + or OR) are grouped together. +
      • +
    • Each group is evaluated as a single expression using its shared conjunction.
      • +
    • The final search combines those groups in order.
      • +
    +

    + In other words, the conjunction applies between consecutive items, and the system will + internally create the correct logical structure based on how you arranged them. +

    -
    + {query.size === 0 ? ( Your query is currently empty. Create a criteria above to enable the{' '} @@ -49,46 +72,66 @@ export const QueryEditor = memo(function QueryEditor({ ) : undefined}
    - - - - - - - - - - - - {Array.from(query.entries(), ([id, query], index) => ( - - ))} - -
    KeyOperatorValueRemove
    +
    + {/* +
    +
    Key
    +
    Operator
    +
    Value
    +
    Remove
    + */} + {Array.from(query.entries(), ([id, criteria], index) => { + const changed = lastconjuction !== criteria.conjunction; + lastconjuction = criteria.conjunction; + return ( + + {changed && } + + + ); + })} +
    ); }); +const CriteriaSeparator = ({ text }: { text: string }) => { + return
    {text}
    ; +}; + +function reorderMapByIndex(map: Map, fromIndex: number, toIndex: number): Map { + const entries = Array.from(map.entries()); + const [moved] = entries.splice(fromIndex, 1); + entries.splice(toIndex, 0, moved); + return new Map(entries); +} + export interface EditableCriteriaProps { index: number; id: ID; criteria: Criteria; dispatch: QueryDispatch; + totalCriterias: number; } // The main Criteria component, finds whatever input fields for the key should be rendered -export const EditableCriteria = ({ index, id, criteria, dispatch }: EditableCriteriaProps) => { +export const EditableCriteria = (props: EditableCriteriaProps) => { + const { index, id, criteria, dispatch, totalCriterias } = props; const setCriteria = (fn: (criteria: Criteria) => Criteria) => { const c = fn(criteria); dispatch((query) => new Map(query.set(id, c))); }; + const setIndex = (newIndex: number) => { + newIndex = clamp(newIndex - 1, 0, totalCriterias - 1); + dispatch((query) => reorderMapByIndex(query, index, newIndex)); + }; const { extraPropertyStore } = useStore(); const epID = 'extraProperty' in criteria ? criteria.extraProperty : undefined; const extraProperty = useMemo( @@ -97,55 +140,55 @@ export const EditableCriteria = ({ index, id, criteria, dispatch }: EditableCrit ); return ( - - - {index + 1} - - - - - - - - - - - - - - +
    + + + + + + +
    ); }; diff --git a/src/frontend/containers/AdvancedSearch/data.ts b/src/frontend/containers/AdvancedSearch/data.ts index 80adde64..90b5f97f 100644 --- a/src/frontend/containers/AdvancedSearch/data.ts +++ b/src/frontend/containers/AdvancedSearch/data.ts @@ -2,6 +2,7 @@ import { generateWidgetId } from 'widgets/utility'; import { ExtraPropertyOperatorType, NumberOperatorType, + SearchConjunction, StringOperatorType, } from '../../../api/data-storage-search'; import { FileDTO, IMG_EXTENSIONS } from '../../../api/file'; @@ -38,6 +39,7 @@ interface Field { key: K; operator: O; value: V; + conjunction: SearchConjunction; } interface ExtraPropertyField< @@ -67,15 +69,16 @@ export type ExtraPropertyID = ID | undefined; export function defaultQuery(key: Key, extraPropertyType?: ExtraPropertyType): Criteria { if (key === 'name' || key === 'absolutePath') { - return { id: generateId(), key, operator: 'contains', value: '' }; + return { id: generateId(), key, operator: 'contains', value: '', conjunction: 'and' }; } else if (key === 'tags') { - return { id: generateId(), key, operator: 'contains', value: undefined }; + return { id: generateId(), key, operator: 'contains', value: undefined, conjunction: 'and' }; } else if (key === 'extension') { return { id: generateId(), key, operator: 'equals', value: IMG_EXTENSIONS[0], + conjunction: 'and', }; } else if (key === 'dateAdded') { return { @@ -83,6 +86,7 @@ export function defaultQuery(key: Key, extraPropertyType?: ExtraPropertyType): C key, operator: 'equals', value: new Date(), + conjunction: 'and', }; } else if (key === 'extraProperties') { if (extraPropertyType !== undefined) { @@ -93,6 +97,7 @@ export function defaultQuery(key: Key, extraPropertyType?: ExtraPropertyType): C key: 'extraProperties', value: 0, operator: 'equals', + conjunction: 'and', }; } else if (extraPropertyType === ExtraPropertyType.text) { return { @@ -101,6 +106,7 @@ export function defaultQuery(key: Key, extraPropertyType?: ExtraPropertyType): C key: 'extraProperties', value: '', operator: 'contains', + conjunction: 'and', }; } } @@ -110,9 +116,16 @@ export function defaultQuery(key: Key, extraPropertyType?: ExtraPropertyType): C key: 'extraProperties', value: 0, operator: 'equals', + conjunction: 'and', }; } else { - return { id: generateId(), key: key, operator: 'greaterThanOrEquals', value: 0 }; + return { + id: generateId(), + key: key, + operator: 'greaterThanOrEquals', + value: 0, + conjunction: 'and', + }; } } diff --git a/src/frontend/containers/AdvancedSearch/index.tsx b/src/frontend/containers/AdvancedSearch/index.tsx index 16b1c90b..17d8be6c 100644 --- a/src/frontend/containers/AdvancedSearch/index.tsx +++ b/src/frontend/containers/AdvancedSearch/index.tsx @@ -49,7 +49,7 @@ export const AdvancedSearchDialog = observer(() => { - +
    +
    + ); +}; + +export interface EditableCriteriaGroupProps { + groupId: string; + group: CriteriaGroup; + path: string; + setQuery: QueryDispatch; } +export const EditableCriteriaGroup = React.memo(function EditableCriteriaGroup( + props: EditableCriteriaGroupProps, +) { + const { group, groupId, path, setQuery } = props; + const handleChangeConjunction = useCallback( + (conjunction: SearchConjunction) => { + setQuery((query) => { + const critPath = getPathByIndexPath(query, parseIndexPath(path)); + if (!critPath) { + return query; + } + return updateNode(query, critPath, (node) => (node ? { ...node, conjunction } : null)); + }); + }, + [path, setQuery], + ); + return ( +
    + {path !== '' && ( + + )} + {Array.from(group.children.entries(), ([nodeCompId, node], nodeIndex) => ( + + {nodeIndex > 0 && ( + + )} + {/*critIndex > 1 && ( + + )*/} + {isCriteriaGroup(node) ? ( + + ) : ( + + )} + + ))} +
    + ); +}); + export interface EditableCriteriaProps { - index: number; - id: ID; + critId: ID; criteria: Criteria; + path: string; dispatch: QueryDispatch; - totalCriterias: number; } // The main Criteria component, finds whatever input fields for the key should be rendered -export const EditableCriteria = (props: EditableCriteriaProps) => { - const { index, id, criteria, dispatch, totalCriterias } = props; +export const EditableCriteria = React.memo(function EditableCriteria(props: EditableCriteriaProps) { + const { critId, criteria, path, dispatch } = props; const setCriteria = (fn: (criteria: Criteria) => Criteria) => { - const c = fn(criteria); - dispatch((query) => new Map(query.set(id, c))); + dispatch((query) => { + const critPath = getPathByIndexPath(query, parseIndexPath(path)); + if (!critPath) { + return query; + } + return updateNode(query, critPath, (node) => + node ? (!isCriteriaGroup(node) ? { ...node, ...fn(node) } : { ...node }) : null, + ); + }); }; - const setIndex = (newIndex: number) => { - newIndex = clamp(newIndex - 1, 0, totalCriterias - 1); - dispatch((query) => reorderMapByIndex(query, index, newIndex)); + const setIndex = (toIndexPat: CritIndexPath) => { + const critIndexPat = parseIndexPath(path); + dispatch((query) => moveNodeByIndexPath(query, critIndexPat, toIndexPat)); }; const { extraPropertyStore } = useStore(); const epID = 'extraProperty' in criteria ? criteria.extraProperty : undefined; @@ -142,31 +260,25 @@ export const EditableCriteria = (props: EditableCriteriaProps) => { return (
    - { />
    ); -}; +}); type QueryMatchProps = { searchMatchAny: boolean; diff --git a/src/frontend/containers/AdvancedSearch/SearchItemDialog.tsx b/src/frontend/containers/AdvancedSearch/SearchItemDialog.tsx index 947ec590..590224b8 100644 --- a/src/frontend/containers/AdvancedSearch/SearchItemDialog.tsx +++ b/src/frontend/containers/AdvancedSearch/SearchItemDialog.tsx @@ -4,13 +4,12 @@ import React, { useCallback, useRef, useState } from 'react'; import { Button } from 'widgets/button'; import { IconSet } from 'widgets/icons'; import { Dialog } from 'widgets/popovers'; -import { ID } from '../../../api/id'; import { useStore } from '../../contexts/StoreContext'; import { ClientFileSearchItem } from '../../entities/SearchItem'; import { useAutorun } from '../../hooks/mobx'; import CriteriaBuilder from './CriteriaBuilder'; -import { QueryEditor, QueryMatch } from './QueryEditor'; -import { Criteria, fromCriteria, intoCriteria } from './data'; +import { QueryEditor } from './QueryEditor'; +import { queryFromCriteria, intoGroup, Query, getemptyQuery } from './data'; interface ISearchItemDialogProps { searchItem: ClientFileSearchItem; @@ -24,21 +23,15 @@ const SearchItemDialog = observer(({ searchItem, onClose // Copy state of search item: only update the ClientSearchItem on submit. const [name, setName] = useState(searchItem.name); - const [searchMatchAny, setSearchMatchAny] = useState(searchItem.matchAny); - const toggle = useCallback(() => setSearchMatchAny((v) => !v), []); - const [query, setQuery] = useState(new Map()); + const [query, setQuery] = useState(getemptyQuery()); const keySelector = useRef(null); const nameInput = useRef(null); // Initialize form with current queries. When the form is closed, all inputs // are unmounted to save memory. useAutorun(() => { - const map = new Map(); - for (const criteria of searchItem.criteria) { - const [id, query] = fromCriteria(criteria); - map.set(id, query); - } + const map = queryFromCriteria(searchItem.rootGroup); // Focus and select the input text so the user can rename immediately after creating a new search item requestAnimationFrame(() => requestAnimationFrame(() => { @@ -50,12 +43,11 @@ const SearchItemDialog = observer(({ searchItem, onClose }); const handleSubmit = useCallback(async () => { + searchItem.setRootGroup(intoGroup(query, tagStore)); searchItem.setName(name); - searchItem.setMatchAny(searchMatchAny); - searchItem.setCriteria(Array.from(query.values(), (vals) => intoCriteria(vals, tagStore))); searchStore.save(searchItem); onClose(); - }, [name, onClose, query, searchItem, searchMatchAny, searchStore, tagStore]); + }, [name, onClose, query, searchItem, searchStore, tagStore]); return ( (({ searchItem, onClose - -
    diff --git a/src/frontend/containers/AdvancedSearch/data.ts b/src/frontend/containers/AdvancedSearch/data.ts index 90b5f97f..52e4e7d5 100644 --- a/src/frontend/containers/AdvancedSearch/data.ts +++ b/src/frontend/containers/AdvancedSearch/data.ts @@ -18,11 +18,35 @@ import { } from '../../entities/SearchCriteria'; import TagStore from '../../stores/TagStore'; import { ExtraPropertyType, ExtraPropertyValue } from 'src/api/extraProperty'; +import { ClientSearchGroup, isClientSearchGroup } from 'src/frontend/entities/SearchItem'; +import { clamp } from 'common/core'; + +export type Query = CriteriaGroup; +export type QueryDispatch = React.Dispatch>; export function generateCriteriaId() { return generateWidgetId('__criteria'); } +export function generateGroupId() { + return generateWidgetId('__group'); +} + +export type CriteriaNode = Criteria | CriteriaGroup; + +export type GroupMap = Map; + +export type CriteriaGroup = { + id: ID; + name: string; + conjunction: SearchConjunction; + children: GroupMap; +}; + +export function isCriteriaGroup(obj: any): obj is CriteriaGroup { + return obj && typeof obj === 'object' && 'children' in obj; +} + export type Criteria = | Field<'name' | 'absolutePath', StringOperatorType, string> | Field<'tags', TagOperatorType, TagValue> @@ -39,7 +63,6 @@ interface Field { key: K; operator: O; value: V; - conjunction: SearchConjunction; } interface ExtraPropertyField< @@ -67,18 +90,26 @@ export type Value = string | number | Date | TagValue | ExtraPropertyValue; export type TagValue = ID | undefined; export type ExtraPropertyID = ID | undefined; +export function getemptyQuery(): Query { + return { + id: generateGroupId(), + name: '', + conjunction: 'and', + children: new Map(), + }; +} + export function defaultQuery(key: Key, extraPropertyType?: ExtraPropertyType): Criteria { if (key === 'name' || key === 'absolutePath') { - return { id: generateId(), key, operator: 'contains', value: '', conjunction: 'and' }; + return { id: generateId(), key, operator: 'contains', value: '' }; } else if (key === 'tags') { - return { id: generateId(), key, operator: 'contains', value: undefined, conjunction: 'and' }; + return { id: generateId(), key, operator: 'contains', value: undefined }; } else if (key === 'extension') { return { id: generateId(), key, operator: 'equals', value: IMG_EXTENSIONS[0], - conjunction: 'and', }; } else if (key === 'dateAdded') { return { @@ -86,7 +117,6 @@ export function defaultQuery(key: Key, extraPropertyType?: ExtraPropertyType): C key, operator: 'equals', value: new Date(), - conjunction: 'and', }; } else if (key === 'extraProperties') { if (extraPropertyType !== undefined) { @@ -97,7 +127,6 @@ export function defaultQuery(key: Key, extraPropertyType?: ExtraPropertyType): C key: 'extraProperties', value: 0, operator: 'equals', - conjunction: 'and', }; } else if (extraPropertyType === ExtraPropertyType.text) { return { @@ -106,7 +135,6 @@ export function defaultQuery(key: Key, extraPropertyType?: ExtraPropertyType): C key: 'extraProperties', value: '', operator: 'contains', - conjunction: 'and', }; } } @@ -116,7 +144,6 @@ export function defaultQuery(key: Key, extraPropertyType?: ExtraPropertyType): C key: 'extraProperties', value: 0, operator: 'equals', - conjunction: 'and', }; } else { return { @@ -124,7 +151,6 @@ export function defaultQuery(key: Key, extraPropertyType?: ExtraPropertyType): C key: key, operator: 'greaterThanOrEquals', value: 0, - conjunction: 'and', }; } } @@ -171,6 +197,24 @@ export function fromCriteria(criteria: ClientFileSearchCriteria): [ID, Criteria] return [generateCriteriaId(), query]; } +/** Converts a ClientSearchGroup tree into a Query tree */ +export function queryFromCriteria(criteria: ClientSearchGroup): Query { + return { + id: generateId(), + name: criteria.name, + conjunction: criteria.conjunction, + children: new Map( + criteria.children.map<[ID, Criteria | CriteriaGroup]>((child) => { + if (isClientSearchGroup(child)) { + return [generateGroupId(), queryFromCriteria(child) as CriteriaGroup]; + } else { + return fromCriteria(child); + } + }), + ), + }; +} + //prettier-ignore export function intoCriteria(query: Criteria, tagStore: TagStore): ClientFileSearchCriteria { if (query.key === 'name' || query.key === 'absolutePath' || query.key === 'extension') { @@ -195,3 +239,258 @@ export function intoCriteria(query: Criteria, tagStore: TagStore): ClientFileSea return new ClientTagSearchCriteria(query.id, 'tags'); } } + +export function intoGroup(query: Query, tagStore: TagStore): ClientSearchGroup { + const nodeId = generateId(); + const group = new ClientSearchGroup(nodeId, query.name, query.conjunction, []); + + for (const crit of query.children.values()) { + if (isCriteriaGroup(crit)) { + group.insertNode(nodeId, intoGroup(crit, tagStore)); + } else { + group.insertNode(nodeId, intoCriteria(crit, tagStore)); + } + } + return group; +} + +export type CritPath = string[]; +export type CritIndexPath = number[]; + +export function getPathByIndexPath(query: Query, indexPath: CritIndexPath): CritPath | null { + let children = Array.from(query.children.entries()); + const path: CritPath = []; + for (const index of indexPath) { + if (index < 0 || index >= children.length) { + // if index out of range, add a new group id and return, later when used into updateNode it will get undefined node + // and will decide what to do with if in each updateNode fn argument + path.push(generateGroupId()); + return path; + } + const [groupKey, node] = children[index]; + if (isCriteriaGroup(node)) { + path.push(groupKey); + children = Array.from(node.children.entries()); + } else { + // if is not group, add the id, stop the loop and return + path.push(groupKey); + return path; + } + } + return path; +} + +export function cloneGroup(group: CriteriaGroup): CriteriaGroup { + return { + ...group, + children: new Map(group.children), + }; +} + +export function getNode( + query: Query, + path: CritPath, + limit: number = path.length, +): CriteriaGroup | Criteria | null { + if (path.length === 0) { + return query; + } + const len = path.length; + const normalizedLimit = Math.min(len, Math.max(0, limit < 0 ? len + limit : limit)); + + let current: CriteriaGroup | Criteria | null = query; + for (let i = 0; i < normalizedLimit; i++) { + if (!isCriteriaGroup(current)) { + return null; + } + const next = current.children.get(path[i]); + current = next ?? null; + } + + return current; +} + +/** it search for a node given a path, rebuilding the path's nodes along + * the way and update the target node with the provided updater function */ +export function updateNode( + query: Query, + path: CritPath, + fn: (node: CriteriaNode | undefined) => CriteriaNode | null = (node) => + node ? { ...node } : null, +): Query { + let children = new Map(query.children); + query.children = children; + for (const id of path) { + let node = children.get(id); + if (id === path.at(-1)) { + const updated = fn(node); + if (updated === null) { + // delete node + children.delete(id); + } else { + children.set(id, updated); + } + } else if (node && isCriteriaGroup(node)) { + node = { ...node, children: new Map(node.children) }; + children.set(id, node); + children = node.children; + } + } + if (path.length === 0) { + const updated = fn(query); + return updated && isCriteriaGroup(updated) ? { ...updated } : { ...query }; + } + return { ...query }; +} + +export function deleteNode( + query: Query, + path: CritPath, + deletedCallback?: (deletedNode: CriteriaNode | undefined) => void, +): Query { + const parentPath = path.slice(0, -1); + const targetId = path.at(-1); + return updateNode(query, parentPath, (parent) => { + if (!parent) { + return null; + } + if (targetId && isCriteriaGroup(parent)) { + const newChildren = new Map(parent.children); + deletedCallback?.(newChildren.get(targetId)); + newChildren.delete(targetId); + if (newChildren.size === 0 && parentPath.length > 0) { + // prevent empty groups except for root + return null; + } + return { ...parent, children: newChildren }; + } + return { ...parent }; + }); +} + +export function insertNode( + query: Query, + path: CritPath, + node: CriteriaNode, + nodeId: string, + at?: number, + parentIndex?: number, +): Query { + const toParentId = path.at(-1); + let generatedGroupToInsert: CriteriaGroup | undefined; + query = updateNode(query, path, (parent) => { + if (isCriteriaGroup(parent)) { + // if parent is group insert into it + const newChildren = new Map(parent.children); + const entries = Array.from(newChildren.entries()); + const insertAt = clamp(at ?? entries.length, 0, entries.length); + entries.splice(insertAt, 0, [nodeId as ID, node]); + return { ...parent, children: new Map(entries) }; + } else { + // if parent is crieria insert both into new group + // if parent is null insert into new group + const entries: [string, CriteriaNode][] = parent + ? [[toParentId ?? generateCriteriaId(), parent]] + : []; + entries.splice(at ?? entries.length, 0, [nodeId as ID, node]); + const newGroup: CriteriaGroup = { + id: generateId(), + name: '', + conjunction: 'and', + children: new Map(entries), + }; + generatedGroupToInsert = newGroup; + // return null to delete the previous criteria node + return null; + } + }); + if (!generatedGroupToInsert) { + return query; + } + const newGroupToInsert: CriteriaGroup = { ...generatedGroupToInsert }; + // if new group was created, insert it into the query + query = insertNode(query, path.slice(0, -1), newGroupToInsert, generateGroupId(), parentIndex); + /* + query = updateNode(query, path.slice(0, -1), (parent) => { + if (!parent) { + return null; + } + if (isCriteriaGroup(parent)) { + const newChildren = new Map(parent.children); + newChildren.set(generateGroupId(), newGroupToInsert); + return { ...parent, children: newChildren }; + } + return parent; + });*/ + return query; +} + +export function moveNode( + query: Query, + fromPath: CritPath, + toPath: CritPath, + at?: number, + toParentIndex?: number, // used to preserve index when moving into a criteria node +): Query { + // get and remove from 'from parent' node: + const nodeId = fromPath.at(-1); + let nodeToMove: CriteriaNode | undefined; + query = deleteNode(query, fromPath, (deletedNode) => { + nodeToMove = deletedNode; + }); + if (!nodeToMove) { + return query; + } + const newNodeToMove: CriteriaNode = { ...nodeToMove }; + // insert into 'to parent' node: + query = insertNode( + query, + toPath, + newNodeToMove, + nodeId ?? (isCriteriaGroup(newNodeToMove) ? generateGroupId() : generateCriteriaId()), + at, + toParentIndex, + ); + return query; +} + +export function moveNodeByIndexPath( + query: Query, + fromIndexPath: CritIndexPath, + toIndexPath: CritIndexPath, +): Query { + if (fromIndexPath.length === 0 || toIndexPath.length === 0) { + return query; + } + const fromPath = getPathByIndexPath(query, fromIndexPath); + // ignore last index since it is the 'at' argument + const toPath = getPathByIndexPath(query, toIndexPath.slice(0, -1)); + if (!fromPath || !toPath) { + return query; + } + const at = toIndexPath[toIndexPath.length - 1]; + const parentIndex = toIndexPath[toIndexPath.length - 2]; + return moveNode(query, fromPath, toPath, at, parentIndex); +} + +export function appendCriteriaByIndexPath( + query: Query, + criteria: Criteria, + toIndexPath?: CritIndexPath, +) { + const toPath = toIndexPath ? getPathByIndexPath(query, toIndexPath) : undefined; + const critCompId = generateCriteriaId(); + if (!toPath || !toIndexPath) { + return { ...query, children: new Map(query.children.set(critCompId, criteria)) }; + } else { + const parentIndex = toIndexPath[toIndexPath.length - 1]; + return insertNode(query, toPath, criteria, critCompId, undefined, parentIndex); + } +} + +export function parseIndexPath(pathStr: string): CritIndexPath { + if (pathStr === '') { + return []; + } + return pathStr.split('.').map((i) => parseInt(i, 10)); +} diff --git a/src/frontend/containers/AdvancedSearch/index.tsx b/src/frontend/containers/AdvancedSearch/index.tsx index 17d8be6c..161492ac 100644 --- a/src/frontend/containers/AdvancedSearch/index.tsx +++ b/src/frontend/containers/AdvancedSearch/index.tsx @@ -1,41 +1,35 @@ import { observer } from 'mobx-react-lite'; import React, { useCallback, useRef, useState } from 'react'; -import { ID } from 'src/api/id'; import { useStore } from 'src/frontend/contexts/StoreContext'; import { useAutorun } from 'src/frontend/hooks/mobx'; import { Button, IconSet } from 'widgets'; import { Dialog } from 'widgets/popovers'; import CriteriaBuilder from './CriteriaBuilder'; -import { Criteria, fromCriteria, intoCriteria } from './data'; -import { QueryEditor, QueryMatch } from './QueryEditor'; +import { queryFromCriteria, intoGroup, Query, getemptyQuery } from './data'; +import { QueryEditor } from './QueryEditor'; export const AdvancedSearchDialog = observer(() => { const { uiStore, tagStore } = useStore(); - const [query, setQuery] = useState(new Map()); + const [query, setQuery] = useState(getemptyQuery()); const keySelector = useRef(null); - // Initialize form with current queries. When the form is closed, all inputs // are unmounted to save memory. useAutorun(() => { - const map = new Map(); + let newQuery: Query = getemptyQuery(); if (uiStore.isAdvancedSearchOpen) { - for (const criteria of uiStore.searchCriteriaList) { - const [id, query] = fromCriteria(criteria); - map.set(id, query); - } + newQuery = queryFromCriteria(uiStore.searchRootGroup); requestAnimationFrame(() => requestAnimationFrame(() => keySelector.current?.focus())); } - setQuery(map); + setQuery(newQuery); }); const search = useCallback(() => { - uiStore.replaceSearchCriterias( - Array.from(query.values(), (vals) => intoCriteria(vals, tagStore)), - ); + //uiStore.replaceSearchRootConjuction(rootConjunction); + uiStore.replaceSearchCriterias(intoGroup(query, tagStore)); uiStore.closeAdvancedSearch(); }, [query, tagStore, uiStore]); - const reset = useRef(() => setQuery(new Map())).current; + const reset = useRef(() => setQuery(getemptyQuery())).current; return ( { text="Search" icon={IconSet.SEARCH} onClick={search} - disabled={query.size === 0} + disabled={query.children.size === 0} /> diff --git a/src/frontend/containers/AppToolbar/Searchbar.tsx b/src/frontend/containers/AppToolbar/Searchbar.tsx index d6d0d2b7..a84b8704 100644 --- a/src/frontend/containers/AppToolbar/Searchbar.tsx +++ b/src/frontend/containers/AppToolbar/Searchbar.tsx @@ -6,7 +6,7 @@ const SEARCHBAR_ID = 'toolbar-searchbar'; const Searchbar = observer(() => { const { uiStore } = useStore(); - const searchCriteriaList = uiStore.searchCriteriaList; + const searchCriteriaList = uiStore.searchRootGroup.children; // Only show quick search bar when all criteria are tags, // otherwise show a search bar that opens to the advanced search form @@ -16,6 +16,7 @@ const Searchbar = observer(() => { searchCriteriaList.length === 0 || searchCriteriaList.every( (crit) => + crit instanceof ClientTagSearchCriteria && crit.key === 'tags' && crit.operator === 'containsRecursively' && (crit as ClientTagSearchCriteria).value, @@ -128,7 +129,7 @@ const QuickSearchList = observer(() => { onSelect={handleSelect} onDeselect={handleDeselect} onTagClick={uiStore.toggleAdvancedSearch} - onClear={uiStore.clearSearchCriteriaList} + onClear={uiStore.clearSearchCriteriaTree} ignoreOnBlur={ingnoreOnBlur} renderCreateOption={renderCreateOption} extraIconButtons={} @@ -263,41 +264,35 @@ const CriteriaList = observer(() => {
    - {uiStore.searchCriteriaList.map((c, i) => ( + {uiStore.searchRootGroup.getLabels(CustomKeyDict, rootStore).map((label) => ( uiStore.removeSearchCriteriaByIndex(i)} + key={`${label.id}`} + text={label.label} + onRemove={() => uiStore.removeSearchCriteriaById(label.id)} // Italicize system tags (for now only "Untagged images") - className={ - c instanceof ClientTagSearchCriteria && c.isSystemTag() ? 'italic' : undefined - } + className={label.isSystemTag ? 'italic' : undefined} /> ))}
    - {uiStore.searchCriteriaList.length > 1 ? ( - { - uiStore.toggleSearchMatchAny(); - fileStore.refetch(); - e.stopPropagation(); - e.preventDefault(); - // TODO: search input element keeps focus after click??? - }} - className="btn-icon-large" - /> - ) : ( - <> - )} + { + uiStore.toggleSearchMatchAny(); + fileStore.refetch(); + e.stopPropagation(); + e.preventDefault(); + // TODO: search input element keeps focus after click??? + }} + className="btn-icon-large" + /> { - uiStore.clearSearchCriteriaList(); + uiStore.clearSearchCriteriaTree(); e.stopPropagation(); e.preventDefault(); }} diff --git a/src/frontend/containers/Outliner/LocationsPanel/index.tsx b/src/frontend/containers/Outliner/LocationsPanel/index.tsx index 7db12f85..b8660bb6 100644 --- a/src/frontend/containers/Outliner/LocationsPanel/index.tsx +++ b/src/frontend/containers/Outliner/LocationsPanel/index.tsx @@ -17,7 +17,10 @@ import DropContext from '../../../contexts/DropContext'; import { useStore } from '../../../contexts/StoreContext'; import { DnDLocationType, useLocationDnD } from '../../../contexts/TagDnDContext'; import { ClientLocation, ClientSubLocation } from '../../../entities/Location'; -import { ClientStringSearchCriteria } from '../../../entities/SearchCriteria'; +import { + ClientStringSearchCriteria, + ClientTagSearchCriteria, +} from '../../../entities/SearchCriteria'; import { useAutorun } from '../../../hooks/mobx'; import LocationStore from '../../../stores/LocationStore'; import { IExpansionState } from '../../types'; @@ -248,7 +251,7 @@ const SubLocation = observer((props: { nodeData: ClientSubLocation; treeData: IT const handleClick = useCallback( (event: React.MouseEvent) => { existingSearchCrit // toggle search - ? uiStore.removeSearchCriteria(existingSearchCrit) + ? uiStore.removeSearchCriteria(existingSearchCrit as ClientTagSearchCriteria) : event.ctrlKey // otherwise add/replace depending on ctrl ? uiStore.addSearchCriteria(pathCriteria(nodeData.path)) : uiStore.replaceSearchCriteria(pathCriteria(nodeData.path)); @@ -319,7 +322,7 @@ const Location = observer( const handleClick = useCallback( (event: React.MouseEvent) => { existingSearchCrit // toggle search - ? uiStore.removeSearchCriteria(existingSearchCrit) + ? uiStore.removeSearchCriteria(existingSearchCrit as ClientTagSearchCriteria) : event.ctrlKey ? uiStore.addSearchCriteria(pathCriteria(nodeData.path)) : uiStore.replaceSearchCriteria(pathCriteria(nodeData.path)); diff --git a/src/frontend/containers/Outliner/SavedSearchesPanel/index.tsx b/src/frontend/containers/Outliner/SavedSearchesPanel/index.tsx index 969cf7df..3fca315b 100644 --- a/src/frontend/containers/Outliner/SavedSearchesPanel/index.tsx +++ b/src/frontend/containers/Outliner/SavedSearchesPanel/index.tsx @@ -13,7 +13,11 @@ import { SavedSearchRemoval } from '../../../components/RemovalAlert'; import { useStore } from '../../../contexts/StoreContext'; import { DnDSearchType, SearchDnDProvider, useSearchDnD } from '../../../contexts/TagDnDContext'; import { ClientFileSearchCriteria, CustomKeyDict } from '../../../entities/SearchCriteria'; -import { ClientFileSearchItem } from '../../../entities/SearchItem'; +import { + ClientFileSearchItem, + ClientSearchGroup, + isClientSearchGroup, +} from '../../../entities/SearchItem'; import { useAutorun } from '../../../hooks/mobx'; import SearchItemDialog from '../../AdvancedSearch/SearchItemDialog'; import { IExpansionState } from '../../types'; @@ -44,7 +48,7 @@ const isExpanded = (nodeData: ClientFileSearchItem, treeData: ITreeData) => !!treeData.expansion[nodeData.id]; const customKeys = ( - search: (crits: ClientFileSearchCriteria[], searchMatchAny: boolean) => void, + search: (crits: ClientFileSearchCriteria[] | ClientSearchGroup, searchMatchAny: boolean) => void, event: React.KeyboardEvent, nodeData: ClientFileSearchItem | ClientFileSearchCriteria, treeData: ITreeData, @@ -59,7 +63,7 @@ const customKeys = ( case 'Enter': event.stopPropagation(); if (nodeData instanceof ClientFileSearchItem) { - search(nodeData.criteria, nodeData.matchAny); + search(nodeData.rootGroup, nodeData.rootGroup.conjunction === 'or'); } else { // TODO: ctrl/shift adds onto search search([nodeData], false); @@ -82,25 +86,29 @@ const customKeys = ( } }; -const SearchCriteriaLabel = ({ nodeData, treeData }: { nodeData: any; treeData: any }) => ( - +const SearchItemNodeLabel = ({ nodeData, treeData }: { nodeData: any; treeData: any }) => ( + ); const SearchItemLabel = ({ nodeData, treeData }: { nodeData: any; treeData: any }) => ( ); +const mapNode = (node: ClientFileSearchCriteria | ClientSearchGroup): ITreeItem => { + return { + id: `${node.id}`, + nodeData: node, + label: SearchItemNodeLabel, + children: isClientSearchGroup(node) ? node.children.map((ch) => mapNode(ch)) : [], + isExpanded, + }; +}; + const mapItem = (item: ClientFileSearchItem): ITreeItem => ({ id: item.id, label: SearchItemLabel, nodeData: item, - children: item.criteria.map((c, i) => ({ - id: `${item.id}-${i}`, - nodeData: c, - label: SearchCriteriaLabel, - children: [], - isExpanded: () => false, - })), + children: item.rootGroup.children.map((ch) => mapNode(ch)), isExpanded, }); @@ -158,16 +166,13 @@ const SearchItem = observer( (e: React.MouseEvent) => { runInAction(() => { if (!e.ctrlKey) { - uiStore.replaceSearchCriterias(nodeData.criteria.toJSON()); - if (uiStore.searchMatchAny !== nodeData.matchAny) { - uiStore.toggleSearchMatchAny(); - } + uiStore.replaceSearchCriterias(nodeData.rootGroup); } else { - uiStore.toggleSearchCriterias(nodeData.criteria.toJSON()); + uiStore.toggleSearchCriterias(nodeData.rootGroup); } }); }, - [nodeData.criteria, nodeData.matchAny, uiStore], + [nodeData.rootGroup, uiStore], ); const handleEdit = useCallback( @@ -235,7 +240,7 @@ const SearchItem = observer( onDrop={handleDrop} > {/* {IconSet.SEARCH} */} - {nodeData.matchAny ? IconSet.SEARCH_ANY : IconSet.SEARCH_ALL} + {nodeData.rootGroup.conjunction === 'or' ? IconSet.SEARCH_ANY : IconSet.SEARCH_ALL}
    {nodeData.name}