From 6d1dc0ccd3ae3546bc414d26e04e21629cf102e2 Mon Sep 17 00:00:00 2001 From: vipzero Date: Thu, 28 Apr 2022 08:59:52 +0900 Subject: [PATCH 001/105] Update readme.md --- readme.md | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/readme.md b/readme.md index 12916c0e..a3d553f1 100644 --- a/readme.md +++ b/readme.md @@ -47,7 +47,25 @@ iTunes API (認証なし) - musixmatch: アートワーク・アルバム名・歌詞の出だし(有料 API では FULL) -## 検索文字生成部分や調整しているアルゴリズムなど -https://github.com/vipzero/haikei-server/wiki/algo +調整・アルゴリズム関係 +## 画像検索文字列の生成部分 + +方針 + +- 関連性のあるものが出るように +- meme やキャプ画像が出るように +- 平凡な公式タイトル画像以外が出るように + +コード + +https://github.com/vipzero/haikei-server/blob/main/src/utils/makeSearchWord.ts + +## 画像選択 +imagemin などで Optimize 後のメタデータで +ソートして上から3つ + +コード + +https://github.com/vipzero/haikei-server/blob/main/src/imageIo/uploadManage.ts From 01767548367ca474387e236c77d76ecec7e52ac0 Mon Sep 17 00:00:00 2001 From: vipzero Date: Thu, 28 Apr 2022 18:32:48 +0900 Subject: [PATCH 002/105] Create LICENSE --- LICENSE | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..095e0d97 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 vipzero + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From c5b4e80d95a4de1fc49e974a671a199392d0bf7b Mon Sep 17 00:00:00 2001 From: vipzero Date: Thu, 28 Apr 2022 19:19:39 +0900 Subject: [PATCH 003/105] update sample --- .envrc.sample | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.envrc.sample b/.envrc.sample index 03566970..34323436 100644 --- a/.envrc.sample +++ b/.envrc.sample @@ -17,8 +17,8 @@ export SPOTIFY_CLIENT_ID="" export SPOTIFY_CLIENT_SECRET="" export MUSIXMATCH_API_KEY="" -export EVENT_ID="" +export EVENT_ID="todo" export STRAGE_URL="" -export THREAD_TITLE_WORD="" +export THREAD_TITLE_WORD="ならアニソン" export DIRECT_MODE=0 \ No newline at end of file From 5482df4271424d2975af2d894191defcc346c4e7 Mon Sep 17 00:00:00 2001 From: vipzero Date: Thu, 28 Apr 2022 20:25:20 +0900 Subject: [PATCH 004/105] refactor --- package.json | 6 +- src/index.ts | 24 +++--- src/state/store.ts | 5 +- yarn.lock | 190 +++++++++++++++++++++++++-------------------- 4 files changed, 125 insertions(+), 100 deletions(-) diff --git a/package.json b/package.json index 9e015807..bfbfd1e1 100644 --- a/package.json +++ b/package.json @@ -41,9 +41,9 @@ "@types/imagemin-mozjpeg": "8.0.1", "@types/node": "17.0.24", "@types/sharp": "0.28.5", - "@typescript-eslint/eslint-plugin": "4.29.1", - "@typescript-eslint/parser": "4.29.1", - "eslint": "8.13.0", + "@typescript-eslint/eslint-plugin": "5.21.0", + "@typescript-eslint/parser": "5.21.0", + "eslint": "8.14.0", "firebase-admin": "9.11.0", "imagemin": "7.0.1", "imagemin-gifsicle": "7.0.0", diff --git a/src/index.ts b/src/index.ts index e067caa0..1fc826d3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -14,7 +14,7 @@ import { import { getAlbum } from './service/itunes' import { getLyricsSafe } from './service/jlyricnet' // import { spotifySearchSongInfo } from './spotify' -import { store } from './state/store' +import { Store, store } from './state/store' import { Song } from './types/index' import { sleep } from './utils' import { anaCounts } from './utils/wordCounts' @@ -36,11 +36,14 @@ async function prepareImages(q: string) { return uploads.map((u) => u.downloadUrl) } -async function receiveIcy(icy: string) { - const time = Date.now() +export async function icyToSong( + icy: string, + time: number, + store: Store +): Promise { info(icy) - if (store.isDuplicate(icy)) return // 起動時の重複登録を防ぐ + if (store.isDuplicate(icy)) return false // 起動時の重複登録を防ぐ addHistoryNow(icy) @@ -54,12 +57,8 @@ async function receiveIcy(icy: string) { const imageSearchWord = makeSearchQuery(song) const imageLinksSync = prepareImages(imageSearchWord) - // const spoinfo = spotifySearchSongInfo(song.title, song.artist) - // if (spoinfo) song.artwork = spoinfo.artwork - const albumInfosSync = getAlbum(icy) const lyricsSync = getLyricsSafe(song.title, song.artist) - // const lyric = lyrics ? lyrics.lyric : null const [imageLinks, albumInfos, { creators }] = await Promise.all([ imageLinksSync, @@ -75,9 +74,14 @@ async function receiveIcy(icy: string) { time, imageSearchWord, } + return compSong +} - songPrint(compSong) - saveMusic(compSong) +async function receiveIcy(icy: string) { + const song = await icyToSong(icy, Date.now(), store) + if (!song) return + songPrint(song) + saveMusic(song) } async function main() { diff --git a/src/state/store.ts b/src/state/store.ts index 7f383521..9960c6ee 100644 --- a/src/state/store.ts +++ b/src/state/store.ts @@ -1,7 +1,6 @@ -import { UploadFile } from './../types/index' -import { Counts } from '../types/index' +import { Counts, UploadFile } from '../types' -class Store { +export class Store { counts: Counts isFirst: boolean firstIcy: string diff --git a/yarn.lock b/yarn.lock index 1b100086..f1cf53a7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -501,10 +501,10 @@ dependencies: "@cspotcode/source-map-consumer" "0.8.0" -"@eslint/eslintrc@^1.2.1": - version "1.2.1" - resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-1.2.1.tgz#8b5e1c49f4077235516bc9ec7d41378c0f69b8c6" - integrity sha512-bxvbYnBPN1Gibwyp6NrpnFzA3YtRL3BBAyEAFVIpNTm2Rn4Vy87GA5M4aSn3InRrlsbX5N0GW7XIx+U4SAEKdQ== +"@eslint/eslintrc@^1.2.2": + version "1.2.2" + resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-1.2.2.tgz#4989b9e8c0216747ee7cca314ae73791bb281aae" + integrity sha512-lTVWHs7O2hjBFZunXTZYnYqtB9GakA1lnxIf+gKq2nY5gxkkNi/lQvveW6t8gFdOHTg6nG50Xs95PrLqVpcaLg== dependencies: ajv "^6.12.4" debug "^4.3.2" @@ -1280,10 +1280,10 @@ jest-matcher-utils "^27.0.0" pretty-format "^27.0.0" -"@types/json-schema@^7.0.7": - version "7.0.9" - resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.9.tgz#97edc9037ea0c38585320b28964dde3b39e4660d" - integrity sha512-qcUXuemtEu+E5wZSJHNxUXeCZhAfXKQ41D+duX+VYPde7xyEVZci+/oXKJL13tnRs9lR2pr4fod59GT6/X1/yQ== +"@types/json-schema@^7.0.9": + version "7.0.11" + resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.11.tgz#d421b6c527a3037f7c84433fd2c4229e016863d3" + integrity sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ== "@types/keyv@*": version "3.1.2" @@ -1381,74 +1381,85 @@ dependencies: "@types/yargs-parser" "*" -"@typescript-eslint/eslint-plugin@4.29.1": - version "4.29.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-4.29.1.tgz#808d206e2278e809292b5de752a91105da85860b" - integrity sha512-AHqIU+SqZZgBEiWOrtN94ldR3ZUABV5dUG94j8Nms9rQnHFc8fvDOue/58K4CFz6r8OtDDc35Pw9NQPWo0Ayrw== +"@typescript-eslint/eslint-plugin@5.21.0": + version "5.21.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.21.0.tgz#bfc22e0191e6404ab1192973b3b4ea0461c1e878" + integrity sha512-fTU85q8v5ZLpoZEyn/u1S2qrFOhi33Edo2CZ0+q1gDaWWm0JuPh3bgOyU8lM0edIEYgKLDkPFiZX2MOupgjlyg== dependencies: - "@typescript-eslint/experimental-utils" "4.29.1" - "@typescript-eslint/scope-manager" "4.29.1" - debug "^4.3.1" + "@typescript-eslint/scope-manager" "5.21.0" + "@typescript-eslint/type-utils" "5.21.0" + "@typescript-eslint/utils" "5.21.0" + debug "^4.3.2" functional-red-black-tree "^1.0.1" - regexpp "^3.1.0" + ignore "^5.1.8" + regexpp "^3.2.0" semver "^7.3.5" tsutils "^3.21.0" -"@typescript-eslint/experimental-utils@4.29.1": - version "4.29.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/experimental-utils/-/experimental-utils-4.29.1.tgz#0af2b17b0296b60c6b207f11062119fa9c5a8994" - integrity sha512-kl6QG6qpzZthfd2bzPNSJB2YcZpNOrP6r9jueXupcZHnL74WiuSjaft7WSu17J9+ae9zTlk0KJMXPUj0daBxMw== +"@typescript-eslint/parser@5.21.0": + version "5.21.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-5.21.0.tgz#6cb72673dbf3e1905b9c432175a3c86cdaf2071f" + integrity sha512-8RUwTO77hstXUr3pZoWZbRQUxXcSXafZ8/5gpnQCfXvgmP9gpNlRGlWzvfbEQ14TLjmtU8eGnONkff8U2ui2Eg== dependencies: - "@types/json-schema" "^7.0.7" - "@typescript-eslint/scope-manager" "4.29.1" - "@typescript-eslint/types" "4.29.1" - "@typescript-eslint/typescript-estree" "4.29.1" - eslint-scope "^5.1.1" - eslint-utils "^3.0.0" + "@typescript-eslint/scope-manager" "5.21.0" + "@typescript-eslint/types" "5.21.0" + "@typescript-eslint/typescript-estree" "5.21.0" + debug "^4.3.2" -"@typescript-eslint/parser@4.29.1": - version "4.29.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-4.29.1.tgz#17dfbb45c9032ffa0fe15881d20fbc2a4bdeb02d" - integrity sha512-3fL5iN20hzX3Q4OkG7QEPFjZV2qsVGiDhEwwh+EkmE/w7oteiOvUNzmpu5eSwGJX/anCryONltJ3WDmAzAoCMg== - dependencies: - "@typescript-eslint/scope-manager" "4.29.1" - "@typescript-eslint/types" "4.29.1" - "@typescript-eslint/typescript-estree" "4.29.1" - debug "^4.3.1" - -"@typescript-eslint/scope-manager@4.29.1": - version "4.29.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-4.29.1.tgz#f25da25bc6512812efa2ce5ebd36619d68e61358" - integrity sha512-Hzv/uZOa9zrD/W5mftZa54Jd5Fed3tL6b4HeaOpwVSabJK8CJ+2MkDasnX/XK4rqP5ZTWngK1ZDeCi6EnxPQ7A== - dependencies: - "@typescript-eslint/types" "4.29.1" - "@typescript-eslint/visitor-keys" "4.29.1" - -"@typescript-eslint/types@4.29.1": - version "4.29.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-4.29.1.tgz#94cce6cf7cc83451df03339cda99d326be2feaf5" - integrity sha512-Jj2yu78IRfw4nlaLtKjVaGaxh/6FhofmQ/j8v3NXmAiKafbIqtAPnKYrf0sbGjKdj0hS316J8WhnGnErbJ4RCA== - -"@typescript-eslint/typescript-estree@4.29.1": - version "4.29.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-4.29.1.tgz#7b32a25ff8e51f2671ccc6b26cdbee3b1e6c5e7f" - integrity sha512-lIkkrR9E4lwZkzPiRDNq0xdC3f2iVCUjw/7WPJ4S2Sl6C3nRWkeE1YXCQ0+KsiaQRbpY16jNaokdWnm9aUIsfw== - dependencies: - "@typescript-eslint/types" "4.29.1" - "@typescript-eslint/visitor-keys" "4.29.1" - debug "^4.3.1" - globby "^11.0.3" - is-glob "^4.0.1" +"@typescript-eslint/scope-manager@5.21.0": + version "5.21.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-5.21.0.tgz#a4b7ed1618f09f95e3d17d1c0ff7a341dac7862e" + integrity sha512-XTX0g0IhvzcH/e3393SvjRCfYQxgxtYzL3UREteUneo72EFlt7UNoiYnikUtmGVobTbhUDByhJ4xRBNe+34kOQ== + dependencies: + "@typescript-eslint/types" "5.21.0" + "@typescript-eslint/visitor-keys" "5.21.0" + +"@typescript-eslint/type-utils@5.21.0": + version "5.21.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/type-utils/-/type-utils-5.21.0.tgz#ff89668786ad596d904c21b215e5285da1b6262e" + integrity sha512-MxmLZj0tkGlkcZCSE17ORaHl8Th3JQwBzyXL/uvC6sNmu128LsgjTX0NIzy+wdH2J7Pd02GN8FaoudJntFvSOw== + dependencies: + "@typescript-eslint/utils" "5.21.0" + debug "^4.3.2" + tsutils "^3.21.0" + +"@typescript-eslint/types@5.21.0": + version "5.21.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-5.21.0.tgz#8cdb9253c0dfce3f2ab655b9d36c03f72e684017" + integrity sha512-XnOOo5Wc2cBlq8Lh5WNvAgHzpjnEzxn4CJBwGkcau7b/tZ556qrWXQz4DJyChYg8JZAD06kczrdgFPpEQZfDsA== + +"@typescript-eslint/typescript-estree@5.21.0": + version "5.21.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-5.21.0.tgz#9f0c233e28be2540eaed3df050f0d54fb5aa52de" + integrity sha512-Y8Y2T2FNvm08qlcoSMoNchh9y2Uj3QmjtwNMdRQkcFG7Muz//wfJBGBxh8R7HAGQFpgYpdHqUpEoPQk+q9Kjfg== + dependencies: + "@typescript-eslint/types" "5.21.0" + "@typescript-eslint/visitor-keys" "5.21.0" + debug "^4.3.2" + globby "^11.0.4" + is-glob "^4.0.3" semver "^7.3.5" tsutils "^3.21.0" -"@typescript-eslint/visitor-keys@4.29.1": - version "4.29.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-4.29.1.tgz#0615be8b55721f5e854f3ee99f1a714f2d093e5d" - integrity sha512-zLqtjMoXvgdZY/PG6gqA73V8BjqPs4af1v2kiiETBObp+uC6gRYnJLmJHxC0QyUrrHDLJPIWNYxoBV3wbcRlag== +"@typescript-eslint/utils@5.21.0": + version "5.21.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/utils/-/utils-5.21.0.tgz#51d7886a6f0575e23706e5548c7e87bce42d7c18" + integrity sha512-q/emogbND9wry7zxy7VYri+7ydawo2HDZhRZ5k6yggIvXa7PvBbAAZ4PFH/oZLem72ezC4Pr63rJvDK/sTlL8Q== dependencies: - "@typescript-eslint/types" "4.29.1" - eslint-visitor-keys "^2.0.0" + "@types/json-schema" "^7.0.9" + "@typescript-eslint/scope-manager" "5.21.0" + "@typescript-eslint/types" "5.21.0" + "@typescript-eslint/typescript-estree" "5.21.0" + eslint-scope "^5.1.1" + eslint-utils "^3.0.0" + +"@typescript-eslint/visitor-keys@5.21.0": + version "5.21.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-5.21.0.tgz#453fb3662409abaf2f8b1f65d515699c888dd8ae" + integrity sha512-SX8jNN+iHqAF0riZQMkm7e8+POXa/fXw5cxL+gjpyP+FI+JVNhii53EmQgDAfDcBpFekYSlO0fGytMQwRiMQCA== + dependencies: + "@typescript-eslint/types" "5.21.0" + eslint-visitor-keys "^3.0.0" abab@^2.0.3, abab@^2.0.5: version "2.0.5" @@ -2379,7 +2390,7 @@ debug@2: dependencies: ms "2.0.0" -debug@4, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2: +debug@4, debug@^4.1.0, debug@^4.1.1, debug@^4.3.2: version "4.3.2" resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.2.tgz#f0a49c18ac8779e31d4a0c6029dfb76873c7428b" integrity sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw== @@ -2777,17 +2788,17 @@ eslint-visitor-keys@^2.0.0: resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-2.1.0.tgz#f65328259305927392c938ed44eb0a5c9b2bd303" integrity sha512-0rSmRBzXgDzIsD6mGdJgevzgezI534Cer5L/vyMX0kHzT/jiB43jRhd9YUlMGYLQy2zprNmoT8qasCGtY+QaKw== -eslint-visitor-keys@^3.3.0: +eslint-visitor-keys@^3.0.0, eslint-visitor-keys@^3.3.0: version "3.3.0" resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-3.3.0.tgz#f6480fa6b1f30efe2d1968aa8ac745b862469826" integrity sha512-mQ+suqKJVyeuwGYHAdjMFqjCyfl8+Ldnxuyp3ldiMBFKkvytrXUZWaiPCEav8qDHKty44bD+qV1IP4T+w+xXRA== -eslint@8.13.0: - version "8.13.0" - resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.13.0.tgz#6fcea43b6811e655410f5626cfcf328016badcd7" - integrity sha512-D+Xei61eInqauAyTJ6C0q6x9mx7kTUC1KZ0m0LSEexR0V+e94K12LmWX076ZIsldwfQ2RONdaJe0re0TRGQbRQ== +eslint@8.14.0: + version "8.14.0" + resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.14.0.tgz#62741f159d9eb4a79695b28ec4989fcdec623239" + integrity sha512-3/CE4aJX7LNEiE3i6FeodHmI/38GZtWCsAtsymScmzYapx8q1nVVb+eLcLSzATmCPXw5pT4TqVs1E0OmxAd9tw== dependencies: - "@eslint/eslintrc" "^1.2.1" + "@eslint/eslintrc" "^1.2.2" "@humanwhocodes/config-array" "^0.9.2" ajv "^6.10.0" chalk "^4.0.0" @@ -2989,7 +3000,7 @@ fast-deep-equal@^3.1.1, fast-deep-equal@^3.1.3: resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz#3a7d56b559d6cbc3eb512325244e619a65c6c525" integrity sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q== -fast-glob@^3.0.3, fast-glob@^3.1.1: +fast-glob@^3.0.3: version "3.2.7" resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.2.7.tgz#fd6cb7a2d7e9aa7a7846111e85a196d6b2f766a1" integrity sha512-rYGMRwip6lUMvYD3BTScMwT1HtAs2d71SMv66Vrxs0IekGZEjhM0pcMfjQPnknBt2zeCwQMEupiN02ZP4DiT1Q== @@ -3000,6 +3011,17 @@ fast-glob@^3.0.3, fast-glob@^3.1.1: merge2 "^1.3.0" micromatch "^4.0.4" +fast-glob@^3.2.9: + version "3.2.11" + resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.2.11.tgz#a1172ad95ceb8a16e20caa5c5e56480e5129c1d9" + integrity sha512-xrO3+1bxSo3ZVHAnqzyuewYT6aMFHRAd4Kcs92MAonjwQZLsK9d0SF1IyQ3k5PoirxTW0Oe/RqFgMQ6TcNE5Ew== + dependencies: + "@nodelib/fs.stat" "^2.0.2" + "@nodelib/fs.walk" "^1.2.3" + glob-parent "^5.1.2" + merge2 "^1.3.0" + micromatch "^4.0.4" + fast-json-stable-stringify@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz#874bf69c6f404c2b5d99c481341399fd55892633" @@ -3420,16 +3442,16 @@ globby@^10.0.0: merge2 "^1.2.3" slash "^3.0.0" -globby@^11.0.3: - version "11.0.4" - resolved "https://registry.yarnpkg.com/globby/-/globby-11.0.4.tgz#2cbaff77c2f2a62e71e9b2813a67b97a3a3001a5" - integrity sha512-9O4MVG9ioZJ08ffbcyVYyLOJLk5JQ688pJ4eMGLpdWLHq/Wr1D9BlriLQyL0E+jbkuePVZXYFj47QM/v093wHg== +globby@^11.0.4: + version "11.1.0" + resolved "https://registry.yarnpkg.com/globby/-/globby-11.1.0.tgz#bd4be98bb042f83d796f7e3811991fbe82a0d34b" + integrity sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g== dependencies: array-union "^2.1.0" dir-glob "^3.0.1" - fast-glob "^3.1.1" - ignore "^5.1.4" - merge2 "^1.3.0" + fast-glob "^3.2.9" + ignore "^5.2.0" + merge2 "^1.4.1" slash "^3.0.0" google-auth-library@^7.0.0, google-auth-library@^7.0.2, google-auth-library@^7.3.0: @@ -3742,12 +3764,12 @@ ieee754@^1.1.13, ieee754@^1.2.1: resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA== -ignore@^5.1.1, ignore@^5.1.4: +ignore@^5.1.1: version "5.1.8" resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.1.8.tgz#f150a8b50a34289b33e22f5889abd4d8016f0e57" integrity sha512-BMpfD7PpiETpBl/A6S498BaIJ6Y/ABT93ETbby2fP00v4EbvPBXWEoaR1UBPKs3iR53pJY7EtZk5KACI57i1Uw== -ignore@^5.2.0: +ignore@^5.1.8, ignore@^5.2.0: version "5.2.0" resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.2.0.tgz#6d3bac8fa7fe0d45d9f9be7bac2fc279577e345a" integrity sha512-CmxgYGiEPCLhfLnpPp1MoRmifwEIOgjcHXxOBjv7mY96c+eWScsOP9c112ZyLdWHi0FxHjI+4uVhKYp/gcdRmQ== @@ -4969,7 +4991,7 @@ merge-stream@^2.0.0: resolved "https://registry.yarnpkg.com/merge-stream/-/merge-stream-2.0.0.tgz#52823629a14dd00c9770fb6ad47dc6310f2c1f60" integrity sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w== -merge2@^1.2.3, merge2@^1.3.0: +merge2@^1.2.3, merge2@^1.3.0, merge2@^1.4.1: version "1.4.1" resolved "https://registry.yarnpkg.com/merge2/-/merge2-1.4.1.tgz#4368892f885e907455a6fd7dc55c0c9d404990ae" integrity sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg== @@ -5798,7 +5820,7 @@ redent@^3.0.0: indent-string "^4.0.0" strip-indent "^3.0.0" -regexpp@^3.1.0, regexpp@^3.2.0: +regexpp@^3.2.0: version "3.2.0" resolved "https://registry.yarnpkg.com/regexpp/-/regexpp-3.2.0.tgz#0425a2768d8f23bad70ca4b90461fa2f1213e1b2" integrity sha512-pq2bWo9mVD43nbts2wGv17XLiNLya+GklZ8kaDLV2Z08gDCsGpnKn9BFMepvWuHCbyVvY7J5o5+BVvoQbmlJLg== From 11af8694482685ccfb1f4987b4d2d5b0e769e4f2 Mon Sep 17 00:00:00 2001 From: vipzero Date: Fri, 29 Apr 2022 14:26:20 +0900 Subject: [PATCH 005/105] add alogrithm description --- src/utils/makeSearchWord.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/utils/makeSearchWord.ts b/src/utils/makeSearchWord.ts index ef1a1a24..4d4223ab 100644 --- a/src/utils/makeSearchWord.ts +++ b/src/utils/makeSearchWord.ts @@ -2,10 +2,16 @@ import { getSyncConf } from './syncConf' import { SongSeed } from '../types/index' import { pickCharaIcy } from '.' +/* anison DB から曲情報を取れた場合アニメタイトル、カテゴリ(ゲーム,アニメ,SF,映画)などを使用できる */ +/* アニメカテゴリ場合は↓を検索クエリに追加する */ const animeExt = ['アニメ', 'meme', 'キャプ画', 'キャラ', 'かわいい'].join( ' OR ' ) -const gameExt = ['ゲーム'].join(' OR ') // 実験 + +/* ゲームカテゴリ場合は↓を検索クエリに追加する */ +const gameExt = ['ゲーム'].join(' OR ') + +/* アニメ情報が取れなかった曲は icy に↓を検索クエリに追加する */ const icyOpt = `(名シーン OR キャラ) (キャプ画像 OR 壁紙) かわいい` export function makeSearchQuery(song: SongSeed): string { @@ -21,7 +27,6 @@ export function makeSearchQuery(song: SongSeed): string { if (!category) return animeTitle if (category.includes('アニメ')) { - // 実験 return `${animeTitle} AND (${animeExt})` } if (category.includes('ゲーム')) { From 484f1f5257d65565836c269440dc1618f625b404 Mon Sep 17 00:00:00 2001 From: vipzero Date: Sun, 1 May 2022 00:28:47 +0900 Subject: [PATCH 006/105] add archive script --- package.json | 2 ++ scripts/archiveEvent/index.ts | 37 +++++++++++++++++++++++ src/service/firebase.ts | 55 ++++++++++++++++++++++++++++++++--- src/types/index.ts | 1 + yarn.lock | 5 ++++ 5 files changed, 96 insertions(+), 4 deletions(-) create mode 100644 scripts/archiveEvent/index.ts diff --git a/package.json b/package.json index bfbfd1e1..05ac1a06 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "chch": "2.8.0", "cheerio": "1.0.0-rc.10", "csv-parse": "4.16.0", + "csv-stringify": "^6.0.5", "date-fns": "2.23.0", "file-type": "16.5.3", "got": "11.8.2", @@ -23,6 +24,7 @@ "setup": "ts-node scripts/setupEvent.ts", "setup:ee": "ts-node scripts/setupEasterEgg.ts", "setup:anison": "./scripts/updateAnisonDb.mjs", + "close:archive": "ts-node ./scripts/archiveEvent/index.ts", "import2": "ts-node scripts/batch/importHistoryFromCells.ts", "map:watch": "./scripts/hourly-yarn-map.sh", "map": "yarn map-load-thread && yarn map-save", diff --git a/scripts/archiveEvent/index.ts b/scripts/archiveEvent/index.ts new file mode 100644 index 00000000..8839ebf5 --- /dev/null +++ b/scripts/archiveEvent/index.ts @@ -0,0 +1,37 @@ +import { assert } from 'console' +import { stringify } from 'csv-stringify/sync' +import { writeFile } from 'fs/promises' + +import { + archiveUrl, + loadHistEventSongs, + uploadStorageArchive, +} from '../../src/service/firebase' +import { log } from '../../src/utils/logger' + +const archiveEventCsv = async (id: string) => { + log({ id }) + const data = await loadHistEventSongs(id) + const dataWithN = data.map((d) => ({ ...d, n: d.n || 0 })) + const csvText = stringify(dataWithN, { + header: true, + columns: ['time', 'title', 'n'].map((key) => ({ key, header: key })), + }) + + await saveFile(id, csvText) +} + +const saveFile = async (id: string, text: string) => { + const paths = archiveUrl(id) + const { localFile } = paths + + await writeFile(localFile, text) + await uploadStorageArchive(paths) + log({ paths }) +} + +const eventId = process.argv[2] + +assert(eventId, 'need arg [event_id]') + +archiveEventCsv(eventId).then(() => log('doae')) diff --git a/src/service/firebase.ts b/src/service/firebase.ts index 76f2ad7e..2257a8d7 100644 --- a/src/service/firebase.ts +++ b/src/service/firebase.ts @@ -1,11 +1,22 @@ import admin from 'firebase-admin' -import { Count, Counts, HistTop, Song } from '../types/index' -import { error, info, log, warn } from '../utils/logger' +import { Count, Counts, HistoryRaw, HistTop, Song } from '../types/index' import { chunk } from '../utils' +import { error, info, log, warn } from '../utils/logger' import { CacheFile } from './../types/index' export { admin } +// const P_SONG = 'song' +// const P_FEEDBACK = 'feedback' +// const P_VOTE = 'vote' +// const P_BOOKS = 'books' +// const P_TABLE = 'table' +// const P_COUNTS = 'counts' +// const P_CVOTE = 'cvote' + +const P_SONGS = 'songs' +const P_HIST = 'hist' + const { SERVICE_ACCOUNT_FILE_PATH, EVENT_ID } = process.env if (!SERVICE_ACCOUNT_FILE_PATH || !EVENT_ID) { error('SetupErorr', 'empty envvar SERVICE_ACCOUNT_FILE_PATH or EVENT_ID') @@ -62,8 +73,18 @@ export const saveMusic = (song: Song) => { // } } -export const histSongsRef = () => - fdb.collection('hist').doc(EVENT_ID).collection('songs') +export const histSongsRef = (eid = EVENT_ID) => + fdb.collection(P_HIST).doc(eid).collection(P_SONGS) + +export const loadHistEventSongs = async (eid: string) => { + const snaps = await histSongsRef(eid).orderBy('time', 'asc').get() + const lines: HistoryRaw[] = [] + snaps.docs.forEach((doc) => { + const d = doc.data() as HistoryRaw + lines.push(d) + }) + return lines +} export const loadHistoryTimes = async () => { const histSnaps = await histSongsRef() @@ -232,4 +253,30 @@ export const uploadStorage = async (file: CacheFile, id: string) => { return { downloadUrl, path: destination, tmpFilePath } } +const distPath = `archive` + +type StoragePaths = { + url: string + localFile: string + destination: string + filename: string +} +export const archiveUrl = (eid: string): StoragePaths => { + const localFile = `data/archvie_${eid}.csv` + const filename = `hist_${eid}.csv` + const destination = `${distPath}/${filename}` + const url = `${process.env.STRAGE_URL}/${destination}` + return { url, destination, filename, localFile } +} +export const uploadStorageArchive = async ({ + localFile, + destination, +}: StoragePaths) => { + await bucket.upload(localFile, { + contentType: 'text/csv', + destination, + predefinedAcl: 'publicRead', + }) +} + export const addHistoryNow = (title: string) => addHistory(title, +new Date()) diff --git a/src/types/index.ts b/src/types/index.ts index bb4c984c..0a33cc41 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -70,6 +70,7 @@ export type Song = Partial & (SongFull | SongMiss) export type HistoryRaw = { title: string time: number + n?: number } export type HistTop = { diff --git a/yarn.lock b/yarn.lock index f1cf53a7..b79464f8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2350,6 +2350,11 @@ csv-parse@*, csv-parse@4.16.0: resolved "https://registry.yarnpkg.com/csv-parse/-/csv-parse-4.16.0.tgz#b4c875e288a41f7ff917cb0d7d45880d563034f6" integrity sha512-Zb4tGPANH4SW0LgC9+s9Mnequs9aqn7N3/pCqNbVjs2XhEF6yWNU2Vm4OGl1v2Go9nw8rXt87Cm2QN/o6Vpqgg== +csv-stringify@^6.0.5: + version "6.0.5" + resolved "https://registry.yarnpkg.com/csv-stringify/-/csv-stringify-6.0.5.tgz#3474a4fe784249eb5c91d455f616e1f70961cdc0" + integrity sha512-7xpV3uweJCFF/Ssn56l3xsR/k2r3UqszwjEhej9qEn2cCPzyK1WyHCgoUVzBA792x8HbwonNX7CU9XM2K5s5yw== + currently-unhandled@^0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/currently-unhandled/-/currently-unhandled-0.4.1.tgz#988df33feab191ef799a61369dd76c17adf957ea" From ce9c312ba5a66df333770b68b0f355bb693d6edf Mon Sep 17 00:00:00 2001 From: vipzero Date: Sun, 1 May 2022 14:15:38 +0900 Subject: [PATCH 007/105] add batch --- scripts/batch/importHistoryFromCells2.ts | 31 ++++++++++++++++++++++++ src/service/firebase.ts | 8 ++++-- 2 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 scripts/batch/importHistoryFromCells2.ts diff --git a/scripts/batch/importHistoryFromCells2.ts b/scripts/batch/importHistoryFromCells2.ts new file mode 100644 index 00000000..f7109fe7 --- /dev/null +++ b/scripts/batch/importHistoryFromCells2.ts @@ -0,0 +1,31 @@ +import { readFileSync } from 'fs' +import { addHistory } from '../../src/service/firebase' +import { sleep } from '../../src/utils' +import { anaCounts } from '../../src/utils/wordCounts' + +const parseLine = (text: string) => { + const [time, title] = text.split(',') + + return { time, title } +} + +const importFile = './data/20220501_0200-0800.mid.csv' +const text = readFileSync(importFile, 'utf8') + +const lines = text.trim().split('\r\n') + +async function main() { + // for (let i = 0; i < 3; i++) { + for (let i = 0; i < lines.length; i++) { + const line = lines[i] + const { title, time } = parseLine(line) + + await sleep(200) + await addHistory(title.trim(), Number(time), 0) + anaCounts(title, {}) + + process.stdout.write('.') + } +} + +main().then(() => console.log('fin')) diff --git a/src/service/firebase.ts b/src/service/firebase.ts index 2257a8d7..98bdaf9f 100644 --- a/src/service/firebase.ts +++ b/src/service/firebase.ts @@ -96,8 +96,12 @@ export const loadHistoryTimes = async () => { return times } -export const addHistory = (title: string, time: number | null) => { - return histSongsRef().doc(String(time)).set({ title, time, n: null }) +export const addHistory = ( + title: string, + time: number | null, + n: null | number = null +) => { + return histSongsRef().doc(String(time)).set({ title, time, n }) } export const loadAllIcy = async () => { From 3e536d1d046f0c642070646ab0120fe1b98bb659 Mon Sep 17 00:00:00 2001 From: vipzero Date: Sun, 1 May 2022 23:47:19 +0900 Subject: [PATCH 008/105] add ee --- scripts/setupEasterEgg.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/setupEasterEgg.ts b/scripts/setupEasterEgg.ts index 00a2e607..8012fa06 100644 --- a/scripts/setupEasterEgg.ts +++ b/scripts/setupEasterEgg.ts @@ -44,6 +44,7 @@ async function main() { mk: 0, rk: 0, }, gabudoro: { gb: 0, vi: 0, st: 0, rf: 0 }, + yamajo: { ur: 0, nn: 0, mi: 0, mr: 0, na: 0, as: 0, si: 0 }, } const tasks = Object.entries(votes).map(async ([animeId, inits]) => { From b8d9e6f7ae5de93a6a9166aae10dccfbe6dc8e36 Mon Sep 17 00:00:00 2001 From: vipzero Date: Mon, 2 May 2022 15:25:44 +0900 Subject: [PATCH 009/105] add reset book count --- scripts/setupEvent.ts | 1 + src/service/firebase.ts | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/setupEvent.ts b/scripts/setupEvent.ts index 45d8a632..845bf8fd 100644 --- a/scripts/setupEvent.ts +++ b/scripts/setupEvent.ts @@ -4,6 +4,7 @@ const eventId = process.env.EVENT_ID async function main() { if (!eventId) return const hist = await fdb.collection('hist').doc(eventId).get() + await fdb.collection('yo').doc('current').set({ bookCount: 0 }) if (hist.exists) return await fdb.collection('hist').doc(eventId).set({ lasttime: 0 }) diff --git a/src/service/firebase.ts b/src/service/firebase.ts index 98bdaf9f..97ae6298 100644 --- a/src/service/firebase.ts +++ b/src/service/firebase.ts @@ -16,6 +16,8 @@ export { admin } const P_SONGS = 'songs' const P_HIST = 'hist' +const P_YO = 'yo' +const P_CURRENT = 'current' const { SERVICE_ACCOUNT_FILE_PATH, EVENT_ID } = process.env if (!SERVICE_ACCOUNT_FILE_PATH || !EVENT_ID) { @@ -75,6 +77,7 @@ export const saveMusic = (song: Song) => { export const histSongsRef = (eid = EVENT_ID) => fdb.collection(P_HIST).doc(eid).collection(P_SONGS) +export const bookCountDocRef = () => fdb.collection(P_YO).doc(P_CURRENT) export const loadHistEventSongs = async (eid: string) => { const snaps = await histSongsRef(eid).orderBy('time', 'asc').get() @@ -96,12 +99,14 @@ export const loadHistoryTimes = async () => { return times } -export const addHistory = ( +export const addHistory = async ( title: string, time: number | null, n: null | number = null ) => { - return histSongsRef().doc(String(time)).set({ title, time, n }) + await bookCountDocRef().update({ bookCount: 0 }) + + return await histSongsRef().doc(String(time)).set({ title, time, n }) } export const loadAllIcy = async () => { From ab446b4241eb3476bd1158fb1891cebe3d113294 Mon Sep 17 00:00:00 2001 From: vipzero Date: Mon, 2 May 2022 15:25:52 +0900 Subject: [PATCH 010/105] add pefo --- src/index.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/index.ts b/src/index.ts index 1fc826d3..3de2d66c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -78,14 +78,27 @@ export async function icyToSong( } async function receiveIcy(icy: string) { + performance.mark('e2') + performance.measure('first', 's2', 'e2') + log(performance.getEntriesByName('first')[0]) + const song = await icyToSong(icy, Date.now(), store) if (!song) return songPrint(song) saveMusic(song) } +performance.mark('s1') + async function main() { const res = await getCurrentPlay() + performance.mark('e1') + performance.measure('launch', 's1', 'e1') + + log(performance.getEntriesByName('launch')[0]) + + performance.mark('s2') + store.counts = (await init()).counts store.setFirstIcy(res.icy) if (!url) { From da5d24c495950237668083a9184d45aeda6b1122 Mon Sep 17 00:00:00 2001 From: vipzero Date: Mon, 2 May 2022 15:35:51 +0900 Subject: [PATCH 011/105] add tests --- package.json | 3 ++- src/index.ts | 4 +++- src/service/musixmatch.ts | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 05ac1a06..0eead94c 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,8 @@ "map-load-thread": "ts-node scripts/loadThreads.ts", "map-save": "ts-node scripts/makePostCountMap.ts", "active-thread": "chch dump-threads |jq -r '.threads[] | \"\\(.url),\\(.title)\"' |grep \"ならアニソン\"", - "test": "jest" + "test": "jest", + "test:type": "npx tsc --noEmit" }, "devDependencies": { "@swc/core": "^1.2.168", diff --git a/src/index.ts b/src/index.ts index 3de2d66c..9894fa39 100644 --- a/src/index.ts +++ b/src/index.ts @@ -80,7 +80,9 @@ export async function icyToSong( async function receiveIcy(icy: string) { performance.mark('e2') performance.measure('first', 's2', 'e2') - log(performance.getEntriesByName('first')[0]) + if (performance.getEntriesByName('first').length === 1) { + log(performance.getEntriesByName('first')[0]) + } const song = await icyToSong(icy, Date.now(), store) if (!song) return diff --git a/src/service/musixmatch.ts b/src/service/musixmatch.ts index 4f24e794..2d1b37c6 100644 --- a/src/service/musixmatch.ts +++ b/src/service/musixmatch.ts @@ -1,5 +1,5 @@ -import { error, log } from './logger' import axios from 'axios' +import { log, error } from '../utils/logger' const apikey = process.env.MUSIXMATCH_API_KEY From afd6d4c06e2789d47d3bb32a98214724abaa5138 Mon Sep 17 00:00:00 2001 From: vipzero Date: Mon, 2 May 2022 15:52:06 +0900 Subject: [PATCH 012/105] add esbuild --- package.json | 6 ++- yarn.lock | 131 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 136 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 0eead94c..ac6db8b1 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,8 @@ "csv-parse": "4.16.0", "csv-stringify": "^6.0.5", "date-fns": "2.23.0", + "esbuild": "^0.14.38", + "esbuild-register": "^3.3.2", "file-type": "16.5.3", "got": "11.8.2", "iconv-lite": "0.6.3", @@ -19,7 +21,9 @@ "sharp": "0.28.3" }, "scripts": { - "start": "ts-node src/index.ts", + "start": "yarn start:esbuild", + "start:esbuild": "node -r esbuild-register src/index.ts", + "start:ts-node": "ts-node src/index.ts", "get_history": "curl http://w.hoshinoa.me:8000/play.txt -o data/history.txt", "setup": "ts-node scripts/setupEvent.ts", "setup:ee": "ts-node scripts/setupEasterEgg.ts", diff --git a/yarn.lock b/yarn.lock index b79464f8..95dcfae9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2733,6 +2733,137 @@ error-ex@^1.2.0, error-ex@^1.3.1: dependencies: is-arrayish "^0.2.1" +esbuild-android-64@0.14.38: + version "0.14.38" + resolved "https://registry.yarnpkg.com/esbuild-android-64/-/esbuild-android-64-0.14.38.tgz#5b94a1306df31d55055f64a62ff6b763a47b7f64" + integrity sha512-aRFxR3scRKkbmNuGAK+Gee3+yFxkTJO/cx83Dkyzo4CnQl/2zVSurtG6+G86EQIZ+w+VYngVyK7P3HyTBKu3nw== + +esbuild-android-arm64@0.14.38: + version "0.14.38" + resolved "https://registry.yarnpkg.com/esbuild-android-arm64/-/esbuild-android-arm64-0.14.38.tgz#78acc80773d16007de5219ccce544c036abd50b8" + integrity sha512-L2NgQRWuHFI89IIZIlpAcINy9FvBk6xFVZ7xGdOwIm8VyhX1vNCEqUJO3DPSSy945Gzdg98cxtNt8Grv1CsyhA== + +esbuild-darwin-64@0.14.38: + version "0.14.38" + resolved "https://registry.yarnpkg.com/esbuild-darwin-64/-/esbuild-darwin-64-0.14.38.tgz#e02b1291f629ebdc2aa46fabfacc9aa28ff6aa46" + integrity sha512-5JJvgXkX87Pd1Og0u/NJuO7TSqAikAcQQ74gyJ87bqWRVeouky84ICoV4sN6VV53aTW+NE87qLdGY4QA2S7KNA== + +esbuild-darwin-arm64@0.14.38: + version "0.14.38" + resolved "https://registry.yarnpkg.com/esbuild-darwin-arm64/-/esbuild-darwin-arm64-0.14.38.tgz#01eb6650ec010b18c990e443a6abcca1d71290a9" + integrity sha512-eqF+OejMI3mC5Dlo9Kdq/Ilbki9sQBw3QlHW3wjLmsLh+quNfHmGMp3Ly1eWm981iGBMdbtSS9+LRvR2T8B3eQ== + +esbuild-freebsd-64@0.14.38: + version "0.14.38" + resolved "https://registry.yarnpkg.com/esbuild-freebsd-64/-/esbuild-freebsd-64-0.14.38.tgz#790b8786729d4aac7be17648f9ea8e0e16475b5e" + integrity sha512-epnPbhZUt93xV5cgeY36ZxPXDsQeO55DppzsIgWM8vgiG/Rz+qYDLmh5ts3e+Ln1wA9dQ+nZmVHw+RjaW3I5Ig== + +esbuild-freebsd-arm64@0.14.38: + version "0.14.38" + resolved "https://registry.yarnpkg.com/esbuild-freebsd-arm64/-/esbuild-freebsd-arm64-0.14.38.tgz#b66340ab28c09c1098e6d9d8ff656db47d7211e6" + integrity sha512-/9icXUYJWherhk+y5fjPI5yNUdFPtXHQlwP7/K/zg8t8lQdHVj20SqU9/udQmeUo5pDFHMYzcEFfJqgOVeKNNQ== + +esbuild-linux-32@0.14.38: + version "0.14.38" + resolved "https://registry.yarnpkg.com/esbuild-linux-32/-/esbuild-linux-32-0.14.38.tgz#7927f950986fd39f0ff319e92839455912b67f70" + integrity sha512-QfgfeNHRFvr2XeHFzP8kOZVnal3QvST3A0cgq32ZrHjSMFTdgXhMhmWdKzRXP/PKcfv3e2OW9tT9PpcjNvaq6g== + +esbuild-linux-64@0.14.38: + version "0.14.38" + resolved "https://registry.yarnpkg.com/esbuild-linux-64/-/esbuild-linux-64-0.14.38.tgz#4893d07b229d9cfe34a2b3ce586399e73c3ac519" + integrity sha512-uuZHNmqcs+Bj1qiW9k/HZU3FtIHmYiuxZ/6Aa+/KHb/pFKr7R3aVqvxlAudYI9Fw3St0VCPfv7QBpUITSmBR1Q== + +esbuild-linux-arm64@0.14.38: + version "0.14.38" + resolved "https://registry.yarnpkg.com/esbuild-linux-arm64/-/esbuild-linux-arm64-0.14.38.tgz#8442402e37d0b8ae946ac616784d9c1a2041056a" + integrity sha512-HlMGZTEsBrXrivr64eZ/EO0NQM8H8DuSENRok9d+Jtvq8hOLzrxfsAT9U94K3KOGk2XgCmkaI2KD8hX7F97lvA== + +esbuild-linux-arm@0.14.38: + version "0.14.38" + resolved "https://registry.yarnpkg.com/esbuild-linux-arm/-/esbuild-linux-arm-0.14.38.tgz#d5dbf32d38b7f79be0ec6b5fb2f9251fd9066986" + integrity sha512-FiFvQe8J3VKTDXG01JbvoVRXQ0x6UZwyrU4IaLBZeq39Bsbatd94Fuc3F1RGqPF5RbIWW7RvkVQjn79ejzysnA== + +esbuild-linux-mips64le@0.14.38: + version "0.14.38" + resolved "https://registry.yarnpkg.com/esbuild-linux-mips64le/-/esbuild-linux-mips64le-0.14.38.tgz#95081e42f698bbe35d8ccee0e3a237594b337eb5" + integrity sha512-qd1dLf2v7QBiI5wwfil9j0HG/5YMFBAmMVmdeokbNAMbcg49p25t6IlJFXAeLzogv1AvgaXRXvgFNhScYEUXGQ== + +esbuild-linux-ppc64le@0.14.38: + version "0.14.38" + resolved "https://registry.yarnpkg.com/esbuild-linux-ppc64le/-/esbuild-linux-ppc64le-0.14.38.tgz#dceb0a1b186f5df679618882a7990bd422089b47" + integrity sha512-mnbEm7o69gTl60jSuK+nn+pRsRHGtDPfzhrqEUXyCl7CTOCLtWN2bhK8bgsdp6J/2NyS/wHBjs1x8aBWwP2X9Q== + +esbuild-linux-riscv64@0.14.38: + version "0.14.38" + resolved "https://registry.yarnpkg.com/esbuild-linux-riscv64/-/esbuild-linux-riscv64-0.14.38.tgz#61fb8edb75f475f9208c4a93ab2bfab63821afd2" + integrity sha512-+p6YKYbuV72uikChRk14FSyNJZ4WfYkffj6Af0/Tw63/6TJX6TnIKE+6D3xtEc7DeDth1fjUOEqm+ApKFXbbVQ== + +esbuild-linux-s390x@0.14.38: + version "0.14.38" + resolved "https://registry.yarnpkg.com/esbuild-linux-s390x/-/esbuild-linux-s390x-0.14.38.tgz#34c7126a4937406bf6a5e69100185fd702d12fe0" + integrity sha512-0zUsiDkGJiMHxBQ7JDU8jbaanUY975CdOW1YDrurjrM0vWHfjv9tLQsW9GSyEb/heSK1L5gaweRjzfUVBFoybQ== + +esbuild-netbsd-64@0.14.38: + version "0.14.38" + resolved "https://registry.yarnpkg.com/esbuild-netbsd-64/-/esbuild-netbsd-64-0.14.38.tgz#322ea9937d9e529183ee281c7996b93eb38a5d95" + integrity sha512-cljBAApVwkpnJZfnRVThpRBGzCi+a+V9Ofb1fVkKhtrPLDYlHLrSYGtmnoTVWDQdU516qYI8+wOgcGZ4XIZh0Q== + +esbuild-openbsd-64@0.14.38: + version "0.14.38" + resolved "https://registry.yarnpkg.com/esbuild-openbsd-64/-/esbuild-openbsd-64-0.14.38.tgz#1ca29bb7a2bf09592dcc26afdb45108f08a2cdbd" + integrity sha512-CDswYr2PWPGEPpLDUO50mL3WO/07EMjnZDNKpmaxUPsrW+kVM3LoAqr/CE8UbzugpEiflYqJsGPLirThRB18IQ== + +esbuild-register@^3.3.2: + version "3.3.2" + resolved "https://registry.yarnpkg.com/esbuild-register/-/esbuild-register-3.3.2.tgz#1c3dc7179cabb4c7bd640a393eb916b18b12a223" + integrity sha512-jceAtTO6zxPmCfSD5cBb3rgIK1vmuqCKYwgylHiS1BF4pq0jJiJb4K2QMuqF4BEw7XDBRatYzip0upyTzfkgsQ== + +esbuild-sunos-64@0.14.38: + version "0.14.38" + resolved "https://registry.yarnpkg.com/esbuild-sunos-64/-/esbuild-sunos-64-0.14.38.tgz#c9446f7d8ebf45093e7bb0e7045506a88540019b" + integrity sha512-2mfIoYW58gKcC3bck0j7lD3RZkqYA7MmujFYmSn9l6TiIcAMpuEvqksO+ntBgbLep/eyjpgdplF7b+4T9VJGOA== + +esbuild-windows-32@0.14.38: + version "0.14.38" + resolved "https://registry.yarnpkg.com/esbuild-windows-32/-/esbuild-windows-32-0.14.38.tgz#f8e9b4602fd0ccbd48e5c8d117ec0ba4040f2ad1" + integrity sha512-L2BmEeFZATAvU+FJzJiRLFUP+d9RHN+QXpgaOrs2klshoAm1AE6Us4X6fS9k33Uy5SzScn2TpcgecbqJza1Hjw== + +esbuild-windows-64@0.14.38: + version "0.14.38" + resolved "https://registry.yarnpkg.com/esbuild-windows-64/-/esbuild-windows-64-0.14.38.tgz#280f58e69f78535f470905ce3e43db1746518107" + integrity sha512-Khy4wVmebnzue8aeSXLC+6clo/hRYeNIm0DyikoEqX+3w3rcvrhzpoix0S+MF9vzh6JFskkIGD7Zx47ODJNyCw== + +esbuild-windows-arm64@0.14.38: + version "0.14.38" + resolved "https://registry.yarnpkg.com/esbuild-windows-arm64/-/esbuild-windows-arm64-0.14.38.tgz#d97e9ac0f95a4c236d9173fa9f86c983d6a53f54" + integrity sha512-k3FGCNmHBkqdJXuJszdWciAH77PukEyDsdIryEHn9cKLQFxzhT39dSumeTuggaQcXY57UlmLGIkklWZo2qzHpw== + +esbuild@^0.14.38: + version "0.14.38" + resolved "https://registry.yarnpkg.com/esbuild/-/esbuild-0.14.38.tgz#99526b778cd9f35532955e26e1709a16cca2fb30" + integrity sha512-12fzJ0fsm7gVZX1YQ1InkOE5f9Tl7cgf6JPYXRJtPIoE0zkWAbHdPHVPPaLi9tYAcEBqheGzqLn/3RdTOyBfcA== + optionalDependencies: + esbuild-android-64 "0.14.38" + esbuild-android-arm64 "0.14.38" + esbuild-darwin-64 "0.14.38" + esbuild-darwin-arm64 "0.14.38" + esbuild-freebsd-64 "0.14.38" + esbuild-freebsd-arm64 "0.14.38" + esbuild-linux-32 "0.14.38" + esbuild-linux-64 "0.14.38" + esbuild-linux-arm "0.14.38" + esbuild-linux-arm64 "0.14.38" + esbuild-linux-mips64le "0.14.38" + esbuild-linux-ppc64le "0.14.38" + esbuild-linux-riscv64 "0.14.38" + esbuild-linux-s390x "0.14.38" + esbuild-netbsd-64 "0.14.38" + esbuild-openbsd-64 "0.14.38" + esbuild-sunos-64 "0.14.38" + esbuild-windows-32 "0.14.38" + esbuild-windows-64 "0.14.38" + esbuild-windows-arm64 "0.14.38" + escalade@^3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.1.tgz#d8cfdc7000965c5a0174b4a82eaa5c0552742e40" From c8748b26d52c16d9fdba2a742e5cae50ed08be99 Mon Sep 17 00:00:00 2001 From: vipzero Date: Mon, 2 May 2022 19:04:38 +0900 Subject: [PATCH 013/105] add ee --- scripts/setupEasterEgg.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/setupEasterEgg.ts b/scripts/setupEasterEgg.ts index 8012fa06..172d8db3 100644 --- a/scripts/setupEasterEgg.ts +++ b/scripts/setupEasterEgg.ts @@ -45,6 +45,7 @@ async function main() { }, gabudoro: { gb: 0, vi: 0, st: 0, rf: 0 }, yamajo: { ur: 0, nn: 0, mi: 0, mr: 0, na: 0, as: 0, si: 0 }, + bryunhild: { nk: 0, kn: 0, kz: 0, kt: 0, nn: 0, ht: 0, mt: 0 }, } const tasks = Object.entries(votes).map(async ([animeId, inits]) => { From 8c36f0382ef79abbdea2cf10d1752d3bd6f2ede1 Mon Sep 17 00:00:00 2001 From: vipzero Date: Mon, 2 May 2022 19:04:42 +0900 Subject: [PATCH 014/105] update normalize --- package.json | 1 + src/tests/normalize.test.ts | 8 ++++++++ src/utils/index.ts | 22 ++++++++++++++-------- yarn.lock | 9 ++++++++- 4 files changed, 31 insertions(+), 9 deletions(-) create mode 100644 src/tests/normalize.test.ts diff --git a/package.json b/package.json index ac6db8b1..6583f670 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "got": "11.8.2", "iconv-lite": "0.6.3", "icy": "2.1.0", + "jaco": "^4.0.0", "jest": "^27.5.1", "node-spotify-api": "1.1.1", "sharp": "0.28.3" diff --git a/src/tests/normalize.test.ts b/src/tests/normalize.test.ts new file mode 100644 index 00000000..c2b127f0 --- /dev/null +++ b/src/tests/normalize.test.ts @@ -0,0 +1,8 @@ +import { textNormalize } from '../utils' +test('textNormalize', () => { + const generalParret = textNormalize('［【test】］') + expect(generalParret).toMatchInlineSnapshot(`"［(test)］"`) + + const wided = textNormalize('ＴEｓtﾃｽトT　！？') + expect(wided).toMatchInlineSnapshot(`"testテストt !?"`) +}) diff --git a/src/utils/index.ts b/src/utils/index.ts index a5ae8498..3d344513 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -1,4 +1,6 @@ import iconv from 'iconv-lite' +import toWideKatakana from 'jaco/fn/toWideKatakana' +import toNarrowAlphanumeric from 'jaco/fn/toNarrowAlphanumeric' export const isHit = (long: string, short: string) => long.indexOf(short) >= 0 @@ -9,14 +11,18 @@ export const sleep = (msec: number) => new Promise((rs) => setTimeout(rs, msec)) // frontend と同じ必要あり export function textNormalize(s: string) { - return s - .trim() - .toLowerCase() - .replace('　', ' ') - .replace(/[（【「[]/, '(') - .replace(/[）】」\]]/, ')') - .replace('！', '!') - .replace('？', '?') + return toNarrowAlphanumeric( + toWideKatakana( + s + .trim() + .toLowerCase() + .replace('　', ' ') + .replace(/[（【「[]/, '(') + .replace(/[）】」\]]/, ')') + .replace('！', '!') + .replace('？', '?') + ) + ) } const SP = /CV ?|[（［([](CV)? ?|[〈〉（）［］()【】[\]、・：．]| x | - |feat\.?/gi diff --git a/yarn.lock b/yarn.lock index 95dcfae9..3b439e8c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2207,7 +2207,7 @@ combined-stream@^1.0.6, combined-stream@^1.0.8, combined-stream@~1.0.6: dependencies: delayed-stream "~1.0.0" -commander@^2.8.1: +commander@^2.15.1, commander@^2.8.1: version "2.20.3" resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33" integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ== @@ -4238,6 +4238,13 @@ isurl@^1.0.0-alpha5: has-to-string-tag-x "^1.2.0" is-object "^1.0.1" +jaco@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/jaco/-/jaco-4.0.0.tgz#7522a3dd4d7295b3e33b0ae82c08b34db99738e5" + integrity sha512-RBtY32bY4ICqgPVTmWOLoxn3rZQ+sFPtkcmCA2HJY7ZCCKkCMoMhv/zw0G+eGCzIb7Vkpx5dqSu4cwuS6c6l1w== + dependencies: + commander "^2.15.1" + jest-changed-files@^27.5.1: version "27.5.1" resolved "https://registry.yarnpkg.com/jest-changed-files/-/jest-changed-files-27.5.1.tgz#a348aed00ec9bf671cc58a66fcbe7c3dfd6a68f5" From 297a3641009545d01d87fb1237d08195a7991c11 Mon Sep 17 00:00:00 2001 From: vipzero Date: Mon, 2 May 2022 20:56:56 +0900 Subject: [PATCH 015/105] add book init --- src/service/firebase.ts | 2 +- src/types/index.ts | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/service/firebase.ts b/src/service/firebase.ts index 97ae6298..8570fb33 100644 --- a/src/service/firebase.ts +++ b/src/service/firebase.ts @@ -106,7 +106,7 @@ export const addHistory = async ( ) => { await bookCountDocRef().update({ bookCount: 0 }) - return await histSongsRef().doc(String(time)).set({ title, time, n }) + return await histSongsRef().doc(String(time)).set({ title, time, n, b: 0 }) } export const loadAllIcy = async () => { diff --git a/src/types/index.ts b/src/types/index.ts index 0a33cc41..ee8cf650 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -71,6 +71,7 @@ export type HistoryRaw = { title: string time: number n?: number + b?: number } export type HistTop = { From f52da9b842054b0f3b9bfa475ad5e6bbcf48738e Mon Sep 17 00:00:00 2001 From: vipzero Date: Tue, 3 May 2022 19:10:48 +0900 Subject: [PATCH 016/105] add ee --- scripts/setupEasterEgg.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/setupEasterEgg.ts b/scripts/setupEasterEgg.ts index 172d8db3..41dd7def 100644 --- a/scripts/setupEasterEgg.ts +++ b/scripts/setupEasterEgg.ts @@ -46,6 +46,12 @@ async function main() { gabudoro: { gb: 0, vi: 0, st: 0, rf: 0 }, yamajo: { ur: 0, nn: 0, mi: 0, mr: 0, na: 0, as: 0, si: 0 }, bryunhild: { nk: 0, kn: 0, kz: 0, kt: 0, nn: 0, ht: 0, mt: 0 }, + // prettier-ignore + imasmlpr: { mr: 0, sy: 0, kt: 0, ar: 0, kn: 0, em: 0, um: 0, mn: 0, nr: 0, no: 0, mt: 0, ik: 0, yr: 0, }, + // prettier-ignore + imasmlfa: { jr: 0, tm: 0, sz: 0, sb: 0, ay: 0, mz: 0, ro: 0, sh: 0, mm: 0, em: 0, cz: 0, rk: 0, tg: 0, }, + // prettier-ignore + imasmlan: { hn: 0, sr: 0, an: 0, tm: 0, ri: 0, el: 0, ak: 0, kn: 0, tb: 0, kl: 0, hk: 0, my: 0, ko: 0, }, } const tasks = Object.entries(votes).map(async ([animeId, inits]) => { From 18e93fff17e7e0732e58531c06511806d872c86d Mon Sep 17 00:00:00 2001 From: vipzero Date: Tue, 3 May 2022 19:47:26 +0900 Subject: [PATCH 017/105] fix time --- src/index.ts | 11 ++++++----- src/service/firebase.ts | 2 -- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/index.ts b/src/index.ts index 9894fa39..3707b43c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,11 +1,8 @@ import { findSong } from './anisonDb/findSong' -import subscribeIcy from './streaming/icy' import { uploadByUrlAll } from './imageIo/uploadManage' -import { error, info, log, songPrint } from './utils/logger' -import { makeSearchQuery } from './utils/makeSearchWord' import { getImageLinks } from './service/customImageSearch' import { - addHistoryNow, + addHistory, deleteFile, getCurrentPlay, init, @@ -15,8 +12,11 @@ import { getAlbum } from './service/itunes' import { getLyricsSafe } from './service/jlyricnet' // import { spotifySearchSongInfo } from './spotify' import { Store, store } from './state/store' +import subscribeIcy from './streaming/icy' import { Song } from './types/index' import { sleep } from './utils' +import { error, info, log, songPrint } from './utils/logger' +import { makeSearchQuery } from './utils/makeSearchWord' import { anaCounts } from './utils/wordCounts' const url = process.env.URL @@ -45,9 +45,10 @@ export async function icyToSong( if (store.isDuplicate(icy)) return false // 起動時の重複登録を防ぐ - addHistoryNow(icy) + addHistory(icy, time) const song = findSong(icy) + const additionals: string[] = [song.animeTitle, song.title].filter( Boolean ) as string[] diff --git a/src/service/firebase.ts b/src/service/firebase.ts index 8570fb33..8b000e56 100644 --- a/src/service/firebase.ts +++ b/src/service/firebase.ts @@ -287,5 +287,3 @@ export const uploadStorageArchive = async ({ predefinedAcl: 'publicRead', }) } - -export const addHistoryNow = (title: string) => addHistory(title, +new Date()) From b31b4a148d6f5b4f0d786568388f45fb8a81ac96 Mon Sep 17 00:00:00 2001 From: vipzero Date: Wed, 4 May 2022 11:25:32 +0900 Subject: [PATCH 018/105] add ee --- scripts/setupEasterEgg.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/setupEasterEgg.ts b/scripts/setupEasterEgg.ts index 41dd7def..3e0a110c 100644 --- a/scripts/setupEasterEgg.ts +++ b/scripts/setupEasterEgg.ts @@ -52,11 +52,14 @@ async function main() { imasmlfa: { jr: 0, tm: 0, sz: 0, sb: 0, ay: 0, mz: 0, ro: 0, sh: 0, mm: 0, em: 0, cz: 0, rk: 0, tg: 0, }, // prettier-ignore imasmlan: { hn: 0, sr: 0, an: 0, tm: 0, ri: 0, el: 0, ak: 0, kn: 0, tb: 0, kl: 0, hk: 0, my: 0, ko: 0, }, + // prettier-ignore + rakupro: { mm: 0, si: 0, kt: 0, ym: 0, mk: 0, mo: 0, nn: 0, rr: 0, rt: 0, ma: 0, rn: 0, sz: 0, }, } const tasks = Object.entries(votes).map(async ([animeId, inits]) => { const animeRef = fdb.collection('cvote').doc(animeId) const doc = await animeRef.get() + if (doc.exists) return await animeRef.set(inits) From 52b12eb9c2dc98dea7e199c8e417b6c68eb972ee Mon Sep 17 00:00:00 2001 From: vipzero Date: Wed, 4 May 2022 11:25:37 +0900 Subject: [PATCH 019/105] update normalize --- src/utils/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/utils/index.ts b/src/utils/index.ts index 3d344513..04e0b946 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -19,6 +19,7 @@ export function textNormalize(s: string) { .replace('　', ' ') .replace(/[（【「[]/, '(') .replace(/[）】」\]]/, ')') + .replace('〜', '~') .replace('！', '!') .replace('？', '?') ) From 4bcc72566e193173e416808af2e923ae611252c2 Mon Sep 17 00:00:00 2001 From: vipzero Date: Wed, 4 May 2022 18:04:04 +0900 Subject: [PATCH 020/105] update algo --- readme.md | 5 +- src/index.ts | 2 +- src/tests/makeSearchQuery.test.ts | 86 +++++++++++++++++-------------- src/utils/makeSearchWord.ts | 44 ++++++++++++---- 4 files changed, 85 insertions(+), 52 deletions(-) diff --git a/readme.md b/readme.md index a3d553f1..e27bfc65 100644 --- a/readme.md +++ b/readme.md @@ -46,8 +46,6 @@ iTunes API (認証なし) - spotify: アートワーク・アルバム名・(邦楽・アニソンあまり取れないため) - musixmatch: アートワーク・アルバム名・歌詞の出だし(有料 API では FULL) - - 調整・アルゴリズム関係 ## 画像検索文字列の生成部分 @@ -63,8 +61,9 @@ iTunes API (認証なし) https://github.com/vipzero/haikei-server/blob/main/src/utils/makeSearchWord.ts ## 画像選択 + imagemin などで Optimize 後のメタデータで -ソートして上から3つ +ソートして上から 3 つ コード diff --git a/src/index.ts b/src/index.ts index 3707b43c..cf60f555 100644 --- a/src/index.ts +++ b/src/index.ts @@ -55,7 +55,7 @@ export async function icyToSong( const { wordCounts, counts } = anaCounts(icy, store.counts || {}, additionals) store.counts = counts - const imageSearchWord = makeSearchQuery(song) + const imageSearchWord = makeSearchQuery(song, Math.random()) const imageLinksSync = prepareImages(imageSearchWord) const albumInfosSync = getAlbum(icy) diff --git a/src/tests/makeSearchQuery.test.ts b/src/tests/makeSearchQuery.test.ts index 3622f580..75701b61 100644 --- a/src/tests/makeSearchQuery.test.ts +++ b/src/tests/makeSearchQuery.test.ts @@ -1,60 +1,70 @@ import { makeSearchQuery } from '../utils/makeSearchWord' import { pickCharaIcy } from '../utils' +const seed = 0.123456789 describe('makeSearchQuery', () => { test('lost animeTitle', () => { expect( - makeSearchQuery({ - artist: '蒼井翔太', - title: 'give me ? me', - icy: '蒼井翔太 - give me ? me', - }) - ).toMatchInlineSnapshot( - `"蒼井翔太 give me ? me (名シーン OR キャラ) (キャプ画像 OR 壁紙) かわいい"` - ) +makeSearchQuery( +{ + artist: '蒼井翔太', + title: 'give me ? me', + icy: '蒼井翔太 - give me ? me' }, + +seed)). + +toMatchInlineSnapshot(`"(蒼井翔太 OR give me ? me) (名シーン OR キャラ) (キャプ画像 OR 壁紙)"`) }) test('lost animeTitle and detect chara', () => { expect( - makeSearchQuery({ - icy: 'せーので跳べって言ってんの! - 本城香澄(CV:岩橋由佳)', - }) - ).toMatchInlineSnapshot( - `"本城香澄 (名シーン OR キャラ) (キャプ画像 OR 壁紙) かわいい"` - ) +makeSearchQuery( +{ + icy: 'せーので跳べって言ってんの! - 本城香澄(CV:岩橋由佳)' }, + +seed)). + +toMatchInlineSnapshot(`"本城香澄 (名シーン OR キャラ) (キャプ画像 OR 壁紙) キャプ画"`) }) test('has animeTitle', () => { expect( - makeSearchQuery({ - artist: 'artist', - title: 'title', - animeTitle: 'アニメタイトルあり', - icy: 'artist - title', - }) + makeSearchQuery( + { + artist: 'artist', + title: 'title', + animeTitle: 'アニメタイトルあり', + icy: 'artist - title', + }, + seed + ) ).toMatchInlineSnapshot(`"アニメタイトルあり"`) }) test('has animeTitle and category anime', () => { expect( - makeSearchQuery({ - artist: 'artist', - title: 'title', - animeTitle: 'アニメタイトルあり', - category: 'ほにゃららアニメ', - icy: 'artist - title', - }) - ).toMatchInlineSnapshot( - `"アニメタイトルあり AND (アニメ OR meme OR キャプ画 OR キャラ OR かわいい)"` - ) +makeSearchQuery( +{ + artist: 'artist', + title: 'title', + animeTitle: 'アニメタイトルあり', + category: 'ほにゃららアニメ', + icy: 'artist - title' }, + +seed)). + +toMatchInlineSnapshot(`"アニメタイトルあり AND (アニメ OR キャプ画 OR かわいい OR ネタ画像)"`) }) test('has animeTitle and category game', () => { expect( - makeSearchQuery({ - title: '飯島真理', - artist: '飯島真理', - category: 'ゲーム', - gameType: 'シューティング', - animeTitle: '超時空要塞マクロス 愛・おぼえていますか', - icy: '飯島真理 - 愛・おぼえていますか', - }) + makeSearchQuery( + { + title: '飯島真理', + artist: '飯島真理', + category: 'ゲーム', + gameType: 'シューティング', + animeTitle: '超時空要塞マクロス 愛・おぼえていますか', + icy: '飯島真理 - 愛・おぼえていますか', + }, + seed + ) ).toMatchInlineSnapshot( `"超時空要塞マクロス 愛・おぼえていますか AND (ゲーム)"` ) diff --git a/src/utils/makeSearchWord.ts b/src/utils/makeSearchWord.ts index 4d4223ab..029e762b 100644 --- a/src/utils/makeSearchWord.ts +++ b/src/utils/makeSearchWord.ts @@ -1,33 +1,57 @@ -import { getSyncConf } from './syncConf' -import { SongSeed } from '../types/index' import { pickCharaIcy } from '.' +import { SongSeed } from '../types/index' +import { getSyncConf } from './syncConf' /* anison DB から曲情報を取れた場合アニメタイトル、カテゴリ(ゲーム,アニメ,SF,映画)などを使用できる */ + +/* ランダムに追加する */ +const animeExt = [ + 'meme', + 'キャプ画', + 'キャラ', + 'かわいい', + 'グッズ', + 'ネタ画像', + '壁紙', + '相関図', + '名シーン', +] /* アニメカテゴリ場合は↓を検索クエリに追加する */ -const animeExt = ['アニメ', 'meme', 'キャプ画', 'キャラ', 'かわいい'].join( - ' OR ' -) +const animeExtBase = ['アニメ'].join(' OR ') + +export const choise = (arr: T[], seed: number) => + arr[Math.floor(arr.length * seed)] || arr[0] /* ゲームカテゴリ場合は↓を検索クエリに追加する */ const gameExt = ['ゲーム'].join(' OR ') /* アニメ情報が取れなかった曲は icy に↓を検索クエリに追加する */ -const icyOpt = `(名シーン OR キャラ) (キャプ画像 OR 壁紙) かわいい` +const icyOpt = `(名シーン OR キャラ) (キャプ画像 OR 壁紙)` -export function makeSearchQuery(song: SongSeed): string { +export function makeSearchQuery(song: SongSeed, seed: number): string { + const seed2 = (seed * 100) % 1 + const seed3 = (seed * 10000) % 1 const { icy, category, animeTitle } = song if (getSyncConf().simpleSearch) return icy.replace(/-/g, ' ') if (!animeTitle) { const charaName = pickCharaIcy(icy).join(' ') - if (charaName) return `${charaName} ${icyOpt}` + if (charaName) return `${charaName} ${icyOpt} ${choise(animeExt, seed)}` - return icy.replace(' - ', ' ') + ` ${icyOpt}` + return ( + `(${icy + .split(' - ') + .map((v) => v.substring(0, 16)) + .join(' OR ')})` + ` ${icyOpt}` + ) } if (!category) return animeTitle if (category.includes('アニメ')) { - return `${animeTitle} AND (${animeExt})` + return `${animeTitle} AND (${animeExtBase} OR ${choise( + animeExt, + seed + )} OR ${choise(animeExt, seed2)} OR ${choise(animeExt, seed3)})` } if (category.includes('ゲーム')) { return `${animeTitle} AND (${gameExt})` From 9ab2243ed18fca4ae5a306a73bfbf885c91c8405 Mon Sep 17 00:00:00 2001 From: vipzero Date: Wed, 4 May 2022 19:44:40 +0900 Subject: [PATCH 021/105] add word --- scripts/playground.ts | 7 +++++++ src/utils/makeSearchWord.ts | 1 + 2 files changed, 8 insertions(+) diff --git a/scripts/playground.ts b/scripts/playground.ts index 26e24bce..0c5b85bf 100644 --- a/scripts/playground.ts +++ b/scripts/playground.ts @@ -77,3 +77,10 @@ // import { getSyncConf } from './../src/utils/syncConf' // console.log(getSyncConf()) + +import { makeSearchQuery } from '../src/utils/makeSearchWord' + +const song = { + icy: '愛美 - LIVE for LIFE ～狼たちの夜～', +} +console.log(makeSearchQuery(song, Math.random())) diff --git a/src/utils/makeSearchWord.ts b/src/utils/makeSearchWord.ts index 029e762b..df6da9c6 100644 --- a/src/utils/makeSearchWord.ts +++ b/src/utils/makeSearchWord.ts @@ -15,6 +15,7 @@ const animeExt = [ '壁紙', '相関図', '名シーン', + '作画', ] /* アニメカテゴリ場合は↓を検索クエリに追加する */ const animeExtBase = ['アニメ'].join(' OR ') From c8c678b1d8c6b57f9d4a80e3123166b13f4ce162 Mon Sep 17 00:00:00 2001 From: vipzero Date: Thu, 5 May 2022 00:24:06 +0900 Subject: [PATCH 022/105] update search word --- scripts/setupEasterEgg.ts | 2 ++ src/tests/makeSearchQuery.test.ts | 4 ++-- src/utils/makeSearchWord.ts | 14 ++++++++------ 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/scripts/setupEasterEgg.ts b/scripts/setupEasterEgg.ts index 3e0a110c..eee3fc15 100644 --- a/scripts/setupEasterEgg.ts +++ b/scripts/setupEasterEgg.ts @@ -54,6 +54,8 @@ async function main() { imasmlan: { hn: 0, sr: 0, an: 0, tm: 0, ri: 0, el: 0, ak: 0, kn: 0, tb: 0, kl: 0, hk: 0, my: 0, ko: 0, }, // prettier-ignore rakupro: { mm: 0, si: 0, kt: 0, ym: 0, mk: 0, mo: 0, nn: 0, rr: 0, rt: 0, ma: 0, rn: 0, sz: 0, }, + // prettier-ignore + shining: { hn: 0, ss: 0, mr: 0, hy: 0, gs: 0, rr: 0, kk: 0, uo: 0, ms: 0, hr: 0, er: 0, te: 0, }, } const tasks = Object.entries(votes).map(async ([animeId, inits]) => { diff --git a/src/tests/makeSearchQuery.test.ts b/src/tests/makeSearchQuery.test.ts index 75701b61..8843bc1e 100644 --- a/src/tests/makeSearchQuery.test.ts +++ b/src/tests/makeSearchQuery.test.ts @@ -13,7 +13,7 @@ makeSearchQuery( seed)). -toMatchInlineSnapshot(`"(蒼井翔太 OR give me ? me) (名シーン OR キャラ) (キャプ画像 OR 壁紙)"`) +toMatchInlineSnapshot(`"(蒼井翔太 OR give me ? me) キャプ画 OR かわいい OR ネタ画像"`) }) test('lost animeTitle and detect chara', () => { expect( @@ -23,7 +23,7 @@ makeSearchQuery( seed)). -toMatchInlineSnapshot(`"本城香澄 (名シーン OR キャラ) (キャプ画像 OR 壁紙) キャプ画"`) +toMatchInlineSnapshot(`"本城香澄 (名シーン OR キャラ) (キャプ画像 OR 壁紙)"`) }) test('has animeTitle', () => { expect( diff --git a/src/utils/makeSearchWord.ts b/src/utils/makeSearchWord.ts index df6da9c6..9ba4df03 100644 --- a/src/utils/makeSearchWord.ts +++ b/src/utils/makeSearchWord.ts @@ -34,25 +34,27 @@ export function makeSearchQuery(song: SongSeed, seed: number): string { const seed3 = (seed * 10000) % 1 const { icy, category, animeTitle } = song if (getSyncConf().simpleSearch) return icy.replace(/-/g, ' ') + const opts = [ + choise(animeExt, seed), + choise(animeExt, seed2), + choise(animeExt, seed3), + ] if (!animeTitle) { const charaName = pickCharaIcy(icy).join(' ') - if (charaName) return `${charaName} ${icyOpt} ${choise(animeExt, seed)}` + if (charaName) return `${charaName} ${icyOpt}` return ( `(${icy .split(' - ') .map((v) => v.substring(0, 16)) - .join(' OR ')})` + ` ${icyOpt}` + .join(' OR ')})` + ` ${opts.join(' OR ')}` ) } if (!category) return animeTitle if (category.includes('アニメ')) { - return `${animeTitle} AND (${animeExtBase} OR ${choise( - animeExt, - seed - )} OR ${choise(animeExt, seed2)} OR ${choise(animeExt, seed3)})` + return `${animeTitle} AND (${animeExtBase} OR ${opts.join(' OR ')})` } if (category.includes('ゲーム')) { return `${animeTitle} AND (${gameExt})` From a07befb66643ce1abd1fb684d29f4f1ea4ba1c09 Mon Sep 17 00:00:00 2001 From: vipzero Date: Fri, 6 May 2022 13:02:51 +0900 Subject: [PATCH 023/105] update random --- src/tests/makeSearchQuery.test.ts | 4 ++-- src/tests/random.test.ts | 12 ++++++++++++ src/utils/index.ts | 13 ------------- src/utils/makeSearchWord.ts | 14 ++++---------- src/utils/random.ts | 11 +++++++++++ 5 files changed, 29 insertions(+), 25 deletions(-) create mode 100644 src/tests/random.test.ts create mode 100644 src/utils/random.ts diff --git a/src/tests/makeSearchQuery.test.ts b/src/tests/makeSearchQuery.test.ts index 8843bc1e..e94f82a9 100644 --- a/src/tests/makeSearchQuery.test.ts +++ b/src/tests/makeSearchQuery.test.ts @@ -13,7 +13,7 @@ makeSearchQuery( seed)). -toMatchInlineSnapshot(`"(蒼井翔太 OR give me ? me) キャプ画 OR かわいい OR ネタ画像"`) +toMatchInlineSnapshot(`"(蒼井翔太 OR give me ? me) キャラ OR 作画 OR 壁紙"`) }) test('lost animeTitle and detect chara', () => { expect( @@ -50,7 +50,7 @@ makeSearchQuery( seed)). -toMatchInlineSnapshot(`"アニメタイトルあり AND (アニメ OR キャプ画 OR かわいい OR ネタ画像)"`) +toMatchInlineSnapshot(`"アニメタイトルあり AND (アニメ OR キャラ OR 作画 OR 壁紙)"`) }) test('has animeTitle and category game', () => { expect( diff --git a/src/tests/random.test.ts b/src/tests/random.test.ts new file mode 100644 index 00000000..8258f7f4 --- /dev/null +++ b/src/tests/random.test.ts @@ -0,0 +1,12 @@ +import { shuffle } from '../utils/random' + +describe('shuffle', () => { + it('shuffle', () => { + const a = [1, 2, 3, 4, 5] + const b = shuffle(a, 'a') + const c = shuffle(a, 'b') + + expect(b).toStrictEqual([2, 1, 5, 3, 4]) + expect(c).toStrictEqual([2, 3, 1, 5, 4]) + }) +}) diff --git a/src/utils/index.ts b/src/utils/index.ts index 04e0b946..a74f72d5 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -37,19 +37,6 @@ const trimWord = (v: string) => const parseWords = (s: string) => s.replace(SP, ',').split(',').map(trimWord).filter(isTagWord) -export const shuffle = (arr: T[]): T[] => { - const a = [...arr] - for (let i = a.length - 1; i > 0; i--) { - const r = Math.floor(Math.random() * (i + 1)) - const tmp = a[i] - a[i] = a[r] - a[r] = tmp - } - return a -} - -export const sample = (arr: T[], n = 1) => shuffle(arr).slice(0, n) - export const uniq = (arr: T[]) => Array.from(new Set(arr)) export const uniqo = (arr: string[]) => { const obj: Record = {} diff --git a/src/utils/makeSearchWord.ts b/src/utils/makeSearchWord.ts index 9ba4df03..ff517ea3 100644 --- a/src/utils/makeSearchWord.ts +++ b/src/utils/makeSearchWord.ts @@ -1,6 +1,7 @@ import { pickCharaIcy } from '.' import { SongSeed } from '../types/index' import { getSyncConf } from './syncConf' +import { shuffle } from './random' /* anison DB から曲情報を取れた場合アニメタイトル、カテゴリ(ゲーム,アニメ,SF,映画)などを使用できる */ @@ -16,13 +17,11 @@ const animeExt = [ '相関図', '名シーン', '作画', + 'wallpaper', ] /* アニメカテゴリ場合は↓を検索クエリに追加する */ const animeExtBase = ['アニメ'].join(' OR ') -export const choise = (arr: T[], seed: number) => - arr[Math.floor(arr.length * seed)] || arr[0] - /* ゲームカテゴリ場合は↓を検索クエリに追加する */ const gameExt = ['ゲーム'].join(' OR ') @@ -30,15 +29,10 @@ const gameExt = ['ゲーム'].join(' OR ') const icyOpt = `(名シーン OR キャラ) (キャプ画像 OR 壁紙)` export function makeSearchQuery(song: SongSeed, seed: number): string { - const seed2 = (seed * 100) % 1 - const seed3 = (seed * 10000) % 1 const { icy, category, animeTitle } = song if (getSyncConf().simpleSearch) return icy.replace(/-/g, ' ') - const opts = [ - choise(animeExt, seed), - choise(animeExt, seed2), - choise(animeExt, seed3), - ] + const r = shuffle(animeExt, `${seed}`) + const opts = [r[0], r[1], r[2]] if (!animeTitle) { const charaName = pickCharaIcy(icy).join(' ') diff --git a/src/utils/random.ts b/src/utils/random.ts new file mode 100644 index 00000000..8a9edced --- /dev/null +++ b/src/utils/random.ts @@ -0,0 +1,11 @@ +import crypto from 'crypto' + +const random = (seed: string) => + crypto.createHash('sha256').update(seed).digest().readUInt32LE() + +export const shuffle = (a: T[], seed: string) => { + return a + .map((v) => ({ v, r: random(seed + v) })) + .sort((a, b) => a.r - b.r) + .map(({ v }) => v) +} From ba47c591416385b584461f0eb3889061b23270e7 Mon Sep 17 00:00:00 2001 From: vipzero Date: Fri, 6 May 2022 15:04:52 +0900 Subject: [PATCH 024/105] update seed algo --- src/tests/makeSearchQuery.test.ts | 4 ++-- src/tests/random.test.ts | 20 ++++++++++++++++++-- src/utils/random.ts | 2 +- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/tests/makeSearchQuery.test.ts b/src/tests/makeSearchQuery.test.ts index e94f82a9..9377afbf 100644 --- a/src/tests/makeSearchQuery.test.ts +++ b/src/tests/makeSearchQuery.test.ts @@ -13,7 +13,7 @@ makeSearchQuery( seed)). -toMatchInlineSnapshot(`"(蒼井翔太 OR give me ? me) キャラ OR 作画 OR 壁紙"`) +toMatchInlineSnapshot(`"(蒼井翔太 OR give me ? me) ネタ画像 OR 壁紙 OR 作画"`) }) test('lost animeTitle and detect chara', () => { expect( @@ -50,7 +50,7 @@ makeSearchQuery( seed)). -toMatchInlineSnapshot(`"アニメタイトルあり AND (アニメ OR キャラ OR 作画 OR 壁紙)"`) +toMatchInlineSnapshot(`"アニメタイトルあり AND (アニメ OR ネタ画像 OR 壁紙 OR 作画)"`) }) test('has animeTitle and category game', () => { expect( diff --git a/src/tests/random.test.ts b/src/tests/random.test.ts index 8258f7f4..4830be41 100644 --- a/src/tests/random.test.ts +++ b/src/tests/random.test.ts @@ -6,7 +6,23 @@ describe('shuffle', () => { const b = shuffle(a, 'a') const c = shuffle(a, 'b') - expect(b).toStrictEqual([2, 1, 5, 3, 4]) - expect(c).toStrictEqual([2, 3, 1, 5, 4]) + expect(b).toMatchInlineSnapshot(` +Array [ + 3, + 2, + 4, + 5, + 1, +] +`) + expect(c).toMatchInlineSnapshot(` +Array [ + 3, + 4, + 2, + 1, + 5, +] +`) }) }) diff --git a/src/utils/random.ts b/src/utils/random.ts index 8a9edced..1eb44aa9 100644 --- a/src/utils/random.ts +++ b/src/utils/random.ts @@ -5,7 +5,7 @@ const random = (seed: string) => export const shuffle = (a: T[], seed: string) => { return a - .map((v) => ({ v, r: random(seed + v) })) + .map((v, i) => ({ v, r: random(seed + `${i}`) })) .sort((a, b) => a.r - b.r) .map(({ v }) => v) } From ff9cc527375ff0ba732b7ae7596f7ef9df9b8e44 Mon Sep 17 00:00:00 2001 From: vipzero Date: Fri, 6 May 2022 18:51:50 +0900 Subject: [PATCH 025/105] add doc --- readme.md | 60 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/readme.md b/readme.md index e27bfc65..bee1579b 100644 --- a/readme.md +++ b/readme.md @@ -68,3 +68,63 @@ imagemin などで Optimize 後のメタデータで コード https://github.com/vipzero/haikei-server/blob/main/src/imageIo/uploadManage.ts + +## フロー + +### Event 作成 + +```mermaid + +sequenceDiagram + +participant you_ as You +participant envB as BackEnd環境設定 +participant appB as haikei-server +participant envF as FronEnd環境設定 +participant fbdb as Firestore +participant andb as AnisonDb + +you_ ->> envB : .envrc EVENT_ID 更新 +you_ ->> envF : .envrc, src/config 更新 +you_ ->> appB : yarn setup +appB ->> fbdb : event 作成 +you_ ->> appB : yarn setup:anison +appB ->> andb : get +andb ->> envB : 同期 +``` + +FrontEnd のセットアップ + +- rekka-haikei の `src/config/incdex.ts` +- 開始時間・終了時間 +- (必要であれば) `Address.tsx` のリンク・`firebase.json` のリダイレクト + +### 動作中 + +```mermaid + +sequenceDiagram + +participant you_ as You +participant appB as haikei-server +participant fbhi as Firestore/hist +participant fbso as Firestore/song +participant fbco as Firestore/counts +participant strm as Stream + +you_ ->> appB : yarn start +appB ->> fbhi : get [eventId] +fbhi ->> appB : lastSong +appB ->> fbco : get [eventId] +fbco ->> appB : all cache + +appB ->> strm : subscribe + +loop update +strm ->> appB : icy +appB ->> appB : 解析 +appB ->> fbhi : update +appB ->> fbso : push +appB ->> fbco : update +end +``` From 9251e33d059b8e7a3b28a5a53a1401625b2b6950 Mon Sep 17 00:00:00 2001 From: vipzero Date: Fri, 6 May 2022 21:00:38 +0900 Subject: [PATCH 026/105] add words --- src/utils/makeSearchWord.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/utils/makeSearchWord.ts b/src/utils/makeSearchWord.ts index ff517ea3..f6dd5e17 100644 --- a/src/utils/makeSearchWord.ts +++ b/src/utils/makeSearchWord.ts @@ -18,6 +18,12 @@ const animeExt = [ '名シーン', '作画', 'wallpaper', + '敵', + 'ボス', + 'ヒット', + '似てる', + '2話', + 'ゲーム', ] /* アニメカテゴリ場合は↓を検索クエリに追加する */ const animeExtBase = ['アニメ'].join(' OR ') @@ -32,7 +38,7 @@ export function makeSearchQuery(song: SongSeed, seed: number): string { const { icy, category, animeTitle } = song if (getSyncConf().simpleSearch) return icy.replace(/-/g, ' ') const r = shuffle(animeExt, `${seed}`) - const opts = [r[0], r[1], r[2]] + const opts = [r[0], r[1], r[2], r[3]] if (!animeTitle) { const charaName = pickCharaIcy(icy).join(' ') From d1017fa6b2abde25b4177362fb3377dde93845b2 Mon Sep 17 00:00:00 2001 From: vipzero Date: Sat, 7 May 2022 20:05:34 +0900 Subject: [PATCH 027/105] add ep word to search --- src/utils/makeSearchWord.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/utils/makeSearchWord.ts b/src/utils/makeSearchWord.ts index f6dd5e17..e373fc35 100644 --- a/src/utils/makeSearchWord.ts +++ b/src/utils/makeSearchWord.ts @@ -22,9 +22,10 @@ const animeExt = [ 'ボス', 'ヒット', '似てる', - '2話', 'ゲーム', ] +const range = (n: number) => [...Array(n).keys()] +const animeExtEp = [...[...range(8)].map((i) => `${i}話`), '映画'] /* アニメカテゴリ場合は↓を検索クエリに追加する */ const animeExtBase = ['アニメ'].join(' OR ') @@ -38,7 +39,8 @@ export function makeSearchQuery(song: SongSeed, seed: number): string { const { icy, category, animeTitle } = song if (getSyncConf().simpleSearch) return icy.replace(/-/g, ' ') const r = shuffle(animeExt, `${seed}`) - const opts = [r[0], r[1], r[2], r[3]] + const rEp = shuffle(animeExtEp, `${seed}`) + const opts = [r[0], r[1], r[2], rEp[0]] if (!animeTitle) { const charaName = pickCharaIcy(icy).join(' ') From 1119ec5f4d7bc765615c49732ed71fe20107e594 Mon Sep 17 00:00:00 2001 From: vipzero Date: Sun, 8 May 2022 17:36:34 +0900 Subject: [PATCH 028/105] update normalize --- src/tests/normalize.test.ts | 3 +++ src/utils/index.ts | 12 ++++++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/tests/normalize.test.ts b/src/tests/normalize.test.ts index c2b127f0..fde30b9d 100644 --- a/src/tests/normalize.test.ts +++ b/src/tests/normalize.test.ts @@ -5,4 +5,7 @@ test('textNormalize', () => { const wided = textNormalize('ＴEｓtﾃｽトT　！？') expect(wided).toMatchInlineSnapshot(`"testテストt !?"`) + + const multiple = textNormalize('〜スヤスヤ生活〜') + expect(multiple).toMatchInlineSnapshot(`"~スヤスヤ生活~"`) }) diff --git a/src/utils/index.ts b/src/utils/index.ts index a74f72d5..2ab39e00 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -16,12 +16,12 @@ export function textNormalize(s: string) { s .trim() .toLowerCase() - .replace('　', ' ') - .replace(/[（【「[]/, '(') - .replace(/[）】」\]]/, ')') - .replace('〜', '~') - .replace('！', '!') - .replace('？', '?') + .replace(/　/g, ' ') + .replace(/[（【「[]/g, '(') + .replace(/[）】」\]]/g, ')') + .replace(/〜/g, '~') + .replace(/！/g, '!') + .replace(/？/g, '?') ) ) } From 3b9f7fdcad8e2531afd2a26f7220f9e49436abd7 Mon Sep 17 00:00:00 2001 From: vipzero Date: Sun, 8 May 2022 17:36:53 +0900 Subject: [PATCH 029/105] update snap --- src/tests/makeSearchQuery.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/makeSearchQuery.test.ts b/src/tests/makeSearchQuery.test.ts index 9377afbf..b727a089 100644 --- a/src/tests/makeSearchQuery.test.ts +++ b/src/tests/makeSearchQuery.test.ts @@ -13,7 +13,7 @@ makeSearchQuery( seed)). -toMatchInlineSnapshot(`"(蒼井翔太 OR give me ? me) ネタ画像 OR 壁紙 OR 作画"`) +toMatchInlineSnapshot(`"(蒼井翔太 OR give me ? me) ネタ画像 OR 壁紙 OR 作画 OR 5話"`) }) test('lost animeTitle and detect chara', () => { expect( @@ -50,7 +50,7 @@ makeSearchQuery( seed)). -toMatchInlineSnapshot(`"アニメタイトルあり AND (アニメ OR ネタ画像 OR 壁紙 OR 作画)"`) +toMatchInlineSnapshot(`"アニメタイトルあり AND (アニメ OR ネタ画像 OR 壁紙 OR 作画 OR 5話)"`) }) test('has animeTitle and category game', () => { expect( From c0e1fa957d5810e97132498d0ced7ab2bd70a376 Mon Sep 17 00:00:00 2001 From: vipzero Date: Wed, 8 Jun 2022 23:21:19 +0900 Subject: [PATCH 030/105] add ee --- scripts/setupEasterEgg.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/setupEasterEgg.ts b/scripts/setupEasterEgg.ts index eee3fc15..1621b9c3 100644 --- a/scripts/setupEasterEgg.ts +++ b/scripts/setupEasterEgg.ts @@ -56,6 +56,7 @@ async function main() { rakupro: { mm: 0, si: 0, kt: 0, ym: 0, mk: 0, mo: 0, nn: 0, rr: 0, rt: 0, ma: 0, rn: 0, sz: 0, }, // prettier-ignore shining: { hn: 0, ss: 0, mr: 0, hy: 0, gs: 0, rr: 0, kk: 0, uo: 0, ms: 0, hr: 0, er: 0, te: 0, }, + aobuta: { mi: 0, sk: 0, tm: 0, ft: 0, nd: 0, ke: 0 }, } const tasks = Object.entries(votes).map(async ([animeId, inits]) => { From 8fac032a6e705444931aee45dafacf1bb1c032a7 Mon Sep 17 00:00:00 2001 From: vipzero Date: Fri, 5 Aug 2022 20:22:48 +0900 Subject: [PATCH 031/105] add ee --- scripts/setupEasterEgg.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/setupEasterEgg.ts b/scripts/setupEasterEgg.ts index 1621b9c3..ba16bc26 100644 --- a/scripts/setupEasterEgg.ts +++ b/scripts/setupEasterEgg.ts @@ -53,6 +53,8 @@ async function main() { // prettier-ignore imasmlan: { hn: 0, sr: 0, an: 0, tm: 0, ri: 0, el: 0, ak: 0, kn: 0, tb: 0, kl: 0, hk: 0, my: 0, ko: 0, }, // prettier-ignore + imasmlau: { ah: 0, ch: 0, yh: 0, yy: 0, rt: 0, az: 0, io: 0, ma: 0, am: 0, mm: 0, mi: 0, hb: 0, tn: 0, }, + // prettier-ignore rakupro: { mm: 0, si: 0, kt: 0, ym: 0, mk: 0, mo: 0, nn: 0, rr: 0, rt: 0, ma: 0, rn: 0, sz: 0, }, // prettier-ignore shining: { hn: 0, ss: 0, mr: 0, hy: 0, gs: 0, rr: 0, kk: 0, uo: 0, ms: 0, hr: 0, er: 0, te: 0, }, From ba5c10da3253771dbbcef668866300aedc3adc54 Mon Sep 17 00:00:00 2001 From: vipzero Date: Fri, 5 Aug 2022 22:04:20 +0900 Subject: [PATCH 032/105] update ee --- scripts/setupEasterEgg.ts | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/scripts/setupEasterEgg.ts b/scripts/setupEasterEgg.ts index ba16bc26..b582ef43 100644 --- a/scripts/setupEasterEgg.ts +++ b/scripts/setupEasterEgg.ts @@ -47,13 +47,11 @@ async function main() { yamajo: { ur: 0, nn: 0, mi: 0, mr: 0, na: 0, as: 0, si: 0 }, bryunhild: { nk: 0, kn: 0, kz: 0, kt: 0, nn: 0, ht: 0, mt: 0 }, // prettier-ignore - imasmlpr: { mr: 0, sy: 0, kt: 0, ar: 0, kn: 0, em: 0, um: 0, mn: 0, nr: 0, no: 0, mt: 0, ik: 0, yr: 0, }, - // prettier-ignore - imasmlfa: { jr: 0, tm: 0, sz: 0, sb: 0, ay: 0, mz: 0, ro: 0, sh: 0, mm: 0, em: 0, cz: 0, rk: 0, tg: 0, }, - // prettier-ignore - imasmlan: { hn: 0, sr: 0, an: 0, tm: 0, ri: 0, el: 0, ak: 0, kn: 0, tb: 0, kl: 0, hk: 0, my: 0, ko: 0, }, - // prettier-ignore - imasmlau: { ah: 0, ch: 0, yh: 0, yy: 0, rt: 0, az: 0, io: 0, ma: 0, am: 0, mm: 0, mi: 0, hb: 0, tn: 0, }, + imasml: { + mr: 0, sy: 0, kt: 0, ar: 0, km: 0, er: 0, um: 0, mn: 0, nr: 0, no: 0, mt: 0, ik: 0, yr: 0, + jr: 0, tk: 0, sz: 0, sb: 0, ay: 0, mz: 0, ro: 0, sh: 0, mk: 0, em: 0, cz: 0, rk: 0, tg: 0, + hn: 0, sr: 0, an: 0, tm: 0, ri: 0, el: 0, ak: 0, kn: 0, tb: 0, kl: 0, hk: 0, my: 0, ko: 0, + ah: 0, ch: 0, yh: 0, yy: 0, rt: 0, az: 0, io: 0, ma: 0, am: 0, mm: 0, mi: 0, hb: 0, tn: 0, }, // prettier-ignore rakupro: { mm: 0, si: 0, kt: 0, ym: 0, mk: 0, mo: 0, nn: 0, rr: 0, rt: 0, ma: 0, rn: 0, sz: 0, }, // prettier-ignore From 0bc42a07aab89299d7057eadf985f61416e3ca63 Mon Sep 17 00:00:00 2001 From: vipzero Date: Mon, 8 Aug 2022 16:01:50 +0900 Subject: [PATCH 033/105] add emol --- package.json | 1 + src/index.ts | 19 +++++++++------ src/service/firebase.ts | 14 +++++++---- src/service/jlyricnet.ts | 2 +- src/types/index.ts | 1 + yarn.lock | 51 ++++++++++++++++++++++++++++++++++++++-- 6 files changed, 74 insertions(+), 14 deletions(-) diff --git a/package.json b/package.json index 6583f670..e28bb9c6 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "icy": "2.1.0", "jaco": "^4.0.0", "jest": "^27.5.1", + "jptext-to-emoji": "^0.2.1", "node-spotify-api": "1.1.1", "sharp": "0.28.3" }, diff --git a/src/index.ts b/src/index.ts index cf60f555..86b71b46 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,3 +1,4 @@ +import { emojify } from 'jptext-to-emoji' import { findSong } from './anisonDb/findSong' import { uploadByUrlAll } from './imageIo/uploadManage' import { getImageLinks } from './service/customImageSearch' @@ -13,7 +14,7 @@ import { getLyricsSafe } from './service/jlyricnet' // import { spotifySearchSongInfo } from './spotify' import { Store, store } from './state/store' import subscribeIcy from './streaming/icy' -import { Song } from './types/index' +import { Emol, Song } from './types/index' import { sleep } from './utils' import { error, info, log, songPrint } from './utils/logger' import { makeSearchQuery } from './utils/makeSearchWord' @@ -40,7 +41,7 @@ export async function icyToSong( icy: string, time: number, store: Store -): Promise { +): Promise<[Song, Emol] | false> { info(icy) if (store.isDuplicate(icy)) return false // 起動時の重複登録を防ぐ @@ -61,11 +62,14 @@ export async function icyToSong( const albumInfosSync = getAlbum(icy) const lyricsSync = getLyricsSafe(song.title, song.artist) - const [imageLinks, albumInfos, { creators }] = await Promise.all([ + const [imageLinks, albumInfos, { creators, lyric }] = await Promise.all([ imageLinksSync, albumInfosSync, lyricsSync, ]) + const emol = { + text: lyric === null ? '' : await emojify(lyric, { onlyEmoji: true }), + } const compSong: Song = { ...song, imageLinks, @@ -75,7 +79,7 @@ export async function icyToSong( time, imageSearchWord, } - return compSong + return [compSong, emol] } async function receiveIcy(icy: string) { @@ -85,10 +89,11 @@ async function receiveIcy(icy: string) { log(performance.getEntriesByName('first')[0]) } - const song = await icyToSong(icy, Date.now(), store) - if (!song) return + const res = await icyToSong(icy, Date.now(), store) + if (!res) return + const [song, emol] = res songPrint(song) - saveMusic(song) + saveMusic(song, emol) } performance.mark('s1') diff --git a/src/service/firebase.ts b/src/service/firebase.ts index 8b000e56..4e5a227b 100644 --- a/src/service/firebase.ts +++ b/src/service/firebase.ts @@ -1,5 +1,5 @@ import admin from 'firebase-admin' -import { Count, Counts, HistoryRaw, HistTop, Song } from '../types/index' +import { Count, Counts, Emol, HistoryRaw, HistTop, Song } from '../types/index' import { chunk } from '../utils' import { error, info, log, warn } from '../utils/logger' import { CacheFile } from './../types/index' @@ -15,6 +15,8 @@ export { admin } // const P_CVOTE = 'cvote' const P_SONGS = 'songs' +const P_SONG = 'song' +const P_EMOL = 'emol' const P_HIST = 'hist' const P_YO = 'yo' const P_CURRENT = 'current' @@ -49,25 +51,29 @@ export const init = async () => { } export const getCurrentPlay = async () => { - const res = await fdb.collection('song').doc(EVENT_ID).get() + const res = await fdb.collection(P_SONG).doc(EVENT_ID).get() return res.data() || { icy: '' } } const saveSong = (song: Song) => { fdb - .collection('song') + .collection(P_SONG) .doc(EVENT_ID) .set({ ...removeUndefined(song), }) } +const saveEmol = (emol: Emol) => { + fdb.collection(P_EMOL).doc(EVENT_ID).set(emol) +} // const saveLyric = (text) => { // // console.log(lyric) // fdb.collection('song').doc('lyric').set({ text }) // } -export const saveMusic = (song: Song) => { +export const saveMusic = (song: Song, emol: Emol) => { saveSong(song) + saveEmol(emol) // if (lyric) { // saveLyric(lyric) // } else { diff --git a/src/service/jlyricnet.ts b/src/service/jlyricnet.ts index 2fe6498d..811c8aca 100644 --- a/src/service/jlyricnet.ts +++ b/src/service/jlyricnet.ts @@ -55,7 +55,7 @@ function scrapeLyrics(html: string) { export async function getLyricsSafe(title?: string, artist?: string) { const res = await getLyrics(title, artist) - if (!res) return { creators: {}, lyrics: null } + if (!res) return { creators: {}, lyric: null } return res } export async function getLyrics(title?: string, artist?: string) { diff --git a/src/types/index.ts b/src/types/index.ts index ee8cf650..a18d2d4e 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -98,3 +98,4 @@ export type CacheFile = { width: number height: number } +export type Emol = { text: string } diff --git a/yarn.lock b/yarn.lock index 3b439e8c..63f27e70 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1668,6 +1668,13 @@ async-retry@^1.3.1: dependencies: retry "0.12.0" +async@^2.0.1: + version "2.6.4" + resolved "https://registry.yarnpkg.com/async/-/async-2.6.4.tgz#706b7ff6084664cd7eae713f6f965433b5504221" + integrity sha512-mzo5dfJYwAn29PeiJ0zvwTo04zj8HDJj0Mn8TD7sno7q12prdbnasKJHhkm2c1LgrhlJ0teaea8860oxi51mGA== + dependencies: + lodash "^4.17.14" + asynckit@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" @@ -2619,6 +2626,11 @@ dot-prop@^5.2.0: dependencies: is-obj "^2.0.0" +doublearray@0.0.2: + version "0.0.2" + resolved "https://registry.yarnpkg.com/doublearray/-/doublearray-0.0.2.tgz#63186fe8d34413276d3621f6aa0ec5f79e227ef9" + integrity sha512-aw55FtZzT6AmiamEj2kvmR6BuFqvYgKZUkfQ7teqVRNqD5UE0rw8IeW/3gieHNKQ5sPuDKlljWEn4bzv5+1bHw== + download@^6.2.2: version "6.2.5" resolved "https://registry.yarnpkg.com/download/-/download-6.2.5.tgz#acd6a542e4cd0bb42ca70cfc98c9e43b07039714" @@ -4238,7 +4250,7 @@ isurl@^1.0.0-alpha5: has-to-string-tag-x "^1.2.0" is-object "^1.0.1" -jaco@^4.0.0: +jaco@4.0.0, jaco@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/jaco/-/jaco-4.0.0.tgz#7522a3dd4d7295b3e33b0ae82c08b34db99738e5" integrity sha512-RBtY32bY4ICqgPVTmWOLoxn3rZQ+sFPtkcmCA2HJY7ZCCKkCMoMhv/zw0G+eGCzIb7Vkpx5dqSu4cwuS6c6l1w== @@ -4657,6 +4669,14 @@ jose@^2.0.5: dependencies: "@panva/asn1.js" "^1.0.0" +jptext-to-emoji@^0.2.1: + version "0.2.1" + resolved "https://registry.yarnpkg.com/jptext-to-emoji/-/jptext-to-emoji-0.2.1.tgz#f6f26d2f8faf3ef6e214b8815e576abebfba2831" + integrity sha512-694E81Rg8d/3gDsJ2ZGLLPEJFNzo2985tdT+uxQbOmF/H9xrTA2SfTCnVRIzFXDtTOr9zkrPTzxW142Pw0gNWA== + dependencies: + jaco "4.0.0" + kuromojin "3.0.0" + js-tokens@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499" @@ -4874,6 +4894,23 @@ kleur@^3.0.3: resolved "https://registry.yarnpkg.com/kleur/-/kleur-3.0.3.tgz#a79c9ecc86ee1ce3fa6206d1216c501f147fc07e" integrity sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w== +kuromoji@0.1.2: + version "0.1.2" + resolved "https://registry.yarnpkg.com/kuromoji/-/kuromoji-0.1.2.tgz#293f0d6706df006112137980588d5daac26d0790" + integrity sha512-V0dUf+C2LpcPEXhoHLMAop/bOht16Dyr+mDiIE39yX3vqau7p80De/koFqpiTcL1zzdZlc3xuHZ8u5gjYRfFaQ== + dependencies: + async "^2.0.1" + doublearray "0.0.2" + zlibjs "^0.3.1" + +kuromojin@3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/kuromojin/-/kuromojin-3.0.0.tgz#54a1a6643110f49f741c4beb82fef400d1cd498b" + integrity sha512-3h3qnn/NVVhqoKFP4oc7e6apO2B01Atc056oiVlIY7Uoup4rhrnBe28g3y9lK1HTmLDQEejvXB+3I3qxAneF7A== + dependencies: + kuromoji "0.1.2" + lru_map "^0.4.1" + leven@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/leven/-/leven-3.1.0.tgz#77891de834064cccba82ae7842bb6b14a13ed7f2" @@ -4973,7 +5010,7 @@ lodash.once@^4.0.0: resolved "https://registry.yarnpkg.com/lodash.once/-/lodash.once-4.1.1.tgz#0dd3971213c7c56df880977d504c88fb471a97ac" integrity sha1-DdOXEhPHxW34gJd9UEyI+0cal6w= -lodash@^4.17.19, lodash@^4.17.20, lodash@^4.7.0: +lodash@^4.17.14, lodash@^4.17.19, lodash@^4.17.20, lodash@^4.7.0: version "4.17.21" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== @@ -5060,6 +5097,11 @@ lru-memoizer@^2.1.4: lodash.clonedeep "^4.5.0" lru-cache "~4.0.0" +lru_map@^0.4.1: + version "0.4.1" + resolved "https://registry.yarnpkg.com/lru_map/-/lru_map-0.4.1.tgz#f7b4046283c79fb7370c36f8fca6aee4324b0a98" + integrity sha512-I+lBvqMMFfqaV8CJCISjI3wbjmwVu/VyOoU7+qtu9d7ioW5klMgsTTiUOUp+DJvfTTzKXoPbyC6YfgkNcyPSOg== + make-dir@^1.0.0, make-dir@^1.2.0: version "1.3.0" resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-1.3.0.tgz#79c1033b80515bd6d24ec9933e860ca75ee27f0c" @@ -7147,3 +7189,8 @@ yocto-queue@^0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-0.1.0.tgz#0294eb3dee05028d31ee1a5fa2c556a6aaf10a1b" integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q== + +zlibjs@^0.3.1: + version "0.3.1" + resolved "https://registry.yarnpkg.com/zlibjs/-/zlibjs-0.3.1.tgz#50197edb28a1c42ca659cc8b4e6a9ddd6d444554" + integrity sha512-+J9RrgTKOmlxFSDHo0pI1xM6BLVUv+o0ZT9ANtCxGkjIVCCUdx9alUF8Gm+dGLKbkkkidWIHFDZHDMpfITt4+w== From efe0070b0fc88e2329d1c6ae4b7858a8939975a7 Mon Sep 17 00:00:00 2001 From: vipzero Date: Mon, 8 Aug 2022 16:55:44 +0900 Subject: [PATCH 034/105] fix emol service --- package.json | 2 +- scripts/playground.ts | 16 +++++++++++----- src/index.ts | 6 ++---- src/service/emol.ts | 11 +++++++++++ yarn.lock | 8 ++++---- 5 files changed, 29 insertions(+), 14 deletions(-) create mode 100644 src/service/emol.ts diff --git a/package.json b/package.json index e28bb9c6..d30d7397 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "icy": "2.1.0", "jaco": "^4.0.0", "jest": "^27.5.1", - "jptext-to-emoji": "^0.2.1", + "jptext-to-emoji": "0.2.5", "node-spotify-api": "1.1.1", "sharp": "0.28.3" }, diff --git a/scripts/playground.ts b/scripts/playground.ts index 0c5b85bf..28c431b1 100644 --- a/scripts/playground.ts +++ b/scripts/playground.ts @@ -78,9 +78,15 @@ // import { getSyncConf } from './../src/utils/syncConf' // console.log(getSyncConf()) -import { makeSearchQuery } from '../src/utils/makeSearchWord' +// import { makeSearchQuery } from '../src/utils/makeSearchWord' +// +// const song = { +// icy: '愛美 - LIVE for LIFE ～狼たちの夜～', +// } +// console.log(makeSearchQuery(song, Math.random())) + +import { emojify } from 'jptext-to-emoji' -const song = { - icy: '愛美 - LIVE for LIFE ～狼たちの夜～', -} -console.log(makeSearchQuery(song, Math.random())) +const text = + '水面が揺らぐ\n風の輪が拡がる\n触れ合った指先の\n青い電流\n\n見つめあうだけで\n孤独な加速度が\n一瞬に砕け散る\nあなたが好きよ\n\n透明な真珠のように\n宙に浮く涙\n悲劇だってかまわない\nあなたと生きたい\n\nキラッ!\n流星にまたがって\nあなたに急降下　ah ah\n濃紺の星空に\n私たち花火みたい\n心が光の矢を放つ\n\n会話などなしに\n内側に潜って\n考えが読み取れる\n不思議な夜\n\nあなたの名　呪文みたいに\n無限のリピート\n憎らしくて手の甲に\n爪をたててみる\n\nキラッ!\n身体ごと透き通り\n絵のように漂う　uh uh\nけし粒の生命でも\n私たち瞬いてる\n魂に銀河　雪崩れてく\n\n流星にまたがって\nあなたは急上昇　oh oh\n濃紺の星空に\n私たち花火みたい\n心が光の矢を放つ\n\nけし粒の生命でも\n私たち瞬いてる\n魂に銀河　雪崩れてく\n魂に銀河　雪崩れてく' +emojify(text, { onlyEmoji: true }).then(console.log) diff --git a/src/index.ts b/src/index.ts index 86b71b46..7e6dd5f4 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,7 +1,7 @@ -import { emojify } from 'jptext-to-emoji' import { findSong } from './anisonDb/findSong' import { uploadByUrlAll } from './imageIo/uploadManage' import { getImageLinks } from './service/customImageSearch' +import { makeEmol } from './service/emol' import { addHistory, deleteFile, @@ -67,9 +67,7 @@ export async function icyToSong( albumInfosSync, lyricsSync, ]) - const emol = { - text: lyric === null ? '' : await emojify(lyric, { onlyEmoji: true }), - } + const emol = await makeEmol(lyric) const compSong: Song = { ...song, imageLinks, diff --git a/src/service/emol.ts b/src/service/emol.ts new file mode 100644 index 00000000..5abfd2aa --- /dev/null +++ b/src/service/emol.ts @@ -0,0 +1,11 @@ +import { emojify } from 'jptext-to-emoji' + +export const makeEmol = async (lyric: string | null) => { + if (lyric === null) return { text: '' } + const text = ( + await Promise.all( + lyric.split('\n').map((line) => (line ? emojify(line) : '')) + ) + ).join('\n') + return { text } +} diff --git a/yarn.lock b/yarn.lock index 63f27e70..ab6fa3e8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4669,10 +4669,10 @@ jose@^2.0.5: dependencies: "@panva/asn1.js" "^1.0.0" -jptext-to-emoji@^0.2.1: - version "0.2.1" - resolved "https://registry.yarnpkg.com/jptext-to-emoji/-/jptext-to-emoji-0.2.1.tgz#f6f26d2f8faf3ef6e214b8815e576abebfba2831" - integrity sha512-694E81Rg8d/3gDsJ2ZGLLPEJFNzo2985tdT+uxQbOmF/H9xrTA2SfTCnVRIzFXDtTOr9zkrPTzxW142Pw0gNWA== +jptext-to-emoji@0.2.5: + version "0.2.5" + resolved "https://registry.yarnpkg.com/jptext-to-emoji/-/jptext-to-emoji-0.2.5.tgz#079177b2de15b09451e033a0214fa63114c2b916" + integrity sha512-t8oPc+x5Sz4Utius5tkdf4SBG7dY3oyWEfSLYUMdemrtTQun0ZLafAJp7XPTaU7rzGPb0Hh6iji1rhu5U0GexA== dependencies: jaco "4.0.0" kuromojin "3.0.0" From a1e322c74c8fd537787a2ff6941a1f063f32ae86 Mon Sep 17 00:00:00 2001 From: vipzero Date: Mon, 8 Aug 2022 17:00:49 +0900 Subject: [PATCH 035/105] fix emol text --- src/service/emol.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/service/emol.ts b/src/service/emol.ts index 5abfd2aa..c93f0773 100644 --- a/src/service/emol.ts +++ b/src/service/emol.ts @@ -4,8 +4,12 @@ export const makeEmol = async (lyric: string | null) => { if (lyric === null) return { text: '' } const text = ( await Promise.all( - lyric.split('\n').map((line) => (line ? emojify(line) : '')) + lyric + .split('\n') + .map((line) => (line ? emojify(line, { onlyEmoji: true }) : '')) ) - ).join('\n') + ) + .filter(Boolean) + .join('\n') return { text } } From 6e543845ba874630b27e32e6906c524bb6fb0a74 Mon Sep 17 00:00:00 2001 From: vipzero Date: Mon, 8 Aug 2022 18:11:14 +0900 Subject: [PATCH 036/105] update --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index d30d7397..4e4f8d6d 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "icy": "2.1.0", "jaco": "^4.0.0", "jest": "^27.5.1", - "jptext-to-emoji": "0.2.5", + "jptext-to-emoji": "0.2.6", "node-spotify-api": "1.1.1", "sharp": "0.28.3" }, diff --git a/yarn.lock b/yarn.lock index ab6fa3e8..44a2d815 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4669,10 +4669,10 @@ jose@^2.0.5: dependencies: "@panva/asn1.js" "^1.0.0" -jptext-to-emoji@0.2.5: - version "0.2.5" - resolved "https://registry.yarnpkg.com/jptext-to-emoji/-/jptext-to-emoji-0.2.5.tgz#079177b2de15b09451e033a0214fa63114c2b916" - integrity sha512-t8oPc+x5Sz4Utius5tkdf4SBG7dY3oyWEfSLYUMdemrtTQun0ZLafAJp7XPTaU7rzGPb0Hh6iji1rhu5U0GexA== +jptext-to-emoji@0.2.6: + version "0.2.6" + resolved "https://registry.yarnpkg.com/jptext-to-emoji/-/jptext-to-emoji-0.2.6.tgz#8da8908f080836a88f85b2b3b197ccb03f51d34a" + integrity sha512-mzRKebmM2gRBRAFIbBciaoVcwg/u0bEB8XUIO8O7L8+ka/Lo+JElROjeEXd6rtqiYj2Axtp67MNuXOwdH50W9Q== dependencies: jaco "4.0.0" kuromojin "3.0.0" From a77797578b3811def80133423ac5c3946e9aa836 Mon Sep 17 00:00:00 2001 From: vipzero Date: Thu, 11 Aug 2022 12:38:49 +0900 Subject: [PATCH 037/105] add filter --- scripts/loadThreads.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/loadThreads.ts b/scripts/loadThreads.ts index 6ce91841..4cf2b16d 100644 --- a/scripts/loadThreads.ts +++ b/scripts/loadThreads.ts @@ -23,7 +23,8 @@ async function main() { if (!data[thread.url]) data[thread.url] = {} - for (const post of thread.posts) { + // const idFilter = (p) => !p.userId.match(/^.{9}a/) + for (const post of thread.posts.filter((p) => p)) { data[thread.url][post.timestamp] = true } sleep(1000) From 57ff7163d5de51ef69b484176778a02fae719a8e Mon Sep 17 00:00:00 2001 From: vipzero Date: Thu, 11 Aug 2022 18:43:22 +0900 Subject: [PATCH 038/105] add keep phatse --- scripts/playground.ts | 7 ++++--- src/service/emol.ts | 12 +++++++++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/scripts/playground.ts b/scripts/playground.ts index 28c431b1..26e7bef5 100644 --- a/scripts/playground.ts +++ b/scripts/playground.ts @@ -85,8 +85,9 @@ // } // console.log(makeSearchQuery(song, Math.random())) -import { emojify } from 'jptext-to-emoji' +import { makeEmol } from '../src/service/emol' const text = - '水面が揺らぐ\n風の輪が拡がる\n触れ合った指先の\n青い電流\n\n見つめあうだけで\n孤独な加速度が\n一瞬に砕け散る\nあなたが好きよ\n\n透明な真珠のように\n宙に浮く涙\n悲劇だってかまわない\nあなたと生きたい\n\nキラッ!\n流星にまたがって\nあなたに急降下　ah ah\n濃紺の星空に\n私たち花火みたい\n心が光の矢を放つ\n\n会話などなしに\n内側に潜って\n考えが読み取れる\n不思議な夜\n\nあなたの名　呪文みたいに\n無限のリピート\n憎らしくて手の甲に\n爪をたててみる\n\nキラッ!\n身体ごと透き通り\n絵のように漂う　uh uh\nけし粒の生命でも\n私たち瞬いてる\n魂に銀河　雪崩れてく\n\n流星にまたがって\nあなたは急上昇　oh oh\n濃紺の星空に\n私たち花火みたい\n心が光の矢を放つ\n\nけし粒の生命でも\n私たち瞬いてる\n魂に銀河　雪崩れてく\n魂に銀河　雪崩れてく' -emojify(text, { onlyEmoji: true }).then(console.log) + '水面が 揺らぐ\n風の輪が 拡がる\n触れ合った指先の\n青い電流\n\n見つめあうだけで\n孤独な加速度が\n一瞬に砕け散る\nあなたが好きよ\n\n透明な真珠のように\n宙に浮く涙\n悲劇だってかまわない\nあなたと生きたい\n\nキラッ!\n流星にまたがって\nあなたに急降下　ah ah\n濃紺の星空に\n私たち花火みたい\n心が光の矢を放つ\n\n会話などなしに\n内側に潜って\n考えが読み取れる\n不思議な夜\n\nあなたの名　呪文みたいに\n無限のリピート\n憎らしくて手の甲に\n爪をたててみる\n\nキラッ!\n身体ごと透き通り\n絵のように漂う　uh uh\nけし粒の生命でも\n私たち瞬いてる\n魂に銀河　雪崩れてく\n\n流星にまたがって\nあなたは急上昇　oh oh\n濃紺の星空に\n私たち花火みたい\n心が光の矢を放つ\n\nけし粒の生命でも\n私たち瞬いてる\n魂に銀河　雪崩れてく\n魂に銀河　雪崩れてく' +// emojify(text, { onlyEmoji: true }).then(console.log) +makeEmol(text).then(console.log) diff --git a/src/service/emol.ts b/src/service/emol.ts index c93f0773..6a766273 100644 --- a/src/service/emol.ts +++ b/src/service/emol.ts @@ -6,7 +6,17 @@ export const makeEmol = async (lyric: string | null) => { await Promise.all( lyric .split('\n') - .map((line) => (line ? emojify(line, { onlyEmoji: true }) : '')) + .map(async (line) => + ( + await Promise.all( + line + .split(/[ \u3000]/) + .map((word) => + word ? emojify(word, { onlyEmoji: false }) : '' + ) + ) + ).join(' ') + ) ) ) .filter(Boolean) From b5554205f17b916254cb061f25d77de059c61b53 Mon Sep 17 00:00:00 2001 From: vipzero Date: Thu, 11 Aug 2022 23:33:39 +0900 Subject: [PATCH 039/105] add wikipedia --- scripts/loadThreads.ts | 7 ++++++- src/service/jlyricnet.ts | 6 +++--- src/utils/wikipedia.ts | 45 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 4 deletions(-) create mode 100644 src/utils/wikipedia.ts diff --git a/scripts/loadThreads.ts b/scripts/loadThreads.ts index 4cf2b16d..066e8dae 100644 --- a/scripts/loadThreads.ts +++ b/scripts/loadThreads.ts @@ -4,6 +4,9 @@ import chch from 'chch' import { sleep } from '../src/utils' import { loadData, saveData } from './postTimeUtil' +const filterNonGreeting = (s: string) => + !(s.includes('>>') && /^おつ|^よろ|^云々/m.exec(s)) + async function main() { const search = process.env.THREAD_TITLE_WORD const currentTreads = search @@ -24,7 +27,9 @@ async function main() { if (!data[thread.url]) data[thread.url] = {} // const idFilter = (p) => !p.userId.match(/^.{9}a/) - for (const post of thread.posts.filter((p) => p)) { + for (const post of thread.posts.filter((v) => + filterNonGreeting(v.message) + )) { data[thread.url][post.timestamp] = true } sleep(1000) diff --git a/src/service/jlyricnet.ts b/src/service/jlyricnet.ts index 811c8aca..e6165f47 100644 --- a/src/service/jlyricnet.ts +++ b/src/service/jlyricnet.ts @@ -1,5 +1,5 @@ import axios from 'axios' -import cheerio from 'cheerio' +import { load } from 'cheerio' function searchJlyrics(title: string, artist: string | null) { const params: Record = { @@ -25,12 +25,12 @@ function searchJlyrics(title: string, artist: string | null) { // } function scrapeFirstResult(html: string) { - const $ = cheerio.load(html) + const $ = load(html) return $('#mnb a').attr('href') } function scrapeLyrics(html: string) { - const $ = cheerio.load(html) + const $ = load(html) $('#Lyric').find('br').replaceWith('\n') $('.lbdy').find('br').replaceWith('\n') diff --git a/src/utils/wikipedia.ts b/src/utils/wikipedia.ts new file mode 100644 index 00000000..eb670283 --- /dev/null +++ b/src/utils/wikipedia.ts @@ -0,0 +1,45 @@ +import axios from 'axios' + +const qp = { + format: 'json', + action: 'query', +} + +const url = 'https://ja.wikipedia.org/w/api.php' +export const makePageUrl = (pageId: string | number) => + `${url}/?${new URLSearchParams({ + ...qp, + prop: 'revisions', + rvprop: 'content', + rvparse: '', + pageids: String(pageId), + }).toString()}` +export const makeSearchUrl = (title: string) => + `${url}/?${new URLSearchParams({ + ...qp, + list: 'search', + srsearch: title, + }).toString()}` + +type SearchResponse = { query: { search: { pageId: number }[] } } + +type PageResponse = { + query: { + pages: { + [pageId: number | string]: { + pageid: number + ns: 0 + title: string + revisions: string[] + } + } + } +} + +export const searchWikipedia = async (title: string) => { + const res = await axios.get(makeSearchUrl(title)) + if (res.data.query.search.length === 0) return null + const { pageId } = res.data.query.search[0] + + await axios.get(makePageUrl(pageId)) +} From 3f4c265de3c84e61ec66e64f81e30a2bb9830b2f Mon Sep 17 00:00:00 2001 From: vipzero Date: Fri, 12 Aug 2022 23:04:38 +0900 Subject: [PATCH 040/105] update wikipedia --- src/utils/wikipedia.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/utils/wikipedia.ts b/src/utils/wikipedia.ts index eb670283..f1c665d8 100644 --- a/src/utils/wikipedia.ts +++ b/src/utils/wikipedia.ts @@ -1,4 +1,5 @@ import axios from 'axios' +import { load } from 'cheerio' const qp = { format: 'json', @@ -30,7 +31,7 @@ type PageResponse = { pageid: number ns: 0 title: string - revisions: string[] + revisions: { '*': string }[] } } } @@ -41,5 +42,9 @@ export const searchWikipedia = async (title: string) => { if (res.data.query.search.length === 0) return null const { pageId } = res.data.query.search[0] - await axios.get(makePageUrl(pageId)) + const res2 = await axios.get(makePageUrl(pageId)) + const html = res2.data.query.pages[pageId].revisions[0]['*'] + + const $ = load(html) + $ } From c9244a64a99ce30d1273b7cb0546386feebae926 Mon Sep 17 00:00:00 2001 From: vipzero Date: Sat, 13 Aug 2022 13:04:26 +0900 Subject: [PATCH 041/105] update --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 4e4f8d6d..20bb59cf 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "icy": "2.1.0", "jaco": "^4.0.0", "jest": "^27.5.1", - "jptext-to-emoji": "0.2.6", + "jptext-to-emoji": "0.3.0", "node-spotify-api": "1.1.1", "sharp": "0.28.3" }, diff --git a/yarn.lock b/yarn.lock index 44a2d815..e8405f87 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4669,10 +4669,10 @@ jose@^2.0.5: dependencies: "@panva/asn1.js" "^1.0.0" -jptext-to-emoji@0.2.6: - version "0.2.6" - resolved "https://registry.yarnpkg.com/jptext-to-emoji/-/jptext-to-emoji-0.2.6.tgz#8da8908f080836a88f85b2b3b197ccb03f51d34a" - integrity sha512-mzRKebmM2gRBRAFIbBciaoVcwg/u0bEB8XUIO8O7L8+ka/Lo+JElROjeEXd6rtqiYj2Axtp67MNuXOwdH50W9Q== +jptext-to-emoji@0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/jptext-to-emoji/-/jptext-to-emoji-0.3.0.tgz#002fa4335c5b200f7b51e423640768e88c7961c7" + integrity sha512-RFwsfR+lm/tXQufRfpXso05pSwEqWctpOlhk0VXzpawpiZmH+j4iS9ERWE+DgOvcbDzRxeARQ9Mv25nNwJbcRQ== dependencies: jaco "4.0.0" kuromojin "3.0.0" From c3d0ef0ec46397b5ddbbbad90f7da9f167d4e511 Mon Sep 17 00:00:00 2001 From: vipzero Date: Wed, 17 Aug 2022 00:16:54 +0900 Subject: [PATCH 042/105] add filtering --- scripts/loadThreads.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/loadThreads.ts b/scripts/loadThreads.ts index 066e8dae..a6b14c4f 100644 --- a/scripts/loadThreads.ts +++ b/scripts/loadThreads.ts @@ -26,9 +26,8 @@ async function main() { if (!data[thread.url]) data[thread.url] = {} - // const idFilter = (p) => !p.userId.match(/^.{9}a/) - for (const post of thread.posts.filter((v) => - filterNonGreeting(v.message) + for (const post of thread.posts.filter((p) => + filterNonGreeting(p.message) )) { data[thread.url][post.timestamp] = true } From 537b3208d249c38fc1d77d09db9e0a8a4dbcb69c Mon Sep 17 00:00:00 2001 From: vipzero Date: Fri, 9 Dec 2022 16:41:47 +0900 Subject: [PATCH 043/105] valid --- scripts/archiveEvent/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/archiveEvent/index.ts b/scripts/archiveEvent/index.ts index 8839ebf5..70f544fc 100644 --- a/scripts/archiveEvent/index.ts +++ b/scripts/archiveEvent/index.ts @@ -11,6 +11,7 @@ import { log } from '../../src/utils/logger' const archiveEventCsv = async (id: string) => { log({ id }) + if (!id) return const data = await loadHistEventSongs(id) const dataWithN = data.map((d) => ({ ...d, n: d.n || 0 })) const csvText = stringify(dataWithN, { From 03ebe0f6d0a04f33d2fc53ffd915addecd9d0497 Mon Sep 17 00:00:00 2001 From: vipzero Date: Wed, 21 Dec 2022 23:24:50 +0900 Subject: [PATCH 044/105] udpate docs --- maintanace-flow.md | 97 +++++++++++++++++++++++++++++++++++++++++++ readme.md | 60 +------------------------- scripts/setupEvent.ts | 11 ++++- 3 files changed, 108 insertions(+), 60 deletions(-) create mode 100644 maintanace-flow.md diff --git a/maintanace-flow.md b/maintanace-flow.md new file mode 100644 index 00000000..01a407cf --- /dev/null +++ b/maintanace-flow.md @@ -0,0 +1,97 @@ +## メンテフロー + +### Event 作成 + +``` +$ vi .envrc EVENT_ID +EVENT_ID=xxxxxx +$ yarn setup # EVENT_ID から動作 +$ yarn setup:anison +``` + +```mermaid +sequenceDiagram + +participant you_ as You +participant envB as BackEnd環境設定 +participant appB as haikei-server +participant fbdb as Firestore +participant andb as AnisonDb + +you_ ->> envB : .envrc EVENT_ID 更新 +you_ ->> appB : yarn setup +envB ->> appB : EVENT_ID +appB ->> fbdb : event 追加 +``` + +```mermaid +sequenceDiagram + +participant you_ as You +participant envB as BackEnd環境設定 data/*.csv +participant appB as haikei-server +participant andb as AnisonDb + +you_ ->> appB : yarn setup:anison +appB ->> andb : get +andb ->> envB : 同期 +``` + +### FrontEnd のセットアップ + +rekka-haikei の `src/config/incdex.ts` + +- イベント追加 `xxxxxx` +- 終了時間指定 + +### サーバー起動 + +```mermaid + +sequenceDiagram + +participant you_ as You +participant appB as haikei-server +participant fbhi as Firestore/hist +participant fbco as Firestore/counts +participant strm as Stream + +you_ ->> appB : yarn start +appB ->> fbhi : get [eventId] +fbhi ->> appB : lastSong +appB ->> fbco : get [eventId] +fbco ->> appB : all cache + +appB ->> strm : subscribe + +``` + +### 動作中 + +```mermaid + +sequenceDiagram + +participant strm as Stream +participant appB as haikei-server +participant fbhi as Firestore/hist +participant fbso as Firestore/song +participant fbco as Firestore/counts + + +strm ->> appB : icy +appB ->> appB : 解析 +appB ->> fbhi : update +appB ->> fbso : push +appB ->> fbco : update + +``` + +### 終了 + +アーカイブする。 +イベント中の履歴は Firestore からロードするが、アーカイブ後は履歴を Cloud Storage からダウンロードするようになる。 + +``` +yarn close:archive # EVENT_ID から動作 +``` diff --git a/readme.md b/readme.md index bee1579b..53ec166c 100644 --- a/readme.md +++ b/readme.md @@ -69,62 +69,6 @@ imagemin などで Optimize 後のメタデータで https://github.com/vipzero/haikei-server/blob/main/src/imageIo/uploadManage.ts -## フロー +## メンテフロー -### Event 作成 - -```mermaid - -sequenceDiagram - -participant you_ as You -participant envB as BackEnd環境設定 -participant appB as haikei-server -participant envF as FronEnd環境設定 -participant fbdb as Firestore -participant andb as AnisonDb - -you_ ->> envB : .envrc EVENT_ID 更新 -you_ ->> envF : .envrc, src/config 更新 -you_ ->> appB : yarn setup -appB ->> fbdb : event 作成 -you_ ->> appB : yarn setup:anison -appB ->> andb : get -andb ->> envB : 同期 -``` - -FrontEnd のセットアップ - -- rekka-haikei の `src/config/incdex.ts` -- 開始時間・終了時間 -- (必要であれば) `Address.tsx` のリンク・`firebase.json` のリダイレクト - -### 動作中 - -```mermaid - -sequenceDiagram - -participant you_ as You -participant appB as haikei-server -participant fbhi as Firestore/hist -participant fbso as Firestore/song -participant fbco as Firestore/counts -participant strm as Stream - -you_ ->> appB : yarn start -appB ->> fbhi : get [eventId] -fbhi ->> appB : lastSong -appB ->> fbco : get [eventId] -fbco ->> appB : all cache - -appB ->> strm : subscribe - -loop update -strm ->> appB : icy -appB ->> appB : 解析 -appB ->> fbhi : update -appB ->> fbso : push -appB ->> fbco : update -end -``` +[./maintanace-flow.md](./maintanace-flow.md) diff --git a/scripts/setupEvent.ts b/scripts/setupEvent.ts index 845bf8fd..621bd3d8 100644 --- a/scripts/setupEvent.ts +++ b/scripts/setupEvent.ts @@ -1,11 +1,18 @@ +/* eslint-disable no-console */ import { fdb } from '../src/service/firebase' const eventId = process.env.EVENT_ID async function main() { - if (!eventId) return + if (!eventId) { + console.warn(`EVENT_ID が設定されてません`) + return + } const hist = await fdb.collection('hist').doc(eventId).get() await fdb.collection('yo').doc('current').set({ bookCount: 0 }) - if (hist.exists) return + if (hist.exists) { + console.warn(`${eventId} は登録済み`) + return + } await fdb.collection('hist').doc(eventId).set({ lasttime: 0 }) From 43b7645a5b0c5a324bd7913a508713bab990d297 Mon Sep 17 00:00:00 2001 From: vipzero Date: Sat, 24 Dec 2022 08:47:41 +0900 Subject: [PATCH 045/105] add restore --- scripts/batch/importHistoryFromCells3.ts | 32 +++++++++++++++++++++ scripts/batch/restoreByRes.ts | 36 ++++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 scripts/batch/importHistoryFromCells3.ts create mode 100644 scripts/batch/restoreByRes.ts diff --git a/scripts/batch/importHistoryFromCells3.ts b/scripts/batch/importHistoryFromCells3.ts new file mode 100644 index 00000000..b52a531e --- /dev/null +++ b/scripts/batch/importHistoryFromCells3.ts @@ -0,0 +1,32 @@ +import { readFileSync } from 'fs' +import { addHistory } from '../../src/service/firebase' +import { sleep } from '../../src/utils' +import { anaCounts } from '../../src/utils/wordCounts' + +const parseLine = (text: string) => { + const [title, artist, time] = text.split(',') + + return { time, title: `${title} - ${artist}` } +} + +const importFile = './data/archive/history_1214.txt' +const text = readFileSync(importFile, 'utf8') + +const lines = text.trim().split('\n') + +async function main() { + // for (let i = 0; i < 3; i++) { + for (let i = 0; i < lines.length; i++) { + const line = lines[i] + const { title, time } = parseLine(line) + + await sleep(200) + + await addHistory(title.trim(), Number(time), 0) + anaCounts(title, {}) + + process.stdout.write('.') + } +} + +main().then(() => console.log('fin')) diff --git a/scripts/batch/restoreByRes.ts b/scripts/batch/restoreByRes.ts new file mode 100644 index 00000000..bf60849b --- /dev/null +++ b/scripts/batch/restoreByRes.ts @@ -0,0 +1,36 @@ +/* eslint-disable no-console */ +import { loadActiveUrls } from './../threadUrls' +import chch from 'chch' +import { Post } from 'chch/dist/types' +import { sleep } from '../../src/utils' + +const filterRes = (p: Post) => + p.name.raw.includes(process.env.TARGET_NAME || '---') + +async function main() { + const search = process.env.THREAD_TITLE_WORD + const currentTreads = search + ? (await chch.getThreads(null)).threads + .filter((t) => t.title.includes(search)) + .map((t) => t.url) + : [] + + const urls = loadActiveUrls() + .map((v) => v.text) + .concat(currentTreads) + console.log(urls) + + for (const url of urls) { + const thread = await chch.getThread(url) + + for (const post of thread.posts.filter(filterRes)) { + const line1 = post.message.split('\n')[0] + if (line1.includes('　')) + console.log(line1.split('　').join(',') + ',' + post.timestamp) + } + sleep(1000) + } +} + +// watch() +main().then(() => console.log('fin')) From 20524462b6d0df24ff12fa67e363565cf0d3c230 Mon Sep 17 00:00:00 2001 From: vipzero Date: Sat, 24 Dec 2022 09:39:04 +0900 Subject: [PATCH 046/105] add normalize case --- scripts/playground.ts | 16 +++++++++++----- src/utils/index.ts | 3 ++- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/scripts/playground.ts b/scripts/playground.ts index 26e7bef5..e7b4492a 100644 --- a/scripts/playground.ts +++ b/scripts/playground.ts @@ -1,5 +1,8 @@ +/* eslint-disable no-irregular-whitespace */ /* eslint-disable no-console */ +import { findSong } from '../src/anisonDb/findSong' + // import { pathQueue, push } from './state/pathQueue' // const { spotifySearchSongInfo } = require('./spotify') // const { getMusixMatch, getMusixLyrics } = require('./musixmatch') @@ -85,9 +88,12 @@ // } // console.log(makeSearchQuery(song, Math.random())) -import { makeEmol } from '../src/service/emol' +// import { makeEmol } from '../src/service/emol' + +// const text = +// '水面が 揺らぐ\n風の輪が 拡がる\n触れ合った指先の\n青い電流\n\n見つめあうだけで\n孤独な加速度が\n一瞬に砕け散る\nあなたが好きよ\n\n透明な真珠のように\n宙に浮く涙\n悲劇だってかまわない\nあなたと生きたい\n\nキラッ!\n流星にまたがって\nあなたに急降下　ah ah\n濃紺の星空に\n私たち花火みたい\n心が光の矢を放つ\n\n会話などなしに\n内側に潜って\n考えが読み取れる\n不思議な夜\n\nあなたの名　呪文みたいに\n無限のリピート\n憎らしくて手の甲に\n爪をたててみる\n\nキラッ!\n身体ごと透き通り\n絵のように漂う　uh uh\nけし粒の生命でも\n私たち瞬いてる\n魂に銀河　雪崩れてく\n\n流星にまたがって\nあなたは急上昇　oh oh\n濃紺の星空に\n私たち花火みたい\n心が光の矢を放つ\n\nけし粒の生命でも\n私たち瞬いてる\n魂に銀河　雪崩れてく\n魂に銀河　雪崩れてく' +// // emojify(text, { onlyEmoji: true }).then(console.log) +// makeEmol(text).then(console.log) -const text = - '水面が 揺らぐ\n風の輪が 拡がる\n触れ合った指先の\n青い電流\n\n見つめあうだけで\n孤独な加速度が\n一瞬に砕け散る\nあなたが好きよ\n\n透明な真珠のように\n宙に浮く涙\n悲劇だってかまわない\nあなたと生きたい\n\nキラッ!\n流星にまたがって\nあなたに急降下　ah ah\n濃紺の星空に\n私たち花火みたい\n心が光の矢を放つ\n\n会話などなしに\n内側に潜って\n考えが読み取れる\n不思議な夜\n\nあなたの名　呪文みたいに\n無限のリピート\n憎らしくて手の甲に\n爪をたててみる\n\nキラッ!\n身体ごと透き通り\n絵のように漂う　uh uh\nけし粒の生命でも\n私たち瞬いてる\n魂に銀河　雪崩れてく\n\n流星にまたがって\nあなたは急上昇　oh oh\n濃紺の星空に\n私たち花火みたい\n心が光の矢を放つ\n\nけし粒の生命でも\n私たち瞬いてる\n魂に銀河　雪崩れてく\n魂に銀河　雪崩れてく' -// emojify(text, { onlyEmoji: true }).then(console.log) -makeEmol(text).then(console.log) +const icy = `READY STEADY GO - L'Arc～en～Ciel` +console.log(findSong(icy)) diff --git a/src/utils/index.ts b/src/utils/index.ts index 2ab39e00..60002f42 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -16,10 +16,11 @@ export function textNormalize(s: string) { s .trim() .toLowerCase() + // eslint-disable-next-line no-irregular-whitespace .replace(/　/g, ' ') .replace(/[（【「[]/g, '(') .replace(/[）】」\]]/g, ')') - .replace(/〜/g, '~') + .replace(/[〜～]/g, '~') .replace(/！/g, '!') .replace(/？/g, '?') ) From d2cb41d50ee8f8d5844f24864a608b14af2f5670 Mon Sep 17 00:00:00 2001 From: vipzero Date: Mon, 26 Dec 2022 15:29:29 +0900 Subject: [PATCH 047/105] update query --- src/tests/makeSearchQuery.test.ts | 57 ++++++++++++++++++------------- src/utils/makeSearchWord.ts | 2 +- 2 files changed, 34 insertions(+), 25 deletions(-) diff --git a/src/tests/makeSearchQuery.test.ts b/src/tests/makeSearchQuery.test.ts index b727a089..4014b05f 100644 --- a/src/tests/makeSearchQuery.test.ts +++ b/src/tests/makeSearchQuery.test.ts @@ -5,25 +5,31 @@ const seed = 0.123456789 describe('makeSearchQuery', () => { test('lost animeTitle', () => { expect( -makeSearchQuery( -{ - artist: '蒼井翔太', - title: 'give me ? me', - icy: '蒼井翔太 - give me ? me' }, - -seed)). + makeSearchQuery( + { + artist: '蒼井翔太', + title: 'give me ? me', + icy: '蒼井翔太 - give me ? me', + }, -toMatchInlineSnapshot(`"(蒼井翔太 OR give me ? me) ネタ画像 OR 壁紙 OR 作画 OR 5話"`) + seed + ) + ).toMatchInlineSnapshot( + `"(蒼井翔太 OR give me ? me) ネタ画像 OR 壁紙 OR 作画 OR 5話"` + ) }) test('lost animeTitle and detect chara', () => { expect( -makeSearchQuery( -{ - icy: 'せーので跳べって言ってんの! - 本城香澄(CV:岩橋由佳)' }, - -seed)). + makeSearchQuery( + { + icy: 'せーので跳べって言ってんの! - 本城香澄(CV:岩橋由佳)', + }, -toMatchInlineSnapshot(`"本城香澄 (名シーン OR キャラ) (キャプ画像 OR 壁紙)"`) + seed + ) + ).toMatchInlineSnapshot( + `"本城香澄 (名シーン OR キャラ) (キャプ画像 OR 壁紙)"` + ) }) test('has animeTitle', () => { expect( @@ -40,17 +46,20 @@ toMatchInlineSnapshot(`"本城香澄 (名シーン OR キャラ) (キャプ画 }) test('has animeTitle and category anime', () => { expect( -makeSearchQuery( -{ - artist: 'artist', - title: 'title', - animeTitle: 'アニメタイトルあり', - category: 'ほにゃららアニメ', - icy: 'artist - title' }, - -seed)). + makeSearchQuery( + { + artist: 'artist', + title: 'title', + animeTitle: 'アニメタイトルあり', + category: 'ほにゃららアニメ', + icy: 'artist - title', + }, -toMatchInlineSnapshot(`"アニメタイトルあり AND (アニメ OR ネタ画像 OR 壁紙 OR 作画 OR 5話)"`) + seed + ) + ).toMatchInlineSnapshot( + `"アニメタイトルあり アニメ AND (ネタ画像 OR 壁紙 OR 作画 OR 5話)"` + ) }) test('has animeTitle and category game', () => { expect( diff --git a/src/utils/makeSearchWord.ts b/src/utils/makeSearchWord.ts index e373fc35..1ad32824 100644 --- a/src/utils/makeSearchWord.ts +++ b/src/utils/makeSearchWord.ts @@ -56,7 +56,7 @@ export function makeSearchQuery(song: SongSeed, seed: number): string { if (!category) return animeTitle if (category.includes('アニメ')) { - return `${animeTitle} AND (${animeExtBase} OR ${opts.join(' OR ')})` + return `${animeTitle} ${animeExtBase} AND (${opts.join(' OR ')})` } if (category.includes('ゲーム')) { return `${animeTitle} AND (${gameExt})` From 5e843d3229210d41d288be393fa5dc0513583b60 Mon Sep 17 00:00:00 2001 From: vipzero Date: Mon, 26 Dec 2022 15:30:53 +0900 Subject: [PATCH 048/105] update query --- src/tests/makeSearchQuery.test.ts | 24 +++++++++++------------- src/utils/makeSearchWord.ts | 2 +- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/tests/makeSearchQuery.test.ts b/src/tests/makeSearchQuery.test.ts index 4014b05f..a3b50333 100644 --- a/src/tests/makeSearchQuery.test.ts +++ b/src/tests/makeSearchQuery.test.ts @@ -46,20 +46,18 @@ describe('makeSearchQuery', () => { }) test('has animeTitle and category anime', () => { expect( - makeSearchQuery( - { - artist: 'artist', - title: 'title', - animeTitle: 'アニメタイトルあり', - category: 'ほにゃららアニメ', - icy: 'artist - title', - }, +makeSearchQuery( +{ + artist: 'artist', + title: 'title', + animeTitle: 'アニメタイトルあり', + category: 'ほにゃららアニメ', + icy: 'artist - title' }, - seed - ) - ).toMatchInlineSnapshot( - `"アニメタイトルあり アニメ AND (ネタ画像 OR 壁紙 OR 作画 OR 5話)"` - ) + +seed)). + +toMatchInlineSnapshot(`"アニメタイトルあり アニメ -公式 AND (ネタ画像 OR 壁紙 OR 作画 OR 5話)"`) }) test('has animeTitle and category game', () => { expect( diff --git a/src/utils/makeSearchWord.ts b/src/utils/makeSearchWord.ts index 1ad32824..ae617e78 100644 --- a/src/utils/makeSearchWord.ts +++ b/src/utils/makeSearchWord.ts @@ -27,7 +27,7 @@ const animeExt = [ const range = (n: number) => [...Array(n).keys()] const animeExtEp = [...[...range(8)].map((i) => `${i}話`), '映画'] /* アニメカテゴリ場合は↓を検索クエリに追加する */ -const animeExtBase = ['アニメ'].join(' OR ') +const animeExtBase = 'アニメ -公式' /* ゲームカテゴリ場合は↓を検索クエリに追加する */ const gameExt = ['ゲーム'].join(' OR ') From e1e57cbfb72bdb296e8f235c218643f1b9903abe Mon Sep 17 00:00:00 2001 From: vipzero Date: Tue, 27 Dec 2022 19:20:14 +0900 Subject: [PATCH 049/105] add black list --- src/service/customImageSearch.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/service/customImageSearch.ts b/src/service/customImageSearch.ts index 0c060fc8..938439af 100644 --- a/src/service/customImageSearch.ts +++ b/src/service/customImageSearch.ts @@ -17,7 +17,13 @@ export const getImage = (q: string) => { }) } -const blackList = ['static.wikia.nocookie.net', 'amazon.com'] +const blackList = [ + 'static.wikia.nocookie.net', + 'amazon.com', + 'fril.jp', + 'shopping.yahoo.co.jp', + 'static.mercdn.net', +] const white = (v: string) => !blackList.some((blink) => v.includes(blink)) export const getImageLinks = async (q: string) => { From f1fbf5da34fb4ad505f71ad8f755b0ab9f44e7ea Mon Sep 17 00:00:00 2001 From: vipzero Date: Wed, 28 Dec 2022 09:14:02 +0900 Subject: [PATCH 050/105] swithc mode --- src/index.ts | 2 +- src/service/customImageSearch.ts | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/index.ts b/src/index.ts index 7e6dd5f4..8ce7e298 100644 --- a/src/index.ts +++ b/src/index.ts @@ -28,7 +28,7 @@ store.onExpiredStorageUrl = (urls) => { }) } -const DIRECT_MODE = Boolean(process.env.DIRECT_MODE) +const DIRECT_MODE = Boolean(Number(process.env.DIRECT_MODE)) async function prepareImages(q: string) { const googleImageLinks = await getImageLinks(q) if (DIRECT_MODE) return googleImageLinks diff --git a/src/service/customImageSearch.ts b/src/service/customImageSearch.ts index 938439af..0c343aff 100644 --- a/src/service/customImageSearch.ts +++ b/src/service/customImageSearch.ts @@ -23,6 +23,7 @@ const blackList = [ 'fril.jp', 'shopping.yahoo.co.jp', 'static.mercdn.net', + 'auctions.c.yimg.jp', ] const white = (v: string) => !blackList.some((blink) => v.includes(blink)) From 55e00f3d44658f3f3833a5f055befdf175204684 Mon Sep 17 00:00:00 2001 From: vipzero Date: Wed, 28 Dec 2022 13:03:08 +0900 Subject: [PATCH 051/105] catch imgmin error --- src/imageIo/download.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/imageIo/download.ts b/src/imageIo/download.ts index f309a6ec..1dbde03f 100644 --- a/src/imageIo/download.ts +++ b/src/imageIo/download.ts @@ -40,7 +40,14 @@ export const downloadOptimize = async ( } ) if (res === 'SaveError') return false - await imageMin(filePath) + if ( + !(await imageMin(filePath).catch((e) => { + warn('ImageMinError', e) + return false + })) + ) { + return false + } const shapeRes = await sharpMin(filePath).catch((e) => { warn('UnsupportedError', e) return false as const From 2dc72c3fc47f764144356cb32a97b1df30989b63 Mon Sep 17 00:00:00 2001 From: vipzero Date: Wed, 28 Dec 2022 13:38:07 +0900 Subject: [PATCH 052/105] fix to buffer --- src/imageIo/download.ts | 7 +------ src/imageIo/imagemin.ts | 37 +++++++++++++++++++++---------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/imageIo/download.ts b/src/imageIo/download.ts index 1dbde03f..1d6bc0b8 100644 --- a/src/imageIo/download.ts +++ b/src/imageIo/download.ts @@ -40,12 +40,7 @@ export const downloadOptimize = async ( } ) if (res === 'SaveError') return false - if ( - !(await imageMin(filePath).catch((e) => { - warn('ImageMinError', e) - return false - })) - ) { + if (!(await imageMin(filePath))) { return false } const shapeRes = await sharpMin(filePath).catch((e) => { diff --git a/src/imageIo/imagemin.ts b/src/imageIo/imagemin.ts index e5ac0f41..336c648b 100644 --- a/src/imageIo/imagemin.ts +++ b/src/imageIo/imagemin.ts @@ -1,22 +1,27 @@ -import { error } from '../utils/logger' +import { readFile, writeFile } from 'fs/promises' import imagemin from 'imagemin' +import imageminGifsicle from 'imagemin-gifsicle' import imageminMozjpeg from 'imagemin-mozjpeg' import imageminPngquant from 'imagemin-pngquant' -import imageminGifsicle from 'imagemin-gifsicle' +import { error } from '../utils/logger' -// const imageminSvgo = require('imagemin-svgo'); +export async function imageMin(path: string) { + const originalFileBuffer = await readFile(path) -export function imageMin(path: string) { - return imagemin([path], { - destination: 'tmp', - plugins: [ - imageminMozjpeg({ quality: 50 }), - imageminPngquant({ quality: [0.5, 0.6] }), - imageminGifsicle(), - // imageminSvgo(), - ], - }).catch((e) => { - error('ImageMin', e) - return false - }) + const modifiedFileBuffer = await imagemin + .buffer(originalFileBuffer, { + plugins: [ + imageminMozjpeg({ quality: 50 }), + imageminPngquant({ quality: [0.5, 0.6] }), + imageminGifsicle(), + // imageminSvgo(), + ], + }) + .catch((e) => { + error('ImageMin', e) + return false + }) + if (typeof modifiedFileBuffer === 'boolean') return false + await writeFile(path, modifiedFileBuffer) + return true } From 1fe67523d4754fa8a8f59430a6ffcfe43c869f1a Mon Sep 17 00:00:00 2001 From: vipzero Date: Wed, 28 Dec 2022 13:57:43 +0900 Subject: [PATCH 053/105] non-skip imagemin failed --- src/imageIo/download.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/imageIo/download.ts b/src/imageIo/download.ts index 1d6bc0b8..f309a6ec 100644 --- a/src/imageIo/download.ts +++ b/src/imageIo/download.ts @@ -40,9 +40,7 @@ export const downloadOptimize = async ( } ) if (res === 'SaveError') return false - if (!(await imageMin(filePath))) { - return false - } + await imageMin(filePath) const shapeRes = await sharpMin(filePath).catch((e) => { warn('UnsupportedError', e) return false as const From 0f76a4250a220c49b3311f6915da6d219585cd1b Mon Sep 17 00:00:00 2001 From: vipzero Date: Wed, 28 Dec 2022 21:07:04 +0900 Subject: [PATCH 054/105] remove emol --- src/index.ts | 15 ++++++--------- src/service/firebase.ts | 19 +++---------------- src/streaming/icy.ts | 4 +--- 3 files changed, 10 insertions(+), 28 deletions(-) diff --git a/src/index.ts b/src/index.ts index 8ce7e298..894ed691 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,7 +1,6 @@ import { findSong } from './anisonDb/findSong' import { uploadByUrlAll } from './imageIo/uploadManage' import { getImageLinks } from './service/customImageSearch' -import { makeEmol } from './service/emol' import { addHistory, deleteFile, @@ -11,10 +10,9 @@ import { } from './service/firebase' import { getAlbum } from './service/itunes' import { getLyricsSafe } from './service/jlyricnet' -// import { spotifySearchSongInfo } from './spotify' import { Store, store } from './state/store' import subscribeIcy from './streaming/icy' -import { Emol, Song } from './types/index' +import { Song } from './types/index' import { sleep } from './utils' import { error, info, log, songPrint } from './utils/logger' import { makeSearchQuery } from './utils/makeSearchWord' @@ -41,7 +39,7 @@ export async function icyToSong( icy: string, time: number, store: Store -): Promise<[Song, Emol] | false> { +): Promise<[Song] | false> { info(icy) if (store.isDuplicate(icy)) return false // 起動時の重複登録を防ぐ @@ -62,12 +60,11 @@ export async function icyToSong( const albumInfosSync = getAlbum(icy) const lyricsSync = getLyricsSafe(song.title, song.artist) - const [imageLinks, albumInfos, { creators, lyric }] = await Promise.all([ + const [imageLinks, albumInfos, { creators }] = await Promise.all([ imageLinksSync, albumInfosSync, lyricsSync, ]) - const emol = await makeEmol(lyric) const compSong: Song = { ...song, imageLinks, @@ -77,7 +74,7 @@ export async function icyToSong( time, imageSearchWord, } - return [compSong, emol] + return [compSong] } async function receiveIcy(icy: string) { @@ -89,9 +86,9 @@ async function receiveIcy(icy: string) { const res = await icyToSong(icy, Date.now(), store) if (!res) return - const [song, emol] = res + const [song] = res songPrint(song) - saveMusic(song, emol) + saveMusic(song) } performance.mark('s1') diff --git a/src/service/firebase.ts b/src/service/firebase.ts index 4e5a227b..e08a432a 100644 --- a/src/service/firebase.ts +++ b/src/service/firebase.ts @@ -1,5 +1,5 @@ import admin from 'firebase-admin' -import { Count, Counts, Emol, HistoryRaw, HistTop, Song } from '../types/index' +import { Count, Counts, HistoryRaw, HistTop, Song } from '../types/index' import { chunk } from '../utils' import { error, info, log, warn } from '../utils/logger' import { CacheFile } from './../types/index' @@ -16,7 +16,7 @@ export { admin } const P_SONGS = 'songs' const P_SONG = 'song' -const P_EMOL = 'emol' +// const P_EMOL = 'emol' const P_HIST = 'hist' const P_YO = 'yo' const P_CURRENT = 'current' @@ -63,22 +63,9 @@ const saveSong = (song: Song) => { ...removeUndefined(song), }) } -const saveEmol = (emol: Emol) => { - fdb.collection(P_EMOL).doc(EVENT_ID).set(emol) -} -// const saveLyric = (text) => { -// // console.log(lyric) -// fdb.collection('song').doc('lyric').set({ text }) -// } -export const saveMusic = (song: Song, emol: Emol) => { +export const saveMusic = (song: Song) => { saveSong(song) - saveEmol(emol) - // if (lyric) { - // saveLyric(lyric) - // } else { - // saveLyric('no lyric') - // } } export const histSongsRef = (eid = EVENT_ID) => diff --git a/src/streaming/icy.ts b/src/streaming/icy.ts index 838bab0e..b84b1dc0 100644 --- a/src/streaming/icy.ts +++ b/src/streaming/icy.ts @@ -1,4 +1,3 @@ -import { log } from '../utils/logger' import { sjisToUtf8 } from '../utils' const icy = require('icy') @@ -22,8 +21,7 @@ function subscribeIcy( // connect to the remote stream icy.get(url, (res: IcyRes) => { // log the HTTP response headers - log() - res.headers + // log(res.headers) // log any "metadata" events that happen res.on('metadata', (metadata) => { From 8591eb31eb0dff1af0aa8f37813ab471b12edb13 Mon Sep 17 00:00:00 2001 From: vipzero Date: Wed, 28 Dec 2022 21:29:35 +0900 Subject: [PATCH 055/105] update sotrage url --- .envrc.sample | 3 ++- src/index.ts | 21 +++++---------------- src/service/firebase.ts | 9 ++++++--- 3 files changed, 13 insertions(+), 20 deletions(-) diff --git a/.envrc.sample b/.envrc.sample index 34323436..853e23d8 100644 --- a/.envrc.sample +++ b/.envrc.sample @@ -18,7 +18,8 @@ export SPOTIFY_CLIENT_SECRET="" export MUSIXMATCH_API_KEY="" export EVENT_ID="todo" -export STRAGE_URL="" +export STORAGE_ID="" +export STORAGE_URL="" export THREAD_TITLE_WORD="ならアニソン" export DIRECT_MODE=0 \ No newline at end of file diff --git a/src/index.ts b/src/index.ts index 894ed691..4086eaef 100644 --- a/src/index.ts +++ b/src/index.ts @@ -27,10 +27,13 @@ store.onExpiredStorageUrl = (urls) => { } const DIRECT_MODE = Boolean(Number(process.env.DIRECT_MODE)) +console.log({ DIRECT_MODE }) async function prepareImages(q: string) { const googleImageLinks = await getImageLinks(q) + console.log(googleImageLinks) if (DIRECT_MODE) return googleImageLinks const uploads = await uploadByUrlAll(googleImageLinks) + console.log(uploads) store.addQueue(uploads) return uploads.map((u) => u.downloadUrl) } @@ -44,7 +47,7 @@ export async function icyToSong( if (store.isDuplicate(icy)) return false // 起動時の重複登録を防ぐ - addHistory(icy, time) + // addHistory(icy, time) const song = findSong(icy) @@ -78,29 +81,15 @@ export async function icyToSong( } async function receiveIcy(icy: string) { - performance.mark('e2') - performance.measure('first', 's2', 'e2') - if (performance.getEntriesByName('first').length === 1) { - log(performance.getEntriesByName('first')[0]) - } - const res = await icyToSong(icy, Date.now(), store) if (!res) return const [song] = res songPrint(song) - saveMusic(song) + // saveMusic(song) } -performance.mark('s1') - async function main() { const res = await getCurrentPlay() - performance.mark('e1') - performance.measure('launch', 's1', 'e1') - - log(performance.getEntriesByName('launch')[0]) - - performance.mark('s2') store.counts = (await init()).counts store.setFirstIcy(res.icy) diff --git a/src/service/firebase.ts b/src/service/firebase.ts index e08a432a..767f75b4 100644 --- a/src/service/firebase.ts +++ b/src/service/firebase.ts @@ -33,7 +33,10 @@ const credential = admin.credential.cert(serviceAccount) admin.initializeApp({ credential }) export const fdb = admin.firestore() -export const bucket = admin.storage().bucket('rekka-haikei.appspot.com') +const storageId = process.env.STORAGE_ID || '' +const storageUrl = process.env.STORAGE_URL || '' +const bucketUrl = storageUrl + storageId +export const bucket = admin.storage().bucket(storageId) type Obj = { [key: string]: number | string | object } const removeUndefined = (obj: Obj) => { @@ -251,7 +254,7 @@ export const uploadStorage = async (file: CacheFile, id: string) => { predefinedAcl: 'publicRead', }) - const downloadUrl = `${process.env.STRAGE_URL}${destination}` + const downloadUrl = `${bucketUrl}/${destination}` return { downloadUrl, path: destination, tmpFilePath } } @@ -267,7 +270,7 @@ export const archiveUrl = (eid: string): StoragePaths => { const localFile = `data/archvie_${eid}.csv` const filename = `hist_${eid}.csv` const destination = `${distPath}/${filename}` - const url = `${process.env.STRAGE_URL}/${destination}` + const url = `${bucketUrl}/${destination}` return { url, destination, filename, localFile } } export const uploadStorageArchive = async ({ From c2aaa98cf4b19e66c0aa02f2a88e1fb2e1700837 Mon Sep 17 00:00:00 2001 From: vipzero Date: Wed, 28 Dec 2022 21:34:44 +0900 Subject: [PATCH 056/105] fix --- src/index.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/index.ts b/src/index.ts index 4086eaef..0355e862 100644 --- a/src/index.ts +++ b/src/index.ts @@ -27,13 +27,10 @@ store.onExpiredStorageUrl = (urls) => { } const DIRECT_MODE = Boolean(Number(process.env.DIRECT_MODE)) -console.log({ DIRECT_MODE }) async function prepareImages(q: string) { const googleImageLinks = await getImageLinks(q) - console.log(googleImageLinks) if (DIRECT_MODE) return googleImageLinks const uploads = await uploadByUrlAll(googleImageLinks) - console.log(uploads) store.addQueue(uploads) return uploads.map((u) => u.downloadUrl) } @@ -47,7 +44,7 @@ export async function icyToSong( if (store.isDuplicate(icy)) return false // 起動時の重複登録を防ぐ - // addHistory(icy, time) + addHistory(icy, time) const song = findSong(icy) @@ -85,7 +82,7 @@ async function receiveIcy(icy: string) { if (!res) return const [song] = res songPrint(song) - // saveMusic(song) + saveMusic(song) } async function main() { From 12acb6d0ba96c180da704cbdbbb0227e6eba8b21 Mon Sep 17 00:00:00 2001 From: vipzero Date: Thu, 29 Dec 2022 14:47:58 +0900 Subject: [PATCH 057/105] silent error --- src/imageIo/download.ts | 16 ++++++++-------- src/utils/logger.ts | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/imageIo/download.ts b/src/imageIo/download.ts index f309a6ec..0a5a0d2b 100644 --- a/src/imageIo/download.ts +++ b/src/imageIo/download.ts @@ -19,14 +19,14 @@ export const downloadOptimize = async ( const uuid = uuidv4() const filePath = `tmp/${uuid}` const stream = got.stream(url) - const fileTypePromise = fromStream(stream).catch((e) => { - if (e.message.includes('End-Of-Stream')) { - warn(`EndOfStreamError`, `${url} ${filePath}`) - return - } else { - error(`FileTypeError`, `${url} ${filePath}`) - log(JSON.stringify(e)) - } + const fileTypePromise = fromStream(stream).catch(() => { + // if (e.message.includes('End-Of-Stream')) { + // warn(`EndOfStreamError`, `${url} ${filePath}`) + // return + // } else { + // error(`FileTypeError`, `${url} ${filePath}`) + // log(JSON.stringify(e)) + // } return fileTypeDefault }) diff --git a/src/utils/logger.ts b/src/utils/logger.ts index cf26575f..6db967bf 100644 --- a/src/utils/logger.ts +++ b/src/utils/logger.ts @@ -9,7 +9,7 @@ export const error = (key: string, description: string) => { } export const warn = (key: string, description: string) => { - console.warn(chalk.red(`${key}: ${description}`)) + console.warn(chalk.yellow(`${key}: ${description}`)) } // eslint-disable-next-line @typescript-eslint/no-explicit-any From 902cb12d69a2df22fa429e3160bf433e6619ced9 Mon Sep 17 00:00:00 2001 From: vipzero Date: Thu, 29 Dec 2022 16:17:28 +0900 Subject: [PATCH 058/105] add word counting additionals --- src/index.ts | 51 ++++++++++++++++++------------------ src/service/firebase.ts | 4 +-- src/tests/parseWords.test.ts | 22 ++++++++++++++++ src/utils/index.ts | 12 +++++++-- src/utils/wordCounts.ts | 4 +-- 5 files changed, 61 insertions(+), 32 deletions(-) diff --git a/src/index.ts b/src/index.ts index 0355e862..f0823bab 100644 --- a/src/index.ts +++ b/src/index.ts @@ -12,8 +12,8 @@ import { getAlbum } from './service/itunes' import { getLyricsSafe } from './service/jlyricnet' import { Store, store } from './state/store' import subscribeIcy from './streaming/icy' -import { Song } from './types/index' -import { sleep } from './utils' +import { Counts, Song } from './types/index' +import { nonEmpty, sleep } from './utils' import { error, info, log, songPrint } from './utils/logger' import { makeSearchQuery } from './utils/makeSearchWord' import { anaCounts } from './utils/wordCounts' @@ -38,33 +38,25 @@ async function prepareImages(q: string) { export async function icyToSong( icy: string, time: number, - store: Store -): Promise<[Song] | false> { - info(icy) - - if (store.isDuplicate(icy)) return false // 起動時の重複登録を防ぐ - - addHistory(icy, time) - + prevCounts: Counts = {} +): Promise<[Song, Counts] | false> { const song = findSong(icy) - const additionals: string[] = [song.animeTitle, song.title].filter( - Boolean - ) as string[] - const { wordCounts, counts } = anaCounts(icy, store.counts || {}, additionals) - store.counts = counts - const imageSearchWord = makeSearchQuery(song, Math.random()) - const imageLinksSync = prepareImages(imageSearchWord) - - const albumInfosSync = getAlbum(icy) - const lyricsSync = getLyricsSafe(song.title, song.artist) const [imageLinks, albumInfos, { creators }] = await Promise.all([ - imageLinksSync, - albumInfosSync, - lyricsSync, + prepareImages(imageSearchWord), + getAlbum(icy), + getLyricsSafe(song.title, song.artist), ]) + const additionals: string[] = nonEmpty([ + song.animeTitle, + song.title, + song.artist, + ...Object.values(creators), + ]) + const { wordCounts, counts } = anaCounts([icy], prevCounts, additionals) + const compSong: Song = { ...song, imageLinks, @@ -74,15 +66,22 @@ export async function icyToSong( time, imageSearchWord, } - return [compSong] + return [compSong, counts] } async function receiveIcy(icy: string) { - const res = await icyToSong(icy, Date.now(), store) + info(icy) + + if (store.isDuplicate(icy)) return false // 起動時の重複登録を防ぐ + + const time = Date.now() + const res = await icyToSong(icy, time, store.counts) if (!res) return - const [song] = res + const [song, counts] = res + store.counts = counts songPrint(song) saveMusic(song) + addHistory(icy, time) } async function main() { diff --git a/src/service/firebase.ts b/src/service/firebase.ts index 767f75b4..0d5e89d9 100644 --- a/src/service/firebase.ts +++ b/src/service/firebase.ts @@ -11,7 +11,6 @@ export { admin } // const P_VOTE = 'vote' // const P_BOOKS = 'books' // const P_TABLE = 'table' -// const P_COUNTS = 'counts' // const P_CVOTE = 'cvote' const P_SONGS = 'songs' @@ -19,6 +18,7 @@ const P_SONG = 'song' // const P_EMOL = 'emol' const P_HIST = 'hist' const P_YO = 'yo' +const P_COUNTS = 'counts' const P_CURRENT = 'current' const { SERVICE_ACCOUNT_FILE_PATH, EVENT_ID } = process.env @@ -138,7 +138,7 @@ export const setupCount = async (counts: Counts, lasttime: number) => { ents.forEach(([k, v]) => { if (!k || !v) return batch.set( - fdb.collection('hist').doc(EVENT_ID).collection('counts').doc(), + fdb.collection('hist').doc(EVENT_ID).collection(P_COUNTS).doc(), { word: k, count: v, diff --git a/src/tests/parseWords.test.ts b/src/tests/parseWords.test.ts index dc35a72a..38f650e5 100644 --- a/src/tests/parseWords.test.ts +++ b/src/tests/parseWords.test.ts @@ -14,6 +14,28 @@ Array [ "沼倉愛美", "Ray of bullet", ] +`) + + expect(parseCountWords([q, q])).toMatchInlineSnapshot(` +Array [ + "イリス", + "フレイア", + "日高里菜", + "物部深月", + "沼倉愛美", + "Ray of bullet", +] +`) + + expect(parseCountWords([q, ''])).toMatchInlineSnapshot(` +Array [ + "イリス", + "フレイア", + "日高里菜", + "物部深月", + "沼倉愛美", + "Ray of bullet", +] `) q = diff --git a/src/utils/index.ts b/src/utils/index.ts index 60002f42..8bb945e0 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -53,8 +53,13 @@ export const uniqo = (arr: string[]) => { return Object.values(obj) } -export const parseCountWords = (icy: string, additional: string[] = []) => { - const words = parseWords(icy) +export const parseCountWords = ( + icys: string | string[], + additional: string[] = [] +) => { + const words = (Array.isArray(icys) ? icys : [icys]) + .map((icy) => parseWords(icy)) + .flat() const entries = uniqo([...additional, ...words]) return entries @@ -92,3 +97,6 @@ export const pickChara = (s: string): string[] => { export const flatten = (arr: T[][]) => arr.reduce((a, b) => a.concat(b), []) + +export const nonEmpty = (strs: (string | undefined)[]) => + strs.filter(Boolean) as string[] diff --git a/src/utils/wordCounts.ts b/src/utils/wordCounts.ts index 03404cf5..40dea437 100644 --- a/src/utils/wordCounts.ts +++ b/src/utils/wordCounts.ts @@ -16,11 +16,11 @@ export function saveCountsFile(counts: Counts, time = Date.now()) { } export function anaCounts( - icy: string, + icys: string[], countsOld: Counts, additionals: string[] = [] ) { - const entries = parseCountWords(icy, additionals) + const entries = parseCountWords(icys, additionals) const counts = { ...countsOld } const entriesNoms = entries.map(textNormalize) From b612e5f2ef9d098f2d96f55ccb771cb37b00dafb Mon Sep 17 00:00:00 2001 From: vipzero Date: Thu, 29 Dec 2022 22:03:58 +0900 Subject: [PATCH 059/105] update word count algo --- scripts/batch/importHistoryFromCells.ts | 2 +- scripts/batch/importHistoryFromCells2.ts | 2 +- scripts/batch/importHistoryFromCells3.ts | 2 +- src/index.ts | 10 +++--- src/service/firebase.ts | 44 +++++++++++++++++++++--- src/utils/wordCounts.ts | 6 ++-- 6 files changed, 53 insertions(+), 13 deletions(-) diff --git a/scripts/batch/importHistoryFromCells.ts b/scripts/batch/importHistoryFromCells.ts index 2a57abfd..69fd6b76 100644 --- a/scripts/batch/importHistoryFromCells.ts +++ b/scripts/batch/importHistoryFromCells.ts @@ -22,7 +22,7 @@ async function main() { const { title, time } = parseLine(line) await sleep(200) await addHistory(title, time) - anaCounts(title, {}) + anaCounts([title], {}, [], true) process.stdout.write('.') } diff --git a/scripts/batch/importHistoryFromCells2.ts b/scripts/batch/importHistoryFromCells2.ts index f7109fe7..513e4068 100644 --- a/scripts/batch/importHistoryFromCells2.ts +++ b/scripts/batch/importHistoryFromCells2.ts @@ -22,7 +22,7 @@ async function main() { await sleep(200) await addHistory(title.trim(), Number(time), 0) - anaCounts(title, {}) + anaCounts([title], {}, [], true) process.stdout.write('.') } diff --git a/scripts/batch/importHistoryFromCells3.ts b/scripts/batch/importHistoryFromCells3.ts index b52a531e..a05213e0 100644 --- a/scripts/batch/importHistoryFromCells3.ts +++ b/scripts/batch/importHistoryFromCells3.ts @@ -23,7 +23,7 @@ async function main() { await sleep(200) await addHistory(title.trim(), Number(time), 0) - anaCounts(title, {}) + anaCounts([title], {}, [], true) process.stdout.write('.') } diff --git a/src/index.ts b/src/index.ts index f0823bab..91c8c344 100644 --- a/src/index.ts +++ b/src/index.ts @@ -10,7 +10,7 @@ import { } from './service/firebase' import { getAlbum } from './service/itunes' import { getLyricsSafe } from './service/jlyricnet' -import { Store, store } from './state/store' +import { store } from './state/store' import subscribeIcy from './streaming/icy' import { Counts, Song } from './types/index' import { nonEmpty, sleep } from './utils' @@ -51,11 +51,13 @@ export async function icyToSong( ]) const additionals: string[] = nonEmpty([ song.animeTitle, - song.title, - song.artist, ...Object.values(creators), ]) - const { wordCounts, counts } = anaCounts([icy], prevCounts, additionals) + const { wordCounts, counts } = anaCounts( + [icy, song.artist || ''], + prevCounts, + additionals + ) const compSong: Song = { ...song, diff --git a/src/service/firebase.ts b/src/service/firebase.ts index 0d5e89d9..ecc9646b 100644 --- a/src/service/firebase.ts +++ b/src/service/firebase.ts @@ -76,7 +76,10 @@ export const histSongsRef = (eid = EVENT_ID) => export const bookCountDocRef = () => fdb.collection(P_YO).doc(P_CURRENT) export const loadHistEventSongs = async (eid: string) => { - const snaps = await histSongsRef(eid).orderBy('time', 'asc').get() + const snaps = await histSongsRef(eid) + .orderBy('time', 'asc') + .where('time', '<', 1672298643907) + .get() const lines: HistoryRaw[] = [] snaps.docs.forEach((doc) => { const d = doc.data() as HistoryRaw @@ -118,7 +121,7 @@ export const loadWordCounts = async () => { const snap = await fdb .collection('hist') .doc(EVENT_ID) - .collection('counts') + .collection(P_COUNTS) .get() const hist = await fdb.collection('hist').doc(EVENT_ID).get() const lasttime = (hist.exists && (hist.data() as HistTop).lasttime) || 0 @@ -173,6 +176,39 @@ export const setupHistN = async (ns: Record) => { } } +export const countupWordsEntry = async (words: Record) => { + const batch = fdb.batch() + const check: Record = {} + for (const ws of chunk(Object.entries(words), 10)) { + const docs = await fdb + .collection('hist') + .doc(EVENT_ID) + .collection(P_COUNTS) + .where( + 'word', + 'in', + ws.map((v) => v[0]) + ) + .get() + docs.forEach((doc) => { + const wk = (doc.data() as Count).word + check[wk] = true + batch.update(doc.ref, { + count: admin.firestore.FieldValue.increment(words[wk]), + }) + }) + } + Object.entries(words).forEach(([word, count]) => { + if (check[word]) return + batch.set(fdb.collection('hist').doc(EVENT_ID).collection(P_COUNTS).doc(), { + word, + count, + }) + }) + + await batch.commit() +} + export const countupWords = async (words: string[]) => { const batch = fdb.batch() const check: Record = {} @@ -180,7 +216,7 @@ export const countupWords = async (words: string[]) => { const docs = await fdb .collection('hist') .doc(EVENT_ID) - .collection('counts') + .collection(P_COUNTS) .where('word', 'in', ws) .get() docs.forEach((doc) => { @@ -190,7 +226,7 @@ export const countupWords = async (words: string[]) => { } words.forEach((word) => { if (check[word]) return - batch.set(fdb.collection('hist').doc(EVENT_ID).collection('counts').doc(), { + batch.set(fdb.collection('hist').doc(EVENT_ID).collection(P_COUNTS).doc(), { word, count: 1, }) diff --git a/src/utils/wordCounts.ts b/src/utils/wordCounts.ts index 40dea437..d311b341 100644 --- a/src/utils/wordCounts.ts +++ b/src/utils/wordCounts.ts @@ -18,13 +18,15 @@ export function saveCountsFile(counts: Counts, time = Date.now()) { export function anaCounts( icys: string[], countsOld: Counts, - additionals: string[] = [] + additionals: string[] = [], + write = false ) { + console.log({ icys, additionals }) const entries = parseCountWords(icys, additionals) const counts = { ...countsOld } const entriesNoms = entries.map(textNormalize) - countupWords(entriesNoms) + if (write) countupWords(entriesNoms) entriesNoms.forEach((v) => { counts[v] = (countsOld[v] || 0) + 1 }) From ca2386121cf90dfc5068feeb9e4bd3753d731d9d Mon Sep 17 00:00:00 2001 From: vipzero Date: Thu, 29 Dec 2022 22:19:36 +0900 Subject: [PATCH 060/105] fix parser --- src/tests/parseWords.test.ts | 20 ++++++++++++++++++++ src/utils/index.ts | 2 +- src/utils/wordCounts.ts | 1 - 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/tests/parseWords.test.ts b/src/tests/parseWords.test.ts index 38f650e5..250efeb3 100644 --- a/src/tests/parseWords.test.ts +++ b/src/tests/parseWords.test.ts @@ -159,6 +159,26 @@ Array [ "めぐみん", "ダクネス", ] +`) + + q = + '劇団ひととせ(桜木ひな子(cv:m・a・o)/夏川くいな(cv:富田美憂)/柊真雪(cv:小倉唯)/萩野千秋(cv:東城日沙子)/中島ゆあ(cv:高野麻里佳))' + expect(parseCountWords(q)).toMatchInlineSnapshot(` +Array [ + "劇団ひととせ", + "桜木ひな子", + "m", + "a", + "o", + "夏川くいな", + "富田美憂", + "柊真雪", + "小倉唯", + "萩野千秋", + "東城日沙子", + "中島ゆあ", + "高野麻里佳", +] `) }) diff --git a/src/utils/index.ts b/src/utils/index.ts index 8bb945e0..2acb32b0 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -32,7 +32,7 @@ const isTagWord = (v: string) => !!v && v !== '-' const trimWord = (v: string) => v .trim() - .replace(/^[.:：&＆]/, '') + .replace(/^[.:：&＆/]/, '') .trim() const parseWords = (s: string) => diff --git a/src/utils/wordCounts.ts b/src/utils/wordCounts.ts index d311b341..c6c035b6 100644 --- a/src/utils/wordCounts.ts +++ b/src/utils/wordCounts.ts @@ -21,7 +21,6 @@ export function anaCounts( additionals: string[] = [], write = false ) { - console.log({ icys, additionals }) const entries = parseCountWords(icys, additionals) const counts = { ...countsOld } const entriesNoms = entries.map(textNormalize) From a59c3621b24d0b403c20d4776ecbbee7b3d47156 Mon Sep 17 00:00:00 2001 From: vipzero Date: Thu, 29 Dec 2022 22:19:43 +0900 Subject: [PATCH 061/105] add replacement batch --- scripts/batch/restoreHistoryTags.ts | 31 +++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 scripts/batch/restoreHistoryTags.ts diff --git a/scripts/batch/restoreHistoryTags.ts b/scripts/batch/restoreHistoryTags.ts new file mode 100644 index 00000000..5dab5898 --- /dev/null +++ b/scripts/batch/restoreHistoryTags.ts @@ -0,0 +1,31 @@ +import { assert } from 'console' +import { findSong } from '../../src/anisonDb/findSong' + +import { loadHistEventSongs } from '../../src/service/firebase' +import { nonEmpty } from '../../src/utils' +import { log } from '../../src/utils/logger' +import { anaCounts } from '../../src/utils/wordCounts' + +const restoreHistoryTags = async (id: string) => { + // log({ id }) + if (!id) return + const data = await loadHistEventSongs(id) + + let counts = {} + + data.map((s) => { + const icy = s.title + const song = findSong(icy) + + const additionals: string[] = nonEmpty([song.animeTitle]) + const cres = anaCounts([song.artist || ''], counts, additionals) + counts = cres.counts + }) + log(JSON.stringify(counts)) +} + +const eventId = process.argv[2] + +assert(eventId, 'need arg [event_id]') + +restoreHistoryTags(eventId).then(() => log('doae')) From db184dc3dbf9ccc1fc0d9a6f2e3cb0b8480faceb Mon Sep 17 00:00:00 2001 From: vipzero Date: Thu, 29 Dec 2022 22:20:51 +0900 Subject: [PATCH 062/105] fix time filter --- src/service/firebase.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/service/firebase.ts b/src/service/firebase.ts index ecc9646b..e89176fc 100644 --- a/src/service/firebase.ts +++ b/src/service/firebase.ts @@ -76,10 +76,7 @@ export const histSongsRef = (eid = EVENT_ID) => export const bookCountDocRef = () => fdb.collection(P_YO).doc(P_CURRENT) export const loadHistEventSongs = async (eid: string) => { - const snaps = await histSongsRef(eid) - .orderBy('time', 'asc') - .where('time', '<', 1672298643907) - .get() + const snaps = await histSongsRef(eid).orderBy('time', 'asc').get() const lines: HistoryRaw[] = [] snaps.docs.forEach((doc) => { const d = doc.data() as HistoryRaw From 2f05c352e0440579102ced9068b0c7924d32e88d Mon Sep 17 00:00:00 2001 From: vipzero Date: Thu, 29 Dec 2022 22:30:26 +0900 Subject: [PATCH 063/105] add convert tags --- src/tests/convertTimeTags.test.ts | 40 +++++++++++++++++++++++++++++++ src/utils/index.ts | 6 +++++ 2 files changed, 46 insertions(+) create mode 100644 src/tests/convertTimeTags.test.ts diff --git a/src/tests/convertTimeTags.test.ts b/src/tests/convertTimeTags.test.ts new file mode 100644 index 00000000..a09baf16 --- /dev/null +++ b/src/tests/convertTimeTags.test.ts @@ -0,0 +1,40 @@ +import { convertTimeTags } from '../utils' + +test('convertTimeTags', () => { + expect(convertTimeTags('2000-01-01')).toMatchInlineSnapshot(` +Array [ + "[2000]", + "[2000-01]", + "[2000-s1]", +] +`) + expect(convertTimeTags('2010-05-20')).toMatchInlineSnapshot(` +Array [ + "[2010]", + "[2010-05]", + "[2010-s2]", +] +`) + const range = (n: number) => [...Array(n).keys()] + + const res = range(12).map((i) => + convertTimeTags(`2000-${String(i + 1).padStart(2, '0')}-01`) + ) + + expect(res.map((v) => v[2])).toMatchInlineSnapshot(` +Array [ + "[2000-s1]", + "[2000-s1]", + "[2000-s1]", + "[2000-s2]", + "[2000-s2]", + "[2000-s2]", + "[2000-s3]", + "[2000-s3]", + "[2000-s3]", + "[2000-s4]", + "[2000-s4]", + "[2000-s4]", +] +`) +}) diff --git a/src/utils/index.ts b/src/utils/index.ts index 2acb32b0..921c9213 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -100,3 +100,9 @@ export const flatten = (arr: T[][]) => export const nonEmpty = (strs: (string | undefined)[]) => strs.filter(Boolean) as string[] + +// YYYY-MM-DD -> [YYYY, YYYY-MM, YYYY-SX] +export const convertTimeTags = (songDate: string) => { + const [y, m] = songDate.split('-') + return [`[${y}]`, `[${y}-${m}]`, `[${y}-S${Math.floor((+m - 1) / 3) + 1}]`] +} From 70a7379ed176edb8e1c6e63f9796e95648bcac8a Mon Sep 17 00:00:00 2001 From: vipzero Date: Thu, 29 Dec 2022 22:36:11 +0900 Subject: [PATCH 064/105] add timeTag --- scripts/batch/restoreHistoryTags.ts | 25 ++++++++++++++++--------- src/index.ts | 4 +++- src/utils/index.ts | 3 ++- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/scripts/batch/restoreHistoryTags.ts b/scripts/batch/restoreHistoryTags.ts index 5dab5898..0a90e0ec 100644 --- a/scripts/batch/restoreHistoryTags.ts +++ b/scripts/batch/restoreHistoryTags.ts @@ -1,27 +1,34 @@ import { assert } from 'console' import { findSong } from '../../src/anisonDb/findSong' -import { loadHistEventSongs } from '../../src/service/firebase' -import { nonEmpty } from '../../src/utils' +import { + countupWordsEntry, + loadHistEventSongs +} from '../../src/service/firebase' +import { convertTimeTags } from '../../src/utils' import { log } from '../../src/utils/logger' -import { anaCounts } from '../../src/utils/wordCounts' +// 年代をタグにして補完する const restoreHistoryTags = async (id: string) => { // log({ id }) if (!id) return const data = await loadHistEventSongs(id) - let counts = {} + const counts: Record = {} data.map((s) => { const icy = s.title - const song = findSong(icy) - - const additionals: string[] = nonEmpty([song.animeTitle]) - const cres = anaCounts([song.artist || ''], counts, additionals) - counts = cres.counts + const { date } = findSong(icy) + + if (!date) return + const tags = convertTimeTags(date) + tags.map((v) => { + if (!counts[v]) counts[v] = 0 + counts[v] += 1 + }) }) log(JSON.stringify(counts)) + countupWordsEntry(counts) } const eventId = process.argv[2] diff --git a/src/index.ts b/src/index.ts index 91c8c344..e064f60c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -13,7 +13,7 @@ import { getLyricsSafe } from './service/jlyricnet' import { store } from './state/store' import subscribeIcy from './streaming/icy' import { Counts, Song } from './types/index' -import { nonEmpty, sleep } from './utils' +import { convertTimeTags, nonEmpty, sleep } from './utils' import { error, info, log, songPrint } from './utils/logger' import { makeSearchQuery } from './utils/makeSearchWord' import { anaCounts } from './utils/wordCounts' @@ -49,8 +49,10 @@ export async function icyToSong( getAlbum(icy), getLyricsSafe(song.title, song.artist), ]) + const additionals: string[] = nonEmpty([ song.animeTitle, + ...convertTimeTags(song.date), ...Object.values(creators), ]) const { wordCounts, counts } = anaCounts( diff --git a/src/utils/index.ts b/src/utils/index.ts index 921c9213..dc541b9e 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -102,7 +102,8 @@ export const nonEmpty = (strs: (string | undefined)[]) => strs.filter(Boolean) as string[] // YYYY-MM-DD -> [YYYY, YYYY-MM, YYYY-SX] -export const convertTimeTags = (songDate: string) => { +export const convertTimeTags = (songDate?: string) => { + if (!songDate) return [] const [y, m] = songDate.split('-') return [`[${y}]`, `[${y}-${m}]`, `[${y}-S${Math.floor((+m - 1) / 3) + 1}]`] } From 2920ad31aaa8f1d444a691ea2dff1073faaa7d05 Mon Sep 17 00:00:00 2001 From: vipzero Date: Thu, 29 Dec 2022 23:32:55 +0900 Subject: [PATCH 065/105] parse creators --- src/index.ts | 3 +-- src/service/firebase.ts | 6 ++++-- src/utils/wordCounts.ts | 1 + 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/index.ts b/src/index.ts index e064f60c..7797d67b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -53,10 +53,9 @@ export async function icyToSong( const additionals: string[] = nonEmpty([ song.animeTitle, ...convertTimeTags(song.date), - ...Object.values(creators), ]) const { wordCounts, counts } = anaCounts( - [icy, song.artist || ''], + [icy, song.artist || '', ...nonEmpty(Object.values(creators))], prevCounts, additionals ) diff --git a/src/service/firebase.ts b/src/service/firebase.ts index e89176fc..04e90b23 100644 --- a/src/service/firebase.ts +++ b/src/service/firebase.ts @@ -1,6 +1,6 @@ import admin from 'firebase-admin' import { Count, Counts, HistoryRaw, HistTop, Song } from '../types/index' -import { chunk } from '../utils' +import { chunk, textNormalize } from '../utils' import { error, info, log, warn } from '../utils/logger' import { CacheFile } from './../types/index' @@ -124,7 +124,9 @@ export const loadWordCounts = async () => { const lasttime = (hist.exists && (hist.data() as HistTop).lasttime) || 0 const counts: Counts = {} - snap.docs.forEach((v) => (counts[v.data().word] = v.data().count)) + snap.docs.forEach( + (v) => (counts[textNormalize(v.data().word)] = v.data().count) + ) return { counts, lasttime } } diff --git a/src/utils/wordCounts.ts b/src/utils/wordCounts.ts index c6c035b6..6c94faf9 100644 --- a/src/utils/wordCounts.ts +++ b/src/utils/wordCounts.ts @@ -26,6 +26,7 @@ export function anaCounts( const entriesNoms = entries.map(textNormalize) if (write) countupWords(entriesNoms) + entriesNoms.forEach((v) => { counts[v] = (countsOld[v] || 0) + 1 }) From a3d31288332e557d3d88e8e7d3786966620979b9 Mon Sep 17 00:00:00 2001 From: vipzero Date: Sat, 31 Dec 2022 07:39:00 +0900 Subject: [PATCH 066/105] add Timeout --- src/imageIo/download.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/imageIo/download.ts b/src/imageIo/download.ts index 0a5a0d2b..101a2eb7 100644 --- a/src/imageIo/download.ts +++ b/src/imageIo/download.ts @@ -18,7 +18,7 @@ export const downloadOptimize = async ( ): Promise => { const uuid = uuidv4() const filePath = `tmp/${uuid}` - const stream = got.stream(url) + const stream = got.stream(url, { timeout: { request: 1000 } }) const fileTypePromise = fromStream(stream).catch(() => { // if (e.message.includes('End-Of-Stream')) { // warn(`EndOfStreamError`, `${url} ${filePath}`) From 45ddda6bd7ce787abddc507c3af4760f3a34fc4e Mon Sep 17 00:00:00 2001 From: vipzero Date: Sun, 1 Jan 2023 10:49:11 +0900 Subject: [PATCH 067/105] update file-type detection --- src/imageIo/download.ts | 39 +++++++++++++++++++++------------------ src/types/index.ts | 5 +---- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/imageIo/download.ts b/src/imageIo/download.ts index 101a2eb7..71534a24 100644 --- a/src/imageIo/download.ts +++ b/src/imageIo/download.ts @@ -1,35 +1,38 @@ -import { error, warn, log } from '../utils/logger' -import { fromStream } from 'file-type' import fs from 'fs' import got from 'got' import stream from 'stream' import { promisify } from 'util' +import { CacheFile } from '../types' +import { error, log, warn } from '../utils/logger' import { imageMin } from './imagemin' import { sharpMin } from './sharp' -import { CacheFile } from '../types' const uuidv4 = require('uuid/v4') const pipeline = promisify(stream.pipeline) -const fileTypeDefault = { ext: 'png', mime: 'image/png' } + +const mimeJpg = { ext: 'jpg', mime: 'image/jpeg' } +const mimePng = { ext: 'png', mime: 'image/png' } +const mimeWebp = { ext: 'webp', mime: 'image/webp' } +const mimeGif = { ext: 'gif', mime: 'image/gif' } +const mimeSvg = { ext: 'svg', mime: 'image/svg+xml' } + +const mimeMap: Record = { + jpeg: mimeJpg, + jpg: mimeJpg, // will unuse + png: mimePng, + webp: mimeWebp, + gif: mimeGif, + svg: mimeSvg, +} +const fileTypeDefault = mimePng export const downloadOptimize = async ( url: string ): Promise => { const uuid = uuidv4() const filePath = `tmp/${uuid}` - const stream = got.stream(url, { timeout: { request: 1000 } }) - const fileTypePromise = fromStream(stream).catch(() => { - // if (e.message.includes('End-Of-Stream')) { - // warn(`EndOfStreamError`, `${url} ${filePath}`) - // return - // } else { - // error(`FileTypeError`, `${url} ${filePath}`) - // log(JSON.stringify(e)) - // } - - return fileTypeDefault - }) + const stream = got.stream(url, { timeout: { request: 5000 } }) const res = await pipeline(stream, fs.createWriteStream(filePath)).catch( (e) => { @@ -47,9 +50,9 @@ export const downloadOptimize = async ( }) if (!shapeRes) return false - const { size, height, width } = shapeRes + const { size, height, width, format } = shapeRes - const fileType = (await fileTypePromise) || fileTypeDefault + const fileType = mimeMap[format] || fileTypeDefault return { filePath, fileType, size, height, width } } diff --git a/src/types/index.ts b/src/types/index.ts index a18d2d4e..aa687128 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -90,10 +90,7 @@ export type Count = { export type CacheFile = { filePath: string - fileType: { - ext: string - mime: string - } + fileType: { ext: string; mime: string } size: number width: number height: number From 306396ca16d1fb92739312efe3e40f50f454d54e Mon Sep 17 00:00:00 2001 From: vipzero Date: Sun, 1 Jan 2023 10:49:28 +0900 Subject: [PATCH 068/105] remove log --- src/service/firebase.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/service/firebase.ts b/src/service/firebase.ts index 04e90b23..200e3ab8 100644 --- a/src/service/firebase.ts +++ b/src/service/firebase.ts @@ -263,7 +263,7 @@ export const countupWords = async (words: string[]) => { // } export const deleteFile = (path: string) => { - info(`delete op: ${path}`) + // info(`delete op: ${path}`) return bucket .file(path) From 9755cf9e1dcbdd844ed0ed3b8d018ba1949b7788 Mon Sep 17 00:00:00 2001 From: vipzero Date: Sun, 1 Jan 2023 10:49:56 +0900 Subject: [PATCH 069/105] add ee --- scripts/setupEasterEgg.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/setupEasterEgg.ts b/scripts/setupEasterEgg.ts index b582ef43..8ca539f6 100644 --- a/scripts/setupEasterEgg.ts +++ b/scripts/setupEasterEgg.ts @@ -57,6 +57,8 @@ async function main() { // prettier-ignore shining: { hn: 0, ss: 0, mr: 0, hy: 0, gs: 0, rr: 0, kk: 0, uo: 0, ms: 0, hr: 0, er: 0, te: 0, }, aobuta: { mi: 0, sk: 0, tm: 0, ft: 0, nd: 0, ke: 0 }, + // prettier-ignore + milgram: { es: 0, hr: 0, yn: 0, fu: 0, mu: 0, sd: 0, mh: 0, kz: 0, am: 0, mk: 0, kt: 0, }, } const tasks = Object.entries(votes).map(async ([animeId, inits]) => { From 362a3e5d4e1edbc5c1ba881de560baf9ee70e5e1 Mon Sep 17 00:00:00 2001 From: vipzero Date: Sun, 1 Jan 2023 17:13:36 +0900 Subject: [PATCH 070/105] update image optimaze --- package.json | 1 + src/imageIo/download.ts | 43 ++- src/imageIo/jimp.ts | 29 ++ src/imageIo/uploadManage.ts | 20 +- src/service/firebase.ts | 2 +- src/types/index.ts | 1 + yarn.lock | 526 +++++++++++++++++++++++++++++++++++- 7 files changed, 602 insertions(+), 20 deletions(-) create mode 100644 src/imageIo/jimp.ts diff --git a/package.json b/package.json index 20bb59cf..130e2108 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "icy": "2.1.0", "jaco": "^4.0.0", "jest": "^27.5.1", + "jimp": "^0.16.2", "jptext-to-emoji": "0.3.0", "node-spotify-api": "1.1.1", "sharp": "0.28.3" diff --git a/src/imageIo/download.ts b/src/imageIo/download.ts index 71534a24..be18843b 100644 --- a/src/imageIo/download.ts +++ b/src/imageIo/download.ts @@ -4,7 +4,7 @@ import stream from 'stream' import { promisify } from 'util' import { CacheFile } from '../types' import { error, log, warn } from '../utils/logger' -import { imageMin } from './imagemin' +import { jimpHash } from './jimp' import { sharpMin } from './sharp' const uuidv4 = require('uuid/v4') @@ -26,24 +26,32 @@ const mimeMap: Record = { svg: mimeSvg, } const fileTypeDefault = mimePng +const gotOption = { timeout: { request: 5000 } } export const downloadOptimize = async ( url: string ): Promise => { - const uuid = uuidv4() - const filePath = `tmp/${uuid}` - const stream = got.stream(url, { timeout: { request: 5000 } }) + const filePath = `tmp/${uuidv4()}` + const stream = got.stream(url, gotOption) - const res = await pipeline(stream, fs.createWriteStream(filePath)).catch( - (e) => { + let ts = Date.now() + let res + try { + res = await pipeline(stream, fs.createWriteStream(filePath)).catch((e) => { error(`DownloadSaveError`, `${url} ${filePath}`) - log(typeof e) log(JSON.stringify(e)) return 'SaveError' as const - } - ) + }) + } catch (e) { + warn(`out-DownloadSaveError`, JSON.stringify(e)) + res = 'SaveError' + } + log(` dw: ${Date.now() - ts}ms`) if (res === 'SaveError') return false - await imageMin(filePath) + // await imageMin(filePath) + + ts = Date.now() + const shapeRes = await sharpMin(filePath).catch((e) => { warn('UnsupportedError', e) return false as const @@ -51,8 +59,19 @@ export const downloadOptimize = async ( if (!shapeRes) return false const { size, height, width, format } = shapeRes - const fileType = mimeMap[format] || fileTypeDefault - return { filePath, fileType, size, height, width } + log(` shape: ${Date.now() - ts}ms`) + + ts = Date.now() + const resj = await jimpHash(filePath, fileType.mime).catch((e) => { + warn('JimpError', e) + return false as const + }) + if (!resj) return false + const { hash } = resj + log(` jimp: ${Date.now() - ts}ms`) + log(` size: ${width}x${height}`) + + return { filePath, fileType, size, height, width, hash } } diff --git a/src/imageIo/jimp.ts b/src/imageIo/jimp.ts new file mode 100644 index 00000000..4df69c1f --- /dev/null +++ b/src/imageIo/jimp.ts @@ -0,0 +1,29 @@ +import { writeFile } from 'fs/promises' +import { compareHashes, read } from 'jimp' +import { warn } from '../utils/logger' + +export async function jimpHash(path: string, mime: string) { + const img = await read(path).catch(() => false as const) + if (img === false) { + warn('read error', path) + + return false + } + + // const res = await img.scaleToFit(1500, 1500).quality(60) + const res = await img.quality(60) + const res2 = await writeFile(path, await res.getBufferAsync(mime)).catch( + () => false as const + ) + if (res2 === false) return false + + return { hash: res.hash(), height: res.getHeight(), width: res.getWidth() } +} + +export async function isUniqueHash(check: string, targets: string[]) { + return targets.every((v) => { + const p = compareHashes(check, v) + const dup = p < 0.15 + return !dup + }) +} diff --git a/src/imageIo/uploadManage.ts b/src/imageIo/uploadManage.ts index 2e13f8e7..f71362f0 100644 --- a/src/imageIo/uploadManage.ts +++ b/src/imageIo/uploadManage.ts @@ -1,27 +1,35 @@ import { unlink } from 'fs/promises' -import { error } from '../utils/logger' import { UploadFile } from '../types/index' +import { error } from '../utils/logger' import { uploadStorage } from './../service/firebase' import { CacheFile } from './../types/index' import { downloadOptimize } from './download' +import { isUniqueHash } from './jimp' +const nonFalse = (v: T | false): v is T => v !== false export const uploadByUrlAll = async (urls: string[]) => { const timeId = +new Date() - const downloads: CacheFile[] = [] + const downloads: CacheFile[] = ( + await Promise.all(urls.map(downloadOptimize)) + ).filter((v) => nonFalse(v)) as CacheFile[] + for (const url of urls) { - // console.log(url) const res = await downloadOptimize(url) if (!res) continue downloads.push(res) - // console.log(res) } const uploads: UploadFile[] = [] - const selects = choiceImage(downloads) + const hashs: string[] = [] for (const [i, file] of selects.entries()) { + if (!(await isUniqueHash(file.hash, hashs))) { + continue + } + hashs.push(file.hash) + const id = `${timeId}_${i}` const res = await uploadStorage(file, id).catch((e) => { error('UploadError', e) @@ -29,7 +37,7 @@ export const uploadByUrlAll = async (urls: string[]) => { }) if (!res) continue uploads.push(res) - if (uploads.length >= 3) break + if (uploads.length >= 5) break } downloads.map((f) => unlink(f.filePath)) diff --git a/src/service/firebase.ts b/src/service/firebase.ts index 200e3ab8..0c240b50 100644 --- a/src/service/firebase.ts +++ b/src/service/firebase.ts @@ -1,7 +1,7 @@ import admin from 'firebase-admin' import { Count, Counts, HistoryRaw, HistTop, Song } from '../types/index' import { chunk, textNormalize } from '../utils' -import { error, info, log, warn } from '../utils/logger' +import { error, log, warn } from '../utils/logger' import { CacheFile } from './../types/index' export { admin } diff --git a/src/types/index.ts b/src/types/index.ts index aa687128..1d343713 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -94,5 +94,6 @@ export type CacheFile = { size: number width: number height: number + hash: string } export type Emol = { text: string } diff --git a/yarn.lock b/yarn.lock index e8405f87..b259f3aa 100644 --- a/yarn.lock +++ b/yarn.lock @@ -419,6 +419,13 @@ dependencies: "@babel/helper-plugin-utils" "^7.14.5" +"@babel/runtime@^7.7.2": + version "7.20.7" + resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.20.7.tgz#fcb41a5a70550e04a7b708037c7c32f7f356d8fd" + integrity sha512-UF0tvkUtxwAgZ5W/KrkHf0Rn0fdnLDU9ScxBrEVNUprE/MzirjK4MJUX1/BVDv00Sv8cljtukVK1aky++X1SjQ== + dependencies: + regenerator-runtime "^0.13.11" + "@babel/template@^7.14.5", "@babel/template@^7.3.3": version "7.14.5" resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.14.5.tgz#a9bc9d8b33354ff6e55a9c60d1109200a68974f4" @@ -859,6 +866,296 @@ "@types/yargs" "^16.0.0" chalk "^4.0.0" +"@jimp/bmp@^0.16.2": + version "0.16.2" + resolved "https://registry.yarnpkg.com/@jimp/bmp/-/bmp-0.16.2.tgz#3982879b10626fc8cf1b4ab8627158bad142ec9d" + integrity sha512-4g9vW45QfMoGhLVvaFj26h4e7cC+McHUQwyFQmNTLW4FfC1OonN9oUr2m/FEDGkTYKR7aqdXR5XUqqIkHWLaFw== + dependencies: + "@babel/runtime" "^7.7.2" + "@jimp/utils" "^0.16.2" + bmp-js "^0.1.0" + +"@jimp/core@^0.16.2": + version "0.16.2" + resolved "https://registry.yarnpkg.com/@jimp/core/-/core-0.16.2.tgz#4f8e83a4af76a60610e794362d1deb5afaa03353" + integrity sha512-dp7HcyUMzjXphXYodI6PaXue+I9PXAavbb+AN+1XqFbotN22Z12DosNPEyy+UhLY/hZiQQqUkEaJHkvV31rs+w== + dependencies: + "@babel/runtime" "^7.7.2" + "@jimp/utils" "^0.16.2" + any-base "^1.1.0" + buffer "^5.2.0" + exif-parser "^0.1.12" + file-type "^9.0.0" + load-bmfont "^1.3.1" + mkdirp "^0.5.1" + phin "^2.9.1" + pixelmatch "^4.0.2" + tinycolor2 "^1.4.1" + +"@jimp/custom@^0.16.2": + version "0.16.2" + resolved "https://registry.yarnpkg.com/@jimp/custom/-/custom-0.16.2.tgz#e1ba6874551dd4d748825680c3a16bb7cd3595b6" + integrity sha512-GtNwOs4hcVS2GIbqRUf42rUuX07oLB92cj7cqxZb0ZGWwcwhnmSW0TFLAkNafXmqn9ug4VTpNvcJSUdiuECVKg== + dependencies: + "@babel/runtime" "^7.7.2" + "@jimp/core" "^0.16.2" + +"@jimp/gif@^0.16.2": + version "0.16.2" + resolved "https://registry.yarnpkg.com/@jimp/gif/-/gif-0.16.2.tgz#c049cf0fc781233aca418f130f8664c4cbab64c1" + integrity sha512-TMdyT9Q0paIKNtT7c5KzQD29CNCsI/t8ka28jMrBjEK7j5RRTvBfuoOnHv7pDJRCjCIqeUoaUSJ7QcciKic6CA== + dependencies: + "@babel/runtime" "^7.7.2" + "@jimp/utils" "^0.16.2" + gifwrap "^0.9.2" + omggif "^1.0.9" + +"@jimp/jpeg@^0.16.2": + version "0.16.2" + resolved "https://registry.yarnpkg.com/@jimp/jpeg/-/jpeg-0.16.2.tgz#1060cff9700d08802a0932a397cfb61a34b1d058" + integrity sha512-BW5gZydgq6wdIwHd+3iUNgrTklvoQc/FUKSj9meM6A0FU21lUaansRX5BDdJqHkyXJLnnlDGwDt27J+hQuBAVw== + dependencies: + "@babel/runtime" "^7.7.2" + "@jimp/utils" "^0.16.2" + jpeg-js "^0.4.2" + +"@jimp/plugin-blit@^0.16.2": + version "0.16.2" + resolved "https://registry.yarnpkg.com/@jimp/plugin-blit/-/plugin-blit-0.16.2.tgz#65e683f3f2860a59999b6af068efde3625f86cf7" + integrity sha512-Z31rRfV80gC/r+B/bOPSVVpJEWXUV248j7MdnMOFLu4vr8DMqXVo9jYqvwU/s4LSTMAMXqm4Jg6E/jQfadPKAg== + dependencies: + "@babel/runtime" "^7.7.2" + "@jimp/utils" "^0.16.2" + +"@jimp/plugin-blur@^0.16.2": + version "0.16.2" + resolved "https://registry.yarnpkg.com/@jimp/plugin-blur/-/plugin-blur-0.16.2.tgz#05533c19973a16feb037d175bb77e4532f144e45" + integrity sha512-ShkJCAzRI+1fAKPuLLgEkixpSpVmKTYaKEFROUcgmrv9AansDXGNCupchqVMTdxf8zPyW8rR1ilvG3OJobufLQ== + dependencies: + "@babel/runtime" "^7.7.2" + "@jimp/utils" "^0.16.2" + +"@jimp/plugin-circle@^0.16.2": + version "0.16.2" + resolved "https://registry.yarnpkg.com/@jimp/plugin-circle/-/plugin-circle-0.16.2.tgz#f66c7b8562ccced02688612f548b76952b14ab70" + integrity sha512-6T4z/48F4Z5+YwAVCLOvXQcyGmo0E3WztxCz6XGQf66r4JJK78+zcCDYZFLMx0BGM0091FogNK4QniP8JaOkrA== + dependencies: + "@babel/runtime" "^7.7.2" + "@jimp/utils" "^0.16.2" + +"@jimp/plugin-color@^0.16.2": + version "0.16.2" + resolved "https://registry.yarnpkg.com/@jimp/plugin-color/-/plugin-color-0.16.2.tgz#925d3b2fa41807c7119197bdf9c5694d92efe3be" + integrity sha512-6oBV0g0J17/7E+aTquvUsgSc85nUbUi+64tIK5eFIDzvjhlqhjGNJYlc46KJMCWIs61qRJayQoZdL/iT/iQuGQ== + dependencies: + "@babel/runtime" "^7.7.2" + "@jimp/utils" "^0.16.2" + tinycolor2 "^1.4.1" + +"@jimp/plugin-contain@^0.16.2": + version "0.16.2" + resolved "https://registry.yarnpkg.com/@jimp/plugin-contain/-/plugin-contain-0.16.2.tgz#e5cf5ca7cc3eec1306cb1b92dbd2a1fad6146a94" + integrity sha512-pLcxO3hVN3LCEhMNvpZ9B7xILHVlS433Vv16zFFJxLRqZdYvPLsc+ZzJhjAiHHuEjVblQrktHE3LGeQwGJPo0w== + dependencies: + "@babel/runtime" "^7.7.2" + "@jimp/utils" "^0.16.2" + +"@jimp/plugin-cover@^0.16.2": + version "0.16.2" + resolved "https://registry.yarnpkg.com/@jimp/plugin-cover/-/plugin-cover-0.16.2.tgz#c4aadfaad718a14838219889936ad39a18021df4" + integrity sha512-gzWM7VvYeI8msyiwbUZxH+sGQEgO6Vd6adGxZ0CeKX00uQOe5lDzxb1Wjx7sHcJGz8a/5fmAuwz7rdDtpDUbkw== + dependencies: + "@babel/runtime" "^7.7.2" + "@jimp/utils" "^0.16.2" + +"@jimp/plugin-crop@^0.16.2": + version "0.16.2" + resolved "https://registry.yarnpkg.com/@jimp/plugin-crop/-/plugin-crop-0.16.2.tgz#2dd716b93a865b839143016acac53681d85362c3" + integrity sha512-qCd3hfMEE+Z2EuuyXewgXRTtKJGIerWzc1zLEJztsUkPz5i73IGgkOL+mrNutZwGaXZbm+8SwUaGb46sxAO6Tw== + dependencies: + "@babel/runtime" "^7.7.2" + "@jimp/utils" "^0.16.2" + +"@jimp/plugin-displace@^0.16.2": + version "0.16.2" + resolved "https://registry.yarnpkg.com/@jimp/plugin-displace/-/plugin-displace-0.16.2.tgz#e4852c48f4b2095a4bcc61c8c1a5faa9618773ef" + integrity sha512-6nXdvNNjCdD95v2o3/jPeur903dz08lG4Y8gmr5oL2yVv9LSSbMonoXYrR/ASesdyXqGdXJLU4NL+yZs4zUqbQ== + dependencies: + "@babel/runtime" "^7.7.2" + "@jimp/utils" "^0.16.2" + +"@jimp/plugin-dither@^0.16.2": + version "0.16.2" + resolved "https://registry.yarnpkg.com/@jimp/plugin-dither/-/plugin-dither-0.16.2.tgz#e5cf77f5b0b8a4247c171b7e234c99031b6a59f3" + integrity sha512-DERpIzy21ZanMkVsD0Tdy8HQLbD1E41OuvIzaMRoW4183PA6AgGNlrQoFTyXmzjy6FTy1SxaQgTEdouInAWZ9Q== + dependencies: + "@babel/runtime" "^7.7.2" + "@jimp/utils" "^0.16.2" + +"@jimp/plugin-fisheye@^0.16.2": + version "0.16.2" + resolved "https://registry.yarnpkg.com/@jimp/plugin-fisheye/-/plugin-fisheye-0.16.2.tgz#ec6cab102959fd67a4061e6812db6135731f7731" + integrity sha512-Df7PsGIwiIpQu3EygYCnaJyTfOwvwtYV3cmYJS7yFLtdiFUuod+hlSo5GkwEPLAy+QBxhUbDuUqnsWo4NQtbiQ== + dependencies: + "@babel/runtime" "^7.7.2" + "@jimp/utils" "^0.16.2" + +"@jimp/plugin-flip@^0.16.2": + version "0.16.2" + resolved "https://registry.yarnpkg.com/@jimp/plugin-flip/-/plugin-flip-0.16.2.tgz#3d6f5eac4a8d7d62251aba55259ecb4f8dfe42cf" + integrity sha512-+2uC8ioVQUr06mnjSWraskz2L33nJHze35LkQ8ZNsIpoZLkgvfiWatqAs5bj+1jGI/9kxoCFAaT1Is0f+a4/rw== + dependencies: + "@babel/runtime" "^7.7.2" + "@jimp/utils" "^0.16.2" + +"@jimp/plugin-gaussian@^0.16.2": + version "0.16.2" + resolved "https://registry.yarnpkg.com/@jimp/plugin-gaussian/-/plugin-gaussian-0.16.2.tgz#6546886e8b0acfebf285c5aabc4fea476dc54159" + integrity sha512-2mnuDSg4ZEH8zcJig7DZZf4st/cYmQ5UYJKP76iGhZ+6JDACk6uejwAgT5xHecNhkVAaXMdCybA2eknH/9OE1w== + dependencies: + "@babel/runtime" "^7.7.2" + "@jimp/utils" "^0.16.2" + +"@jimp/plugin-invert@^0.16.2": + version "0.16.2" + resolved "https://registry.yarnpkg.com/@jimp/plugin-invert/-/plugin-invert-0.16.2.tgz#6ca4f7b204c5d674d093d9aa4c32bf20a924a0ee" + integrity sha512-xFvHbVepTY/nus+6yXiYN1iq+UBRkT0MdnObbiQPstUrAsz0Imn6MWISsnAyMvcNxHGrxaxjuU777JT/esM0gg== + dependencies: + "@babel/runtime" "^7.7.2" + "@jimp/utils" "^0.16.2" + +"@jimp/plugin-mask@^0.16.2": + version "0.16.2" + resolved "https://registry.yarnpkg.com/@jimp/plugin-mask/-/plugin-mask-0.16.2.tgz#b352392bc8773f6b21b34901ed17f2bb90a8047e" + integrity sha512-AbdO85xxhfgEDdxYKpUotEI9ixiCMaIpfYHD5a5O/VWeimz2kuwhcrzlHGiyq1kKAgRcl0WEneTCZAHVSyvPKA== + dependencies: + "@babel/runtime" "^7.7.2" + "@jimp/utils" "^0.16.2" + +"@jimp/plugin-normalize@^0.16.2": + version "0.16.2" + resolved "https://registry.yarnpkg.com/@jimp/plugin-normalize/-/plugin-normalize-0.16.2.tgz#e36a8ecaea6acb4711c543212863a570fe19901f" + integrity sha512-+ItBWFwmB0Od7OfOtTYT1gm543PpHUgU8/DN55z83l1JqS0OomDJAe7BmCppo2405TN6YtVm/csXo7p4iWd/SQ== + dependencies: + "@babel/runtime" "^7.7.2" + "@jimp/utils" "^0.16.2" + +"@jimp/plugin-print@^0.16.2": + version "0.16.2" + resolved "https://registry.yarnpkg.com/@jimp/plugin-print/-/plugin-print-0.16.2.tgz#8873338941498997cb2a0d2820e9d58d7c03ba61" + integrity sha512-ifTGEeJ5UZTCiqC70HMeU3iXk/vsOmhWiwVGOXSFXhFeE8ZpDWvlmBsrMYnRrJGuaaogHOIrrQPI+kCdDBSBIQ== + dependencies: + "@babel/runtime" "^7.7.2" + "@jimp/utils" "^0.16.2" + load-bmfont "^1.4.0" + +"@jimp/plugin-resize@^0.16.2": + version "0.16.2" + resolved "https://registry.yarnpkg.com/@jimp/plugin-resize/-/plugin-resize-0.16.2.tgz#7bcca41d9959667fb1e6e87bd6073ce0dbc43bc4" + integrity sha512-gE4N9l6xuwzacFZ2EPCGZCJ/xR+aX2V7GdMndIl/6kYIw5/eib1SFuF9AZLvIPSFuE1FnGo8+vT0pr++SSbhYg== + dependencies: + "@babel/runtime" "^7.7.2" + "@jimp/utils" "^0.16.2" + +"@jimp/plugin-rotate@^0.16.2": + version "0.16.2" + resolved "https://registry.yarnpkg.com/@jimp/plugin-rotate/-/plugin-rotate-0.16.2.tgz#deba6956eaf1d127e91389c53d5c6f59ef80d17f" + integrity sha512-/CTEYkR1HrgmnE0VqPhhbBARbDAfFX590LWGIpxcYIYsUUGQCadl+8Qo4UX13FH0Nt8UHEtPA+O2x08uPYg9UA== + dependencies: + "@babel/runtime" "^7.7.2" + "@jimp/utils" "^0.16.2" + +"@jimp/plugin-scale@^0.16.2": + version "0.16.2" + resolved "https://registry.yarnpkg.com/@jimp/plugin-scale/-/plugin-scale-0.16.2.tgz#d297e6a83f860b5e29bc5bd30ec1556561cb71ab" + integrity sha512-3inuxfrlquyLaqFdiiiQNJUurR0WbvN5wAf1qcYX2LubG1AG8grayYD6H7XVoxfUGTZXh1kpmeirEYlqA2zxcw== + dependencies: + "@babel/runtime" "^7.7.2" + "@jimp/utils" "^0.16.2" + +"@jimp/plugin-shadow@^0.16.2": + version "0.16.2" + resolved "https://registry.yarnpkg.com/@jimp/plugin-shadow/-/plugin-shadow-0.16.2.tgz#2365b0d4ade0f9641cf48b887431fe478a7ace45" + integrity sha512-Q0aIs2/L6fWMcEh9Ms73u34bT1hyUMw/oxaVoIzOLo6/E8YzCs2Bi63H0/qaPS0MQpEppI++kvosPbblABY79w== + dependencies: + "@babel/runtime" "^7.7.2" + "@jimp/utils" "^0.16.2" + +"@jimp/plugin-threshold@^0.16.2": + version "0.16.2" + resolved "https://registry.yarnpkg.com/@jimp/plugin-threshold/-/plugin-threshold-0.16.2.tgz#3b851659ab1db195b2b4e6c9901f19996a086568" + integrity sha512-gyOwmBgjtMPvcuyOhkP6dOGWbQdaTfhcBRN22mYeI/k/Wh/Zh1OI21F6eKLApsVRmg15MoFnkrCz64RROC34sw== + dependencies: + "@babel/runtime" "^7.7.2" + "@jimp/utils" "^0.16.2" + +"@jimp/plugins@^0.16.2": + version "0.16.2" + resolved "https://registry.yarnpkg.com/@jimp/plugins/-/plugins-0.16.2.tgz#bba2a7247f926fe7e13e35b24ca9552b0aae4312" + integrity sha512-zCvYtCgctmC0tkYEu+y+kSwSIZBsNznqJ3/3vkpzxdyjd6wCfNY5Qc/68MPrLc1lmdeGo4cOOTYHG7Vc6myzRw== + dependencies: + "@babel/runtime" "^7.7.2" + "@jimp/plugin-blit" "^0.16.2" + "@jimp/plugin-blur" "^0.16.2" + "@jimp/plugin-circle" "^0.16.2" + "@jimp/plugin-color" "^0.16.2" + "@jimp/plugin-contain" "^0.16.2" + "@jimp/plugin-cover" "^0.16.2" + "@jimp/plugin-crop" "^0.16.2" + "@jimp/plugin-displace" "^0.16.2" + "@jimp/plugin-dither" "^0.16.2" + "@jimp/plugin-fisheye" "^0.16.2" + "@jimp/plugin-flip" "^0.16.2" + "@jimp/plugin-gaussian" "^0.16.2" + "@jimp/plugin-invert" "^0.16.2" + "@jimp/plugin-mask" "^0.16.2" + "@jimp/plugin-normalize" "^0.16.2" + "@jimp/plugin-print" "^0.16.2" + "@jimp/plugin-resize" "^0.16.2" + "@jimp/plugin-rotate" "^0.16.2" + "@jimp/plugin-scale" "^0.16.2" + "@jimp/plugin-shadow" "^0.16.2" + "@jimp/plugin-threshold" "^0.16.2" + timm "^1.6.1" + +"@jimp/png@^0.16.2": + version "0.16.2" + resolved "https://registry.yarnpkg.com/@jimp/png/-/png-0.16.2.tgz#45af82656aad2fde0489687a538f2af903867a1b" + integrity sha512-sFOtOSz/tzDwXEChFQ/Nxe+0+vG3Tj0eUxnZVDUG/StXE9dI8Bqmwj3MIa0EgK5s+QG3YlnDOmlPUa4JqmeYeQ== + dependencies: + "@babel/runtime" "^7.7.2" + "@jimp/utils" "^0.16.2" + pngjs "^3.3.3" + +"@jimp/tiff@^0.16.2": + version "0.16.2" + resolved "https://registry.yarnpkg.com/@jimp/tiff/-/tiff-0.16.2.tgz#613870065fe1387f6a09fe9d8230c00c35b7b640" + integrity sha512-ADcdqmtZF+U2YoaaHTzFX8D6NFpmN4WZUT0BPMerEuY7Cq8QoLYU22z2h034FrVW+Rbi1b3y04sB9iDiQAlf2w== + dependencies: + "@babel/runtime" "^7.7.2" + utif "^2.0.1" + +"@jimp/types@^0.16.2": + version "0.16.2" + resolved "https://registry.yarnpkg.com/@jimp/types/-/types-0.16.2.tgz#e245281495d0c92cd73174f7ac359211882288c7" + integrity sha512-0Ue5Sq0XnDF6TirisWv5E+8uOnRcd8vRLuwocJOhF76NIlcQrz+5r2k2XWKcr3d+11n28dHLXW5TKSqrUopxhA== + dependencies: + "@babel/runtime" "^7.7.2" + "@jimp/bmp" "^0.16.2" + "@jimp/gif" "^0.16.2" + "@jimp/jpeg" "^0.16.2" + "@jimp/png" "^0.16.2" + "@jimp/tiff" "^0.16.2" + timm "^1.6.1" + +"@jimp/utils@^0.16.2": + version "0.16.2" + resolved "https://registry.yarnpkg.com/@jimp/utils/-/utils-0.16.2.tgz#e78cb82c46f608b72179a31581065bf75b35166c" + integrity sha512-XENrPvmigiXZQ8E2nxJqO6UVvWBLzbNwyYi3Y8Q1IECoYhYI3kgOQ0fmy4G269Vz1V0omh1bNmC42r4OfXg1Jg== + dependencies: + "@babel/runtime" "^7.7.2" + regenerator-runtime "^0.13.3" + "@jridgewell/resolve-uri@^3.0.3": version "3.0.5" resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.0.5.tgz#68eb521368db76d040a6315cdb24bf2483037b9c" @@ -1317,6 +1614,11 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-16.4.13.tgz#7dfd9c14661edc65cccd43a29eb454174642370d" integrity sha512-bLL69sKtd25w7p1nvg9pigE4gtKVpGTPojBFLMkGHXuUgap2sLqQt2qUnqmVCDfzGUL0DRNZP+1prIZJbMeAXg== +"@types/node@16.9.1": + version "16.9.1" + resolved "https://registry.yarnpkg.com/@types/node/-/node-16.9.1.tgz#0611b37db4246c937feef529ddcc018cf8e35708" + integrity sha512-QpLcX9ZSsq3YYUUnD3nFDY8H7wctAhQj/TFKL8Ya8v5fMm3CFXxo8zStsLAl780ltoYoo1WvKUVGBQK+1ifr7g== + "@types/node@17.0.24": version "17.0.24" resolved "https://registry.yarnpkg.com/@types/node/-/node-17.0.24.tgz#20ba1bf69c1b4ab405c7a01e950c4f446b05029f" @@ -1579,6 +1881,11 @@ ansi-styles@^5.0.0: resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-5.2.0.tgz#07449690ad45777d1924ac2abb2fc8895dba836b" integrity sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA== +any-base@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/any-base/-/any-base-1.1.0.tgz#ae101a62bc08a597b4c9ab5b7089d456630549fe" + integrity sha512-uMgjozySS8adZZYePpaWs8cxB9/kdzmpX6SgJZ+wbz1K5eYk5QMYDVJaZKhxyIHUdnnJkfR7SVgStgH7LkGUyg== + anymatch@^3.0.3: version "3.1.2" resolved "https://registry.yarnpkg.com/anymatch/-/anymatch-3.1.2.tgz#c0557c096af32f106198f4f4e2a383537e378716" @@ -1857,6 +2164,11 @@ bluebird@^3.5.0: resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f" integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg== +bmp-js@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/bmp-js/-/bmp-js-0.1.0.tgz#e05a63f796a6c1ff25f4771ec7adadc148c07233" + integrity sha512-vHdS19CnY3hwiNdkaqk93DvjVLfbEcI8mys4UjuWrlX1haDmroo8o4xCzh4wD6DGV6HxRCyauwhHRqMTfERtjw== + boolbase@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/boolbase/-/boolbase-1.0.0.tgz#68dff5fbe60c51eb37725ea9e3ed310dcc1e776e" @@ -1934,6 +2246,11 @@ buffer-equal-constant-time@1.0.1: resolved "https://registry.yarnpkg.com/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz#f8e71132f7ffe6e01a5c9697a4c6f3e48d5cc819" integrity sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk= +buffer-equal@0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/buffer-equal/-/buffer-equal-0.0.1.tgz#91bc74b11ea405bc916bc6aa908faafa5b4aac4b" + integrity sha512-RgSV6InVQ9ODPdLWJ5UAqBqJBOg370Nz6ZQtRzpt6nUjc8v0St97uJ4PYC6NztqIScrAXafKM3mZPMygSe1ggA== + buffer-fill@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/buffer-fill/-/buffer-fill-1.0.0.tgz#f8f78b76789888ef39f205cd637f68e702122b2c" @@ -1944,7 +2261,7 @@ buffer-from@^1.0.0: resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.2.tgz#2b146a6fd72e80b4f55d255f35ed59a3a9a41bd5" integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ== -buffer@^5.2.1, buffer@^5.5.0: +buffer@^5.2.0, buffer@^5.2.1, buffer@^5.5.0: version "5.7.1" resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.7.1.tgz#ba62e7c13133053582197160851a8f648e99eed0" integrity sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ== @@ -2591,6 +2908,11 @@ dom-serializer@^1.0.1, dom-serializer@^1.3.2: domhandler "^4.2.0" entities "^2.0.0" +dom-walk@^0.1.0: + version "0.1.2" + resolved "https://registry.yarnpkg.com/dom-walk/-/dom-walk-0.1.2.tgz#0c548bef048f4d1f2a97249002236060daa3fd84" + integrity sha512-6QvTW9mrGeIegrFXdtQi9pk7O/nSK6lSdXW2eqUspN5LWD7UTji2Fqw5V2YLjBpHEoU9Xl/eUWNpDeZvoyOv2w== + domelementtype@^2.0.1, domelementtype@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-2.2.0.tgz#9a0b6c2782ed6a1c7323d42267183df9bd8b1d57" @@ -3093,6 +3415,11 @@ executable@^4.1.0: dependencies: pify "^2.2.0" +exif-parser@^0.1.12: + version "0.1.12" + resolved "https://registry.yarnpkg.com/exif-parser/-/exif-parser-0.1.12.tgz#58a9d2d72c02c1f6f02a0ef4a9166272b7760922" + integrity sha512-c2bQfLNbMzLPmzQuOr8fy0csy84WmwnER81W88DzTp9CYNPJ6yzOj2EZAh9pywYpqHnshVLHQJ8WzldAyfY+Iw== + exit@^0.1.2: version "0.1.2" resolved "https://registry.yarnpkg.com/exit/-/exit-0.1.2.tgz#0632638f8d877cc82107d30a0fff1a17cba1cd0c" @@ -3272,6 +3599,11 @@ file-type@^8.1.0: resolved "https://registry.yarnpkg.com/file-type/-/file-type-8.1.0.tgz#244f3b7ef641bbe0cca196c7276e4b332399f68c" integrity sha512-qyQ0pzAy78gVoJsmYeNgl8uH8yKhr1lVhW7JbzJmnlRi0I4R2eEDEJZVKG8agpDnLpacwNbDhLNG/LMdxHD2YQ== +file-type@^9.0.0: + version "9.0.0" + resolved "https://registry.yarnpkg.com/file-type/-/file-type-9.0.0.tgz#a68d5ad07f486414dfb2c8866f73161946714a18" + integrity sha512-Qe/5NJrgIOlwijpq3B7BEpzPFcgzggOTagZmkXQY4LA6bsXKTUstK7Wp12lEJ/mLKTpvIZxmIuRcLYWT6ov9lw== + filename-reserved-regex@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/filename-reserved-regex/-/filename-reserved-regex-2.0.0.tgz#abf73dfab735d045440abfea2d91f389ebbfa229" @@ -3533,6 +3865,14 @@ gifsicle@^5.0.0: execa "^5.0.0" logalot "^2.0.0" +gifwrap@^0.9.2: + version "0.9.4" + resolved "https://registry.yarnpkg.com/gifwrap/-/gifwrap-0.9.4.tgz#f4eb6169ba027d61df64aafbdcb1f8ae58ccc0c5" + integrity sha512-MDMwbhASQuVeD4JKd1fKgNgCRL3fGqMM4WaqpNhWO0JiMOAjbQdumbs4BbBZEy9/M00EHEjKN3HieVhCUlwjeQ== + dependencies: + image-q "^4.0.0" + omggif "^1.0.10" + github-from-package@0.0.0: version "0.0.0" resolved "https://registry.yarnpkg.com/github-from-package/-/github-from-package-0.0.0.tgz#97fb5d96bfde8973313f20e8288ef9a167fa64ce" @@ -3564,6 +3904,14 @@ glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4: once "^1.3.0" path-is-absolute "^1.0.0" +global@~4.4.0: + version "4.4.0" + resolved "https://registry.yarnpkg.com/global/-/global-4.4.0.tgz#3e7b105179006a323ed71aafca3e9c57a5cc6406" + integrity sha512-wv/LAoHdRE3BeTGz53FAamhGlPLhlssK45usmGFThIi4XqnBmjKQ16u+RNbP7WvigRZDxUsM0J3gcQ5yicaL0w== + dependencies: + min-document "^2.19.0" + process "^0.11.10" + globals@^11.1.0: version "11.12.0" resolved "https://registry.yarnpkg.com/globals/-/globals-11.12.0.tgz#ab8795338868a0babd8525758018c2a7eb95c42e" @@ -3922,6 +4270,13 @@ ignore@^5.1.8, ignore@^5.2.0: resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.2.0.tgz#6d3bac8fa7fe0d45d9f9be7bac2fc279577e345a" integrity sha512-CmxgYGiEPCLhfLnpPp1MoRmifwEIOgjcHXxOBjv7mY96c+eWScsOP9c112ZyLdWHi0FxHjI+4uVhKYp/gcdRmQ== +image-q@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/image-q/-/image-q-4.0.0.tgz#31e075be7bae3c1f42a85c469b4732c358981776" + integrity sha512-PfJGVgIfKQJuq3s0tTDOKtztksibuUEbJQIYT3by6wctQo+Rdlh7ef4evJ5NCdxY4CfMbvFkocEwbl4BF8RlJw== + dependencies: + "@types/node" "16.9.1" + imagemin-gifsicle@7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/imagemin-gifsicle/-/imagemin-gifsicle-7.0.0.tgz#1a7ab136a144c4678657ba3b6c412f80805d26b0" @@ -4079,6 +4434,11 @@ is-fullwidth-code-point@^3.0.0: resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz#f116f8064fe90b3f7844a38997c0b75051269f1d" integrity sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg== +is-function@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/is-function/-/is-function-1.0.2.tgz#4f097f30abf6efadac9833b17ca5dc03f8144e08" + integrity sha512-lw7DUp0aWXYg+CBCN+JKkcE0Q2RayZnSvnZBlwgxHBQhqt5pZNVy4Ri7H9GmmXkdu7LUthszM+Tor1u/2iBcpQ== + is-generator-fn@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/is-generator-fn/-/is-generator-fn-2.1.0.tgz#7d140adc389aaf3011a8f2a2a4cfa6faadffb118" @@ -4662,6 +5022,17 @@ jest@^27.5.1: import-local "^3.0.2" jest-cli "^27.5.1" +jimp@^0.16.2: + version "0.16.2" + resolved "https://registry.yarnpkg.com/jimp/-/jimp-0.16.2.tgz#c03e296381ae37586e27f209d134d4596d112f7b" + integrity sha512-UpItBk81a92f8oEyoGYbO3YK4QcM0hoIyuGHmShoF9Ov63P5Qo7Q/X2xsAgnODmSuDJFOtrPtJd5GSWW4LKdOQ== + dependencies: + "@babel/runtime" "^7.7.2" + "@jimp/custom" "^0.16.2" + "@jimp/plugins" "^0.16.2" + "@jimp/types" "^0.16.2" + regenerator-runtime "^0.13.3" + jose@^2.0.5: version "2.0.5" resolved "https://registry.yarnpkg.com/jose/-/jose-2.0.5.tgz#29746a18d9fff7dcf9d5d2a6f62cb0c7cd27abd3" @@ -4669,6 +5040,11 @@ jose@^2.0.5: dependencies: "@panva/asn1.js" "^1.0.0" +jpeg-js@^0.4.2: + version "0.4.4" + resolved "https://registry.yarnpkg.com/jpeg-js/-/jpeg-js-0.4.4.tgz#a9f1c6f1f9f0fa80cdb3484ed9635054d28936aa" + integrity sha512-WZzeDOEtTOBK4Mdsar0IqEU5sMr3vSV2RqkAIzUEV2BHnUfKGyswWFPFwK5EeDo93K3FohSHbLAjj0s1Wzd+dg== + jptext-to-emoji@0.3.0: version "0.3.0" resolved "https://registry.yarnpkg.com/jptext-to-emoji/-/jptext-to-emoji-0.3.0.tgz#002fa4335c5b200f7b51e423640768e88c7961c7" @@ -4942,6 +5318,20 @@ lines-and-columns@^1.1.6: resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.1.6.tgz#1c00c743b433cd0a4e80758f7b64a57440d9ff00" integrity sha1-HADHQ7QzzQpOgHWPe2SldEDZ/wA= +load-bmfont@^1.3.1, load-bmfont@^1.4.0: + version "1.4.1" + resolved "https://registry.yarnpkg.com/load-bmfont/-/load-bmfont-1.4.1.tgz#c0f5f4711a1e2ccff725a7b6078087ccfcddd3e9" + integrity sha512-8UyQoYmdRDy81Brz6aLAUhfZLwr5zV0L3taTQ4hju7m6biuwiWiJXjPhBJxbUQJA8PrkvJ/7Enqmwk2sM14soA== + dependencies: + buffer-equal "0.0.1" + mime "^1.3.4" + parse-bmfont-ascii "^1.0.3" + parse-bmfont-binary "^1.0.5" + parse-bmfont-xml "^1.1.4" + phin "^2.9.1" + xhr "^2.0.1" + xtend "^4.0.0" + load-json-file@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/load-json-file/-/load-json-file-1.1.0.tgz#956905708d58b4bab4c2261b04f59f31c99374c0" @@ -5201,6 +5591,11 @@ mime-types@^2.0.8, mime-types@^2.1.12, mime-types@~2.1.19: dependencies: mime-db "1.49.0" +mime@^1.3.4: + version "1.6.0" + resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1" + integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg== + mime@^2.2.0: version "2.5.2" resolved "https://registry.yarnpkg.com/mime/-/mime-2.5.2.tgz#6e3dc6cc2b9510643830e5f19d5cb753da5eeabe" @@ -5226,6 +5621,13 @@ mimic-response@^3.1.0: resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-3.1.0.tgz#2d1d59af9c1b129815accc2c46a022a5ce1fa3c9" integrity sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ== +min-document@^2.19.0: + version "2.19.0" + resolved "https://registry.yarnpkg.com/min-document/-/min-document-2.19.0.tgz#7bd282e3f5842ed295bb748cdd9f1ffa2c824685" + integrity sha512-9Wy1B3m3f66bPPmU5hdA4DR4PB2OfDU/+GS3yAB7IQozE3tqXaVv2zOjgla7MEGSRv95+ILmOuvhLkOK6wJtCQ== + dependencies: + dom-walk "^0.1.0" + min-indent@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/min-indent/-/min-indent-1.0.1.tgz#a63f681673b30571fbe8bc25686ae746eefa9869" @@ -5252,11 +5654,23 @@ minimist@^1.1.3, minimist@^1.2.0, minimist@^1.2.3, minimist@^1.2.5: resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.5.tgz#67d66014b66a6a8aaa0c083c5fd58df4e4e97602" integrity sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw== +minimist@^1.2.6: + version "1.2.7" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.7.tgz#daa1c4d91f507390437c6a8bc01078e7000c4d18" + integrity sha512-bzfL1YUZsP41gmu/qjrEk0Q6i2ix/cVeAhbCbqH9u3zYutS1cLg00qhrD0M2MVdCcx4Sc0UpP2eBWo9rotpq6g== + mkdirp-classic@^0.5.2, mkdirp-classic@^0.5.3: version "0.5.3" resolved "https://registry.yarnpkg.com/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz#fa10c9115cc6d8865be221ba47ee9bed78601113" integrity sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A== +mkdirp@^0.5.1: + version "0.5.6" + resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.6.tgz#7def03d2432dcae4ba1d611445c48396062255f6" + integrity sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw== + dependencies: + minimist "^1.2.6" + mozjpeg@^7.0.0: version "7.1.0" resolved "https://registry.yarnpkg.com/mozjpeg/-/mozjpeg-7.1.0.tgz#23f202f3e48e98f02ed84f415358d4cbfab66c19" @@ -5444,6 +5858,11 @@ object-hash@^2.1.1: resolved "https://registry.yarnpkg.com/object-hash/-/object-hash-2.2.0.tgz#5ad518581eefc443bd763472b8ff2e9c2c0d54a5" integrity sha512-gScRMn0bS5fH+IuwyIFgnh9zBdo4DV+6GhygmWM9HyNJSgS0hScp1f5vjtm7oIIOiT9trXrShAkLFSc2IqKNgw== +omggif@^1.0.10, omggif@^1.0.9: + version "1.0.10" + resolved "https://registry.yarnpkg.com/omggif/-/omggif-1.0.10.tgz#ddaaf90d4a42f532e9e7cb3a95ecdd47f17c7b19" + integrity sha512-LMJTtvgc/nugXj0Vcrrs68Mn2D1r0zf630VNtqtpI1FEO7e+O9FP4gqs9AcnBaSEeoHIPm28u6qgPR0oyEpGSw== + once@^1.3.0, once@^1.3.1, once@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" @@ -5592,6 +6011,11 @@ p-try@^2.0.0: resolved "https://registry.yarnpkg.com/p-try/-/p-try-2.2.0.tgz#cb2868540e313d61de58fafbe35ce9004d5540e6" integrity sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ== +pako@^1.0.5: + version "1.0.11" + resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.11.tgz#6c9599d340d54dfd3946380252a35705a6b992bf" + integrity sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw== + parent-module@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/parent-module/-/parent-module-1.0.1.tgz#691d2709e78c79fae3a156622452d00762caaaa2" @@ -5599,6 +6023,29 @@ parent-module@^1.0.0: dependencies: callsites "^3.0.0" +parse-bmfont-ascii@^1.0.3: + version "1.0.6" + resolved "https://registry.yarnpkg.com/parse-bmfont-ascii/-/parse-bmfont-ascii-1.0.6.tgz#11ac3c3ff58f7c2020ab22769079108d4dfa0285" + integrity sha512-U4RrVsUFCleIOBsIGYOMKjn9PavsGOXxbvYGtMOEfnId0SVNsgehXh1DxUdVPLoxd5mvcEtvmKs2Mmf0Mpa1ZA== + +parse-bmfont-binary@^1.0.5: + version "1.0.6" + resolved "https://registry.yarnpkg.com/parse-bmfont-binary/-/parse-bmfont-binary-1.0.6.tgz#d038b476d3e9dd9db1e11a0b0e53a22792b69006" + integrity sha512-GxmsRea0wdGdYthjuUeWTMWPqm2+FAd4GI8vCvhgJsFnoGhTrLhXDDupwTo7rXVAgaLIGoVHDZS9p/5XbSqeWA== + +parse-bmfont-xml@^1.1.4: + version "1.1.4" + resolved "https://registry.yarnpkg.com/parse-bmfont-xml/-/parse-bmfont-xml-1.1.4.tgz#015319797e3e12f9e739c4d513872cd2fa35f389" + integrity sha512-bjnliEOmGv3y1aMEfREMBJ9tfL3WR0i0CKPj61DnSLaoxWR3nLrsQrEbCId/8rF4NyRF0cCqisSVXyQYWM+mCQ== + dependencies: + xml-parse-from-string "^1.0.0" + xml2js "^0.4.5" + +parse-headers@^2.0.0: + version "2.0.5" + resolved "https://registry.yarnpkg.com/parse-headers/-/parse-headers-2.0.5.tgz#069793f9356a54008571eb7f9761153e6c770da9" + integrity sha512-ft3iAoLOB/MlwbNXgzy43SWGP6sQki2jQvAyBg/zDFAgr9bfNWZIUj42Kw2eJIl8kEi4PbgE6U1Zau/HwI75HA== + parse-json@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/parse-json/-/parse-json-2.2.0.tgz#f480f40434ef80741f8469099f8dea18f55a4dc9" @@ -5689,6 +6136,11 @@ performance-now@^2.1.0: resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" integrity sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns= +phin@^2.9.1: + version "2.9.3" + resolved "https://registry.yarnpkg.com/phin/-/phin-2.9.3.tgz#f9b6ac10a035636fb65dfc576aaaa17b8743125c" + integrity sha512-CzFr90qM24ju5f88quFC/6qohjC144rehe5n6DH900lgXmUe86+xCKc10ev56gRKC4/BkHUoG4uSiQgBiIXwDA== + picocolors@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.0.tgz#cb5bdc74ff3f51892236eaf79d68bc44564ab81c" @@ -5731,6 +6183,13 @@ pirates@^4.0.4: resolved "https://registry.yarnpkg.com/pirates/-/pirates-4.0.5.tgz#feec352ea5c3268fb23a37c702ab1699f35a5f3b" integrity sha512-8V9+HQPupnaXMA23c5hvl69zXvTwTzyAYasnkb0Tts4XvO4CliqONMOnvlq26rkhLC3nWDFBJf73LU1e1VZLaQ== +pixelmatch@^4.0.2: + version "4.0.2" + resolved "https://registry.yarnpkg.com/pixelmatch/-/pixelmatch-4.0.2.tgz#8f47dcec5011b477b67db03c243bc1f3085e8854" + integrity sha512-J8B6xqiO37sU/gkcMglv6h5Jbd9xNER7aHzpfRdNmV4IbQBzBpe4l9XmbG+xPF/znacgu2jfEw+wHffaq/YkXA== + dependencies: + pngjs "^3.0.0" + pkg-dir@^4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/pkg-dir/-/pkg-dir-4.2.0.tgz#f099133df7ede422e81d1d8448270eeb3e4261f3" @@ -5738,6 +6197,11 @@ pkg-dir@^4.2.0: dependencies: find-up "^4.0.0" +pngjs@^3.0.0, pngjs@^3.3.3: + version "3.4.0" + resolved "https://registry.yarnpkg.com/pngjs/-/pngjs-3.4.0.tgz#99ca7d725965fb655814eaf65f38f12bbdbf555f" + integrity sha512-NCrCHhWmnQklfH4MtJMRjZ2a8c80qXeMlQMv2uVp9ISJMTt562SbGd6n2oq0PaPgKm7Z6pL9E2UlLIhC+SHL3w== + pngquant-bin@^6.0.0: version "6.0.0" resolved "https://registry.yarnpkg.com/pngquant-bin/-/pngquant-bin-6.0.0.tgz#aff0d7e61095feb96ced379ad8c7294ad3dd1712" @@ -5801,6 +6265,11 @@ process-nextick-args@~2.0.0: resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2" integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag== +process@^0.11.10: + version "0.11.10" + resolved "https://registry.yarnpkg.com/process/-/process-0.11.10.tgz#7332300e840161bda3e69a1d1d91a7d4bc16f182" + integrity sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A== + prompts@^2.0.1: version "2.4.1" resolved "https://registry.yarnpkg.com/prompts/-/prompts-2.4.1.tgz#befd3b1195ba052f9fd2fde8a486c4e82ee77f61" @@ -6005,6 +6474,11 @@ redent@^3.0.0: indent-string "^4.0.0" strip-indent "^3.0.0" +regenerator-runtime@^0.13.11, regenerator-runtime@^0.13.3: + version "0.13.11" + resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz#f6dca3e7ceec20590d07ada785636a90cdca17f9" + integrity sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg== + regexpp@^3.2.0: version "3.2.0" resolved "https://registry.yarnpkg.com/regexpp/-/regexpp-3.2.0.tgz#0425a2768d8f23bad70ca4b90461fa2f1213e1b2" @@ -6166,6 +6640,11 @@ safe-buffer@>=5.1.0, safe-buffer@^5.0.1, safe-buffer@^5.1.1, safe-buffer@^5.1.2, resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== +sax@>=0.6.0: + version "1.2.4" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" + integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw== + saxes@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/saxes/-/saxes-5.0.1.tgz#eebab953fa3b7608dbe94e5dadb15c888fa6696d" @@ -6721,6 +7200,16 @@ timed-out@^4.0.0, timed-out@^4.0.1: resolved "https://registry.yarnpkg.com/timed-out/-/timed-out-4.0.1.tgz#f32eacac5a175bea25d7fab565ab3ed8741ef56f" integrity sha1-8y6srFoXW+ol1/q1Zas+2HQe9W8= +timm@^1.6.1: + version "1.7.1" + resolved "https://registry.yarnpkg.com/timm/-/timm-1.7.1.tgz#96bab60c7d45b5a10a8a4d0f0117c6b7e5aff76f" + integrity sha512-IjZc9KIotudix8bMaBW6QvMuq64BrJWFs1+4V0lXwWGQZwH+LnX87doAYhem4caOEusRP9/g6jVDQmZ8XOk1nw== + +tinycolor2@^1.4.1: + version "1.5.1" + resolved "https://registry.yarnpkg.com/tinycolor2/-/tinycolor2-1.5.1.tgz#df11c5f14e6b7fdd8a9c27c2c6a5f2565fb776b7" + integrity sha512-BHlrsGeYN2OpkRpfAgkEwCMu6w8Quq8JkK/mp4c55NZP7OwceJObR1CPZt62TqiA0Y3J5pwuDX+fXDqc35REtg== + tmpl@1.0.x: version "1.0.4" resolved "https://registry.yarnpkg.com/tmpl/-/tmpl-1.0.4.tgz#23640dd7b42d00433911140820e5cf440e521dd1" @@ -6947,6 +7436,13 @@ url-to-options@^1.0.1: resolved "https://registry.yarnpkg.com/url-to-options/-/url-to-options-1.0.1.tgz#1505a03a289a48cbd7a434efbaeec5055f5633a9" integrity sha1-FQWgOiiaSMvXpDTvuu7FBV9WM6k= +utif@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/utif/-/utif-2.0.1.tgz#9e1582d9bbd20011a6588548ed3266298e711759" + integrity sha512-Z/S1fNKCicQTf375lIP9G8Sa1H/phcysstNrrSdZKj1f9g58J4NMgb5IgiEZN9/nLMPDwF0W7hdOe9Qq2IYoLg== + dependencies: + pako "^1.0.5" + util-deprecate@^1.0.1, util-deprecate@~1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" @@ -7124,11 +7620,39 @@ xdg-basedir@^4.0.0: resolved "https://registry.yarnpkg.com/xdg-basedir/-/xdg-basedir-4.0.0.tgz#4bc8d9984403696225ef83a1573cbbcb4e79db13" integrity sha512-PSNhEJDejZYV7h50BohL09Er9VaIefr2LMAf3OEmpCkjOi34eYyQYAXUTjEQtZJTKcF0E2UKTh+osDLsgNim9Q== +xhr@^2.0.1: + version "2.6.0" + resolved "https://registry.yarnpkg.com/xhr/-/xhr-2.6.0.tgz#b69d4395e792b4173d6b7df077f0fc5e4e2b249d" + integrity sha512-/eCGLb5rxjx5e3mF1A7s+pLlR6CGyqWN91fv1JgER5mVWg1MZmlhBvy9kjcsOdRk8RrIujotWyJamfyrp+WIcA== + dependencies: + global "~4.4.0" + is-function "^1.0.1" + parse-headers "^2.0.0" + xtend "^4.0.0" + xml-name-validator@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a" integrity sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw== +xml-parse-from-string@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/xml-parse-from-string/-/xml-parse-from-string-1.0.1.tgz#a9029e929d3dbcded169f3c6e28238d95a5d5a28" + integrity sha512-ErcKwJTF54uRzzNMXq2X5sMIy88zJvfN2DmdoQvy7PAFJ+tPRU6ydWuOKNMyfmOjdyBQTFREi60s0Y0SyI0G0g== + +xml2js@^0.4.5: + version "0.4.23" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66" + integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug== + dependencies: + sax ">=0.6.0" + xmlbuilder "~11.0.0" + +xmlbuilder@~11.0.0: + version "11.0.1" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3" + integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA== + xmlchars@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb" From 666b0fad61a778f5b1f0291e35dfd0bbc938f2b3 Mon Sep 17 00:00:00 2001 From: vipzero Date: Mon, 2 Jan 2023 12:56:35 +0900 Subject: [PATCH 071/105] fix test --- src/tests/imageIO.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/tests/imageIO.test.ts b/src/tests/imageIO.test.ts index fb2d2e85..bcf17056 100644 --- a/src/tests/imageIO.test.ts +++ b/src/tests/imageIO.test.ts @@ -1,4 +1,5 @@ import { choiceImage } from '../imageIo/uploadManage' +import { CacheFile } from '../types' test('choiceImage', () => { const basePng = { @@ -6,8 +7,9 @@ test('choiceImage', () => { size: 100_000, height: 700, width: 800, + hash: 'ffffff', } - const downloads = [ + const downloads: CacheFile[] = [ { ...basePng, filePath: 'will skipped', From da4aa3d31cc9fd9c8bf6380771db81c2615f9eb5 Mon Sep 17 00:00:00 2001 From: vipzero Date: Mon, 2 Jan 2023 12:57:00 +0900 Subject: [PATCH 072/105] add artwork to imageLinks --- src/index.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/index.ts b/src/index.ts index 7797d67b..e4b5ac77 100644 --- a/src/index.ts +++ b/src/index.ts @@ -59,6 +59,9 @@ export async function icyToSong( prevCounts, additionals ) + if (albumInfos?.artworkUrl100) { + imageLinks.push(albumInfos.artworkUrl100) + } const compSong: Song = { ...song, From d666e530d435a468c48f1405c511c71030b3e44e Mon Sep 17 00:00:00 2001 From: vipzero Date: Mon, 2 Jan 2023 13:17:38 +0900 Subject: [PATCH 073/105] fix download --- src/imageIo/download.ts | 13 +++++++++---- src/imageIo/uploadManage.ts | 10 +++------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/imageIo/download.ts b/src/imageIo/download.ts index be18843b..0792a43b 100644 --- a/src/imageIo/download.ts +++ b/src/imageIo/download.ts @@ -26,11 +26,12 @@ const mimeMap: Record = { svg: mimeSvg, } const fileTypeDefault = mimePng -const gotOption = { timeout: { request: 5000 } } +const gotOption = { timeout: { request: 3000 } } export const downloadOptimize = async ( url: string ): Promise => { + // log('s: ' + url) const filePath = `tmp/${uuidv4()}` const stream = got.stream(url, gotOption) @@ -38,8 +39,12 @@ export const downloadOptimize = async ( let res try { res = await pipeline(stream, fs.createWriteStream(filePath)).catch((e) => { - error(`DownloadSaveError`, `${url} ${filePath}`) - log(JSON.stringify(e)) + if (e.name === 'TimeoutError') { + log(`Timeout`, `${url}`) + } else { + error(`DownloadSaveError`, `${url} ${filePath}`) + log(JSON.stringify(e)) + } return 'SaveError' as const }) } catch (e) { @@ -71,7 +76,7 @@ export const downloadOptimize = async ( if (!resj) return false const { hash } = resj log(` jimp: ${Date.now() - ts}ms`) - log(` size: ${width}x${height}`) + // log(` size: ${width}x${height}`) return { filePath, fileType, size, height, width, hash } } diff --git a/src/imageIo/uploadManage.ts b/src/imageIo/uploadManage.ts index f71362f0..e4b172c0 100644 --- a/src/imageIo/uploadManage.ts +++ b/src/imageIo/uploadManage.ts @@ -1,6 +1,6 @@ import { unlink } from 'fs/promises' import { UploadFile } from '../types/index' -import { error } from '../utils/logger' +import { error, log } from '../utils/logger' import { uploadStorage } from './../service/firebase' import { CacheFile } from './../types/index' import { downloadOptimize } from './download' @@ -10,16 +10,12 @@ const nonFalse = (v: T | false): v is T => v !== false export const uploadByUrlAll = async (urls: string[]) => { const timeId = +new Date() + log(urls) + log(urls.length) const downloads: CacheFile[] = ( await Promise.all(urls.map(downloadOptimize)) ).filter((v) => nonFalse(v)) as CacheFile[] - for (const url of urls) { - const res = await downloadOptimize(url) - if (!res) continue - downloads.push(res) - } - const uploads: UploadFile[] = [] const selects = choiceImage(downloads) const hashs: string[] = [] From ee96adae3618721f7006770619978accef5c2cf8 Mon Sep 17 00:00:00 2001 From: vipzero Date: Mon, 2 Jan 2023 16:32:48 +0900 Subject: [PATCH 074/105] fix tests --- src/tests/convertTimeTags.test.ts | 28 ++++++++++++++-------------- src/tests/imageIO.test.ts | 6 ++++++ src/utils/makeSearchWord.ts | 6 +++--- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/src/tests/convertTimeTags.test.ts b/src/tests/convertTimeTags.test.ts index a09baf16..001fa3a0 100644 --- a/src/tests/convertTimeTags.test.ts +++ b/src/tests/convertTimeTags.test.ts @@ -5,14 +5,14 @@ test('convertTimeTags', () => { Array [ "[2000]", "[2000-01]", - "[2000-s1]", + "[2000-S1]", ] `) expect(convertTimeTags('2010-05-20')).toMatchInlineSnapshot(` Array [ "[2010]", "[2010-05]", - "[2010-s2]", + "[2010-S2]", ] `) const range = (n: number) => [...Array(n).keys()] @@ -23,18 +23,18 @@ Array [ expect(res.map((v) => v[2])).toMatchInlineSnapshot(` Array [ - "[2000-s1]", - "[2000-s1]", - "[2000-s1]", - "[2000-s2]", - "[2000-s2]", - "[2000-s2]", - "[2000-s3]", - "[2000-s3]", - "[2000-s3]", - "[2000-s4]", - "[2000-s4]", - "[2000-s4]", + "[2000-S1]", + "[2000-S1]", + "[2000-S1]", + "[2000-S2]", + "[2000-S2]", + "[2000-S2]", + "[2000-S3]", + "[2000-S3]", + "[2000-S3]", + "[2000-S4]", + "[2000-S4]", + "[2000-S4]", ] `) }) diff --git a/src/tests/imageIO.test.ts b/src/tests/imageIO.test.ts index bcf17056..15f1059d 100644 --- a/src/tests/imageIO.test.ts +++ b/src/tests/imageIO.test.ts @@ -52,6 +52,7 @@ Array [ "ext": "png", "mime": "image/png", }, + "hash": "ffffff", "height": 700, "size": 100000, "width": 800, @@ -62,6 +63,7 @@ Array [ "ext": "png", "mime": "image/png", }, + "hash": "ffffff", "height": 1000, "size": 100000, "width": 30, @@ -72,6 +74,7 @@ Array [ "ext": "jpg", "mime": "image/jpeg", }, + "hash": "ffffff", "height": 700, "size": 100000, "width": 800, @@ -82,6 +85,7 @@ Array [ "ext": "png", "mime": "image/png", }, + "hash": "ffffff", "height": 500, "size": 100000, "width": 500, @@ -92,6 +96,7 @@ Array [ "ext": "png", "mime": "image/png", }, + "hash": "ffffff", "height": 700, "size": 555117, "width": 800, @@ -102,6 +107,7 @@ Array [ "ext": "png", "mime": "image/png", }, + "hash": "ffffff", "height": 30, "size": 100000, "width": 80, diff --git a/src/utils/makeSearchWord.ts b/src/utils/makeSearchWord.ts index ae617e78..c29c0371 100644 --- a/src/utils/makeSearchWord.ts +++ b/src/utils/makeSearchWord.ts @@ -7,14 +7,14 @@ import { shuffle } from './random' /* ランダムに追加する */ const animeExt = [ - 'meme', + 'キービジュアル', 'キャプ画', - 'キャラ', + 'イラストレーター', 'かわいい', 'グッズ', 'ネタ画像', '壁紙', - '相関図', + '原作', '名シーン', '作画', 'wallpaper', From 08052d3dc622cec8b90804d74ae822c02a78a8bb Mon Sep 17 00:00:00 2001 From: vipzero Date: Mon, 2 Jan 2023 16:35:26 +0900 Subject: [PATCH 075/105] refactor --- src/service/firebase.ts | 35 ++++++++++++++--------------------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/src/service/firebase.ts b/src/service/firebase.ts index 0c240b50..8084232f 100644 --- a/src/service/firebase.ts +++ b/src/service/firebase.ts @@ -6,13 +6,6 @@ import { CacheFile } from './../types/index' export { admin } -// const P_SONG = 'song' -// const P_FEEDBACK = 'feedback' -// const P_VOTE = 'vote' -// const P_BOOKS = 'books' -// const P_TABLE = 'table' -// const P_CVOTE = 'cvote' - const P_SONGS = 'songs' const P_SONG = 'song' // const P_EMOL = 'emol' @@ -107,7 +100,7 @@ export const addHistory = async ( export const loadAllIcy = async () => { const snap = await fdb - .collection('hist') + .collection(P_HIST) .doc(EVENT_ID) .collection('songs') .get() @@ -116,11 +109,11 @@ export const loadAllIcy = async () => { export const loadWordCounts = async () => { const snap = await fdb - .collection('hist') + .collection(P_HIST) .doc(EVENT_ID) .collection(P_COUNTS) .get() - const hist = await fdb.collection('hist').doc(EVENT_ID).get() + const hist = await fdb.collection(P_HIST).doc(EVENT_ID).get() const lasttime = (hist.exists && (hist.data() as HistTop).lasttime) || 0 const counts: Counts = {} @@ -140,7 +133,7 @@ export const setupCount = async (counts: Counts, lasttime: number) => { ents.forEach(([k, v]) => { if (!k || !v) return batch.set( - fdb.collection('hist').doc(EVENT_ID).collection(P_COUNTS).doc(), + fdb.collection(P_HIST).doc(EVENT_ID).collection(P_COUNTS).doc(), { word: k, count: v, @@ -149,14 +142,14 @@ export const setupCount = async (counts: Counts, lasttime: number) => { }) await batch.commit() } - fdb.collection('hist').doc(EVENT_ID).set({ lasttime }, { merge: true }) + fdb.collection(P_HIST).doc(EVENT_ID).set({ lasttime }, { merge: true }) } export const setupHistN500 = async (ns: Record) => { const batch = fdb.batch() for (const [id, n] of Object.entries(ns)) { batch.update( - fdb.collection('hist').doc(EVENT_ID).collection('song').doc(id), + fdb.collection(P_HIST).doc(EVENT_ID).collection('song').doc(id), { n } ) } @@ -180,7 +173,7 @@ export const countupWordsEntry = async (words: Record) => { const check: Record = {} for (const ws of chunk(Object.entries(words), 10)) { const docs = await fdb - .collection('hist') + .collection(P_HIST) .doc(EVENT_ID) .collection(P_COUNTS) .where( @@ -199,7 +192,7 @@ export const countupWordsEntry = async (words: Record) => { } Object.entries(words).forEach(([word, count]) => { if (check[word]) return - batch.set(fdb.collection('hist').doc(EVENT_ID).collection(P_COUNTS).doc(), { + batch.set(fdb.collection(P_HIST).doc(EVENT_ID).collection(P_COUNTS).doc(), { word, count, }) @@ -213,7 +206,7 @@ export const countupWords = async (words: string[]) => { const check: Record = {} for (const ws of chunk(words, 10)) { const docs = await fdb - .collection('hist') + .collection(P_HIST) .doc(EVENT_ID) .collection(P_COUNTS) .where('word', 'in', ws) @@ -225,12 +218,12 @@ export const countupWords = async (words: string[]) => { } words.forEach((word) => { if (check[word]) return - batch.set(fdb.collection('hist').doc(EVENT_ID).collection(P_COUNTS).doc(), { + batch.set(fdb.collection(P_HIST).doc(EVENT_ID).collection(P_COUNTS).doc(), { word, count: 1, }) }) - batch.update(fdb.collection('hist').doc(EVENT_ID), { lasttime: +new Date() }) + batch.update(fdb.collection(P_HIST).doc(EVENT_ID), { lasttime: +new Date() }) await batch.commit() } @@ -239,7 +232,7 @@ export const countupWords = async (words: string[]) => { // const check = {} // for (const ws of chunk(Object.keys(words), 10)) { // const docs = await fdb -// .collection('hist') +// .collection(P_HIST) // .doc(EVENT_ID) // .collection('counts') // .where('word', 'in', ws) @@ -253,12 +246,12 @@ export const countupWords = async (words: string[]) => { // } // Object.keys(words).forEach((word) => { // if (check[word]) return -// batch.set(fdb.collection('hist').doc(EVENT_ID).collection('counts').doc(), { +// batch.set(fdb.collection(P_HIST).doc(EVENT_ID).collection('counts').doc(), { // word, // count: 1, // }) // }) -// batch.update(fdb.collection('hist').doc(EVENT_ID), { lasttime: +new Date() }) +// batch.update(fdb.collection(P_HIST).doc(EVENT_ID), { lasttime: +new Date() }) // await batch.commit() // } From 02b975e5409efe52e0b811ab8610e4d3af29ae41 Mon Sep 17 00:00:00 2001 From: vipzero Date: Mon, 2 Jan 2023 17:57:23 +0900 Subject: [PATCH 076/105] update warnDec logger --- src/imageIo/jimp.ts | 4 ++-- src/service/firebase.ts | 4 ++-- src/utils/logger.ts | 4 +++- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/imageIo/jimp.ts b/src/imageIo/jimp.ts index 4df69c1f..0e0397ec 100644 --- a/src/imageIo/jimp.ts +++ b/src/imageIo/jimp.ts @@ -1,11 +1,11 @@ import { writeFile } from 'fs/promises' import { compareHashes, read } from 'jimp' -import { warn } from '../utils/logger' +import { warnDesc } from '../utils/logger' export async function jimpHash(path: string, mime: string) { const img = await read(path).catch(() => false as const) if (img === false) { - warn('read error', path) + warnDesc('read error', path) return false } diff --git a/src/service/firebase.ts b/src/service/firebase.ts index 8084232f..5d7d9841 100644 --- a/src/service/firebase.ts +++ b/src/service/firebase.ts @@ -1,7 +1,7 @@ import admin from 'firebase-admin' import { Count, Counts, HistoryRaw, HistTop, Song } from '../types/index' import { chunk, textNormalize } from '../utils' -import { error, log, warn } from '../utils/logger' +import { error, log, warnDesc } from '../utils/logger' import { CacheFile } from './../types/index' export { admin } @@ -263,7 +263,7 @@ export const deleteFile = (path: string) => { .delete() .catch((e) => { if (e.code === 404) { - warn(`NoDeleteTargetWarn`, `no delete target ${path}`) + warnDesc(`NoDeleteTargetWarn`, `no delete target ${path}`) return } error(`DeleteFileError`, path) diff --git a/src/utils/logger.ts b/src/utils/logger.ts index 6db967bf..41ad12e2 100644 --- a/src/utils/logger.ts +++ b/src/utils/logger.ts @@ -7,8 +7,10 @@ export const info = (str: string | number | object) => export const error = (key: string, description: string) => { console.error(chalk.red(`${key}: ${description}`)) } +export const warn = (str: string | number | object) => + console.warn(chalk.yellow(str)) -export const warn = (key: string, description: string) => { +export const warnDesc = (key: string, description: string) => { console.warn(chalk.yellow(`${key}: ${description}`)) } From 383a194c314546d8f65d773ddb2d5a04268b4628 Mon Sep 17 00:00:00 2001 From: vipzero Date: Mon, 2 Jan 2023 17:57:34 +0900 Subject: [PATCH 077/105] update performance log --- src/imageIo/download.ts | 41 +++++++++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/src/imageIo/download.ts b/src/imageIo/download.ts index 0792a43b..f769c28f 100644 --- a/src/imageIo/download.ts +++ b/src/imageIo/download.ts @@ -3,7 +3,7 @@ import got from 'got' import stream from 'stream' import { promisify } from 'util' import { CacheFile } from '../types' -import { error, log, warn } from '../utils/logger' +import { error, info, log, warn, warnDesc } from '../utils/logger' import { jimpHash } from './jimp' import { sharpMin } from './sharp' @@ -28,14 +28,33 @@ const mimeMap: Record = { const fileTypeDefault = mimePng const gotOption = { timeout: { request: 3000 } } +const putil = () => { + let prev = performance.now() + + return { + mark(name: string) { + const cur = performance.now() + const ms = Math.floor(cur - prev) + if (ms < 1000) { + info(`${name}${ms}ms`) + } else { + warn(`${name}${ms}ms`) + } + + prev = cur + }, + } +} + export const downloadOptimize = async ( url: string ): Promise => { + const time = putil() + // log('s: ' + url) const filePath = `tmp/${uuidv4()}` const stream = got.stream(url, gotOption) - let ts = Date.now() let res try { res = await pipeline(stream, fs.createWriteStream(filePath)).catch((e) => { @@ -48,17 +67,16 @@ export const downloadOptimize = async ( return 'SaveError' as const }) } catch (e) { - warn(`out-DownloadSaveError`, JSON.stringify(e)) + warnDesc(`out-DownloadSaveError`, JSON.stringify(e)) res = 'SaveError' } - log(` dw: ${Date.now() - ts}ms`) + + time.mark(` dw: `) if (res === 'SaveError') return false // await imageMin(filePath) - ts = Date.now() - const shapeRes = await sharpMin(filePath).catch((e) => { - warn('UnsupportedError', e) + warnDesc('UnsupportedError', e) return false as const }) if (!shapeRes) return false @@ -66,17 +84,16 @@ export const downloadOptimize = async ( const { size, height, width, format } = shapeRes const fileType = mimeMap[format] || fileTypeDefault - log(` shape: ${Date.now() - ts}ms`) + time.mark(` shape: `) - ts = Date.now() const resj = await jimpHash(filePath, fileType.mime).catch((e) => { - warn('JimpError', e) + warnDesc('JimpError', e) return false as const }) if (!resj) return false const { hash } = resj - log(` jimp: ${Date.now() - ts}ms`) - // log(` size: ${width}x${height}`) + time.mark(` jimp: `) + // time.mark(` size: `) return { filePath, fileType, size, height, width, hash } } From 7824f80dc9cb65b71b3dbe01b69adc90f6f64323 Mon Sep 17 00:00:00 2001 From: vipzero Date: Tue, 3 Jan 2023 12:16:06 +0900 Subject: [PATCH 078/105] fix catch --- src/imageIo/download.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/imageIo/download.ts b/src/imageIo/download.ts index f769c28f..3c58fd0e 100644 --- a/src/imageIo/download.ts +++ b/src/imageIo/download.ts @@ -53,10 +53,10 @@ export const downloadOptimize = async ( // log('s: ' + url) const filePath = `tmp/${uuidv4()}` - const stream = got.stream(url, gotOption) let res try { + const stream = got.stream(url, gotOption) res = await pipeline(stream, fs.createWriteStream(filePath)).catch((e) => { if (e.name === 'TimeoutError') { log(`Timeout`, `${url}`) From d3bc2fbc32378550b7b378c2867ecbd4e532d22e Mon Sep 17 00:00:00 2001 From: vipzero Date: Fri, 28 Apr 2023 13:45:50 +0900 Subject: [PATCH 079/105] fix doc --- maintanace-flow.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maintanace-flow.md b/maintanace-flow.md index 01a407cf..1d2d597d 100644 --- a/maintanace-flow.md +++ b/maintanace-flow.md @@ -93,5 +93,5 @@ appB ->> fbco : update イベント中の履歴は Firestore からロードするが、アーカイブ後は履歴を Cloud Storage からダウンロードするようになる。 ``` -yarn close:archive # EVENT_ID から動作 +yarn close:archive {event_ia} ``` From 4fafae5a23dfd292144cfcb3ceb7fbb0d53438a9 Mon Sep 17 00:00:00 2001 From: vipzero Date: Fri, 28 Apr 2023 13:49:21 +0900 Subject: [PATCH 080/105] fix archive --- scripts/updateAnisonDb.mjs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/updateAnisonDb.mjs b/scripts/updateAnisonDb.mjs index 5afc950e..beca749c 100755 --- a/scripts/updateAnisonDb.mjs +++ b/scripts/updateAnisonDb.mjs @@ -8,9 +8,10 @@ const files = ['anison', 'program', 'sf', 'game'] for (const file of files) { await $`wget http://anison.info/data/download/${file}.zip` - await $`unzip ${file}.zip` + await $`unzip -o ${file}.zip` await $`mv -f ${file}.csv ../data` } await cd('../') -await rm('workspace') +await $`rm -rf workspace` +// await rm('workspace') From 538df59e2e772cda66c66f71265d566d7b3bccef Mon Sep 17 00:00:00 2001 From: vipzero Date: Fri, 28 Apr 2023 19:55:28 +0900 Subject: [PATCH 081/105] add jimp timeout --- src/imageIo/download.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/imageIo/download.ts b/src/imageIo/download.ts index 3c58fd0e..d53c6e2e 100644 --- a/src/imageIo/download.ts +++ b/src/imageIo/download.ts @@ -6,6 +6,7 @@ import { CacheFile } from '../types' import { error, info, log, warn, warnDesc } from '../utils/logger' import { jimpHash } from './jimp' import { sharpMin } from './sharp' +import { sleep } from '../utils' const uuidv4 = require('uuid/v4') @@ -86,10 +87,18 @@ export const downloadOptimize = async ( time.mark(` shape: `) - const resj = await jimpHash(filePath, fileType.mime).catch((e) => { + const jimpTask = jimpHash(filePath, fileType.mime).catch((e) => { warnDesc('JimpError', e) return false as const }) + const resj = await Promise.race([ + jimpTask, + (async () => { + await sleep(10000) + return false as const + })(), + ]) + if (!resj) return false const { hash } = resj time.mark(` jimp: `) From 5b39895e13ea8b82d09f2f30608e2de50950a01b Mon Sep 17 00:00:00 2001 From: vipzero Date: Sat, 29 Apr 2023 01:23:49 +0900 Subject: [PATCH 082/105] add infos --- src/imageIo/download.ts | 2 +- src/imageIo/jimp.ts | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/imageIo/download.ts b/src/imageIo/download.ts index d53c6e2e..bae9b781 100644 --- a/src/imageIo/download.ts +++ b/src/imageIo/download.ts @@ -27,7 +27,7 @@ const mimeMap: Record = { svg: mimeSvg, } const fileTypeDefault = mimePng -const gotOption = { timeout: { request: 3000 } } +const gotOption = { timeout: { response: 10_000 } } const putil = () => { let prev = performance.now() diff --git a/src/imageIo/jimp.ts b/src/imageIo/jimp.ts index 0e0397ec..8847f405 100644 --- a/src/imageIo/jimp.ts +++ b/src/imageIo/jimp.ts @@ -1,6 +1,6 @@ import { writeFile } from 'fs/promises' import { compareHashes, read } from 'jimp' -import { warnDesc } from '../utils/logger' +import { info, warnDesc } from '../utils/logger' export async function jimpHash(path: string, mime: string) { const img = await read(path).catch(() => false as const) @@ -16,6 +16,9 @@ export async function jimpHash(path: string, mime: string) { () => false as const ) if (res2 === false) return false + const befSize = Buffer.byteLength(await img.getBufferAsync(mime)) + const aftSize = Buffer.byteLength(await res.getBufferAsync(mime)) + info(`${befSize} => ${aftSize}`) return { hash: res.hash(), height: res.getHeight(), width: res.getWidth() } } From a35e5e5d62bcbfaf709a45ad3b1ce6252342b8d2 Mon Sep 17 00:00:00 2001 From: vipzero Date: Sat, 29 Apr 2023 01:51:27 +0900 Subject: [PATCH 083/105] update compact algo --- src/imageIo/jimp.ts | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/imageIo/jimp.ts b/src/imageIo/jimp.ts index 8847f405..8b5562e1 100644 --- a/src/imageIo/jimp.ts +++ b/src/imageIo/jimp.ts @@ -1,8 +1,14 @@ -import { writeFile } from 'fs/promises' +import { statSync } from 'fs' import { compareHashes, read } from 'jimp' import { info, warnDesc } from '../utils/logger' +const QUALITY_MAP: Record = { + 'image/jpeg': 50, + 'image/png': 70, +} + export async function jimpHash(path: string, mime: string) { + const befSize = statSync(path).size const img = await read(path).catch(() => false as const) if (img === false) { warnDesc('read error', path) @@ -10,15 +16,9 @@ export async function jimpHash(path: string, mime: string) { return false } - // const res = await img.scaleToFit(1500, 1500).quality(60) - const res = await img.quality(60) - const res2 = await writeFile(path, await res.getBufferAsync(mime)).catch( - () => false as const - ) - if (res2 === false) return false - const befSize = Buffer.byteLength(await img.getBufferAsync(mime)) - const aftSize = Buffer.byteLength(await res.getBufferAsync(mime)) - info(`${befSize} => ${aftSize}`) + const res = await img.quality(QUALITY_MAP[mime] || 80).write(path) + const aftSize = statSync(path).size + info(`${befSize / 1000}KB => ${aftSize / 1000}KB`) return { hash: res.hash(), height: res.getHeight(), width: res.getWidth() } } From 58f8567bff700bd1b239626b7e47f65b2e422f75 Mon Sep 17 00:00:00 2001 From: vipzero Date: Sat, 29 Apr 2023 01:59:32 +0900 Subject: [PATCH 084/105] add shape timeout --- src/imageIo/download.ts | 13 ++++--------- src/utils/index.ts | 13 +++++++++++++ 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/imageIo/download.ts b/src/imageIo/download.ts index bae9b781..ccfb11f9 100644 --- a/src/imageIo/download.ts +++ b/src/imageIo/download.ts @@ -6,7 +6,7 @@ import { CacheFile } from '../types' import { error, info, log, warn, warnDesc } from '../utils/logger' import { jimpHash } from './jimp' import { sharpMin } from './sharp' -import { sleep } from '../utils' +import { raseTimeout, sleep } from '../utils' const uuidv4 = require('uuid/v4') @@ -76,10 +76,11 @@ export const downloadOptimize = async ( if (res === 'SaveError') return false // await imageMin(filePath) - const shapeRes = await sharpMin(filePath).catch((e) => { + const shapeTask = sharpMin(filePath).catch((e) => { warnDesc('UnsupportedError', e) return false as const }) + const shapeRes = await raseTimeout(shapeTask, 10000, false as const) if (!shapeRes) return false const { size, height, width, format } = shapeRes @@ -91,13 +92,7 @@ export const downloadOptimize = async ( warnDesc('JimpError', e) return false as const }) - const resj = await Promise.race([ - jimpTask, - (async () => { - await sleep(10000) - return false as const - })(), - ]) + const resj = await raseTimeout(jimpTask, 10000, false as const) if (!resj) return false const { hash } = resj diff --git a/src/utils/index.ts b/src/utils/index.ts index dc541b9e..ad896ded 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -107,3 +107,16 @@ export const convertTimeTags = (songDate?: string) => { const [y, m] = songDate.split('-') return [`[${y}]`, `[${y}-${m}]`, `[${y}-S${Math.floor((+m - 1) / 3) + 1}]`] } + +export const raseTimeout = async ( + task: Promise, + msec: number, + alt: U +): Promise => { + const timeout = async () => { + await sleep(msec) + return alt + } + + return await Promise.race([task, timeout()]) +} From f3300bf7608015d9e3650732b91c0132e845ec9e Mon Sep 17 00:00:00 2001 From: vipzero Date: Sat, 29 Apr 2023 02:04:05 +0900 Subject: [PATCH 085/105] add self download timeout --- src/imageIo/download.ts | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/imageIo/download.ts b/src/imageIo/download.ts index ccfb11f9..6a3c1309 100644 --- a/src/imageIo/download.ts +++ b/src/imageIo/download.ts @@ -46,15 +46,7 @@ const putil = () => { }, } } - -export const downloadOptimize = async ( - url: string -): Promise => { - const time = putil() - - // log('s: ' + url) - const filePath = `tmp/${uuidv4()}` - +export const download = async (url: string, filePath: string) => { let res try { const stream = got.stream(url, gotOption) @@ -71,9 +63,21 @@ export const downloadOptimize = async ( warnDesc(`out-DownloadSaveError`, JSON.stringify(e)) res = 'SaveError' } + return res +} + +export const downloadOptimize = async ( + url: string +): Promise => { + const time = putil() + + // log('s: ' + url) + const filePath = `tmp/${uuidv4()}` + + const res = await raseTimeout(download(url, filePath), 10000, false as const) time.mark(` dw: `) - if (res === 'SaveError') return false + if (res === 'SaveError' || !res) return false // await imageMin(filePath) const shapeTask = sharpMin(filePath).catch((e) => { From 67c459a8c3eac5fc3f5231b711204ce441360d2a Mon Sep 17 00:00:00 2001 From: vipzero Date: Sat, 29 Apr 2023 02:16:22 +0900 Subject: [PATCH 086/105] fix promise --- src/imageIo/download.ts | 24 +++++++++++++----------- src/utils/index.ts | 8 ++------ 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/src/imageIo/download.ts b/src/imageIo/download.ts index 6a3c1309..f5173ee9 100644 --- a/src/imageIo/download.ts +++ b/src/imageIo/download.ts @@ -3,10 +3,10 @@ import got from 'got' import stream from 'stream' import { promisify } from 'util' import { CacheFile } from '../types' +import { raseTimeout } from '../utils' import { error, info, log, warn, warnDesc } from '../utils/logger' import { jimpHash } from './jimp' import { sharpMin } from './sharp' -import { raseTimeout, sleep } from '../utils' const uuidv4 = require('uuid/v4') @@ -47,18 +47,20 @@ const putil = () => { } } export const download = async (url: string, filePath: string) => { - let res + let res: string | boolean = true try { const stream = got.stream(url, gotOption) - res = await pipeline(stream, fs.createWriteStream(filePath)).catch((e) => { - if (e.name === 'TimeoutError') { - log(`Timeout`, `${url}`) - } else { - error(`DownloadSaveError`, `${url} ${filePath}`) - log(JSON.stringify(e)) - } - return 'SaveError' as const - }) + res = await pipeline(stream, fs.createWriteStream(filePath)) + .catch((e) => { + if (e.name === 'TimeoutError') { + log(`Timeout`, `${url}`) + } else { + error(`DownloadSaveError`, `${url} ${filePath}`) + log(JSON.stringify(e)) + } + return 'SaveError' as const + }) + .then(() => true) } catch (e) { warnDesc(`out-DownloadSaveError`, JSON.stringify(e)) res = 'SaveError' diff --git a/src/utils/index.ts b/src/utils/index.ts index ad896ded..037ee636 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -108,15 +108,11 @@ export const convertTimeTags = (songDate?: string) => { return [`[${y}]`, `[${y}-${m}]`, `[${y}-S${Math.floor((+m - 1) / 3) + 1}]`] } -export const raseTimeout = async ( - task: Promise, - msec: number, - alt: U -): Promise => { +export const raseTimeout = (task: Promise, msec: number, alt: U) => { const timeout = async () => { await sleep(msec) return alt } - return await Promise.race([task, timeout()]) + return Promise.race([task, timeout()]) } From a071bce79bf143ddff644d5511da6e01d4abb3ce Mon Sep 17 00:00:00 2001 From: vipzero Date: Sat, 29 Apr 2023 22:51:33 +0900 Subject: [PATCH 087/105] change log timing --- src/imageIo/download.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/imageIo/download.ts b/src/imageIo/download.ts index f5173ee9..b9120742 100644 --- a/src/imageIo/download.ts +++ b/src/imageIo/download.ts @@ -78,8 +78,8 @@ export const downloadOptimize = async ( const res = await raseTimeout(download(url, filePath), 10000, false as const) - time.mark(` dw: `) if (res === 'SaveError' || !res) return false + time.mark(` dw: `) // await imageMin(filePath) const shapeTask = sharpMin(filePath).catch((e) => { From bd31168cbb12fd0cda47a392a7308222bbbfe6f2 Mon Sep 17 00:00:00 2001 From: vipzero Date: Mon, 1 May 2023 01:10:03 +0900 Subject: [PATCH 088/105] update new table logger --- src/imageIo/download.ts | 17 +++++++++----- src/imageIo/uploadManage.ts | 4 +++- src/utils/logger.ts | 2 ++ src/utils/tableTimeLogger.ts | 43 ++++++++++++++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 6 deletions(-) create mode 100644 src/utils/tableTimeLogger.ts diff --git a/src/imageIo/download.ts b/src/imageIo/download.ts index b9120742..3114a047 100644 --- a/src/imageIo/download.ts +++ b/src/imageIo/download.ts @@ -7,6 +7,10 @@ import { raseTimeout } from '../utils' import { error, info, log, warn, warnDesc } from '../utils/logger' import { jimpHash } from './jimp' import { sharpMin } from './sharp' +import { + ImageSetupTimeTable, + imageSetupTimeTable, +} from '../utils/tableTimeLogger' const uuidv4 = require('uuid/v4') @@ -69,9 +73,10 @@ export const download = async (url: string, filePath: string) => { } export const downloadOptimize = async ( - url: string + url: string, + tt: ImageSetupTimeTable ): Promise => { - const time = putil() + tt.init(url) // log('s: ' + url) const filePath = `tmp/${uuidv4()}` @@ -79,7 +84,8 @@ export const downloadOptimize = async ( const res = await raseTimeout(download(url, filePath), 10000, false as const) if (res === 'SaveError' || !res) return false - time.mark(` dw: `) + tt.mark(url, 'dw') + // await imageMin(filePath) const shapeTask = sharpMin(filePath).catch((e) => { @@ -92,17 +98,18 @@ export const downloadOptimize = async ( const { size, height, width, format } = shapeRes const fileType = mimeMap[format] || fileTypeDefault - time.mark(` shape: `) + tt.mark(url, 'shape') const jimpTask = jimpHash(filePath, fileType.mime).catch((e) => { warnDesc('JimpError', e) return false as const }) + const resj = await raseTimeout(jimpTask, 10000, false as const) if (!resj) return false const { hash } = resj - time.mark(` jimp: `) + tt.mark(url, 'jimp') // time.mark(` size: `) return { filePath, fileType, size, height, width, hash } diff --git a/src/imageIo/uploadManage.ts b/src/imageIo/uploadManage.ts index e4b172c0..9fe8c0e4 100644 --- a/src/imageIo/uploadManage.ts +++ b/src/imageIo/uploadManage.ts @@ -5,6 +5,7 @@ import { uploadStorage } from './../service/firebase' import { CacheFile } from './../types/index' import { downloadOptimize } from './download' import { isUniqueHash } from './jimp' +import { imageSetupTimeTable } from '../utils/tableTimeLogger' const nonFalse = (v: T | false): v is T => v !== false export const uploadByUrlAll = async (urls: string[]) => { @@ -12,8 +13,9 @@ export const uploadByUrlAll = async (urls: string[]) => { log(urls) log(urls.length) + const tt = imageSetupTimeTable() const downloads: CacheFile[] = ( - await Promise.all(urls.map(downloadOptimize)) + await Promise.all(urls.map((url) => downloadOptimize(url, tt))) ).filter((v) => nonFalse(v)) as CacheFile[] const uploads: UploadFile[] = [] diff --git a/src/utils/logger.ts b/src/utils/logger.ts index 41ad12e2..53275b3f 100644 --- a/src/utils/logger.ts +++ b/src/utils/logger.ts @@ -1,6 +1,8 @@ /* eslint-disable no-console */ import chalk from 'chalk' +export const logLine = (str: string) => process.stdout.write(chalk.gray(str)) + export const log = console.log export const info = (str: string | number | object) => console.log(chalk.gray(str)) diff --git a/src/utils/tableTimeLogger.ts b/src/utils/tableTimeLogger.ts new file mode 100644 index 00000000..05a527df --- /dev/null +++ b/src/utils/tableTimeLogger.ts @@ -0,0 +1,43 @@ +import { logLine, log } from './logger' + +export type ImageSetupTimeTable = ReturnType +export const imageSetupTimeTable = () => { + const data = new Map; prev: number }>() + + // const decoTime = (ms: number) => { + // if (ms < 1000) { + // info(`${ms}ms`) + // } else { + // warn(`${ms}ms`) + // } + // } + + return { + init(name: string) { + if (!data.has(name)) { + data.set(name, { times: new Map(), prev: performance.now() }) + } + }, + mark(name: string, step: 'dw' | 'shape' | 'jimp') { + const item = data.get(name) + if (!item) throw new Error() + const cur = performance.now() + const ms = Math.floor(cur - item.prev) + data.get(name)?.times.set(step, ms) + + item.prev = cur + logLine('.') + }, + print() { + const res = [...data.entries()] + .map(([name, item]) => { + const times = [...item.times.entries()] + .map(([step, ms]) => `${step}: ${ms}ms`) + .join(', ') + return `${name}: ${times}` + }) + .join('\n') + log(res) + }, + } +} From 5ae6182d941c6bf093c98dd15a0b6c2d09e7928a Mon Sep 17 00:00:00 2001 From: vipzero Date: Mon, 1 May 2023 01:30:35 +0900 Subject: [PATCH 089/105] update marker --- scripts/playground.ts | 22 ++++++++++++++++-- src/imageIo/download.ts | 2 +- src/utils/tableTimeLogger.ts | 44 ++++++++++++++++++++++++------------ 3 files changed, 51 insertions(+), 17 deletions(-) diff --git a/scripts/playground.ts b/scripts/playground.ts index e7b4492a..03407e06 100644 --- a/scripts/playground.ts +++ b/scripts/playground.ts @@ -2,6 +2,7 @@ /* eslint-disable no-console */ import { findSong } from '../src/anisonDb/findSong' +import { imageSetupTimeTable } from '../src/utils/tableTimeLogger' // import { pathQueue, push } from './state/pathQueue' // const { spotifySearchSongInfo } = require('./spotify') @@ -95,5 +96,22 @@ import { findSong } from '../src/anisonDb/findSong' // // emojify(text, { onlyEmoji: true }).then(console.log) // makeEmol(text).then(console.log) -const icy = `READY STEADY GO - L'Arc～en～Ciel` -console.log(findSong(icy)) +// const icy = `READY STEADY GO - L'Arc～en～Ciel` +// console.log(findSong(icy)) + +const tt = imageSetupTimeTable() +tt.init('http://example.com/aaaaaaaaaaa/bbbbbbbbbbbb/a.jpg') +tt.init('http://example.com/aaaaaaaaaaa/bbbbbbbbbbbb/b.jpg') +tt.init('http://example.com/c.jpg') + +tt.mark('http://example.com/aaaaaaaaaaa/bbbbbbbbbbbb/a.jpg', 'dw') +tt.mark('http://example.com/aaaaaaaaaaa/bbbbbbbbbbbb/a.jpg', 'sharp') +tt.mark('http://example.com/aaaaaaaaaaa/bbbbbbbbbbbb/a.jpg', 'jimp') +tt.mark('http://example.com/aaaaaaaaaaa/bbbbbbbbbbbb/b.jpg', 'dw') +tt.mark('http://example.com/aaaaaaaaaaa/bbbbbbbbbbbb/b.jpg', 'sharp') +// tt.mark('http://example.com/aaaaaaaaaaa/bbbbbbbbbbbb/b.jpg', 'jimp') +tt.mark('http://example.com/c.jpg', 'dw') +tt.mark('http://example.com/c.jpg', 'sharp') +tt.mark('http://example.com/c.jpg', 'jimp') + +tt.print() diff --git a/src/imageIo/download.ts b/src/imageIo/download.ts index 3114a047..52591ba8 100644 --- a/src/imageIo/download.ts +++ b/src/imageIo/download.ts @@ -98,7 +98,7 @@ export const downloadOptimize = async ( const { size, height, width, format } = shapeRes const fileType = mimeMap[format] || fileTypeDefault - tt.mark(url, 'shape') + tt.mark(url, 'sharp') const jimpTask = jimpHash(filePath, fileType.mime).catch((e) => { warnDesc('JimpError', e) diff --git a/src/utils/tableTimeLogger.ts b/src/utils/tableTimeLogger.ts index 05a527df..2af565e0 100644 --- a/src/utils/tableTimeLogger.ts +++ b/src/utils/tableTimeLogger.ts @@ -1,16 +1,18 @@ import { logLine, log } from './logger' +import chalk from 'chalk' + export type ImageSetupTimeTable = ReturnType export const imageSetupTimeTable = () => { const data = new Map; prev: number }>() - // const decoTime = (ms: number) => { - // if (ms < 1000) { - // info(`${ms}ms`) - // } else { - // warn(`${ms}ms`) - // } - // } + const decoTime = (ms: number) => { + if (ms < 1000) { + return chalk.gray(`${ms}ms`.padStart(8)) + } else { + return chalk.yellow(`${ms}ms`.padStart(8)) + } + } return { init(name: string) { @@ -18,23 +20,37 @@ export const imageSetupTimeTable = () => { data.set(name, { times: new Map(), prev: performance.now() }) } }, - mark(name: string, step: 'dw' | 'shape' | 'jimp') { + mark(name: string, step: 'dw' | 'sharp' | 'jimp') { const item = data.get(name) if (!item) throw new Error() const cur = performance.now() - const ms = Math.floor(cur - item.prev) - data.get(name)?.times.set(step, ms) + const ms = Math.floor(cur - item.prev) + 1000 + item.times.set(step, ms) item.prev = cur logLine('.') }, print() { + log('') + log( + [ + 'url'.padStart(30), + 'dw'.padStart(8), + 'sharp'.padStart(8), + 'jimp'.padStart(8), + ].join(', ') + ) const res = [...data.entries()] - .map(([name, item]) => { - const times = [...item.times.entries()] - .map(([step, ms]) => `${step}: ${ms}ms`) + .map(([id, item]) => { + const cols = [ + item.times.get('dw'), + item.times.get('sharp'), + item.times.get('jimp'), + ] + const times = cols + .map((ms) => (ms ? decoTime(ms) : '-'.padStart(8))) .join(', ') - return `${name}: ${times}` + return `${id.substring(0, 30).padStart(30)}, ${times}` }) .join('\n') log(res) From 4b3405cc58bb6b38da85db2d5ef42674c0505568 Mon Sep 17 00:00:00 2001 From: vipzero Date: Mon, 1 May 2023 01:33:45 +0900 Subject: [PATCH 090/105] add id shorter --- src/utils/tableTimeLogger.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/utils/tableTimeLogger.ts b/src/utils/tableTimeLogger.ts index 2af565e0..ec4183ac 100644 --- a/src/utils/tableTimeLogger.ts +++ b/src/utils/tableTimeLogger.ts @@ -50,10 +50,19 @@ export const imageSetupTimeTable = () => { const times = cols .map((ms) => (ms ? decoTime(ms) : '-'.padStart(8))) .join(', ') - return `${id.substring(0, 30).padStart(30)}, ${times}` + return `${printId(id).substring(0, 30).padStart(30)}, ${times}` }) .join('\n') log(res) }, } } +const printId = (str: string) => { + const id = str.substring(0, 30) + if (str.length <= 30) return str + return ( + str.substring(0, 20 - 5) + + ' ... ' + + str.substring(str.length - 10, str.length) + ) +} From 65296d7cd447d01d213198320dc7424b2bd927ea Mon Sep 17 00:00:00 2001 From: vipzero Date: Mon, 1 May 2023 01:34:26 +0900 Subject: [PATCH 091/105] add print --- src/imageIo/uploadManage.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/imageIo/uploadManage.ts b/src/imageIo/uploadManage.ts index 9fe8c0e4..b3f39ed8 100644 --- a/src/imageIo/uploadManage.ts +++ b/src/imageIo/uploadManage.ts @@ -17,6 +17,7 @@ export const uploadByUrlAll = async (urls: string[]) => { const downloads: CacheFile[] = ( await Promise.all(urls.map((url) => downloadOptimize(url, tt))) ).filter((v) => nonFalse(v)) as CacheFile[] + tt.print() const uploads: UploadFile[] = [] const selects = choiceImage(downloads) From f04dd3b2deb2610bba1edde12061e44e415e1307 Mon Sep 17 00:00:00 2001 From: vipzero Date: Mon, 1 May 2023 01:37:17 +0900 Subject: [PATCH 092/105] remove size log --- src/imageIo/jimp.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/imageIo/jimp.ts b/src/imageIo/jimp.ts index 8b5562e1..540bc16b 100644 --- a/src/imageIo/jimp.ts +++ b/src/imageIo/jimp.ts @@ -1,6 +1,5 @@ -import { statSync } from 'fs' import { compareHashes, read } from 'jimp' -import { info, warnDesc } from '../utils/logger' +import { warnDesc } from '../utils/logger' const QUALITY_MAP: Record = { 'image/jpeg': 50, @@ -8,7 +7,7 @@ const QUALITY_MAP: Record = { } export async function jimpHash(path: string, mime: string) { - const befSize = statSync(path).size + // const befSize = statSync(path).size const img = await read(path).catch(() => false as const) if (img === false) { warnDesc('read error', path) @@ -17,8 +16,8 @@ export async function jimpHash(path: string, mime: string) { } const res = await img.quality(QUALITY_MAP[mime] || 80).write(path) - const aftSize = statSync(path).size - info(`${befSize / 1000}KB => ${aftSize / 1000}KB`) + // const aftSize = statSync(path).size + // info(`${befSize / 1000}KB => ${aftSize / 1000}KB`) return { hash: res.hash(), height: res.getHeight(), width: res.getWidth() } } From 45edd2cea4eae1f447db8b5517ef9f992ce8bfd0 Mon Sep 17 00:00:00 2001 From: vipzero Date: Mon, 1 May 2023 01:39:55 +0900 Subject: [PATCH 093/105] off printer --- src/imageIo/download.ts | 25 ++++--------------------- src/imageIo/uploadManage.ts | 2 +- 2 files changed, 5 insertions(+), 22 deletions(-) diff --git a/src/imageIo/download.ts b/src/imageIo/download.ts index 52591ba8..439fcf12 100644 --- a/src/imageIo/download.ts +++ b/src/imageIo/download.ts @@ -33,23 +33,6 @@ const mimeMap: Record = { const fileTypeDefault = mimePng const gotOption = { timeout: { response: 10_000 } } -const putil = () => { - let prev = performance.now() - - return { - mark(name: string) { - const cur = performance.now() - const ms = Math.floor(cur - prev) - if (ms < 1000) { - info(`${name}${ms}ms`) - } else { - warn(`${name}${ms}ms`) - } - - prev = cur - }, - } -} export const download = async (url: string, filePath: string) => { let res: string | boolean = true try { @@ -76,7 +59,7 @@ export const downloadOptimize = async ( url: string, tt: ImageSetupTimeTable ): Promise => { - tt.init(url) + // tt.init(url) // log('s: ' + url) const filePath = `tmp/${uuidv4()}` @@ -84,7 +67,7 @@ export const downloadOptimize = async ( const res = await raseTimeout(download(url, filePath), 10000, false as const) if (res === 'SaveError' || !res) return false - tt.mark(url, 'dw') + // tt.mark(url, 'dw') // await imageMin(filePath) @@ -98,7 +81,7 @@ export const downloadOptimize = async ( const { size, height, width, format } = shapeRes const fileType = mimeMap[format] || fileTypeDefault - tt.mark(url, 'sharp') + // tt.mark(url, 'sharp') const jimpTask = jimpHash(filePath, fileType.mime).catch((e) => { warnDesc('JimpError', e) @@ -109,7 +92,7 @@ export const downloadOptimize = async ( if (!resj) return false const { hash } = resj - tt.mark(url, 'jimp') + // tt.mark(url, 'jimp') // time.mark(` size: `) return { filePath, fileType, size, height, width, hash } diff --git a/src/imageIo/uploadManage.ts b/src/imageIo/uploadManage.ts index b3f39ed8..279e49a8 100644 --- a/src/imageIo/uploadManage.ts +++ b/src/imageIo/uploadManage.ts @@ -17,7 +17,7 @@ export const uploadByUrlAll = async (urls: string[]) => { const downloads: CacheFile[] = ( await Promise.all(urls.map((url) => downloadOptimize(url, tt))) ).filter((v) => nonFalse(v)) as CacheFile[] - tt.print() + // tt.print() const uploads: UploadFile[] = [] const selects = choiceImage(downloads) From e9076ac832a4351e018aaf76fd7e04a6d8a5e901 Mon Sep 17 00:00:00 2001 From: vipzero Date: Mon, 1 May 2023 02:04:39 +0900 Subject: [PATCH 094/105] update image table log This reverts commit f04dd3b2deb2610bba1edde12061e44e415e1307. --- scripts/playground.ts | 4 +- src/imageIo/download.ts | 25 ++++++---- src/imageIo/jimp.ts | 3 -- src/imageIo/uploadManage.ts | 6 +-- src/types/index.ts | 17 +++++++ src/utils/tableTimeLogger.ts | 92 +++++++++++++++--------------------- 6 files changed, 75 insertions(+), 72 deletions(-) diff --git a/scripts/playground.ts b/scripts/playground.ts index 03407e06..54e2c53c 100644 --- a/scripts/playground.ts +++ b/scripts/playground.ts @@ -2,7 +2,7 @@ /* eslint-disable no-console */ import { findSong } from '../src/anisonDb/findSong' -import { imageSetupTimeTable } from '../src/utils/tableTimeLogger' +import { printImageSetupTimeTable } from '../src/utils/tableTimeLogger' // import { pathQueue, push } from './state/pathQueue' // const { spotifySearchSongInfo } = require('./spotify') @@ -99,7 +99,7 @@ import { imageSetupTimeTable } from '../src/utils/tableTimeLogger' // const icy = `READY STEADY GO - L'Arc～en～Ciel` // console.log(findSong(icy)) -const tt = imageSetupTimeTable() +const tt = printImageSetupTimeTable() tt.init('http://example.com/aaaaaaaaaaa/bbbbbbbbbbbb/a.jpg') tt.init('http://example.com/aaaaaaaaaaa/bbbbbbbbbbbb/b.jpg') tt.init('http://example.com/c.jpg') diff --git a/src/imageIo/download.ts b/src/imageIo/download.ts index 439fcf12..e984a3da 100644 --- a/src/imageIo/download.ts +++ b/src/imageIo/download.ts @@ -2,14 +2,14 @@ import fs from 'fs' import got from 'got' import stream from 'stream' import { promisify } from 'util' -import { CacheFile } from '../types' +import { CacheFile, CacheFileStat } from '../types' import { raseTimeout } from '../utils' import { error, info, log, warn, warnDesc } from '../utils/logger' import { jimpHash } from './jimp' import { sharpMin } from './sharp' import { ImageSetupTimeTable, - imageSetupTimeTable, + printImageSetupTimeTable, } from '../utils/tableTimeLogger' const uuidv4 = require('uuid/v4') @@ -56,18 +56,24 @@ export const download = async (url: string, filePath: string) => { } export const downloadOptimize = async ( - url: string, - tt: ImageSetupTimeTable + url: string ): Promise => { // tt.init(url) // log('s: ' + url) + const stat: CacheFileStat = { + url, + times: { prev: performance.now(), dw: 0, sharp: 0, jimp: 0 }, + size: { before: 0, sharped: 0, sharpReport: 0, jimped: 0 }, + } + const filePath = `tmp/${uuidv4()}` const res = await raseTimeout(download(url, filePath), 10000, false as const) if (res === 'SaveError' || !res) return false - // tt.mark(url, 'dw') + stat.times.dw = performance.now() - stat.times.prev + stat.times.prev = performance.now() // await imageMin(filePath) @@ -81,7 +87,8 @@ export const downloadOptimize = async ( const { size, height, width, format } = shapeRes const fileType = mimeMap[format] || fileTypeDefault - // tt.mark(url, 'sharp') + stat.times.sharp = performance.now() - stat.times.prev + stat.times.prev = performance.now() const jimpTask = jimpHash(filePath, fileType.mime).catch((e) => { warnDesc('JimpError', e) @@ -92,8 +99,8 @@ export const downloadOptimize = async ( if (!resj) return false const { hash } = resj - // tt.mark(url, 'jimp') - // time.mark(` size: `) + stat.times.jimp = performance.now() - stat.times.prev + stat.times.prev = performance.now() - return { filePath, fileType, size, height, width, hash } + return { filePath, fileType, size, height, width, hash, stat } } diff --git a/src/imageIo/jimp.ts b/src/imageIo/jimp.ts index 540bc16b..f47fd4f1 100644 --- a/src/imageIo/jimp.ts +++ b/src/imageIo/jimp.ts @@ -7,7 +7,6 @@ const QUALITY_MAP: Record = { } export async function jimpHash(path: string, mime: string) { - // const befSize = statSync(path).size const img = await read(path).catch(() => false as const) if (img === false) { warnDesc('read error', path) @@ -16,8 +15,6 @@ export async function jimpHash(path: string, mime: string) { } const res = await img.quality(QUALITY_MAP[mime] || 80).write(path) - // const aftSize = statSync(path).size - // info(`${befSize / 1000}KB => ${aftSize / 1000}KB`) return { hash: res.hash(), height: res.getHeight(), width: res.getWidth() } } diff --git a/src/imageIo/uploadManage.ts b/src/imageIo/uploadManage.ts index 279e49a8..03eaf896 100644 --- a/src/imageIo/uploadManage.ts +++ b/src/imageIo/uploadManage.ts @@ -5,7 +5,7 @@ import { uploadStorage } from './../service/firebase' import { CacheFile } from './../types/index' import { downloadOptimize } from './download' import { isUniqueHash } from './jimp' -import { imageSetupTimeTable } from '../utils/tableTimeLogger' +import { printImageSetupTimeTable } from '../utils/tableTimeLogger' const nonFalse = (v: T | false): v is T => v !== false export const uploadByUrlAll = async (urls: string[]) => { @@ -13,11 +13,11 @@ export const uploadByUrlAll = async (urls: string[]) => { log(urls) log(urls.length) - const tt = imageSetupTimeTable() const downloads: CacheFile[] = ( - await Promise.all(urls.map((url) => downloadOptimize(url, tt))) + await Promise.all(urls.map((url) => downloadOptimize(url))) ).filter((v) => nonFalse(v)) as CacheFile[] // tt.print() + printImageSetupTimeTable(downloads.map((v) => v.stat)) const uploads: UploadFile[] = [] const selects = choiceImage(downloads) diff --git a/src/types/index.ts b/src/types/index.ts index 1d343713..f4af3d6c 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -95,5 +95,22 @@ export type CacheFile = { width: number height: number hash: string + stat: CacheFileStat +} + +export type CacheFileStat = { + url: string + times: { + prev: number + dw: number + sharp: number + jimp: number + } + size: { + before: number + sharped: number + sharpReport: number + jimped: number + } } export type Emol = { text: string } diff --git a/src/utils/tableTimeLogger.ts b/src/utils/tableTimeLogger.ts index ec4183ac..6c19775a 100644 --- a/src/utils/tableTimeLogger.ts +++ b/src/utils/tableTimeLogger.ts @@ -1,64 +1,46 @@ -import { logLine, log } from './logger' +import { CacheFileStat } from '../types' +import { log } from './logger' import chalk from 'chalk' -export type ImageSetupTimeTable = ReturnType -export const imageSetupTimeTable = () => { - const data = new Map; prev: number }>() - - const decoTime = (ms: number) => { - if (ms < 1000) { - return chalk.gray(`${ms}ms`.padStart(8)) - } else { - return chalk.yellow(`${ms}ms`.padStart(8)) - } - } - - return { - init(name: string) { - if (!data.has(name)) { - data.set(name, { times: new Map(), prev: performance.now() }) - } - }, - mark(name: string, step: 'dw' | 'sharp' | 'jimp') { - const item = data.get(name) - if (!item) throw new Error() - const cur = performance.now() - const ms = Math.floor(cur - item.prev) + 1000 - item.times.set(step, ms) - - item.prev = cur - logLine('.') - }, - print() { - log('') - log( - [ - 'url'.padStart(30), - 'dw'.padStart(8), - 'sharp'.padStart(8), - 'jimp'.padStart(8), - ].join(', ') - ) - const res = [...data.entries()] - .map(([id, item]) => { - const cols = [ - item.times.get('dw'), - item.times.get('sharp'), - item.times.get('jimp'), - ] - const times = cols - .map((ms) => (ms ? decoTime(ms) : '-'.padStart(8))) - .join(', ') - return `${printId(id).substring(0, 30).padStart(30)}, ${times}` - }) - .join('\n') - log(res) - }, +export type ImageSetupTimeTable = ReturnType +const decoTime = (ms: number) => { + if (ms < 1000) { + return chalk.gray(`${ms}ms`.padStart(8)) + } else { + return chalk.yellow(`${ms}ms`.padStart(8)) } } +export const printImageSetupTimeTable = (status: CacheFileStat[]) => { + log('') + log( + [ + 'url'.padStart(30), + 'dw'.padStart(8), + 'sharp'.padStart(8), + 'jimp'.padStart(8), + ].join(', ') + ) + const res = status + .map((item) => { + const cols = [item.times.dw, item.times.sharp, item.times.jimp] + const times = cols + .map((ms) => (ms ? decoTime(ms) : '-'.padStart(8))) + .join(', ') + const sizes = `${printSize(item.size.before)} => ${printSize( + item.size.sharpReport + )} => ${printSize(item.size.sharped)} => ${printSize(item.size.jimped)}` + return `${printId(item.url) + .substring(0, 30) + .padStart(30)}, ${times}, ${sizes}` + }) + .join('\n') + log(res) +} +const printSize = (size: number) => { + return chalk.gray(`${(size / 1024).toFixed(1)}KB`) +} const printId = (str: string) => { - const id = str.substring(0, 30) if (str.length <= 30) return str return ( str.substring(0, 20 - 5) + From 190ba8a45cfa554c628dcc82404251ee26ea2796 Mon Sep 17 00:00:00 2001 From: vipzero Date: Mon, 1 May 2023 02:08:51 +0900 Subject: [PATCH 095/105] fix logs --- src/imageIo/download.ts | 6 +++++- src/utils/tableTimeLogger.ts | 5 +++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/imageIo/download.ts b/src/imageIo/download.ts index e984a3da..0e452cc9 100644 --- a/src/imageIo/download.ts +++ b/src/imageIo/download.ts @@ -1,4 +1,4 @@ -import fs from 'fs' +import fs, { statSync } from 'fs' import got from 'got' import stream from 'stream' import { promisify } from 'util' @@ -74,6 +74,7 @@ export const downloadOptimize = async ( if (res === 'SaveError' || !res) return false stat.times.dw = performance.now() - stat.times.prev stat.times.prev = performance.now() + stat.size.before = statSync(filePath).size // await imageMin(filePath) @@ -89,6 +90,8 @@ export const downloadOptimize = async ( stat.times.sharp = performance.now() - stat.times.prev stat.times.prev = performance.now() + stat.size.sharped = statSync(filePath).size + stat.size.sharpReport = size const jimpTask = jimpHash(filePath, fileType.mime).catch((e) => { warnDesc('JimpError', e) @@ -101,6 +104,7 @@ export const downloadOptimize = async ( const { hash } = resj stat.times.jimp = performance.now() - stat.times.prev stat.times.prev = performance.now() + stat.size.jimped = statSync(filePath).size return { filePath, fileType, size, height, width, hash, stat } } diff --git a/src/utils/tableTimeLogger.ts b/src/utils/tableTimeLogger.ts index 6c19775a..fab0e047 100644 --- a/src/utils/tableTimeLogger.ts +++ b/src/utils/tableTimeLogger.ts @@ -5,10 +5,11 @@ import chalk from 'chalk' export type ImageSetupTimeTable = ReturnType const decoTime = (ms: number) => { + const s = `${Math.floor(ms)}ms`.padStart(8) if (ms < 1000) { - return chalk.gray(`${ms}ms`.padStart(8)) + return chalk.gray(s) } else { - return chalk.yellow(`${ms}ms`.padStart(8)) + return chalk.yellow(s) } } export const printImageSetupTimeTable = (status: CacheFileStat[]) => { From bb6e64169ed2ac594d21f0b8bf32c65867d16fd3 Mon Sep 17 00:00:00 2001 From: vipzero Date: Mon, 1 May 2023 02:16:06 +0900 Subject: [PATCH 096/105] refactor --- scripts/playground.ts | 20 -------------------- src/imageIo/download.ts | 11 +---------- 2 files changed, 1 insertion(+), 30 deletions(-) diff --git a/scripts/playground.ts b/scripts/playground.ts index 54e2c53c..7f4ebba1 100644 --- a/scripts/playground.ts +++ b/scripts/playground.ts @@ -1,9 +1,6 @@ /* eslint-disable no-irregular-whitespace */ /* eslint-disable no-console */ -import { findSong } from '../src/anisonDb/findSong' -import { printImageSetupTimeTable } from '../src/utils/tableTimeLogger' - // import { pathQueue, push } from './state/pathQueue' // const { spotifySearchSongInfo } = require('./spotify') // const { getMusixMatch, getMusixLyrics } = require('./musixmatch') @@ -98,20 +95,3 @@ import { printImageSetupTimeTable } from '../src/utils/tableTimeLogger' // const icy = `READY STEADY GO - L'Arc～en～Ciel` // console.log(findSong(icy)) - -const tt = printImageSetupTimeTable() -tt.init('http://example.com/aaaaaaaaaaa/bbbbbbbbbbbb/a.jpg') -tt.init('http://example.com/aaaaaaaaaaa/bbbbbbbbbbbb/b.jpg') -tt.init('http://example.com/c.jpg') - -tt.mark('http://example.com/aaaaaaaaaaa/bbbbbbbbbbbb/a.jpg', 'dw') -tt.mark('http://example.com/aaaaaaaaaaa/bbbbbbbbbbbb/a.jpg', 'sharp') -tt.mark('http://example.com/aaaaaaaaaaa/bbbbbbbbbbbb/a.jpg', 'jimp') -tt.mark('http://example.com/aaaaaaaaaaa/bbbbbbbbbbbb/b.jpg', 'dw') -tt.mark('http://example.com/aaaaaaaaaaa/bbbbbbbbbbbb/b.jpg', 'sharp') -// tt.mark('http://example.com/aaaaaaaaaaa/bbbbbbbbbbbb/b.jpg', 'jimp') -tt.mark('http://example.com/c.jpg', 'dw') -tt.mark('http://example.com/c.jpg', 'sharp') -tt.mark('http://example.com/c.jpg', 'jimp') - -tt.print() diff --git a/src/imageIo/download.ts b/src/imageIo/download.ts index 0e452cc9..64668817 100644 --- a/src/imageIo/download.ts +++ b/src/imageIo/download.ts @@ -4,13 +4,9 @@ import stream from 'stream' import { promisify } from 'util' import { CacheFile, CacheFileStat } from '../types' import { raseTimeout } from '../utils' -import { error, info, log, warn, warnDesc } from '../utils/logger' +import { error, log, warnDesc } from '../utils/logger' import { jimpHash } from './jimp' import { sharpMin } from './sharp' -import { - ImageSetupTimeTable, - printImageSetupTimeTable, -} from '../utils/tableTimeLogger' const uuidv4 = require('uuid/v4') @@ -58,9 +54,6 @@ export const download = async (url: string, filePath: string) => { export const downloadOptimize = async ( url: string ): Promise => { - // tt.init(url) - - // log('s: ' + url) const stat: CacheFileStat = { url, times: { prev: performance.now(), dw: 0, sharp: 0, jimp: 0 }, @@ -76,8 +69,6 @@ export const downloadOptimize = async ( stat.times.prev = performance.now() stat.size.before = statSync(filePath).size - // await imageMin(filePath) - const shapeTask = sharpMin(filePath).catch((e) => { warnDesc('UnsupportedError', e) return false as const From c911e4072e19f0bb9ec82c6ea3196d64e0cd9ee5 Mon Sep 17 00:00:00 2001 From: vipzero Date: Tue, 2 May 2023 12:54:06 +0900 Subject: [PATCH 097/105] add fix count script --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 130e2108..eaf00e54 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ "map": "yarn map-load-thread && yarn map-save", "map-load-thread": "ts-node scripts/loadThreads.ts", "map-save": "ts-node scripts/makePostCountMap.ts", + "fix:counts": "ts-node scripts/batch/migrateWordCounts.ts", "active-thread": "chch dump-threads |jq -r '.threads[] | \"\\(.url),\\(.title)\"' |grep \"ならアニソン\"", "test": "jest", "test:type": "npx tsc --noEmit" From c263e3d6df4807306174f966ef7ad875166be1e6 Mon Sep 17 00:00:00 2001 From: vipzero Date: Sat, 6 May 2023 01:47:58 +0900 Subject: [PATCH 098/105] add fix history 4 --- package.json | 2 ++ scripts/batch/importHistoryFromCells4.ts | 38 ++++++++++++++++++++++++ yarn.lock | 10 +++++++ 3 files changed, 50 insertions(+) create mode 100644 scripts/batch/importHistoryFromCells4.ts diff --git a/package.json b/package.json index eaf00e54..302c0376 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "jest": "^27.5.1", "jimp": "^0.16.2", "jptext-to-emoji": "0.3.0", + "luxon": "^3.3.0", "node-spotify-api": "1.1.1", "sharp": "0.28.3" }, @@ -50,6 +51,7 @@ "@types/imagemin": "7.0.1", "@types/imagemin-gifsicle": "7.0.1", "@types/imagemin-mozjpeg": "8.0.1", + "@types/luxon": "^3.3.0", "@types/node": "17.0.24", "@types/sharp": "0.28.5", "@typescript-eslint/eslint-plugin": "5.21.0", diff --git a/scripts/batch/importHistoryFromCells4.ts b/scripts/batch/importHistoryFromCells4.ts new file mode 100644 index 00000000..b7f2718b --- /dev/null +++ b/scripts/batch/importHistoryFromCells4.ts @@ -0,0 +1,38 @@ +import { readFileSync } from 'fs' +import { addHistory } from '../../src/service/firebase' +import { sleep } from '../../src/utils' +import { anaCounts } from '../../src/utils/wordCounts' +import { DateTime } from 'luxon' + +const parseLine = (text: string) => { + const [time, ...icys] = text.split(' ') + const icy = icys.join(' ') + + return { time, title: icy } +} + +const importFile = './data/archive/history_2023gw0501.utf.txt' +const text = readFileSync(importFile, 'utf8') + +const lines = text.trim().split('\r\n') + +async function main() { + // for (let i = 0; i < 3; i++) { + for (let i = 0; i < lines.length; i++) { + const line = lines[i] + const { title, time: timeStr } = parseLine(line) + + await sleep(200) + // 01/May/2023:05:19:32 + const time = DateTime.fromFormat(timeStr, 'dd/MMM/yyyy:HH:mm:ss') + .setLocale('ja') + .toMillis() + + await addHistory(title.trim(), Number(time), 0) + anaCounts([title], {}, [], true) + + process.stdout.write('.') + } +} + +main().then(() => console.log('fin')) diff --git a/yarn.lock b/yarn.lock index b259f3aa..2c8588ce 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1594,6 +1594,11 @@ resolved "https://registry.yarnpkg.com/@types/long/-/long-4.0.1.tgz#459c65fa1867dafe6a8f322c4c51695663cc55e9" integrity sha512-5tXH6Bx/kNGd3MgffdmP4dy2Z+G4eaXw0SE81Tq3BNadtnMR5/ySMzX4SLEzHJzSmPNn4HIdpQsBvXMUykr58w== +"@types/luxon@^3.3.0": + version "3.3.0" + resolved "https://registry.yarnpkg.com/@types/luxon/-/luxon-3.3.0.tgz#a61043a62c0a72696c73a0a305c544c96501e006" + integrity sha512-uKRI5QORDnrGFYgcdAVnHvEIvEZ8noTpP/Bg+HeUzZghwinDlIS87DEenV5r1YoOF9G4x600YsUXLWZ19rmTmg== + "@types/mime@^1": version "1.3.2" resolved "https://registry.yarnpkg.com/@types/mime/-/mime-1.3.2.tgz#93e25bf9ee75fe0fd80b594bc4feb0e862111b5a" @@ -5492,6 +5497,11 @@ lru_map@^0.4.1: resolved "https://registry.yarnpkg.com/lru_map/-/lru_map-0.4.1.tgz#f7b4046283c79fb7370c36f8fca6aee4324b0a98" integrity sha512-I+lBvqMMFfqaV8CJCISjI3wbjmwVu/VyOoU7+qtu9d7ioW5klMgsTTiUOUp+DJvfTTzKXoPbyC6YfgkNcyPSOg== +luxon@^3.3.0: + version "3.3.0" + resolved "https://registry.yarnpkg.com/luxon/-/luxon-3.3.0.tgz#d73ab5b5d2b49a461c47cedbc7e73309b4805b48" + integrity sha512-An0UCfG/rSiqtAIiBPO0Y9/zAnHUZxAMiCpTd5h2smgsj7GGmcenvrvww2cqNA8/4A5ZrD1gJpHN2mIHZQF+Mg== + make-dir@^1.0.0, make-dir@^1.2.0: version "1.3.0" resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-1.3.0.tgz#79c1033b80515bd6d24ec9933e860ca75ee27f0c" From 0acf2fa46511b320729965c0433284b6489796e9 Mon Sep 17 00:00:00 2001 From: vipzero Date: Sat, 6 May 2023 01:55:17 +0900 Subject: [PATCH 099/105] safe --- scripts/batch/importHistoryFromCells4.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/batch/importHistoryFromCells4.ts b/scripts/batch/importHistoryFromCells4.ts index b7f2718b..b63e55c2 100644 --- a/scripts/batch/importHistoryFromCells4.ts +++ b/scripts/batch/importHistoryFromCells4.ts @@ -27,11 +27,14 @@ async function main() { const time = DateTime.fromFormat(timeStr, 'dd/MMM/yyyy:HH:mm:ss') .setLocale('ja') .toMillis() + if (i === 0) console.log('start1: ', time) + if (i === 1) console.log('start2: ', time) + if (i === lines.length - 1) console.log('end: ', time) - await addHistory(title.trim(), Number(time), 0) - anaCounts([title], {}, [], true) + // await addHistory(title.trim(), Number(time), 0) + // anaCounts([title], {}, [], true) - process.stdout.write('.') + // process.stdout.write('.') } } From bb9a6ac3115f26a0b6385024d27bf01d3096c927 Mon Sep 17 00:00:00 2001 From: vipzero Date: Sat, 6 May 2023 17:34:49 +0900 Subject: [PATCH 100/105] add empty search word --- .envrc.sample | 2 ++ scripts/batch/importHistoryFromCells4.ts | 4 ++-- src/utils/makeSearchWord.ts | 3 +++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.envrc.sample b/.envrc.sample index 853e23d8..1b4876d0 100644 --- a/.envrc.sample +++ b/.envrc.sample @@ -21,5 +21,7 @@ export EVENT_ID="todo" export STORAGE_ID="" export STORAGE_URL="" +export EMPTY_MODE_SEARCH_WORD="探偵 クイズ 推理 アニメ " # クイズなど + export THREAD_TITLE_WORD="ならアニソン" export DIRECT_MODE=0 \ No newline at end of file diff --git a/scripts/batch/importHistoryFromCells4.ts b/scripts/batch/importHistoryFromCells4.ts index b63e55c2..75f82803 100644 --- a/scripts/batch/importHistoryFromCells4.ts +++ b/scripts/batch/importHistoryFromCells4.ts @@ -1,7 +1,7 @@ import { readFileSync } from 'fs' -import { addHistory } from '../../src/service/firebase' +// import { addHistory } from '../../src/service/firebase' import { sleep } from '../../src/utils' -import { anaCounts } from '../../src/utils/wordCounts' +// import { anaCounts } from '../../src/utils/wordCounts' import { DateTime } from 'luxon' const parseLine = (text: string) => { diff --git a/src/utils/makeSearchWord.ts b/src/utils/makeSearchWord.ts index c29c0371..57d5c532 100644 --- a/src/utils/makeSearchWord.ts +++ b/src/utils/makeSearchWord.ts @@ -37,6 +37,9 @@ const icyOpt = `(名シーン OR キャラ) (キャプ画像 OR 壁紙)` export function makeSearchQuery(song: SongSeed, seed: number): string { const { icy, category, animeTitle } = song + if (icy === '') { + return process.env.EMPTY_MODE_SEARCH_WORD || 'アニメ' + } if (getSyncConf().simpleSearch) return icy.replace(/-/g, ' ') const r = shuffle(animeExt, `${seed}`) const rEp = shuffle(animeExtEp, `${seed}`) From 274ede4d6da129231ae405dfe7ed7ee6b6e441ee Mon Sep 17 00:00:00 2001 From: vipzero Date: Sat, 6 May 2023 17:37:27 +0900 Subject: [PATCH 101/105] update empty word --- src/utils/makeSearchWord.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/utils/makeSearchWord.ts b/src/utils/makeSearchWord.ts index 57d5c532..214d0b97 100644 --- a/src/utils/makeSearchWord.ts +++ b/src/utils/makeSearchWord.ts @@ -37,14 +37,18 @@ const icyOpt = `(名シーン OR キャラ) (キャプ画像 OR 壁紙)` export function makeSearchQuery(song: SongSeed, seed: number): string { const { icy, category, animeTitle } = song - if (icy === '') { - return process.env.EMPTY_MODE_SEARCH_WORD || 'アニメ' - } if (getSyncConf().simpleSearch) return icy.replace(/-/g, ' ') const r = shuffle(animeExt, `${seed}`) const rEp = shuffle(animeExtEp, `${seed}`) const opts = [r[0], r[1], r[2], rEp[0]] + if (icy === '') { + return ( + process.env.EMPTY_MODE_SEARCH_WORD || + 'アニメ' + ` AND (${opts.join(' OR ')})` + ) + } + if (!animeTitle) { const charaName = pickCharaIcy(icy).join(' ') if (charaName) return `${charaName} ${icyOpt}` From 5e8d67594a98d9c7180c58dc36f56e7b422ce1a3 Mon Sep 17 00:00:00 2001 From: vipzero Date: Sat, 6 May 2023 18:22:13 +0900 Subject: [PATCH 102/105] fix empty search word --- src/utils/makeSearchWord.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/makeSearchWord.ts b/src/utils/makeSearchWord.ts index 214d0b97..e7e7aa45 100644 --- a/src/utils/makeSearchWord.ts +++ b/src/utils/makeSearchWord.ts @@ -44,8 +44,8 @@ export function makeSearchQuery(song: SongSeed, seed: number): string { if (icy === '') { return ( - process.env.EMPTY_MODE_SEARCH_WORD || - 'アニメ' + ` AND (${opts.join(' OR ')})` + (process.env.EMPTY_MODE_SEARCH_WORD || 'アニメ') + + ` AND (${opts.join(' OR ')})` ) } From 11cfc140d22c0491cb930c9803016bb4de14cbfc Mon Sep 17 00:00:00 2001 From: vipzero Date: Sat, 6 May 2023 19:02:56 +0900 Subject: [PATCH 103/105] fix ignore search not query enable --- src/service/customImageSearch.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/service/customImageSearch.ts b/src/service/customImageSearch.ts index 0c343aff..e3ff4897 100644 --- a/src/service/customImageSearch.ts +++ b/src/service/customImageSearch.ts @@ -30,7 +30,7 @@ const white = (v: string) => !blackList.some((blink) => v.includes(blink)) export const getImageLinks = async (q: string) => { // マイナス検索を省く - const res = await getImage(q.replace(/-/g, ' ')).catch((e) => { + const res = await getImage(q.replace(/ -/g, ' ')).catch((e) => { error('GetImageError', q) log(e) return false as const From a858e59318f352a457e7b57dbb72d39d261442ec Mon Sep 17 00:00:00 2001 From: vipzero Date: Sat, 6 May 2023 19:07:42 +0900 Subject: [PATCH 104/105] fix ignore pattenr --- src/service/customImageSearch.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/service/customImageSearch.ts b/src/service/customImageSearch.ts index e3ff4897..975e55fd 100644 --- a/src/service/customImageSearch.ts +++ b/src/service/customImageSearch.ts @@ -30,7 +30,7 @@ const white = (v: string) => !blackList.some((blink) => v.includes(blink)) export const getImageLinks = async (q: string) => { // マイナス検索を省く - const res = await getImage(q.replace(/ -/g, ' ')).catch((e) => { + const res = await getImage(q).catch((e) => { error('GetImageError', q) log(e) return false as const From 8343495ebe22693f6741b5a80fd418e9929946b9 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 6 May 2023 11:56:34 +0000 Subject: [PATCH 105/105] Update definitelyTyped --- package.json | 6 +- yarn.lock | 153 ++++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 143 insertions(+), 16 deletions(-) diff --git a/package.json b/package.json index 302c0376..92ca0bfd 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "license": "MIT", "type": "commonjs", "dependencies": { - "@types/jest": "^27.4.1", + "@types/jest": "^29.0.0", "axios": "0.21.1", "chalk": "4.1.2", "chch": "2.8.0", @@ -48,11 +48,11 @@ "@swc/helpers": "^0.3.8", "@swc/jest": "^0.2.20", "@types/csv-parse": "1.2.2", - "@types/imagemin": "7.0.1", + "@types/imagemin": "8.0.1", "@types/imagemin-gifsicle": "7.0.1", "@types/imagemin-mozjpeg": "8.0.1", "@types/luxon": "^3.3.0", - "@types/node": "17.0.24", + "@types/node": "18.16.5", "@types/sharp": "0.28.5", "@typescript-eslint/eslint-plugin": "5.21.0", "@typescript-eslint/parser": "5.21.0", diff --git a/yarn.lock b/yarn.lock index 2c8588ce..372843e7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -753,6 +753,13 @@ "@types/node" "*" jest-mock "^27.5.1" +"@jest/expect-utils@^29.5.0": + version "29.5.0" + resolved "https://registry.yarnpkg.com/@jest/expect-utils/-/expect-utils-29.5.0.tgz#f74fad6b6e20f924582dc8ecbf2cb800fe43a036" + integrity sha512-fmKzsidoXQT2KwnrwE0SQq3uj8Z763vzR8LnLBwC2qYWEFpjX8daRsk6rHUM1QvNlEW/UJXNXm59ztmJJWs2Mg== + dependencies: + jest-get-type "^29.4.3" + "@jest/fake-timers@^27.5.1": version "27.5.1" resolved "https://registry.yarnpkg.com/@jest/fake-timers/-/fake-timers-27.5.1.tgz#76979745ce0579c8a94a4678af7a748eda8ada74" @@ -805,6 +812,13 @@ terminal-link "^2.0.0" v8-to-istanbul "^8.1.0" +"@jest/schemas@^29.4.3": + version "29.4.3" + resolved "https://registry.yarnpkg.com/@jest/schemas/-/schemas-29.4.3.tgz#39cf1b8469afc40b6f5a2baaa146e332c4151788" + integrity sha512-VLYKXQmtmuEz6IxJsrZwzG9NvtkQsWNnWMsKxqWNu3+CnfzJQhp0WDDKWLVV9hLKr0l3SLLFRqcYHjhtyuDVxg== + dependencies: + "@sinclair/typebox" "^0.25.16" + "@jest/source-map@^27.5.1": version "27.5.1" resolved "https://registry.yarnpkg.com/@jest/source-map/-/source-map-27.5.1.tgz#6608391e465add4205eae073b55e7f279e04e8cf" @@ -866,6 +880,18 @@ "@types/yargs" "^16.0.0" chalk "^4.0.0" +"@jest/types@^29.5.0": + version "29.5.0" + resolved "https://registry.yarnpkg.com/@jest/types/-/types-29.5.0.tgz#f59ef9b031ced83047c67032700d8c807d6e1593" + integrity sha512-qbu7kN6czmVRc3xWFQcAN03RAUamgppVUdXrvl1Wr3jlNF93o9mJbGcDWrwGB6ht44u7efB1qCFgVQmca24Uog== + dependencies: + "@jest/schemas" "^29.4.3" + "@types/istanbul-lib-coverage" "^2.0.0" + "@types/istanbul-reports" "^3.0.0" + "@types/node" "*" + "@types/yargs" "^17.0.8" + chalk "^4.0.0" + "@jimp/bmp@^0.16.2": version "0.16.2" resolved "https://registry.yarnpkg.com/@jimp/bmp/-/bmp-0.16.2.tgz#3982879b10626fc8cf1b4ab8627158bad142ec9d" @@ -1253,6 +1279,11 @@ resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.0.tgz#a777360b5b39a1a2e5106f8e858f2fd2d060c570" integrity sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA= +"@sinclair/typebox@^0.25.16": + version "0.25.24" + resolved "https://registry.yarnpkg.com/@sinclair/typebox/-/typebox-0.25.24.tgz#8c7688559979f7079aacaf31aa881c3aa410b718" + integrity sha512-XJfwUVUKDHF5ugKwIcxEgc9k8b7HbznCp6eUfWgu710hMPNIO4aw4/zB5RogDQz8nd6gyCDpU9O/m6qYEWY6yQ== + "@sindresorhus/is@^0.7.0": version "0.7.0" resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-0.7.0.tgz#9a06f4f137ee84d7df0460c1fdb1135ffa6c50fd" @@ -1543,13 +1574,20 @@ dependencies: "@types/imagemin" "*" -"@types/imagemin@*", "@types/imagemin@7.0.1": +"@types/imagemin@*": version "7.0.1" resolved "https://registry.yarnpkg.com/@types/imagemin/-/imagemin-7.0.1.tgz#11ca1e65ccb3871a8469d9b23033b95d3838eda0" integrity sha512-xEn5+M3lDBtI3JxLy6eU3ksoVurygnlG7OYhTqJfGGP4PcvYnfn+IABCmMve7ziM/SneHDm5xgJFKC8hCYPicw== dependencies: "@types/node" "*" +"@types/imagemin@8.0.1": + version "8.0.1" + resolved "https://registry.yarnpkg.com/@types/imagemin/-/imagemin-8.0.1.tgz#83795a10cdbe31a42ab304a7f8425588212c6a81" + integrity sha512-DSpM//dRPzme7doePGkmR1uoquHi0h0ElaA5qFnxHECfFcB8z/jhMI8eqmxWNpHn9ZG18p4PC918sZLhR0cr5A== + dependencies: + "@types/node" "*" + "@types/istanbul-lib-coverage@*", "@types/istanbul-lib-coverage@^2.0.0", "@types/istanbul-lib-coverage@^2.0.1": version "2.0.3" resolved "https://registry.yarnpkg.com/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.3.tgz#4ba8ddb720221f432e443bd5f9117fd22cfd4762" @@ -1569,13 +1607,13 @@ dependencies: "@types/istanbul-lib-report" "*" -"@types/jest@^27.4.1": - version "27.4.1" - resolved "https://registry.yarnpkg.com/@types/jest/-/jest-27.4.1.tgz#185cbe2926eaaf9662d340cc02e548ce9e11ab6d" - integrity sha512-23iPJADSmicDVrWk+HT58LMJtzLAnB2AgIzplQuq/bSrGaxCrlvRFjGbXmamnnk/mAmCdLStiGqggu28ocUyiw== +"@types/jest@^29.0.0": + version "29.5.1" + resolved "https://registry.yarnpkg.com/@types/jest/-/jest-29.5.1.tgz#83c818aa9a87da27d6da85d3378e5a34d2f31a47" + integrity sha512-tEuVcHrpaixS36w7hpsfLBLpjtMRJUE09/MHXn923LOVojDwyC14cWcfc0rDs0VEfUyYmt/+iX1kxxp+gZMcaQ== dependencies: - jest-matcher-utils "^27.0.0" - pretty-format "^27.0.0" + expect "^29.0.0" + pretty-format "^29.0.0" "@types/json-schema@^7.0.9": version "7.0.11" @@ -1624,10 +1662,10 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-16.9.1.tgz#0611b37db4246c937feef529ddcc018cf8e35708" integrity sha512-QpLcX9ZSsq3YYUUnD3nFDY8H7wctAhQj/TFKL8Ya8v5fMm3CFXxo8zStsLAl780ltoYoo1WvKUVGBQK+1ifr7g== -"@types/node@17.0.24": - version "17.0.24" - resolved "https://registry.yarnpkg.com/@types/node/-/node-17.0.24.tgz#20ba1bf69c1b4ab405c7a01e950c4f446b05029f" - integrity sha512-aveCYRQbgTH9Pssp1voEP7HiuWlD2jW2BO56w+bVrJn04i61yh6mRfoKO6hEYQD9vF+W8Chkwc6j1M36uPkx4g== +"@types/node@18.16.5": + version "18.16.5" + resolved "https://registry.yarnpkg.com/@types/node/-/node-18.16.5.tgz#bf64e42719dc2e74da24709a2e1c0b50a966120a" + integrity sha512-seOA34WMo9KB+UA78qaJoCO20RJzZGVXQ5Sh6FWu0g/hfT44nKXnej3/tCQl7FL97idFpBhisLYCTB50S0EirA== "@types/normalize-package-data@^2.4.0": version "2.4.1" @@ -1688,6 +1726,13 @@ dependencies: "@types/yargs-parser" "*" +"@types/yargs@^17.0.8": + version "17.0.24" + resolved "https://registry.yarnpkg.com/@types/yargs/-/yargs-17.0.24.tgz#b3ef8d50ad4aa6aecf6ddc97c580a00f5aa11902" + integrity sha512-6i0aC7jV6QzQB8ne1joVZ0eSFIstHsCrobmOtghM11yGlH0j43FKL2UhWdELkyps0zuf7qVTUVCCR+tgSlyLLw== + dependencies: + "@types/yargs-parser" "*" + "@typescript-eslint/eslint-plugin@5.21.0": version "5.21.0" resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.21.0.tgz#bfc22e0191e6404ab1192973b3b4ea0461c1e878" @@ -2885,6 +2930,11 @@ diff-sequences@^27.5.1: resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-27.5.1.tgz#eaecc0d327fd68c8d9672a1e64ab8dccb2ef5327" integrity sha512-k1gCAXAsNgLwEL+Y8Wvl+M6oEFj5bgazfZULpS5CneoPPXRaCCW7dm+q21Ky2VEE5X+VeRDBVg1Pcvvsr4TtNQ== +diff-sequences@^29.4.3: + version "29.4.3" + resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-29.4.3.tgz#9314bc1fabe09267ffeca9cbafc457d8499a13f2" + integrity sha512-ofrBgwpPhCD85kMKtE9RYFFq6OC1A89oW2vvgWZNCwxrUpRUILopY7lsYyMDSjc8g6U6aiO0Qubg6r4Wgt5ZnA== + diff@^4.0.1: version "4.0.2" resolved "https://registry.yarnpkg.com/diff/-/diff-4.0.2.tgz#60f3aecb89d5fae520c11aa19efc2bb982aade7d" @@ -3445,6 +3495,17 @@ expect@^27.5.1: jest-matcher-utils "^27.5.1" jest-message-util "^27.5.1" +expect@^29.0.0: + version "29.5.0" + resolved "https://registry.yarnpkg.com/expect/-/expect-29.5.0.tgz#68c0509156cb2a0adb8865d413b137eeaae682f7" + integrity sha512-yM7xqUrCO2JdpFo4XpM82t+PJBFybdqoQuJLDGeDX2ij8NZzqRHyu3Hp188/JX7SWqud+7t4MUdvcgGBICMHZg== + dependencies: + "@jest/expect-utils" "^29.5.0" + jest-get-type "^29.4.3" + jest-matcher-utils "^29.5.0" + jest-message-util "^29.5.0" + jest-util "^29.5.0" + ext-list@^2.0.0: version "2.2.2" resolved "https://registry.yarnpkg.com/ext-list/-/ext-list-2.2.2.tgz#0b98e64ed82f5acf0f2931babf69212ef52ddd37" @@ -4714,6 +4775,16 @@ jest-diff@^27.5.1: jest-get-type "^27.5.1" pretty-format "^27.5.1" +jest-diff@^29.5.0: + version "29.5.0" + resolved "https://registry.yarnpkg.com/jest-diff/-/jest-diff-29.5.0.tgz#e0d83a58eb5451dcc1fa61b1c3ee4e8f5a290d63" + integrity sha512-LtxijLLZBduXnHSniy0WMdaHjmQnt3g5sa16W4p0HqukYTTsyTW3GD1q41TyGl5YFXj/5B2U6dlh5FM1LIMgxw== + dependencies: + chalk "^4.0.0" + diff-sequences "^29.4.3" + jest-get-type "^29.4.3" + pretty-format "^29.5.0" + jest-docblock@^27.5.1: version "27.5.1" resolved "https://registry.yarnpkg.com/jest-docblock/-/jest-docblock-27.5.1.tgz#14092f364a42c6108d42c33c8cf30e058e25f6c0" @@ -4762,6 +4833,11 @@ jest-get-type@^27.5.1: resolved "https://registry.yarnpkg.com/jest-get-type/-/jest-get-type-27.5.1.tgz#3cd613c507b0f7ace013df407a1c1cd578bcb4f1" integrity sha512-2KY95ksYSaK7DMBWQn6dQz3kqAf3BB64y2udeG+hv4KfSOb9qwcYQstTJc1KCbsix+wLZWZYN8t7nwX3GOBLRw== +jest-get-type@^29.4.3: + version "29.4.3" + resolved "https://registry.yarnpkg.com/jest-get-type/-/jest-get-type-29.4.3.tgz#1ab7a5207c995161100b5187159ca82dd48b3dd5" + integrity sha512-J5Xez4nRRMjk8emnTpWrlkyb9pfRQQanDrvWHhsR1+VUfbwxi30eVcZFlcdGInRibU4G5LwHXpI7IRHU0CY+gg== + jest-haste-map@^27.5.1: version "27.5.1" resolved "https://registry.yarnpkg.com/jest-haste-map/-/jest-haste-map-27.5.1.tgz#9fd8bd7e7b4fa502d9c6164c5640512b4e811e7f" @@ -4813,7 +4889,7 @@ jest-leak-detector@^27.5.1: jest-get-type "^27.5.1" pretty-format "^27.5.1" -jest-matcher-utils@^27.0.0, jest-matcher-utils@^27.5.1: +jest-matcher-utils@^27.5.1: version "27.5.1" resolved "https://registry.yarnpkg.com/jest-matcher-utils/-/jest-matcher-utils-27.5.1.tgz#9c0cdbda8245bc22d2331729d1091308b40cf8ab" integrity sha512-z2uTx/T6LBaCoNWNFWwChLBKYxTMcGBRjAt+2SbP929/Fflb9aa5LGma654Rz8z9HLxsrUaYzxE9T/EFIL/PAw== @@ -4823,6 +4899,16 @@ jest-matcher-utils@^27.0.0, jest-matcher-utils@^27.5.1: jest-get-type "^27.5.1" pretty-format "^27.5.1" +jest-matcher-utils@^29.5.0: + version "29.5.0" + resolved "https://registry.yarnpkg.com/jest-matcher-utils/-/jest-matcher-utils-29.5.0.tgz#d957af7f8c0692c5453666705621ad4abc2c59c5" + integrity sha512-lecRtgm/rjIK0CQ7LPQwzCs2VwW6WAahA55YBuI+xqmhm7LAaxokSB8C97yJeYyT+HvQkH741StzpU41wohhWw== + dependencies: + chalk "^4.0.0" + jest-diff "^29.5.0" + jest-get-type "^29.4.3" + pretty-format "^29.5.0" + jest-message-util@^27.5.1: version "27.5.1" resolved "https://registry.yarnpkg.com/jest-message-util/-/jest-message-util-27.5.1.tgz#bdda72806da10d9ed6425e12afff38cd1458b6cf" @@ -4838,6 +4924,21 @@ jest-message-util@^27.5.1: slash "^3.0.0" stack-utils "^2.0.3" +jest-message-util@^29.5.0: + version "29.5.0" + resolved "https://registry.yarnpkg.com/jest-message-util/-/jest-message-util-29.5.0.tgz#1f776cac3aca332ab8dd2e3b41625435085c900e" + integrity sha512-Kijeg9Dag6CKtIDA7O21zNTACqD5MD/8HfIV8pdD94vFyFuer52SigdC3IQMhab3vACxXMiFk+yMHNdbqtyTGA== + dependencies: + "@babel/code-frame" "^7.12.13" + "@jest/types" "^29.5.0" + "@types/stack-utils" "^2.0.0" + chalk "^4.0.0" + graceful-fs "^4.2.9" + micromatch "^4.0.4" + pretty-format "^29.5.0" + slash "^3.0.0" + stack-utils "^2.0.3" + jest-mock@^27.5.1: version "27.5.1" resolved "https://registry.yarnpkg.com/jest-mock/-/jest-mock-27.5.1.tgz#19948336d49ef4d9c52021d34ac7b5f36ff967d6" @@ -4984,6 +5085,18 @@ jest-util@^27.5.1: graceful-fs "^4.2.9" picomatch "^2.2.3" +jest-util@^29.5.0: + version "29.5.0" + resolved "https://registry.yarnpkg.com/jest-util/-/jest-util-29.5.0.tgz#24a4d3d92fc39ce90425311b23c27a6e0ef16b8f" + integrity sha512-RYMgG/MTadOr5t8KdhejfvUU82MxsCu5MF6KuDUHl+NuwzUt+Sm6jJWxTJVrDR1j5M/gJVCPKQEpWXY+yIQ6lQ== + dependencies: + "@jest/types" "^29.5.0" + "@types/node" "*" + chalk "^4.0.0" + ci-info "^3.2.0" + graceful-fs "^4.2.9" + picomatch "^2.2.3" + jest-validate@^27.5.1: version "27.5.1" resolved "https://registry.yarnpkg.com/jest-validate/-/jest-validate-27.5.1.tgz#9197d54dc0bdb52260b8db40b46ae668e04df067" @@ -6261,7 +6374,7 @@ prepend-http@^2.0.0: resolved "https://registry.yarnpkg.com/prepend-http/-/prepend-http-2.0.0.tgz#e92434bfa5ea8c19f41cdfd401d741a3c819d897" integrity sha1-6SQ0v6XqjBn0HN/UAddBo8gZ2Jc= -pretty-format@^27.0.0, pretty-format@^27.5.1: +pretty-format@^27.5.1: version "27.5.1" resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-27.5.1.tgz#2181879fdea51a7a5851fb39d920faa63f01d88e" integrity sha512-Qb1gy5OrP5+zDf2Bvnzdl3jsTf1qXVMazbvCoKhtKqVs4/YK4ozX4gKQJJVyNe+cajNPn0KoC0MC3FUmaHWEmQ== @@ -6270,6 +6383,15 @@ pretty-format@^27.0.0, pretty-format@^27.5.1: ansi-styles "^5.0.0" react-is "^17.0.1" +pretty-format@^29.0.0, pretty-format@^29.5.0: + version "29.5.0" + resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-29.5.0.tgz#283134e74f70e2e3e7229336de0e4fce94ccde5a" + integrity sha512-V2mGkI31qdttvTFX7Mt4efOqHXqJWMu4/r66Xh3Z3BwZaPfPJgp6/gbwoujRpPUtfEF6AUUWx3Jim3GCw5g/Qw== + dependencies: + "@jest/schemas" "^29.4.3" + ansi-styles "^5.0.0" + react-is "^18.0.0" + process-nextick-args@~2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2" @@ -6393,6 +6515,11 @@ react-is@^17.0.1: resolved "https://registry.yarnpkg.com/react-is/-/react-is-17.0.2.tgz#e691d4a8e9c789365655539ab372762b0efb54f0" integrity sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w== +react-is@^18.0.0: + version "18.2.0" + resolved "https://registry.yarnpkg.com/react-is/-/react-is-18.2.0.tgz#199431eeaaa2e09f86427efbb4f1473edb47609b" + integrity sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w== + read-pkg-up@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/read-pkg-up/-/read-pkg-up-1.0.1.tgz#9d63c13276c065918d57f002a57f40a1b643fb02"