diff --git a/.github/workflows/check-domain.yaml b/.github/workflows/check-domain.yaml new file mode 100644 index 000000000..10ba65717 --- /dev/null +++ b/.github/workflows/check-domain.yaml @@ -0,0 +1,179 @@ +name: Check Domains + +on: + schedule: + - cron: "0 * * * *" # every hour + workflow_dispatch: + +permissions: + issues: write + contents: read + +jobs: + check: + runs-on: ubuntu-latest + + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - uses: denoland/setup-deno@v2 + with: + deno-version: v2.6.3 + + - name: Run domain checker + id: run + env: + EXCLUDE: ${{ vars.SCAN_SOURCES_EXCLUDE || '' }} + run: | + cd check-domains + deno task start + + - name: Create issues per domain + uses: actions/github-script@v7 + env: + AUTO_ASSIGN: ${{ vars.ISSUE_AUTO_ASSIGN || '' }} + with: + script: | + const fs = require("fs"); + + const data = JSON.parse(fs.readFileSync("check-domain-output.json", "utf8")); + + const deiced = data?.deiced || {}; + const changed = data?.changed || {}; + + const LABEL_TYPE = "domain" + const AUTO_ASSIGN = process.env.AUTO_ASSIGN || null; + + const owner = context.repo.owner; + const repo = context.repo.repo; + + const branch = context.ref.replace("refs/heads/", ""); + + const activeIssues = new Set(); + + const issues = await github.rest.issues.listForRepo({ + owner, + repo, + state: "open" + }); + + // Helper to build direct link to the source folder + const sourceFolderLink = (name) => + `https://github.com/${owner}/${repo}/tree/${branch}/sources/${name}`; + + // Helper to create issue with labels + const createIssue = async (title, body, extraLabels = []) => { + const exists = issues.data.find(i => i.title === title); + if (exists) { + activeIssues.add(exists.number); + console.log("Issue already exists:", title); + return; + } + + const labels = ["ci", LABEL_TYPE, ...extraLabels]; + + const newIssue = await github.rest.issues.create({ + owner, + repo, + title, + body, + labels + }); + + activeIssues.add(newIssue.data.number); + + console.log("Issue created:", title); + + if (AUTO_ASSIGN) { + console.log("Auto-assign to:", AUTO_ASSIGN); + + await github.rest.issues.addAssignees({ + owner, + repo, + issue_number: newIssue.data.number, + assignees: [AUTO_ASSIGN] + }); + } + }; + + // ---- Handle deiced ---- + for (const source of Object.keys(deiced)) { + const urls = deiced[source]; + const folderUrl = sourceFolderLink(source); + + const title = `source(${source}): domain deiced`; + + const body = [ + `## ❄️ Domain Deiced`, + `**Source:** ${source}`, + ``, + `### Source Folder`, + `${folderUrl}`, + ``, + `### URLs Checked`, + urls.map(u => "- " + u).join("\n"), + ``, + `### Timestamp`, + new Date().toISOString() + ].join("\n"); + + await createIssue(title, body, ["deiced"]); + } + + // ---- Handle changed ---- + for (const source of Object.keys(changed)) { + const urls = changed[source]; + const original = urls[0]; + const redirected = urls[1]; + const folderUrl = sourceFolderLink(source); + + const title = `source(${source}): changed → \`${redirected}\``; + + const body = [ + `## 🔀 Domain Redirect Detected`, + `**Source:** ${source}`, + ``, + `### Source Folder`, + `${folderUrl}`, + ``, + `### Original`, + `- ${original}`, + ``, + `### Redirected To`, + `- ${redirected}`, + ``, + `### Timestamp`, + new Date().toISOString() + ].join("\n"); + + await createIssue(title, body, ["changed"]); + } + + // ---- Auto-close stale fixed issues ---- + console.log("Checking for stale issues to close..."); + + for (const issue of issues.data) { + if (!issue.labels.some(l => l.name === LABEL_TYPE)) continue; + + // If this issue wasn't registered as active, it is now fixed + if (!activeIssues.has(issue.number)) { + console.log("Closing stale issue:", issue.title); + + // Comment before close + await github.rest.issues.createComment({ + owner, + repo, + issue_number: issue.number, + body: "Automatically closed because the domain is now resolved. 🟢" + }); + + // Close it + await github.rest.issues.update({ + owner, + repo, + issue_number: issue.number, + state: "closed" + }); + } + } diff --git a/.gitignore b/.gitignore index d71350931..24d4ae328 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,5 @@ templates/*/target .zed .ropeproject /target + +/check-domain-output.json diff --git a/check-domains/deno.json b/check-domains/deno.json new file mode 100644 index 000000000..545a00307 --- /dev/null +++ b/check-domains/deno.json @@ -0,0 +1,11 @@ +{ + "tasks": { + "dev": "deno run --allow-read --allow-env --allow-write=../check-domain-output.json --allow-net --watch main.ts", + "start": "deno run --allow-read --allow-env --allow-write=../check-domain-output.json --allow-net main.ts" + }, + "imports": { + "@std/assert": "jsr:@std/assert@1", + "p-limit": "npm:p-limit@^7.2.0", + "ts-retry": "npm:ts-retry@^6.0.0" + } +} diff --git a/check-domains/deno.lock b/check-domains/deno.lock new file mode 100644 index 000000000..786e58e21 --- /dev/null +++ b/check-domains/deno.lock @@ -0,0 +1,41 @@ +{ + "version": "5", + "specifiers": { + "jsr:@std/assert@1": "1.0.16", + "jsr:@std/internal@^1.0.12": "1.0.12", + "npm:p-limit@^7.2.0": "7.2.0", + "npm:ts-retry@6": "6.0.0" + }, + "jsr": { + "@std/assert@1.0.16": { + "integrity": "6a7272ed1eaa77defe76e5ff63ca705d9c495077e2d5fd0126d2b53fc5bd6532", + "dependencies": [ + "jsr:@std/internal" + ] + }, + "@std/internal@1.0.12": { + "integrity": "972a634fd5bc34b242024402972cd5143eac68d8dffaca5eaa4dba30ce17b027" + } + }, + "npm": { + "p-limit@7.2.0": { + "integrity": "sha512-ATHLtwoTNDloHRFFxFJdHnG6n2WUeFjaR8XQMFdKIv0xkXjrER8/iG9iu265jOM95zXHAfv9oTkqhrfbIzosrQ==", + "dependencies": [ + "yocto-queue" + ] + }, + "ts-retry@6.0.0": { + "integrity": "sha512-WsVRE/P+VNYbiQC3E6TeIXBRCQj7vzjN4MlXd84AC88K7WwuWShN7A3Q/QSV/yd1hjO8qn2Cevdqny2HMwKUaA==" + }, + "yocto-queue@1.2.2": { + "integrity": "sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ==" + } + }, + "workspace": { + "dependencies": [ + "jsr:@std/assert@1", + "npm:p-limit@^7.2.0", + "npm:ts-retry@6" + ] + } +} diff --git a/check-domains/domain-alive.ts b/check-domains/domain-alive.ts new file mode 100644 index 000000000..c461d6090 --- /dev/null +++ b/check-domains/domain-alive.ts @@ -0,0 +1,30 @@ +// ドメインが生きているかどうかをDNSで確認する +// google dns? + +import type { LookupAddress } from "node:dns"; +import { lookup } from "node:dns/promises"; + +export interface AliveOk { + alive: true; + records: LookupAddress[]; +} +export interface AliveFailed { + alive: false; + domains: string[]; + err: unknown; +} +export async function checkDomainAlive( + url: string +): Promise { + try { + const result = await lookup(new URL(url).hostname, { all: true }); + + // console.log("DNS Records:", result); + console.log("Domain %s is alive ✓", url); + + return { alive: true, records: result }; + } catch (err) { + console.log("DNS lookup failed ✗", err); + return { alive: false, domains: [url], err }; + } +} diff --git a/check-domains/domain-redirect.ts b/check-domains/domain-redirect.ts new file mode 100644 index 000000000..0e0260a7a --- /dev/null +++ b/check-domains/domain-redirect.ts @@ -0,0 +1,53 @@ +// HTTPステータスとリダイレクトを確認する関数 + +import type { AliveFailed } from "./domain-alive.ts"; + +export interface DomainOk { + alive: true; + from: string; + location: string; +} +export async function checkDomainHttp( + domain: string +): Promise { + const url = domain.startsWith("http") ? domain : `https://${domain}`; + + try { + const res = await fetch(url, { + redirect: "manual", // do NOT auto redirect + method: "GET", + }); + + // Redirect? + const location = res.headers.get("location"); + + if (location) { + const objUrl = new URL(url); + const objLocation = new URL(location, url); + + if ( + !checkDomainEqual(objUrl, objLocation, "protocol") || + !checkDomainEqual(objUrl, objLocation, "hostname") || + !checkDomainEqual(objUrl, objLocation, "port") || + // !checkDomainEqual(objUrl, objLocation, "pathname") || + !checkDomainEqual(objUrl, objLocation, "username") || + !checkDomainEqual(objUrl, objLocation, "password") + ) { + console.log("%s redirected to: %s", url, location); + return { + alive: true, + from: url, + location, + }; + } + } + // deno-lint-ignore no-explicit-any + } catch (err) { + console.log("HTTP check failed ✗", err instanceof Error ? err.message : String(err)); + return { alive: false, err, domains: [url] }; + } +} + +function checkDomainEqual(u1: URL, u2: URL, name: keyof URL): boolean { + return u1[name] === u2[name]; +} diff --git a/check-domains/main.ts b/check-domains/main.ts new file mode 100644 index 000000000..7a61facb7 --- /dev/null +++ b/check-domains/main.ts @@ -0,0 +1,102 @@ +import { dirname, join } from "node:path"; + +import { checkDomainAlive, type AliveFailed } from "./domain-alive.ts"; +import { checkDomainHttp, type DomainOk } from "./domain-redirect.ts"; +import pLimit from "p-limit"; +import { retryAsync } from "ts-retry"; +import { readdir } from "node:fs/promises"; +import process from "node:process"; + +const sources = join(dirname(import.meta.dirname ?? ""), "sources"); + +async function checkSource( + name: string +): Promise { + const meta = JSON.parse( + await Deno.readTextFile(join(sources, name, "res/source.json")).catch( + (err) => { + console.warn(`Read file ${name}/res/source.json failed: ${err}`); + return Promise.reject(err); + } + ) + ) as { + info: { + url?: string; + urls?: string[]; + }; + }; + + const urls = meta.info.urls ?? [meta.info.url!]; + + // check first domain alive? + for (const url of urls.slice(0, 1)) { + const output = await retryAsync( + async () => { + const look = await checkDomainAlive(url); + if (!look.alive) return look; + + const changed = await checkDomainHttp(url); + if (changed) return changed; + }, + { maxTry: 3 } + ); + if (output) return output; + } +} + +const listSources = await readdir(sources); +const limit = pLimit(10); + +console.log("Total size: %d", listSources.length); + +const excludes = process.env.EXCLUDE?.split(",").filter(Boolean); + +const sourcesDomainChanged: Map = new Map(); +const sourcesDomainDeiced: Map = new Map(); + +await Promise.all( + listSources.map((name) => + limit(async () => { + if (excludes?.includes(name)) { + console.info("Skip %s", name); + + return; + } + + try { + const output = await checkSource(name); + if (output) { + if (output.alive) { + sourcesDomainChanged.set(name, [output.from, output.location]); + } else { + sourcesDomainDeiced.set(name, output.domains); + } + } + } catch (error) { + console.log(error); + } + }) + ) +); + +const outfile = await Deno.open( + join(dirname(import.meta.dirname ?? ""), "check-domain-output.json"), + { write: true, create: true } +); +await outfile.write( + new TextEncoder().encode( + JSON.stringify( + { + deiced: Object.fromEntries(sourcesDomainDeiced), + changed: Object.fromEntries(sourcesDomainChanged), + }, + null, + 2 + ) + ) +); + +console.log(`Ok Report: + [+] ${sourcesDomainChanged.size} source changed domain + [+] ${sourcesDomainDeiced.size} source deiced domain +`); diff --git a/sources/ar.promanga/res/source.json b/sources/ar.promanga/res/source.json index 27c8df469..f7d494f00 100644 --- a/sources/ar.promanga/res/source.json +++ b/sources/ar.promanga/res/source.json @@ -2,8 +2,8 @@ "info": { "id": "ar.promanga", "name": "Pro Manga", - "version": 1, - "url": "https://promanga.net", + "version": 2, + "url": "https://prochan.net", "contentRating": 1, "languages": ["ar"] } diff --git a/sources/ar.promanga/src/lib.rs b/sources/ar.promanga/src/lib.rs index b4f6fbb76..9a3714ed8 100644 --- a/sources/ar.promanga/src/lib.rs +++ b/sources/ar.promanga/src/lib.rs @@ -2,7 +2,7 @@ use aidoku::{prelude::*, Source}; use iken::{Iken, Impl, Params}; -const BASE_URL: &str = "https://promanga.net"; +const BASE_URL: &str = "https://prochan.net"; struct ProManga; diff --git a/sources/ja.manga1000/res/source.json b/sources/ja.manga1000/res/source.json index 1589e51f6..4c6ed422a 100644 --- a/sources/ja.manga1000/res/source.json +++ b/sources/ja.manga1000/res/source.json @@ -2,8 +2,8 @@ "info": { "id": "ja.manga1000", "name": "Manga1000", - "version": 1, - "url": "https://manga1000.top", + "version": 2, + "url": "https://hachiraw.win", "contentRating": 1, "languages": ["ja"] }, diff --git a/sources/ja.manga1000/src/lib.rs b/sources/ja.manga1000/src/lib.rs index 533529ba1..62ef02511 100644 --- a/sources/ja.manga1000/src/lib.rs +++ b/sources/ja.manga1000/src/lib.rs @@ -2,7 +2,7 @@ use aidoku::{prelude::*, Source}; use liliana::{Impl, Liliana, Params}; -const BASE_URL: &str = "https://manga1000.top"; +const BASE_URL: &str = "https://hachiraw.win"; struct Manga1000; diff --git a/sources/vi.dilib/res/source.json b/sources/vi.dilib/res/source.json index f1a9e4223..00b9e86e0 100644 --- a/sources/vi.dilib/res/source.json +++ b/sources/vi.dilib/res/source.json @@ -2,6 +2,7 @@ "info": { "id": "vi.dilib", "name": "Digital Library", + "url": "https://dilib.vn", "version": 3, "contentRating": 1, "languages": [ diff --git a/sources/vi.hakovn/.cargo/config.toml b/sources/vi.hakovn/.cargo/config.toml new file mode 100644 index 000000000..f137b5a99 --- /dev/null +++ b/sources/vi.hakovn/.cargo/config.toml @@ -0,0 +1,5 @@ +[build] +target = "wasm32-unknown-unknown" + +[target.wasm32-unknown-unknown] +runner = "aidoku-test-runner" diff --git a/sources/vi.hakovn/Cargo.lock b/sources/vi.hakovn/Cargo.lock new file mode 100644 index 000000000..5913f419a --- /dev/null +++ b/sources/vi.hakovn/Cargo.lock @@ -0,0 +1,407 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aidoku" +version = "0.3.0" +source = "git+https://github.com/Aidoku/aidoku-rs.git#96d7d8c784bfb1327dabe1bef5cc0ffcaa334f62" +dependencies = [ + "euclid", + "hashbrown", + "itoa", + "num-traits", + "paste", + "postcard", + "serde", + "serde_json", + "talc", + "thiserror", +] + +[[package]] +name = "aidoku-test" +version = "1.0.0" +source = "git+https://github.com/Aidoku/aidoku-rs.git#96d7d8c784bfb1327dabe1bef5cc0ffcaa334f62" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "atomic-polyfill" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cf2bce30dfe09ef0bfaef228b9d414faaf7e563035494d7fe092dba54b300f4" +dependencies = [ + "critical-section", +] + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chrono" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" +dependencies = [ + "num-traits", +] + +[[package]] +name = "cobs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" +dependencies = [ + "thiserror", +] + +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + +[[package]] +name = "euclid" +version = "0.22.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad9cdb4b747e485a12abb0e6566612956c7a1bafa3bdb8d682c5b6d403589e48" +dependencies = [ + "num-traits", +] + +[[package]] +name = "hakovn" +version = "0.1.0" +dependencies = [ + "aidoku", + "aidoku-test", + "wpcomics", +] + +[[package]] +name = "hash32" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", + "allocator-api2", + "serde", +] + +[[package]] +name = "heapless" +version = "0.7.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f" +dependencies = [ + "atomic-polyfill", + "hash32", + "rustc_version", + "serde", + "spin", + "stable_deref_trait", +] + +[[package]] +name = "itoa" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "libm" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "memchr" +version = "2.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "postcard" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" +dependencies = [ + "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", + "heapless", + "serde", +] + +[[package]] +name = "proc-macro2" +version = "1.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "semver" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.145" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", + "serde_core", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "syn" +version = "2.0.111" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "390cc9a294ab71bdb1aa2e99d13be9c753cd2d7bd6560c77118597410c4d2e87" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "talc" +version = "4.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3ae828aa394de34c7de08f522d1b86bd1c182c668d27da69caadda00590f26d" +dependencies = [ + "lock_api", +] + +[[package]] +name = "thiserror" +version = "2.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "unicode-ident" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wpcomics" +version = "0.1.0" +dependencies = [ + "aidoku", + "chrono", +] + +[[package]] +name = "zerocopy" +version = "0.8.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd74ec98b9250adb3ca554bdde269adf631549f51d8a8f8f0a10b50f1cb298c3" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a8d209fdf45cf5138cbb5a506f6b52522a25afccc534d1475dad8e31105c6a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/sources/vi.hakovn/Cargo.toml b/sources/vi.hakovn/Cargo.toml new file mode 100644 index 000000000..d7efda0a4 --- /dev/null +++ b/sources/vi.hakovn/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "hakovn" +version = "0.1.0" +edition = "2024" + +[dependencies] +aidoku = { git = "https://github.com/Aidoku/aidoku-rs.git" } +wpcomics = { path = "../../templates/wpcomics" } + +[dev-dependencies] +aidoku = { git = "https://github.com/Aidoku/aidoku-rs.git", features = ["test"] } +aidoku-test = { git = "https://github.com/Aidoku/aidoku-rs.git" } + +[lib] +crate-type = ["cdylib"] + +[profile.dev] +panic = "abort" + +[profile.release] +panic = "abort" +opt-level = "s" +strip = true +lto = true diff --git a/sources/vi.hakovn/res/filters.json b/sources/vi.hakovn/res/filters.json new file mode 100644 index 000000000..74da5ed60 --- /dev/null +++ b/sources/vi.hakovn/res/filters.json @@ -0,0 +1,156 @@ +[ + { + "type": "text", + "id": "author", + "title": "Tác giả" + }, + { + "type": "text", + "id": "illustrator", + "title": "Họa sĩ" + }, + { + "type": "sort", + "id": "sort", + "title": "Sắp xếp theo", + "options": ["Tất cả", "Đang tiến hành", "Tạm ngưng", "Hoàn thành"], + "ids": ["0", "1", "2", "3"] + }, + { + "type": "multi-select", + "id": "genres[]", + "title": "Thể loại", + "isGenre": true, + "canExclude": true, + "options": [ + "Action", + "Adapted to Anime", + "Adapted to Drama CD", + "Adapted to Manga", + "Adapted to Manhua", + "Adapted to Manhwa", + "Adult", + "Adventure", + "Age Gap", + "Boys Love", + "Character Growth", + "Chinese Novel", + "Comedy", + "Cooking", + "Different Social Status", + "Drama", + "Ecchi", + "English Novel", + "Fanfiction", + "Fantasy", + "Female Protagonist", + "Game", + "Gender Bender", + "Harem", + "Historical", + "Horror", + "Incest", + "Isekai", + "Josei", + "Korean Novel", + "Magic", + "Martial Arts", + "Mature", + "Mecha", + "Military", + "Misunderstanding", + "Mystery", + "Netorare", + "One shot", + "Otome Game", + "Parody", + "Psychological", + "Reverse Harem", + "Romance", + "School Life", + "Science Fiction", + "Seinen", + "Shoujo", + "Shoujo ai", + "Shounen", + "Shounen ai", + "Slice of Life", + "Slow Life", + "Sports", + "Super Power", + "Supernatural", + "Suspense", + "Tragedy", + "Wars", + "Web Novel", + "Workplace", + "Yandere", + "Yuri" + ], + "ids": [ + "1", + "49", + "51", + "50", + "64", + "65", + "28", + "2", + "52", + "60", + "54", + "39", + "3", + "43", + "56", + "4", + "5", + "40", + "62", + "6", + "59", + "45", + "7", + "8", + "35", + "9", + "10", + "30", + "33", + "34", + "44", + "37", + "27", + "11", + "36", + "58", + "12", + "32", + "38", + "46", + "61", + "23", + "47", + "22", + "13", + "14", + "31", + "15", + "16", + "26", + "17", + "18", + "55", + "19", + "24", + "20", + "25", + "21", + "53", + "29", + "57", + "63", + "48" + ] + } +] diff --git a/sources/vi.hakovn/res/icon.png b/sources/vi.hakovn/res/icon.png new file mode 100644 index 000000000..6f12d2fe8 Binary files /dev/null and b/sources/vi.hakovn/res/icon.png differ diff --git a/sources/vi.hakovn/res/source.json b/sources/vi.hakovn/res/source.json new file mode 100644 index 000000000..8455d2324 --- /dev/null +++ b/sources/vi.hakovn/res/source.json @@ -0,0 +1,18 @@ +{ + "info": { + "id": "vi.hakovn", + "name": "HakoVN", + "version":4, + "contentRating": 1, + "urls": [ + "https://docln.sbs", + "https://hako.vn", + "https://ln.hako.vn", + "https://docln.net" + ], + "languages": ["vi"] + }, + "config": { + "allowsBaseUrlSelect": true + } +} diff --git a/sources/vi.hakovn/src/lib.rs b/sources/vi.hakovn/src/lib.rs new file mode 100644 index 000000000..d6261aa5d --- /dev/null +++ b/sources/vi.hakovn/src/lib.rs @@ -0,0 +1,391 @@ +#![no_std] +use aidoku::{ + alloc::{borrow::ToOwned, string::ToString, *}, + helpers::uri::QueryParameters, + imports::{ + defaults::defaults_get, + html::{Element, Html}, + }, + prelude::*, + Chapter, FilterValue, Manga, Page, PageContent, Result, Source, Viewer, +}; +use wpcomics::{helpers::extract_f32_from_string, Impl, Params, WpComics}; + +const USER_AGENT: &str = "Mozilla/5.0 (iPhone; CPU iPhone OS 17_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/300.0.598994205 Mobile/15E148 Safari/604"; + +fn remove_node(node: Element, content_html: &mut String) { + if let Some(node_html) = node.outer_html() { + *content_html = content_html.replace(&node_html, ""); + } +} + +struct Hako; + +impl Impl for Hako { + fn new() -> Self { + Self + } + + fn params(&self) -> Params { + let manga_details_cover_transformer = |style: String| { + // style="background-image: url('https://...');" + let start = style.find("url('").map(|i| i + 5).unwrap_or(0); + let end = style[start..] + .find("')") + .map(|i| start + i) + .unwrap_or(style.len()); + style[start..end].to_string() + }; + Params { + base_url: defaults_get::("url").unwrap_or_default().into(), + viewer: Viewer::RightToLeft, + + next_page: ".next:not(.disabled)", + manga_cell: ".row .thumb-item-flow", + manga_cell_title: ".series-title a", + manga_cell_url: ".series-title a", + manga_cell_image: ".content.img-in-ratio", + manga_cell_image_attr: "abs:data-bg", + manga_parse_id: |url| { + url.split_once("//") + .map(|(_, rest)| rest) + .unwrap_or_default() + .split_once('/') + .map(|(_, path)| path) + .unwrap_or_default() + .trim_start_matches('/') + .to_string() + }, + chapter_parse_id: |url| { + url.trim_start_matches('/') + .split("/") + .last() + .unwrap_or_default() + .to_string() + }, + + manga_details_title: ".series-name > a", + manga_details_cover: ".series-cover .img-in-ratio", + manga_details_cover_attr: "style", + manga_details_cover_transformer, + manga_details_authors: ".info-name:contains(Tác giả) + span", + manga_details_description: ".summary-wrapper", + manga_details_tags: "a.series-gerne-item", + manga_details_tags_splitter: "", + manga_details_status: ".info-name:contains(Tình trạng) + span", + + user_agent: Some(USER_AGENT), + + get_search_url: |params, q, page, filters| { + let mut excluded_tags: Vec = Vec::new(); + let mut included_tags: Vec = Vec::new(); + let mut query = QueryParameters::new(); + query.push("query", Some(&q.to_owned().unwrap_or_default())); + query.push("keywords", Some(&q.to_owned().unwrap_or_default())); + query.push("title", Some(&q.unwrap_or_default())); + query.push("page", Some(&page.to_string())); + + if filters.is_empty() { + return Ok(format!("{}/tim-kiem?{query}", params.base_url,)); + } + + for filter in filters { + match filter { + FilterValue::Text { id, value, .. } => { + query.push(&id, core::prelude::v1::Some(&value)); + } + FilterValue::MultiSelect { + included, excluded, .. + } => { + for tag in included { + included_tags.push(tag); + } + for tag in excluded { + excluded_tags.push(tag); + } + } + FilterValue::Select { id, value } => { + query.push(&id, Some(&value)); + } + FilterValue::Sort { id, index, .. } => { + query.push(&id, Some(&index.to_string())); + } + _ => {} + } + } + + Ok(format!( + "{}/tim-kiem-nang-cao/?selectgenres={}&rejectgenres={}&{}", + params.base_url, + included_tags.join(","), + excluded_tags.join(","), + query + )) + }, + + home_manga_link: ".series-title a", + home_chapter_link: ".chapter-title a", + + home_sliders_selector: ".slider", + home_sliders_title_selector: "h2", + home_sliders_item_selector: ".popular-thumb-item", + + home_grids_selector: ".index-section", + home_grids_title_selector: ".section-title", + home_grids_item_selector: ".thumb-item-flow", + + home_manga_cover_selector: ".content.img-in-ratio", + home_manga_cover_slider_attr: Some("style"), + home_manga_cover_slider_transformer: manga_details_cover_transformer, + home_manga_cover_attr: "abs:data-bg", + time_formats: Some(["%d/%m/%Y", "%m-%d-%Y", "%Y-%d-%m"].to_vec()), + + ..Default::default() + } + } + + fn get_chapter_list( + &self, + cache: &mut wpcomics::Cache, + params: &Params, + url: String, + ) -> Result> { + let html = self.cache_manga_page(cache, params, url.as_str())?; + + let html = Html::parse_with_url(html, url)?; + let title_untrimmed = (params.manga_details_title_transformer)( + html.select(params.manga_details_title) + .and_then(|v| v.text()) + .unwrap_or_default(), + ); + let title = title_untrimmed.trim(); + + let Some(volumes_iter) = html.select(".volume-list") else { + return Ok(vec![]); + }; + + let mut chapters = volumes_iter + .filter_map(|volume_node| { + let Some(chapters_iter) = volume_node.select(".list-chapters > li") else { + return None; + }; + + let volume_title = volume_node + .select_first(".sect-title") + .and_then(|v| v.text()) + .unwrap_or_default(); + let volume_number = if volume_title.to_lowercase().contains("one shot") { + -1.0 + } else { + extract_f32_from_string(&title, &volume_title) + .first() + .map(|v| *v) + .unwrap_or(-1.0) + }; + let volume_title = + if let Some((_, rest)) = volume_title.split_once(&volume_number.to_string()) { + rest.trim_start_matches([':', '-', ' ']).trim().to_string() + } else { + volume_title.replace(title, "").trim().to_string() + }; + let volume_thumbnail = + volume_node + .select_first(".content.img-in-ratio") + .and_then(|node| { + let style = node.attr("style")?; + let url = (params.home_manga_cover_slider_transformer)(style); + Some(url) + }); + + Some( + chapters_iter + .filter_map(|chapter_node| { + let anchor_node = chapter_node.select_first("a")?; + + let chapter_url = anchor_node.attr("abs:href")?; + + let chapter_id = (params.chapter_parse_id)(chapter_url.to_owned()); + let chapter_title = anchor_node.text().unwrap_or_default(); + let chapter_title = chapter_title.trim(); + let chapter_number = extract_f32_from_string(&title, &chapter_title) + .first() + .map(|v| *v) + .unwrap_or(-1.0); + let chapter_title = if let Some((_, rest)) = + chapter_title.split_once(&chapter_number.to_string()) + { + rest.trim_start_matches([':', '-', ' ']).trim().to_string() + } else { + chapter_title.replace(title, "").trim().to_string() + }; + + let date_updated = (params.time_converter)( + params, + &chapter_node + .select(".chapter-time")? + .text() + .unwrap_or_default(), + ); + + let chapter = Chapter { + key: chapter_id, + title: Some( + format!( + "{}{}{}", + chapter_title, + if volume_title.is_empty() { "" } else { " - " }, + volume_title + ) + .to_string(), + ), + volume_number: if volume_number < 0.0 { + None + } else { + Some(volume_number) + }, + chapter_number: if chapter_number < 0.0 { + None + } else { + Some(chapter_number) + }, + date_uploaded: Some(date_updated), + url: Some(chapter_url), + thumbnail: volume_thumbnail.to_owned(), + ..Default::default() + }; + + Some(chapter) + }) + .collect::>(), + ) + }) + .flatten() + .collect::>(); + chapters.reverse(); + + Ok(chapters) + } + + fn get_page_list( + &self, + cache: &mut wpcomics::Cache, + params: &Params, + manga: Manga, + chapter: Chapter, + ) -> Result> { + let mut pages: Vec = Vec::new(); + + let url = (params.page_list_page)(params, &manga, &chapter); + let html = self.create_request(cache, params, &url, None)?.html()?; + + let Some(content) = html.select_first("#chapter-content") else { + bail!("Failed to get chapter content"); + }; + let Some(mut content_html) = content.html() else { + bail!("Failed to get chapter content HTML"); + }; + + // modify html + if let Some(list) = + content.select(".d-none, script, #chapter-content > a[target='__blank']") + { + for node in list { + remove_node(node, &mut content_html); + } + } + if let Some(list) = content.select("[id^=\"note\"]") { + for node in list { + let none_print_node = node.select(".none-print.inline"); + if let Some(none_print_node) = none_print_node { + for node in none_print_node { + remove_node(node, &mut content_html); + } + } + + let note_content_node = node.select_first(".note-content").and_then(|v| v.parent()); + if let Some(note_content_node) = note_content_node { + remove_node(note_content_node, &mut content_html); + } + } + } + + if let Some(styles_node) = content.select("[style]") { + for style_node in styles_node { + if let Some(style) = style_node.attr("style") { + let has_display_none = style.contains("display:") + && style[style.find("display:").unwrap_or_default()..].contains("none"); + if has_display_none { + remove_node(style_node, &mut content_html); + } + } + } + } + + // edit notes + if let Some(notes) = content.select("[id^=\"note\"]") { + let ids = notes + .into_iter() + .filter_map(|node| node.attr("id")) + .collect::>(); + + // Replace occurrences like [note123] with an anchor only if the id exists + let original = content_html.clone(); + content_html = String::new(); + let mut last_idx: usize = 0; + + while let Some(rel_start) = original[last_idx..].find('[') { + let start = last_idx + rel_start; + // find closing bracket after start + if let Some(rel_end) = original[start + 1..].find(']') { + let end = start + 1 + rel_end; // index of ']' + // append text before '[' + content_html.push_str(&original[last_idx..start]); + let inner = &original[start + 1..end]; + let is_note = inner.len() > 4 + && inner.starts_with("note") + && inner[4..].chars().all(|c| c.is_ascii_digit()); + if is_note && ids.iter().any(|id| id == inner) { + content_html.push_str(&format!( + "**", + id = inner + )); + } else { + // not a matching note id — keep original including brackets + content_html.push_str(&original[start..=end]); + } + last_idx = end + 1; + continue; + } + // no closing bracket found; stop searching + break; + } + // append remaining tail + content_html.push_str(&original[last_idx..]); + } + + // remove comments + while let Some(start) = content_html.find("") { + let end_pos = start + end + 3; + content_html.drain(start..end_pos); + } else { + break; + } + } + + // end modify html + + let description = html.select_first("h6.title-item").and_then(|v| v.text()); + + pages.push(Page { + content: PageContent::Text(format!("{content_html}")), + has_description: description.is_some(), + description, + ..Default::default() + }); + + Ok(pages) + } +} + +register_source!(WpComics, ImageRequestProvider, DeepLinkHandler, Home); diff --git a/sources/vi.truyenqq2/res/source.json b/sources/vi.truyenqq2/res/source.json index d42cc9347..80b9cc24a 100644 --- a/sources/vi.truyenqq2/res/source.json +++ b/sources/vi.truyenqq2/res/source.json @@ -2,8 +2,8 @@ "info": { "id": "vi.truyenqq2", "name": "TruyenQQ2", - "version": 2, - "url": "https://truyenqq.online", + "version": 3, + "url": "https://www.truyenqq.online", "contentRating": 1, "languages": [ "vi" diff --git a/sources/vi.truyenqq2/src/lib.rs b/sources/vi.truyenqq2/src/lib.rs index 03398c385..361ab3153 100644 --- a/sources/vi.truyenqq2/src/lib.rs +++ b/sources/vi.truyenqq2/src/lib.rs @@ -9,7 +9,7 @@ use aidoku::{ use wpcomics::{Impl, Params, WpComics}; const USER_AGENT: &str = "Mozilla/5.0 (iPhone; CPU iPhone OS 17_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/300.0.598994205 Mobile/15E148 Safari/604"; -const BASE_URL: &str = "https://truyenqq.online"; +const BASE_URL: &str = "https://www.truyenqq.online"; fn get_visit_read_id() -> String { defaults_get::("visitReadId")