diff --git a/.github/ISSUE_TEMPLATE/01_report_issue.yml b/.github/ISSUE_TEMPLATE/01_report_issue.yml index 69fafbe75..b16f99ba3 100644 --- a/.github/ISSUE_TEMPLATE/01_report_issue.yml +++ b/.github/ISSUE_TEMPLATE/01_report_issue.yml @@ -63,7 +63,7 @@ body: description: | You can find your Tsundoku/Tsundoku fork version number in **More → About**. placeholder: | - Example: "Tsundoku 0.1.0" + Example: "Tsundoku 0.2.0" validations: required: true @@ -95,7 +95,7 @@ body: required: true - label: I have written a short but informative title. required: true - - label: I have updated the app to version **[0.1.0](https://github.com/tsundoku-otaku/tsundoku/releases/latest)**. + - label: I have updated the app to version **[0.2.0](https://github.com/tsundoku-otaku/tsundoku/releases/latest)**. required: true - label: I have updated the extension to the latest version (as listed on the [extensions listing](https://novelsourcery.github.io/extensions/) page). required: true diff --git a/.github/ISSUE_TEMPLATE/05_request_feature.yml b/.github/ISSUE_TEMPLATE/05_request_feature.yml index 902e3c574..14909f822 100644 --- a/.github/ISSUE_TEMPLATE/05_request_feature.yml +++ b/.github/ISSUE_TEMPLATE/05_request_feature.yml @@ -53,7 +53,7 @@ body: required: true - label: If this is an issue with the app itself, I should be opening an issue in the [app repository](https://github.com/tsundoku-otaku/tsundoku/issues/). required: true - - label: I have updated the app to version **[0.1.1](https://github.com/tsundoku-otaku/tsundoku/releases/latest)**. + - label: I have updated the app to version **[0.2.0](https://github.com/tsundoku-otaku/tsundoku/releases/latest)**. required: true - label: I will correctly fill out all of the requested information in this form. required: true diff --git a/.github/ISSUE_TEMPLATE/06_request_meta.yml b/.github/ISSUE_TEMPLATE/06_request_meta.yml index 137db0e69..5160ee304 100644 --- a/.github/ISSUE_TEMPLATE/06_request_meta.yml +++ b/.github/ISSUE_TEMPLATE/06_request_meta.yml @@ -33,7 +33,7 @@ body: required: true - label: If this is an issue with the app itself, I should be opening an issue in the [app repository](https://github.com/tsundoku-otaku/tsundoku/issues/). required: true - - label: I have updated the app to version **[0.1.1](https://github.com/tsundoku-otaku/tsundoku/releases/latest)**. + - label: I have updated the app to version **[0.2.0](https://github.com/tsundoku-otaku/tsundoku/releases/latest)**. required: true - label: I have updated the extension to the latest version (as listed on the [extensions listing](https://novelsourcery.github.io/extensions/) page). required: true diff --git a/.github/always_build.json b/.github/always_build.json deleted file mode 100644 index 0d4f101c7..000000000 --- a/.github/always_build.json +++ /dev/null @@ -1,2 +0,0 @@ -[ -] diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 0eff6a604..62f4b3357 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,9 +1,9 @@ Checklist: -- [ ] Updated `extVersionCode` value in `build.gradle` for individual extensions -- [ ] Updated `overrideVersionCode` or `baseVersionCode` as needed for all multisrc extensions +- [ ] Updated `versionCode` value in `build.gradle.kts` +- [ ] Updated `baseVersionCode` in `build.gradle.kts` (if updated multisrc theme code) - [ ] Referenced all related issues in the PR body (e.g. "Closes #xyz") -- [ ] Added the `isNsfw = true` flag in `build.gradle` when appropriate +- [ ] Set the `contentWarning` configuration in `build.gradle.kts` appropriately - [ ] Have not changed source names - [ ] Have explicitly kept the `id` if a source's name or language were changed - [ ] Have tested the modifications by compiling and running the extension through Android Studio diff --git a/.github/scripts/commit-repo.sh b/.github/scripts/commit-repo.sh deleted file mode 100755 index b5bc8dfd0..000000000 --- a/.github/scripts/commit-repo.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -set -e - -git config --global user.email "<>" -git config --global user.name "GitHub Actions Bot" -git status -if [ -n "$(git status --porcelain)" ]; then - git add . - git commit -m "Update extensions repo" - git push - - curl https://purge.jsdelivr.net/gh/novelsourcery/extensions@repo/index.min.json -else - echo "No changes to commit" -fi diff --git a/.github/scripts/create-repo.py b/.github/scripts/create-repo.py deleted file mode 100644 index 5c25c3a27..000000000 --- a/.github/scripts/create-repo.py +++ /dev/null @@ -1,85 +0,0 @@ -import json -import os -import re -import subprocess -from pathlib import Path -from zipfile import ZipFile - -PACKAGE_NAME_REGEX = re.compile(r"package: name='([^']+)'") -VERSION_CODE_REGEX = re.compile(r"versionCode='([^']+)'") -VERSION_NAME_REGEX = re.compile(r"versionName='([^']+)'") -IS_NSFW_REGEX = re.compile(r"'tachiyomi.novelextension.nsfw' value='([^']+)'") -IS_NOVEL_REGEX = re.compile(r"'tachiyomi.novelextension.novel' value='([^']+)'") -APPLICATION_LABEL_REGEX = re.compile(r"^application-label:'([^']+)'", re.MULTILINE) -APPLICATION_ICON_320_REGEX = re.compile(r"^application-icon-320:'([^']+)'", re.MULTILINE) -LANGUAGE_REGEX = re.compile(r"tsundoku-([^.]+)") - -*_, ANDROID_BUILD_TOOLS = (Path(os.environ["ANDROID_HOME"]) / "build-tools").iterdir() -REPO_DIR = Path("repo") -REPO_APK_DIR = REPO_DIR / "apk" -REPO_ICON_DIR = REPO_DIR / "icon" - -REPO_ICON_DIR.mkdir(parents=True, exist_ok=True) - -with open("output.json", encoding="utf-8") as f: - inspector_data = json.load(f) - -index_data = [] - -for apk in REPO_APK_DIR.iterdir(): - badging = subprocess.check_output( - [ - ANDROID_BUILD_TOOLS / "aapt", - "dump", - "--include-meta-data", - "badging", - apk, - ] - ).decode() - - package_info = next(x for x in badging.splitlines() if x.startswith("package: ")) - package_name = PACKAGE_NAME_REGEX.search(package_info).group(1) - application_icon = APPLICATION_ICON_320_REGEX.search(badging).group(1) - - with ZipFile(apk) as z, z.open(application_icon) as i, ( - REPO_ICON_DIR / f"{package_name}.png" - ).open("wb") as f: - f.write(i.read()) - - language = LANGUAGE_REGEX.search(apk.name).group(1) - sources = inspector_data[package_name] - - if len(sources) == 1: - source_language = sources[0]["lang"] - - if ( - source_language != language - and source_language not in {"all", "other"} - and language not in {"all", "other"} - ): - language = source_language - - extension_data = { - "name": APPLICATION_LABEL_REGEX.search(badging).group(1), - "pkg": package_name, - "apk": apk.name, - "lang": language, - "code": int(VERSION_CODE_REGEX.search(package_info).group(1)), - "version": VERSION_NAME_REGEX.search(package_info).group(1), - "nsfw": int(IS_NSFW_REGEX.search(badging).group(1)), - "isNovel": bool(int(m.group(1))) if (m := IS_NOVEL_REGEX.search(badging)) else True, - "sources": [ - { - "name": source["name"], - "lang": source["lang"], - "id": source["id"], - "baseUrl": source["baseUrl"], - } - for source in sources - ], - } - - index_data.append(extension_data) - -with REPO_DIR.joinpath("index.min.json").open("w", encoding="utf-8") as index_file: - json.dump(index_data, index_file, ensure_ascii=False, separators=(",", ":")) diff --git a/.github/scripts/generate-build-matrices.py b/.github/scripts/generate-build-matrices.py index 4b797ab4c..ad6ae8f25 100644 --- a/.github/scripts/generate-build-matrices.py +++ b/.github/scripts/generate-build-matrices.py @@ -5,14 +5,13 @@ import subprocess import sys from pathlib import Path -from typing import NoReturn EXTENSION_REGEX = re.compile(r"^src/(?P\w+)/(?P\w+)") MULTISRC_LIB_REGEX = re.compile(r"^lib-multisrc/(?P\w+)") LIB_REGEX = re.compile(r"^lib/(?P\w+)") MODULE_REGEX = re.compile(r"^:src:(?P\w+):(?P\w+)$") CORE_FILES_REGEX = re.compile( - r"^(common/|core/|gradle/|build\.gradle\.kts|common\.gradle|gradle\.properties|settings\.gradle\.kts|.github/scripts)" + r"^(common/|compiler/|core/|gradle/|build\.gradle\.kts|gradle\.properties|settings\.gradle\.kts|.github/scripts)" ) def run_command(command: str) -> str: @@ -98,7 +97,10 @@ def resolve_ext(multisrcs: set[str], libs: set[str]) -> set[tuple[str, str]]: patterns = [] if multisrc_pattern: + # Most extensions still use the legacy Groovy `themePkg` property; only a + # handful have migrated to the modern DSL's `theme` property so far. patterns.append(rf"themePkg\s*=\s*['\"]({multisrc_pattern})['\"]") + patterns.append(rf"theme\s*=\s*['\"]({multisrc_pattern})['\"]") if lib_pattern: patterns.append(rf"project\([\"']:(?:lib):({lib_pattern})[\"']\)") @@ -108,7 +110,9 @@ def resolve_ext(multisrcs: set[str], libs: set[str]) -> set[tuple[str, str]]: for lang in Path("src").iterdir(): for extension in lang.iterdir(): - build_file = extension / "build.gradle" + build_file = extension / "build.gradle.kts" + if not build_file.is_file(): + build_file = extension / "build.gradle" if not build_file.is_file(): continue @@ -179,13 +183,6 @@ def get_module_list(ref: str) -> tuple[list[str], list[str]]: modules.update([f":src:{lang}:{extension}" for lang, extension in extensions]) deleted.update([f"{lang}.{extension}" for lang, extension in extensions]) - if os.getenv("IS_PR_CHECK") != "true": - with Path(".github/always_build.json").open() as always_build_file: - always_build = json.load(always_build_file) - for extension in always_build: - modules.add(":src:" + extension.replace(".", ":")) - deleted.add(extension) - return list(modules), list(deleted) def get_all_modules() -> tuple[list[str], list[str]]: diff --git a/.github/scripts/index.proto b/.github/scripts/index.proto new file mode 100644 index 000000000..88ca2feab --- /dev/null +++ b/.github/scripts/index.proto @@ -0,0 +1,65 @@ +syntax = "proto3"; + +// Upstream source: https://github.com/mihonapp/tachiyomix/blob/e4d546c/index/index.proto +// +// divergence: Resources.jarUrl (field 501), Extension.isNovel (field 8000, reserved 8000+ block +// for fork fields so they never collide with new upstream fields) +// +// Regenerate the Python bindings after editing: +// protoc --python_out=. --pyi_out=. index.proto + +message Index { + string name = 1; + string badgeLabel = 2; + string signingKey = 3; + Contact contact = 4; + oneof extensions { + ExtensionList extensionList = 101; + string extensionListUrl = 102; + } +} + +message Contact { + string website = 1; + optional string discord = 2; +} + +message ExtensionList { + repeated Extension extensions = 1; +} + +message Extension { + string name = 1; + string packageName = 2; + Resources resources = 3; + string extensionLib = 4; + int64 versionCode = 5; + string versionName = 6; + ContentWarning contentWarning = 7; + repeated Source sources = 8; + + // Fork field, reserved 8000+ block so it never collides with new upstream fields. + bool isNovel = 8000; +} + +message Resources { + string apkUrl = 1; + string iconUrl = 2; + string jarUrl = 501; +} + +message Source { + int64 id = 1; + string name = 2; + string language = 3; + string homeUrl = 4; + repeated string mirrorUrls = 5; + optional string message = 7; +} + +enum ContentWarning { + CONTENT_WARNING_UNSPECIFIED = 0; + CONTENT_WARNING_SAFE = 1; + CONTENT_WARNING_MIXED = 2; + CONTENT_WARNING_NSFW = 3; +} diff --git a/.github/scripts/index_pb2.py b/.github/scripts/index_pb2.py index 677dff05d..85519c499 100644 --- a/.github/scripts/index_pb2.py +++ b/.github/scripts/index_pb2.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # NO CHECKED-IN PROTOBUF GENCODE -# source: https://github.com/mihonapp/tachiyomix/blob/661fbb2/index/index.proto -# Protobuf Python Version: 7.35.0 +# source: index.proto +# Protobuf Python Version: 7.35.1 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -13,7 +13,7 @@ _runtime_version.Domain.PUBLIC, 7, 35, - 0, + 1, '', 'index.proto' ) @@ -24,15 +24,15 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0bindex.proto\"\xab\x01\n\x05Index\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x12\n\nbadgeLabel\x18\x02 \x01(\t\x12\x12\n\nsigningKey\x18\x03 \x01(\t\x12\x19\n\x07\x63ontact\x18\x04 \x01(\x0b\x32\x08.Contact\x12\'\n\rextensionList\x18\x65 \x01(\x0b\x32\x0e.ExtensionListH\x00\x12\x1a\n\x10\x65xtensionListUrl\x18\x66 \x01(\tH\x00\x42\x0c\n\nextensions\"<\n\x07\x43ontact\x12\x0f\n\x07website\x18\x01 \x01(\t\x12\x14\n\x07\x64iscord\x18\x02 \x01(\tH\x00\x88\x01\x01\x42\n\n\x08_discord\"/\n\rExtensionList\x12\x1e\n\nextensions\x18\x01 \x03(\x0b\x32\n.Extension\"\xd0\x01\n\tExtension\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0bpackageName\x18\x02 \x01(\t\x12\x1d\n\tresources\x18\x03 \x01(\x0b\x32\n.Resources\x12\x14\n\x0c\x65xtensionLib\x18\x04 \x01(\t\x12\x13\n\x0bversionCode\x18\x05 \x01(\x03\x12\x13\n\x0bversionName\x18\x06 \x01(\t\x12\'\n\x0e\x63ontentWarning\x18\x07 \x01(\x0e\x32\x0f.ContentWarning\x12\x18\n\x07sources\x18\x08 \x03(\x0b\x32\x07.Source\",\n\tResources\x12\x0e\n\x06\x61pkUrl\x18\x01 \x01(\t\x12\x0f\n\x07iconUrl\x18\x02 \x01(\t\"{\n\x06Source\x12\n\n\x02id\x18\x01 \x01(\x03\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x10\n\x08language\x18\x03 \x01(\t\x12\x0f\n\x07homeUrl\x18\x04 \x01(\t\x12\x12\n\nmirrorUrls\x18\x05 \x03(\t\x12\x14\n\x07message\x18\x07 \x01(\tH\x00\x88\x01\x01\x42\n\n\x08_message*_\n\x0e\x43ontentWarning\x12\x18\n\x14\x43ONTENT_WARNING_SAFE\x10\x00\x12\x19\n\x15\x43ONTENT_WARNING_MIXED\x10\x01\x12\x18\n\x14\x43ONTENT_WARNING_NSFW\x10\x02\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0bindex.proto\"\xab\x01\n\x05Index\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x12\n\nbadgeLabel\x18\x02 \x01(\t\x12\x12\n\nsigningKey\x18\x03 \x01(\t\x12\x19\n\x07\x63ontact\x18\x04 \x01(\x0b\x32\x08.Contact\x12\'\n\rextensionList\x18\x65 \x01(\x0b\x32\x0e.ExtensionListH\x00\x12\x1a\n\x10\x65xtensionListUrl\x18\x66 \x01(\tH\x00\x42\x0c\n\nextensions\"<\n\x07\x43ontact\x12\x0f\n\x07website\x18\x01 \x01(\t\x12\x14\n\x07\x64iscord\x18\x02 \x01(\tH\x00\x88\x01\x01\x42\n\n\x08_discord\"/\n\rExtensionList\x12\x1e\n\nextensions\x18\x01 \x03(\x0b\x32\n.Extension\"\xe2\x01\n\tExtension\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0bpackageName\x18\x02 \x01(\t\x12\x1d\n\tresources\x18\x03 \x01(\x0b\x32\n.Resources\x12\x14\n\x0c\x65xtensionLib\x18\x04 \x01(\t\x12\x13\n\x0bversionCode\x18\x05 \x01(\x03\x12\x13\n\x0bversionName\x18\x06 \x01(\t\x12\'\n\x0e\x63ontentWarning\x18\x07 \x01(\x0e\x32\x0f.ContentWarning\x12\x18\n\x07sources\x18\x08 \x03(\x0b\x32\x07.Source\x12\x10\n\x07isNovel\x18\xc0> \x01(\x08\"=\n\tResources\x12\x0e\n\x06\x61pkUrl\x18\x01 \x01(\t\x12\x0f\n\x07iconUrl\x18\x02 \x01(\t\x12\x0f\n\x06jarUrl\x18\xf5\x03 \x01(\t\"{\n\x06Source\x12\n\n\x02id\x18\x01 \x01(\x03\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x10\n\x08language\x18\x03 \x01(\t\x12\x0f\n\x07homeUrl\x18\x04 \x01(\t\x12\x12\n\nmirrorUrls\x18\x05 \x03(\t\x12\x14\n\x07message\x18\x07 \x01(\tH\x00\x88\x01\x01\x42\n\n\x08_message*\x80\x01\n\x0e\x43ontentWarning\x12\x1f\n\x1b\x43ONTENT_WARNING_UNSPECIFIED\x10\x00\x12\x18\n\x14\x43ONTENT_WARNING_SAFE\x10\x01\x12\x19\n\x15\x43ONTENT_WARNING_MIXED\x10\x02\x12\x18\n\x14\x43ONTENT_WARNING_NSFW\x10\x03\x62\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'index_pb2', _globals) if not _descriptor._USE_C_DESCRIPTORS: DESCRIPTOR._loaded_options = None - _globals['_CONTENTWARNING']._serialized_start=682 - _globals['_CONTENTWARNING']._serialized_end=777 + _globals['_CONTENTWARNING']._serialized_start=718 + _globals['_CONTENTWARNING']._serialized_end=846 _globals['_INDEX']._serialized_start=16 _globals['_INDEX']._serialized_end=187 _globals['_CONTACT']._serialized_start=189 @@ -40,9 +40,9 @@ _globals['_EXTENSIONLIST']._serialized_start=251 _globals['_EXTENSIONLIST']._serialized_end=298 _globals['_EXTENSION']._serialized_start=301 - _globals['_EXTENSION']._serialized_end=509 - _globals['_RESOURCES']._serialized_start=511 - _globals['_RESOURCES']._serialized_end=555 - _globals['_SOURCE']._serialized_start=557 - _globals['_SOURCE']._serialized_end=680 + _globals['_EXTENSION']._serialized_end=527 + _globals['_RESOURCES']._serialized_start=529 + _globals['_RESOURCES']._serialized_end=590 + _globals['_SOURCE']._serialized_start=592 + _globals['_SOURCE']._serialized_end=715 # @@protoc_insertion_point(module_scope) diff --git a/.github/scripts/index_pb2.pyi b/.github/scripts/index_pb2.pyi index 398ea82de..576d46dcb 100644 --- a/.github/scripts/index_pb2.pyi +++ b/.github/scripts/index_pb2.pyi @@ -9,9 +9,11 @@ DESCRIPTOR: _descriptor.FileDescriptor class ContentWarning(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): __slots__ = () + CONTENT_WARNING_UNSPECIFIED: _ClassVar[ContentWarning] CONTENT_WARNING_SAFE: _ClassVar[ContentWarning] CONTENT_WARNING_MIXED: _ClassVar[ContentWarning] CONTENT_WARNING_NSFW: _ClassVar[ContentWarning] +CONTENT_WARNING_UNSPECIFIED: ContentWarning CONTENT_WARNING_SAFE: ContentWarning CONTENT_WARNING_MIXED: ContentWarning CONTENT_WARNING_NSFW: ContentWarning @@ -47,7 +49,7 @@ class ExtensionList(_message.Message): def __init__(self, extensions: _Optional[_Iterable[_Union[Extension, _Mapping]]] = ...) -> None: ... class Extension(_message.Message): - __slots__ = ("name", "packageName", "resources", "extensionLib", "versionCode", "versionName", "contentWarning", "sources") + __slots__ = ("name", "packageName", "resources", "extensionLib", "versionCode", "versionName", "contentWarning", "sources", "isNovel") NAME_FIELD_NUMBER: _ClassVar[int] PACKAGENAME_FIELD_NUMBER: _ClassVar[int] RESOURCES_FIELD_NUMBER: _ClassVar[int] @@ -56,6 +58,7 @@ class Extension(_message.Message): VERSIONNAME_FIELD_NUMBER: _ClassVar[int] CONTENTWARNING_FIELD_NUMBER: _ClassVar[int] SOURCES_FIELD_NUMBER: _ClassVar[int] + ISNOVEL_FIELD_NUMBER: _ClassVar[int] name: str packageName: str resources: Resources @@ -64,15 +67,18 @@ class Extension(_message.Message): versionName: str contentWarning: ContentWarning sources: _containers.RepeatedCompositeFieldContainer[Source] - def __init__(self, name: _Optional[str] = ..., packageName: _Optional[str] = ..., resources: _Optional[_Union[Resources, _Mapping]] = ..., extensionLib: _Optional[str] = ..., versionCode: _Optional[int] = ..., versionName: _Optional[str] = ..., contentWarning: _Optional[_Union[ContentWarning, str]] = ..., sources: _Optional[_Iterable[_Union[Source, _Mapping]]] = ...) -> None: ... + isNovel: bool + def __init__(self, name: _Optional[str] = ..., packageName: _Optional[str] = ..., resources: _Optional[_Union[Resources, _Mapping]] = ..., extensionLib: _Optional[str] = ..., versionCode: _Optional[int] = ..., versionName: _Optional[str] = ..., contentWarning: _Optional[_Union[ContentWarning, str]] = ..., sources: _Optional[_Iterable[_Union[Source, _Mapping]]] = ..., isNovel: _Optional[bool] = ...) -> None: ... class Resources(_message.Message): - __slots__ = ("apkUrl", "iconUrl") + __slots__ = ("apkUrl", "iconUrl", "jarUrl") APKURL_FIELD_NUMBER: _ClassVar[int] ICONURL_FIELD_NUMBER: _ClassVar[int] + JARURL_FIELD_NUMBER: _ClassVar[int] apkUrl: str iconUrl: str - def __init__(self, apkUrl: _Optional[str] = ..., iconUrl: _Optional[str] = ...) -> None: ... + jarUrl: str + def __init__(self, apkUrl: _Optional[str] = ..., iconUrl: _Optional[str] = ..., jarUrl: _Optional[str] = ...) -> None: ... class Source(_message.Message): __slots__ = ("id", "name", "language", "homeUrl", "mirrorUrls", "message") diff --git a/.github/scripts/merge-repo.py b/.github/scripts/merge-repo.py deleted file mode 100644 index 7545e0944..000000000 --- a/.github/scripts/merge-repo.py +++ /dev/null @@ -1,106 +0,0 @@ -import gzip -import html -import sys -import json -from pathlib import Path -import re -import shutil - -from google.protobuf import json_format - -import index_pb2 - -REMOTE_REPO: Path = Path.cwd() -LOCAL_REPO: Path = REMOTE_REPO.parent.joinpath("main/repo") - -to_delete: list[str] = json.loads(sys.argv[1]) - -for module in to_delete: - apk_name = f"tsundoku-{module}-v*.*.*.apk" - icon_name = f"eu.kanade.tachiyomi.novelextension.{module}.png" - for file in REMOTE_REPO.joinpath("apk").glob(apk_name): - print(file.name) - file.unlink(missing_ok=True) - for file in REMOTE_REPO.joinpath("icon").glob(icon_name): - print(file.name) - file.unlink(missing_ok=True) - -shutil.copytree(src=LOCAL_REPO.joinpath("apk"), dst=REMOTE_REPO.joinpath("apk"), dirs_exist_ok = True) -shutil.copytree(src=LOCAL_REPO.joinpath("icon"), dst=REMOTE_REPO.joinpath("icon"), dirs_exist_ok = True) - -REMOTE_REPO.joinpath("index.json").unlink(missing_ok=True) - -with REMOTE_REPO.joinpath("index.min.json").open() as remote_index_file: - remote_index = json.load(remote_index_file) - -with LOCAL_REPO.joinpath("index.min.json").open() as local_index_file: - local_index = json.load(local_index_file) - -legacy_index = [ - item for item in remote_index - if not any([item["pkg"].endswith(f".{module}") for module in to_delete]) -] -legacy_index.extend(local_index) -legacy_index.sort(key=lambda x: x["pkg"]) - -def extract_extension_lib(version: str) -> str: - if match := re.search(r'(\d+)\.(\d+)', version): - return f"{match.group(1)}.{match.group(2)}" - - raise ValueError(f"Version {version} doesn't contain MAJOR.MINOR") - -index = index_pb2.Index( - name = "NovelSourcery", - badgeLabel = "NS", - signingKey = "4281820d4866bb71bed3dec5224aad9cf4633d44a113682cfb0c3b1cfd71702d", - contact=index_pb2.Contact( - website="https://novelsourcery.github.io", - discord="https://discord.gg/JG2K2jTjd6" - ), - extensionList=index_pb2.ExtensionList( - extensions=[ - index_pb2.Extension( - name=extension["name"].replace("Tachiyomi: ", ""), - packageName=extension["pkg"], - resources=index_pb2.Resources( - apkUrl=f"https://raw.githubusercontent.com/novelsourcery/extensions/refs/heads/repo/apk/{extension["apk"]}", - iconUrl=f"https://raw.githubusercontent.com/novelsourcery/extensions/refs/heads/repo/icon/{extension["pkg"]}.png", - ), - extensionLib=extract_extension_lib(extension["version"]), - versionCode=extension["code"], - versionName=extension["version"], - contentWarning=index_pb2.CONTENT_WARNING_NSFW if extension["nsfw"] == 1 else index_pb2.CONTENT_WARNING_SAFE, - sources=[ - index_pb2.Source( - id=int(source["id"]), - name=source["name"], - language=source["lang"], - homeUrl=source["baseUrl"], - ) - for source in extension["sources"] - ] - ) - for extension in legacy_index - ] - ) -) - -with REMOTE_REPO.joinpath("index.json").open("w", encoding="utf-8") as index_file: - index_file.write(json_format.MessageToJson(index, always_print_fields_with_no_presence=False, preserving_proto_field_name=True)) - -with REMOTE_REPO.joinpath("index.pb").open("wb") as index_pb_file: - index_pb_file.write(index.SerializeToString()) - -with REMOTE_REPO.joinpath("index.pb.gz").open("wb") as index_pb_file: - index_pb_file.write(gzip.compress(index.SerializeToString())) - -with REMOTE_REPO.joinpath("index.min.json").open("w", encoding="utf-8") as index_min_file: - json.dump(legacy_index, index_min_file, ensure_ascii=False, separators=(",", ":")) - -with REMOTE_REPO.joinpath("index.html").open("w", encoding="utf-8") as index_html_file: - index_html_file.write('\n\n\n\napks\n\n\n
\n')
-    for entry in legacy_index:
-        apk_escaped = 'apk/' + html.escape(entry["apk"])
-        name_escaped = html.escape(entry["name"])
-        index_html_file.write(f'{name_escaped}\n')
-    index_html_file.write('
\n\n\n') diff --git a/.github/scripts/move-built-apks.py b/.github/scripts/move-built-apks.py deleted file mode 100644 index edc938ad1..000000000 --- a/.github/scripts/move-built-apks.py +++ /dev/null @@ -1,12 +0,0 @@ -from pathlib import Path -import shutil - -REPO_APK_DIR = Path("repo/apk") - -shutil.rmtree(REPO_APK_DIR, ignore_errors=True) -REPO_APK_DIR.mkdir(parents=True, exist_ok=True) - -for apk in Path.home().joinpath("apk-artifacts").glob("**/*.apk"): - apk_name = apk.name.replace("-release.apk", ".apk") - - shutil.move(apk, REPO_APK_DIR.joinpath(apk_name)) diff --git a/.github/scripts/publish-repo.py b/.github/scripts/publish-repo.py new file mode 100644 index 000000000..cb84cb5c6 --- /dev/null +++ b/.github/scripts/publish-repo.py @@ -0,0 +1,204 @@ +import gzip +import html +import json +import os +import re +import subprocess +import sys +from functools import cache +from pathlib import Path +from zipfile import ZipFile + +from google.protobuf import json_format + +import index_pb2 + +APPLICATION_ICON_320_REGEX = re.compile( + r"^application-icon-320:'([^']+)'", re.MULTILINE +) +LANGUAGE_REGEX = re.compile(r"tsundoku-([^.]+)") + + +@cache +def aapt() -> Path: + *_, build_tools = (Path(os.environ["ANDROID_HOME"]) / "build-tools").iterdir() + return build_tools / "aapt" + + +# Artifacts downloaded from the build jobs: one APK per extension plus the source metadata JSON +# emitted by each assembleRelease. +ARTIFACTS_DIR = Path.home() / "apk-artifacts" + +# The checked-out `repo` branch we publish into (the working directory). +REPO_DIR = Path.cwd() +REPO_APK_DIR = REPO_DIR / "apk" +REPO_JAR_DIR = REPO_DIR / "jar" +REPO_ICON_DIR = REPO_DIR / "icon" +REPO_APK_DIR.mkdir(parents=True, exist_ok=True) +REPO_JAR_DIR.mkdir(parents=True, exist_ok=True) +REPO_ICON_DIR.mkdir(parents=True, exist_ok=True) + +APK_BASE_URL = "https://cdn.jsdelivr.net/gh/novelsourcery/extensions@repo/apk" +JAR_BASE_URL = "https://raw.githubusercontent.com/novelsourcery/extensions/repo/jar" +ICON_BASE_URL = "https://cdn.jsdelivr.net/gh/novelsourcery/extensions@repo/icon" + +to_delete: list[str] = json.loads(sys.argv[1]) + +# Drop apks/icons for modules that were deleted or rebuilt (rebuilt ones are re-added below). +for module in to_delete: + for file in REPO_APK_DIR.glob(f"tsundoku-{module}-v*.*.*.apk"): + print(f"removing {file.name}") + file.unlink(missing_ok=True) + for file in REPO_JAR_DIR.glob(f"tsundoku-{module}-v*.*.*.jar"): + print(f"removing {file.name}") + file.unlink(missing_ok=True) + for file in REPO_ICON_DIR.glob(f"eu.kanade.tachiyomi.novelextension.{module}.png"): + print(f"removing {file.name}") + file.unlink(missing_ok=True) + +# Build index entries for the freshly built apks. Each extension's metadata comes from the +# source-info JSON emitted by its assembleRelease task (see GenerateSourceInfoTask); its APK is a +# sibling in the same build dir. aapt reads the icon out of the APK +new_extensions: list[index_pb2.Extension] = [] + +for info_file in ARTIFACTS_DIR.glob("**/keiyoushi-source-info.json"): + with info_file.open(encoding="utf-8") as f: + info = json.load(f) + package_name = info["packageName"] + apk = next((info_file.parent / "outputs/apk/release").glob("*.apk"), None) + if apk is None: + raise FileNotFoundError( + f"{package_name}: no release apk found under {info_file.parent}" + ) + + apk_name = apk.name.replace("-release.apk", ".apk") + (REPO_APK_DIR / apk_name).write_bytes(apk.read_bytes()) + + jar = next((info_file.parent / "outputs/jar/release").glob("*.jar"), None) + if jar is None: + raise FileNotFoundError( + f"{package_name}: no release jar found under {info_file.parent}" + ) + (REPO_JAR_DIR / jar.name).write_bytes(jar.read_bytes()) + + badging = subprocess.check_output( + [aapt(), "dump", "--include-meta-data", "badging", apk] + ).decode() + application_icon = APPLICATION_ICON_320_REGEX.search(badging).group(1) + with ( + ZipFile(apk) as z, + z.open(application_icon) as i, + (REPO_ICON_DIR / f"{package_name}.png").open("wb") as f, + ): + f.write(i.read()) + + new_extensions.append( + index_pb2.Extension( + name=info["name"], + packageName=package_name, + resources=index_pb2.Resources( + apkUrl=f"{APK_BASE_URL}/{apk_name}", + jarUrl=f"{JAR_BASE_URL}/{jar.name}", + iconUrl=f"{ICON_BASE_URL}/{package_name}.png", + ), + extensionLib=info["extensionLib"], + versionCode=info["versionCode"], + versionName=info["versionName"], + contentWarning=info["contentWarning"], + isNovel=info.get("isNovel", True), + sources=[ + index_pb2.Source( + id=int(source["id"]), + name=source["name"], + language=source["lang"], + homeUrl=source["baseUrl"], + mirrorUrls=source.get("mirrorUrls", []), + ) + for source in info["sources"] + ], + ) + ) + +# Merge with the already-published index, dropping the deleted/rebuilt modules. +with REPO_DIR.joinpath("index.json").open() as f: + remote_proto = json_format.Parse(f.read(), index_pb2.Index()) + +all_extensions = [ + ext + for ext in remote_proto.extensionList.extensions + if not any(ext.packageName.endswith(f".{module}") for module in to_delete) +] +all_extensions.extend(new_extensions) +all_extensions.sort(key=lambda ext: ext.packageName) + +index = index_pb2.Index( + name="NovelSourcery", + badgeLabel="NS", + signingKey="4281820d4866bb71bed3dec5224aad9cf4633d44a113682cfb0c3b1cfd71702d", + contact=index_pb2.Contact( + website="https://novelsourcery.github.io", discord="https://discord.gg/JG2K2jTjd6" + ), + extensionList=index_pb2.ExtensionList(extensions=all_extensions), +) + +with REPO_DIR.joinpath("index.json").open("w", encoding="utf-8") as f: + f.write( + json_format.MessageToJson( + index, + always_print_fields_with_no_presence=False, + preserving_proto_field_name=True, + ) + ) + +with REPO_DIR.joinpath("index.pb").open("wb") as f: + f.write(gzip.compress(index.SerializeToString())) + + +def get_legacy_lang(ext) -> str: + apk_filename = ext.resources.apkUrl.split("/")[-1] + lang = LANGUAGE_REGEX.search(apk_filename).group(1) + if len(ext.sources) == 1: + source_language = ext.sources[0].language + if ( + source_language != lang + and source_language not in {"all", "other"} + and lang not in {"all", "other"} + ): + lang = source_language + return lang + + +legacy_json_index = [ + { + "name": f"Tachiyomi: {ext.name}", + "pkg": ext.packageName, + "apk": ext.resources.apkUrl.split("/")[-1], + "lang": get_legacy_lang(ext), + "code": ext.versionCode, + "version": ext.versionName, + "nsfw": 1 if ext.contentWarning > 2 else 0, + "sources": [ + { + "name": source.name, + "lang": source.language, + "id": str(source.id), + "baseUrl": source.homeUrl, + } + for source in ext.sources + ], + } + for ext in all_extensions +] + +with REPO_DIR.joinpath("index.min.json").open("w", encoding="utf-8") as f: + json.dump(legacy_json_index, f, ensure_ascii=False, separators=(",", ":")) + +with REPO_DIR.joinpath("index.html").open("w", encoding="utf-8") as f: + f.write( + '\n\n\n\napks\n\n\n
\n'
+    )
+    for ext in all_extensions:
+        apk_escaped = html.escape(ext.resources.apkUrl)
+        name_escaped = html.escape(f"Tachiyomi: {ext.name}")
+        f.write(f'{name_escaped}\n')
+    f.write("
\n\n\n") diff --git a/.github/workflows/build_pull_request.yml b/.github/workflows/build_pull_request.yml index 30568f319..a7cceb857 100644 --- a/.github/workflows/build_pull_request.yml +++ b/.github/workflows/build_pull_request.yml @@ -28,7 +28,7 @@ jobs: delete: ${{ steps.generate-matrices.outputs.delete }} steps: - name: Checkout PR - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: fetch-depth: 0 persist-credentials: false @@ -48,12 +48,12 @@ jobs: matrix: ${{ fromJSON(needs.prepare.outputs.matrix) }} steps: - name: Checkout PR - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Set up Java - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + uses: actions/setup-java@0f481fcb613427c0f801b606911222b5b6f3083a # v5.5.0 with: java-version: 17 distribution: temurin @@ -74,7 +74,9 @@ jobs: if: "github.repository == 'novelsourcery/extensions-source'" with: name: "individual-apks-${{ matrix.chunk.number }}" - path: "**/*.apk" + path: | + **/build/outputs/apk/release/*.apk + **/build/outputs/jar/release/*.jar retention-days: 1 inspect: @@ -90,7 +92,7 @@ jobs: path: ~/apk-artifacts - name: Set up JDK - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + uses: actions/setup-java@1bcf9fb12cf4aa7d266a90ae39939e61372fe520 # v5.4.0 with: java-version: 17 distribution: temurin diff --git a/.github/workflows/build_push.yml b/.github/workflows/build_push.yml index 849cd2c69..0825f5bad 100644 --- a/.github/workflows/build_push.yml +++ b/.github/workflows/build_push.yml @@ -32,7 +32,7 @@ jobs: delete: ${{ steps.generate-matrices.outputs.delete }} steps: - name: Checkout main branch - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: fetch-depth: 0 persist-credentials: false @@ -57,12 +57,12 @@ jobs: matrix: ${{ fromJSON(needs.prepare.outputs.matrix) }} steps: - name: Checkout main branch - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Set up Java - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + uses: actions/setup-java@0f481fcb613427c0f801b606911222b5b6f3083a # v5.5.0 with: java-version: 17 distribution: temurin @@ -90,7 +90,10 @@ jobs: if: "github.repository == 'novelsourcery/extensions-source'" with: name: "individual-apks-${{ matrix.chunk.number }}" - path: "**/*.apk" + path: | + **/build/outputs/apk/release/*.apk + **/build/outputs/jar/release/*.jar + **/build/keiyoushi-source-info.json retention-days: 1 - name: Clean up CI files @@ -99,39 +102,32 @@ jobs: publish: name: Publish extension repo needs: [prepare, build] - if: "github.repository == 'novelsourcery/extensions-source'" + # Run whenever there's anything to publish, new/changed builds OR only deletions + if: >- + always() && + github.repository == 'novelsourcery/extensions-source' && + needs.prepare.result == 'success' && + needs.build.result != 'failure' && + needs.build.result != 'cancelled' && + needs.prepare.outputs.delete != '[]' runs-on: ubuntu-latest timeout-minutes: 15 steps: - name: Download APK artifacts + if: needs.build.result == 'success' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: path: ~/apk-artifacts - - name: Set up JDK - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 - with: - java-version: 17 - distribution: temurin - - name: Checkout main branch - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: ref: main path: main persist-credentials: false - - name: Create repo artifacts - run: | - cd main - python ./.github/scripts/move-built-apks.py - INSPECTOR_LINK="$(curl -s "https://api.github.com/repos/NovelSourcery/extensions-inspector/releases/latest" | jq -r '.assets[0].browser_download_url')" - curl -L "$INSPECTOR_LINK" -o ./Inspector.jar - java -jar ./Inspector.jar "repo/apk" "output.json" "tmp" - python ./.github/scripts/create-repo.py - - name: Checkout repo branch - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: repository: novelsourcery/extensions token: ${{ secrets.BOT_PAT }} @@ -139,11 +135,23 @@ jobs: path: repo persist-credentials: true - - name: Merge and deploy repo + - name: Publish and deploy repo env: DELETE: ${{ needs.prepare.outputs.delete }} run: | cd repo pip install protobuf - python ../main/.github/scripts/merge-repo.py "$DELETE" - ../main/.github/scripts/commit-repo.sh + python ../main/.github/scripts/publish-repo.py "$DELETE" + git config --global user.email "" + git config --global user.name "github-actions" + git status + if [ -n "$(git status --porcelain)" ]; then + git add . + git commit -m "Update extensions repo" + git push + curl https://purge.jsdelivr.net/gh/novelsourcery/extensions@repo/index.min.json + curl https://purge.jsdelivr.net/gh/novelsourcery/extensions@repo/index.json + curl https://purge.jsdelivr.net/gh/novelsourcery/extensions@repo/index.pb + else + echo "No changes to commit" + fi diff --git a/.github/workflows/codeberg_mirror.yml b/.github/workflows/codeberg_mirror.yml index 8f63d1441..70aadd2be 100644 --- a/.github/workflows/codeberg_mirror.yml +++ b/.github/workflows/codeberg_mirror.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: fetch-depth: 0 persist-credentials: false diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index a1145401f..631d07144 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -19,9 +19,9 @@ jobs: timeout-minutes: 5 steps: - name: Checkout repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Run zizmor - uses: zizmorcore/zizmor-action@5f14fd08f7cf1cb1609c1e344975f152c7ee938d # v0.5.6 + uses: zizmorcore/zizmor-action@192e21d79ab29983730a13d1382995c2307fbcaa # v0.5.7 diff --git a/.gitignore b/.gitignore index e014f939c..300e08f0f 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,8 @@ gen generated-src/ .kotlin *.jks +__pycache__/ + +# kei-sync-vscode build/install artifacts (see scripts/kei-sync-vscode/README.md) +scripts/kei-sync-vscode/node_modules/ +scripts/kei-sync-vscode/*.vsix diff --git a/.kei-sync b/.kei-sync new file mode 100644 index 000000000..b964f41f7 --- /dev/null +++ b/.kei-sync @@ -0,0 +1 @@ +38fe0f76733322b53e7c1665f3e9abab140c81f6 diff --git a/.kei-sync-ignore b/.kei-sync-ignore new file mode 100644 index 000000000..71d0bbcc5 --- /dev/null +++ b/.kei-sync-ignore @@ -0,0 +1,49 @@ +# Fork-only content that never exists upstream -- excluded from sync-kei.sh's +# diff scope and its --status "local-only files" report. See: +# scripts/sync-kei.sh --help +# +# This is for PERMANENT exclusions only. For a one-off judgment call made +# while resolving a specific sync, use the kei-sync-vscode helper instead +# (scripts/kei-sync-vscode/), which records per-attempt decisions rather +# than a standing exclusion. + +# Shared fork-only libraries -- specific subdirs, not a blanket `lib` +# exclusion, so any NEW lib/ content kei adds in the future still shows up +# normally (we generally want kei's lib/ updates verbatim -- these are +# libraries authored/borrowed as a group, not modified per-file the way our +# own novel-specific bits are, so a name match here would mean we made it, +# not that kei's version should be skipped). +lib/chapterutils +lib/clipstudioreader +lib/siteparsers +lib/synchrony +lib/unpacker + +# Fork-only sync tooling -- never exists in kei's tree +scripts + +# Sync bookkeeping files -- obviously never exist upstream +.kei-sync +.kei-sync-ignore + +# Fork-only novel-specific core utilities and legacy DSL support +core/src/main/AndroidManifest.xml +core/src/main/kotlin/eu/kanade/tachiyomi/source/NovelSource.kt +core/src/main/kotlin/eu/kanade/tachiyomi/source/SourceTracker.kt +core/src/main/kotlin/keiyoushi/utils/ChapterName.kt +core/src/main/kotlin/keiyoushi/utils/Jsoup.kt +core/src/main/kotlin/keiyoushi/utils/SChapterExt.kt +core/src/main/kotlin/keiyoushi/utils/SMangaExt.kt +gradle/build-logic/src/main/kotlin/PluginExtensionLegacy.kt +gradle/ns.versions.toml + +# kei's own kei.versions.toml -- we never carry this. Our fork registers the +# "kei" version catalog name against gradle/ns.versions.toml instead (see +# settings.gradle.kts), so kei-verbatim code that references the "kei" +# catalog still resolves correctly without needing this file. +gradle/kei.versions.toml + +# Fork-only repo policy/config +common.gradle +.github/CODEOWNERS +.github/workflows/stale_issue_moderator.yml diff --git a/.kei-sync-rewrite b/.kei-sync-rewrite new file mode 100644 index 000000000..a93c56beb --- /dev/null +++ b/.kei-sync-rewrite @@ -0,0 +1,54 @@ +# Permanent local overrides -- tab-separated \t +# pairs, applied to both the last-synced and target blobs (never to the +# worktree) before diffing, so a routine upstream touch-up near one of these +# never generates a hunk in the first place. Use this for values we will +# NEVER take from upstream, where the resolution is always "keep ours" no +# matter what upstream's new value is (repo names, product branding, signing +# keys, CDN paths) -- not for genuine code changes we might sometimes want. +# See scripts/sync-kei.sh --help. +# +# Order matters: a longer/more-specific pattern must come before a shorter +# one it contains (e.g. the *-source repo name before the bare repo name), +# or the shorter rule will partially consume the longer one's match first. +# '#' starts a comment only at the beginning of a line; empty lines ignored. + +# Matching is case-insensitive (see apply_rewrite), so a single rule covers +# e.g. both "keiyoushi/..." and "Keiyoushi/..."; replacement text is always +# output exactly as written below regardless of which casing matched. + +# The real org is "NovelSourcery" (capitalized) -- that's the casing used +# in plain github.com/... links (issue templates, CONTRIBUTING.md). The +# lowercase "novelsourcery" below is deliberately separate: confirmed fine +# for raw.githubusercontent.com/jsdelivr/github.io/github.repository-yaml- +# comparison URL types specifically, not a casing mistake to "fix" into +# matching the org name. Each org-name rule below is scoped to the specific +# surrounding context it's known correct in (a domain prefix, or the +# `github.repository ==` comparison) rather than matching the bare +# "keiyoushi/..." substring everywhere -- an unscoped rule can only pick one +# output casing, but this string legitimately needs different casing in +# different contexts, so a generic rule is guaranteed to be wrong somewhere. +keiyoushi extension NovelSourcery extension +github\.com/keiyoushi/extensions-source github.com/NovelSourcery/extensions-source +github\.com/keiyoushi/extensions github.com/NovelSourcery/extensions +github\.repository == 'keiyoushi/extensions-source' github.repository == 'novelsourcery/extensions-source' +156378334\+keiyoushi-bot@users\.noreply\.github\.com +keiyoushi-bot github-actions +cdn\.jsdelivr\.net/gh/keiyoushi cdn.jsdelivr.net/gh/novelsourcery +raw\.githubusercontent\.com/keiyoushi raw.githubusercontent.com/novelsourcery +9add655a78e96c4ec7a53ef89dccb557cb5d767489fac5e785d671a5a75d4da2 4281820d4866bb71bed3dec5224aad9cf4633d44a113682cfb0c3b1cfd71702d +keiyoushi\.github\.io novelsourcery.github.io +discord\.gg/3FbCpdKbdY discord.gg/JG2K2jTjd6 +mihonapp/mihon tsundoku-otaku/tsundoku + +# Version placeholders: our version number and kei's have no 1:1 +# correspondence, so unlike the rules above this needs BOTH sides mapped to +# the identical placeholder text (not one side mapped to the other's +# current value), or they'd still diff against each other every time either +# number changes. +Mihon [0-9]+\.[0-9]+(\.[0-9]+)? PLACEHOLDER_APP_VERSION +Tsundoku [0-9]+\.[0-9]+(\.[0-9]+)? PLACEHOLDER_APP_VERSION + +# Same idea, for the ISSUE_TEMPLATE checklist item's bracket-link form +# (relies on the mihonapp/mihon rule above having already normalized the +# repo path on both sides). +\[[0-9]+\.[0-9]+(\.[0-9]+)?\]\(https://github\.com/tsundoku-otaku/tsundoku/releases/latest\) [PLACEHOLDER_APP_VERSION](PLACEHOLDER_APP_LINK) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a1d614423..b20a9d917 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,12 +1,12 @@ # Contributing -This guide has some instructions and tips on how to create a new NovelSourcery extension. Please **read -it carefully** if you're a new contributor or don't have any experience on the required languages -and knowledges. +This guide provides instructions and tips on creating a new NovelSourcery extension. Please **read +it carefully** if you are a new contributor or lack experience with the required languages +and knowledge. -This guide is not definitive and it's being updated over time. If you find any issues in it, feel -free to report it through a [Meta Issue](https://github.com/NovelSourcery/extensions-source/issues/new?assignees=&labels=Meta+request&template=06_request_meta.yml) -or fixing it directly by submitting a Pull Request. +This guide is not definitive and is updated over time. If you find any issues, feel +free to report them through a [Meta Issue](https://github.com/NovelSourcery/extensions-source/issues/new?assignees=&labels=Meta+request&template=06_request_meta.yml) +or fix them directly by submitting a Pull Request. ## Table of Contents @@ -18,10 +18,15 @@ or fixing it directly by submitting a Pull Request. - [Getting help](#getting-help) - [Writing an extension](#writing-an-extension) - [Setting up a new Gradle module](#setting-up-a-new-gradle-module) + - [Using ext-bootstrap.py](#using-ext-bootstrappy) - [Loading a subset of Gradle modules](#loading-a-subset-of-gradle-modules) - [Extension file structure](#extension-file-structure) - - [AndroidManifest.xml (optional)](#androidmanifestxml-optional) - - [build.gradle](#buildgradle) + - [build.gradle.kts](#buildgradlekts) + - [Source declaration](#source-declaration) + - [Annotate your source class](#annotate-your-source-class) + - [Declare sources in build.gradle.kts](#declare-sources-in-buildgradlekts) + - [baseUrl modes](#baseurl-modes) + - [Multiple sources from one class](#multiple-sources-from-one-class) - [Core dependencies](#core-dependencies) - [Extension API](#extension-api) - [lib tools](#lib-tools) @@ -35,8 +40,13 @@ or fixing it directly by submitting a Pull Request. - [Protobuf parsing and serialization - `parseAsProto` / `toRequestBodyProto`](#protobuf-parsing-and-serialization---parseasproto--torequestbodyproto) - [Date parsing - `tryParse`](#date-parsing---tryparse) - [Filter helpers - `firstInstance` / `firstInstanceOrNull`](#filter-helpers---firstinstance--firstinstanceornull) + - [SharedPreferences - `getPreferences` / `getPreferencesLazy`](#sharedpreferences---getpreferences--getpreferenceslazy) - [Next.js data extraction - `extractNextJs` / `extractNextJsRsc`](#nextjs-data-extraction---extractnextjs--extractnextjsrsc) - [Extracting URLs - `setUrlWithoutDomain` + `absUrl`](#extracting-urls---seturlwithoutdomain--absurl) + - [GraphQL Requests - `graphQLPost` / `parseGraphQLAs`](#graphql-requests---graphqlpost--parsegraphqlas) + - [GraphQL GET requests - `graphQLGet`](#graphql-get-requests---graphqlget) + - [JsonElement accessor helpers](#jsonelement-accessor-helpers) + - [ZIP streaming - `readZipDirectory` / `readZipEntry`](#zip-streaming---readzipdirectory--readzipentry) - [Additional dependencies](#additional-dependencies) - [Extension main class](#extension-main-class) - [Main class key variables](#main-class-key-variables) @@ -69,7 +79,7 @@ or fixing it directly by submitting a Pull Request. - [Logs](#logs) - [Inspecting network calls](#inspecting-network-calls) - [Using external network inspecting tools](#using-external-network-inspecting-tools) - - [Setup your proxy server](#setup-your-proxy-server) + - [Set up your proxy server](#set-up-your-proxy-server) - [OkHttp proxy setup](#okhttp-proxy-setup) - [Building](#building) - [Submitting the changes](#submitting-the-changes) @@ -106,94 +116,93 @@ navigate and build. This will also reduce disk usage and network traffic. 1. Do a partial clone. - ```bash - git clone --filter=blob:none --sparse - cd extensions-source/ - ``` + ```bash + git clone --filter=blob:none --sparse + cd extensions-source/ + ``` 2. Configure sparse checkout. - There are two modes of pattern matching. The default is cone mode. - Cone mode enables significantly faster pattern matching for big monorepos - and the sparse index feature to make Git commands more responsive. - In this mode, you can only filter by file path, which is less flexible - and might require more work when the project structure changes. - - You can skip this code block to use legacy mode if you want easier filters. - It won't be much slower as the repo doesn't have that many files. - - To enable cone mode together with sparse index, follow these steps: - - ```bash - git sparse-checkout set --cone --sparse-index - # add project folders - git sparse-checkout add common core gradle lib lib-multisrc utils - # add a single source - git sparse-checkout add src// - ``` - - To remove a source, open `.git/info/sparse-checkout` and delete the exact - lines you typed when adding it. Don't touch the other auto-generated lines - unless you fully understand how cone mode works, or you might break it. - - To use the legacy non-cone mode, follow these steps: - - ```bash - # enable sparse checkout - git sparse-checkout set --no-cone - # edit sparse checkout filter - vim .git/info/sparse-checkout - # alternatively, if you have VS Code installed - code .git/info/sparse-checkout - ``` - - Here's an example: - - ```bash - /* - !/src/* - !/multisrc-lib/* - # allow a single source - /src// - # allow a multisrc theme - /lib-multisrc/ - # or type the source name directly - - ``` - - Explanation: the rules are like `gitignore`. We first exclude all sources - while retaining project folders, then add the needed sources back manually. + There are two modes of pattern matching. The default is cone mode. + Cone mode enables significantly faster pattern matching for big monorepos + and the sparse index feature to make Git commands more responsive. + In this mode, you can only filter by file path, which is less flexible + and might require more work when the project structure changes. + + Cone mode is the recommended standard for pattern matching and responsiveness. Using non-cone mode is deprecated and discouraged. + + ```bash + git sparse-checkout set --cone --sparse-index + # add project folders + git sparse-checkout add common compiler core gradle lib lib-multisrc + # add a single source + git sparse-checkout add src// + ``` + + To remove a source, open `.git/info/sparse-checkout` and delete the exact + lines you typed when adding it. Don't touch the other auto-generated lines + unless you fully understand how cone mode works, or you might break it. + + ### Non-cone mode (Deprecated/Discouraged) + + Using non-cone mode is deprecated and not recommended. If you still need it, follow these steps: + + ```bash + # enable sparse checkout + git sparse-checkout set --no-cone + # edit sparse checkout filter + vim .git/info/sparse-checkout + # alternatively, if you have VS Code installed + code .git/info/sparse-checkout + ``` + + Here's an example: + + ```bash + /* + !/src/* + !/multisrc-lib/* + # allow a single source + /src// + # allow a multisrc theme + /lib-multisrc/ + # or type the source name directly + + ``` + + Explanation: the rules are like `gitignore`. We first exclude all sources + while retaining project folders, then add the needed sources back manually. 3. Configure remotes. - ```bash - # add upstream - git remote add upstream - # optionally disable push to upstream - git remote set-url --push upstream no_pushing - # optionally fetch main only (ignore all other branches) - git config remote.upstream.fetch "+refs/heads/main:refs/remotes/upstream/main" - # update remotes - git remote update - # track main of upstream instead of fork - git branch main -u upstream/main - ``` + ```bash + # add upstream + git remote add upstream + # optionally disable push to upstream + git remote set-url --push upstream no_pushing + # optionally fetch main only (ignore all other branches) + git config remote.upstream.fetch "+refs/heads/main:refs/remotes/upstream/main" + # update remotes + git remote update + # track main of upstream instead of fork + git branch main -u upstream/main + ``` 4. Useful configurations. (optional) - ```bash - # prune obsolete remote branches on fetch - git config remote.origin.prune true - # fast-forward only when pulling main branch - git config pull.ff only - # Add an alias to sync main branch without fetching useless blobs. - # If you run `git pull` to fast-forward in a blobless clone like this, - # all blobs (files) in the new commits are still fetched regardless of - # sparse rules, which makes the local repo accumulate unused files. - # Use `git sync-main` to avoid this. Be careful if you have changes - # on main branch, which is bad practice. - git config alias.sync-main '!git switch main && git fetch upstream && git reset --keep FETCH_HEAD' - ``` + ```bash + # prune obsolete remote branches on fetch + git config remote.origin.prune true + # fast-forward only when pulling main branch + git config pull.ff only + # Add an alias to sync main branch without fetching useless blobs. + # If you run `git pull` to fast-forward in a blobless clone like this, + # all blobs (files) in the new commits are still fetched regardless of + # sparse rules, which makes the local repo accumulate unused files. + # Use `git sync-main` to avoid this. Be careful if you have changes + # on main branch, which is bad practice. + git config alias.sync-main '!git switch main && git fetch upstream && git reset --keep FETCH_HEAD' + ``` 5. Later, if you change the sparse checkout filter, run `git sparse-checkout reapply`. @@ -203,19 +212,20 @@ Read more on [sparse checkout](https://github.blog/2020-01-17-bring-your-monorepo-down-to-size-with-sparse-checkout/), [sparse index](https://github.blog/2021-11-10-make-your-monorepo-feel-small-with-gits-sparse-index/), and [negative refspecs](https://github.blog/2020-10-19-git-2-29-released/#user-content-negative-refspecs). + ## Getting help -- Join [the Discord server](https://discord.gg/3FbCpdKbdY) for online help and to ask questions while -developing your extension. When doing so, please ask them in the `#programming` channel. -- There are some features and tricks that are not explored in this document. Refer to existing -extension code for examples. +- Join [the Discord server](https://discord.gg/JG2K2jTjd6) for online help and to ask questions during + development. Please ask questions in the `#programming` channel. +- There are features and tricks not explored in this document; refer to existing + extension code for examples. ## Writing an extension -The quickest way to get started is to copy an existing extension's folder structure and renaming it -as needed. We also recommend reading through a few existing extensions' code before you start. +The quickest way to get started is by using the [ext-bootstrap.py](#using-ext-bootstrappy) script. +We also recommend reading through the code of a few existing extensions before beginning. ### Setting up a new Gradle module @@ -226,6 +236,36 @@ The `` used in the folder inside `src` should be the major `language` part you will be creating a `pt-BR` source, use `` here as `pt` only. Inside the source class, use the full locale string instead. +#### Using ext-bootstrap.py + +Instead of setting this up by hand, you can use the `ext-bootstrap.py` script to scaffold a new +extension module automatically: + +```console +$ python ext-bootstrap.py -n "My Source" -l en -u https://mysource.com +``` + +This creates `src///build.gradle.kts` along with the extension's package +directory and a starter source class implementing `HttpSource`. + +Available options: + +| Flag | Description | +|------------------------------|--------------------------------------------------------------------------| +| `-n`, `--extname` | Extension name | +| `-l`, `--lang`, `--language` | Extension language (2- or 3-letter ISO code, or `all`) | +| `-u`, `--baseurl` | Extension base URL (must be `https://`) | +| `--source-name` | Source name (defaults to `--extname`) | +| `-c`, `--content-warning` | `SAFE`, `MIXED`, or `NSFW` (default: `SAFE`) | +| `-m`, `--multisrc` | Name of an existing multisrc theme to base the source on | +| `--path` | Path to the extension repo directory (defaults to the current directory) | + +For example, to scaffold a source based on the `madara` multisrc theme: + +```console +$ python ext-bootstrap.py -n "My Source" -l en -u https://mysource.com -m madara +``` + ### Loading a subset of Gradle modules By default, all individual and multisrc extensions are loaded for local development. @@ -240,8 +280,7 @@ The simplest extension structure looks like this: ```console $ tree src/// src/// -├── AndroidManifest.xml (optional) -├── build.gradle +├── build.gradle.kts ├── res │   ├── mipmap-hdpi │   │   └── ic_launcher.png @@ -262,8 +301,7 @@ src/// └── ├── .kt ├── .kt (optional) - ├── .kt (optional) - └── .kt (optional) + └── .kt (optional) ``` @@ -272,40 +310,180 @@ should be adapted from the site name, and can only contain lowercase ASCII lette Your extension code must be placed in the package `eu.kanade.tachiyomi.novelextension..`. > [!TIP] -> Additional files in the extension package (like `Dto.kt`, `Filters.kt`, `UrlActivity.kt`) +> Additional files in the extension package (like `Dto.kt`, `Filters.kt`) > should NOT repeat the extension name (e.g. use `Dto.kt` instead of `MySourceNameDto.kt`). > Note: While older extensions might use the repeated name pattern, avoiding it is a newly enforced convention to maintain consistency across the repository. -#### AndroidManifest.xml (optional) - -You only need to create this file if you want to add deep linking to your extension. -See [URL intent filter](#url-intent-filter) for more information. +#### build.gradle.kts -#### build.gradle +Extensions' `build.gradle.kts` should look like this: -Make sure that your new extension's `build.gradle` file follows the following structure: +```kotlin +import io.github.keiyoushi.gradle.api.ContentWarning -```groovy -ext { - extName = '' - extClass = '.' - extVersionCode = 1 - isNsfw = true +plugins { + alias(ns.plugins.extension) } -apply plugin: "kei.plugins.extension.legacy" +keiyoushi { + name = "Example" // Replace with your actual source name + versionCode = 1 + contentWarning = ContentWarning.NSFW // Options: ContentWarning.SAFE, ContentWarning.MIXED, ContentWarning.NSFW + libVersion = "1.4" + + source { + name = "Example" // Optional, defaults to the top-level name + lang = "en" + baseUrl = "https://example.com" + } +} ``` -| Field | Description | -|------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `extName` | The name of the extension. Should be romanized if site name is not in English. | -| `extClass` | Points to the class that implements `Source`. You can use a relative path starting with a dot (the package name is the base path). This is used to find and instantiate the source(s). | -| `extVersionCode` | The extension version code. This must be a positive integer and incremented with any change to the code. Do not bump for changes that do not affect users, such as changing a private function to a public function. | -| `isNsfw` | Flag to indicate that a source contains NSFW content. Should always be set explicitly to either `true` or `false`. Falls back to `false` if not set. | +At least one `source {}` block is required for every extension. + +| Field | Description | +|------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `name` | The name of the extension. Should be romanized if site name is not in English. | +| `versionCode` | The extension version code. This must be a positive integer and incremented with any change to the code. Do not bump for changes that do not affect users, such as changing a private function to a public function. | +| `contentWarning` | Content safety classification. Must be set explicitly to one of `ContentWarning.SAFE`, `ContentWarning.MIXED`, or `ContentWarning.NSFW`. | +| `libVersion` | The extension library version. Always set to `"1.4"`. | +| `theme` | Name of a multi-source theme from `lib-multisrc/` to inherit from (e.g. `"madara"`). When set, the extension's version code is `theme.baseVersionCode + versionCode`. | +| `source {}` | Declares one source (or multiple, for multi-language or multi-mirror extensions) using KSP code generation. This block is mandatory. See [Source declaration](#source-declaration). | +| `deeplink {}` | Declares a URL deeplink intent filter. See [URL intent filter](#url-intent-filter). | -The extension's version name is generated automatically by concatenating `1.4` and `extVersionCode`. +The extension's version name is generated automatically by concatenating `libVersion` and the calculated version code. With the example used above, the version would be `1.4.1`. +### Source declaration + +Sources are registered through `source {}` blocks in `build.gradle.kts`, combined with the `@Source` annotation on your source class. The build system uses KSP to generate a subclass (`ExtensionGenerated`) that automatically injects `name`, `lang`, `id`, and `baseUrl`- you no longer need to declare them manually in Kotlin. + +#### Annotate your source class + +Add `@Source` to your main class and remove any manual declarations of `name`, `lang`, `id`, and `baseUrl`: + +```kotlin +import keiyoushi.annotation.Source + +@Source +abstract class MySource : HttpSource() { + // name, lang, id, and baseUrl are injected automatically - do not declare them here. + // All other overrides go here as normal. +} +``` + +#### Declare sources in build.gradle.kts + +Add one or more `source {}` blocks inside `keiyoushi {}`: + +```kotlin +keiyoushi { + name = "My Source" + versionCode = 1 + contentWarning = ContentWarning.SAFE + libVersion = "1.4" + + source { + lang = "en" + baseUrl = "https://example.com" + } +} +``` + +| Field | Description | +|---------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `name` | The source name shown in the app. Optional; defaults to the top-level extension `name`. | +| `lang` | ISO 639-1 language code. Required. | +| `baseUrl` | The source's base URL. See [baseUrl modes](#baseurl-modes) below. | +| `id` | Explicit source ID. Optional; auto-computed from `name + lang + versionId` if omitted. Set this explicitly when renaming a source to preserve users' libraries. | +| `versionId` | Integer used as a seed for auto-computing `id`. Defaults to `1`. Only bump this if the source's URL structure fundamentally changes and old entries can no longer be redirected. | + +A source class may compute `name` and/or `baseUrl` itself by declaring `override val name` / `override val baseUrl`; codegen detects the override and skips generating that property (the DSL value is then used only for metadata such as the repo index and deeplink hosts). **This is discouraged** — prefer letting the DSL own `name` and `baseUrl`, and only override them in the source class when you have a very specific reason. `baseUrl` may only be overridden when the DSL declares a plain static `baseUrl` — the `mirrors`/`custom` modes generate preference infrastructure and cannot be hand-overridden. `id` and `lang` are always owned by the DSL; overriding them in the source class is an error. + +#### baseUrl modes + +The `baseUrl` field inside `source {}` supports three modes: + +**Static** (single URL, no preferences UI): + +```kotlin +source { + lang = "en" + baseUrl = "https://example.com" +} +``` + +**Mirrors** (user picks a mirror from a list - preference UI is generated automatically): + +```kotlin +source { + lang = "en" + baseUrl { + mirrors( + "https://example.com", + "https://mirror1.com", + "https://mirror2.com", + ) + } +} +``` + +The first url is the default; each mirror's host is shown in the picker. To show custom labels instead, pass `"label" to "url"` pairs — either all urls are naked or all are labeled, not a mix: + +```kotlin +baseUrl { + mirrors( + "Main" to "https://example.com", + "Mirror" to "https://mirror1.com", + ) +} +``` + +The extension automatically implements `ConfigurableSource` and adds a "Preferred mirror" `ListPreference` to the settings screen. You do not need to write any `SharedPreferences` code or add `setupPreferenceScreen`. If your class already implements `ConfigurableSource`, `super.setupPreferenceScreen(screen)` is called so your own preferences are preserved. + +**Custom URL** (user can enter any URL - preference UI with validation is generated automatically): + +```kotlin +source { + lang = "en" + baseUrl { + custom("https://example.com") + } +} +``` + +Like `mirrors`, the extension automatically implements `ConfigurableSource` and adds a validated "Custom base URL" `EditTextPreference`. The default URL is restored automatically if the hardcoded default changes in a future update. + +> [!IMPORTANT] +> When using `mirrors(...)` or `custom(...)`, **do not** implement mirror/URL selection manually in your class using `SharedPreferences` or a `ListPreference` - the generated code handles it. Doing both will create duplicate preferences. + +#### Multiple sources from one class + +To expose multiple sources (previously done with `SourceFactory`), add multiple `source {}` blocks. Each block generates an anonymous inner class that subclasses your `@Source` class: + +```kotlin +keiyoushi { + name = "Example" + versionCode = 1 + contentWarning = ContentWarning.SAFE + libVersion = "1.4" + + source { + name = "Example EN" + lang = "en" + baseUrl = "https://en.example.com" + } + + source { + name = "Example JP" + lang = "ja" + baseUrl = "https://jp.example.com" + } +} +``` + +The generated `ExtensionGenerated` class implements `SourceFactory` automatically. You do not need to implement `SourceFactory` yourself. + ### Core dependencies #### Extension API @@ -324,16 +502,23 @@ use case. Each lib is self-documented via KDoc comments and/or a README in its o #### Available libs -| Module | Description | -|---|---| -| [`lib-cookieinterceptor`](https://github.com/NovelSourcery/extensions-source/tree/main/lib/cookieinterceptor) | Injects cookies into OkHttp requests for a given domain | -| [`lib-cryptoaes`](https://github.com/NovelSourcery/extensions-source/tree/main/lib/cryptoaes) | AES-CBC decryption compatible with CryptoJS; JSFuck deobfuscation | -| [`lib-dataimage`](https://github.com/NovelSourcery/extensions-source/tree/main/lib/dataimage) | Decodes base64 `data:image` strings into mock URLs that OkHttp can handle | -| [`lib-randomua`](https://github.com/NovelSourcery/extensions-source/tree/main/lib/randomua) | Fetches and rotates real-world User-Agent strings (requires overriding `getMangaUrl()`) | -| [`lib-synchrony`](https://github.com/NovelSourcery/extensions-source/tree/main/lib/synchrony) | JavaScript deobfuscation via the Synchrony engine (QuickJS sandbox) | -| [`lib-textinterceptor`](https://github.com/NovelSourcery/extensions-source/tree/main/lib/textinterceptor) | Renders plain text or HTML as a PNG image page | -| [`lib-unpacker`](https://github.com/NovelSourcery/extensions-source/tree/main/lib/unpacker) | Unpacks Dean Edwards-packed JavaScript; substring extraction helpers | -| [`lib-zipinterceptor`](https://github.com/NovelSourcery/extensions-source/tree/main/lib/zipinterceptor) | Decodes, stitches, and processes multi-page ZIP/AVIF/SVG image archives | +| Module | Description | +|-----------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------| +| [`lib-cookieinterceptor`](https://github.com/NovelSourcery/extensions-source/tree/main/lib/cookieinterceptor) | Injects cookies into OkHttp requests for a given domain | +| [`lib-cryptoaes`](https://github.com/NovelSourcery/extensions-source/tree/main/lib/cryptoaes) | AES-CBC decryption compatible with CryptoJS; JSFuck deobfuscation | +| [`lib-dataimage`](https://github.com/NovelSourcery/extensions-source/tree/main/lib/dataimage) | Decodes base64 `data:image` strings into mock URLs that OkHttp can handle | +| [`lib-e4p`](https://github.com/NovelSourcery/extensions-source/tree/main/lib/e4p) | Decodes and decrypts E4P-format manga page archives (TIFF/XEBP) | +| [`lib-i18n`](https://github.com/NovelSourcery/extensions-source/tree/main/lib/i18n) | Internationalization helper (`Intl`) for multi-language UI strings in extensions | +| [`lib-lzstring`](https://github.com/NovelSourcery/extensions-source/tree/main/lib/lzstring) | LZ-String decompression and compression | +| [`lib-publus`](https://github.com/NovelSourcery/extensions-source/tree/main/lib/publus) | Handles Publus DRM-protected reader decryption, unscrambling, and page loading | +| [`lib-randomua`](https://github.com/NovelSourcery/extensions-source/tree/main/lib/randomua) | Fetches and rotates real-world User-Agent strings (requires overriding `getMangaUrl()`) | +| [`lib-secretstream`](https://github.com/NovelSourcery/extensions-source/tree/main/lib/secretstream) | ChaCha20/Poly1305/X25519 cryptography for secret-stream encrypted sources | +| [`lib-seedrandom`](https://github.com/NovelSourcery/extensions-source/tree/main/lib/seedrandom) | Seeded deterministic pseudo-random number generation (ARC4-based) | +| [`lib-speedbinb`](https://github.com/NovelSourcery/extensions-source/tree/main/lib/speedbinb) | Processes, decrypts, and descrambles SpeedBinb reader payloads | +| [`lib-synchrony`](https://github.com/NovelSourcery/extensions-source/tree/main/lib/synchrony) | JavaScript deobfuscation via the Synchrony engine (QuickJS sandbox) | +| [`lib-textinterceptor`](https://github.com/NovelSourcery/extensions-source/tree/main/lib/textinterceptor) | Renders plain text or HTML as a PNG image page | +| [`lib-unpacker`](https://github.com/NovelSourcery/extensions-source/tree/main/lib/unpacker) | Unpacks Dean Edwards-packed JavaScript; substring extraction helpers | +| [`lib-zipinterceptor`](https://github.com/NovelSourcery/extensions-source/tree/main/lib/zipinterceptor) | Decodes, stitches, and processes multi-page ZIP/AVIF/SVG image archives | > [!IMPORTANT] > If your module uses `:lib:randomua`, the Spotless check requires your extension to override the `getMangaUrl()` method in your main class, or the build will fail. @@ -343,19 +528,22 @@ use case. Each lib is self-documented via KDoc comments and/or a README in its o #### Adding a lib dependency -Declare the module in your extension's `build.gradle`: +Declare the module in your extension's `build.gradle.kts`: -```groovy +```kotlin dependencies { - implementation(project(':lib:')) + implementation(project(":lib:")) } ``` +> [!TIP] +> For multi-source themes in `lib-multisrc/`, use `api()` instead of `implementation()` so the dependency is transitively available to all extensions using the theme. + For example: -```groovy +```kotlin dependencies { - implementation(project(':lib:dataimage')) + implementation(project(":lib:dataimage")) } ``` @@ -395,7 +583,7 @@ plugins { } dependencies { - implementation(project(":lib:")) + implementation(project(":lib:other-lib")) // Replace with the actual other library name } ``` @@ -498,7 +686,7 @@ val dto = base64String.decodeProtoBase64() // Creating a RequestBody for a POST request (defaults to application/protobuf): val requestBody = myRequestDto.toRequestBodyProto() -```` +``` If you only need to work with raw bytes, you can also use `.decodeProto()` and `.encodeProto()` directly on a `ByteArray`. @@ -520,14 +708,14 @@ private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale chapter.date_upload = dateFormat.tryParse(dateStr) ``` -**Do not** write manual try/catch blocks or null-guards around `SimpleDateFormat.parse()` - +**Do not** write manual try/catch blocks or null-guards around `SimpleDateFormat.parse()`; `tryParse` handles both. Also, always declare your `SimpleDateFormat` as a class-level or file-level `val` so it is not reconstructed for every chapter. Two common mistakes to avoid: -- **Always set `Locale.ROOT`**, unless the pattern contains locale-sensitive text (such as month names) - in which case use the appropriate locale. -- **Set the timezone** if known. Either if the site's region is known, or because the pattern uses a literal `'Z'`. +- **Always set `Locale.ROOT`**, unless the pattern contains locale-sensitive text (such as month names), in which case use the appropriate locale. +- **Set the timezone** if known, either if the site's region is known or because the pattern uses a literal `'Z'`. ```kotlin // Wrong: 'Z' is treated as a literal character, timezone defaults to device local time @@ -551,7 +739,7 @@ import tachiyomi.utils.firstInstanceOrNull val genreFilter = filters.firstInstanceOrNull() ``` -**SharedPreferences - `getPreferences` / `getPreferencesLazy`** +##### SharedPreferences - `getPreferences` / `getPreferencesLazy` Use these instead of accessing `Injekt` manually. @@ -559,13 +747,13 @@ Use these instead of accessing `Injekt` manually. import tachiyomi.utils.getPreferences import tachiyomi.utils.getPreferencesLazy -// Eager: -private val preferences = getPreferences() - -// Lazy (recommended for most cases): +// Inside your HttpSource class: private val preferences by getPreferencesLazy() ``` +> [!NOTE] +> `getPreferences()` and `getPreferencesLazy()` are extension functions on `HttpSource`. If you need to access preferences from a context without a source receiver (e.g. inside a helper class), use the top-level `getPreferences(sourceId)` function instead. + ##### Next.js data extraction - `extractNextJs` / `extractNextJsRsc` If the site is built with Next.js, use `tachiyomi.utils.extractNextJs` on a `Document` or `Response`, @@ -616,7 +804,13 @@ val request = graphQLPost( url = "$baseUrl/graphql", headers = headers, operationName = "SearchManga", - query = $$"""query SearchManga($page: Int!) { ... }""", + query = $$""" + query SearchManga($page: Int!) { + mangas(page: $page) { + id + } + } + """, variables = variables ) @@ -624,6 +818,102 @@ val request = graphQLPost( val data = response.parseGraphQLAs() ``` +##### GraphQL GET requests - `graphQLGet` + +For sources that send GraphQL over HTTP GET instead of POST, use `graphQLGet` with the same signature as `graphQLPost`: + +```kotlin +import keiyoushi.utils.graphQLGet + +val request = graphQLGet( + url = "$baseUrl/graphql", + headers = headers, + query = $$""" + query SearchManga($page: Int!) { + mangas(page: $page) { + id + } + } + """, + variables = variables +) +``` + +For sources that use [Automatic Persisted Queries (APQ)](https://www.apollographql.com/docs/kotlin/advanced/persisted-queries/), pass the result of `persistedQueryExtension(sha256Hash)` as the `extensions` parameter and omit `query`. This works for both `graphQLPost` and `graphQLGet`. + +```kotlin +import keiyoushi.utils.persistedQueryExtension + +val request = graphQLPost( + url = "$baseUrl/graphql", + headers = headers, + operationName = "SearchManga", + variables = variables, + extensions = persistedQueryExtension("abc123sha256...") +) +``` + +To automatically throw `GraphQLException` for every request on a client rather than parsing per-response, add `GraphQLErrorInterceptor` to the `OkHttpClient`: + +```kotlin +import keiyoushi.utils.GraphQLErrorInterceptor + +override val client = network.client.newBuilder() + .addInterceptor(GraphQLErrorInterceptor()) + .build() +``` + +##### JsonElement accessor helpers + +`keiyoushi.utils` provides concise read-only accessors for traversing raw `JsonElement` trees. Import individually as needed: + +```kotlin +import keiyoushi.utils.array +import keiyoushi.utils.boolean +import keiyoushi.utils.get +import keiyoushi.utils.int +import keiyoushi.utils.long +import keiyoushi.utils.obj +import keiyoushi.utils.string + +val root: JsonElement = response.parseAs() +val title = root["data"]["title"].string +val count = root["data"]["count"].int +val items = root["data"]["items"].array +val nested = root["data"]["meta"].obj +``` + +`element[key]` returns `JsonElement?` (null-safe). The terminal accessors (`.string`, `.int`, `.long`, `.boolean`) throw if the element is null. `JsonObject` also has `getStringOrNull`, `getIntOrNull`, `getLongOrNull`, and `getBooleanOrNull` variants for optional fields. + +Prefer these over writing `element.jsonObject["key"]?.jsonPrimitive?.content` manually. + +##### ZIP streaming - `readZipDirectory` / `readZipEntry` + +For sources that serve manga pages as remote ZIP archives, the `keiyoushi.zip` package lets you read the central directory and individual entries using HTTP Range requests - no need to download the entire file. Import from `keiyoushi.zip`: + +```kotlin +import keiyoushi.zip.readZipDirectory +import keiyoushi.zip.readZipEntry +import keiyoushi.zip.range + +// 1. Fetch the ZIP central directory (two Range requests at most). +val directory = readZipDirectory(totalFileSizeInBytes) { byteRange -> + client.newCall( + GET(zipUrl, headers).newBuilder().range(byteRange).build() + ).execute().body.source().buffer() +} + +// 2. Find an entry by name and read its decompressed bytes. +val entry = directory.entries.first { it.name == "001.jpg" } +val imageBytes = readZipEntry(entry) { byteRange -> + client.newCall( + GET(zipUrl, headers).newBuilder().range(byteRange).build() + ).execute().body.source().buffer() +}.buffer().readByteArray() +``` + +`readZipDirectory` resolves every entry's offset to an absolute file position and handles ZIP64 archives. `readZipEntry` fetches only the bytes for that one entry. Use this instead of downloading the full ZIP into a `ZipInputStream`, which forces the entire archive into memory. + #### Additional dependencies If you find yourself needing additional functionality, you can add more dependencies to your `build.gradle` @@ -638,23 +928,31 @@ Notice that we're using `compileOnly` instead of `implementation` if the app alr You could use `implementation` instead for a new dependency, or you prefer not to rely on whatever the main app has at the expense of app size. +> [!TIP] +> Use `compileOnlyApi` (not `compileOnly`) when a dependency is provided by the app but also needs to be visible to consumers of your module (e.g. when building a theme or a library). + > [!IMPORTANT] > Using `compileOnly` restricts you to versions that must be compatible with those used in > [the latest stable version of the app](https://github.com/tsundoku-otaku/tsundoku/releases/latest). ### Extension main class -The class which is referenced and defined by `extClass` in `build.gradle`. This class should implement -either `SourceFactory` or `HttpSource`. +The class annotated with `@Source` and referenced by your `source {}` block(s). This class should implement `HttpSource`. -| Class | Description | -|--------------------|----------------------------------------------------------------------------------------------------------------------------------| -| `SourceFactory` | Used to expose multiple `Source`s. Use this in case of a source that supports multiple languages or mirrors of the same website. | -| `HttpSource` | For online source, where requests are made using HTTP. | -| `ParsedHttpSource` | Deprecated, use `HttpSource` instead. | +> [!NOTE] +> `className` is set to `ExtensionGenerated` automatically by the build system. + +| Class | Description | +|--------------------|---------------------------------------------------------------------------------------------------------------------------------------| +| `SourceFactory` | **Obsolete.** Used to expose multiple `Source`s manually. With `source {}` blocks, this is generated automatically. | +| `HttpSource` | For online source, where requests are made using HTTP. Use this directly or extend a theme base class. | +| `ParsedHttpSource` | Deprecated, use `HttpSource` instead. | #### Main class key variables +> [!IMPORTANT] +> Since `source {}` blocks are required, these fields are generated and injected automatically. You can access them within your class (as they are part of the `HttpSource` contract), but **you must not declare or override them manually**. + | Field | Description | |-----------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------| | `name` | Name displayed in the "Sources" tab in the app. | @@ -664,30 +962,30 @@ either `SourceFactory` or `HttpSource`. ### HTML and Image Processing -- **Parsing partial HTML:** If an API returns a JSON response containing an HTML string, use `Jsoup.parseBodyFragment(html, baseUrl)` instead of `Jsoup.parse(html)`. Passing the `baseUrl` ensures that `abs:href` and `absUrl()` can correctly resolve relative links. +- **Parsing partial HTML:** If an API returns a JSON response containing an HTML string, use `Jsoup.parseBodyFragment(html, baseUrl)` instead of `Jsoup.parse(html)`. Passing the `baseUrl` ensures that `abs:href` and `absUrl()` correctly resolve relative links. -- **Formatting Chapter Numbers:** Do not write custom `DecimalFormat` logic just to remove trailing zeros from float chapter numbers. Simply use `.toString().removeSuffix(".0")`. +- **Formatting Chapter Numbers:** Do not write custom `DecimalFormat` logic solely to remove trailing zeros from float chapter numbers. Instead, use `.toString().removeSuffix(".0")`. - **Generating Page lists:** The app ignores the `index` passed to the `Page` object, but you must ensure the list itself is sorted correctly according to the source. You can use Kotlin's `mapIndexed` to easily instantiate `Page` objects, or rely on the index provided by the source API if available: - ```kotlin - return document.select(".pages img").mapIndexed { index, img -> - Page(index, imageUrl = img.attr("abs:src")) - } - ``` + ```kotlin + return document.select(".pages img").mapIndexed { index, img -> + Page(index, imageUrl = img.attr("abs:src")) + } + ``` -- **Memory-efficient Image Interceptors:** When implementing interceptors for descrambling, stitching, or decrypting images, avoid loading the entire image into a `ByteArray`, as this can cause `OutOfMemoryError` on low-end devices. Prefer stream-based processing instead: +- **Memory-efficient Image Interceptors:** When implementing interceptors for descrambling, stitching, or decrypting images, avoid loading the entire image into a `ByteArray`, as this can cause `OutOfMemoryError` on low-end devices. Prefer stream-based processing: - **Read:** Use `response.body.byteStream()` with `BitmapFactory.decodeStream()` to decode images directly from the stream. - **Write:** Write the processed bitmap into an Okio `Buffer` via `output.outputStream()` and convert it using `asResponseBody(mediaType)`. - **Decryption:** Use Okio's `cipherSource` extension for stream-based decryption rather than decrypting a full byte array in memory. - Note: `readByteArray()` should generally be avoided here because it forces full in-memory buffering of the image. Streaming directly keeps memory usage lower and more stable. - - Always wrap network responses in `response.use { ... }` to ensure the response body is properly closed and to prevent memory leaks. - - If applicable, call `bitmap.recycle()` after you're done with it to free native memory early. + - Always wrap network responses in `response.use { ... }` to ensure the response body is properly closed and memory leaks are prevented. + - If applicable, call `bitmap.recycle()` after use to free native memory early. - **Do not manually check for Cloudflare:** Do not manually check for Cloudflare challenges (e.g., checking for "Just a moment..." text) in `parse` methods. The app handles this before calling the parser. - **Prefer stable selectors:** Avoid relying on volatile auto-generated CSS class names (e.g., `styles_Card__jN8og`) or complex regex for parsing. Prefer stable structural selectors. -- **Use `ownText()` to avoid mutation:** To get text from an element without including text from its children, use `.ownText()`. This avoids having to select and remove child elements (`.select().remove()`) or mutate the document. +- **Use `ownText()` to avoid mutation:** To get text from an element without including text from its children, use `.ownText()`. This avoids the need to select and remove child elements (`.select().remove()`) or mutate the document. - **Parse status using `.lowercase()`:** When comparing strings for status parsing (e.g., `contains("ongoing")`), prefer calling `.lowercase()` on the source string once instead of using `ignoreCase = true` on multiple `contains` checks. ### OkHttp and Network @@ -702,13 +1000,14 @@ either `SourceFactory` or `HttpSource`. // Prefer: return GET("$baseUrl/manga", headers) ``` -- **GraphQL Queries:** If you are sending GraphQL requests, use Kotlin's raw multi-dollar string interpolation (`$$"""..."""`) for your queries. This prevents having to escape every JSON variable `$` symbol manually. For building the request and parsing the response, prefer the `graphQLPost` and `parseGraphQLAs` helpers in `tachiyomi.utils`. + +- **GraphQL Queries:** If you are sending GraphQL requests, use Kotlin's raw multi-dollar string interpolation (`$$"""..."""`) for your queries. This prevents having to escape every JSON variable `$` symbol manually. For building the request and parsing the response, prefer the `graphQLPost` and `parseGraphQLAs` helpers in `keiyoushi.utils`. - **Empty checks on `.text()`:** Because Jsoup's `.text()` automatically trims whitespace, you can use `.isNotEmpty()` instead of `.isNotBlank()` when checking for empty strings. The same applies to `.ownText()`. This also means you should not use `.trim()` with these functions. - **Use `network.client` for Cloudflare:** When overriding the client for sources, simply use `override val client = network.client.newBuilder()...`. - **Never use `Thread.sleep()`:** Do not use `Thread.sleep()` for rate limiting. Use the `tachiyomi.network.rateLimit` builder extension function on your `OkHttpClient.Builder` instead. - **Avoid synchronous calls in `parse` methods:** Do not call `client.newCall(...).execute()` inside parsing methods like `pageListParse` or `chapterListParse`. Make the request part of the standard flow by overriding the corresponding request method (e.g., `pageListRequest`) or `fetchImageUrl`. - **Pass `HttpUrl` directly:** The `GET()` and `POST()` helpers accept an `HttpUrl` object. Do not call `.toString()` on a built `HttpUrl` before passing it. -- **Use `HttpUrl` for URL manipulation:** When parsing or extracting parts of a URL, prefer using `HttpUrl` methods (like `pathSegments()` or `queryParameter()`) over manual string splitting (e.g., `.split("/")`) or regex. This ensures proper separation of concerns and protects against unexpected inputs-such as URL fragments or query parameters-without you needing to manually account for all edge cases. +- **Use `HttpUrl` for URL manipulation:** When parsing or extracting parts of a URL, prefer using `HttpUrl` methods (like `pathSegments` property, `encodedPathSegments`, or `queryParameter("id")`) over manual string splitting (e.g., `.split("/")`) or regex. This ensures proper separation of concerns and protects against unexpected inputs-such as URL fragments or query parameters-without you needing to manually account for all edge cases. - **Use `CookieInterceptor` for custom cookies:** When you need to inject custom cookies into requests, use the `lib-cookieinterceptor` dependency instead of manually adding `Cookie` headers. Manually setting the `Cookie` header overrides all cookies (including Cloudflare cookies set via WebView), breaking login and challenge solving. ### Extension call flow @@ -718,12 +1017,12 @@ either `SourceFactory` or `HttpSource`. a.k.a. the Browse source entry point in the app (invoked by tapping on the source name). - The app calls `fetchPopularManga` which should return a `MangasPage` containing the first batch of -found `SManga` entries. + found `SManga` entries. - This method supports pagination. When user scrolls the manga list and more results must be fetched, - the app calls it again with increasing `page` values (starting with `page=1`). This continues while - `MangasPage.hasNextPage` is passed as `true` and `MangasPage.mangas` is not empty. + the app calls it again with increasing `page` values (starting with `page=1`). This continues while + `MangasPage.hasNextPage` is passed as `true` and `MangasPage.mangas` is not empty. - To show the list properly, the app needs `url`, `title` and `thumbnail_url`. You **must** set them -here. The rest of the fields could be filled later (refer to Manga Details below). + here. The rest of the fields could be filled later (refer to Manga Details below). #### Latest Manga @@ -736,7 +1035,7 @@ the source name). #### Manga Search - When the user searches inside the app, `fetchSearchManga` will be called and the rest of the flow -is similar to what happens with `fetchPopularManga`. + is similar to what happens with `fetchPopularManga`. - If search functionality is not available, return `Observable.just(MangasPage(emptyList(), false))` - `getFilterList` will be called to get all filters and filter types. @@ -774,70 +1073,71 @@ open class UriPartFilter(displayName: String, private val vals: Array - - - - - - - - - - - - - - - - -``` - -The `AndroidManifest.xml` file will contain an `android:name` attribute that refers to the path of your `UrlActivity.kt` file. For example, if the extension is Riztranslation, the `android:name` will be `.id.riztranslation.UrlActivity`. +Extensions can handle URLs from browsers or other apps by declaring deeplinks in `build.gradle.kts`. When a matching URL is opened on the device, Mihon launches and receives the URL as a search query. -Next, you have the `` element; you can have it multiple times, which allows you to specify the URL that can be opened in Tsundoku. You can read more about this in Android's [`` documentation](https://developer.android.com/guide/topics/manifest/data-element). +Add one or more `deeplink {}` blocks inside the `keiyoushi {}` block: -Now, as for `UrlActivity`, you can just use the example below. +```kotlin +keiyoushi { + name = "My Source" + // ... + + deeplink { + host("example.com") + path("/manga/..*") + path("/chapter/..*") + } +} +``` -> [!CAUTION] -> The activity does not support any Kotlin Intrinsics specific methods or calls, -> and using them will cause crashes in the activity. Consider using Java's equivalent -> methods instead, such as using `String`'s `equals()` instead of using `==`. -> -> You can use Kotlin Intrinsics in the extension source class, this limitation only -> applies to the activity classes. +| DSL call | Description | +|-----------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `host("example.com")` | A hostname to match. Call multiple times to register multiple hosts. If omitted, the host is derived from `baseUrl` automatically. | +| `path("/manga/..*")` | A path pattern in Android [`pathPattern`](https://developer.android.com/guide/topics/manifest/data-element#path) syntax. Call multiple times to match multiple paths. At least one `path()` call is required; a `deeplink {}` block with no paths produces no intent filter. | -To explain how it works, it will trigger Tsundoku's `SEARCH` action, passing the URL as a query and specifying that it comes from your extension to narrow down the search. Avoid putting any logic in this file; instead, implement it in your extension's class. +Multiple `deeplink {}` blocks create independent intent filters, which is useful when different hosts or path groups must be handled separately: ```kotlin -class UrlActivity : Activity() { - override fun onCreate(savedInstanceState: Bundle?) { - super.onCreate(savedInstanceState) - val intentData = intent?.data?.toString() - if (intentData != null) { - val mainIntent = Intent().apply { - action = "eu.kanade.tachiyomi.SEARCH" - putExtra("query", intentData) - putExtra("filter", packageName) - } - try { - startActivity(mainIntent) - } catch (e: Throwable) { - Log.e("RiztranslationUrl", e.toString()) - } - } else { - Log.e("RiztranslationUrl", "could not parse uri from intent $intent") - } +deeplink { + host("example.com") + path("/manga/..*") +} - finish() - exitProcess(0) - } +deeplink { + host("cdn.example.com") + path("/images/..*") } ``` -Now all you need to do is adapt the search function (`fetchSearchManga`) in your extension so that, given a URL, it returns a single manga that matches that URL. For example: +No `AndroidManifest.xml` or `UrlActivity.kt` is needed; they are generated and provided automatically by the build system. + +If the extension uses a theme (via `theme = ""`), deeplinks defined in the theme's `build.gradle.kts` are automatically merged, so individual extensions using that theme do not need to repeat shared URL patterns. + +Once deeplinks are declared, implement URL handling inside `fetchSearchManga`. When a deeplink is triggered, the app fires a search with the full URL as the query: ```kotlin -if (query.startsWith("https://")) { - val url = query.toHttpUrlOrNull() - if (url != null && url.host == baseUrl.toHttpUrl().host) { - val typeIndex = url.pathSegments.indexOfFirst { it == "detail" || it == "view" } - if (typeIndex != -1 && typeIndex + 1 < url.pathSize) { - val id = url.pathSegments[typeIndex + 1] - return GET("$apiUrl/Book?select=id,judul,cover&type=not.ilike.*novel*&id=eq.$id", apiHeaders) +override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable { + if (query.startsWith("https://")) { + val url = query.toHttpUrlOrNull() + if (url != null && url.host == baseUrl.toHttpUrl().host) { + val typeIndex = url.pathSegments.indexOfFirst { it == "detail" || it == "view" } + if (typeIndex != -1 && typeIndex + 1 < url.pathSize) { + val id = url.pathSegments[typeIndex + 1] + val manga = SManga.create().apply { + this@apply.url = "/Book?select=id,judul,cover&type=not.ilike.*novel*&id=eq.$id" + initialized = true + } + return fetchMangaDetails(manga) + .map { + it.url = manga.url + it.initialized = true + MangasPage(listOf(it), false) + } + } + + throw Exception("Unsupported url") } } + // normal search flow... } ``` -To test if the URL intent filter is working as expected, you can try opening the website in a browser -and navigating to the endpoint that was added as a filter or clicking a hyperlink. Alternatively, -you can use the `adb` command below. +> [!NOTE] +> Avoid checking for hardcoded host strings (e.g., `url.host == "site.com"`). Prefer dynamically comparing against the source's `baseUrl` to maintain mirror support. + +To test whether the URL intent filter is working as expected, use the `adb` command below: ```bash adb shell am start -d "" -a android.intent.action.VIEW ``` -You can find a complete example of how URLs work in the [Riztranslation extension](https://github.com/NovelSourcery/extensions-source/tree/main/src/id/riztranslation). #### Update strategy -In some cases, titles in a source will always have the same chapter list (i.e., they are immutable). -These do not need to be included in global app updates. Excluding them saves a lot of network requests -and prevents unnecessary load on the source servers. To change the update strategy of a `SManga`, -use the `update_strategy` field. You can find below a description of the current possible values. +In some cases, titles in a source always have the same chapter list (i.e., they are immutable). +These do not need inclusion in global app updates. Excluding them saves network requests +and prevents unnecessary load on source servers. To change the update strategy of a `SManga`, +use the `update_strategy` field. Description of the current possible values follows: - `UpdateStrategy.ALWAYS_UPDATE`: Titles marked as always update will be included in the library -update if they aren't excluded by additional restrictions. -- `UpdateStrategy.ONLY_FETCH_ONCE`: Titles marked as only fetch once will be automatically skipped -during library updates. Useful for cases where the series is previously known to be finished and have -only a single chapter, for example. + update unless excluded by additional restrictions. +- `UpdateStrategy.ONLY_FETCH_ONCE`: Titles marked as only fetch once are automatically skipped + during library updates. This is useful for cases where the series is known to be finished and has + only a single chapter, for example. If not set, it defaults to `ALWAYS_UPDATE`. #### Renaming existing sources -There are some cases where existing sources change their names on the website. To correctly reflect -these changes in the extension, you need to explicitly set the `id` to the same old value, otherwise -it will get changed by the new `name` value and users will be forced to migrate back to the source. +If existing sources change their names on the website, you must explicitly set the `id` to the previous value to reflect these changes correctly. Otherwise, it will change based on the new `name` value, forcing users to re-migrate to the source. -To get the current `id` value before the name change, you can search the source name in the [repository JSON file](https://github.com/novelsourcery/extensions/blob/repo/index.json) -by looking at the `sources` attribute of the extension. When you have the `id` copied, you can -override it in the source: +To get the current `id` value before a name change, search the source name in the [repository JSON file](https://github.com/NovelSourcery/extensions/blob/repo/index.json) +under the `sources` attribute of the extension. + +**If you are using `source {}` blocks**, set `id` directly in the block: ```kotlin -override val id: Long = +source { + name = "New Name" // or lang = "xx" if lang is what changed + lang = "en" + baseUrl = "https://example.com" + id = 1234567890123456789L // Replace with the actual old source ID +} ``` -Then the class name and the `name` attribute value can be changed. Also don't forget to update the -extension name and class name in the individual Gradle file. +The class name and the `name` attribute value can then be changed. Also, update the extension name and class name in the individual Gradle file. > [!IMPORTANT] -> The package name **needs** to be the same (even if it has the old name), otherwise users will not -> receive the extension update when it gets published in the repository. +> The package name **must** remain the same (even if it uses the old name); otherwise, users will not +> receive the extension update when published in the repository. -The `id` also needs to be explicitly set to the old value if you're changing the `lang` attribute. +The `id` also must be explicitly set to the old value if you change the `lang` attribute. > [!NOTE] -> If the source has also changed their theme you can instead just change -> the `name` field in the source class and in the Gradle file. By doing so -> a new `id` will be generated and users will be forced to migrate. +> If the source has also changed its theme, you can simply change +> the `name` field in the source class and the Gradle file. By doing so, +> a new `id` is generated and users will be forced to migrate. ## Multi-source themes -The `lib-multisrc` directory houses source code that is useful in situations where multiple source -sites use the same site generator tool (usually a CMS) for bootstrapping their website and this makes -them similar enough to prompt code reuse through inheritance/composition; which from now on we will -use the general **theme** term to refer to. +The `lib-multisrc` directory houses source code useful when multiple source +sites use the same site generator tool (usually a CMS). Their similarity prompts code reuse through inheritance or composition, referred to here as a **theme**. -Themes are provided as libraries within `lib-multisrc`. You can apply a theme to an extension by specifying the `themePkg` property in its `build.gradle` file. +Themes are provided as libraries within `lib-multisrc`. You can apply a theme to an extension by specifying the `theme` property in its `build.gradle.kts` file. ### Creating a new theme -To create a new theme, you need to set up a new module inside the `lib-multisrc` directory. The structure is similar to a regular extension, but it acts as a base library that other extensions can depend on. +To create a new theme, set up a new module inside the `lib-multisrc` directory. The structure is similar to a regular extension, but it acts as a base library that other extensions can depend on. #### Theme directory structure @@ -1051,48 +1324,67 @@ $ tree lib-multisrc// lib-multisrc// ├── build.gradle.kts └── src - └── main - └── java - └── eu - └── kanade - └── tachiyomi - └── multisrc - └── - └── .kt + └── eu + └── kanade + └── tachiyomi + └── multisrc + └── + └── .kt ``` -`` should be adapted from the CMS/theme name, and can only contain lowercase ASCII letters and digits. Your theme code must be placed in the package `eu.kanade.tachiyomi.multisrc.`. +`` should be adapted from the CMS or theme name and can only contain lowercase ASCII letters and digits. Your theme code must be placed in the package `eu.kanade.tachiyomi.multisrc.`. #### Theme build.gradle.kts -Make sure that your new theme's `build.gradle.kts` file follows this structure: +Ensure that your new theme's `build.gradle.kts` file follows this structure: ```kotlin plugins { alias(ns.plugins.multisrc) } -baseVersionCode = 1 +keiyoushi { + baseVersionCode = 1 + libVersion = "1.4" +} +``` + +If the CMS generates URLs with a consistent structure shared by all sites built on it, you can declare deeplinks here as well. Every extension using this theme inherits them automatically: + +```kotlin +keiyoushi { + baseVersionCode = 1 + libVersion = "1.4" + + deeplink { + path("/manga/..*") + path("/chapter/..*") + } +} ``` -| Field | Description | -|-------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `baseVersionCode` | The base version code for the theme. This must be a positive integer and **incremented** whenever a change is made to the theme's implementation that affects the extensions. | +When no `host()` is specified in a theme `deeplink {}` block, the host is resolved at build time from each individual extension's `baseUrl`, so the same path patterns apply to every site without hardcoding hostnames in the theme. + +| Field | Description | +|-------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `baseVersionCode` | The theme's base version code. This must be a positive integer and **incremented** whenever a change is made to the theme implementation that affects the extensions. | +| `libVersion` | The extension library version. Always set to `"1.4"`. | +| `deeplink {}` | Declares URL deeplink patterns inherited by all extensions using this theme. See [URL intent filter](#url-intent-filter). | #### Theme main class -The main class of the theme (e.g., `.kt`) contains the default implementation for the source sites. It should be declared as an `abstract class` extending `HttpSource`, allowing individual extensions to inherit and override its properties and methods. +The theme's main class (e.g., `.kt`) contains the default implementation for the source sites. It should be declared as an `abstract class` extending `HttpSource`, allowing individual extensions to inherit and override its properties and methods. ```kotlin -package eu.kanade.tachiyomi.multisrc. +package eu.kanade.tachiyomi.multisrc.example // Replace 'example' with your theme name import eu.kanade.tachiyomi.source.online.HttpSource -abstract class ( - override val name: String, - override val baseUrl: String, - override val lang: String, -) : HttpSource() { +abstract class ExampleTheme : HttpSource() { + + // name, lang, and baseUrl are inherited from HttpSource. They are automatically + // overridden and injected by the KSP generator via each extension's `source {}` + // block. Do not declare them in the constructor. // Theme default implementation... @@ -1101,25 +1393,42 @@ abstract class ( ### Using a Theme -To use a theme in your extension, follow the regular extension creation steps and add the `themePkg` property to your `build.gradle`: +To use a theme in your extension, follow the regular extension creation steps and configure `theme` in your `build.gradle.kts`: -```groovy -ext { - extName = '' - extClass = '.' - themePkg = '' - overrideVersionCode = 1 - isNsfw = true +```kotlin +// build.gradle.kts +keiyoushi { + name = "Example" // Replace with your actual source name + theme = "example" // Replace with your actual theme name + versionCode = 1 + contentWarning = ContentWarning.NSFW + libVersion = "1.4" + + source { + lang = "en" + baseUrl = "https://example.com" + } } +``` + +```kotlin +// MySource.kt +package eu.kanade.tachiyomi.extension.en.mysource -apply plugin: "kei.plugins.extension.legacy" +import keiyoushi.annotation.Source + +@Source +abstract class MySource : ExampleTheme() { // Replace 'ExampleTheme' with your theme name + // name, lang, id, baseUrl injected automatically + // override theme defaults as needed +} ``` -Notice that instead of `extVersionCode`, extensions using a theme must use `overrideVersionCode`. The final extension version code (`extVersionCode`) is automatically calculated during the build process as `theme.baseVersionCode + ext.overrideVersionCode`. +The final extension version code is automatically calculated during the build as `theme.baseVersionCode + versionCode`. -Because themes are provided as libraries, your extension's main class will directly inherit from the theme's base class. +Because themes are provided as libraries, your extension's main class inherits directly from the theme's base class. -Any site-specific overrides, custom functions, or custom icons are implemented directly in your extension's module (`src//`) by overriding the inherited theme properties and functions. +Site-specific overrides, custom functions, or custom icons are implemented directly in your extension's module (`src//`) by overriding the inherited theme properties and functions. ## Running @@ -1138,27 +1447,27 @@ For other builds, replace `app.tsundoku.dev` with the corresponding package IDs - Preview build: `app.tsundoku.debug` -If the extension builds and runs successfully, then the code changes should be ready to test in your local app. +If the extension builds and runs successfully, the code changes should be ready to test in your local app. > [!IMPORTANT] -> If you're deploying to Android 11 or higher, enable the `Always install with package manager` option in the run configurations. Without this option enabled, you might face issues such as Android Studio running an older version of the extension without the modifications you might have done. +> If you are deploying to Android 11 or higher, enable the `Always install with package manager` option in the run configurations. Otherwise, you might face issues such as Android Studio running an older version of the extension without your modifications. ## Debugging ### Android Debugger > [!NOTE] -> It is generally recommended to rely on logging instead of the Android Debugger. Using standard logs (like `Log.d` or viewing OkHttp logs) is typically much faster, easier to set up, and is more than sufficient for debugging web scraping logic. +> It is generally recommended to rely on logging instead of the Android Debugger. Using standard logs (like `Log.d` or viewing OkHttp logs) is typically much faster, easier to set up, and sufficient for debugging web scraping logic. > [!IMPORTANT] -> If you didn't **build the main app** from source with **debug enabled** and are using a release/beta APK, you **need a rooted device**. -> If you are using an **emulator** instead, make sure you choose a profile **without Google Play**. +> If you did not **build the main app** from source with **debug enabled** and are using a release or beta APK, you **need a rooted device**. +> If you are using an **emulator**, ensure you choose a profile **without Google Play**. -Follow the steps above for building and running locally if you haven't already. Debugging will not work if you did not follow the steps above. +Follow the steps above for building and running locally if you haven't already. Debugging will not work if you did not follow those steps. You can leverage the Android Debugger to add breakpoints and step through your extension while debugging. -You *cannot* simply use Android Studio's `Debug 'module.name'` -> this will most likely result in an +You _cannot_ simply use Android Studio's `Debug 'module.name'`; this will likely result in an error while launching. Instead, once you've built and installed your extension on the target device, use @@ -1170,7 +1479,7 @@ Inside the `Attach Debugger to Android Process` window, once the app is running ### Logs -You can also elect to simply rely on logs printed from your extension, which +You can also elect to rely on logs printed from your extension, which show up in the [`Logcat`](https://developer.android.com/studio/debug/am-logcat) panel of Android Studio. ### Inspecting network calls @@ -1179,38 +1488,34 @@ One of the easiest ways to inspect network issues (such as HTTP errors 404, 429, is to use the [`Logcat`](https://developer.android.com/studio/debug/am-logcat) panel of Android Studio and filter by the `OkHttpClient` tag. -To be able to check the calls made by OkHttp, you need to enable verbose logging in the app, which is -not enabled by default. To enable it, go to -More -> Settings -> Advanced -> Verbose logging. After enabling it, don't forget to restart the app. +To check the calls made by OkHttp, you must enable verbose logging in the app; it is not enabled by default. To enable it, go to More → Settings → Advanced → Verbose logging. Afterward, restart the app. -Inspecting the Logcat allows you to get a good look at the call flow and is more than enough in most -cases where issues occur. However, alternatively, you can also use an external tool like `mitm-proxy`. +Inspecting the Logcat allows you to see the call flow and is sufficient in most +cases. Alternatively, you can use an external tool like `mitmproxy`. For that, refer to the subsequent sections. -On newer Android Studio versions, you can use its built-in Network Inspector inside the -App Inspection tool window. This feature provides a nice GUI to inspect the requests made in the app. +On newer Android Studio versions, you can use the built-in Network Inspector inside the +App Inspection tool window. This feature provides a GUI to inspect the requests made in the app. To use it, follow the [official documentation](https://developer.android.com/studio/debug/network-profiler) and select the app's package name in the process list. ### Using external network inspecting tools -If you want a deeper look into the network flow, such as inspecting the request and response bodies -you can use an external tool like `mitm-proxy`. +If you want a deeper look into the network flow, such as inspecting the request and response bodies, +you can use an external tool like `mitmproxy`. -#### Setup your proxy server +#### Set up your proxy server -We are going to use [mitm-proxy](https://mitmproxy.org/) but you can replace it with any other Web -Debugger (i.e. Charles, Burp Suite, Fiddler etc). To install and execute, follow the commands below. +We are going to use [mitmproxy](https://mitmproxy.org/), but you can replace it with any other Web +Debugger (e.g., Charles, Burp Suite, Fiddler). To install and execute, follow the commands below. -```console -# Install the tool. -$ sudo pip3 install mitmproxy -# Execute the web interface and the proxy. -$ mitmweb -``` +> [!WARNING] +> Do NOT use `sudo pip` to install `mitmproxy`. Follow the [official installation methods](https://docs.mitmproxy.org/stable/overview-installation/). + +After installing it, you can run `mitmweb` to open the web interface. -Alternatively, you can also use the Docker image: +Alternatively, use the Docker image: ```bash $ docker run --rm -it -p 8080:8080 \ @@ -1223,14 +1528,19 @@ After installing and running, open your browser and navigate to [!CAUTION] +> The following code disables certificate and hostname verification. Use it **only** in a local debug build, **never** submit it as production source code, and **remove it** before opening a Pull Request. + +For that, add this code inside your source class: ```kotlin package eu.kanade.tachiyomi.novelextension.en.mysource import android.annotation.SuppressLint import eu.kanade.tachiyomi.source.online.HttpSource +import keiyoushi.annotation.Source import okhttp3.OkHttpClient import java.net.InetSocketAddress import java.net.Proxy @@ -1240,7 +1550,8 @@ import javax.net.ssl.SSLContext import javax.net.ssl.TrustManager import javax.net.ssl.X509TrustManager -class MySource : HttpSource() { +@Source +abstract class MySource : HttpSource() { private fun OkHttpClient.Builder.ignoreAllSSLErrors(): OkHttpClient.Builder { val naiveTrustManager = @SuppressLint("CustomX509TrustManager") object : X509TrustManager { @@ -1266,9 +1577,9 @@ class MySource : HttpSource() { } ``` -Note: `10.0.2.2` is usually the address of your loopback interface in the android emulator. If -the app tells you that it's unable to connect to 10.0.2.2:8080 you will likely need to change it -(the same if you are using hardware device). +Note: `10.0.2.2` is usually the address of your loopback interface in the Android emulator. If +the app tells you that it's unable to connect to 10.0.2.2:8080, you will likely need to change it +(the same if you are using a hardware device). If all went well, you should see all requests and responses made by the source in the web interface of `mitmweb`. @@ -1278,9 +1589,8 @@ of `mitmweb`. APKs can be created in Android Studio via `Build > Build Bundle(s) / APK(s) > Build APK(s)` or `Build > Generate Signed Bundle / APK`. -If for some reason you decide to build the APK from the command line, you can use the following -command (because you're doing things differently than expected, I assume you have some -knowledge of gradlew and your OS): +If you decide to build the APK from the command line, use the following +command: ```console // For a single apk, use this command @@ -1289,31 +1599,25 @@ $ ./gradlew src:::assembleDebug ## Submitting the changes -When you feel confident about your changes, submit a new Pull Request so your code can be reviewed -and merged if it's approved. We encourage following a [GitHub Standard Fork & Pull Request Workflow](https://gist.github.com/Chaser324/ce0505fbed06b947d962) -and following the good practices of the workflow, such as not committing directly to `main`: always -create a new branch for your changes. +When you feel confident about your changes, submit a new Pull Request for review. We encourage following a [GitHub Standard Fork & Pull Request Workflow](https://gist.github.com/Chaser324/ce0505fbed06b947d962); avoid committing directly to `main` and always create a new branch for your changes. -If you are more comfortable about using Git GUI-based tools, you can refer to [this guide](https://learntodroid.com/how-to-use-git-and-github-in-android-studio/) -about the Git integration inside Android Studio, specifically the "How to Contribute to an to Existing -Git Repository in Android Studio" section of the guide. +If you prefer using Git GUI-based tools, refer to [this guide](https://learntodroid.com/how-to-use-git-and-github-in-android-studio/) +about Git integration in Android Studio. Specifically, check the "How to Contribute to an Existing +Git Repository in Android Studio" section. > [!IMPORTANT] -> Make sure you have generated the extension icon using the linked Icon Generator tool in the [Tools](#tools) -> section. The icon **must follow the pattern** adopted by all other extensions: a square with rounded -> corners. Make sure to remove the generated `web_hi_res_512.png`. +> Ensure you have generated the extension icon using the Icon Generator tool in the [Tools](#tools) +> section. The icon **must follow the pattern** adopted by all extensions: a square with rounded +> corners. Remove the generated `web_hi_res_512.png`. -Please **do test your changes by compiling it through Android Studio** before submitting it. Obvious -untested PRs will not be merged, such as ones created with the GitHub web interface. Also make sure -to follow the PR checklist available in the PR body field when creating a new PR. As a reference, you -can find it below. +Please **test your changes by compiling through Android Studio** before submitting. Untested PRs will not be merged. Also, ensure you follow the PR checklist in the PR body field when creating a new PR; it is provided below for reference. ### Pull Request checklist -- Updated `extVersionCode` value in `build.gradle` for individual extensions -- Updated `overrideVersionCode` or `baseVersionCode` as needed for all multisrc extensions +- Updated `versionCode` value in `build.gradle.kts` +- Updated `baseVersionCode` in `build.gradle.kts` (if updated multisrc theme code) - Referenced all related issues in the PR body (e.g. "Closes #xyz") -- Added the `isNsfw = true` flag in `build.gradle` when appropriate +- Set the `contentWarning` configuration in `build.gradle.kts` appropriately - Have not changed source names - Have explicitly kept the `id` if a source's name or language were changed - Have tested the modifications by compiling and running the extension through Android Studio diff --git a/common/proguard-rules.pro b/common/proguard-rules.pro index 65433f578..0527233ab 100644 --- a/common/proguard-rules.pro +++ b/common/proguard-rules.pro @@ -1,6 +1,5 @@ #-dontobfuscate -dontoptimize --dontpreverify ## Partially based on https://android.googlesource.com/platform/tools/base/+/refs/heads/mirror-goog-studio-main/build-system/gradle-core/src/main/resources/com/android/build/gradle/proguard-common.txt @@ -49,4 +48,4 @@ } -if @kotlinx.serialization.Serializable class ** --keep,allowshrinking,allowoptimization,allowobfuscation,allowaccessmodification class <1> +-keep,allowshrinking,allowoptimization,allowobfuscation class <1> diff --git a/compiler/build.gradle.kts b/compiler/build.gradle.kts new file mode 100644 index 000000000..3ed7f0687 --- /dev/null +++ b/compiler/build.gradle.kts @@ -0,0 +1,10 @@ +plugins { + kotlin("jvm") + kotlin("plugin.serialization") +} + +dependencies { + implementation(libs.ksp.api) + implementation(libs.kotlinpoet.ksp) + implementation(libs.kotlin.json) +} diff --git a/compiler/src/main/kotlin/keiyoushi/processor/SourceProcessor.kt b/compiler/src/main/kotlin/keiyoushi/processor/SourceProcessor.kt new file mode 100644 index 000000000..50176a0b9 --- /dev/null +++ b/compiler/src/main/kotlin/keiyoushi/processor/SourceProcessor.kt @@ -0,0 +1,443 @@ +package keiyoushi.processor + +import com.google.devtools.ksp.getAllSuperTypes +import com.google.devtools.ksp.processing.CodeGenerator +import com.google.devtools.ksp.processing.Dependencies +import com.google.devtools.ksp.processing.KSPLogger +import com.google.devtools.ksp.processing.Resolver +import com.google.devtools.ksp.processing.SymbolProcessor +import com.google.devtools.ksp.processing.SymbolProcessorEnvironment +import com.google.devtools.ksp.processing.SymbolProcessorProvider +import com.google.devtools.ksp.symbol.KSAnnotated +import com.google.devtools.ksp.symbol.KSClassDeclaration +import com.google.devtools.ksp.symbol.Modifier +import com.squareup.kotlinpoet.ClassName +import com.squareup.kotlinpoet.CodeBlock +import com.squareup.kotlinpoet.FileSpec +import com.squareup.kotlinpoet.FunSpec +import com.squareup.kotlinpoet.KModifier +import com.squareup.kotlinpoet.MemberName +import com.squareup.kotlinpoet.ParameterizedTypeName.Companion.parameterizedBy +import com.squareup.kotlinpoet.PropertySpec +import com.squareup.kotlinpoet.TypeSpec +import com.squareup.kotlinpoet.asClassName +import com.squareup.kotlinpoet.buildCodeBlock +import com.squareup.kotlinpoet.ksp.toClassName +import com.squareup.kotlinpoet.ksp.writeTo +import java.io.File +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json + +@Serializable +data class PartialLocaleStrings( + val mirrorTitle: String? = null, + val customUrlTitle: String? = null, + val customUrlDialogMessage: String? = null, +) + +data class LocaleStrings( + val mirrorTitle: String, + val customUrlTitle: String, + val customUrlDialogMessage: String, +) + +@Serializable +data class BaseUrlSpecData( + val type: String, + val defaultUrl: String, + val mirrors: List = emptyList(), +) + +@Serializable +data class MirrorData( + val url: String, + val label: String = "", +) + +@Serializable +data class SourceDef( + val name: String, + val lang: String, + val id: Long, + val baseUrl: BaseUrlSpecData, +) + +private const val HTTP_SOURCE = "eu.kanade.tachiyomi.source.online.HttpSource" + +private fun KSClassDeclaration.derivesFromHttpSource(): Boolean = + getAllSuperTypes().any { it.declaration.qualifiedName?.asString() == HTTP_SOURCE } + +private val configurable = ClassName("eu.kanade.tachiyomi.source", "ConfigurableSource") +private val preferenceScreen = ClassName("androidx.preference", "PreferenceScreen") +private val mirrorPrefsClass = ClassName("keiyoushi.source", "MirrorPreferences") +private val customUrlPrefsClass = ClassName("keiyoushi.source", "CustomUrlPreferences") +private val getPreferencesFn = MemberName("keiyoushi.utils", "getPreferences") + +class SourceProcessor( + private val codeGenerator: CodeGenerator, + private val options: Map, + private val logger: KSPLogger, +) : SymbolProcessor { + + private val translations: Map by lazy { + val path = options["kei_translations"] ?: return@lazy emptyMap() + runCatching { + Json.decodeFromString>(File(path).readText()) + }.getOrElse { + logger.warn("kei_translations: ${it.message}") + emptyMap() + } + } + + private fun stringsForLang(lang: String): LocaleStrings { + val en = translations.getValue("en") + val locale = translations[lang] ?: translations[lang.substringBefore("-")] + return LocaleStrings( + mirrorTitle = locale?.mirrorTitle ?: en.mirrorTitle!!, + customUrlTitle = locale?.customUrlTitle ?: en.customUrlTitle!!, + customUrlDialogMessage = locale?.customUrlDialogMessage ?: en.customUrlDialogMessage!!, + ) + } + + private var invoked = false + + override fun process(resolver: Resolver): List { + if (invoked) return emptyList() + invoked = true + + val annotatedClasses = resolver + .getSymbolsWithAnnotation("keiyoushi.annotation.Source") + .filterIsInstance() + .toList() + + val sourcesJson = options["kei_sources"] + + if (annotatedClasses.isEmpty() && sourcesJson == null) return emptyList() + + if (annotatedClasses.isEmpty()) { + logger.error("source {} blocks present but no @Source class found — annotate your source class with @Source") + return emptyList() + } + + if (annotatedClasses.size > 1) { + val names = annotatedClasses.joinToString { it.qualifiedName?.asString() ?: it.simpleName.asString() } + logger.error("exactly one @Source class allowed per module, found: $names", annotatedClasses.first()) + return emptyList() + } + + val annotated = annotatedClasses.single() + + if (sourcesJson == null) { + logger.error("@Source found but no source {} blocks in build.gradle.kts", annotated) + return emptyList() + } + + val sources = Json.decodeFromString>(sourcesJson) + if (sources.isEmpty()) { + logger.error("@Source found but source list is empty", annotated) + return emptyList() + } + + val pkg = annotated.packageName.asString() + val annotatedClass = annotated.toClassName() + val superTypeNames = annotated.getAllSuperTypes() + .mapNotNull { it.declaration.qualifiedName?.asString() } + .toSet() + val isConfigurable = "eu.kanade.tachiyomi.source.ConfigurableSource" in superTypeNames + + if (HTTP_SOURCE !in superTypeNames) { + logger.error("@Source class must derive from HttpSource", annotated) + return emptyList() + } + + // any overrides below HttpSource + val overridden = annotated.getAllProperties() + .filter { Modifier.OVERRIDE in it.modifiers && Modifier.ABSTRACT !in it.modifiers } + .filter { prop -> + val owner = prop.parentDeclaration as? KSClassDeclaration + owner != null && owner.qualifiedName?.asString() != HTTP_SOURCE && owner.derivesFromHttpSource() + } + .map { it.simpleName.asString() } + .toSet() + + val fileProps = mutableListOf() + + val isConcrete = Modifier.ABSTRACT !in annotated.modifiers + val generatedClass = if (sources.size == 1 && !isConcrete) { + buildSingleSourceClass(annotatedClass, sources.single(), isConfigurable, overridden, annotated, fileProps) + } else { + buildSourceFactoryClass(annotatedClass, sources, isConfigurable, overridden, annotated, fileProps) + } + + FileSpec.builder(pkg, "ExtensionGenerated") + .apply { fileProps.forEach(::addProperty) } + .addType(generatedClass) + .build() + .writeTo(codeGenerator, Dependencies(false, annotated.containingFile!!)) + + return emptyList() + } + + private fun buildSingleSourceClass( + annotatedClass: ClassName, + source: SourceDef, + isConfigurable: Boolean, + overridden: Set, + node: KSClassDeclaration, + fileProps: MutableList, + ): TypeSpec = TypeSpec.classBuilder("ExtensionGenerated") + .addModifiers(KModifier.INTERNAL) + .superclass(annotatedClass) + .applySourceMembers(source, "", fileProps, isConfigurable, overridden, node) + .build() + + private fun buildSourceFactoryClass( + annotatedClass: ClassName, + sources: List, + isConfigurable: Boolean, + overridden: Set, + node: KSClassDeclaration, + fileProps: MutableList, + ): TypeSpec { + val sourceFactoryType = ClassName("eu.kanade.tachiyomi.source", "SourceFactory") + val sourceType = ClassName("eu.kanade.tachiyomi.source", "Source") + + // A concrete @Source class is instantiated directly (preserving its FQN) instead of + // being wrapped in an anonymous subclass, whose qualifiedName would be null and break + // app-side deligated sources or enhanced trackers + val isConcrete = Modifier.ABSTRACT !in node.modifiers + val ctorParams = node.primaryConstructor?.parameters.orEmpty() + .mapNotNull { it.name?.asString() } + .toSet() + + if (isConcrete) { + validateConcreteSource(node, sources, overridden, ctorParams) + } + + val createSourcesCode = CodeBlock.builder() + .add("return listOf(\n") + .indent() + .apply { + for ((index, source) in sources.withIndex()) { + if (isConcrete) { + add("%L,\n", buildConcreteSource(annotatedClass, source, ctorParams)) + } else { + add( + "%L,\n", + TypeSpec.anonymousClassBuilder() + .superclass(annotatedClass) + .applySourceMembers(source, index.toString(), fileProps, isConfigurable, overridden, node) + .build(), + ) + } + } + } + .unindent() + .add(")") + .build() + + return TypeSpec.classBuilder("ExtensionGenerated") + .addModifiers(KModifier.INTERNAL) + .addSuperinterface(sourceFactoryType) + .addFunction( + FunSpec.builder("createSources") + .addModifiers(KModifier.OVERRIDE) + .returns(List::class.asClassName().parameterizedBy(sourceType)) + .addCode(createSourcesCode) + .build(), + ) + .build() + } + + private fun validateConcreteSource( + node: KSClassDeclaration, + sources: List, + overridden: Set, + ctorParams: Set, + ) { + val className = node.simpleName.asString() + + if (sources.any { it.baseUrl.type != "static" }) { + logger.error( + "A concrete @Source class ($className) only supports a static baseUrl; " + + "make it abstract to use a mirror/custom baseUrl.", + node, + ) + } + + if ("id" !in ctorParams || "lang" !in ctorParams) { + logger.error( + "A concrete @Source class ($className) must declare `override val lang` and " + + "`override val id` as primary constructor parameters.", + node, + ) + } + + if ("versionId" in overridden || "versionId" in ctorParams) { + logger.error( + "versionId is owned by the DSL; remove `versionId` from $className " + + "(set 'versionId = …' in the source { } block if you need a specific value).", + node, + ) + } + + if ("name" !in ctorParams && "name" in overridden) { + logger.warn("name is provided by $className; the DSL name is used for metadata only.", node) + } + + if ("baseUrl" !in ctorParams && "baseUrl" in overridden) { + logger.warn("baseUrl is provided by $className; the DSL baseUrl is used for metadata/hosts only.", node) + } + } + + private fun buildConcreteSource( + annotatedClass: ClassName, + source: SourceDef, + ctorParams: Set, + ): CodeBlock = buildCodeBlock { + add("%T(\n", annotatedClass) + indent() + if ("name" in ctorParams) add("name = %S,\n", source.name) + if ("lang" in ctorParams) add("lang = %S,\n", source.lang) + if ("id" in ctorParams) add("id = %LL,\n", source.id) + if ("baseUrl" in ctorParams) add("baseUrl = %S,\n", source.baseUrl.defaultUrl) + unindent() + add(")") + } + + private fun TypeSpec.Builder.applySourceMembers( + source: SourceDef, + suffix: String, + fileProps: MutableList, + isConfigurable: Boolean, + overridden: Set, + node: KSClassDeclaration, + ): TypeSpec.Builder = apply { + val className = node.simpleName.asString() + + if ("name" in overridden) { + logger.warn("name is provided by $className; skipping generated name (DSL name is used for metadata only)", node) + } else { + addProperty( + PropertySpec.builder("name", String::class.asClassName(), KModifier.OVERRIDE) + .getter(FunSpec.getterBuilder().addStatement("return %S", source.name).build()) + .build(), + ) + } + + if ("lang" in overridden) { + logger.error("lang is owned by the DSL; remove 'override val lang' from $className", node) + } else { + addProperty( + PropertySpec.builder("lang", String::class.asClassName(), KModifier.OVERRIDE) + .getter(FunSpec.getterBuilder().addStatement("return %S", source.lang).build()) + .build(), + ) + } + + if ("versionId" in overridden) { + logger.error("versionId is owned by the DSL; remove 'override val versionId' from $className (set 'versionId = …' in the source { } block)", node) + } + + if ("id" in overridden) { + logger.error("id is owned by the DSL; remove 'override val id' from $className (set 'id = …' in the source { } block if you need a specific value)", node) + } else { + addProperty( + PropertySpec.builder("id", Long::class.asClassName(), KModifier.OVERRIDE) + .getter(FunSpec.getterBuilder().addStatement("return %LL", source.id).build()) + .build(), + ) + } + + val urlSpec = source.baseUrl + when { + "baseUrl" in overridden && urlSpec.type == "static" -> + logger.warn("baseUrl is provided by $className; skipping generated baseUrl (DSL baseUrl is used for metadata/hosts only)", node) + "baseUrl" in overridden -> + logger.error("baseUrl is overridden in $className but the DSL declares a ${urlSpec.type} baseUrl, which is generated.", node) + urlSpec.type == "static" -> { + addProperty( + PropertySpec.builder("baseUrl", String::class.asClassName(), KModifier.OVERRIDE) + .getter(FunSpec.getterBuilder().addStatement("return %S", urlSpec.defaultUrl).build()) + .build(), + ) + } + urlSpec.type == "mirrors" -> { + val strings = stringsForLang(source.lang) + val prefsName = "mirrorPrefs$suffix" + val initializer = CodeBlock.builder() + .add("%T(\n", mirrorPrefsClass) + .indent() + .add("preferences = %M(%LL),\n", getPreferencesFn, source.id) + .add("mirrors = arrayOf(\n") + .indent() + .apply { + urlSpec.mirrors.forEach { mirror -> + add("%S to %S,\n", mirror.label, mirror.url) + } + } + .unindent() + .add("),\n") + .add("title = %S,\n", strings.mirrorTitle) + .unindent() + .add(")") + .build() + fileProps += PropertySpec.builder(prefsName, mirrorPrefsClass) + .addModifiers(KModifier.PRIVATE) + .initializer(initializer) + .build() + addProperty( + PropertySpec.builder("baseUrl", String::class.asClassName(), KModifier.OVERRIDE) + .getter(FunSpec.getterBuilder().addStatement("return %N.baseUrl", prefsName).build()) + .build(), + ) + addPreferenceScreen(isConfigurable) { addStatement("%N.setupPreferenceScreen(screen)", prefsName) } + if (!isConfigurable) addSuperinterface(configurable) + } + urlSpec.type == "custom" -> { + val strings = stringsForLang(source.lang) + val prefsName = "customUrlPrefs$suffix" + val initializer = CodeBlock.builder() + .add("%T(\n", customUrlPrefsClass) + .indent() + .add("preferences = %M(%LL),\n", getPreferencesFn, source.id) + .add("defaultUrl = %S,\n", urlSpec.defaultUrl) + .add("title = %S,\n", strings.customUrlTitle) + .add("dialogMessage = %S,\n", strings.customUrlDialogMessage) + .unindent() + .add(")") + .build() + fileProps += PropertySpec.builder(prefsName, customUrlPrefsClass) + .addModifiers(KModifier.PRIVATE) + .initializer(initializer) + .build() + addProperty( + PropertySpec.builder("baseUrl", String::class.asClassName(), KModifier.OVERRIDE) + .getter(FunSpec.getterBuilder().addStatement("return %N.baseUrl", prefsName).build()) + .build(), + ) + addPreferenceScreen(isConfigurable) { addStatement("%N.setupPreferenceScreen(screen)", prefsName) } + if (!isConfigurable) addSuperinterface(configurable) + } + } + } + + private fun TypeSpec.Builder.addPreferenceScreen( + callSuper: Boolean, + addPrefs: FunSpec.Builder.() -> Unit, + ) { + addFunction( + FunSpec.builder("setupPreferenceScreen") + .addModifiers(KModifier.OVERRIDE) + .addParameter("screen", preferenceScreen) + .apply(addPrefs) + .apply { if (callSuper) addStatement("super.setupPreferenceScreen(screen)") } + .build(), + ) + } + + class Provider : SymbolProcessorProvider { + override fun create(environment: SymbolProcessorEnvironment): SymbolProcessor = + SourceProcessor(environment.codeGenerator, environment.options, environment.logger) + } +} diff --git a/compiler/src/main/resources/META-INF/services/com.google.devtools.ksp.processing.SymbolProcessorProvider b/compiler/src/main/resources/META-INF/services/com.google.devtools.ksp.processing.SymbolProcessorProvider new file mode 100644 index 000000000..618574dee --- /dev/null +++ b/compiler/src/main/resources/META-INF/services/com.google.devtools.ksp.processing.SymbolProcessorProvider @@ -0,0 +1 @@ +keiyoushi.processor.SourceProcessor$Provider diff --git a/core/build.gradle.kts b/core/build.gradle.kts index 21caeba47..5d53f8171 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -19,8 +19,10 @@ android { } dependencies { - compileOnly(libs.bundles.common) + // :core targets the 1.6 lib line specifically for awaitSuccess - it doesn't touch + // RefreshContext, so this is independent of what individual extensions compile against. + compileOnly(libs.bundles.common16) - testImplementation(libs.bundles.common) + testImplementation(libs.bundles.common16) testImplementation(libs.junit) } diff --git a/core/src/main/AndroidManifest.xml b/core/src/main/AndroidManifest.xml index 4712f467b..5588a152a 100644 --- a/core/src/main/AndroidManifest.xml +++ b/core/src/main/AndroidManifest.xml @@ -9,6 +9,10 @@ + + + + diff --git a/core/src/main/kotlin/keiyoushi/annotation/Source.kt b/core/src/main/kotlin/keiyoushi/annotation/Source.kt new file mode 100644 index 000000000..ceca76f8b --- /dev/null +++ b/core/src/main/kotlin/keiyoushi/annotation/Source.kt @@ -0,0 +1,5 @@ +package keiyoushi.annotation + +@Retention(AnnotationRetention.SOURCE) +@Target(AnnotationTarget.CLASS) +annotation class Source diff --git a/core/src/main/kotlin/keiyoushi/network/OkHttp.kt b/core/src/main/kotlin/keiyoushi/network/OkHttp.kt new file mode 100644 index 000000000..258ba818e --- /dev/null +++ b/core/src/main/kotlin/keiyoushi/network/OkHttp.kt @@ -0,0 +1,328 @@ +package keiyoushi.network + +import eu.kanade.tachiyomi.network.await +import eu.kanade.tachiyomi.network.awaitSuccess +import eu.kanade.tachiyomi.source.online.HttpSource +import okhttp3.CacheControl +import okhttp3.Headers +import okhttp3.HttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.RequestBody +import okhttp3.Response +import kotlin.time.Duration.Companion.minutes + +/** + * Default cache control. + */ +private val DEFAULT_CACHE_CONTROL = CacheControl.Builder().maxAge(10.minutes).build() + +/** + * Executes a GET request asynchronously and returns the response. + * + * @param url The [HttpUrl] to request. + * @param headers The headers to include in the request. + * @param cacheControl The cache control settings for the request. + * @param ensureSuccess If true, throws an exception if the response code is not 2xx. + * @return The HTTP [Response]. + */ +suspend fun OkHttpClient.get( + url: HttpUrl, + headers: Headers, + cacheControl: CacheControl = DEFAULT_CACHE_CONTROL, + ensureSuccess: Boolean = true, +): Response { + val request = Request.Builder() + .url(url) + .headers(headers) + .cacheControl(cacheControl) + .build() + val call = newCall(request) + + return if (ensureSuccess) { + call.awaitSuccess() + } else { + call.await() + } +} + +/** + * Executes a GET request asynchronously using a URL string and returns the response. + * + * @param url The URL string to request. + * @param headers The headers to include in the request. + * @param cacheControl The cache control settings for the request. + * @param ensureSuccess If true, throws an exception if the response code is not 2xx. + * @return The HTTP [Response]. + */ +suspend fun OkHttpClient.get( + url: String, + headers: Headers, + cacheControl: CacheControl = DEFAULT_CACHE_CONTROL, + ensureSuccess: Boolean = true, +): Response = get(url.toHttpUrl(), headers, cacheControl, ensureSuccess) + +/** + * Executes a GET request asynchronously, automatically retrieving the headers from the + * current [HttpSource] context receiver. + * + * @param url The [HttpUrl] to request. + * @param cacheControl The cache control settings for the request. + * @param ensureSuccess If true, throws an exception if the response code is not 2xx. + * @return The HTTP [Response]. + */ +context(source: HttpSource) +suspend fun OkHttpClient.get( + url: HttpUrl, + cacheControl: CacheControl = DEFAULT_CACHE_CONTROL, + ensureSuccess: Boolean = true, +): Response = get(url, source.headers, cacheControl, ensureSuccess) + +/** + * Executes a GET request asynchronously using a URL string, automatically retrieving the + * headers from the current [HttpSource] context receiver. + * + * @param url The URL string to request. + * @param cacheControl The cache control settings for the request. + * @param ensureSuccess If true, throws an exception if the response code is not 2xx. + * @return The HTTP [Response]. + */ +context(source: HttpSource) +suspend fun OkHttpClient.get( + url: String, + cacheControl: CacheControl = DEFAULT_CACHE_CONTROL, + ensureSuccess: Boolean = true, +): Response = get(url, source.headers, cacheControl, ensureSuccess) + +/** + * Executes a POST request asynchronously and returns the response. + * + * @param url The [HttpUrl] to request. + * @param headers The headers to include in the request. + * @param body The request body payload. + * @param ensureSuccess If true, throws an exception if the response code is not 2xx. + * @return The HTTP [Response]. + */ +suspend fun OkHttpClient.post( + url: HttpUrl, + headers: Headers, + body: RequestBody, + ensureSuccess: Boolean = true, +): Response { + val request = Request.Builder() + .url(url) + .headers(headers) + .post(body) + .build() + val call = newCall(request) + + return if (ensureSuccess) { + call.awaitSuccess() + } else { + call.await() + } +} + +/** + * Executes a POST request asynchronously using a URL string and returns the response. + * + * @param url The URL string to request. + * @param headers The headers to include in the request. + * @param body The request body payload. + * @param ensureSuccess If true, throws an exception if the response code is not 2xx. + * @return The HTTP [Response]. + */ +suspend fun OkHttpClient.post( + url: String, + headers: Headers, + body: RequestBody, + ensureSuccess: Boolean = true, +): Response = post(url.toHttpUrl(), headers, body, ensureSuccess) + +/** + * Executes a POST request asynchronously, automatically retrieving the + * headers from the current [HttpSource] context receiver. + * + * @param url The [HttpUrl] to request. + * @param body The request body payload. + * @param ensureSuccess If true, throws an exception if the response code is not 2xx. + * @return The HTTP [Response]. + */ +context(source: HttpSource) +suspend fun OkHttpClient.post( + url: HttpUrl, + body: RequestBody, + ensureSuccess: Boolean = true, +): Response = post(url, source.headers, body, ensureSuccess) + +/** + * Executes a POST request asynchronously using a URL string, automatically retrieving the + * headers from the current [HttpSource] context receiver. + * + * @param url The URL string to request. + * @param body The request body payload. + * @param ensureSuccess If true, throws an exception if the response code is not 2xx. + * @return The HTTP [Response]. + */ +context(source: HttpSource) +suspend fun OkHttpClient.post( + url: String, + body: RequestBody, + ensureSuccess: Boolean = true, +): Response = post(url, source.headers, body, ensureSuccess) + +/** + * Executes a PUT request asynchronously and returns the response. + * + * @param url The [HttpUrl] to request. + * @param headers The headers to include in the request. + * @param body The request body payload. + * @param ensureSuccess If true, throws an exception if the response code is not 2xx. + * @return The HTTP [Response]. + */ +suspend fun OkHttpClient.put( + url: HttpUrl, + headers: Headers, + body: RequestBody, + ensureSuccess: Boolean = true, +): Response { + val request = Request.Builder() + .url(url) + .headers(headers) + .put(body) + .build() + val call = newCall(request) + + return if (ensureSuccess) { + call.awaitSuccess() + } else { + call.await() + } +} + +/** + * Executes a PUT request asynchronously using a URL string and returns the response. + * + * @param url The URL string to request. + * @param headers The headers to include in the request. + * @param body The request body payload. + * @param ensureSuccess If true, throws an exception if the response code is not 2xx. + * @return The HTTP [Response]. + */ +suspend fun OkHttpClient.put( + url: String, + headers: Headers, + body: RequestBody, + ensureSuccess: Boolean = true, +): Response = put(url.toHttpUrl(), headers, body, ensureSuccess) + +/** + * Executes a PUT request asynchronously using a URL string, automatically retrieving the + * headers from the current [HttpSource] context receiver. + * + * @param url The URL string to request. + * @param body The request body payload. + * @param ensureSuccess If true, throws an exception if the response code is not 2xx. + * @return The HTTP [Response]. + */ +context(source: HttpSource) +suspend fun OkHttpClient.put( + url: String, + body: RequestBody, + ensureSuccess: Boolean = true, +): Response = put(url, source.headers, body, ensureSuccess) + +/** + * Executes a PUT request asynchronously, automatically retrieving the + * headers from the current [HttpSource] context receiver. + * + * @param url The [HttpUrl] to request. + * @param body The request body payload. + * @param ensureSuccess If true, throws an exception if the response code is not 2xx. + * @return The HTTP [Response]. + */ +context(source: HttpSource) +suspend fun OkHttpClient.put( + url: HttpUrl, + body: RequestBody, + ensureSuccess: Boolean = true, +): Response = put(url, source.headers, body, ensureSuccess) + +/** + * Executes a HEAD request asynchronously and returns the response. + * + * @param url The [HttpUrl] to request. + * @param headers The headers to include in the request. + * @param cacheControl The cache control settings for the request. + * @param ensureSuccess If true, throws an exception if the response code is not 2xx. + * @return The HTTP [Response]. + */ +suspend fun OkHttpClient.head( + url: HttpUrl, + headers: Headers, + cacheControl: CacheControl = DEFAULT_CACHE_CONTROL, + ensureSuccess: Boolean = true, +): Response { + val request = Request.Builder() + .url(url) + .headers(headers) + .cacheControl(cacheControl) + .head() + .build() + val call = newCall(request) + + return if (ensureSuccess) { + call.awaitSuccess() + } else { + call.await() + } +} + +/** + * Executes a HEAD request asynchronously using a URL string and returns the response. + * + * @param url The URL string to request. + * @param headers The headers to include in the request. + * @param cacheControl The cache control settings for the request. + * @param ensureSuccess If true, throws an exception if the response code is not 2xx. + * @return The HTTP [Response]. + */ +suspend fun OkHttpClient.head( + url: String, + headers: Headers, + cacheControl: CacheControl = DEFAULT_CACHE_CONTROL, + ensureSuccess: Boolean = true, +): Response = head(url.toHttpUrl(), headers, cacheControl, ensureSuccess) + +/** + * Executes a HEAD request asynchronously using a URL string, automatically retrieving the + * headers from the current [HttpSource] context receiver. + * + * @param url The URL string to request. + * @param cacheControl The cache control settings for the request. + * @param ensureSuccess If true, throws an exception if the response code is not 2xx. + * @return The HTTP [Response]. + */ +context(source: HttpSource) +suspend fun OkHttpClient.head( + url: String, + cacheControl: CacheControl = DEFAULT_CACHE_CONTROL, + ensureSuccess: Boolean = true, +): Response = head(url, source.headers, cacheControl, ensureSuccess) + +/** + * Executes a HEAD request asynchronously, automatically retrieving the + * headers from the current [HttpSource] context receiver. + * + * @param url The [HttpUrl] to request. + * @param cacheControl The cache control settings for the request. + * @param ensureSuccess If true, throws an exception if the response code is not 2xx. + * @return The HTTP [Response]. + */ +context(source: HttpSource) +suspend fun OkHttpClient.head( + url: HttpUrl, + cacheControl: CacheControl = DEFAULT_CACHE_CONTROL, + ensureSuccess: Boolean = true, +): Response = head(url, source.headers, cacheControl, ensureSuccess) diff --git a/core/src/main/kotlin/keiyoushi/network/RateLimit.kt b/core/src/main/kotlin/keiyoushi/network/RateLimit.kt index 44fd7949a..8ddb876dd 100644 --- a/core/src/main/kotlin/keiyoushi/network/RateLimit.kt +++ b/core/src/main/kotlin/keiyoushi/network/RateLimit.kt @@ -69,6 +69,18 @@ fun OkHttpClient.Builder.rateLimit( shouldLimit: (HttpUrl) -> Boolean = { true }, ): RateLimitBuilder = RateLimitBuilder(this, listOf(RateLimitRule(permits, period, interval, shouldLimit))) +/** + * Convenience for a [eu.kanade.tachiyomi.source.RateLimited]-declaring source's self-throttle: + * up to [permits] requests within any [minimumDelayMillis] window, sliding. This is the fallback + * that keeps working even if the host app's own rate limiting is missing or disabled - bundled + * into this module means it's dexed into the extension's own APK, not the app's. + * + * Uses [permits]/[period] rather than [interval] deliberately: an [interval] floor would force a + * minimum gap between *every* dispatch regardless of the permits budget, which would silently + * defeat any permits value above 1 (every request would still wait out the full interval). + */ +fun OkHttpClient.Builder.rateLimit(minimumDelayMillis: Long, permits: Int = 1): RateLimitBuilder = rateLimit(permits = permits, period = minimumDelayMillis.milliseconds) + internal class RateLimitRule( val permits: Int, val period: Duration, diff --git a/core/src/main/kotlin/keiyoushi/source/CustomUrlPreferences.kt b/core/src/main/kotlin/keiyoushi/source/CustomUrlPreferences.kt new file mode 100644 index 000000000..46e5ae9b5 --- /dev/null +++ b/core/src/main/kotlin/keiyoushi/source/CustomUrlPreferences.kt @@ -0,0 +1,89 @@ +package keiyoushi.source + +import android.content.SharedPreferences +import android.text.Editable +import android.text.TextWatcher +import android.widget.Button +import androidx.preference.EditTextPreference +import androidx.preference.PreferenceScreen +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull + +class CustomUrlPreferences( + private val preferences: SharedPreferences, + private val defaultUrl: String, + private val title: String, + private val dialogMessage: String, +) { + private val prefBaseKey = "overrideBaseUrl" + private val prefDefaultKey = "defaultBaseUrl" + + init { + val storedDefault = preferences.getString(prefDefaultKey, null) + if (storedDefault != defaultUrl) { + preferences.edit() + .putString(prefBaseKey, defaultUrl) + .putString(prefDefaultKey, defaultUrl) + .apply() + } + } + + val baseUrl: String + get() = preferences.getString(prefBaseKey, defaultUrl) ?: defaultUrl + + fun setupPreferenceScreen(screen: PreferenceScreen) { + EditTextPreference(screen.context).apply { + key = prefBaseKey + title = this@CustomUrlPreferences.title + dialogTitle = this@CustomUrlPreferences.title + dialogMessage = this@CustomUrlPreferences.dialogMessage + + val currentValue = preferences.getString(prefBaseKey, null) + summary = currentValue?.takeIf { it.isNotBlank() } ?: defaultUrl + setDefaultValue(defaultUrl) + + setOnBindEditTextListener { editText -> + editText.hint = defaultUrl + editText.setHorizontallyScrolling(true) + editText.post { editText.selectAll() } + editText.addTextChangedListener( + object : TextWatcher { + override fun afterTextChanged(editable: Editable?) { + val text = editable?.toString() ?: "" + val isValid = text.isBlank() || text.toHttpUrlOrNull() != null + editText.rootView.findViewById