From 02e4534f2018dfe8cf142690871a50d442f8b942 Mon Sep 17 00:00:00 2001 From: Origo Date: Sun, 12 Jul 2026 14:44:13 +0800 Subject: [PATCH] Stabilize ORSP and add community source registry --- .github/ISSUE_TEMPLATE/source-report.yml | 36 ++ .../source-submission.md | 27 + .github/workflows/registry-health.yml | 41 ++ .github/workflows/registry-pr-check.yml | 62 +++ .github/workflows/validate.yml | 27 +- .gitignore | 3 + CONTRIBUTING.md | 67 ++- ECOSYSTEM.md | 76 +++ GOVERNANCE.md | 81 +++ README.es.md | 10 +- README.ja.md | 10 +- README.ko.md | 10 +- README.md | 42 +- README.zh-CN.md | 40 +- README.zh-TW.md | 10 +- SECURITY.md | 20 +- SPECIFICATION.md | 320 ++++++++--- examples/dart_server.dart | 205 +++++-- openapi.yaml | 170 +++++- registry/POLICY.md | 60 ++ registry/README.md | 84 +++ registry/examples/source-entry.json | 21 + registry/generated/registry.json | 4 + registry/removed/.gitkeep | 1 + registry/schemas/registry.schema.json | 19 + registry/schemas/source-entry.schema.json | 122 +++++ registry/sources/.gitkeep | 1 + registry/status/.gitkeep | 1 + schemas/discovery.schema.json | 32 +- scripts/registry_tool.py | 512 ++++++++++++++++++ scripts/test_registry_tool.py | 75 +++ scripts/validate_contracts.py | 63 +++ tool/conformance_test.dart | 180 ++++++ 33 files changed, 2231 insertions(+), 201 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/source-report.yml create mode 100644 .github/PULL_REQUEST_TEMPLATE/source-submission.md create mode 100644 .github/workflows/registry-health.yml create mode 100644 .github/workflows/registry-pr-check.yml create mode 100644 ECOSYSTEM.md create mode 100644 GOVERNANCE.md create mode 100644 registry/POLICY.md create mode 100644 registry/README.md create mode 100644 registry/examples/source-entry.json create mode 100644 registry/generated/registry.json create mode 100644 registry/removed/.gitkeep create mode 100644 registry/schemas/registry.schema.json create mode 100644 registry/schemas/source-entry.schema.json create mode 100644 registry/sources/.gitkeep create mode 100644 registry/status/.gitkeep create mode 100644 scripts/registry_tool.py create mode 100644 scripts/test_registry_tool.py create mode 100644 scripts/validate_contracts.py create mode 100644 tool/conformance_test.dart diff --git a/.github/ISSUE_TEMPLATE/source-report.yml b/.github/ISSUE_TEMPLATE/source-report.yml new file mode 100644 index 0000000..9f005f1 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/source-report.yml @@ -0,0 +1,36 @@ +name: Report a registry source +description: Report availability, security, maintenance, abuse, or rights concerns +title: "[Source report]: " +labels: [registry, source-report] +body: + - type: input + id: source_id + attributes: + label: Source ID + placeholder: org.example.public-books + validations: + required: true + - type: dropdown + id: category + attributes: + label: Category + options: + - Availability or compatibility + - Security or malicious behavior + - Maintainer or ownership + - Copyright or content rights + - Incorrect registry metadata + validations: + required: true + - type: textarea + id: evidence + attributes: + label: Evidence + description: Include dates, requests, responses, and public references. Do not post secrets. + validations: + required: true + - type: textarea + id: requested_action + attributes: + label: Requested action + description: Explain whether you are requesting correction, warning, suspension, or removal. diff --git a/.github/PULL_REQUEST_TEMPLATE/source-submission.md b/.github/PULL_REQUEST_TEMPLATE/source-submission.md new file mode 100644 index 0000000..0e7a2d7 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE/source-submission.md @@ -0,0 +1,27 @@ +## ORSP source submission + +- Source ID: +- Discovery URL: +- Source homepage: +- Source repository, if public: +- Maintainer GitHub account(s): + +## Content declaration + +- [ ] The source contains only original, public-domain, or properly licensed content. +- [ ] I operate this source or am authorized by its operator to submit it. +- [ ] A public reporting or contact path is available where practical. + +## Compatibility and safety + +- [ ] The source implements discovery, search, detail, catalog, and content. +- [ ] The manifest filename exactly matches the discovery source ID. +- [ ] `testQuery` returns at least one readable book and chapter. +- [ ] The source does not return credentials, cookies, executable rules, or private infrastructure details. +- [ ] I ran `python scripts/registry_tool.py validate`. +- [ ] I ran `python scripts/registry_tool.py build --check`. +- [ ] I ran `python scripts/registry_tool.py check-source registry/sources/.json`. + +## Additional context + +Describe licensing, hosting, regional availability, rate limits, or other facts reviewers should know. diff --git a/.github/workflows/registry-health.yml b/.github/workflows/registry-health.yml new file mode 100644 index 0000000..928e32d --- /dev/null +++ b/.github/workflows/registry-health.yml @@ -0,0 +1,41 @@ +name: Check community registry health + +on: + workflow_dispatch: + schedule: + - cron: '23 3 * * *' + +permissions: + contents: read + +jobs: + health: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.x' + - name: Install registry validator + run: python -m pip install --disable-pip-version-check jsonschema + - name: Validate registry before network access + run: | + python scripts/registry_tool.py validate + python scripts/registry_tool.py build --check + - name: Check public sources + id: health + continue-on-error: true + run: >- + python scripts/registry_tool.py check-all + --status-output registry-health.json + - name: Upload health report + if: always() + uses: actions/upload-artifact@v4 + with: + name: orsp-registry-health + path: registry-health.json + if-no-files-found: error + retention-days: 30 + - name: Fail after preserving the report + if: steps.health.outcome == 'failure' + run: exit 1 diff --git a/.github/workflows/registry-pr-check.yml b/.github/workflows/registry-pr-check.yml new file mode 100644 index 0000000..f8ea557 --- /dev/null +++ b/.github/workflows/registry-pr-check.yml @@ -0,0 +1,62 @@ +name: Check submitted registry sources + +on: + pull_request: + paths: + - 'registry/sources/**' + +permissions: + contents: read + +jobs: + network-check: + runs-on: ubuntu-latest + steps: + - name: Check out trusted validator from the base revision + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.base.sha }} + path: validator + persist-credentials: false + - name: Check out submitted manifests without credentials + uses: actions/checkout@v4 + with: + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.sha }} + path: submission + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: '3.x' + - name: Install trusted validator dependencies + run: python -m pip install --disable-pip-version-check jsonschema + - name: Identify submitted source manifests + run: | + : > submitted-sources.txt + shopt -s dotglob nullglob + for path in submission/registry/sources/*; do + file="${path#submission/}" + if [ "$file" = "registry/sources/.gitkeep" ]; then + continue + fi + if [ -L "$path" ] || [ ! -f "$path" ]; then + echo "Registry source must be a regular file: $file" >&2 + exit 1 + fi + if [[ ! "$file" =~ ^registry/sources/[A-Za-z0-9._-]+\.json$ ]]; then + echo "Invalid registry source path: $file" >&2 + exit 1 + fi + if [ ! -f "validator/$file" ] || ! cmp -s "validator/$file" "$path"; then + echo "$file" >> submitted-sources.txt + fi + done + if [ ! -s submitted-sources.txt ]; then + echo "No added or modified source manifest requires a network check." + fi + - name: Run trusted network checks without repository credentials + run: | + while IFS= read -r file; do + [ -z "$file" ] && continue + python validator/scripts/registry_tool.py check-source "submission/$file" + done < submitted-sources.txt diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index ee3279e..e642193 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -16,15 +16,22 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.x' + - name: Install contract validators + run: >- + python -m pip install --disable-pip-version-check + jsonschema pyyaml openapi-spec-validator - name: Check Dart formatting - run: dart format --output=none --set-exit-if-changed examples/dart_server.dart - - name: Analyze reference server - run: dart analyze examples/dart_server.dart - - name: Validate JSON documents + run: >- + dart format --output=none --set-exit-if-changed + examples/dart_server.dart tool/conformance_test.dart + - name: Analyze Dart implementation + run: dart analyze examples/dart_server.dart tool/conformance_test.dart + - name: Validate schemas, examples, and OpenAPI + run: python scripts/validate_contracts.py + - name: Validate community registry manifests run: | - python -m json.tool schemas/discovery.schema.json > /dev/null - python -m json.tool examples/open-reading-source.json > /dev/null - - name: Validate OpenAPI YAML syntax - run: | - python -m pip install --disable-pip-version-check pyyaml - python -c "import pathlib, yaml; yaml.safe_load(pathlib.Path('openapi.yaml').read_text())" + python scripts/registry_tool.py validate + python scripts/registry_tool.py build --check + python scripts/test_registry_tool.py + - name: Run core conformance tests + run: dart run tool/conformance_test.dart diff --git a/.gitignore b/.gitignore index 360f426..3fb4c66 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,7 @@ .dart_tool/ +.omx/ +__pycache__/ +*.pyc .idea/ .vscode/ build/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 194bdea..d8e083c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,35 +2,68 @@ Thank you for helping improve Open Reading Source Protocol. +Read [GOVERNANCE.md](GOVERNANCE.md) before proposing a protocol change. ORSP +optimizes for interoperability across independently developed readers and +sources, so a useful feature is not automatically suitable for the core. + +## Implementation feedback + +Implementation feedback is especially valuable. Please include: + +1. whether you implemented a source, reader, adapter, registry, or test tool; +2. the language and framework used; +3. the protocol version and capabilities tested; +4. behavior that was ambiguous or differed between implementations; +5. a minimal request and response example where possible. + ## Proposing a protocol change -Open an issue describing: +Open a protocol issue describing: 1. the source-developer or reader problem; -2. why the current protocol cannot solve it; -3. the proposed request or response shape; -4. backward-compatibility impact; -5. security, privacy, and copyright considerations. +2. why the current protocol or a namespaced extension cannot solve it; +3. the proposed request, response, and capability shape; +4. backward- and forward-compatibility impact; +5. security, privacy, accessibility, and copyright considerations; +6. migration and deprecation behavior; +7. implementation and test evidence. -Protocol changes should update the specification, OpenAPI document, relevant -JSON Schemas, and at least one example. +Protocol changes must update the specification, OpenAPI document, relevant +JSON Schemas, examples, and conformance tests in the same pull request. -## Compatibility +## Compatibility rules -- Additive optional fields can usually remain within the current major version. -- Changing required fields, endpoint meaning, or response semantics requires a - new major protocol version. -- Implementations must ignore unknown discovery fields unless a future - capability explicitly says otherwise. +- Minor releases may add optional fields and optional behavior only. +- Existing field and endpoint meanings cannot change within a major version. +- Implementations must ignore unknown JSON object members at every nesting + level. +- Implementations must ignore unknown optional capability tokens. +- Extension capabilities use `reverse.domain:name` and cannot use the reserved + `orsp` namespace. +- Missing and `null` are distinct; null is invalid unless explicitly declared. +- Incompatible changes require a new major protocol version and migration + documentation. ## Local checks ```bash -dart format --output=none --set-exit-if-changed examples/dart_server.dart -dart analyze examples/dart_server.dart -python -m json.tool schemas/discovery.schema.json > /dev/null -python -m json.tool examples/open-reading-source.json > /dev/null +dart format --output=none --set-exit-if-changed \ + examples/dart_server.dart tool/conformance_test.dart +dart analyze examples/dart_server.dart tool/conformance_test.dart +dart run tool/conformance_test.dart +python -m pip install jsonschema pyyaml openapi-spec-validator +python scripts/validate_contracts.py +python scripts/registry_tool.py validate +python scripts/registry_tool.py build --check +python scripts/test_registry_tool.py ``` +The CI workflow runs the same checks. JSON or YAML syntax validation alone is +not considered protocol conformance. + +Community source submissions follow [registry/README.md](registry/README.md) +and should use the source-submission pull-request template. A registry listing +is a separate decision from changing the ORSP protocol. + By contributing, you agree that your contribution is licensed under the MIT License used by this repository. diff --git a/ECOSYSTEM.md b/ECOSYSTEM.md new file mode 100644 index 0000000..5981da1 --- /dev/null +++ b/ECOSYSTEM.md @@ -0,0 +1,76 @@ +# ORSP Ecosystem Architecture + +ORSP separates three concerns so the community can grow without making readers +unsafe or the wire protocol unstable. + +## 1. Wire protocol + +The ORSP core is the HTTP contract implemented by readers and sources. It +defines data and behavior, not scraping programs. A reader should be able to +connect to any conforming source without source-specific code. + +## 2. Server-side adapters + +An adapter translates an existing authorized content service into ORSP. It may +use HTML parsing, vendor APIs, databases, or other source-specific logic on the +server side. Credentials and executable rules remain under the adapter +operator's control and are never sent to readers. + +Future adapter SDKs should provide: + +- typed ORSP response models; +- request validation and pagination helpers; +- caching, rate limiting, and safe upstream-fetch defaults; +- HTML sanitization utilities; +- conformance-test integration; +- migration guides for existing book-source formats. + +Adapter compatibility is separate from the ORSP wire version. A legacy format +can evolve or disappear without forcing every reader to change. + +## 3. Federated registries + +A registry helps users discover sources. It is not the protocol authority and +does not need to be unique. Readers may subscribe to multiple community, +regional, organizational, or private registries. + +A future registry companion specification should describe signed or +integrity-protected entries containing at least: + +- source ID and discovery URL; +- operator and maintainer information; +- supported languages and capabilities; +- last successful conformance result and test-suite version; +- health-check time without exposing private infrastructure; +- content-rights declaration, reporting contact, and removal status; +- optional reputation and community moderation metadata. + +Registry entries must not duplicate the complete discovery document as a new +source of truth. Readers should fetch discovery from the source and compare the +stable source ID. + +This repository now contains an initial PR-based implementation under +[`registry/`](registry/README.md). It is intentionally a community registry +prototype rather than the final companion specification. Operational evidence +from this implementation should guide the federated format. + +## Trust model + +Protocol conformance proves interoperability, not legality, content quality, or +operator trustworthiness. Registries should present these as separate signals. +A source can be technically conforming and still be unsuitable for a registry. + +Readers should clearly show source identity, origin, permissions, security +warnings, and removal information. Private-network sources should require an +explicit client policy. + +## Roadmap + +1. Stabilize ORSP 1.0 and publish a reusable core conformance kit. +2. Build adapter SDKs in at least two server ecosystems. +3. Test multiple independent readers and sources and publish a compatibility + matrix. +4. Draft a federated registry companion specification using real operational + experience. +5. Standardize additional capabilities only after extension implementations + demonstrate compatible semantics. diff --git a/GOVERNANCE.md b/GOVERNANCE.md new file mode 100644 index 0000000..43dfecc --- /dev/null +++ b/GOVERNANCE.md @@ -0,0 +1,81 @@ +# ORSP Governance + +ORSP is developed as an open interoperability standard. The protocol belongs +to its implementers and users rather than to one reader, source, registry, or +company. + +## Principles + +1. **Interoperability before convenience.** A feature must behave consistently + across common client and server languages. +2. **Small stable core.** Optional or experimental behavior begins as a + namespaced extension instead of immediately expanding the core. +3. **Evidence before standardization.** Stable requirements come from working + independent implementations and conformance tests. +4. **Safe readers.** ORSP data is untrusted; the protocol does not distribute + executable scraping rules, credentials, or cookies to clients. +5. **Federation before gatekeeping.** Registries may be operated by many + communities and must not redefine the wire protocol. +6. **Lawful distribution.** The protocol does not grant copyright or access + rights, and governance must include practical reporting and removal paths. + +## Normative sources + +`SPECIFICATION.md` is the normative human-readable contract. The JSON Schemas, +OpenAPI document, and conformance suite are normative where they encode that +contract. If they conflict, the conflict is a protocol bug and must be fixed; +implementers should not silently choose one interpretation. + +README files and translations are informative. A translation must identify the +protocol version it describes and must not introduce new requirements. + +## Change process + +Substantial protocol work uses an ORSP Enhancement Proposal (OEP) issue or +document with one of these states: + +- **Problem:** use cases and interoperability evidence are being collected. +- **Draft:** a concrete contract and compatibility analysis exist. +- **Experimental:** the proposal has a namespaced capability and working code. +- **Candidate:** semantics are frozen and conformance tests exist. +- **Accepted:** the proposal is included in an ORSP release. +- **Deferred:** useful but not ready or not appropriate for the current scope. +- **Rejected:** incompatible with ORSP goals or superseded by another design. +- **Deprecated:** supported for migration but no longer recommended. + +Repository maintainers seek public consensus and record the reason for a +decision. When consensus cannot be reached, maintainers may defer a proposal +rather than prematurely standardize competing behavior. + +## Acceptance gates + +A new stable core feature should have: + +- at least two independent source implementations; +- at least two independent reader implementations; +- implementations spanning at least two programming-language ecosystems; +- machine-readable schemas where applicable; +- positive, negative, compatibility, and security conformance tests; +- documented fallback behavior for older clients; +- migration guidance and an ecosystem-impact summary. + +An urgent security correction may be accepted before all gates are met, but the +exception and follow-up work must be documented. + +## Releases and compatibility + +Protocol releases use `MAJOR.MINOR`. A release candidate should be published +before a stable release so independent implementations can run the conformance +suite. + +Within a major version, existing required behavior cannot be removed or +reinterpreted. Deprecation warnings may be introduced in a minor release, but +removal requires a new major version. Test-suite releases have their own build +or package version and do not change the wire protocol version. + +## Security and conduct + +Security vulnerabilities follow [SECURITY.md](SECURITY.md) and should not be +disclosed in public protocol issues before a fix is available. Participation +must remain technical, respectful, and welcoming to implementers from different +regions, languages, platforms, and project sizes. diff --git a/README.es.md b/README.es.md index 4ccaafb..f55b2a6 100644 --- a/README.es.md +++ b/README.es.md @@ -24,10 +24,10 @@ GET /.well-known/open-reading-source.json También implementa cuatro operaciones principales: ```text -GET /v1/search?q={query}&page=1&pageSize=20 -GET /v1/books/{bookId} -GET /v1/books/{bookId}/chapters -GET /v1/books/{bookId}/chapters/{chapterId} +GET {apiBaseUrl}v1/search?q={query}&page=1&pageSize=20 +GET {apiBaseUrl}v1/books/{bookId} +GET {apiBaseUrl}v1/books/{bookId}/chapters?page=1&pageSize=100 +GET {apiBaseUrl}v1/books/{bookId}/chapters/{chapterId} ``` Consulta la [especificación](SPECIFICATION.md) y la definición @@ -57,7 +57,7 @@ emulador de Android. ## Estado y uso responsable -La versión `1.0` es un borrador público inicial. El flujo básico de lectura ya +La versión `1.0` es un borrador de estabilización. El flujo básico de lectura ya ha sido verificado, pero se aceptan propuestas sobre autenticación, sincronización, paginación y formatos de contenido antes de declarar una versión estable. diff --git a/README.ja.md b/README.ja.md index ad96f4a..a7788cb 100644 --- a/README.ja.md +++ b/README.ja.md @@ -23,10 +23,10 @@ GET /.well-known/open-reading-source.json さらに、4 つの基本エンドポイントを実装します。 ```text -GET /v1/search?q={query}&page=1&pageSize=20 -GET /v1/books/{bookId} -GET /v1/books/{bookId}/chapters -GET /v1/books/{bookId}/chapters/{chapterId} +GET {apiBaseUrl}v1/search?q={query}&page=1&pageSize=20 +GET {apiBaseUrl}v1/books/{bookId} +GET {apiBaseUrl}v1/books/{bookId}/chapters?page=1&pageSize=100 +GET {apiBaseUrl}v1/books/{bookId}/chapters/{chapterId} ``` フィールドとエラー処理の詳細は、[仕様書](SPECIFICATION.md)および @@ -55,7 +55,7 @@ PC では `http://127.0.0.1:8787`、Android エミュレーターでは ## ステータスと責任ある利用 -バージョン `1.0` は初期公開ドラフトです。基本的な読書フローは検証済みですが、安定版の +バージョン `1.0` は安定化ドラフトです。基本的な読書フローは検証済みですが、安定版の 公開前に、認証、同期、ページング、コンテンツ形式についてのフィードバックを歓迎します。 ORSP は、オリジナル、パブリックドメイン、または正規に許諾されたコンテンツを対象とします。 diff --git a/README.ko.md b/README.ko.md index 89be94b..12f6eeb 100644 --- a/README.ko.md +++ b/README.ko.md @@ -22,10 +22,10 @@ GET /.well-known/open-reading-source.json 그리고 네 개의 핵심 엔드포인트를 구현합니다. ```text -GET /v1/search?q={query}&page=1&pageSize=20 -GET /v1/books/{bookId} -GET /v1/books/{bookId}/chapters -GET /v1/books/{bookId}/chapters/{chapterId} +GET {apiBaseUrl}v1/search?q={query}&page=1&pageSize=20 +GET {apiBaseUrl}v1/books/{bookId} +GET {apiBaseUrl}v1/books/{bookId}/chapters?page=1&pageSize=100 +GET {apiBaseUrl}v1/books/{bookId}/chapters/{chapterId} ``` 필드와 오류 처리에 대한 자세한 내용은 [프로토콜 명세](SPECIFICATION.md)와 @@ -54,7 +54,7 @@ PC에서는 `http://127.0.0.1:8787`, Android 에뮬레이터에서는 ## 상태 및 책임 있는 사용 -프로토콜 `1.0`은 초기 공개 초안입니다. 핵심 읽기 흐름은 검증되었지만 안정 버전 발표 전까지 +프로토콜 `1.0`은 안정화 초안입니다. 핵심 읽기 흐름은 검증되었지만 안정 버전 발표 전까지 인증, 동기화, 페이지 처리 및 콘텐츠 형식에 대한 의견을 환영합니다. ORSP는 창작물, 퍼블릭 도메인 및 정식 허가를 받은 콘텐츠를 위한 것입니다. 접근 제어를 diff --git a/README.md b/README.md index ffeedc1..57f5d9e 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,9 @@ **English** · [简体中文](README.zh-CN.md) · [繁體中文](README.zh-TW.md) · [日本語](README.ja.md) · [한국어](README.ko.md) · [Español](README.es.md) · -[Specification](SPECIFICATION.md) · [OpenAPI](openapi.yaml) +[Specification](SPECIFICATION.md) · [OpenAPI](openapi.yaml) · +[Governance](GOVERNANCE.md) · [Ecosystem](ECOSYSTEM.md) · +[Community Registry](registry/README.md) Open Reading Source Protocol (ORSP) is an open HTTP contract for connecting reading applications to independent book and serial-content providers. @@ -26,12 +28,15 @@ GET /.well-known/open-reading-source.json It then implements four standard API operations: ```text -GET /v1/search?q={query}&page=1&pageSize=20 -GET /v1/books/{bookId} -GET /v1/books/{bookId}/chapters -GET /v1/books/{bookId}/chapters/{chapterId} +GET {apiBaseUrl}v1/search?q={query}&page=1&pageSize=20 +GET {apiBaseUrl}v1/books/{bookId} +GET {apiBaseUrl}v1/books/{bookId}/chapters?page=1&pageSize=100 +GET {apiBaseUrl}v1/books/{bookId}/chapters/{chapterId} ``` +These endpoint relations are resolved against the discovered `apiBaseUrl`. +ORSP 1.0 Core Reading sources implement all four capabilities. + Example discovery document: ```json @@ -66,13 +71,34 @@ SPECIFICATION.md Normative protocol specification openapi.yaml OpenAPI 3.1 API description schemas/discovery.schema.json Discovery-document JSON Schema examples/dart_server.dart Runnable reference source +tool/conformance_test.dart End-to-end core conformance checks +GOVERNANCE.md Protocol change and release process +ECOSYSTEM.md Adapter and federated registry architecture +registry/ PR-based community source registry ``` ## Status -Protocol version `1.0` is an early public draft. The discovery format and core -read path are implemented and tested in Open Reading, but feedback from source -developers is welcome before the specification is declared stable. +Protocol version `1.0` is a stabilization draft. Its compatibility rules, +machine-readable contracts, reference source, and core conformance tests are +now developed together. Independent reader and source implementations are +welcome before the specification is declared stable. + +Run the local conformance checks with: + +```bash +dart run tool/conformance_test.dart +``` + +Traditional book-source formats belong in server-side ORSP adapters rather than +inside readers. Source discovery communities can operate multiple federated +registries; registries do not redefine the wire protocol. See +[ECOSYSTEM.md](ECOSYSTEM.md). + +Source operators can submit an ORSP service through a pull request to the +[community registry](registry/README.md). Each source uses one independently +validated manifest file, and the combined reader-facing index is generated +automatically. ## Responsible use diff --git a/README.zh-CN.md b/README.zh-CN.md index 3805a03..a415801 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -2,7 +2,9 @@ [English](README.md) · **简体中文** · [繁體中文](README.zh-TW.md) · [日本語](README.ja.md) · [한국어](README.ko.md) · [Español](README.es.md) · -[协议规范](SPECIFICATION.md) · [OpenAPI](openapi.yaml) +[协议规范](SPECIFICATION.md) · [OpenAPI](openapi.yaml) · +[治理机制](GOVERNANCE.md) · [生态架构](ECOSYSTEM.md) · +[社区书源目录](registry/README.md) Open Reading Source Protocol（ORSP，开放阅读书源协议）是一套用于连接阅读应用与 独立内容服务的开放 HTTP 协议。它让不同书源以统一方式提供搜索、书籍详情、章节目录 @@ -53,13 +55,14 @@ GET /.well-known/open-reading-source.json 协议 1.0 定义四类核心接口： ```text -GET /v1/search?q={query}&page=1&pageSize=20 -GET /v1/books/{bookId} -GET /v1/books/{bookId}/chapters -GET /v1/books/{bookId}/chapters/{chapterId} +GET {apiBaseUrl}v1/search?q={query}&page=1&pageSize=20 +GET {apiBaseUrl}v1/books/{bookId} +GET {apiBaseUrl}v1/books/{bookId}/chapters?page=1&pageSize=100 +GET {apiBaseUrl}v1/books/{bookId}/chapters/{chapterId} ``` -对应能力分别是搜索、书籍详情、章节目录和章节正文。完整字段与错误处理要求请阅读 +这些相对路径必须基于发现文档中的 `apiBaseUrl` 解析。ORSP 1.0 Core Reading 书源必须 +实现全部四项能力。完整字段与错误处理要求请阅读 [协议规范](SPECIFICATION.md)，接口模型可查看 [OpenAPI 3.1 文档](openapi.yaml)。 ## 快速运行测试书源 @@ -81,7 +84,7 @@ dart run examples/dart_server.dart ```bash curl http://127.0.0.1:8787/.well-known/open-reading-source.json -curl "http://127.0.0.1:8787/v1/search?q=星&page=1&pageSize=20" +curl "http://127.0.0.1:8787/api/v1/search?q=协议&page=1&pageSize=20" ``` ## 开发自己的书源 @@ -107,17 +110,34 @@ openapi.yaml OpenAPI 3.1 接口定义 schemas/discovery.schema.json 发现文档 JSON Schema examples/dart_server.dart 可运行的 Dart 参考书源 examples/open-reading-source.json 发现文档示例 +tool/conformance_test.dart 端到端核心一致性测试 +GOVERNANCE.md 协议变更与发布治理 +ECOSYSTEM.md 适配器与联邦书源目录架构 +registry/ 基于 PR 的社区书源目录 ``` ## 当前状态与兼容性 -协议版本 `1.0` 目前是早期公开草案。发现、搜索、详情、目录和正文链路已经完成真实 HTTP -验证，但在宣布稳定版本之前，仍欢迎书源开发者对认证、更新同步、分页、内容格式和错误 -处理提出建议。 +协议版本 `1.0` 目前是稳定化草案。兼容性规则、机器可读契约、参考书源和核心一致性测试 +会同步演进。在宣布稳定版本之前，欢迎独立阅读器和书源实现参与互操作验证。 + +可以运行官方核心测试： + +```bash +dart run tool/conformance_test.dart +``` 同一主版本内应保持向后兼容。新增可选字段通常可以继续使用 1.x；修改必填字段、接口 含义或响应语义则需要新的主版本。 +传统书源格式应通过服务端适配器转换成 ORSP，而不是把抓取脚本下发给阅读器。社区可以 +运营多个联邦式书源目录，目录负责发现、健康状态和信任信息，但不能重新定义通信协议。 +详细边界参见 [ECOSYSTEM.md](ECOSYSTEM.md)。 + +书源维护者可以通过 Pull Request 加入[社区书源目录](registry/README.md)。每个书源使用 +一个独立 Manifest 文件，CI 会检查格式、ID、公网地址安全和完整阅读链路，然后生成供 +阅读器订阅的统一索引。 + ## 合法与负责任地使用 ORSP 面向原创内容、公共领域内容以及已获得合法授权的内容。请勿使用本协议绕过访问 diff --git a/README.zh-TW.md b/README.zh-TW.md index 79ca8fb..5a53171 100644 --- a/README.zh-TW.md +++ b/README.zh-TW.md @@ -31,10 +31,10 @@ GET /.well-known/open-reading-source.json 並實作四個核心端點： ```text -GET /v1/search?q={query}&page=1&pageSize=20 -GET /v1/books/{bookId} -GET /v1/books/{bookId}/chapters -GET /v1/books/{bookId}/chapters/{chapterId} +GET {apiBaseUrl}v1/search?q={query}&page=1&pageSize=20 +GET {apiBaseUrl}v1/books/{bookId} +GET {apiBaseUrl}v1/books/{bookId}/chapters?page=1&pageSize=100 +GET {apiBaseUrl}v1/books/{bookId}/chapters/{chapterId} ``` 完整欄位及錯誤處理請參閱[協議規範](SPECIFICATION.md)與 @@ -63,7 +63,7 @@ dart run examples/dart_server.dart ## 狀態與使用邊界 -協議 `1.0` 目前是早期公開草案。核心讀取流程已完成驗證，但在穩定版本發布前仍歡迎 +協議 `1.0` 目前是穩定化草案。核心讀取流程已完成驗證，但在穩定版本發布前仍歡迎 對認證、同步、分頁及內容格式提出建議。 ORSP 僅適用於原創、公共領域或合法授權內容。請勿用於繞過存取控制、違反服務條款或 diff --git a/SECURITY.md b/SECURITY.md index 674f7c8..3f44415 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -5,8 +5,20 @@ data, credentials, private source URLs, or content-provider infrastructure. Report security problems privately through GitHub's security advisory feature for this repository. Include reproduction steps, affected protocol version, -and the expected impact. +expected impact, and whether the issue affects readers, sources, adapters, or +registries. -ORSP source operators should use HTTPS in production, validate all opaque IDs -and query parameters, sanitize HTML, set response-size limits, and avoid -returning cookies, tokens, internal URLs, or executable scripts. +Source operators should use HTTPS in production, validate all IDs and known +query parameters, sanitize HTML, set response-size and execution-time limits, +and avoid returning cookies, tokens, internal URLs, stack traces, or executable +content. + +Readers should treat discovery URLs and all source responses as untrusted. They +should defend against server-side request forgery, DNS rebinding, loopback and +private-network access unless explicitly allowed, redirect loops, TLS +downgrades, decompression bombs, oversized images, unsafe HTML, and dangerous +URL schemes. + +Adapters should isolate source-specific credentials and parsing logic from the +public ORSP response. Registries should avoid publishing private source +locations or operational details that increase attack surface. diff --git a/SPECIFICATION.md b/SPECIFICATION.md index 656fbd1..fdff2ef 100644 --- a/SPECIFICATION.md +++ b/SPECIFICATION.md @@ -1,64 +1,185 @@ # Open Reading Source Protocol 1.0 -Status: **Early public draft** +Status: **Stabilization draft** The key words MUST, MUST NOT, REQUIRED, SHOULD, SHOULD NOT, and MAY in this -document are to be interpreted as described in RFC 2119 and RFC 8174. +document are to be interpreted as described in RFC 2119 and RFC 8174 when, and +only when, they appear in all capitals. -## 1. Scope +## 1. Scope and design goals -ORSP defines discovery, search, book metadata, chapter catalogs, and chapter -content retrieval. Version 1.0 covers public HTTP(S) sources that do not require -user authentication. +Open Reading Source Protocol (ORSP) defines a safe HTTP boundary between a +reading client and an independently operated content source. ORSP 1.0 covers: -## 2. Discovery +- source discovery; +- book search and metadata; +- chapter catalogs; +- chapter content retrieval. -A source MUST expose a JSON discovery document at: +ORSP 1.0 covers public HTTP(S) sources that do not require user authentication. +It does not define executable scraping rules. A source MAY use an authorized +adapter internally, but clients only receive data described by this protocol. + +The core reading profile is intentionally small. A conforming ORSP 1.0 source +MUST implement the complete discovery, search, detail, catalog, and content +path. Optional features are advertised as capabilities and MUST NOT change the +meaning of core fields or endpoints. + +## 2. Protocol versions and evolution + +Protocol versions use `MAJOR.MINOR`, for example `1.0`. They are not Semantic +Versioning package versions and MUST NOT include a patch component. + +- A major version may make incompatible changes. +- A minor version may add optional object members, optional capability tokens, + optional endpoints, or additional enum values with defined fallback behavior. +- A minor version MUST NOT remove or reinterpret an existing field, make an + optional field required, or change the meaning of an existing HTTP status. + +A client implementing ORSP 1.x SHOULD accept any discovery document with major +version `1`. This is safe only because all 1.x implementations MUST follow the +forward-compatibility rules in this document. + +Clients MUST ignore unknown JSON object members. Clients MUST ignore unknown +optional capability tokens. A source MUST NOT require an extension capability +for discovery, search, detail, catalog, or content in the core reading profile. + +Missing and `null` are different values. Sources MUST omit an unavailable +optional member. A source MUST NOT emit `null` unless the member's schema +explicitly permits it. + +## 3. Discovery + +A source MUST expose a JSON discovery document at the origin-root path: ```text /.well-known/open-reading-source.json ``` -Clients MAY also accept the complete discovery-document URL from a user. +Clients MAY also accept a complete discovery-document URL supplied by a user or +registry. Redirect handling is defined in Section 5. -Required fields: +Required discovery members: -| Field | Requirement | +| Member | Requirement | | --- | --- | | `protocol` | MUST equal `open-reading-source` | -| `protocolVersion` | Semantic version; this specification uses `1.0` | -| `id` | Stable globally unique source identifier; reverse-domain form is recommended | +| `protocolVersion` | ORSP `MAJOR.MINOR`; this specification uses `1.0` | +| `id` | Stable globally unique source identifier | | `name` | Human-readable source name | -| `apiBaseUrl` | Absolute HTTP(S) API base URL | -| `capabilities` | MUST contain `search` | +| `apiBaseUrl` | Absolute HTTP(S) URL ending in `/` | +| `capabilities` | MUST contain all four core capability tokens | + +Core capability tokens are `search`, `detail`, `catalog`, and `content`. -Optional fields are `description`, `iconUrl`, `websiteUrl`, and `languages`. +Extension capability tokens MUST use a collision-resistant namespace in the +form `reverse.domain:name`, for example `org.example:audio`. An extension token +MUST NOT use the `orsp` namespace unless it is defined by a future ORSP +specification. Unknown extension tokens do not make a source incompatible. -Clients implementing version 1.x SHOULD accept discovery documents whose major -version is `1`. A source MUST NOT change its `id` when its host or display name -changes. +Optional discovery members are `description`, `iconUrl`, `websiteUrl`, and +`languages`. Language tags MUST follow BCP 47, such as `en`, `zh-CN`, or +`sr-Latn`. + +`apiBaseUrl` MUST NOT contain user information, a query, or a fragment and MUST +end in `/`. Endpoint paths in this document are resolved relative to that URL. +For example, an API base URL of `https://books.example/api/` and endpoint +relation `v1/search` produce `https://books.example/api/v1/search`. + +The source `id` MUST be between 3 and 200 characters, start and end with an +ASCII letter or digit, and otherwise contain only ASCII letters, digits, `.`, +`_`, or `-`. It MUST remain unchanged when the source host, API URL, or display +name changes. Reverse-domain form is recommended. Source IDs identify a source +only; book and chapter IDs are scoped to that source. The discovery document is formally described by [`schemas/discovery.schema.json`](schemas/discovery.schema.json). -## 3. Common requirements +## 4. Data model and encoding + +### 4.1 JSON and text - Requests and responses MUST use UTF-8. -- JSON responses MUST use `application/json`. -- Clients SHOULD send `X-Open-Reading-Protocol: 1.0`. -- IDs are stable opaque strings. Clients MUST NOT infer their structure. -- Timestamps MUST use ISO 8601. -- Production deployments SHOULD use HTTPS. -- Sources SHOULD return an `ETag` or suitable cache headers where practical. -- Clients MUST treat source HTML as untrusted input. +- JSON responses MUST use `application/json`; media type parameters such as + `charset=utf-8` are allowed. +- JSON object member names are case-sensitive. +- Clients MUST ignore unknown object members at every nesting level. +- Sources SHOULD normalize human-readable text to Unicode NFC. +- Timestamps MUST be RFC 3339 `date-time` values. UTC with a `Z` suffix is + recommended. + +### 4.2 Identifiers + +Book and chapter IDs are stable opaque strings. To produce identical routing +behavior across common HTTP frameworks, wire IDs MUST contain only RFC 3986 +unreserved ASCII characters: letters, digits, `-`, `.`, `_`, and `~`. + +IDs MUST be between 1 and 200 characters. Clients MUST treat each ID as one URL +path segment and MUST NOT infer hierarchy or other meaning from it. A source +MUST NOT reuse an ID for a different logical resource. + +Chapter IDs are unique within a book. Book IDs are unique within a source. + +### 4.3 URLs + +JSON URL members MUST be absolute HTTP(S) URLs unless a field explicitly says +otherwise. Clients MUST reject unsupported URL schemes. Clients SHOULD apply +platform-appropriate limits before downloading covers or other remote assets. + +### 4.4 Enums + +For descriptive enums that include the value `unknown`, clients MUST map an +unrecognized value to `unknown`. For operational enums such as `contentType`, a +client MUST report that the representation is unsupported rather than guessing +how to interpret it. + +## 5. HTTP behavior + +- Core endpoints use `GET` and MUST be safe and idempotent. +- Clients SHOULD send `Accept: application/json`. +- Clients SHOULD send `X-Open-Reading-Protocol` with their highest supported + ORSP 1.x version, for example `1.0`. +- Sources MAY return `X-Open-Reading-Protocol` containing the source protocol + version. +- Unsupported methods MUST return `405 Method Not Allowed` and SHOULD include + an `Allow` header. +- Sources SHOULD support HTTP compression for large JSON responses. + +Clients MAY follow redirects, subject to all of the following: + +- no more than five redirects per request; +- redirect targets MUST use HTTP(S); +- production clients SHOULD NOT follow an HTTPS-to-HTTP downgrade; +- credentials or private headers MUST NOT be forwarded to a different origin. + +Clients SHOULD enforce configurable connection, response-time, decompressed +size, and redirect limits. A client MUST treat all source responses as +untrusted input. -## 4. Search +### 5.1 Caching + +Discovery, detail, catalog, and content responses SHOULD provide an `ETag` or +appropriate `Cache-Control`/`Last-Modified` headers. When a client sends +`If-None-Match` with a current entity tag, a source SHOULD return `304 Not +Modified` with no response body. + +An entity tag identifies the complete selected representation. Sources MUST +change it when a response changes in a way visible to a client. + +## 6. Search + +Endpoint relation: ```text -GET /v1/search?q={query}&page={page}&pageSize={pageSize} +GET v1/search?q={query}&page={page}&pageSize={pageSize} ``` -`q` is required. `page` starts at 1. `pageSize` SHOULD be between 1 and 100. +`q` is required after trimming surrounding whitespace and MUST contain between +1 and 200 Unicode scalar values. `page` starts at 1 and defaults to 1. +`pageSize` defaults to 20 and MUST be between 1 and 100. + +Sources MAY ignore unknown query parameters. Sources MUST return `400` for an +invalid known query parameter. Response: @@ -74,6 +195,7 @@ Response: "categories": ["Fiction"], "status": "completed", "latestChapter": "Chapter 30", + "language": "en", "updatedAt": "2026-07-11T10:00:00Z" } ], @@ -84,23 +206,37 @@ Response: } ``` -Book objects MUST include `id` and `title`. Other fields are optional. +`items`, `page`, `pageSize`, and `hasMore` are required. `pageSize` reports the +requested page capacity, not the number of returned items. `total` is optional +because some sources cannot calculate it efficiently. Search ranking is +source-defined but SHOULD be deterministic for an unchanged dataset and query. + +Book objects MUST include `id` and `title`. Other members are optional. +`language`, when present, MUST be a BCP 47 tag. `status` is one of `ongoing`, +`completed`, `hiatus`, or `unknown`. + +## 7. Book details -## 5. Book details +Endpoint relation: ```text -GET /v1/books/{bookId} +GET v1/books/{bookId} ``` The response uses the same book object as search and MAY provide a longer -description or richer metadata. +description or additional optional metadata. Existing member meanings MUST be +identical in search and detail responses. -## 6. Chapter catalog +## 8. Chapter catalog + +Endpoint relation: ```text -GET /v1/books/{bookId}/chapters +GET v1/books/{bookId}/chapters?page={page}&pageSize={pageSize} ``` +`page` defaults to 1. `pageSize` defaults to 100 and MUST be between 1 and 100. + ```json { "items": [ @@ -110,17 +246,29 @@ GET /v1/books/{bookId}/chapters "order": 1, "updatedAt": "2026-07-11T10:00:00Z" } - ] + ], + "page": 1, + "pageSize": 100, + "total": 1, + "hasMore": false } ``` Chapter objects MUST contain `id`, `title`, and `order`. `order` is a -zero-or-positive integer used for deterministic display ordering. +zero-or-positive integer. Sources SHOULD use unique order values within a book. +When order values are equal, clients MUST use the chapter ID as an ascending +secondary sort key so display order remains deterministic. -## 7. Chapter content +Catalog pages MUST be sorted by `(order, id)`. A source SHOULD return an `ETag` +for each catalog page. A client that observes a changed catalog validator while +loading multiple pages SHOULD restart catalog loading to avoid a mixed view. + +## 9. Chapter content + +Endpoint relation: ```text -GET /v1/books/{bookId}/chapters/{chapterId} +GET v1/books/{bookId}/chapters/{chapterId} ``` ```json @@ -129,55 +277,101 @@ GET /v1/books/{bookId}/chapters/{chapterId} "chapterId": "chapter-1", "title": "Chapter One", "contentType": "text/plain", - "content": "Chapter content" + "content": "Chapter content", + "baseUrl": "https://books.example.org/books/book-1001/" } ``` -`contentType` MUST be one of: +`bookId` and `chapterId` MUST exactly match the requested resource. `baseUrl` is +optional and, when present, MUST be an absolute HTTP(S) URL. + +ORSP 1.0 content types are: + +- `text/plain`; +- `text/markdown`, interpreted as CommonMark; +- `text/html`, containing an HTML fragment rather than a complete document. -- `text/plain` -- `text/markdown` -- `text/html` +Relative links and image URLs are resolved against `baseUrl` when provided, +otherwise against the final chapter request URL. -HTML MUST be a content fragment and MUST NOT contain executable scripts. -Clients SHOULD sanitize it before rendering. +HTML MUST NOT contain scripts or other executable content. Sources SHOULD +remove event-handler attributes, dangerous URL schemes, embedded frames, and +active form controls. Clients MUST still sanitize HTML before rendering it. +Raw HTML embedded in Markdown MUST receive the same treatment. -## 8. Errors +## 10. Errors -Recommended status codes: +Core status codes: | Status | Meaning | | --- | --- | -| `400` | Invalid request | +| `304` | Selected representation has not changed | +| `400` | Invalid request parameter or malformed ID | | `404` | Book, chapter, or route not found | -| `429` | Rate limited; `Retry-After` is recommended | +| `405` | HTTP method not supported | +| `429` | Rate limited; `Retry-After` SHOULD be included | | `500` | Internal source error | -| `503` | Source temporarily unavailable | +| `503` | Source temporarily unavailable; `Retry-After` MAY be included | -Recommended error body: +Except for responses that cannot contain a body, error responses MUST use: ```json { "error": { "code": "BOOK_NOT_FOUND", - "message": "The requested book does not exist" + "message": "The requested book does not exist", + "retryable": false, + "requestId": "request-7f3a" } } ``` -## 9. Security +`code` and `message` are required. Codes MUST use stable uppercase ASCII words +separated by underscores. `message` is intended for developers and diagnostics; +clients SHOULD NOT assume it is localized. `retryable` and `requestId` are +optional. Clients MUST ignore unknown error members and unknown error codes. + +Sources MUST NOT include credentials, cookies, internal URLs, stack traces, or +other sensitive implementation details in error responses. + +## 11. Security and privacy + +Sources MUST validate known query parameters and IDs and SHOULD impose rate, +response-size, execution-time, and upstream-fetch limits. Production sources +SHOULD use HTTPS. + +Clients accepting user- or registry-supplied source URLs SHOULD protect against +server-side request forgery, DNS rebinding, loopback/private-network access, +redirect loops, decompression bombs, and unexpectedly large images or content. +Whether private-network sources are allowed is a client policy decision and +SHOULD be explicit to the user. + +Web-facing sources are responsible for an appropriate CORS policy. CORS is not +required for native clients. + +ORSP does not grant copyright or access rights. Source operators are +responsible for distributing only content they are authorized to provide. + +## 12. Conformance + +An implementation claiming `ORSP 1.0 Core Reading` conformance MUST: + +1. implement discovery and all four core capabilities; +2. satisfy the normative requirements in this document; +3. pass the official core conformance suite for the claimed version; +4. disclose optional extension capabilities separately. -Sources MUST validate query parameters and opaque IDs, SHOULD impose response -size and rate limits, and MUST NOT expose credentials, cookies, internal URLs, -or executable code. Web-facing sources are responsible for an appropriate CORS -policy. +Passing syntax validation alone is not evidence of protocol conformance. +Conformance reports SHOULD record the test-suite version and execution date. -## 10. Future work +## 13. Reserved future work -The following features are intentionally outside version 1.0: +The following features are intentionally outside ORSP 1.0 and require a future +minor version, major version, or companion specification: - authentication and paid-content authorization; -- incremental catalog synchronization; +- cursor-based and incremental catalog synchronization; - offline package/download manifests; -- source health and conformance endpoints; +- source health and registry protocols; +- audio and other media representations; - annotations, highlights, and reading-progress synchronization. diff --git a/examples/dart_server.dart b/examples/dart_server.dart index 2b60710..091b825 100644 --- a/examples/dart_server.dart +++ b/examples/dart_server.dart @@ -2,6 +2,9 @@ import 'dart:async'; import 'dart:convert'; import 'dart:io'; +const protocolVersion = '1.0'; +final _opaqueIdPattern = RegExp(r'^[A-Za-z0-9._~-]{1,200}$'); + class ReferenceSourceServer { HttpServer? _server; @@ -22,37 +25,42 @@ class ReferenceSourceServer { return baseUri; } - Future close() async => _server?.close(force: true); + Future close() async { + await _server?.close(force: true); + _server = null; + } Future _serve(HttpServer server) async { await for (final request in server) { try { _handle(request); - } catch (error) { - _send(request.response, 500, { - 'error': {'code': 'INTERNAL_ERROR', 'message': '$error'}, - }); + } catch (_) { + _error(request, 500, 'INTERNAL_ERROR', retryable: true); } } } void _handle(HttpRequest request) { _cors(request.response); + request.response.headers.set('X-Open-Reading-Protocol', protocolVersion); + if (request.method == 'OPTIONS') { + request.response.statusCode = HttpStatus.noContent; request.response.close(); return; } if (request.method != 'GET') { - _error(request.response, 405, 'METHOD_NOT_ALLOWED'); + request.response.headers.set(HttpHeaders.allowHeader, 'GET, OPTIONS'); + _error(request, 405, 'METHOD_NOT_ALLOWED'); return; } final path = request.uri.path; final segments = request.uri.pathSegments; if (path == '/.well-known/open-reading-source.json') { - _send(request.response, 200, { + _sendJson(request, 200, { 'protocol': 'open-reading-source', - 'protocolVersion': '1.0', + 'protocolVersion': protocolVersion, 'id': 'dev.open-reading.reference-source', 'name': 'ORSP Reference Source', 'description': 'Original sample content for protocol development.', @@ -71,86 +79,203 @@ class ReferenceSourceServer { segments[1] == 'v1' && segments[2] == 'books') { final bookId = segments[3]; + if (!_validId(bookId)) { + _error(request, 400, 'INVALID_BOOK_ID'); + return; + } final book = books[bookId]; if (book == null) { - _error(request.response, 404, 'BOOK_NOT_FOUND'); + _error(request, 404, 'BOOK_NOT_FOUND'); return; } if (segments.length == 4) { - _send(request.response, 200, book.metadata); + _sendJson(request, 200, book.metadata); return; } if (segments.length == 5 && segments[4] == 'chapters') { - _send(request.response, 200, { - 'items': book.chapters.map((chapter) => chapter.metadata).toList(), - }); + _catalog(request, book); return; } if (segments.length == 6 && segments[4] == 'chapters') { + final chapterId = segments[5]; + if (!_validId(chapterId)) { + _error(request, 400, 'INVALID_CHAPTER_ID'); + return; + } final chapter = book.chapters - .where((item) => item.id == segments[5]) + .where((item) => item.id == chapterId) .firstOrNull; if (chapter == null) { - _error(request.response, 404, 'CHAPTER_NOT_FOUND'); + _error(request, 404, 'CHAPTER_NOT_FOUND'); return; } - _send(request.response, 200, { + _sendJson(request, 200, { 'bookId': bookId, 'chapterId': chapter.id, 'title': chapter.title, 'contentType': 'text/plain', 'content': chapter.content, + 'baseUrl': baseUri.resolve('books/$bookId/').toString(), }); return; } } - _error(request.response, 404, 'ROUTE_NOT_FOUND'); + _error(request, 404, 'ROUTE_NOT_FOUND'); } void _search(HttpRequest request) { - final query = (request.uri.queryParameters['q'] ?? '').toLowerCase(); + final rawQuery = request.uri.queryParameters['q']; + if (rawQuery == null) { + _error(request, 400, 'QUERY_REQUIRED'); + return; + } + final query = rawQuery.trim(); + if (query.isEmpty || query.runes.length > 200) { + _error(request, 400, 'INVALID_QUERY'); + return; + } + final pagination = _pagination(request); + if (pagination == null) return; + + final normalizedQuery = query.toLowerCase(); final matches = books.values .where( (book) => '${book.metadata['title']} ${book.metadata['author']}' .toLowerCase() - .contains(query), + .contains(normalizedQuery), ) .map((book) => book.metadata) .toList(); - _send(request.response, 200, { - 'items': matches, - 'page': 1, - 'pageSize': matches.length, + final pageItems = _page(matches, pagination); + _sendJson(request, 200, { + 'items': pageItems, + 'page': pagination.page, + 'pageSize': pagination.pageSize, 'total': matches.length, - 'hasMore': false, + 'hasMore': pagination.end < matches.length, }); } - void _error(HttpResponse response, int status, String code) { - _send(response, status, { - 'error': {'code': code, 'message': code.replaceAll('_', ' ')}, + void _catalog(HttpRequest request, BookData book) { + final pagination = _pagination(request, defaultPageSize: 100); + if (pagination == null) return; + final chapters = [...book.chapters] + ..sort((a, b) { + final byOrder = a.order.compareTo(b.order); + return byOrder != 0 ? byOrder : a.id.compareTo(b.id); + }); + _sendJson(request, 200, { + 'items': _page( + chapters, + pagination, + ).map((chapter) => chapter.metadata).toList(), + 'page': pagination.page, + 'pageSize': pagination.pageSize, + 'total': chapters.length, + 'hasMore': pagination.end < chapters.length, }); } - void _send(HttpResponse response, int status, Object body) { + Pagination? _pagination(HttpRequest request, {int defaultPageSize = 20}) { + final pageText = request.uri.queryParameters['page']; + final pageSizeText = request.uri.queryParameters['pageSize']; + final page = pageText == null ? 1 : int.tryParse(pageText); + final pageSize = pageSizeText == null + ? defaultPageSize + : int.tryParse(pageSizeText); + if (page == null || page < 1) { + _error(request, 400, 'INVALID_PAGE'); + return null; + } + if (pageSize == null || pageSize < 1 || pageSize > 100) { + _error(request, 400, 'INVALID_PAGE_SIZE'); + return null; + } + return Pagination(page, pageSize); + } + + List _page(List items, Pagination pagination) { + if (pagination.start >= items.length) return const []; + final end = pagination.end.clamp(0, items.length); + return items.sublist(pagination.start, end); + } + + bool _validId(String value) => _opaqueIdPattern.hasMatch(value); + + void _error( + HttpRequest request, + int status, + String code, { + bool retryable = false, + }) { + _sendJson(request, status, { + 'error': { + 'code': code, + 'message': code.replaceAll('_', ' '), + 'retryable': retryable, + }, + }, cacheable: false); + } + + void _sendJson( + HttpRequest request, + int status, + Object body, { + bool cacheable = true, + }) { + final encoded = jsonEncode(body); + final response = request.response; + if (cacheable) { + final etag = _etag(encoded); + response.headers + ..set(HttpHeaders.etagHeader, etag) + ..set(HttpHeaders.cacheControlHeader, 'public, max-age=60'); + if (request.headers.value(HttpHeaders.ifNoneMatchHeader) == etag) { + response.statusCode = HttpStatus.notModified; + response.close(); + return; + } + } else { + response.headers.set(HttpHeaders.cacheControlHeader, 'no-store'); + } response ..statusCode = status ..headers.contentType = ContentType.json - ..write(jsonEncode(body)) + ..write(encoded) ..close(); } + String _etag(String value) { + var hash = 0x811c9dc5; + for (final byte in utf8.encode(value)) { + hash ^= byte; + hash = (hash * 0x01000193) & 0xffffffff; + } + return '"${hash.toRadixString(16).padLeft(8, '0')}"'; + } + void _cors(HttpResponse response) { response.headers ..set('Access-Control-Allow-Origin', '*') ..set('Access-Control-Allow-Methods', 'GET, OPTIONS') ..set( 'Access-Control-Allow-Headers', - 'Content-Type, X-Open-Reading-Protocol', - ); + 'Accept, Content-Type, If-None-Match, X-Open-Reading-Protocol', + ) + ..set('Access-Control-Expose-Headers', 'ETag, X-Open-Reading-Protocol'); } } +class Pagination { + final int page; + final int pageSize; + + const Pagination(this.page, this.pageSize); + + int get start => (page - 1) * pageSize; + int get end => start + pageSize; +} + class BookData { final Map metadata; final List chapters; @@ -184,6 +309,7 @@ const books = { 'categories': ['Original', 'Protocol'], 'status': 'completed', 'latestChapter': '共同的语言', + 'language': 'zh-CN', 'updatedAt': '2026-07-11T00:00:00Z', }, [ @@ -196,6 +322,23 @@ const books = { ), ], ), + 'river-of-pages': BookData( + { + 'id': 'river-of-pages', + 'title': '书页之河', + 'author': 'Open Reading', + 'description': 'A second original sample used for pagination tests.', + 'categories': ['Original'], + 'status': 'ongoing', + 'latestChapter': '向远方', + 'language': 'zh-CN', + 'updatedAt': '2026-07-12T00:00:00Z', + }, + [ + ChapterData('source', '源头', 1, '每一条繁华的河流，都从一个可以验证的小源头开始。'), + ChapterData('distance', '向远方', 2, '当规则稳定、道路开放，故事便会流向更多读者。'), + ], + ), }; Future main(List arguments) async { diff --git a/openapi.yaml b/openapi.yaml index 82ac5cb..6217727 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1,8 +1,8 @@ openapi: 3.1.0 info: title: Open Reading Source Protocol - version: 1.0.0 - description: Standard API implemented by an ORSP-compatible book source. + version: '1.0' + description: Machine-readable contract for the ORSP 1.0 Core Reading profile. license: name: MIT identifier: MIT @@ -14,75 +14,133 @@ paths: operationId: searchBooks summary: Search books parameters: + - $ref: '#/components/parameters/ProtocolVersion' - name: q in: query required: true schema: { type: string, minLength: 1, maxLength: 200 } - - name: page - in: query - schema: { type: integer, minimum: 1, default: 1 } - - name: pageSize - in: query - schema: { type: integer, minimum: 1, maximum: 100, default: 20 } + - $ref: '#/components/parameters/Page' + - $ref: '#/components/parameters/PageSize' responses: '200': description: Search results + headers: + ETag: { $ref: '#/components/headers/ETag' } content: application/json: schema: { $ref: '#/components/schemas/SearchPage' } + '304': { $ref: '#/components/responses/NotModified' } '400': { $ref: '#/components/responses/BadRequest' } + '405': { $ref: '#/components/responses/MethodNotAllowed' } '429': { $ref: '#/components/responses/RateLimited' } + '500': { $ref: '#/components/responses/InternalError' } + '503': { $ref: '#/components/responses/Unavailable' } /v1/books/{bookId}: get: operationId: getBook summary: Get book details parameters: + - $ref: '#/components/parameters/ProtocolVersion' - $ref: '#/components/parameters/BookId' responses: '200': description: Book details + headers: + ETag: { $ref: '#/components/headers/ETag' } content: application/json: schema: { $ref: '#/components/schemas/Book' } + '304': { $ref: '#/components/responses/NotModified' } + '400': { $ref: '#/components/responses/BadRequest' } '404': { $ref: '#/components/responses/NotFound' } + '405': { $ref: '#/components/responses/MethodNotAllowed' } + '500': { $ref: '#/components/responses/InternalError' } + '503': { $ref: '#/components/responses/Unavailable' } /v1/books/{bookId}/chapters: get: operationId: listChapters summary: List a book's chapters parameters: + - $ref: '#/components/parameters/ProtocolVersion' - $ref: '#/components/parameters/BookId' + - $ref: '#/components/parameters/Page' + - $ref: '#/components/parameters/CatalogPageSize' responses: '200': - description: Chapter catalog + description: Paginated chapter catalog + headers: + ETag: { $ref: '#/components/headers/ETag' } content: application/json: schema: { $ref: '#/components/schemas/ChapterPage' } + '304': { $ref: '#/components/responses/NotModified' } + '400': { $ref: '#/components/responses/BadRequest' } '404': { $ref: '#/components/responses/NotFound' } + '405': { $ref: '#/components/responses/MethodNotAllowed' } + '500': { $ref: '#/components/responses/InternalError' } + '503': { $ref: '#/components/responses/Unavailable' } /v1/books/{bookId}/chapters/{chapterId}: get: operationId: getChapterContent summary: Get chapter content parameters: + - $ref: '#/components/parameters/ProtocolVersion' - $ref: '#/components/parameters/BookId' - - name: chapterId - in: path - required: true - schema: { type: string, minLength: 1, maxLength: 500 } + - $ref: '#/components/parameters/ChapterId' responses: '200': description: Chapter content + headers: + ETag: { $ref: '#/components/headers/ETag' } content: application/json: schema: { $ref: '#/components/schemas/ChapterContent' } + '304': { $ref: '#/components/responses/NotModified' } + '400': { $ref: '#/components/responses/BadRequest' } '404': { $ref: '#/components/responses/NotFound' } + '405': { $ref: '#/components/responses/MethodNotAllowed' } + '500': { $ref: '#/components/responses/InternalError' } + '503': { $ref: '#/components/responses/Unavailable' } components: parameters: + ProtocolVersion: + name: X-Open-Reading-Protocol + in: header + required: false + description: Highest ORSP 1.x version supported by the client. + schema: { type: string, pattern: '^1\.(?:0|[1-9][0-9]*)$' } + Page: + name: page + in: query + required: false + schema: { type: integer, minimum: 1, default: 1 } + PageSize: + name: pageSize + in: query + required: false + schema: { type: integer, minimum: 1, maximum: 100, default: 20 } + CatalogPageSize: + name: pageSize + in: query + required: false + schema: { type: integer, minimum: 1, maximum: 100, default: 100 } BookId: name: bookId in: path required: true - schema: { type: string, minLength: 1, maxLength: 500 } + schema: { $ref: '#/components/schemas/OpaqueId' } + ChapterId: + name: chapterId + in: path + required: true + schema: { $ref: '#/components/schemas/OpaqueId' } + headers: + ETag: + description: Entity tag for the selected representation. + schema: { type: string } responses: + NotModified: + description: The selected representation has not changed. BadRequest: description: Invalid request content: @@ -93,74 +151,126 @@ components: content: application/json: schema: { $ref: '#/components/schemas/ErrorResponse' } + MethodNotAllowed: + description: HTTP method not supported + headers: + Allow: + schema: { type: string } + content: + application/json: + schema: { $ref: '#/components/schemas/ErrorResponse' } RateLimited: description: Too many requests headers: Retry-After: - schema: { type: integer, minimum: 1 } + schema: + oneOf: + - { type: integer, minimum: 1 } + - { type: string } + content: + application/json: + schema: { $ref: '#/components/schemas/ErrorResponse' } + InternalError: + description: Internal source error + content: + application/json: + schema: { $ref: '#/components/schemas/ErrorResponse' } + Unavailable: + description: Source temporarily unavailable + headers: + Retry-After: + schema: + oneOf: + - { type: integer, minimum: 1 } + - { type: string } content: application/json: schema: { $ref: '#/components/schemas/ErrorResponse' } schemas: + OpaqueId: + type: string + minLength: 1 + maxLength: 200 + pattern: '^[A-Za-z0-9._~-]+$' Book: type: object + additionalProperties: true required: [id, title] properties: - id: { type: string, minLength: 1 } - title: { type: string, minLength: 1 } - author: { type: string } + id: { $ref: '#/components/schemas/OpaqueId' } + title: { type: string, minLength: 1, maxLength: 500 } + author: { type: string, maxLength: 500 } description: { type: string } - coverUrl: { type: string, format: uri } + coverUrl: { type: string, format: uri, pattern: '^https?://' } categories: type: array - items: { type: string } + uniqueItems: true + items: { type: string, minLength: 1, maxLength: 100 } status: { type: string, enum: [ongoing, completed, hiatus, unknown] } - latestChapter: { type: string } + latestChapter: { type: string, maxLength: 500 } + language: + type: string + minLength: 2 + maxLength: 35 + pattern: '^[A-Za-z]{2,8}(?:-[A-Za-z0-9]{1,8})*$' updatedAt: { type: string, format: date-time } SearchPage: type: object + additionalProperties: true required: [items, page, pageSize, hasMore] properties: items: type: array items: { $ref: '#/components/schemas/Book' } page: { type: integer, minimum: 1 } - pageSize: { type: integer, minimum: 0 } + pageSize: { type: integer, minimum: 1, maximum: 100 } total: { type: integer, minimum: 0 } hasMore: { type: boolean } Chapter: type: object + additionalProperties: true required: [id, title, order] properties: - id: { type: string, minLength: 1 } - title: { type: string, minLength: 1 } + id: { $ref: '#/components/schemas/OpaqueId' } + title: { type: string, minLength: 1, maxLength: 500 } order: { type: integer, minimum: 0 } updatedAt: { type: string, format: date-time } ChapterPage: type: object - required: [items] + additionalProperties: true + required: [items, page, pageSize, hasMore] properties: items: type: array items: { $ref: '#/components/schemas/Chapter' } + page: { type: integer, minimum: 1 } + pageSize: { type: integer, minimum: 1, maximum: 100 } + total: { type: integer, minimum: 0 } + hasMore: { type: boolean } ChapterContent: type: object + additionalProperties: true required: [bookId, chapterId, title, contentType, content] properties: - bookId: { type: string, minLength: 1 } - chapterId: { type: string, minLength: 1 } - title: { type: string, minLength: 1 } + bookId: { $ref: '#/components/schemas/OpaqueId' } + chapterId: { $ref: '#/components/schemas/OpaqueId' } + title: { type: string, minLength: 1, maxLength: 500 } contentType: type: string enum: [text/plain, text/markdown, text/html] content: { type: string } + baseUrl: { type: string, format: uri, pattern: '^https?://' } ErrorResponse: type: object + additionalProperties: true required: [error] properties: error: type: object + additionalProperties: true required: [code, message] properties: - code: { type: string } - message: { type: string } + code: { type: string, pattern: '^[A-Z][A-Z0-9]*(?:_[A-Z0-9]+)*$' } + message: { type: string, minLength: 1 } + retryable: { type: boolean } + requestId: { type: string, minLength: 1, maxLength: 200 } diff --git a/registry/POLICY.md b/registry/POLICY.md new file mode 100644 index 0000000..8bfe9a8 --- /dev/null +++ b/registry/POLICY.md @@ -0,0 +1,60 @@ +# Registry Review and Removal Policy + +## Scope + +The registry lists public ORSP sources that implement the complete Core Reading +profile. Listing is discretionary and does not certify content ownership, +security, permanence, or quality. + +## Submission declarations + +Submitters must state that: + +- they operate the source or are authorized by its operator to submit it; +- the source distributes only original, public-domain, or licensed content; +- the provided test query is safe and stable enough for automated checks; +- maintainers will respond to security, availability, and rights reports; +- no secret, cookie, or private infrastructure detail is included. + +## Review + +Automated validation is evidence, not the final decision. Reviewers may request +ownership evidence, clearer licensing information, source-code links, a public +reporting channel, or changes needed to protect users and infrastructure. + +Reviewers should explain rejection or deferral using concrete protocol, safety, +maintenance, or policy reasons. Competing readers and sources must be treated +under the same criteria. + +## Maintainer transfer + +An existing maintainer may add or transfer responsibility through a pull +request. If all maintainers are unreachable, a new operator must demonstrate +control of the discovery endpoint or source domain. The stable source ID should +remain unchanged only when the logical source continues. + +## Operational states + +- `verified`: current compatibility and policy checks pass. +- `degraded`: some checks fail but the core reading path remains usable. +- `offline`: repeated checks cannot reach or use the source. +- `unmaintained`: maintainers do not respond within the published review window. +- `suspended`: hidden or warned due to an active safety, abuse, or rights issue. +- `removed`: no longer distributed by the active registry. + +Except for urgent risks, a source should not be removed after one failed check. +Repeated failures should first mark it degraded, then offline. Restoration uses +the same validation process as a new submission. + +## Emergency suspension + +Maintainers may immediately suspend a source for credential exposure, malicious +content, unsafe redirects, client exploitation, credible copyright complaints, +or deliberate evasion of registry checks. Public notes must avoid disclosing +secrets or details that increase user risk. + +## Appeals + +Source maintainers may open a new pull request or issue with corrected evidence. +An appeal should be reviewed by someone other than the person who made the +original discretionary decision when another reviewer is available. diff --git a/registry/README.md b/registry/README.md new file mode 100644 index 0000000..52f78ce --- /dev/null +++ b/registry/README.md @@ -0,0 +1,84 @@ +# ORSP Community Source Registry + +This directory is the initial community registry for public ORSP sources. It is +a discovery and moderation layer, not a replacement for the ORSP wire protocol. + +Each source is submitted as one JSON file: + +```text +registry/sources/.json +``` + +The filename MUST exactly match the entry's `id`. Do not edit +`registry/generated/registry.json` by hand; it is built from accepted entries. + +## Submit a source + +1. Fork the repository. +2. Copy `registry/examples/source-entry.json` to + `registry/sources/.json`. +3. Replace every example value and choose a stable `testQuery` that returns at + least one book with at least one chapter. +4. Run the local checks below. +5. Open a pull request using `.github/PULL_REQUEST_TEMPLATE/source-submission.md`. + +The discovery document's `id` MUST match both the registry entry and filename. +All four ORSP Core Reading capabilities are required. + +## What registry fields mean + +- `discoveryUrl`: canonical HTTPS discovery-document URL. +- `maintainers`: GitHub accounts responsible for the source entry. +- `description`: registry-facing summary; runtime source metadata still comes + from discovery. +- `languages`: claimed content languages. Network verification compares these + with discovery but discovery remains the runtime source of truth. +- `contentTypes`: operator declaration of the basis for distribution. +- `testQuery`: stable query used only for compatibility and health checks. +- `reportingUrl`: optional abuse, copyright, or availability reporting page. + +Protocol conformance does not prove content legality, quality, or operator +trustworthiness. Those are separate registry and community decisions. + +## Local checks + +```bash +python scripts/registry_tool.py validate +python scripts/registry_tool.py build --check +python scripts/registry_tool.py check-source \ + registry/sources/.json +``` + +The first two commands are safe static checks and run on every pull request. +For submitted manifests, a separate workflow runs the already-reviewed +validator from the pull request's base revision; it does not execute submitted +code or expose repository credentials. The network check rejects private, +loopback, link-local, multicast, and reserved IP addresses, pins the connection +to a validated address, and repeats validation after every redirect. + +## Review and publication + +A pull request is eligible for merge when: + +- the manifest and filename pass static validation; +- source and discovery IDs match; +- discovery and API URLs pass public-network safety checks; +- search, detail, catalog, and content complete successfully; +- the submitter completes the authorization and maintenance declarations; +- no unresolved security, abuse, or copyright concern remains. + +Accepted entries are sorted by source ID in +`registry/generated/registry.json`. Reader applications may consume that file, +but MUST fetch current discovery metadata directly from each source before use. + +## Health and removal + +Automated checks produce operational reports separately from author-controlled +source entries. A single failure does not remove a source. + +Suggested operational states are `verified`, `degraded`, `offline`, +`unmaintained`, `suspended`, and `removed`. Maintainers may suspend a source +immediately for a credible security or legal risk. Removed entries are retained +under `registry/removed/` with a public reason when disclosure is appropriate. + +See [POLICY.md](POLICY.md) for review, transfer, suspension, and removal rules. diff --git a/registry/examples/source-entry.json b/registry/examples/source-entry.json new file mode 100644 index 0000000..ba756b4 --- /dev/null +++ b/registry/examples/source-entry.json @@ -0,0 +1,21 @@ +{ + "schemaVersion": "1.0", + "id": "org.example.public-books", + "discoveryUrl": "https://books.example.org/.well-known/open-reading-source.json", + "maintainers": [ + { + "github": "example-author", + "role": "Source operator" + } + ], + "description": "An example registry entry for a public-domain ORSP source.", + "languages": ["en", "zh-CN"], + "contentTypes": ["public-domain"], + "testQuery": "Example", + "homepage": "https://books.example.org/", + "repository": "https://github.com/example/books-source", + "reportingUrl": "https://books.example.org/report", + "license": "MIT", + "tags": ["books", "public-domain"], + "submittedAt": "2026-07-12" +} diff --git a/registry/generated/registry.json b/registry/generated/registry.json new file mode 100644 index 0000000..8e56845 --- /dev/null +++ b/registry/generated/registry.json @@ -0,0 +1,4 @@ +{ + "schemaVersion": "1.0", + "sources": [] +} diff --git a/registry/removed/.gitkeep b/registry/removed/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/registry/removed/.gitkeep @@ -0,0 +1 @@ + diff --git a/registry/schemas/registry.schema.json b/registry/schemas/registry.schema.json new file mode 100644 index 0000000..807b6fd --- /dev/null +++ b/registry/schemas/registry.schema.json @@ -0,0 +1,19 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/miloquinn/open-reading-source-protocol/registry/schemas/registry.schema.json", + "title": "ORSP Community Registry", + "type": "object", + "additionalProperties": false, + "required": ["schemaVersion", "sources"], + "properties": { + "schemaVersion": { + "const": "1.0" + }, + "sources": { + "type": "array", + "items": { + "$ref": "source-entry.schema.json" + } + } + } +} diff --git a/registry/schemas/source-entry.schema.json b/registry/schemas/source-entry.schema.json new file mode 100644 index 0000000..bf931e4 --- /dev/null +++ b/registry/schemas/source-entry.schema.json @@ -0,0 +1,122 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/miloquinn/open-reading-source-protocol/registry/schemas/source-entry.schema.json", + "title": "ORSP Community Registry Source Entry", + "type": "object", + "additionalProperties": false, + "required": [ + "schemaVersion", + "id", + "discoveryUrl", + "maintainers", + "description", + "languages", + "contentTypes", + "testQuery", + "submittedAt" + ], + "properties": { + "schemaVersion": { + "const": "1.0" + }, + "id": { + "type": "string", + "minLength": 3, + "maxLength": 200, + "pattern": "^[A-Za-z0-9](?:[A-Za-z0-9._-]*[A-Za-z0-9])?$" + }, + "discoveryUrl": { + "type": "string", + "format": "uri", + "pattern": "^https://[^@/?#\\s]+(?:/[^?#\\s]*)?$" + }, + "maintainers": { + "type": "array", + "minItems": 1, + "maxItems": 10, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["github"], + "properties": { + "github": { + "type": "string", + "minLength": 1, + "maxLength": 39, + "pattern": "^[A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?$" + }, + "role": { + "type": "string", + "maxLength": 100 + } + } + } + }, + "description": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "languages": { + "type": "array", + "minItems": 1, + "maxItems": 100, + "uniqueItems": true, + "items": { + "type": "string", + "minLength": 2, + "maxLength": 35, + "pattern": "^[A-Za-z]{2,8}(?:-[A-Za-z0-9]{1,8})*$" + } + }, + "contentTypes": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "enum": ["original", "public-domain", "licensed"] + } + }, + "testQuery": { + "type": "string", + "minLength": 1, + "maxLength": 200, + "description": "A stable query expected to return at least one readable book." + }, + "homepage": { + "type": "string", + "format": "uri", + "pattern": "^https://" + }, + "repository": { + "type": "string", + "format": "uri", + "pattern": "^https://" + }, + "reportingUrl": { + "type": "string", + "format": "uri", + "pattern": "^https://" + }, + "license": { + "type": "string", + "minLength": 1, + "maxLength": 100 + }, + "tags": { + "type": "array", + "maxItems": 20, + "uniqueItems": true, + "items": { + "type": "string", + "minLength": 1, + "maxLength": 50, + "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$" + } + }, + "submittedAt": { + "type": "string", + "format": "date" + } + } +} diff --git a/registry/sources/.gitkeep b/registry/sources/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/registry/sources/.gitkeep @@ -0,0 +1 @@ + diff --git a/registry/status/.gitkeep b/registry/status/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/registry/status/.gitkeep @@ -0,0 +1 @@ + diff --git a/schemas/discovery.schema.json b/schemas/discovery.schema.json index 79e154a..e0a2245 100644 --- a/schemas/discovery.schema.json +++ b/schemas/discovery.schema.json @@ -18,12 +18,13 @@ }, "protocolVersion": { "type": "string", - "pattern": "^1\\.[0-9]+(?:\\.[0-9]+)?$" + "pattern": "^1\\.(?:0|[1-9][0-9]*)$" }, "id": { "type": "string", "minLength": 3, - "maxLength": 200 + "maxLength": 200, + "pattern": "^[A-Za-z0-9](?:[A-Za-z0-9._-]*[A-Za-z0-9])?$" }, "name": { "type": "string", @@ -37,7 +38,7 @@ "apiBaseUrl": { "type": "string", "format": "uri", - "pattern": "^https?://" + "pattern": "^https?://[^@/?#\\s]+(?:/[^?#\\s]*)?/$" }, "iconUrl": { "type": "string", @@ -51,21 +52,36 @@ }, "languages": { "type": "array", + "maxItems": 100, "uniqueItems": true, "items": { "type": "string", "minLength": 2, - "maxLength": 35 + "maxLength": 35, + "pattern": "^[A-Za-z]{2,8}(?:-[A-Za-z0-9]{1,8})*$" } }, "capabilities": { "type": "array", + "minItems": 4, + "maxItems": 64, "uniqueItems": true, - "contains": { - "const": "search" - }, + "allOf": [ + { "contains": { "const": "search" } }, + { "contains": { "const": "detail" } }, + { "contains": { "const": "catalog" } }, + { "contains": { "const": "content" } } + ], "items": { - "enum": ["search", "detail", "catalog", "content"] + "anyOf": [ + { + "enum": ["search", "detail", "catalog", "content"] + }, + { + "type": "string", + "pattern": "^(?!orsp(?:[.:]|$))[a-z0-9]+(?:[.-][a-z0-9]+)+:[A-Za-z0-9][A-Za-z0-9._-]{0,99}$" + } + ] } } } diff --git a/scripts/registry_tool.py b/scripts/registry_tool.py new file mode 100644 index 0000000..517cc68 --- /dev/null +++ b/scripts/registry_tool.py @@ -0,0 +1,512 @@ +"""Build and validate the ORSP community source registry.""" + +from __future__ import annotations + +import argparse +import http.client +import ipaddress +import json +import socket +import ssl +import sys +import time +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Any +from urllib.parse import quote, urlencode, urljoin, urlsplit + +from jsonschema import Draft202012Validator, FormatChecker +from referencing import Registry, Resource + + +ROOT = Path(__file__).resolve().parents[1] +SOURCES_DIR = ROOT / "registry" / "sources" +GENERATED_REGISTRY = ROOT / "registry" / "generated" / "registry.json" +SOURCE_SCHEMA_PATH = ROOT / "registry" / "schemas" / "source-entry.schema.json" +REGISTRY_SCHEMA_PATH = ROOT / "registry" / "schemas" / "registry.schema.json" +DISCOVERY_SCHEMA_PATH = ROOT / "schemas" / "discovery.schema.json" +EXAMPLE_ENTRY_PATH = ROOT / "registry" / "examples" / "source-entry.json" + +CORE_CAPABILITIES = {"search", "detail", "catalog", "content"} +REDIRECT_STATUSES = {301, 302, 303, 307, 308} +MAX_REDIRECTS = 5 +MAX_JSON_BYTES = 2 * 1024 * 1024 +MAX_MANIFEST_BYTES = 64 * 1024 +REQUEST_TIMEOUT_SECONDS = 10.0 + + +class RegistryError(RuntimeError): + pass + + +@dataclass(frozen=True) +class HttpResult: + url: str + status: int + headers: dict[str, str] + body: bytes + elapsed_ms: int + + def json(self) -> Any: + try: + return json.loads(self.body.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise RegistryError(f"{self.url} did not return valid UTF-8 JSON") from error + + +class PinnedHTTPSConnection(http.client.HTTPSConnection): + """HTTPS connection pinned to a previously validated IP address.""" + + def __init__(self, hostname: str, ip_address: str, port: int, timeout: float): + super().__init__(hostname, port=port, timeout=timeout) + self._pinned_ip = ip_address + + def connect(self) -> None: + raw_socket = socket.create_connection( + (self._pinned_ip, self.port), + self.timeout, + self.source_address, + ) + try: + self.sock = self._context.wrap_socket(raw_socket, server_hostname=self.host) + except Exception: + raw_socket.close() + raise + + +def load_json(path: Path) -> Any: + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error: + raise RegistryError(f"Cannot read JSON from {path}: {error}") from error + + +def json_validator(schema_path: Path) -> Draft202012Validator: + return Draft202012Validator( + load_json(schema_path), + format_checker=FormatChecker(), + ) + + +def registry_validator() -> Draft202012Validator: + source_schema = load_json(SOURCE_SCHEMA_PATH) + schema_registry = Registry().with_resource( + source_schema["$id"], + Resource.from_contents(source_schema), + ) + return Draft202012Validator( + load_json(REGISTRY_SCHEMA_PATH), + registry=schema_registry, + format_checker=FormatChecker(), + ) + + +def format_validation_errors( + validator: Draft202012Validator, + value: Any, +) -> list[str]: + errors = [] + for error in sorted(validator.iter_errors(value), key=lambda item: list(item.path)): + location = ".".join(str(part) for part in error.path) or "" + errors.append(f"{location}: {error.message}") + return errors + + +def source_files() -> list[Path]: + return sorted(SOURCES_DIR.glob("*.json"), key=lambda path: path.name.lower()) + + +def validate_sources_directory() -> None: + for path in SOURCES_DIR.iterdir(): + if path.name == ".gitkeep": + continue + if path.is_symlink() or not path.is_file() or path.suffix != ".json": + raise RegistryError(f"Unexpected registry source entry: {path}") + + +def validate_manifest_file(path: Path) -> None: + if path.is_symlink() or not path.is_file(): + raise RegistryError(f"Registry entry must be a regular file: {path}") + if path.stat().st_size > MAX_MANIFEST_BYTES: + raise RegistryError( + f"Registry entry exceeds {MAX_MANIFEST_BYTES} bytes: {path}" + ) + + +def validate_entries() -> list[dict[str, Any]]: + validate_sources_directory() + validator = json_validator(SOURCE_SCHEMA_PATH) + entries: list[dict[str, Any]] = [] + ids: set[str] = set() + failures: list[str] = [] + + example = load_json(EXAMPLE_ENTRY_PATH) + for message in format_validation_errors(validator, example): + failures.append(f"{EXAMPLE_ENTRY_PATH.relative_to(ROOT)}: {message}") + + for path in source_files(): + validate_manifest_file(path) + entry = load_json(path) + errors = format_validation_errors(validator, entry) + failures.extend(f"{path.relative_to(ROOT)}: {message}" for message in errors) + if errors or not isinstance(entry, dict): + continue + source_id = entry["id"] + if path.stem != source_id: + failures.append( + f"{path.relative_to(ROOT)}: filename must be {source_id}.json" + ) + if source_id in ids: + failures.append(f"{path.relative_to(ROOT)}: duplicate source ID {source_id}") + ids.add(source_id) + entries.append(entry) + + if failures: + raise RegistryError("Registry validation failed:\n- " + "\n- ".join(failures)) + return sorted(entries, key=lambda entry: entry["id"]) + + +def generated_document(entries: list[dict[str, Any]]) -> dict[str, Any]: + return {"schemaVersion": "1.0", "sources": entries} + + +def validate_command(_: argparse.Namespace) -> None: + entries = validate_entries() + document = generated_document(entries) + errors = format_validation_errors(registry_validator(), document) + if errors: + raise RegistryError("Generated registry is invalid:\n- " + "\n- ".join(errors)) + print(f"Registry static validation passed: {len(entries)} source(s)") + + +def build_command(args: argparse.Namespace) -> None: + document = generated_document(validate_entries()) + errors = format_validation_errors(registry_validator(), document) + if errors: + raise RegistryError("Generated registry is invalid:\n- " + "\n- ".join(errors)) + encoded = json.dumps(document, ensure_ascii=False, indent=2) + "\n" + if args.check: + current = GENERATED_REGISTRY.read_text(encoding="utf-8") + if current != encoded: + raise RegistryError( + "registry/generated/registry.json is stale; run " + "python scripts/registry_tool.py build" + ) + print("Generated registry is up to date") + return + GENERATED_REGISTRY.write_text(encoded, encoding="utf-8", newline="\n") + print(f"Built registry with {len(document['sources'])} source(s)") + + +def ensure_global_ip(value: str) -> ipaddress.IPv4Address | ipaddress.IPv6Address: + address = ipaddress.ip_address(value) + if not address.is_global: + raise RegistryError(f"Network target is not a public global IP: {address}") + return address + + +def resolve_public_addresses(hostname: str, port: int) -> list[str]: + try: + records = socket.getaddrinfo( + hostname, + port, + type=socket.SOCK_STREAM, + proto=socket.IPPROTO_TCP, + ) + except socket.gaierror as error: + raise RegistryError(f"Cannot resolve {hostname}: {error}") from error + addresses: list[str] = [] + for record in records: + value = record[4][0] + ensure_global_ip(value) + if value not in addresses: + addresses.append(value) + if not addresses: + raise RegistryError(f"No public address found for {hostname}") + return addresses + + +def validate_public_https_url(url: str) -> tuple[str, int, list[str]]: + parsed = urlsplit(url) + if parsed.scheme != "https": + raise RegistryError(f"Only HTTPS registry targets are allowed: {url}") + if not parsed.hostname or parsed.username or parsed.password: + raise RegistryError(f"URL has an invalid authority: {url}") + try: + port = parsed.port or 443 + except ValueError as error: + raise RegistryError(f"URL has an invalid port: {url}") from error + return parsed.hostname, port, resolve_public_addresses(parsed.hostname, port) + + +def fetch(url: str) -> HttpResult: + current_url = url + for redirect_count in range(MAX_REDIRECTS + 1): + parsed = urlsplit(current_url) + hostname, port, addresses = validate_public_https_url(current_url) + target = parsed.path or "/" + if parsed.query: + target += f"?{parsed.query}" + started = time.monotonic() + last_error: Exception | None = None + response: http.client.HTTPResponse | None = None + connection: PinnedHTTPSConnection | None = None + for address in addresses: + try: + connection = PinnedHTTPSConnection( + hostname, + address, + port, + REQUEST_TIMEOUT_SECONDS, + ) + connection.request( + "GET", + target, + headers={ + "Accept": "application/json", + "Accept-Encoding": "identity", + "User-Agent": "ORSP-Registry-Validator/1.0", + "X-Open-Reading-Protocol": "1.0", + }, + ) + response = connection.getresponse() + break + except (OSError, ssl.SSLError, http.client.HTTPException) as error: + last_error = error + if connection is not None: + connection.close() + if response is None or connection is None: + raise RegistryError(f"Cannot connect to {current_url}: {last_error}") + try: + headers = {name.lower(): value for name, value in response.getheaders()} + if response.status in REDIRECT_STATUSES: + location = headers.get("location") + response.read() + if not location: + raise RegistryError(f"Redirect without Location from {current_url}") + if redirect_count == MAX_REDIRECTS: + raise RegistryError(f"Too many redirects while fetching {url}") + current_url = urljoin(current_url, location) + continue + body = response.read(MAX_JSON_BYTES + 1) + if len(body) > MAX_JSON_BYTES: + raise RegistryError(f"Response exceeds {MAX_JSON_BYTES} bytes: {current_url}") + return HttpResult( + current_url, + response.status, + headers, + body, + round((time.monotonic() - started) * 1000), + ) + finally: + connection.close() + raise RegistryError(f"Too many redirects while fetching {url}") + + +def require_json(url: str, label: str) -> tuple[dict[str, Any], HttpResult]: + result = fetch(url) + if result.status != 200: + raise RegistryError(f"{label} returned HTTP {result.status}: {result.url}") + content_type = result.headers.get("content-type", "") + if not content_type.lower().startswith("application/json"): + raise RegistryError(f"{label} did not use application/json: {content_type}") + value = result.json() + if not isinstance(value, dict): + raise RegistryError(f"{label} must return a JSON object") + return value, result + + +def checked_api_url(api_base: str, relation: str, query: dict[str, str] | None = None) -> str: + if not api_base.endswith("/"): + raise RegistryError("Discovery apiBaseUrl must end in /") + url = urljoin(api_base, relation) + validate_public_https_url(url) + if query: + parsed = urlsplit(url) + url = parsed._replace(query=urlencode(query)).geturl() + return url + + +def require_id(value: Any, label: str) -> str: + if not isinstance(value, str) or not value or len(value) > 200: + raise RegistryError(f"{label} is not a valid non-empty ID") + allowed = set("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._~-") + if any(character not in allowed for character in value): + raise RegistryError(f"{label} contains characters outside the ORSP ID alphabet") + return value + + +def check_source(path: Path) -> dict[str, Any]: + validate_manifest_file(path) + entry_validator = json_validator(SOURCE_SCHEMA_PATH) + entry = load_json(path) + errors = format_validation_errors(entry_validator, entry) + if errors: + raise RegistryError(f"Invalid source entry {path}:\n- " + "\n- ".join(errors)) + + timings: dict[str, int] = {} + warnings: list[str] = [] + discovery, result = require_json(entry["discoveryUrl"], "Discovery") + timings["discovery"] = result.elapsed_ms + discovery_errors = format_validation_errors( + json_validator(DISCOVERY_SCHEMA_PATH), + discovery, + ) + if discovery_errors: + raise RegistryError("Invalid discovery document:\n- " + "\n- ".join(discovery_errors)) + if discovery["id"] != entry["id"]: + raise RegistryError( + f"Source ID mismatch: registry={entry['id']} discovery={discovery['id']}" + ) + capabilities = set(discovery["capabilities"]) + if not CORE_CAPABILITIES.issubset(capabilities): + missing = ", ".join(sorted(CORE_CAPABILITIES - capabilities)) + raise RegistryError(f"Discovery is missing core capabilities: {missing}") + discovery_languages = set(discovery.get("languages", [])) + declared_languages = set(entry["languages"]) + if discovery_languages and not declared_languages.issubset(discovery_languages): + warnings.append("Registry languages are not all declared by discovery") + + api_base = discovery["apiBaseUrl"] + search_url = checked_api_url( + api_base, + "v1/search", + {"q": entry["testQuery"], "page": "1", "pageSize": "1"}, + ) + search, result = require_json(search_url, "Search") + timings["search"] = result.elapsed_ms + search_items = search.get("items") + if not isinstance(search_items, list) or not search_items: + raise RegistryError("testQuery did not return a book") + first_book = search_items[0] + if not isinstance(first_book, dict): + raise RegistryError("Search item must be an object") + book_id = require_id(first_book.get("id"), "Search book ID") + + detail_url = checked_api_url(api_base, f"v1/books/{quote(book_id, safe='')}") + detail, result = require_json(detail_url, "Book detail") + timings["detail"] = result.elapsed_ms + if detail.get("id") != book_id: + raise RegistryError("Book detail ID does not match the search result") + + catalog_url = checked_api_url( + api_base, + f"v1/books/{quote(book_id, safe='')}/chapters", + {"page": "1", "pageSize": "1"}, + ) + catalog, result = require_json(catalog_url, "Chapter catalog") + timings["catalog"] = result.elapsed_ms + chapter_items = catalog.get("items") + if not isinstance(chapter_items, list) or not chapter_items: + raise RegistryError("The selected test book has no chapter") + first_chapter = chapter_items[0] + if not isinstance(first_chapter, dict): + raise RegistryError("Chapter item must be an object") + chapter_id = require_id(first_chapter.get("id"), "Chapter ID") + + content_url = checked_api_url( + api_base, + f"v1/books/{quote(book_id, safe='')}/chapters/{quote(chapter_id, safe='')}", + ) + content, result = require_json(content_url, "Chapter content") + timings["content"] = result.elapsed_ms + if content.get("bookId") != book_id or content.get("chapterId") != chapter_id: + raise RegistryError("Chapter content IDs do not match the requested resource") + if content.get("contentType") not in {"text/plain", "text/markdown", "text/html"}: + raise RegistryError("Chapter contentType is not supported by ORSP 1.0") + if not isinstance(content.get("content"), str): + raise RegistryError("Chapter content must be a string") + + return { + "id": entry["id"], + "status": "verified", + "protocolVersion": discovery["protocolVersion"], + "checkedAt": datetime.now(UTC).isoformat().replace("+00:00", "Z"), + "latencyMs": timings, + "warnings": warnings, + } + + +def check_source_command(args: argparse.Namespace) -> None: + report = check_source(Path(args.entry).absolute()) + if args.status_output: + Path(args.status_output).write_text( + json.dumps(report, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + newline="\n", + ) + print(json.dumps(report, ensure_ascii=False, indent=2)) + + +def check_all_command(args: argparse.Namespace) -> None: + reports: list[dict[str, Any]] = [] + failures = 0 + for path in source_files(): + try: + reports.append(check_source(path)) + except RegistryError as error: + failures += 1 + reports.append( + { + "id": path.stem, + "status": "offline", + "checkedAt": datetime.now(UTC).isoformat().replace("+00:00", "Z"), + "error": str(error), + } + ) + output = { + "schemaVersion": "1.0", + "checkedAt": datetime.now(UTC).isoformat().replace("+00:00", "Z"), + "sources": reports, + } + if args.status_output: + Path(args.status_output).write_text( + json.dumps(output, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + newline="\n", + ) + print(json.dumps(output, ensure_ascii=False, indent=2)) + if failures: + raise RegistryError(f"Network validation failed for {failures} source(s)") + + +def parser() -> argparse.ArgumentParser: + value = argparse.ArgumentParser(description=__doc__) + commands = value.add_subparsers(dest="command", required=True) + + validate = commands.add_parser("validate", help="validate source manifests") + validate.set_defaults(function=validate_command) + + build = commands.add_parser("build", help="build the combined registry") + build.add_argument("--check", action="store_true", help="fail if output is stale") + build.set_defaults(function=build_command) + + check_source_parser = commands.add_parser( + "check-source", + help="safely test one source over the network", + ) + check_source_parser.add_argument("entry", help="path to a source entry JSON file") + check_source_parser.add_argument("--status-output") + check_source_parser.set_defaults(function=check_source_command) + + check_all = commands.add_parser( + "check-all", + help="safely test every accepted source over the network", + ) + check_all.add_argument("--status-output") + check_all.set_defaults(function=check_all_command) + return value + + +def main() -> None: + args = parser().parse_args() + try: + args.function(args) + except RegistryError as error: + print(f"ERROR: {error}", file=sys.stderr) + raise SystemExit(1) from error + + +if __name__ == "__main__": + main() diff --git a/scripts/test_registry_tool.py b/scripts/test_registry_tool.py new file mode 100644 index 0000000..b694985 --- /dev/null +++ b/scripts/test_registry_tool.py @@ -0,0 +1,75 @@ +"""Unit tests for registry validation and network-safety helpers.""" + +from __future__ import annotations + +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory + +import registry_tool + + +class RegistryToolTests(unittest.TestCase): + def test_example_and_empty_registry_are_valid(self) -> None: + entries = registry_tool.validate_entries() + self.assertEqual(entries, []) + errors = registry_tool.format_validation_errors( + registry_tool.registry_validator(), + registry_tool.generated_document(entries), + ) + self.assertEqual(errors, []) + + def test_global_ip_is_allowed(self) -> None: + self.assertEqual(str(registry_tool.ensure_global_ip("8.8.8.8")), "8.8.8.8") + + def test_loopback_and_private_ips_are_rejected(self) -> None: + rejected = [ + "127.0.0.1", + "10.0.0.1", + "172.16.0.1", + "192.168.1.1", + "169.254.169.254", + "::1", + "fc00::1", + ] + for value in rejected: + with self.subTest(value=value): + with self.assertRaises(registry_tool.RegistryError): + registry_tool.ensure_global_ip(value) + + def test_id_validation(self) -> None: + self.assertEqual(registry_tool.require_id("book-1", "ID"), "book-1") + for value in ["", "book/1", "book%201", "章节一"]: + with self.subTest(value=value): + with self.assertRaises(registry_tool.RegistryError): + registry_tool.require_id(value, "ID") + + def test_non_https_and_user_info_are_rejected_without_dns(self) -> None: + with self.assertRaises(registry_tool.RegistryError): + registry_tool.validate_public_https_url("http://example.com/source") + with self.assertRaises(registry_tool.RegistryError): + registry_tool.validate_public_https_url( + "https://user:password@example.com/source" + ) + + def test_direct_private_network_url_is_rejected(self) -> None: + with self.assertRaises(registry_tool.RegistryError): + registry_tool.validate_public_https_url("https://127.0.0.1/source") + + def test_unexpected_source_directory_files_are_rejected(self) -> None: + original = registry_tool.SOURCES_DIR + with TemporaryDirectory() as directory: + registry_tool.SOURCES_DIR = Path(directory) + (registry_tool.SOURCES_DIR / "notes.txt").write_text( + "not a manifest", + encoding="utf-8", + ) + try: + with self.assertRaises(registry_tool.RegistryError): + registry_tool.validate_sources_directory() + finally: + registry_tool.SOURCES_DIR = original + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/validate_contracts.py b/scripts/validate_contracts.py new file mode 100644 index 0000000..4e50d62 --- /dev/null +++ b/scripts/validate_contracts.py @@ -0,0 +1,63 @@ +"""Validate ORSP machine-readable contracts and compatibility invariants.""" + +from __future__ import annotations + +import copy +import json +from pathlib import Path + +import yaml +from jsonschema import Draft202012Validator, FormatChecker +from openapi_spec_validator import validate_spec + + +ROOT = Path(__file__).resolve().parents[1] + + +def load_json(relative_path: str) -> object: + return json.loads((ROOT / relative_path).read_text(encoding="utf-8")) + + +def expect_invalid(validator: Draft202012Validator, value: object, name: str) -> None: + if validator.is_valid(value): + raise AssertionError(f"Expected invalid discovery fixture: {name}") + + +def main() -> None: + schema = load_json("schemas/discovery.schema.json") + discovery = load_json("examples/open-reading-source.json") + validator = Draft202012Validator(schema, format_checker=FormatChecker()) + validator.validate(discovery) + + extension_fixture = copy.deepcopy(discovery) + extension_fixture["capabilities"].append("org.example:audio") + validator.validate(extension_fixture) + + patch_version = copy.deepcopy(discovery) + patch_version["protocolVersion"] = "1.0.0" + expect_invalid(validator, patch_version, "patch protocol version") + + missing_core = copy.deepcopy(discovery) + missing_core["capabilities"].remove("content") + expect_invalid(validator, missing_core, "missing core capability") + + no_trailing_slash = copy.deepcopy(discovery) + no_trailing_slash["apiBaseUrl"] = "https://books.example.org/api" + expect_invalid(validator, no_trailing_slash, "API URL without trailing slash") + + api_user_info = copy.deepcopy(discovery) + api_user_info["apiBaseUrl"] = "https://user@books.example.org/api/" + expect_invalid(validator, api_user_info, "API URL containing user information") + + reserved_extension = copy.deepcopy(discovery) + reserved_extension["capabilities"].append("orsp:private") + expect_invalid(validator, reserved_extension, "reserved extension namespace") + + openapi = yaml.safe_load((ROOT / "openapi.yaml").read_text(encoding="utf-8")) + validate_spec(openapi) + + print("ORSP contract validation passed") + + +if __name__ == "__main__": + main() diff --git a/tool/conformance_test.dart b/tool/conformance_test.dart new file mode 100644 index 0000000..8bc7d35 --- /dev/null +++ b/tool/conformance_test.dart @@ -0,0 +1,180 @@ +import 'dart:convert'; +import 'dart:io'; + +import '../examples/dart_server.dart'; + +Future main() async { + final server = ReferenceSourceServer(); + final baseUri = await server.start(port: 0); + final client = HttpClient(); + var checks = 0; + + void check(bool condition, String message) { + checks++; + if (!condition) throw StateError('Conformance check failed: $message'); + } + + try { + final discovery = await request( + client, + baseUri.resolve('.well-known/open-reading-source.json'), + ); + check(discovery.status == 200, 'discovery returns 200'); + check( + discovery.headers + .value(HttpHeaders.contentTypeHeader) + ?.startsWith('application/json') ?? + false, + 'discovery uses application/json', + ); + check( + discovery.headers.value('x-open-reading-protocol') == '1.0', + 'source advertises its protocol version header', + ); + final discoveryJson = discovery.json as Map; + check(discoveryJson['protocol'] == 'open-reading-source', 'protocol token'); + check(discoveryJson['protocolVersion'] == '1.0', 'protocol version'); + final capabilities = (discoveryJson['capabilities'] as List).toSet(); + check( + capabilities.containsAll(['search', 'detail', 'catalog', 'content']), + 'all core capabilities are advertised', + ); + final apiBase = Uri.parse(discoveryJson['apiBaseUrl'] as String); + check(apiBase.path.endsWith('/'), 'apiBaseUrl ends in a slash'); + + final searchPage1 = await request( + client, + apiBase.resolve('v1/search?q=Open%20Reading&page=1&pageSize=1&future=x'), + headers: {'X-Open-Reading-Protocol': '1.0'}, + ); + check(searchPage1.status == 200, 'search returns 200'); + final searchJson1 = searchPage1.json as Map; + check( + (searchJson1['items'] as List).length == 1, + 'search page size honored', + ); + check(searchJson1['pageSize'] == 1, 'search reports requested capacity'); + check(searchJson1['total'] == 2, 'search total is correct'); + check(searchJson1['hasMore'] == true, 'search hasMore is correct'); + + final searchPage2 = await request( + client, + apiBase.resolve('v1/search?q=Open%20Reading&page=2&pageSize=1'), + ); + final searchJson2 = searchPage2.json as Map; + check((searchJson2['items'] as List).length == 1, 'second search page'); + check(searchJson2['hasMore'] == false, 'last search page'); + + final detail = await request( + client, + apiBase.resolve('v1/books/protocol-garden'), + ); + check(detail.status == 200, 'detail returns 200'); + check((detail.json as Map)['id'] == 'protocol-garden', 'detail ID'); + final etag = detail.headers.value(HttpHeaders.etagHeader); + check(etag != null && etag.isNotEmpty, 'detail provides ETag'); + + final notModified = await request( + client, + apiBase.resolve('v1/books/protocol-garden'), + headers: {HttpHeaders.ifNoneMatchHeader: etag!}, + ); + check(notModified.status == 304, 'If-None-Match returns 304'); + check(notModified.body.isEmpty, '304 has no body'); + + final catalog1 = await request( + client, + apiBase.resolve('v1/books/protocol-garden/chapters?page=1&pageSize=1'), + ); + final catalogJson1 = catalog1.json as Map; + check(catalog1.status == 200, 'catalog returns 200'); + check((catalogJson1['items'] as List).length == 1, 'catalog pagination'); + check(catalogJson1['total'] == 2, 'catalog total'); + check(catalogJson1['hasMore'] == true, 'catalog hasMore'); + + final catalog2 = await request( + client, + apiBase.resolve('v1/books/protocol-garden/chapters?page=2&pageSize=1'), + ); + final catalogItems2 = (catalog2.json as Map)['items'] as List; + check(catalogItems2.single['id'] == 'common-language', 'catalog order'); + + final content = await request( + client, + apiBase.resolve('v1/books/protocol-garden/chapters/seed'), + ); + final contentJson = content.json as Map; + check(content.status == 200, 'content returns 200'); + check(contentJson['bookId'] == 'protocol-garden', 'content book ID'); + check(contentJson['chapterId'] == 'seed', 'content chapter ID'); + check(contentJson['contentType'] == 'text/plain', 'content type'); + check(Uri.parse(contentJson['baseUrl'] as String).isAbsolute, 'base URL'); + + final missingQuery = await request(client, apiBase.resolve('v1/search')); + checkError(missingQuery, 400, 'QUERY_REQUIRED', check); + + final invalidPage = await request( + client, + apiBase.resolve('v1/search?q=test&page=0'), + ); + checkError(invalidPage, 400, 'INVALID_PAGE', check); + + final missingBook = await request( + client, + apiBase.resolve('v1/books/does-not-exist'), + ); + checkError(missingBook, 404, 'BOOK_NOT_FOUND', check); + + final wrongMethod = await request( + client, + apiBase.resolve('v1/search?q=test'), + method: 'POST', + ); + checkError(wrongMethod, 405, 'METHOD_NOT_ALLOWED', check); + check( + wrongMethod.headers.value(HttpHeaders.allowHeader) == 'GET, OPTIONS', + '405 includes Allow', + ); + + stdout.writeln('ORSP conformance checks passed: $checks'); + } finally { + client.close(force: true); + await server.close(); + } +} + +void checkError( + TestResponse response, + int status, + String code, + void Function(bool, String) check, +) { + check(response.status == status, '$code returns $status'); + final body = response.json as Map; + final error = body['error'] as Map; + check(error['code'] == code, '$code uses the standard error envelope'); + check(error['message'] is String, '$code includes a message'); +} + +Future request( + HttpClient client, + Uri uri, { + String method = 'GET', + Map headers = const {}, +}) async { + final request = await client.openUrl(method, uri); + headers.forEach(request.headers.set); + final response = await request.close(); + final body = await utf8.decoder.bind(response).join(); + return TestResponse(response.statusCode, response.headers, body); +} + +class TestResponse { + final int status; + final HttpHeaders headers; + final String body; + + const TestResponse(this.status, this.headers, this.body); + + Object? get json => body.isEmpty ? null : jsonDecode(body); +}