From 7194cff6e4b8673c80256d6bf4a1f3f27b57c4a6 Mon Sep 17 00:00:00 2001 From: Iktahana <171251543+Iktahana@users.noreply.github.com> Date: Fri, 24 Jul 2026 02:50:14 +0900 Subject: [PATCH] Prepare Python Swift and Rust package releases --- .github/workflows/ci.yml | 20 ++ .github/workflows/python.yml | 73 +++++ .github/workflows/release-pypi.yml | 263 ++++++++++++++++-- .github/workflows/swift-release.yml | 84 ++++-- .gitignore | 3 + Package.swift | 2 +- docs/mdi-2.0-announcement.md | 4 +- docs/src/content/docs/bindings/swift.md | 10 +- docs/src/content/docs/ja/bindings/swift.md | 12 +- docs/src/content/docs/zh-tw/bindings/swift.md | 10 +- mdi-android-jni/Cargo.lock | 2 +- mdi-core/Cargo.lock | 2 +- mdi-core/Cargo.toml | 2 +- python/Cargo.lock | 4 +- python/Cargo.toml | 4 +- python/pyproject.toml | 3 +- python/tests/test_mdi.py | 23 ++ python/tests/test_package_smoke.py | 171 ++++++++++++ scripts/export-swift-coverage.sh | 7 +- scripts/verify-swift-package-formats.sh | 91 ++++++ scripts/write-swift-package-manifest.sh | 29 +- swift/README.md | 8 +- swift/Sources/MDI/MDI.swift | 40 ++- swift/Sources/MDICore/include/mdi_core.h | 8 + swift/Tests/MDITests/MDITests.swift | 234 +++++++++++++++- 25 files changed, 1049 insertions(+), 60 deletions(-) create mode 100644 python/tests/test_package_smoke.py create mode 100755 scripts/verify-swift-package-formats.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1b53222..8a6dfa0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -112,10 +112,24 @@ jobs: with: targets: aarch64-apple-darwin,x86_64-apple-darwin,aarch64-apple-ios,aarch64-apple-ios-sim,x86_64-apple-ios + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '21' + + - name: Install the official W3C EPUBCheck validator + run: | + curl --fail --location --silent --show-error \ + https://github.com/w3c/epubcheck/releases/download/v5.3.0/epubcheck-5.3.0.zip \ + --output "$RUNNER_TEMP/epubcheck.zip" + ditto -x -k "$RUNNER_TEMP/epubcheck.zip" "$RUNNER_TEMP" + - name: Build the Apple XCFramework run: scripts/package-swift-xcframework.sh "$RUNNER_TEMP/mdi-swift" - name: Test Swift binding with coverage + env: + MDI_SWIFT_SMOKE_OUTPUT: ${{ runner.temp }}/swift-package-smoke run: | mkdir -p swift/Artifacts ditto "$RUNNER_TEMP/mdi-swift/MDICore.xcframework" swift/Artifacts/MDICore.xcframework @@ -123,6 +137,12 @@ jobs: swift test --enable-code-coverage scripts/export-swift-coverage.sh swift/coverage.lcov + - name: Open and validate every Swift package format + run: >- + scripts/verify-swift-package-formats.sh + "$RUNNER_TEMP/swift-package-smoke" + "$RUNNER_TEMP/epubcheck-5.3.0/epubcheck.jar" + - name: Upload Swift coverage to Codecov uses: codecov/codecov-action@v5 with: diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 699fff5..d7a4a93 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -7,11 +7,13 @@ on: - python/** - mdi-core/** - .github/workflows/python.yml + - .github/workflows/release-pypi.yml pull_request: paths: - python/** - mdi-core/** - .github/workflows/python.yml + - .github/workflows/release-pypi.yml jobs: test: @@ -35,6 +37,22 @@ jobs: cache: pip cache-dependency-path: python/pyproject.toml + - uses: actions/setup-java@v4 + if: matrix.python-version == '3.14' + with: + distribution: temurin + java-version: "21" + + - uses: actions/setup-dotnet@v4 + if: matrix.python-version == '3.14' + with: + dotnet-version: "8.0.x" + + - uses: actions/setup-node@v4 + if: matrix.python-version == '3.14' + with: + node-version: "22" + - name: Build the native extension and install test dependencies run: python -m pip install -e '.[test]' @@ -43,6 +61,61 @@ jobs: - name: Run public API tests with coverage gate run: python -m pytest --cov=mdi --cov-branch --cov-report=term-missing --cov-report=xml + - name: Install authoritative publication-format consumers + if: matrix.python-version == '3.14' + working-directory: ${{ github.workspace }} + run: | + sudo apt-get update + sudo apt-get install --yes libreoffice-writer + curl --fail --location --silent --show-error \ + https://github.com/w3c/epubcheck/releases/download/v5.3.0/epubcheck-5.3.0.zip \ + --output "$RUNNER_TEMP/epubcheck.zip" + unzip -q "$RUNNER_TEMP/epubcheck.zip" -d "$RUNNER_TEMP" + npm install --prefix "$RUNNER_TEMP/vnu" --no-save vnu-jar@25.12.31 + + - name: Generate every public Python format + if: matrix.python-version == '3.14' + working-directory: ${{ github.workspace }} + run: | + python python/tests/test_package_smoke.py \ + --output-dir "$RUNNER_TEMP/python-package-smoke" + + - name: Validate HTML with the Nu Html Checker + if: matrix.python-version == '3.14' + working-directory: ${{ github.workspace }} + run: | + java -jar \ + "$RUNNER_TEMP/vnu/node_modules/vnu-jar/build/dist/vnu.jar" \ + --errors-only "$RUNNER_TEMP/python-package-smoke/smoke.html" + + - name: Validate EPUB with the official W3C EPUBCheck + if: matrix.python-version == '3.14' + working-directory: ${{ github.workspace }} + run: | + java -jar "$RUNNER_TEMP/epubcheck-5.3.0/epubcheck.jar" \ + "$RUNNER_TEMP/python-package-smoke/smoke.epub" + + - name: Validate DOCX with the Open XML SDK + if: matrix.python-version == '3.14' + working-directory: ${{ github.workspace }} + run: | + dotnet run \ + --project nodejs/format-contracts/openxml-validator/OpenXmlValidator.csproj \ + -- "$RUNNER_TEMP/python-package-smoke/smoke.docx" + + - name: Open DOCX with LibreOffice Writer + if: matrix.python-version == '3.14' + working-directory: ${{ github.workspace }} + run: | + mkdir -p "$RUNNER_TEMP/libreoffice-output" + soffice \ + "-env:UserInstallation=file://$RUNNER_TEMP/libreoffice-profile" \ + --headless \ + --convert-to pdf \ + --outdir "$RUNNER_TEMP/libreoffice-output" \ + "$RUNNER_TEMP/python-package-smoke/smoke.docx" + test -s "$RUNNER_TEMP/libreoffice-output/smoke.pdf" + - name: Upload Python coverage if: matrix.python-version == '3.14' uses: actions/upload-artifact@v4 diff --git a/.github/workflows/release-pypi.yml b/.github/workflows/release-pypi.yml index 6064215..2965413 100644 --- a/.github/workflows/release-pypi.yml +++ b/.github/workflows/release-pypi.yml @@ -2,23 +2,98 @@ name: Publish illusion-markdown to PyPI on: workflow_dispatch: + inputs: + version: + description: Package version to publish (for example, 2.0.4) + required: true + type: string + release_sha: + description: Commit on main whose complete CI run succeeded + required: true + type: string push: tags: - "illusion-markdown-v*" +concurrency: + group: release-pypi-${{ inputs.version || github.ref_name }} + cancel-in-progress: false + permissions: actions: read contents: read +env: + EXPECTED_VERSION: ${{ inputs.version || github.ref_name }} + RELEASE_SHA: ${{ inputs.release_sha || github.sha }} + RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && format('illusion-markdown-v{0}', inputs.version) || github.ref_name }} + jobs: - verify-ci: - name: Verify complete CI gate + verify-release: + name: Verify version and complete CI gate runs-on: ubuntu-latest steps: - - env: + - uses: actions/checkout@v4 + with: + ref: ${{ env.RELEASE_SHA }} + + - uses: dtolnay/rust-toolchain@stable + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Resolve the expected version + shell: bash + run: | + if [[ "$GITHUB_EVENT_NAME" == "push" ]]; then + EXPECTED_VERSION=${EXPECTED_VERSION#illusion-markdown-v} + echo "EXPECTED_VERSION=$EXPECTED_VERSION" >> "$GITHUB_ENV" + fi + + if [[ ! "$EXPECTED_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Invalid package version: $EXPECTED_VERSION" >&2 + exit 1 + fi + + - name: Verify Python and Cargo package versions agree + run: | + python - <<'PY' + import os + import tomllib + from pathlib import Path + + expected = os.environ["EXPECTED_VERSION"] + manifests = { + "python/pyproject.toml": "project", + "python/Cargo.toml": "package", + } + for filename, table in manifests.items(): + actual = tomllib.loads(Path(filename).read_text())[table]["version"] + if actual != expected: + raise SystemExit(f"{filename} has version {actual}, expected {expected}") + PY + + cargo_version=$( + cargo metadata --locked --no-deps \ + --manifest-path python/Cargo.toml \ + --format-version 1 | + jq -r '.packages[] | select(.name == "illusion-markdown-python") | .version' + ) + test "$cargo_version" = "$EXPECTED_VERSION" + + - name: Verify the release commit passed the complete CI gate + env: GH_TOKEN: ${{ github.token }} - RELEASE_SHA: ${{ github.sha }} run: | + if [[ "$GITHUB_EVENT_NAME" == "workflow_dispatch" ]]; then + main_sha=$(gh api "repos/$GITHUB_REPOSITORY/git/ref/heads/main" --jq '.object.sha') + test "$RELEASE_SHA" = "$main_sha" + fi + + checked_out_sha=$(git rev-parse HEAD) + test "$checked_out_sha" = "$RELEASE_SHA" + count=$( gh api --method GET \ "repos/$GITHUB_REPOSITORY/actions/workflows/ci.yml/runs" \ @@ -29,45 +104,88 @@ jobs: ) test "$count" -gt 0 + - name: Verify the release tag is not attached to a different commit + env: + GH_TOKEN: ${{ github.token }} + run: | + tag_sha=$( + gh api "repos/$GITHUB_REPOSITORY/commits/$RELEASE_TAG" \ + --jq '.sha' 2>/dev/null || true + ) + if [[ -n "$tag_sha" ]]; then + test "$tag_sha" = "$RELEASE_SHA" + fi + build-wheels: - name: Build ${{ matrix.target }} wheel - needs: verify-ci + name: Build and test ${{ matrix.target }} wheel + needs: verify-release runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: include: - os: ubuntu-latest - target: x86_64 + target: linux-x86_64 manylinux: "2014" - os: macos-15-intel - target: x86_64 + target: macos-x86_64 - os: macos-15 - target: aarch64 + target: macos-aarch64 - os: windows-latest - target: x86_64 + target: windows-x86_64 steps: - uses: actions/checkout@v4 + with: + ref: ${{ env.RELEASE_SHA }} + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Resolve the expected version + if: github.event_name == 'push' + shell: bash + run: echo "EXPECTED_VERSION=${EXPECTED_VERSION#illusion-markdown-v}" >> "$GITHUB_ENV" - name: Build wheel uses: PyO3/maturin-action@v1 with: command: build - args: --release --manifest-path python/Cargo.toml --out dist + args: --release --locked --manifest-path python/Cargo.toml --out dist manylinux: ${{ matrix.manylinux }} + - name: Install and smoke-test the wheel + shell: bash + run: | + python -m pip install --no-index --find-links dist "illusion-markdown==$EXPECTED_VERSION" + python python/tests/test_package_smoke.py \ + --expected-version "$EXPECTED_VERSION" + - uses: actions/upload-artifact@v4 with: - name: wheels-${{ matrix.os }}-${{ matrix.target }} + name: wheel-${{ matrix.target }} path: dist/*.whl if-no-files-found: error build-sdist: - name: Build source distribution - needs: verify-ci + name: Build and test source distribution + needs: verify-release runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + ref: ${{ env.RELEASE_SHA }} + + - uses: dtolnay/rust-toolchain@stable + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Resolve the expected version + if: github.event_name == 'push' + shell: bash + run: echo "EXPECTED_VERSION=${EXPECTED_VERSION#illusion-markdown-v}" >> "$GITHUB_ENV" - name: Build source distribution uses: PyO3/maturin-action@v1 @@ -75,25 +193,138 @@ jobs: command: sdist args: --manifest-path python/Cargo.toml --out dist + - name: Install and smoke-test the source distribution + run: | + python -m pip install dist/*.tar.gz + python python/tests/test_package_smoke.py \ + --expected-version "$EXPECTED_VERSION" + - uses: actions/upload-artifact@v4 with: name: source-distribution path: dist/*.tar.gz if-no-files-found: error + validate-formats: + name: Validate installed-package publication formats + needs: [verify-release, build-wheels] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ env.RELEASE_SHA }} + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "21" + + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: "8.0.x" + + - uses: actions/setup-node@v4 + with: + node-version: "22" + + - name: Resolve the expected version + if: github.event_name == 'push' + shell: bash + run: echo "EXPECTED_VERSION=${EXPECTED_VERSION#illusion-markdown-v}" >> "$GITHUB_ENV" + + - uses: actions/download-artifact@v4 + with: + name: wheel-linux-x86_64 + path: dist + + - name: Install the built wheel and authoritative consumers + run: | + python -m pip install --no-index --find-links dist \ + "illusion-markdown==$EXPECTED_VERSION" + sudo apt-get update + sudo apt-get install --yes libreoffice-writer + curl --fail --location --silent --show-error \ + https://github.com/w3c/epubcheck/releases/download/v5.3.0/epubcheck-5.3.0.zip \ + --output "$RUNNER_TEMP/epubcheck.zip" + unzip -q "$RUNNER_TEMP/epubcheck.zip" -d "$RUNNER_TEMP" + npm install --prefix "$RUNNER_TEMP/vnu" --no-save vnu-jar@25.12.31 + + - name: Generate every public format from the installed wheel + run: | + python python/tests/test_package_smoke.py \ + --expected-version "$EXPECTED_VERSION" \ + --output-dir "$RUNNER_TEMP/python-package-smoke" + + - name: Validate HTML with the Nu Html Checker + run: | + java -jar \ + "$RUNNER_TEMP/vnu/node_modules/vnu-jar/build/dist/vnu.jar" \ + --errors-only "$RUNNER_TEMP/python-package-smoke/smoke.html" + + - name: Validate EPUB with the official W3C EPUBCheck + run: | + java -jar "$RUNNER_TEMP/epubcheck-5.3.0/epubcheck.jar" \ + "$RUNNER_TEMP/python-package-smoke/smoke.epub" + + - name: Validate DOCX with the Open XML SDK + run: | + dotnet run \ + --project nodejs/format-contracts/openxml-validator/OpenXmlValidator.csproj \ + -- "$RUNNER_TEMP/python-package-smoke/smoke.docx" + + - name: Open DOCX with LibreOffice Writer + run: | + mkdir -p "$RUNNER_TEMP/libreoffice-output" + soffice \ + "-env:UserInstallation=file://$RUNNER_TEMP/libreoffice-profile" \ + --headless \ + --convert-to pdf \ + --outdir "$RUNNER_TEMP/libreoffice-output" \ + "$RUNNER_TEMP/python-package-smoke/smoke.docx" + test -s "$RUNNER_TEMP/libreoffice-output/smoke.pdf" + publish: - name: Publish to PyPI - needs: [build-wheels, build-sdist] + name: Publish to PyPI and create GitHub release + needs: [build-wheels, build-sdist, validate-formats] runs-on: ubuntu-latest environment: pypi permissions: + actions: read + contents: write id-token: write steps: + - name: Resolve the expected version + if: github.event_name == 'push' + shell: bash + run: echo "EXPECTED_VERSION=${EXPECTED_VERSION#illusion-markdown-v}" >> "$GITHUB_ENV" + - uses: actions/download-artifact@v4 with: pattern: "*" merge-multiple: true path: dist + - name: Check package distributions + run: | + python -m pip install --disable-pip-version-check twine + python -m twine check dist/* + test "$(find dist -maxdepth 1 -name '*.whl' | wc -l)" -eq 4 + test "$(find dist -maxdepth 1 -name '*.tar.gz' | wc -l)" -eq 1 + - name: Publish package distributions to PyPI uses: pypa/gh-action-pypi-publish@release/v1 + + - name: Create the GitHub release + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release create "$RELEASE_TAG" \ + --repo "$GITHUB_REPOSITORY" \ + --target "$RELEASE_SHA" \ + --title "illusion-markdown $EXPECTED_VERSION" \ + --generate-notes \ + dist/* diff --git a/.github/workflows/swift-release.yml b/.github/workflows/swift-release.yml index 7978904..2cfd471 100644 --- a/.github/workflows/swift-release.yml +++ b/.github/workflows/swift-release.yml @@ -9,15 +9,15 @@ on: type: choice options: [prepare, publish] version: - description: Complete SemVer version (for example, 2.0.1) + description: Complete SemVer version (for example, 2.0.3) required: true type: string prepare_run_id: description: Required for publish; the successful prepare workflow run ID required: false type: string - force_retag: - description: "Recovery only: replace an existing tag with the approved main commit" + recover_existing_tag: + description: "Recovery only: allow an unpublished tag that already points to the approved main commit" required: false default: false type: boolean @@ -28,7 +28,7 @@ permissions: pull-requests: write concurrency: - group: swift-release-${{ inputs.version }}-${{ inputs.action }} + group: swift-release-${{ inputs.version }} cancel-in-progress: false jobs: @@ -61,25 +61,51 @@ jobs: with: targets: aarch64-apple-darwin,x86_64-apple-darwin,aarch64-apple-ios,aarch64-apple-ios-sim,x86_64-apple-ios + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '21' + - name: Validate version env: + GH_TOKEN: ${{ github.token }} VERSION: ${{ inputs.version }} - FORCE_RETAG: ${{ inputs.force_retag }} run: | [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] if git rev-parse --verify --quiet "refs/tags/$VERSION"; then - [[ "$FORCE_RETAG" == "true" ]] + echo "tag $VERSION already exists" >&2 + exit 1 fi + if gh release view "$VERSION" --repo "$GITHUB_REPOSITORY"; then + echo "release $VERSION already exists" >&2 + exit 1 + fi + + - name: Install the official W3C EPUBCheck validator + run: | + curl --fail --location --silent --show-error \ + https://github.com/w3c/epubcheck/releases/download/v5.3.0/epubcheck-5.3.0.zip \ + --output "$RUNNER_TEMP/epubcheck.zip" + ditto -x -k "$RUNNER_TEMP/epubcheck.zip" "$RUNNER_TEMP" - name: Build the Apple XCFramework run: scripts/package-swift-xcframework.sh "$RUNNER_TEMP/mdi-swift" - - name: Test against the local XCFramework + - name: Test the local XCFramework with the coverage gate + env: + MDI_SWIFT_SMOKE_OUTPUT: ${{ runner.temp }}/swift-package-smoke run: | mkdir -p swift/Artifacts ditto "$RUNNER_TEMP/mdi-swift/MDICore.xcframework" swift/Artifacts/MDICore.xcframework scripts/write-swift-package-manifest.sh local "swift/Artifacts/MDICore.xcframework" - swift test + swift test --enable-code-coverage + scripts/export-swift-coverage.sh "$RUNNER_TEMP/swift-coverage.lcov" + + - name: Open and validate every Swift package format + run: >- + scripts/verify-swift-package-formats.sh + "$RUNNER_TEMP/swift-package-smoke" + "$RUNNER_TEMP/epubcheck-5.3.0/epubcheck.jar" - name: Archive the XCFramework id: artifact @@ -141,15 +167,43 @@ jobs: - name: Validate approved release inputs env: + GH_TOKEN: ${{ github.token }} VERSION: ${{ inputs.version }} PREPARE_RUN_ID: ${{ inputs.prepare_run_id }} - FORCE_RETAG: ${{ inputs.force_retag }} + RECOVER_EXISTING_TAG: ${{ inputs.recover_existing_tag }} run: | [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] - [[ -n "$PREPARE_RUN_ID" ]] + [[ "$PREPARE_RUN_ID" =~ ^[0-9]+$ ]] + release_sha=$(git rev-parse HEAD) if git rev-parse --verify --quiet "refs/tags/$VERSION"; then - [[ "$FORCE_RETAG" == "true" ]] + [[ "$RECOVER_EXISTING_TAG" == "true" ]] + [[ "$(git rev-list -n 1 "$VERSION")" == "$release_sha" ]] + else + [[ "$RECOVER_EXISTING_TAG" == "false" ]] fi + if gh release view "$VERSION" --repo "$GITHUB_REPOSITORY"; then + echo "release $VERSION already exists" >&2 + exit 1 + fi + + - name: Verify the successful prepare run + env: + GH_TOKEN: ${{ github.token }} + VERSION: ${{ inputs.version }} + PREPARE_RUN_ID: ${{ inputs.prepare_run_id }} + run: | + run_metadata=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$PREPARE_RUN_ID") + [[ "$(jq -r '.path' <<<"$run_metadata")" == ".github/workflows/swift-release.yml" ]] + [[ "$(jq -r '.event' <<<"$run_metadata")" == "workflow_dispatch" ]] + [[ "$(jq -r '.head_branch' <<<"$run_metadata")" == "main" ]] + [[ "$(jq -r '.conclusion' <<<"$run_metadata")" == "success" ]] + prepare_jobs=$( + gh api "repos/$GITHUB_REPOSITORY/actions/runs/$PREPARE_RUN_ID/jobs" \ + -f per_page=100 \ + --method GET \ + --jq "[.jobs[] | select(.name == \"Prepare MDI $VERSION\" and .conclusion == \"success\")] | length" + ) + [[ "$prepare_jobs" == "1" ]] - name: Download the approved XCFramework env: @@ -170,12 +224,8 @@ jobs: - name: Tag the approved main commit env: VERSION: ${{ inputs.version }} - FORCE_RETAG: ${{ inputs.force_retag }} run: | - if [[ "$FORCE_RETAG" == "true" ]]; then - git tag -f "$VERSION" - git push --force origin "refs/tags/$VERSION" - else + if ! git rev-parse --verify --quiet "refs/tags/$VERSION"; then git tag "$VERSION" git push origin "$VERSION" fi @@ -184,4 +234,4 @@ jobs: env: GH_TOKEN: ${{ github.token }} VERSION: ${{ inputs.version }} - run: gh release create "$VERSION" "$RUNNER_TEMP/release/MDICore.xcframework.zip#MDICore.xcframework.zip" --title "MDI $VERSION" --generate-notes + run: gh release create "$VERSION" "$RUNNER_TEMP/release/MDICore.xcframework.zip#MDICore.xcframework.zip" --verify-tag --title "MDI $VERSION" --generate-notes diff --git a/.gitignore b/.gitignore index 3461bf4..f9676ff 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,9 @@ dist .turbo *.tsbuildinfo coverage +.coverage +/output/ +/tmp/ .DS_Store *.log target/ diff --git a/Package.swift b/Package.swift index 2980203..d302e8e 100644 --- a/Package.swift +++ b/Package.swift @@ -10,6 +10,6 @@ let package = Package( targets: [ .binaryTarget(name: "MDICore", url: "https://github.com/illusions-lab/MDI/releases/download/2.0.2/MDICore.xcframework.zip", checksum: "d1fd0fedd6cb5464edbd193efac60052589dc0a71cef380e64f15197c319fc89"), .target(name: "MDI", dependencies: ["MDICore"], path: "swift/Sources/MDI"), - .testTarget(name: "MDITests", dependencies: ["MDI"], path: "swift/Tests/MDITests"), + .testTarget(name: "MDITests", dependencies: ["MDI", "MDICore"], path: "swift/Tests/MDITests"), ] ) diff --git a/docs/mdi-2.0-announcement.md b/docs/mdi-2.0-announcement.md index 218eb54..fa23c85 100644 --- a/docs/mdi-2.0-announcement.md +++ b/docs/mdi-2.0-announcement.md @@ -144,8 +144,8 @@ MDI 固有の記法は導入せず、GFM / Pandoc の脚注記法をそのまま | フレーバー | 用途 | |-----------|------| -| `plain` | ルビを捨てた素のテキスト | -| `ruby-paren` | ルビを全角括弧で表現: `漢字（かんじ）` | +| `txt` | ルビを捨てた素のテキスト | +| `txt-ruby` | ルビを再解析可能な MDI 形式で保持: `{漢字\|かんじ}` | | `narou` | 小説家になろう投稿用（傍点は圏点ルビで表現） | | `kakuyomu` | カクヨム投稿用（傍点は `《《》》` 記法） | | `aozora` | 青空文庫注記形式 | diff --git a/docs/src/content/docs/bindings/swift.md b/docs/src/content/docs/bindings/swift.md index 56daf0e..1c7d7a9 100644 --- a/docs/src/content/docs/bindings/swift.md +++ b/docs/src/content/docs/bindings/swift.md @@ -21,7 +21,7 @@ that uses it: ```swift dependencies: [ - .package(url: "https://github.com/illusions-lab/MDI.git", from: "2.0.2"), + .package(url: "https://github.com/illusions-lab/MDI.git", from: "2.0.3"), ] // In a target: @@ -55,11 +55,17 @@ All renderers take MDI source text: let html = try MDI.renderHTML("{東京|とうきょう} ^12^") let mdi = try MDI.serialize("{東京|とうきょう} ^12^") let text = try MDI.renderText("# Title") +let note = try MDI.renderTextFormat( + "# Title\n\n{東京|とうきょう}", + format: .note +) let epub: Data = try MDI.renderEPUB("# Chapter") let docx: Data = try MDI.renderDOCX("# Chapter") ``` +`MDITextFormat` exposes the same six Rust-owned conventions as the other +bindings: `plain`, `ruby`, `narou`, `kakuyomu`, `aozora`, and `note`. `renderEPUB` and `renderDOCX` return ZIP-based `Data`; write the data to a file with the appropriate extension. @@ -83,7 +89,7 @@ do { ## Development and releases The repository's `swift/Package.swift` is the local development package. CI -builds an XCFramework, runs XCTest with a 90% line-coverage gate for +builds an XCFramework, runs XCTest with a 95% line-coverage gate for `swift/Sources/MDI`, and uploads the report to Codecov. The release workflow creates a manifest pull request and publishes the approved artifact after that PR is merged. It uses GitHub Actions' built-in token; no PAT or second diff --git a/docs/src/content/docs/ja/bindings/swift.md b/docs/src/content/docs/ja/bindings/swift.md index 33363a4..6d19dcf 100644 --- a/docs/src/content/docs/ja/bindings/swift.md +++ b/docs/src/content/docs/ja/bindings/swift.md @@ -17,7 +17,7 @@ Swift は小さな C ABI を通じて Rust の `mdi-core` に解析とレンダ ```swift dependencies: [ - .package(url: "https://github.com/illusions-lab/MDI.git", from: "2.0.2"), + .package(url: "https://github.com/illusions-lab/MDI.git", from: "2.0.3"), ] // target の dependencies: @@ -43,15 +43,21 @@ print(result.diagnostics) let html = try MDI.renderHTML("{東京|とうきょう} ^12^") let mdi = try MDI.serialize("{東京|とうきょう} ^12^") let text = try MDI.renderText("# Title") +let note = try MDI.renderTextFormat( + "# Title\n\n{東京|とうきょう}", + format: .note +) let epub: Data = try MDI.renderEPUB("# Chapter") let docx: Data = try MDI.renderDOCX("# Chapter") ``` +`MDITextFormat` は他の binding と同じ6種類（`plain`、`ruby`、`narou`、 +`kakuyomu`、`aozora`、`note`）を Rust core から提供します。 EPUB と DOCX は ZIP ベースの `Data` を返すため、対応する拡張子でファイルへ書き出してください。 ## エラー -すべての API は `MDIError` を throw します。`core` は Rust core の失敗、`invalidWireFormat` は無効または未対応の native response を表します。CI は XCFramework をビルドし、XCTest を実行して `swift/Sources/MDI` に 90% の line coverage を要求し、Codecov へ送信します。PAT や別リポジトリは必要ありません。 +すべての API は `MDIError` を throw します。`core` は Rust core の失敗、`invalidWireFormat` は無効または未対応の native response を表します。 ```swift do { @@ -64,4 +70,4 @@ do { ## 開発とリリース -リポジトリの `swift/Package.swift` はローカル開発用パッケージです。CI は XCFramework をビルドし、XCTest を実行して `swift/Sources/MDI` に 90% の line-coverage gate を適用し、レポートを Codecov に送信します。release workflow は manifest 用の pull request を作成し、その PR がマージされた後に承認済み artifact を公開します。GitHub Actions 組み込みの token を使うため、PAT や別リポジトリは不要です。 +リポジトリの `swift/Package.swift` はローカル開発用パッケージです。CI は XCFramework をビルドし、XCTest を実行して `swift/Sources/MDI` に 95% の line-coverage gate を適用し、レポートを Codecov に送信します。release workflow は manifest 用の pull request を作成し、その PR がマージされた後に承認済み artifact を公開します。GitHub Actions 組み込みの token を使うため、PAT や別リポジトリは不要です。 diff --git a/docs/src/content/docs/zh-tw/bindings/swift.md b/docs/src/content/docs/zh-tw/bindings/swift.md index d92bfdb..b97bd43 100644 --- a/docs/src/content/docs/zh-tw/bindings/swift.md +++ b/docs/src/content/docs/zh-tw/bindings/swift.md @@ -17,7 +17,7 @@ Swift 會透過精簡的 C ABI 將解析和轉譯交給 Rust 的 `mdi-core`， ```swift dependencies: [ - .package(url: "https://github.com/illusions-lab/MDI.git", from: "2.0.2"), + .package(url: "https://github.com/illusions-lab/MDI.git", from: "2.0.3"), ] // target 的 dependencies： @@ -43,10 +43,16 @@ print(result.diagnostics) let html = try MDI.renderHTML("{東京|とうきょう} ^12^") let mdi = try MDI.serialize("{東京|とうきょう} ^12^") let text = try MDI.renderText("# Title") +let note = try MDI.renderTextFormat( + "# Title\n\n{東京|とうきょう}", + format: .note +) let epub: Data = try MDI.renderEPUB("# Chapter") let docx: Data = try MDI.renderDOCX("# Chapter") ``` +`MDITextFormat` 提供與其他綁定相同的六種 Rust 格式：`plain`、`ruby`、 +`narou`、`kakuyomu`、`aozora` 與 `note`。 EPUB 與 DOCX 回傳 ZIP 格式的 `Data`，請以對應副檔名寫入檔案。 ## 錯誤 @@ -64,4 +70,4 @@ do { ## 開發與發布 -repository 的 `swift/Package.swift` 是本機開發用 package。CI 會建置 XCFramework、跑 XCTest，並對 `swift/Sources/MDI` 強制 90% line coverage、上傳 Codecov。release workflow 會建立 manifest pull request；該 PR 合併後才發布核准的 artifact。它使用 GitHub Actions 內建 token，不需要 PAT 或第二個 repository。 +repository 的 `swift/Package.swift` 是本機開發用 package。CI 會建置 XCFramework、跑 XCTest，並對 `swift/Sources/MDI` 強制 95% line coverage、上傳 Codecov。release workflow 會建立 manifest pull request；該 PR 合併後才發布核准的 artifact。它使用 GitHub Actions 內建 token，不需要 PAT 或第二個 repository。 diff --git a/mdi-android-jni/Cargo.lock b/mdi-android-jni/Cargo.lock index 139b936..d942171 100644 --- a/mdi-android-jni/Cargo.lock +++ b/mdi-android-jni/Cargo.lock @@ -205,7 +205,7 @@ dependencies = [ [[package]] name = "mdi-core" -version = "2.0.4" +version = "2.0.5" dependencies = [ "markdown", "serde", diff --git a/mdi-core/Cargo.lock b/mdi-core/Cargo.lock index 8817aad..0a47e78 100644 --- a/mdi-core/Cargo.lock +++ b/mdi-core/Cargo.lock @@ -166,7 +166,7 @@ dependencies = [ [[package]] name = "mdi-core" -version = "2.0.4" +version = "2.0.5" dependencies = [ "js-sys", "markdown", diff --git a/mdi-core/Cargo.toml b/mdi-core/Cargo.toml index 7e3dda0..69cd832 100644 --- a/mdi-core/Cargo.toml +++ b/mdi-core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mdi-core" -version = "2.0.4" +version = "2.0.5" description = "Language-neutral parser for MDI 2.0 syntax" edition = "2024" rust-version = "1.85" diff --git a/python/Cargo.lock b/python/Cargo.lock index 760f659..a6d45a0 100644 --- a/python/Cargo.lock +++ b/python/Cargo.lock @@ -111,7 +111,7 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "illusion-markdown-python" -version = "2.0.3" +version = "2.0.4" dependencies = [ "mdi-core", "pyo3", @@ -166,7 +166,7 @@ dependencies = [ [[package]] name = "mdi-core" -version = "2.0.4" +version = "2.0.5" dependencies = [ "markdown", "serde", diff --git a/python/Cargo.toml b/python/Cargo.toml index de31c55..9341e48 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "illusion-markdown-python" -version = "2.0.3" +version = "2.0.4" description = "PyO3 bindings for illusion Markdown (MDI)" edition = "2024" license = "MIT" @@ -8,6 +8,8 @@ publish = false readme = "README.md" repository = "https://github.com/illusions-lab/MDI" exclude = [ + ".pytest_cache/**", + "coverage*.xml", "src/mdi/_native.abi3.so", "src/**/__pycache__/**", "tests/**/__pycache__/**", diff --git a/python/pyproject.toml b/python/pyproject.toml index 90ac754..a13cbcd 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "illusion-markdown" -version = "2.0.3" +version = "2.0.4" description = "Python bindings for illusion Markdown (MDI), backed by the Rust core." readme = "README.md" requires-python = ">=3.10" @@ -16,6 +16,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Programming Language :: Rust", "Topic :: Text Processing :: Markup", ] diff --git a/python/tests/test_mdi.py b/python/tests/test_mdi.py index faa05af..9c23e34 100644 --- a/python/tests/test_mdi.py +++ b/python/tests/test_mdi.py @@ -82,6 +82,29 @@ def test_returns_recoverable_diagnostics_with_utf8_byte_spans() -> None: assert result["document"]["span"]["endByte"] == len(source.encode()) +@pytest.mark.parametrize( + "source", + [ + "\\{}《《傍点》》\n\n\\[{東京|とう.きょう}", + "👨‍👩‍👧 [[em:**強調**]] [^n]\n\n[^n]: 注", + "[[indent:2]]\n{𠮟る|しか.る} [[no-break:^12^]]\n\n[[pagebreak:left]]", + "```mdi\n{東京|とうきょう}\n```\n\n| a | b |\n| - | - |\n| [[em:x]] | ^12^ |", + ], +) +def test_keeps_the_js_adversarial_corpus_inside_the_rust_wire_contract( + source: str, +) -> None: + result = mdi.parse(source) + + assert result["document"]["span"] == { + "startByte": 0, + "endByte": len(source.encode()), + } + assert_valid_spans(result["document"], source) + assert mdi.render_html(source).startswith("") + assert mdi.parse(mdi.serialize_mdi(source))["irVersion"] == mdi.MDI_IR_VERSION + + def test_serializes_and_renders_complete_source_in_rust() -> None: source = "# 題\n\n{東京|とうきょう} ^12^" assert "

題

" in mdi.render_html(source) diff --git a/python/tests/test_package_smoke.py b/python/tests/test_package_smoke.py new file mode 100644 index 0000000..760886a --- /dev/null +++ b/python/tests/test_package_smoke.py @@ -0,0 +1,171 @@ +"""Package-level smoke tests shared by pytest and distribution workflows. + +The release jobs execute this file against an installed wheel or sdist. Keep +it limited to the public ``mdi`` API and the Python standard library so that it +cannot accidentally pass by importing repository implementation details. +""" + +from __future__ import annotations + +import argparse +from html.parser import HTMLParser +from importlib.metadata import version +from pathlib import Path +from tempfile import TemporaryDirectory +from xml.etree import ElementTree +from zipfile import ZIP_STORED, ZipFile + +import mdi + + +SOURCE = """--- +title: 東京の夜 +lang: ja +author: MDI +--- + +# {東京|とうきょう}の夜 + +本文 ^12^ + +[[pagebreak]] + +## 第二章 +""" + + +class _TagCollector(HTMLParser): + def __init__(self) -> None: + super().__init__() + self.tags: list[str] = [] + + def handle_starttag( + self, + tag: str, + attrs: list[tuple[str, str | None]], + ) -> None: + del attrs + self.tags.append(tag) + + +def _assert_xml_parts_parse(archive: ZipFile, suffixes: tuple[str, ...]) -> None: + for name in archive.namelist(): + if name.endswith(suffixes): + ElementTree.fromstring(archive.read(name)) + + +def run_package_smoke(output_directory: Path, expected_version: str | None = None) -> None: + """Exercise every format exposed by the installed Python distribution.""" + + output_directory.mkdir(parents=True, exist_ok=True) + if expected_version is not None: + assert version("illusion-markdown") == expected_version + + html = mdi.render_html(SOURCE) + assert html.startswith("") + assert '' in html + assert "東京の夜" in html + assert '' in html + html_parser = _TagCollector() + html_parser.feed(html) + html_parser.close() + assert {"html", "head", "body", "h1", "ruby", "rt"}.issubset(html_parser.tags) + (output_directory / "smoke.html").write_text(html, encoding="utf-8") + + text = mdi.render_text(SOURCE) + assert text == "東京の夜\n本文 12\n\n\f\n第二章\n" + (output_directory / "smoke.txt").write_text(text, encoding="utf-8") + + text_formats = { + "txt": "東京の夜\n\u3000本文 12\n\n第二章", + "txt-ruby": "{東京|とうきょう}の夜\n\u3000本文 12\n\n第二章", + "narou": "｜東京《とうきょう》の夜\n\u3000本文 12\n\n第二章", + "kakuyomu": "｜東京《とうきょう》の夜\n\u3000本文 12\n\n第二章", + "aozora": ( + "｜東京《とうきょう》の夜［＃「東京の夜」は大見出し］\r\n" + "\u3000本文 12［＃「12」は縦中横］\r\n" + "［＃改ページ］\r\n" + "第二章［＃「第二章」は中見出し］" + ), + "note": ( + "## ｜東京《とうきょう》の夜\n\n" + "\u3000本文 12\n\n" + "---\n\n" + "### 第二章" + ), + } + for text_format, expected in text_formats.items(): + rendered = mdi.render_text_format(SOURCE, text_format, "\u3000") # type: ignore[arg-type] + assert rendered == expected + (output_directory / f"smoke-{text_format}.txt").write_bytes(rendered.encode()) + + canonical = mdi.serialize_mdi(SOURCE) + assert mdi.serialize_mdi(canonical) == canonical + parsed = mdi.parse(canonical) + assert parsed["irVersion"] == mdi.MDI_IR_VERSION + assert parsed["syntaxVersion"] == mdi.MDI_SPEC_VERSION + assert [node["type"] for node in parsed["document"]["children"]] == [ + "heading", + "paragraph", + "pagebreak", + "heading", + ] + (output_directory / "smoke.mdi").write_text(canonical, encoding="utf-8") + + epub_path = output_directory / "smoke.epub" + epub_path.write_bytes(mdi.render_epub(SOURCE)) + with ZipFile(epub_path) as archive: + names = set(archive.namelist()) + assert archive.infolist()[0].filename == "mimetype" + assert archive.infolist()[0].compress_type == ZIP_STORED + assert archive.read("mimetype") == b"application/epub+zip" + assert { + "META-INF/container.xml", + "OEBPS/package.opf", + "OEBPS/nav.xhtml", + "OEBPS/chapter-1.xhtml", + "OEBPS/chapter-2.xhtml", + }.issubset(names) + assert "東京の夜" in archive.read("OEBPS/package.opf").decode() + assert "とうきょう" in archive.read("OEBPS/chapter-1.xhtml").decode() + _assert_xml_parts_parse(archive, (".xml", ".opf", ".xhtml")) + + docx_path = output_directory / "smoke.docx" + docx_path.write_bytes(mdi.render_docx(SOURCE)) + with ZipFile(docx_path) as archive: + names = set(archive.namelist()) + assert { + "[Content_Types].xml", + "_rels/.rels", + "docProps/core.xml", + "word/document.xml", + "word/styles.xml", + "word/_rels/document.xml.rels", + }.issubset(names) + document_xml = archive.read("word/document.xml").decode() + assert "東京" in document_xml + assert "とうきょう" in document_xml + assert 'w:type="page"' in document_xml + _assert_xml_parts_parse(archive, (".xml", ".rels")) + + +def test_every_public_format_survives_package_level_smoke(tmp_path: Path) -> None: + run_package_smoke(tmp_path) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--output-dir", type=Path) + parser.add_argument("--expected-version") + args = parser.parse_args() + + if args.output_dir is not None: + run_package_smoke(args.output_dir, args.expected_version) + return + + with TemporaryDirectory(prefix="mdi-python-smoke-") as directory: + run_package_smoke(Path(directory), args.expected_version) + + +if __name__ == "__main__": + main() diff --git a/scripts/export-swift-coverage.sh b/scripts/export-swift-coverage.sh index 8caa13c..4a91b20 100755 --- a/scripts/export-swift-coverage.sh +++ b/scripts/export-swift-coverage.sh @@ -18,13 +18,14 @@ coverage_percent="$(jq -r ' [ .data[].files[] | select(.filename | endswith("/swift/Sources/MDI/MDI.swift")) | .summary.lines - | (.covered / .count * 100) + | if .count > 0 then (.covered / .count * 100) else error("MDI source has no coverable lines") end ] | if length == 1 then .[0] else error("expected exactly one MDI source coverage record") end ' "$coverage_json")" awk -v coverage="$coverage_percent" 'BEGIN { - printf "Swift MDI line coverage: %.2f%%\\n", coverage - exit !(coverage >= 90) + minimum = 95 + printf "Swift MDI line coverage: %.2f%% (required: %.2f%%)\n", coverage, minimum + exit !(coverage >= minimum) }' xcrun llvm-cov export "$test_binary" \ diff --git a/scripts/verify-swift-package-formats.sh b/scripts/verify-swift-package-formats.sh new file mode 100755 index 0000000..008a5d9 --- /dev/null +++ b/scripts/verify-swift-package-formats.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 2 ]]; then + echo "usage: $0 " >&2 + exit 64 +fi + +output_directory="$1" +epubcheck="$2" + +if [[ ! -d "$output_directory" ]]; then + echo "Swift smoke output directory not found: $output_directory" >&2 + exit 66 +fi +if [[ ! -f "$epubcheck" ]]; then + echo "EPUBCheck not found: $epubcheck" >&2 + exit 66 +fi +if [[ "$epubcheck" != *.jar && ! -x "$epubcheck" ]]; then + echo "EPUBCheck is not executable: $epubcheck" >&2 + exit 66 +fi + +for name in \ + document.html \ + document.txt \ + document.mdi \ + document.epub \ + document.docx \ + document-txt.txt \ + document-txt-ruby.txt \ + document-narou.txt \ + document-kakuyomu.txt \ + document-aozora.txt \ + document-note.txt; do + if [[ ! -s "$output_directory/$name" ]]; then + echo "Swift smoke artifact is missing or empty: $name" >&2 + exit 65 + fi +done + +temporary_directory="$(mktemp -d "${TMPDIR:-/tmp}/mdi-swift-consumers.XXXXXX")" +trap 'rm -rf "$temporary_directory"' EXIT + +# TextEdit's command-line frontend uses Apple's native document importers. This +# opens both the HTML and DOCX outputs rather than merely checking signatures. +textutil -convert txt \ + -output "$temporary_directory/html.txt" \ + "$output_directory/document.html" +textutil -convert txt \ + -output "$temporary_directory/docx.txt" \ + "$output_directory/document.docx" +grep -Fq "Swift package contract" "$temporary_directory/html.txt" +grep -Fq "Swift package contract" "$temporary_directory/docx.txt" + +# Plain text and canonical MDI are UTF-8 text contracts. MDI is additionally +# reparsed and checked for canonical idempotence inside MDITests. +iconv -f UTF-8 -t UTF-8 "$output_directory/document.txt" >/dev/null +iconv -f UTF-8 -t UTF-8 "$output_directory/document.mdi" >/dev/null +grep -Fq "東京" "$output_directory/document.txt" +grep -Fq "{東京|とうきょう}" "$output_directory/document.mdi" +for name in \ + document-txt.txt \ + document-txt-ruby.txt \ + document-narou.txt \ + document-kakuyomu.txt \ + document-aozora.txt \ + document-note.txt; do + iconv -f UTF-8 -t UTF-8 "$output_directory/$name" >/dev/null +done +grep -Fq "{東京|とうきょう}" "$output_directory/document-txt-ruby.txt" +grep -Fq "｜東京《とうきょう》" "$output_directory/document-narou.txt" +grep -Fq "｜東京《とうきょう》" "$output_directory/document-kakuyomu.txt" +grep -Fq "｜東京《とうきょう》" "$output_directory/document-aozora.txt" +grep -Fq "｜東京《とうきょう》" "$output_directory/document-note.txt" +grep -Fq "## Swift package contract" "$output_directory/document-note.txt" + +# Exercise the ZIP readers before the format-specific validators/importers so +# corrupt central directories fail with a direct diagnostic. +unzip -tqq "$output_directory/document.epub" +unzip -tqq "$output_directory/document.docx" + +# EPUBCheck is the official EPUB conformance checker maintained by the W3C. +if [[ "$epubcheck" == *.jar ]]; then + java -jar "$epubcheck" "$output_directory/document.epub" +else + "$epubcheck" "$output_directory/document.epub" +fi + +echo "Swift package format smoke passed: HTML, TXT (all 6 conventions), MDI, EPUB, DOCX" diff --git a/scripts/write-swift-package-manifest.sh b/scripts/write-swift-package-manifest.sh index 43136e5..e30235e 100755 --- a/scripts/write-swift-package-manifest.sh +++ b/scripts/write-swift-package-manifest.sh @@ -15,6 +15,18 @@ case "$mode" in echo "usage: $0 local " >&2 exit 64 fi + xcframework_path="$2" + if [[ "$xcframework_path" != /* ]]; then + xcframework_path="$root_directory/$xcframework_path" + fi + if [[ ! -d "$xcframework_path" ]]; then + echo "XCFramework not found: $2" >&2 + exit 66 + fi + if [[ "$2" == *'"'* || "$2" == *$'\n'* ]]; then + echo "XCFramework path cannot contain a quote or newline" >&2 + exit 64 + fi binary_target=".binaryTarget(name: \"MDICore\", path: \"$2\")" ;; remote) @@ -22,6 +34,14 @@ case "$mode" in echo "usage: $0 remote " >&2 exit 64 fi + if [[ "$2" != https://* || "$2" == *'"'* || "$2" == *$'\n'* ]]; then + echo "remote XCFramework URL must be an HTTPS URL without quotes or newlines" >&2 + exit 64 + fi + if [[ ! "$3" =~ ^[0-9a-f]{64}$ ]]; then + echo "checksum must contain exactly 64 lowercase hexadecimal characters" >&2 + exit 64 + fi binary_target=".binaryTarget(name: \"MDICore\", url: \"$2\", checksum: \"$3\")" ;; *) @@ -30,7 +50,10 @@ case "$mode" in ;; esac -cat > "$root_directory/Package.swift" < "$generated_manifest" < MDIParseResult { @@ -100,6 +110,30 @@ public enum MDI { public static func renderHTML(_ source: String) throws -> String { try stringCall(source, operation: mdi_render_html) } public static func serialize(_ source: String) throws -> String { try stringCall(source, operation: mdi_serialize_mdi) } public static func renderText(_ source: String) throws -> String { try stringCall(source, operation: mdi_render_text) } + public static func renderTextFormat( + _ source: String, + format: MDITextFormat, + indentPrefix: String = "" + ) throws -> String { + let sourceBytes = Array(source.utf8) + let formatBytes = Array(format.rawValue.utf8) + let indentBytes = Array(indentPrefix.utf8) + let result = sourceBytes.withUnsafeBufferPointer { sourceBuffer in + formatBytes.withUnsafeBufferPointer { formatBuffer in + indentBytes.withUnsafeBufferPointer { indentBuffer in + mdi_render_text_format( + sourceBuffer.baseAddress, + sourceBuffer.count, + formatBuffer.baseAddress, + formatBuffer.count, + indentBuffer.baseAddress, + indentBuffer.count + ) + } + } + } + return try string(from: data(from: result)) + } public static func renderEPUB(_ source: String) throws -> Data { try call(source, operation: mdi_render_epub) } public static func renderDOCX(_ source: String) throws -> Data { try call(source, operation: mdi_render_docx) } @@ -110,6 +144,10 @@ public enum MDI { return value } + static func coreErrorMessage(from data: Data) -> String { + String(data: data, encoding: .utf8) ?? "Unknown MDI core error" + } + private static func stringCall(_ source: String, operation: MDIFFIOperation) throws -> String { try string(from: call(source, operation: operation)) } @@ -126,7 +164,7 @@ public enum MDI { static func data(from result: mdi_ffi_result) throws -> Data { defer { mdi_free_buffer(result.value); mdi_free_buffer(result.error) } if result.error.len > 0 { - let message = String(data: Data(bytes: result.error.data!, count: result.error.len), encoding: .utf8) ?? "Unknown MDI core error" + let message = coreErrorMessage(from: Data(bytes: result.error.data!, count: result.error.len)) throw MDIError.core(message) } guard result.value.len == 0 || result.value.data != nil else { diff --git a/swift/Sources/MDICore/include/mdi_core.h b/swift/Sources/MDICore/include/mdi_core.h index 4aef36b..afce512 100644 --- a/swift/Sources/MDICore/include/mdi_core.h +++ b/swift/Sources/MDICore/include/mdi_core.h @@ -11,6 +11,14 @@ mdi_ffi_result mdi_parse_json(const uint8_t *data, size_t len); mdi_ffi_result mdi_render_html(const uint8_t *data, size_t len); mdi_ffi_result mdi_serialize_mdi(const uint8_t *data, size_t len); mdi_ffi_result mdi_render_text(const uint8_t *data, size_t len); +mdi_ffi_result mdi_render_text_format( + const uint8_t *data, + size_t len, + const uint8_t *format_data, + size_t format_len, + const uint8_t *indent_data, + size_t indent_len +); mdi_ffi_result mdi_render_epub(const uint8_t *data, size_t len); mdi_ffi_result mdi_render_docx(const uint8_t *data, size_t len); void mdi_free_buffer(mdi_ffi_buffer buffer); diff --git a/swift/Tests/MDITests/MDITests.swift b/swift/Tests/MDITests/MDITests.swift index 1eaf51f..4b63860 100644 --- a/swift/Tests/MDITests/MDITests.swift +++ b/swift/Tests/MDITests/MDITests.swift @@ -4,15 +4,64 @@ import MDICore @testable import MDI final class MDITests: XCTestCase { + private let formatContractSource = """ + --- + title: Swift package contract + lang: ja + --- + + # Swift package contract + + {東京|とうきょう}で第^12^話。 + """ + func testParsesRustOwnedDocument() throws { let result = try MDI.parse("第^12^話") XCTAssertEqual(result.irVersion, mdiIRVersion) XCTAssertEqual(result.syntaxVersion, mdiSpecVersion) XCTAssertTrue(result.capabilities.mdi) XCTAssertTrue(result.capabilities.commonMark) + XCTAssertTrue(result.capabilities.gfm) + XCTAssertTrue(result.capabilities.frontMatter) + XCTAssertTrue(result.capabilities.sourceSpans) XCTAssertTrue(result.diagnostics.isEmpty) } + func testReturnsRecoverableDiagnosticsWithUTF8ByteSpans() throws { + let source = "---\nmdi: '3.0'\n---\n\n👨‍👩‍👧" + let result = try MDI.parse(source) + + XCTAssertEqual(result.diagnostics, [ + MDIDiagnostic( + severity: .warning, + code: "mdi.version.unsupported", + message: "MDI 3.0 is newer than the supported 2.0", + span: MDISourceSpan(startByte: 0, endByte: 18) + ), + ]) + XCTAssertEqual(documentSpan(in: result.document)?.endByte, UInt32(source.utf8.count)) + } + + func testKeepsTheJSAdversarialCorpusInsideTheRustWireContract() throws { + let corpus = [ + "\\{}《《傍点》》\n\n\\[{東京|とう.きょう}", + "👨‍👩‍👧 [[em:**強調**]] [^n]\n\n[^n]: 注", + "[[indent:2]]\n{𠮟る|しか.る} [[no-break:^12^]]\n\n[[pagebreak:left]]", + "```mdi\n{東京|とうきょう}\n```\n\n| a | b |\n| - | - |\n| [[em:x]] | ^12^ |", + ] + + for source in corpus { + let result = try MDI.parse(source) + XCTAssertEqual( + documentSpan(in: result.document), + MDISourceSpan(startByte: 0, endByte: UInt32(source.utf8.count)) + ) + assertValidSpans(in: result.document, sourceByteCount: UInt32(source.utf8.count)) + XCTAssertTrue(try MDI.renderHTML(source).hasPrefix("")) + XCTAssertEqual(try MDI.parse(MDI.serialize(source)).irVersion, mdiIRVersion) + } + } + func testParsePreservesDocumentStructureAndUnicode() throws { let result = try MDI.parse("# 見出し\n\n{東京|とうきょう}で第^12^話") guard case let .object(document) = result.document else { @@ -35,20 +84,113 @@ final class MDITests: XCTestCase { XCTAssertEqual(try MDI.renderText("{東京|とうきょう} ^12^"), "東京 12\n") } + func testRendersEveryPublicTextFormatThroughRust() throws { + let source = "{東京|とうきょう}" + let expected: [MDITextFormat: String] = [ + .plain: "　東京", + .ruby: "　{東京|とうきょう}", + .narou: "　｜東京《とうきょう》", + .kakuyomu: "　｜東京《とうきょう》", + .aozora: "　｜東京《とうきょう》", + .note: "　｜東京《とうきょう》", + ] + + XCTAssertEqual( + Set(MDITextFormat.allCases.map(\.rawValue)), + ["txt", "txt-ruby", "narou", "kakuyomu", "aozora", "note"] + ) + for format in MDITextFormat.allCases { + XCTAssertEqual( + try MDI.renderTextFormat(source, format: format, indentPrefix: "　"), + expected[format] + ) + } + } + func testSerializesMDIThroughRust() throws { let mdi = try MDI.serialize("{東京|とうきょう} ^12^") XCTAssertTrue(mdi.contains("{東京|とうきょう}")) XCTAssertTrue(mdi.contains("^12^")) } - func testReturnsBinaryDocuments() throws { - let epub = try MDI.renderEPUB("# Chapter\n\nText") - let docx = try MDI.renderDOCX("# Chapter\n\nText") + func testAllPublicFormatsPreserveThePackageContract() throws { + let html = try MDI.renderHTML(formatContractSource) + let text = try MDI.renderText(formatContractSource) + let serialized = try MDI.serialize(formatContractSource) + let reparsed = try MDI.parse(serialized) + let epub = try MDI.renderEPUB(formatContractSource) + let docx = try MDI.renderDOCX(formatContractSource) + XCTAssertTrue(html.contains("

Swift package contract

")) + XCTAssertTrue(html.contains("東京")) + XCTAssertTrue(html.contains("12")) + XCTAssertTrue(text.contains("Swift package contract")) + XCTAssertTrue(text.contains("東京で第12話。")) + XCTAssertTrue(serialized.contains("{東京|とうきょう}")) + XCTAssertTrue(serialized.contains("^12^")) + XCTAssertEqual(reparsed.irVersion, mdiIRVersion) + XCTAssertTrue(reparsed.diagnostics.isEmpty) XCTAssertGreaterThan(epub.count, 100) XCTAssertGreaterThan(docx.count, 100) XCTAssertEqual(Array(epub.prefix(2)), [0x50, 0x4b]) XCTAssertEqual(Array(docx.prefix(2)), [0x50, 0x4b]) + +#if os(macOS) + XCTAssertTrue( + try archiveEntries(in: epub, extension: "epub").isSuperset(of: [ + "mimetype", + "META-INF/container.xml", + "OEBPS/package.opf", + "OEBPS/nav.xhtml", + "OEBPS/style.css", + "OEBPS/chapter-1.xhtml", + ]) + ) + XCTAssertTrue( + try archiveEntries(in: docx, extension: "docx").isSuperset(of: [ + "[Content_Types].xml", + "_rels/.rels", + "docProps/core.xml", + "word/document.xml", + ]) + ) +#endif + } + + /// CI supplies `MDI_SWIFT_SMOKE_OUTPUT` and then opens these exact + /// package-level products with EPUBCheck and Apple's document importers. + func testWritesEveryPublicFormatForExternalConsumers() throws { + guard let outputPath = ProcessInfo.processInfo.environment["MDI_SWIFT_SMOKE_OUTPUT"] else { + throw XCTSkip("external format smoke output was not requested") + } + + let output = URL(fileURLWithPath: outputPath, isDirectory: true) + try FileManager.default.createDirectory(at: output, withIntermediateDirectories: true) + var formats: [(String, Data)] = [ + ("document.html", Data(try MDI.renderHTML(formatContractSource).utf8)), + ("document.txt", Data(try MDI.renderText(formatContractSource).utf8)), + ("document.mdi", Data(try MDI.serialize(formatContractSource).utf8)), + ("document.epub", try MDI.renderEPUB(formatContractSource)), + ("document.docx", try MDI.renderDOCX(formatContractSource)), + ] + for format in MDITextFormat.allCases { + formats.append(( + "document-\(format.rawValue).txt", + Data(try MDI.renderTextFormat( + formatContractSource, + format: format, + indentPrefix: "　" + ).utf8) + )) + } + + for (name, contents) in formats { + XCTAssertFalse(contents.isEmpty, "\(name) must not be empty") + try contents.write(to: output.appendingPathComponent(name), options: .atomic) + } + + let serialized = try String(contentsOf: output.appendingPathComponent("document.mdi"), encoding: .utf8) + XCTAssertEqual(try MDI.serialize(serialized), serialized) } func testJSONValueRoundTripsEveryVariant() throws { @@ -103,10 +245,11 @@ final class MDITests: XCTestCase { } } - func testRejectsMalformedNativeStringAndBufferPayloads() { + func testRejectsMalformedNativeStringAndBufferPayloads() throws { XCTAssertThrowsError(try MDI.string(from: Data([0xff]))) { error in XCTAssertEqual(error as? MDIError, .invalidWireFormat("MDI core returned non-UTF-8 string data")) } + XCTAssertEqual(MDI.coreErrorMessage(from: Data([0xff])), "Unknown MDI core error") let invalid = mdi_ffi_result( value: mdi_ffi_buffer(data: nil, len: 1), @@ -115,6 +258,12 @@ final class MDITests: XCTestCase { XCTAssertThrowsError(try MDI.data(from: invalid)) { error in XCTAssertEqual(error as? MDIError, .invalidWireFormat("MDI core returned an invalid buffer")) } + + let empty = mdi_ffi_result( + value: mdi_ffi_buffer(data: nil, len: 0), + error: mdi_ffi_buffer(data: nil, len: 0) + ) + XCTAssertEqual(try MDI.data(from: empty), Data()) } func testForwardsRustCoreErrors() { @@ -125,4 +274,81 @@ final class MDITests: XCTestCase { XCTAssertFalse(message.isEmpty) } } + +#if os(macOS) + private func archiveEntries(in data: Data, extension fileExtension: String) throws -> Set { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("mdi-swift-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + + let archive = directory.appendingPathComponent("document.\(fileExtension)") + try data.write(to: archive, options: .atomic) + + let process = Process() + let standardOutput = Pipe() + let standardError = Pipe() + process.executableURL = URL(fileURLWithPath: "/usr/bin/unzip") + process.arguments = ["-Z1", archive.path] + process.standardOutput = standardOutput + process.standardError = standardError + try process.run() + process.waitUntilExit() + + let output = standardOutput.fileHandleForReading.readDataToEndOfFile() + let error = standardError.fileHandleForReading.readDataToEndOfFile() + guard process.terminationStatus == 0 else { + throw MDIError.invalidWireFormat( + "unzip could not open \(fileExtension): \(String(decoding: error, as: UTF8.self))" + ) + } + return Set(String(decoding: output, as: UTF8.self).split(separator: "\n").map(String.init)) + } +#endif + + private func documentSpan(in value: MDIJSONValue) -> MDISourceSpan? { + guard + case let .object(object) = value, + case let .object(span)? = object["span"], + case let .number(start)? = span["startByte"], + case let .number(end)? = span["endByte"] + else { + return nil + } + return MDISourceSpan(startByte: UInt32(start), endByte: UInt32(end)) + } + + private func assertValidSpans( + in value: MDIJSONValue, + sourceByteCount: UInt32, + file: StaticString = #filePath, + line: UInt = #line + ) { + switch value { + case let .array(values): + for child in values { + assertValidSpans( + in: child, + sourceByteCount: sourceByteCount, + file: file, + line: line + ) + } + case let .object(object): + if let span = documentSpan(in: value) { + XCTAssertLessThanOrEqual(span.startByte, span.endByte, file: file, line: line) + XCTAssertLessThanOrEqual(span.endByte, sourceByteCount, file: file, line: line) + } + for child in object.values { + assertValidSpans( + in: child, + sourceByteCount: sourceByteCount, + file: file, + line: line + ) + } + case .null, .bool, .number, .string: + break + } + } }