diff --git a/.github/workflows/link-check.yml b/.github/workflows/link-check.yml index b01d1e5..38feb79 100644 --- a/.github/workflows/link-check.yml +++ b/.github/workflows/link-check.yml @@ -2,17 +2,215 @@ name: Link check on: pull_request: + paths: + - "**/*.md" + - ".lycheeignore" + - ".github/workflows/link-check.yml" workflow_dispatch: schedule: - - cron: "0 17 1 * *" + - cron: "0 23 * * 0" # 23:00 UTC every Sunday + +permissions: + contents: read + +env: + LYCHEE_ARGS: --cache --max-cache-age 1d --timeout 10 --verbose --no-progress './**/*.md' + PERMANENT_REDIRECT_CODES: 301,308 jobs: - links: + links-pr: + if: github.event_name == 'pull_request' runs-on: ubuntu-latest + steps: + - name: Checkout PR head + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + - name: Restore lychee cache + uses: actions/cache/restore@v4 + with: + path: .lycheecache + key: cache-lychee-pr-${{ github.event.pull_request.number }}-${{ github.run_number }} + restore-keys: cache-lychee-pr-${{ github.event.pull_request.number }}- + + - name: Check links on PR head + uses: lycheeverse/lychee-action@v2 + with: + args: ${{ env.LYCHEE_ARGS }} + format: json + output: ./lychee/pr-head.json + fail: false + + - name: Checkout base branch + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.base.ref }} + + - name: Check links on base branch + uses: lycheeverse/lychee-action@v2 + with: + args: ${{ env.LYCHEE_ARGS }} + format: json + output: ./lychee/pr-base.json + fail: false + + - name: Compare broken links + run: | + python - <<'PY' + import json + import os + from collections import defaultdict + from pathlib import Path + + head = json.loads(Path('lychee/pr-head.json').read_text(encoding='utf-8')) + base = json.loads(Path('lychee/pr-base.json').read_text(encoding='utf-8')) + + def flatten_errors(payload): + rows = set() + for source, items in payload.get('error_map', {}).items(): + for item in items: + status = item.get('status', {}) + rows.add( + ( + source, + item.get('url', ''), + status.get('code'), + status.get('text', ''), + status.get('details', ''), + ) + ) + return rows + + def count_permanent_redirects(payload): + # Only configured permanent redirect codes are counted. + permanent_codes = {int(code) for code in os.getenv('PERMANENT_REDIRECT_CODES', '301,308').split(',')} + count = 0 + for items in payload.get('success_map', {}).values(): + for item in items: + status = item.get('status', {}) + if status.get('code') in permanent_codes: + count += 1 + return count + + new_errors = sorted(flatten_errors(head) - flatten_errors(base)) + permanent_redirects = count_permanent_redirects(head) + + summary_path = os.environ.get('GITHUB_STEP_SUMMARY') + if summary_path: + with open(summary_path, 'a', encoding='utf-8') as summary: + summary.write(f"### Permanent redirects detected (non-failing): {permanent_redirects}\n\n") + if new_errors: + summary.write('### New broken links in this PR\n\n') + by_file = defaultdict(list) + for source, url, code, text, details in new_errors: + by_file[source].append((url, code, text, details)) + for source in sorted(by_file): + summary.write(f"- **{source}**\n") + for url, code, text, details in by_file[source]: + status = text or details or (str(code) if code is not None else 'error') + summary.write(f" - `{url}` ({status})\n") + else: + summary.write('No new broken links compared with the base branch.\n') + + if new_errors: + print('New broken links introduced in this PR:') + for source, url, code, text, details in new_errors: + status = text or details or (str(code) if code is not None else 'error') + print(f"- {source}: {url} ({status})") + raise SystemExit(1) + PY + + - name: Save lychee cache + if: always() + uses: actions/cache/save@v4 + with: + path: .lycheecache + key: cache-lychee-pr-${{ github.event.pull_request.number }}-${{ github.run_number }} + + links-scheduled: + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + permissions: + contents: read + issues: write steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + + - name: Restore lychee cache + uses: actions/cache/restore@v4 with: - python-version: "3.x" + path: .lycheecache + key: cache-lychee-scheduled-${{ github.ref_name }}-${{ github.run_number }} + restore-keys: cache-lychee-scheduled-${{ github.ref_name }}- + - name: Check links - run: python scripts/check_links.py --online --include-resources + id: lychee + uses: lycheeverse/lychee-action@v2 + with: + args: ${{ env.LYCHEE_ARGS }} + format: json + output: ./lychee/scheduled.json + fail: false + + - name: Build issue report + if: steps.lychee.outputs.exit_code != 0 + run: | + python - <<'PY' + import json + import os + from collections import defaultdict + from datetime import date + from pathlib import Path + + payload = json.loads(Path('lychee/scheduled.json').read_text(encoding='utf-8')) + grouped = defaultdict(list) + permanent_redirects = 0 + permanent_codes = {int(code) for code in os.getenv('PERMANENT_REDIRECT_CODES', '301,308').split(',')} + + for source, items in payload.get('error_map', {}).items(): + for item in items: + status = item.get('status', {}) + status_text = status.get('text') or status.get('details') or 'Error' + grouped[source].append((item.get('url', ''), status_text)) + + for items in payload.get('success_map', {}).values(): + for item in items: + status = item.get('status', {}) + if status.get('code') in permanent_codes: + permanent_redirects += 1 + + lines = [ + f"# Weekly link check report ({date.today().isoformat()})", + '', + f"Permanent redirects detected (non-failing): {permanent_redirects}", + '', + 'Broken links grouped by file:', + '', + ] + + for source in sorted(grouped): + lines.append(f"## {source}") + for url, status_text in grouped[source]: + lines.append(f"- `{url}` ({status_text})") + lines.append('') + + Path('lychee/report.md').write_text('\n'.join(lines).rstrip() + '\n', encoding='utf-8') + PY + + - name: Create issue from report + if: steps.lychee.outputs.exit_code != 0 + uses: peter-evans/create-issue-from-file@v5 + with: + title: Scheduled link check report - run ${{ github.run_number }} + content-filepath: ./lychee/report.md + labels: report, automated issue + + - name: Save lychee cache + if: always() + uses: actions/cache/save@v4 + with: + path: .lycheecache + key: cache-lychee-scheduled-${{ github.ref_name }}-${{ github.run_number }} diff --git a/.lycheeignore b/.lycheeignore new file mode 100644 index 0000000..fce57a4 --- /dev/null +++ b/.lycheeignore @@ -0,0 +1,3 @@ +^https://www\.kaggle\.com/learn$ +^https://portswigger\.net/web-security$ +^https://stackoverflow\.com/help/asking$ \ No newline at end of file diff --git a/MAINTENANCE.md b/MAINTENANCE.md index e8e6db7..c12ac7f 100644 --- a/MAINTENANCE.md +++ b/MAINTENANCE.md @@ -122,7 +122,8 @@ Run the online link check: python scripts/check_links.py --online --include-resources ``` -The GitHub Actions workflow in `.github/workflows/link-check.yml` runs the -online check on pull requests, on demand, and monthly. A few sites are -browser-verified but block command-line checks; keep those in the script's -`BROWSER_ONLY_URLS` allowlist only when you have manually verified them. +The GitHub Actions workflow in `.github/workflows/link-check.yml` runs a +weekly scheduled lychee check (11 PM Sunday UTC), plus pull request checks for +Markdown changes. Scheduled failures automatically open an issue with broken +links grouped by file, and pull requests fail only when they introduce new +broken links. Keep known flaky URLs in `.lycheeignore`. diff --git a/README.md b/README.md index d44a241..be714e8 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # Learn to Code +[![Link check](https://github.com/ashleymcnamara/learn_to_code/actions/workflows/link-check.yml/badge.svg)](https://github.com/ashleymcnamara/learn_to_code/actions/workflows/link-check.yml) + A free-first, beginner-friendly guide to learning programming, computer science, AI, and the practical tools developers use every day.