From 6608f951fed99446ffafa9c73e73cb3e06b5e064 Mon Sep 17 00:00:00 2001 From: ReenigneArcher <42013603+ReenigneArcher@users.noreply.github.com> Date: Thu, 19 Mar 2026 18:46:18 -0400 Subject: [PATCH 1/3] feat: migrate to jekyll static site --- .github/workflows/update-pages.yml | 109 +--- .readthedocs.yaml | 25 + gh-pages-template/_config.yml | 2 + gh-pages-template/assets/js/dashboard.js | 532 +++++++++++++++++++ gh-pages-template/index.html | 120 +++++ jupyter_nbconvert_config.py | 8 - notebook/dashboard.ipynb | 328 ------------ package.json | 5 - pyproject.toml | 17 +- src/__init__.py | 1 + src/builder.py | 213 ++++++++ src/dashboard.py | 617 ----------------------- src/helpers.py | 16 +- src/updater.py | 142 ++++-- 14 files changed, 1043 insertions(+), 1092 deletions(-) create mode 100644 .readthedocs.yaml create mode 100644 gh-pages-template/_config.yml create mode 100644 gh-pages-template/assets/js/dashboard.js create mode 100644 gh-pages-template/index.html delete mode 100644 jupyter_nbconvert_config.py delete mode 100644 notebook/dashboard.ipynb delete mode 100644 package.json create mode 100644 src/builder.py delete mode 100644 src/dashboard.py diff --git a/.github/workflows/update-pages.yml b/.github/workflows/update-pages.yml index 33ac5faeb..4a55e517f 100644 --- a/.github/workflows/update-pages.yml +++ b/.github/workflows/update-pages.yml @@ -16,7 +16,7 @@ concurrency: cancel-in-progress: true jobs: - update_pages: + update: permissions: contents: read runs-on: ubuntu-latest @@ -25,48 +25,17 @@ jobs: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Checkout gh-pages - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - ref: gh-pages - path: gh-pages - persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of the personal token - fetch-depth: 0 # otherwise, will fail to push refs to dest repo - - name: Setup python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.14' - - name: Setup node - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 - with: - node-version: 'latest' - - name: Install Python dependencies run: | python -m pip install --upgrade pip setuptools wheel python -m pip install . - - name: Install npm dependencies - run: npm install --ignore-scripts - - - name: Get current date - id: date - run: echo "date=$(date +'%Y-%m-%d')" >> "${GITHUB_OUTPUT}" - - - name: Prepare gh-pages - run: | - # empty contents - rm -f -r ./gh-pages/* - - # no jekyll - touch ./gh-pages/.nojekyll - - # copy dependencies - cp -f ./node_modules/plotly.js/dist/plotly.min.js ./gh-pages/plotly.js - - - name: Convert notebook + - name: Collect data env: DASHBOARD_AUR_REPOS: sunshine,sunshine-bin,sunshine-git CODECOV_TOKEN: ${{ secrets.CODECOV_API_TOKEN }} @@ -78,62 +47,40 @@ jobs: GITHUB_TOKEN: ${{ secrets.GH_BOT_TOKEN || secrets.GITHUB_TOKEN }} PATREON_CAMPAIGN_ID: 6131567 READTHEDOCS_TOKEN: ${{ secrets.READTHEDOCS_TOKEN }} - run: | - jupyter nbconvert \ - --debug \ - --config=./jupyter_nbconvert_config.py \ - --execute \ - --no-input \ - --to=html \ - --output-dir=./gh-pages \ - --output=index \ - ./notebook/dashboard.ipynb + THREADING_EXCEPTION_HANDLER: "true" + run: python -u -m src.updater - name: Cat log + if: always() run: cat ./logs/updater.log - - name: Check notebook for tracebacks - run: | - cat ./gh-pages/index.html - echo "---" - echo "Checking for tracebacks..." - set +e - tracebacks=$(grep -i -E 'Traceback \(most recent call last\):' ./gh-pages/index.html) - set -e - if [ -n "${tracebacks}" ]; then - echo "Tracebacks found:" - echo "${tracebacks}" - exit 1 - fi + - name: Build dashboard data + run: python -u -m src.builder - - name: Archive gh-pages - if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' + - name: Prepare Artifacts # uploading artifacts will fail if not zipped due to very large quantity of files shell: bash - run: | - 7z \ - "-xr!*.git*" \ - a "./gh-pages.zip" "./gh-pages" + run: 7z a build.zip ./gh-pages/* ./gh-pages-template/* - - name: Upload Artifacts - if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' + - name: Upload artifact uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: - name: gh-pages + name: update + path: ${{ github.workspace }}/build.zip if-no-files-found: error - path: | - ${{ github.workspace }}/gh-pages.zip + include-hidden-files: true + retention-days: 1 - - name: Deploy to gh-pages - if: >- - (github.event_name == 'push' && github.ref == 'refs/heads/master') || - github.event_name == 'schedule' || - github.event_name == 'workflow_dispatch' - uses: actions-js/push@5a7cbd780d82c0c937b5977586e641b2fd94acc5 # v1.5 - with: - github_token: ${{ secrets.GH_BOT_TOKEN }} - author_email: ${{ secrets.GH_BOT_EMAIL }} - author_name: ${{ vars.GH_BOT_NAME }} - directory: gh-pages - branch: gh-pages - force: false - message: automatic-update-${{ steps.date.outputs.date }} + call-jekyll-build: + needs: update + permissions: + contents: read + uses: LizardByte/LizardByte.github.io/.github/workflows/jekyll-build.yml@master + secrets: + GH_BOT_EMAIL: ${{ secrets.GH_BOT_EMAIL }} + GH_BOT_TOKEN: ${{ secrets.GH_BOT_TOKEN }} + with: + gh_bot_name: ${{ vars.GH_BOT_NAME }} + site_artifact: 'update' + extract_archive: 'build.zip' + target_branch: 'gh-pages' + clean_gh_pages: true diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 000000000..681f205b8 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,25 @@ +--- +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +version: 2 + +build: + os: ubuntu-24.04 + tools: + ruby: "3.3" + apt_packages: + - 7zip + - jq + jobs: + install: + - | + mkdir -p "./tmp" + branch="master" + base_url="https://raw.githubusercontent.com/LizardByte/LizardByte.github.io" + url="${base_url}/refs/heads/${branch}/scripts/readthedocs_build.sh" + curl -sSL -o "./tmp/readthedocs_build.sh" "${url}" + chmod +x "./tmp/readthedocs_build.sh" + build: + html: + - "./tmp/readthedocs_build.sh" diff --git a/gh-pages-template/_config.yml b/gh-pages-template/_config.yml new file mode 100644 index 000000000..29db11009 --- /dev/null +++ b/gh-pages-template/_config.yml @@ -0,0 +1,2 @@ +--- +site-js: [] # disable crowdin for this site diff --git a/gh-pages-template/assets/js/dashboard.js b/gh-pages-template/assets/js/dashboard.js new file mode 100644 index 000000000..9bf5f1ea7 --- /dev/null +++ b/gh-pages-template/assets/js/dashboard.js @@ -0,0 +1,532 @@ +'use strict'; + +// Config +const _cfg = globalThis.DASHBOARD_CONFIG || {}; +const basePath = _cfg.base_path + ? ('/' + _cfg.base_path).replaceAll(/\/+/g, '/').replace(/\/$/, '') + : '/dashboard'; +const baseUrl = globalThis.location.origin + basePath; + +// Helpers +async function fetchJSON(filename) { + const url = `${baseUrl}/assets/data/${filename}`; + const resp = await fetch(url); + if (!resp.ok) throw new Error(`Failed to fetch ${url}: ${resp.status}`); + return resp.json(); +} + +function isDark() { + const bsTheme = document.documentElement.dataset.bsTheme; + if (bsTheme === 'dark') return true; + if (bsTheme === 'light') return false; + return document.documentElement.classList.contains('dark-mode') + || globalThis.matchMedia('(prefers-color-scheme: dark)').matches; +} + +function plotlyTemplate() { + return isDark() ? 'plotly_dark' : 'plotly_white'; +} + +const LAYOUT = { + paper_bgcolor: 'rgba(0,0,0,0)', + plot_bgcolor: 'rgba(0,0,0,0)', + margin: { t: 30, r: 20, b: 120, l: 60 }, + autosize: true, +}; +const CONFIG = { responsive: true }; + +function themeLayout(overrides = {}, applyFontColor = true) { + const base = { ...LAYOUT, template: plotlyTemplate() }; + if (applyFontColor) { + const textColor = getComputedStyle(document.documentElement).getPropertyValue('--bs-body-color').trim(); + if (textColor) base.font = { color: textColor }; + } + return { ...base, ...overrides }; +} + +function activeRepos(repos) { + return repos.filter(r => !r.archived && !(r.topics || []).includes('package-manager')); +} + +// Summary +function renderSummary(active, prs, metadata) { + const el = document.getElementById('summary-stats'); + if (!el) return; + el.classList.add('justify-content-center'); + const updated = new Date(metadata.updated_at).toUTCString(); + const stat = (val, label) => + `
+

${val}

+ ${label} +
`; + el.innerHTML = + stat(active.length, 'Active Repositories') + + stat(active.filter(r => r.fork).length, 'Forked') + + stat(active.reduce((s, r) => s + r.issues, 0), 'Open Issues') + + stat(active.reduce((s, r) => s + r.prs, 0), 'Open PRs') + + `
+ Last updated: ${updated} +
`; +} + +// Bar charts +function renderBarChart(divId, entries) { + const el = document.getElementById(divId); + if (!el || !entries.length) return; + const sorted = [...entries].sort((a, b) => b.value - a.value); + Plotly.newPlot(divId, [{ + x: sorted.map(d => d.name), + y: sorted.map(d => Math.log1p(d.value)), + text: sorted.map(d => String(d.value)), + type: 'bar', + textposition: 'inside', + marker: { color: '#28a9e6' }, + }], themeLayout({ + yaxis: { showticklabels: false }, + xaxis: { tickangle: -45 }, + }), CONFIG); +} + +// PRs bar chart +function renderPRsBarChart(active, prs) { + const el = document.getElementById('chart-prs'); + if (!el) return; + const counts = {}; + active.forEach(r => { counts[r.name] = { Ready: 0, Draft: 0 }; }); + prs.forEach(pr => { + if (counts[pr.repo]) counts[pr.repo][pr.draft ? 'Draft' : 'Ready']++; + }); + const sorted = active + .map(r => ({ name: r.name, ...counts[r.name], total: counts[r.name].Ready + counts[r.name].Draft })) + .filter(r => r.total > 0) + .sort((a, b) => b.total - a.total); + if (!sorted.length) { + el.innerHTML = '

No open PRs.

'; + return; + } + Plotly.newPlot('chart-prs', ['Ready', 'Draft'].map(status => ({ + name: status, + x: sorted.map(r => r.name), + y: sorted.map(r => r[status]), + text: sorted.map(r => String(r[status])), + type: 'bar', + textposition: 'inside', + })), themeLayout({ + barmode: 'stack', + xaxis: { tickangle: -45 }, + }), CONFIG); +} + +function stripTags(html) { + let out = '', inTag = false; + for (const ch of html) { + if (ch === '<') { inTag = true; } + else if (ch === '>') { inTag = false; } + else if (!inTag) { out += ch; } + } + return out; +} + +// PR table +function renderPRTable(prs) { + const container = document.getElementById('table-prs'); + if (!container) return; + const cols = ['Repo', '#', 'Title', 'Author', 'Labels', 'Assignees', 'Status', 'Created', 'Last Activity', 'Milestone']; + let rows = ''; + prs.forEach(pr => { + const link = `#${pr.number}`; + rows += ` + ${pr.repo} + ${link} + ${pr.title} + ${pr.author} + ${(pr.labels || []).join(', ')} + ${(pr.assignees || []).join(', ')} + ${pr.draft ? 'Draft' : 'Ready'} + ${(pr.created_at || '').substring(0, 10)} + ${(pr.updated_at || '').substring(0, 10)} + ${pr.milestone || ''} + `; + }); + const headerCells = cols.map(c => `${c}`).join(''); + const filterCells = cols.map(() => ``).join(''); + container.innerHTML = ` + + + ${headerCells} + ${filterCells} + + ${rows} +
`; + + const filterInputs = Array.from(container.querySelectorAll('thead tr:nth-child(2) input')); + const tbody = container.querySelector('tbody'); + + if (typeof DataTable !== 'undefined') { + const filterValues = new Array(cols.length).fill(''); + DataTable.ext.search.push((settings, rowData) => { + if (settings.nTable.id !== 'pr-datatable') return true; + return filterValues.every((val, i) => { + if (!val) return true; + return stripTags(String(rowData[i] || '')).toLowerCase().includes(val.toLowerCase()); + }); + }); + const dt = new DataTable('#pr-datatable', { + pageLength: 25, + lengthMenu: [10, 25, 50, 100], + order: [[7, 'asc']], + orderCellsTop: true, + }); + filterInputs.forEach((input, i) => { + input.addEventListener('input', () => { + filterValues[i] = input.value; + dt.draw(); + }); + }); + return dt; + } + + // Fallback: sort by Created date (col 7) ascending, then add paging + per-column filtering. + const allRows = Array.from(tbody.querySelectorAll('tr')); + allRows.sort((a, b) => { + const aVal = a.querySelectorAll('td')[7]?.textContent || ''; + const bVal = b.querySelectorAll('td')[7]?.textContent || ''; + return aVal.localeCompare(bVal); + }); + allRows.forEach(row => tbody.appendChild(row)); + + let pageSize = 10; + let currentPage = 0; + + const lengthSel = document.createElement('select'); + lengthSel.className = 'form-select form-select-sm d-inline-block w-auto'; + [10, 25, 50, 100].forEach(n => { + const opt = document.createElement('option'); + opt.value = String(n); + opt.textContent = String(n); + if (n === 25) opt.selected = true; + lengthSel.appendChild(opt); + }); + const lengthLabel = document.createElement('label'); + lengthLabel.className = 'text-muted small me-auto'; + lengthLabel.append('Show ', lengthSel, ' entries'); + + const infoEl = document.createElement('small'); + infoEl.className = 'text-muted'; + + const pagUl = document.createElement('ul'); + pagUl.className = 'pagination pagination-sm mb-0'; + + const topBar = document.createElement('div'); + topBar.className = 'd-flex align-items-center mb-2'; + topBar.appendChild(lengthLabel); + + const bottomBar = document.createElement('div'); + bottomBar.className = 'd-flex justify-content-between align-items-center mt-2 flex-wrap gap-2'; + bottomBar.append(infoEl, pagUl); + + const table = container.querySelector('table'); + table.before(topBar); + container.appendChild(bottomBar); + + function getFiltered() { + const filters = filterInputs.map(inp => inp.value.toLowerCase()); + return allRows.filter(row => { + const cells = Array.from(row.querySelectorAll('td')); + return filters.every((f, i) => !f || (cells[i]?.textContent || '').toLowerCase().includes(f)); + }); + } + + function renderPage() { + const filtered = getFiltered(); + const totalPages = Math.max(1, Math.ceil(filtered.length / pageSize)); + currentPage = Math.min(currentPage, totalPages - 1); + const start = currentPage * pageSize; + + allRows.forEach(r => { r.style.display = 'none'; }); + filtered.slice(start, start + pageSize).forEach(r => { r.style.display = ''; }); + + const from = filtered.length === 0 ? 0 : start + 1; + const to = Math.min(start + pageSize, filtered.length); + infoEl.textContent = `Showing ${from} to ${to} of ${filtered.length} entries`; + + pagUl.innerHTML = ''; + const addBtn = (label, page, disabled, active) => { + const li = document.createElement('li'); + li.className = `page-item${disabled ? ' disabled' : ''}${active ? ' active' : ''}`; + const btn = document.createElement('button'); + btn.className = 'page-link'; + btn.textContent = label; + btn.disabled = disabled; + if (!disabled) btn.addEventListener('click', () => { currentPage = page; renderPage(); }); + li.appendChild(btn); + pagUl.appendChild(li); + }; + addBtn('Previous', currentPage - 1, currentPage === 0, false); + const startP = Math.max(0, Math.min(currentPage - 2, totalPages - 5)); + for (let p = startP; p < Math.min(totalPages, startP + 5); p++) { + addBtn(String(p + 1), p, false, p === currentPage); + } + addBtn('Next', currentPage + 1, currentPage >= totalPages - 1, false); + } + + lengthSel.addEventListener('change', () => { + pageSize = Number.parseInt(lengthSel.value, 10); + currentPage = 0; + renderPage(); + }); + filterInputs.forEach(inp => inp.addEventListener('input', () => { currentPage = 0; renderPage(); })); + renderPage(); +} + +// License treemap +function renderLicenseChart(repos) { + const licRepos = {}; + repos.forEach(r => { + const lic = r.license || 'No License'; + if (!licRepos[lic]) licRepos[lic] = []; + licRepos[lic].push(r.name); + }); + const ids = [], labels = [], parents = [], values = []; + Object.entries(licRepos).forEach(([lic, rlist]) => { + ids.push(`lic::${lic}`); + labels.push(lic); + parents.push(''); + values.push(rlist.length); + rlist.forEach(name => { + ids.push(`repo::${lic}::${name}`); + labels.push(name); + parents.push(`lic::${lic}`); + values.push(1); + }); + }); + Plotly.newPlot('chart-license', [{ + type: 'treemap', ids, labels, parents, values, + branchvalues: 'total', + tiling: { squarifyratio: 1.618, pad: 2 }, + pathbar: { visible: false }, + }], themeLayout({ + margin: { t: 5, r: 5, b: 5, l: 5 }, + }, false), CONFIG); +} + +// Coverage scatter +function renderCoverageChart(repos) { + const sorted = [...repos].filter(r => r.coverage > 0).sort((a, b) => b.coverage - a.coverage); + if (!sorted.length) return; + Plotly.newPlot('chart-coverage', [{ + x: sorted.map(r => r.name), + y: sorted.map(r => r.coverage), + mode: 'markers', + type: 'scatter', + marker: { + size: sorted.map(r => Math.max(8, 110 - r.coverage)), + color: sorted.map(r => r.coverage), + colorscale: [[0, 'red'], [0.5, 'yellow'], [1, 'green']], + colorbar: { title: 'Coverage %' }, + showscale: true, + }, + text: sorted.map(r => `${r.name}: ${r.coverage.toFixed(1)}%`), + }], themeLayout({ + yaxis: { title: 'Coverage %', range: [0, 105] }, + xaxis: { tickangle: -45 }, + }), CONFIG); +} + +// Coverage history line +function renderCoverageHistory(history) { + const el = document.getElementById('chart-coverage-history'); + if (!el) return; + if (!history?.length) { + el.innerHTML = '

No coverage history available.

'; + return; + } + const byRepo = {}; + history.forEach(({ repo, date, coverage }) => { + if (!byRepo[repo]) byRepo[repo] = { x: [], y: [] }; + byRepo[repo].x.push(date); + byRepo[repo].y.push(coverage); + }); + const traces = Object.entries(byRepo).map(([repo, d]) => ({ + name: repo, x: d.x, y: d.y, mode: 'lines', type: 'scatter', + })); + Plotly.newPlot('chart-coverage-history', traces, themeLayout({ + yaxis: { title: 'Coverage %', range: [0, 100] }, + }), CONFIG); +} + +// Language treemaps +function treemap(data, valueFor) { + const ids = [], labels = [], parents = [], values = []; + const sorted = Object.entries(data).sort((a, b) => valueFor(b) - valueFor(a)); + sorted.forEach(([lang, info]) => { + ids.push(`L::${lang}`); + labels.push(lang); + parents.push(''); + values.push(valueFor([lang, info])); + const rlist = Array.isArray(info) ? info : Object.keys(info.repos); + rlist.forEach(name => { + ids.push(`R::${lang}::${name}`); + labels.push(name); + parents.push(`L::${lang}`); + values.push(Array.isArray(info) ? 1 : info.repos[name]); + }); + }); + return { ids, labels, parents, values }; +} + +function renderLanguageCharts(repos) { + const byBytes = {}, byCount = {}; + repos.forEach(r => { + Object.entries(r.languages || {}).forEach(([lang, bytes]) => { + if (!byBytes[lang]) byBytes[lang] = { total: 0, repos: {} }; + byBytes[lang].total += bytes; + byBytes[lang].repos[r.name] = (byBytes[lang].repos[r.name] || 0) + bytes; + if (!byCount[lang]) byCount[lang] = []; + if (!byCount[lang].includes(r.name)) byCount[lang].push(r.name); + }); + }); + + const treemapTrace = (data) => ({ + type: 'treemap', ...data, + branchvalues: 'total', + tiling: { squarifyratio: 1.618, pad: 2 }, + pathbar: { visible: false }, + }); + const treemapLayout = { margin: { t: 5, r: 5, b: 5, l: 5 } }; + + Plotly.newPlot('chart-languages-bytes', [treemapTrace(treemap(byBytes, ([, info]) => info.total))], + themeLayout(treemapLayout, false), CONFIG); + + Plotly.newPlot('chart-languages-repos', [treemapTrace(treemap(byCount, ([, arr]) => arr.length))], + themeLayout(treemapLayout, false), CONFIG); +} + +// Commit activity charts +function renderCommitActivityChart(commitActivity, active) { + const activeNames = new Set(active.map(r => r.name)); + + const byWeek = {}; + commitActivity.forEach(({ repo, week, total }) => { + if (!activeNames.has(repo)) return; + byWeek[week] = (byWeek[week] || 0) + total; + }); + const weeks = Object.keys(byWeek).sort((a, b) => a.localeCompare(b)); + if (weeks.length) { + Plotly.newPlot('chart-commit-activity-weekly', [{ + x: weeks, + y: weeks.map(w => byWeek[w]), + mode: 'lines', + type: 'scatter', + fill: 'tozeroy', + line: { color: '#28a9e6' }, + }], themeLayout({ + yaxis: { title: 'Commits' }, + }), CONFIG); + } + + const byRepo = {}; + commitActivity.forEach(({ repo, week, total }) => { + if (!activeNames.has(repo)) return; + if (!byRepo[repo]) byRepo[repo] = { x: [], y: [] }; + byRepo[repo].x.push(week); + byRepo[repo].y.push(total); + }); + const repoTraces = Object.entries(byRepo) + .filter(([, d]) => d.x.length > 0) + .map(([repo, d]) => { + const pairs = d.x.map((w, i) => [w, d.y[i]]).sort((a, b) => a[0].localeCompare(b[0])); + return { + name: repo, + x: pairs.map(([w]) => w), + y: pairs.map(([, v]) => v), + mode: 'lines', + type: 'scatter', + }; + }); + if (repoTraces.length) { + Plotly.newPlot('chart-commit-activity-repos', repoTraces, themeLayout({ + yaxis: { title: 'Commits' }, + margin: { t: 30, r: 20, b: 60, l: 60 }, + }), CONFIG); + } +} + +// Docs treemap +function renderDocsChart(repos) { + const hasRTD = repos.filter(r => r.has_readthedocs); + const noRTD = repos.filter(r => !r.has_readthedocs); + const ids = ['true', 'false', + ...hasRTD.map(r => `y::${r.name}`), + ...noRTD.map(r => `n::${r.name}`)]; + const labels = ['Has ReadTheDocs', 'No ReadTheDocs', + ...hasRTD.map(r => r.name), + ...noRTD.map(r => r.name)]; + const parents = ['', '', + ...hasRTD.map(() => 'true'), + ...noRTD.map(() => 'false')]; + const values = [hasRTD.length, noRTD.length, + ...hasRTD.map(() => 1), + ...noRTD.map(() => 1)]; + Plotly.newPlot('chart-docs', [{ + type: 'treemap', ids, labels, parents, values, + branchvalues: 'total', + tiling: { squarifyratio: 1.618, pad: 2 }, + pathbar: { visible: false }, + }], themeLayout({ + margin: { t: 5, r: 5, b: 5, l: 5 }, + }, false), CONFIG); +} + +// Main +document.addEventListener('DOMContentLoaded', async () => { + const loadingEl = document.getElementById('loading-msg'); + const contentEl = document.getElementById('dashboard-content'); + try { + const [repos, prs, metadata, coverageHistory, commitActivity] = await Promise.all([ + fetchJSON('repos.json'), + fetchJSON('prs.json'), + fetchJSON('metadata.json'), + fetchJSON('coverage_history.json').catch(() => []), + fetchJSON('commit_activity.json').catch(() => []), + ]); + + const active = activeRepos(repos); + const activePRs = prs.filter(pr => active.some(r => r.name === pr.repo)); + + // Patch repos whose current coverage is 0 using their latest history value + if (coverageHistory.length) { + const latestByRepo = {}; + coverageHistory.forEach(({ repo, date, coverage }) => { + if (!latestByRepo[repo] || date > latestByRepo[repo].date) { + latestByRepo[repo] = { date, coverage }; + } + }); + active.forEach(r => { + if (!r.coverage && latestByRepo[r.name]) { + r.coverage = latestByRepo[r.name].coverage; + } + }); + } + + if (loadingEl) loadingEl.style.display = 'none'; + if (contentEl) contentEl.style.display = ''; + + renderSummary(active, activePRs, metadata); + renderBarChart('chart-stars', active.map(r => ({ name: r.name, value: r.stars }))); + renderBarChart('chart-forks', active.map(r => ({ name: r.name, value: r.forks }))); + renderBarChart('chart-issues', active.map(r => ({ name: r.name, value: r.issues }))); + renderPRsBarChart(active, activePRs); + renderPRTable(activePRs); + renderLicenseChart(active); + renderCoverageChart(active); + renderCoverageHistory(coverageHistory); + renderLanguageCharts(active); + renderCommitActivityChart(commitActivity, active); + renderDocsChart(active); + + } catch (err) { + if (loadingEl) loadingEl.innerHTML = + `
Failed to load dashboard data: ${err.message}
`; + } +}); diff --git a/gh-pages-template/index.html b/gh-pages-template/index.html new file mode 100644 index 000000000..f6ab367f6 --- /dev/null +++ b/gh-pages-template/index.html @@ -0,0 +1,120 @@ +--- +title: Dashboard +layout: page +full-width: true +after-content: + - donate.html + - support.html + +ext-css: + - href: https://cdn.datatables.net/2.2.2/css/dataTables.bootstrap5.min.css + +ext-js: + - https://cdn.plot.ly/plotly-2.35.3.min.js + - https://cdn.datatables.net/2.2.2/js/dataTables.min.js + - https://cdn.datatables.net/2.2.2/js/dataTables.bootstrap5.min.js + +js: + - /dashboard/assets/js/dashboard.js +--- + + + +
+ +

Loading dashboard data…

+
+ + diff --git a/jupyter_nbconvert_config.py b/jupyter_nbconvert_config.py deleted file mode 100644 index 4cac9eaf1..000000000 --- a/jupyter_nbconvert_config.py +++ /dev/null @@ -1,8 +0,0 @@ -# jupyter imports -from traitlets.config import get_config - -# define the config object -c = get_config() - -c.Templateexporter.exclude_input_prompt = True -c.HTMLExporter.theme = 'dark' diff --git a/notebook/dashboard.ipynb b/notebook/dashboard.ipynb deleted file mode 100644 index 225251dcb..000000000 --- a/notebook/dashboard.ipynb +++ /dev/null @@ -1,328 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "4aed1755a10d75d7", - "metadata": {}, - "source": [ - "
\n", - "

LizardByte Developer Dashboard

\n", - "
\n", - "\n", - "
\n", - "[\n", - " Developer Tools •\n", - " Repository Data •\n", - " Star Gazers •\n", - " Forks •\n", - " Open Issues •\n", - " Open PRs •\n", - " License Distribution •\n", - " Coverage •\n", - " Programming Languages •\n", - " Documentation\n", - "]\n", - "
\n", - "\n", - "
\n", - "

Developer Tools

\n", - " \n", - " \"CodeCov\"\n", - " \n", - " \n", - " \"Crowdin\n", - " \n", - " \n", - " \"Crowdin\n", - " \n", - " \n", - " \"Fedora\n", - " \n", - " \n", - " \"GitHub\n", - " \n", - " \n", - " \"GitHub\n", - " \n", - " \n", - " \"NPM\"\n", - " \n", - " \n", - " \"PyPI\"\n", - " \n", - " \n", - " \"ReadTheDocs\"\n", - " \n", - " \n", - " \"SonarCloud\"\n", - " \n", - "
\n", - "\n", - "
\n", - "

Under Construction

\n", - " \n", - " \"Cloudsmith\"\n", - " \n", - " \n", - " \"renovate\"\n", - " \n", - " \n", - " \"YouTrack\"\n", - " \n", - "
\n", - "\n", - "## Repository Data" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "initial_id", - "metadata": {}, - "outputs": [], - "source": [ - "# Setup the environment\n", - "import os\n", - "import sys\n", - "import time\n", - "\n", - "# disable TQDM output if not in debug mode\n", - "if not os.getenv('ACTIONS_RUNNER_DEBUG') and not os.getenv('ACTIONS_STEP_DEBUG'):\n", - " os.environ['TQDM_DISABLE'] = '1'\n", - "\n", - "# Add the src directory to the path\n", - "sys.path.insert(0, os.path.abspath(os.path.join(os.getcwd(), os.pardir)))\n", - "\n", - "from src import dashboard # noqa: E402" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "55ae039e9f91e18", - "metadata": {}, - "outputs": [], - "source": [ - "# get repo data\n", - "dashboard.init()\n", - "df_all_repos = dashboard.get_repo_data()\n", - "df_repos = dashboard.get_df_repos()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "466d0d2fec2e1828", - "metadata": {}, - "outputs": [], - "source": [ - "# Initial Results\n", - "print(f'Last Updated: {time.strftime(\"%Y-%m-%d %H:%M:%S\", time.gmtime())} UTC')\n", - "print(f'Total Repositories: {len(df_all_repos)}')\n", - "print(f'Archived Repositories: {df_all_repos[\"archived\"].sum()}')\n", - "print(f'Forked Repositories: {df_all_repos[\"fork\"].sum()}')\n", - "print(f'Total Open Issues: {df_all_repos[\"issues\"].apply(len).sum()}')\n", - "print(f'Total Open PRs: {df_all_repos[\"prs\"].apply(len).sum()}')\n", - "print(f'Open issues in active repositories: {df_repos[\"issues\"].apply(len).sum()}')\n", - "print(f'Open PRs in active repositories: {df_repos[\"prs\"].apply(len).sum()}')" - ] - }, - { - "cell_type": "markdown", - "id": "616d87e4b0c3dea5", - "metadata": {}, - "source": [ - "### Star Gazers" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "337aca37b216cc0d", - "metadata": {}, - "outputs": [], - "source": [ - "# Stars\n", - "dashboard.show_star_gazers()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a08398efe666a399", - "metadata": {}, - "outputs": [], - "source": [ - "# Star History Visuals\n", - "dashboard.show_star_history()" - ] - }, - { - "cell_type": "markdown", - "id": "d46c30581ed069fd", - "metadata": {}, - "source": [ - "### Forks" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7c6ffc3f9b56adc6", - "metadata": {}, - "outputs": [], - "source": [ - "# Forks\n", - "dashboard.show_forks()" - ] - }, - { - "cell_type": "markdown", - "id": "89ad81f3285c93d5", - "metadata": {}, - "source": [ - "### Open Issues" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ce4c9d640154b774", - "metadata": {}, - "outputs": [], - "source": [ - "# Open Issues\n", - "dashboard.show_issues()" - ] - }, - { - "cell_type": "markdown", - "id": "63b0eb1841d850be", - "metadata": {}, - "source": [ - "### Open PRs" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4f473fe7a839b01c", - "metadata": {}, - "outputs": [], - "source": [ - "# PR Graph\n", - "dashboard.show_pr_graph()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b416c3c48ae4e103", - "metadata": {}, - "outputs": [], - "source": [ - "# PR Table\n", - "dashboard.show_pr_table()" - ] - }, - { - "cell_type": "markdown", - "id": "3c067ce1b06d52da", - "metadata": {}, - "source": [ - "### License Distribution" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2bd8913a09eb4802", - "metadata": {}, - "outputs": [], - "source": [ - "# License distribution\n", - "dashboard.show_license_distribution()" - ] - }, - { - "cell_type": "markdown", - "id": "1e7ac0186201244b", - "metadata": {}, - "source": [ - "### Coverage" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c8dced4bcfff438", - "metadata": {}, - "outputs": [], - "source": [ - "# Coverage\n", - "dashboard.show_coverage()\n", - "\n", - "# Coverage History\n", - "dashboard.show_coverage_history()" - ] - }, - { - "cell_type": "markdown", - "id": "d9116f5c8e154dae", - "metadata": {}, - "source": [ - "### Programming Languages" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "20e09d93eda0478a", - "metadata": {}, - "outputs": [], - "source": [ - "# Programming Languages\n", - "dashboard.show_language_data()" - ] - }, - { - "cell_type": "markdown", - "id": "a520a320cd823874", - "metadata": {}, - "source": [ - "### Documentation" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ec43a8fd7f9d49bb", - "metadata": {}, - "outputs": [], - "source": [ - "# Docs\n", - "dashboard.show_docs_data()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/package.json b/package.json deleted file mode 100644 index f06816907..000000000 --- a/package.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "dependencies": { - "plotly.js": "2.35.3" - } -} diff --git a/pyproject.toml b/pyproject.toml index 485859c0e..13f39a607 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,12 +14,7 @@ authors = [ dependencies = [ "cloudscraper==1.2.71", - "ipython==9.11.0", - "itables==2.7.1", - "notebook==7.5.5", - "pandas==3.0.1", "pillow==12.1.1", - "plotly==5.24.1", "pygithub==2.8.1", "python-dotenv==1.2.2", "requests==2.32.5", @@ -27,13 +22,7 @@ dependencies = [ "unhandled_exit==1.0.0", ] -[project.optional-dependencies] -dev = [ - "nb-clean==4.0.1", - "nbqa[toolchain]==1.9.1", -] - [project.urls] -Homepage = "https://app.lizardbyte.dev/Sunshine" -Repository = "https://github.com/LizardByte/Sunshine" -Issues = "https://github.com/LizardByte/Sunshine/issues" +Homepage = "https://app.lizardbyte.dev/dashboard" +Repository = "https://github.com/LizardByte/dashboard" +Issues = "https://github.com/LizardByte/dashboard/issues" diff --git a/src/__init__.py b/src/__init__.py index 8ef445a3b..afed78b4c 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -8,3 +8,4 @@ load_dotenv() BASE_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'gh-pages') +TEMPLATE_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'gh-pages-template') diff --git a/src/builder.py b/src/builder.py new file mode 100644 index 000000000..4439a2fcf --- /dev/null +++ b/src/builder.py @@ -0,0 +1,213 @@ +"""Build consolidated dashboard data JSON files from raw collected data.""" + +# standard imports +import json +import os +from datetime import datetime, timezone + +# local imports +from src import BASE_DIR, TEMPLATE_DIR +from src.logger import log + + +def _safe_name(name: str) -> str: + return os.path.basename(name) + + +def _load_rtd_repos(base_dir: str) -> set: + rtd_repos = set() + rtd_path = os.path.join(base_dir, 'readthedocs', 'projects.json') + if not os.path.exists(rtd_path): + return rtd_repos + try: + with open(rtd_path) as f: + rtd_data = json.load(f) + for project in rtd_data: + git_url = project.get('repository', {}).get('url', '') + repo_name = git_url.rsplit('/', 1)[-1].rsplit('.git', 1)[0] + rtd_repos.add(repo_name) + except Exception: + pass + return rtd_repos + + +def _get_coverage(base_dir: str, name: str) -> float: + safe = _safe_name(name) + codecov_path = os.path.join(base_dir, 'codecov', f'{safe}.json') + if not os.path.exists(codecov_path): + return 0.0 + try: + with open(codecov_path) as f: + data = json.load(f) + return float((data.get('totals') or {}).get('coverage', 0) or 0) + except Exception: + return 0.0 + + +def _collect_coverage_history(base_dir: str, name: str) -> list: + safe = _safe_name(name) + trend_path = os.path.join(base_dir, 'codecov', f'{safe}_coverage_trend.json') + if not os.path.exists(trend_path): + return [] + try: + with open(trend_path) as f: + trend_data = json.load(f) + return [ + {'repo': name, 'date': entry.get('timestamp'), 'coverage': float(entry['avg'])} + for entry in trend_data + if entry.get('avg') is not None + ] + except Exception: + return [] + + +def _get_languages(base_dir: str, name: str) -> dict: + safe = _safe_name(name) + lang_path = os.path.join(base_dir, 'github', 'languages', f'{safe}.json') + if not os.path.exists(lang_path): + return {} + try: + with open(lang_path) as f: + return json.load(f) + except Exception: + return {} + + +def _get_prs(base_dir: str, name: str) -> list: + safe = _safe_name(name) + pulls_path = os.path.join(base_dir, 'github', 'pulls', f'{safe}.json') + if not os.path.exists(pulls_path): + return [] + try: + with open(pulls_path) as f: + return json.load(f) + except Exception: + return [] + + +def _get_commit_activity(base_dir: str, name: str) -> list: + """ + Read cached commit activity for a repo and return flat weekly records. + + Parameters + ---------- + base_dir : str + Root directory containing the gh-pages data. + name : str + Repository name. + + Returns + ------- + list + List of dicts with keys ``repo``, ``week`` (ISO date), and ``total``. + """ + safe = _safe_name(name) + path = os.path.join(base_dir, 'github', 'commitActivity', f'{safe}.json') + if not os.path.exists(path): + return [] + try: + with open(path) as f: + data = json.load(f) + result = [] + for entry in data: + week_ts = entry.get('week') + total = entry.get('total', 0) + if week_ts and total > 0: + week_str = datetime.fromtimestamp(week_ts, tz=timezone.utc).strftime('%Y-%m-%d') + result.append({'repo': name, 'week': week_str, 'total': total}) + return result + except Exception: + return [] + + +def _build_repo_entry(repo: dict, coverage: float, languages: dict, prs: list, rtd_repos: set) -> dict: + name = repo['name'] + pr_count = len(prs) + open_issues_total = repo.get('open_issues_count', 0) + issue_count = max(0, open_issues_total - pr_count) + + license_info = repo.get('license') + license_name = 'No License' + if license_info: + license_name = license_info.get('name') or license_info.get('spdx_id') or 'No License' + + return { + 'name': name, + 'stars': repo.get('stargazers_count', 0), + 'forks': repo.get('forks_count', 0), + 'issues': issue_count, + 'prs': pr_count, + 'license': license_name, + 'coverage': coverage, + 'language': repo.get('language'), + 'languages': languages, + 'archived': repo.get('archived', False), + 'fork': repo.get('fork', False), + 'topics': repo.get('topics', []), + 'has_readthedocs': name in rtd_repos, + 'created_at': repo.get('created_at'), + 'updated_at': repo.get('updated_at'), + } + + +def build(): + """ + Read raw data collected by updater.py and write dashboard-ready JSON files + into gh-pages-template/assets/data/ for consumption by the Jekyll site's JavaScript. + """ + log.info('Building dashboard data...') + + repos_path = os.path.join(BASE_DIR, 'github', 'repos.json') + if not os.path.exists(repos_path): + log.error(f'Repos file not found: {repos_path}') + return + + with open(repos_path) as f: + raw_repos = json.load(f) + + rtd_repos = _load_rtd_repos(BASE_DIR) + repos = [] + prs_all = [] + coverage_history = [] + commit_activity = [] + + for repo in raw_repos: + if repo.get('private') or repo.get('archived'): + continue + + name = repo['name'] + coverage = _get_coverage(BASE_DIR, name) + coverage_hist = _collect_coverage_history(BASE_DIR, name) + coverage_history.extend(coverage_hist) + if not coverage and coverage_hist: + coverage = max(coverage_hist, key=lambda e: e.get('date', '')).get('coverage', 0.0) + languages = _get_languages(BASE_DIR, name) + prs = _get_prs(BASE_DIR, name) + commit_activity.extend(_get_commit_activity(BASE_DIR, name)) + + repos.append(_build_repo_entry(repo, coverage, languages, prs, rtd_repos)) + prs_all.extend({'repo': name, **pr} for pr in prs) + + data_dir = os.path.join(TEMPLATE_DIR, 'assets', 'data') + os.makedirs(data_dir, exist_ok=True) + + def write_json(filename, data): + path = os.path.join(data_dir, filename) + with open(path, 'w') as f: + json.dump(data, f) + log.info(f'Written: {path}') + + write_json('repos.json', repos) + write_json('prs.json', prs_all) + write_json('coverage_history.json', coverage_history) + write_json('commit_activity.json', commit_activity) + write_json('metadata.json', { + 'updated_at': datetime.now(timezone.utc).isoformat(), + 'repo_count': len(repos), + }) + + log.info('Dashboard build complete.') + + +if __name__ == '__main__': + build() diff --git a/src/dashboard.py b/src/dashboard.py deleted file mode 100644 index 82f38df24..000000000 --- a/src/dashboard.py +++ /dev/null @@ -1,617 +0,0 @@ -# standard imports -import json -import os - -# lib imports -from github import Github, PaginatedList -from IPython.display import HTML, display -from itables import init_notebook_mode, show -import numpy as np -import pandas as pd -import plotly.express as px -import plotly.graph_objects as go -import plotly.io as pio - -# local imports -from src import BASE_DIR -from src import updater - -# Authenticate with GitHub -token = os.getenv("GITHUB_TOKEN") -g = Github(token) - -# set the default plotly template -pio.templates.default = "plotly_dark" - -# Fetch repository data -org_name = "LizardByte" -org = g.get_organization(org_name) - -# constants -text_template = '%{text}' - - -# globals -updated = False -repos = [] -df_all_repos = pd.DataFrame() -df_repos = pd.DataFrame() -df_pr_details = pd.DataFrame() -df_language_data = pd.DataFrame() -df_docs_data = pd.DataFrame() - - -def init(): - global updated - if not updated: - update_html_head_title() - updater.update() - updated = True - - -def update_html_head_title(): - script = """ - - """ - display(HTML(script)) - - -def get_repos() -> PaginatedList: - global repos - repos = org.get_repos() - return repos - - -def get_repo_data() -> pd.DataFrame: - global df_all_repos - - if not repos: - get_repos() - - # all readthedocs projects - readthedocs_path = os.path.join(BASE_DIR, 'readthedocs', 'projects.json') - with open(readthedocs_path, 'r') as f: - readthedocs_data = json.load(f) - - repo_data = [] - for repo in repos: - # skip private repos - if repo.private: - continue - - # get license - license_name = repo.license.name if repo.license else "No License" - - # split open issues and PRs - open_issues = repo.get_issues(state='open') - open_prs = [issue for issue in open_issues if issue.pull_request is not None] - open_issues = [issue for issue in open_issues if issue.pull_request is None] - - # coverage data - coverage = 0 - try: - with open(os.path.join(BASE_DIR, 'codecov', f'{repo.name}.json')) as f: - coverage_data = json.load(f) - coverage = coverage_data['totals']['coverage'] - except Exception: - pass - - # readthedocs data - readthedocs_project = None - for project in readthedocs_data: - if project['repository']['url'] == repo.clone_url: - readthedocs_project = project - - repo_data.append({ - "repo": repo.name, - "stars": repo.stargazers_count, - "archived": repo.archived, - "fork": repo.fork, - "forks": repo.forks_count, - "issues": open_issues, - "topics": repo.get_topics(), - "languages": repo.get_languages(), - "license": license_name, - "prs": open_prs, - "created_at": repo.created_at, - "updated_at": repo.updated_at, - "coverage": coverage, - "readthedocs": readthedocs_project, - "has_readthedocs": readthedocs_project is not None, - "_repo": repo, - }) - - df_all_repos = pd.DataFrame(repo_data) - return df_all_repos - - -def get_df_repos() -> pd.DataFrame: - global df_repos - - if df_all_repos.empty: - get_repo_data() - - df_repos = df_all_repos[ - (~df_all_repos['archived']) & - (~df_all_repos['topics'].apply(lambda topics: 'package-manager' in topics)) - ] - return df_repos - - -def show_star_gazers(): - if df_repos.empty: - get_df_repos() - - df_stars = df_repos.sort_values( - by='stars', - ascending=False, - ) - df_stars['log_stars'] = np.log1p(df_stars['stars']) - fig = px.bar( - df_stars, - x='repo', - y='log_stars', - title='Stars', - text='stars', - ) - fig.update_traces( - texttemplate=text_template, - textposition='inside', - ) - fig.update_layout( - yaxis_title=None, - yaxis_showticklabels=False, - ) - fig.show() - - -def get_stargazer_data() -> list: - if df_repos.empty: - get_df_repos() - - stargazer_data = [] - for repo in df_repos.to_dict('records'): - stargazers = repo['_repo'].get_stargazers_with_dates() - for stargazer in stargazers: - stargazer_data.append({ - "repo": repo['repo'], - "date": stargazer.starred_at, - }) - - return stargazer_data - - -def show_star_history(): - df_stargazers = pd.DataFrame(get_stargazer_data()) - df_stargazers = df_stargazers.sort_values(by="date") - df_stargazers["cumulative_stars"] = df_stargazers.groupby("repo").cumcount() + 1 - - fig = px.line( - df_stargazers, - x="date", - y="cumulative_stars", - color="repo", - title="Star History", - labels={"date": "Date", "cumulative_stars": "Cumulative Stars"}, - ) - fig.show() - - -def show_forks(): - if df_repos.empty: - get_df_repos() - - df_forks = df_repos.sort_values( - by='forks', - ascending=False, - ) - df_forks['log_forks'] = np.log1p(df_forks['forks']) - fig = px.bar( - df_forks, - x='repo', - y='log_forks', - title='Forks', - text='forks', - ) - fig.update_traces( - texttemplate=text_template, - textposition='inside', - ) - fig.update_layout( - yaxis_title=None, - yaxis_showticklabels=False, - ) - fig.show() - - -def show_issues(): - if df_repos.empty: - get_df_repos() - - df_issues = df_repos.copy() - df_issues['issue_count'] = df_issues['issues'].apply(len) - df_issues = df_issues.sort_values(by='issue_count', ascending=False) - df_issues['log_issues'] = np.log1p(df_issues['issue_count']) - fig = px.bar( - df_issues, - x='repo', - y='log_issues', - title='Open Issues', - text='issue_count', - ) - fig.update_traces( - texttemplate=text_template, - textposition='inside', - ) - fig.update_layout( - yaxis_title=None, - yaxis_showticklabels=False, - ) - fig.show() - - -def get_pr_data() -> pd.DataFrame: - global df_pr_details - - if df_repos.empty: - get_df_repos() - - pr_data = [] - for repo in df_repos.to_dict('records'): - for pr in repo['prs']: - pr_details = repo['_repo'].get_pull(pr.number) - - # Check if the PR has been approved - reviews = pr_details.get_reviews() - approved = any(review.state == 'APPROVED' for review in reviews) - - # Get the milestone - milestone = pr_details.milestone.title if pr_details.milestone else None - - pr_data.append({ - "repo": repo['repo'], - "number": pr_details.number, - "title": pr_details.title, - "author": pr_details.user.login, - "labels": [label.name for label in pr_details.labels], - "assignees": [assignee.login for assignee in pr_details.assignees], - "created_at": pr_details.created_at, - "last_activity": pr_details.updated_at, - "status": "Draft" if pr_details.draft else "Ready", - "approved": approved, - "milestone": milestone, - }) - - df_pr_details = pd.DataFrame(pr_data) - return df_pr_details - - -def show_pr_graph(): - if df_pr_details.empty: - get_pr_data() - - # Group by repository and status to get the count of PRs - df_pr_counts = df_pr_details.groupby(['repo', 'status']).size().reset_index(name='pr_count') - - # Sort repositories by total PR count - df_pr_counts['total_prs'] = df_pr_counts.groupby('repo')['pr_count'].transform('sum') - df_pr_counts = df_pr_counts.sort_values(by='total_prs', ascending=False) - - # Create Stacked Bar Chart - fig_bar = px.bar( - df_pr_counts, - x='repo', - y='pr_count', - color='status', - title='Open Pull Requests', - labels={'pr_count': 'Count of PRs', 'repo': 'Repository', 'status': 'PR Status'}, - text='pr_count', - category_orders={'repo': df_pr_counts['repo'].tolist()}, - ) - - fig_bar.update_layout( - yaxis_title='Open PRs', - xaxis_title='Repository', - ) - fig_bar.update_traces( - texttemplate=text_template, - textposition='inside', - ) - fig_bar.show() - - -def show_pr_table(): - if df_pr_details.empty: - get_pr_data() - - # darken the column filter inputs - css = """ - .dt-column-title input[type="text"] { - background-color: var(--jp-layout-color0); - border-color: rgb(64,67,70); - border-width: 1px; - color: var(--jp-ui-font-color1); - } - """ - display(HTML(f"")) - - init_notebook_mode( - all_interactive=True, - connected=False, - ) - - # Display the DataFrame as an interactive table using itables - table_download_name = "LizardByte-Pull-Requests" - show( - df_pr_details, - buttons=[ - "pageLength", - "copyHtml5", - {"extend": "csvHtml5", "title": table_download_name}, - {"extend": "excelHtml5", "title": table_download_name}, - ], - classes="display compact", - column_filters="header", - layout={"topEnd": None}, - ) - - -def show_license_distribution(): - if df_repos.empty: - get_df_repos() - - license_counts = df_repos.groupby(['license', 'repo']).size().reset_index(name='count') - - fig_treemap = px.treemap( - license_counts, - path=['license', 'repo'], - values='count', - title='License Distribution', - hover_data={'repo': True, 'count': False}, - ) - fig_treemap.show() - - -def show_coverage(): - if df_repos.empty: - get_df_repos() - - df_coverage = df_repos.sort_values( - by='coverage', - ascending=False, - ) - - # inverse marker size, so higher coverage has smaller markers - df_coverage['marker_size'] = df_coverage['coverage'].apply(lambda x: 110 - x if x > 0 else 0) - - fig_scatter = px.scatter( - df_coverage, - x='repo', - y='coverage', - title='Coverage (Current)', - size='marker_size', - color='coverage', - color_continuous_scale=['red', 'yellow', 'green'], # red is low, green is high - ) - fig_scatter.update_layout( - yaxis_title='Coverage Percentage', - xaxis_title='Repository', - ) - fig_scatter.show() - - -def get_coverage_trend_data() -> list: - """ - Get coverage trend data for all repositories from codecov. - """ - if df_repos.empty: - get_df_repos() - - coverage_trend_data = [] - for repo in df_repos.to_dict('records'): - trend_file = os.path.join(BASE_DIR, 'codecov', f'{repo["repo"]}_coverage_trend.json') - - if os.path.exists(trend_file): - try: - with open(trend_file, 'r') as f: - trend_data = json.load(f) - - for entry in trend_data: - # Parse the coverage data - use avg coverage value - if 'avg' in entry and entry['avg'] is not None: - coverage_trend_data.append({ - "repo": repo['repo'], - "date": entry.get('timestamp'), - "coverage": entry['avg'], - }) - except Exception: - # Skip repos with no coverage data or errors - pass - - return coverage_trend_data - - -def show_coverage_history(): - """ - Display coverage history over time for all repositories, similar to star history. - """ - df_coverage_trend = pd.DataFrame(get_coverage_trend_data()) - - if df_coverage_trend.empty: - print("No coverage trend data available.") - return - - # Convert date string to datetime - df_coverage_trend['date'] = pd.to_datetime(df_coverage_trend['date']) - df_coverage_trend = df_coverage_trend.sort_values(by="date") - - fig = px.line( - df_coverage_trend, - x="date", - y="coverage", - color="repo", - title="Coverage (History)", - labels={"date": "Date", "coverage": "Coverage %"}, - ) - fig.update_layout( - yaxis_title='Coverage Percentage', - xaxis_title='Date', - yaxis_range=[0, 100], - ) - fig.show() - - -def get_language_data(): - global df_language_data - - language_data = [] - for repo in df_repos.to_dict('records'): - for language, bytes_of_code in repo['languages'].items(): - language_data.append({ - "repo": repo['repo'], - "language": language, - "bytes_of_code": bytes_of_code, - }) - - df_language_data = pd.DataFrame(language_data) - return df_language_data - - -def show_language_data(): - if df_language_data.empty: - get_language_data() - - # Aggregate data by language and repo - language_counts_bytes = df_language_data.groupby(['language', 'repo']).agg({ - 'bytes_of_code': 'sum' - }).reset_index() - language_counts_repos = df_language_data.groupby(['language', 'repo']).size().reset_index(name='repo_count') - - def create_language_figures(counts: pd.DataFrame, path_key: str, value_key: str): - _fig_treemap = px.treemap( - counts, - path=[path_key, 'repo'], - values=value_key, - ) - _fig_sunburst = px.sunburst( - counts, - path=[path_key, 'repo'], - values=value_key, - ) - return _fig_treemap, _fig_sunburst - - # List of tuples containing the data and titles for each figure - figures_data = [ - (language_counts_bytes, 'language', 'bytes_of_code', 'Programming Languages by Bytes of Code'), - (language_counts_repos, 'language', 'repo_count', 'Programming Languages by Repo Count') - ] - - # Loop through the list to create figures and add traces - for _counts, _path_key, value_key, title in figures_data: - fig_treemap, fig_sunburst = create_language_figures(counts=_counts, path_key=_path_key, value_key=value_key) - - fig = go.Figure() - fig.add_trace(fig_treemap.data[0]) - fig.add_trace(fig_sunburst.data[0]) - fig.data[1].visible = False - - fig.update_layout( - title=title, - updatemenus=[ - { - "buttons": [ - { - "label": "Treemap", - "method": "update", - "args": [ - {"visible": [True, False]}, - ], - }, - { - "label": "Sunburst", - "method": "update", - "args": [ - {"visible": [False, True]}, - ], - }, - ], - "direction": "down", - "showactive": True, - } - ] - ) - fig.show() - - -def get_docs_data(): - global df_docs_data - - docs_data = [] - for repo in df_repos.to_dict('records'): - docs_data.append({ - "repo": repo['repo'], - "has_readthedocs": repo['has_readthedocs'], - }) - - df_docs_data = pd.DataFrame(docs_data) - return df_docs_data - - -def show_docs_data(): - if df_docs_data.empty: - get_docs_data() - readthedocs_counts = df_docs_data.groupby(['has_readthedocs', 'repo']).size().reset_index(name='repo_count') - - def create_figures(counts: pd.DataFrame, path_key: str): - _fig_treemap = px.treemap( - counts, - path=[path_key, 'repo'], - values='repo_count', - ) - _fig_sunburst = px.sunburst( - counts, - path=[path_key, 'repo'], - values='repo_count', - ) - return _fig_treemap, _fig_sunburst - - # List of tuples containing the data and titles for each figure - figures_data = [ - (readthedocs_counts, 'has_readthedocs', 'Uses ReadTheDocs') - ] - - # Loop through the list to create figures and add traces - for _counts, _path_key, title in figures_data: - fig_treemap, fig_sunburst = create_figures(counts=_counts, path_key=_path_key) - - fig = go.Figure() - fig.add_trace(fig_treemap.data[0]) - fig.add_trace(fig_sunburst.data[0]) - fig.data[1].visible = False - - fig.update_layout( - title=title, - updatemenus=[ - { - "buttons": [ - { - "label": "Treemap", - "method": "update", - "args": [{"visible": [True, False]}], - }, - { - "label": "Sunburst", - "method": "update", - "args": [{"visible": [False, True]}], - }, - ], - "direction": "down", - "showactive": True, - } - ] - ) - fig.show() diff --git a/src/helpers.py b/src/helpers.py index dc03010ee..87a2a563c 100644 --- a/src/helpers.py +++ b/src/helpers.py @@ -17,22 +17,32 @@ # constants HTTPS = 'https://' +DEFAULT_TIMEOUT = 30 # seconds # setup requests sessions retry_adapter = HTTPAdapter(max_retries=Retry(total=5, backoff_factor=1)) + +class TimeoutSession(requests.Session): + """A requests.Session that applies a default timeout to every request.""" + + def request(self, *args, **kwargs): + kwargs.setdefault('timeout', DEFAULT_TIMEOUT) + return super().request(*args, **kwargs) + + # cloudscraper session cs = cloudscraper.CloudScraper() # CloudScraper inherits from requests.Session cs.mount(HTTPS, retry_adapter) # requests session -s = requests.Session() +s = TimeoutSession() s.mount(HTTPS, retry_adapter) -class RateLimitedSession(requests.Session): +class RateLimitedSession(TimeoutSession): """ - A requests.Session subclass that implements rate limiting. + A TimeoutSession subclass that implements rate limiting. """ def __init__(self, calls_per_minute=60): super().__init__() diff --git a/src/updater.py b/src/updater.py index f3d656318..91b21400a 100644 --- a/src/updater.py +++ b/src/updater.py @@ -1,9 +1,11 @@ # standard imports +import json import os +from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeout from threading import Thread # lib imports -from github import Github +from github import Auth, Github import requests from tqdm import tqdm import unhandled_exit @@ -80,6 +82,15 @@ def update_codecov(): """ Get code coverage data from Codecov API. """ + archived_repos = set() + repos_path = os.path.join(BASE_DIR, 'github', 'repos.json') + if os.path.exists(repos_path): + try: + with open(repos_path) as f: + archived_repos = {r['name'] for r in json.load(f) if r.get('archived')} + except Exception as e: + log.warning(f'Could not load GitHub repos for archived check: {e}') + headers = { 'Accept': 'application/json', 'Authorization': f'bearer {os.environ["CODECOV_TOKEN"]}', @@ -105,6 +116,9 @@ def update_codecov(): iterable=data['results'], desc='Updating Codecov data', ): + if repo['name'] in archived_repos: + continue + # Get repo details url = f'{base_url}/repos/{repo["name"]}' response = helpers.s.get(url=url, headers=headers) @@ -176,12 +190,37 @@ def update_fb(): helpers.write_json_files(file_path=file_path, data=data) +def _get_stats_with_timeout(repo, timeout=60): + """ + Fetch commit activity for a repo, capping total wait time. + + Parameters + ---------- + repo : + PyGithub Repository object. + timeout : int + Maximum seconds to wait before giving up (GitHub may return 202 while + computing stats, causing PyGithub to retry indefinitely without this guard). + + Returns + ------- + list or None + Weekly commit-activity objects, or None on timeout. + """ + with ThreadPoolExecutor(max_workers=1) as pool: + future = pool.submit(repo.get_stats_commit_activity) + try: + return future.result(timeout=timeout) + except FuturesTimeout: + log.warning(f'Timeout fetching commit activity for {repo.name}, skipping.') + return None + + def update_github(): """ Cache and update GitHub Repo banners and data. """ - # Initialize PyGithub client - g = Github(os.environ["GITHUB_TOKEN"]) + g = Github(auth=Auth.Token(os.environ["GITHUB_TOKEN"]), timeout=30) # Get the user/organization owner = g.get_user(os.environ["GITHUB_REPOSITORY_OWNER"]) @@ -216,12 +255,29 @@ def update_github(): file_path = os.path.join(BASE_DIR, 'github', 'languages', repo.name) helpers.write_json_files(file_path=file_path, data=languages) - # commit activity (last year of activity) - commit_activity = repo.get_stats_commit_activity() - commits = [week.raw_data for week in commit_activity] # Convert PyGithub objects to dict format - - file_path = os.path.join(BASE_DIR, 'github', 'commitActivity', repo.name) - helpers.write_json_files(file_path=file_path, data=commits) + # commit activity (last year, weekly buckets) + commit_activity = _get_stats_with_timeout(repo) + if commit_activity: + commits = [week.raw_data for week in commit_activity] + file_path = os.path.join(BASE_DIR, 'github', 'commitActivity', repo.name) + helpers.write_json_files(file_path=file_path, data=commits) + + # open pull requests + pulls_data = [] + for pr in repo.get_pulls(state='open'): + pulls_data.append({ + 'number': pr.number, + 'title': pr.title, + 'author': pr.user.login, + 'labels': [label.name for label in pr.labels], + 'assignees': [assignee.login for assignee in pr.assignees], + 'created_at': pr.created_at.isoformat(), + 'updated_at': pr.updated_at.isoformat(), + 'draft': pr.draft, + 'milestone': pr.milestone.title if pr.milestone else None, + }) + file_path = os.path.join(BASE_DIR, 'github', 'pulls', repo.name) + helpers.write_json_files(file_path=file_path, data=pulls_data) # openGraphImages - uses GraphQL query = """ @@ -351,69 +407,83 @@ def append_thread_if_env_set( def update(): - threads = [] + # Threads that are fully independent of each other and of GitHub data. + independent_threads = [] append_thread_if_env_set( env_vars=['DASHBOARD_AUR_REPOS'], name='aur', target=update_aur, - threads=threads, + threads=independent_threads, kwargs={'aur_repos': os.getenv('DASHBOARD_AUR_REPOS').split(',')}, ) - append_thread_if_env_set( - env_vars=['CODECOV_TOKEN', 'GITHUB_REPOSITORY_OWNER'], - name='codecov', - target=update_codecov, - threads=threads, - ) append_thread_if_env_set( env_vars=['DISCORD_INVITE'], name='discord', target=update_discord, - threads=threads, + threads=independent_threads, ) append_thread_if_env_set( env_vars=['FACEBOOK_TOKEN', 'FACEBOOK_PAGE_ID'], name='facebook', target=update_fb, - threads=threads, - ) - append_thread_if_env_set( - env_vars=['GITHUB_TOKEN', 'GITHUB_REPOSITORY_OWNER'], - name='github', - target=update_github, - threads=threads, + threads=independent_threads, ) append_thread_if_env_set( env_vars=['PATREON_CAMPAIGN_ID'], name='patreon', target=update_patreon, - threads=threads, + threads=independent_threads, ) append_thread_if_env_set( env_vars=['READTHEDOCS_TOKEN'], name='readthedocs', target=update_readthedocs, - threads=threads, + threads=independent_threads, + ) + + # GitHub must finish before Codecov so that repos.json is up to date and + # update_codecov() can correctly skip archived repos. + github_threads = [] + append_thread_if_env_set( + env_vars=['GITHUB_TOKEN', 'GITHUB_REPOSITORY_OWNER'], + name='github', + target=update_github, + threads=github_threads, + ) + + codecov_threads = [] + append_thread_if_env_set( + env_vars=['CODECOV_TOKEN', 'GITHUB_REPOSITORY_OWNER'], + name='codecov', + target=update_codecov, + threads=codecov_threads, ) # setup threading exception handling if os.getenv('THREADING_EXCEPTION_HANDLER'): unhandled_exit.activate() - for thread in tqdm( - iterable=threads, - desc='Starting threads', - ): + # Phase 1: start independent threads and GitHub in parallel. + for thread in tqdm(iterable=independent_threads + github_threads, desc='Starting threads'): thread.start() - # wait for all threads to finish - for thread in tqdm( - iterable=threads, - desc='Waiting for threads to finish', - ): + # Wait for GitHub before starting Codecov. + for thread in tqdm(iterable=github_threads, desc='Waiting for GitHub thread'): + thread.join() + + # Phase 2: start Codecov now that repos.json is fresh. + for thread in tqdm(iterable=codecov_threads, desc='Starting Codecov thread'): + thread.start() + + # Wait for all remaining threads. + for thread in tqdm(iterable=independent_threads + codecov_threads, desc='Waiting for threads to finish'): thread.join() # deactivate threading exception handling if os.getenv('THREADING_EXCEPTION_HANDLER'): unhandled_exit.deactivate() + + +if __name__ == '__main__': + update() From 31dcee5a579e1ea8f1e0f1aa512a3ec25d4ee5d1 Mon Sep 17 00:00:00 2001 From: ReenigneArcher <42013603+ReenigneArcher@users.noreply.github.com> Date: Thu, 19 Mar 2026 23:27:50 -0400 Subject: [PATCH 2/3] Add star history tracking and chart Introduce star-history collection, caching, and visualization. - Frontend: add a star-history line chart (renderStarHistory) and new DOM element in index.html; dashboard now fetches star_history.json and renders history alongside current stars. - Builder: read per-repo cached star history files and include a consolidated star_history.json in the built site data. - Updater: implement _collect_star_history with an initial seeding strategy (_seed_star_history) that samples stargazer pages, then appends only today's star count on subsequent runs to minimize API calls; cache per-repo starHistory JSON files. Refactor per-repo processing into _process_github_repo and call it from update_github to consolidate logic. This adds persistent, sampled star timelines while keeping API usage low after the initial seed. --- gh-pages-template/assets/js/dashboard.js | 31 ++- gh-pages-template/index.html | 3 + src/builder.py | 31 +++ src/updater.py | 237 +++++++++++++++++------ 4 files changed, 245 insertions(+), 57 deletions(-) diff --git a/gh-pages-template/assets/js/dashboard.js b/gh-pages-template/assets/js/dashboard.js index 9bf5f1ea7..5df7fc125 100644 --- a/gh-pages-template/assets/js/dashboard.js +++ b/gh-pages-template/assets/js/dashboard.js @@ -87,6 +87,33 @@ function renderBarChart(divId, entries) { }), CONFIG); } +// Star history line chart +function renderStarHistory(history) { + const el = document.getElementById('chart-star-history'); + if (!el) return; + if (!history?.length) { + el.innerHTML = '

No star history available.

'; + return; + } + const byRepo = {}; + history.forEach(({ repo, date, stars }) => { + if (!byRepo[repo]) byRepo[repo] = { x: [], y: [] }; + byRepo[repo].x.push(date); + byRepo[repo].y.push(stars); + }); + const traces = Object.entries(byRepo) + .sort((a, b) => { + const lastA = a[1].y.at(-1) ?? 0; + const lastB = b[1].y.at(-1) ?? 0; + return lastB - lastA; + }) + .map(([repo, d]) => ({ name: repo, x: d.x, y: d.y, mode: 'lines+markers', type: 'scatter' })); + Plotly.newPlot('chart-star-history', traces, themeLayout({ + yaxis: { title: 'Stars' }, + margin: { t: 30, r: 20, b: 60, l: 60 }, + }), CONFIG); +} + // PRs bar chart function renderPRsBarChart(active, prs) { const el = document.getElementById('chart-prs'); @@ -483,12 +510,13 @@ document.addEventListener('DOMContentLoaded', async () => { const loadingEl = document.getElementById('loading-msg'); const contentEl = document.getElementById('dashboard-content'); try { - const [repos, prs, metadata, coverageHistory, commitActivity] = await Promise.all([ + const [repos, prs, metadata, coverageHistory, commitActivity, starHistory] = await Promise.all([ fetchJSON('repos.json'), fetchJSON('prs.json'), fetchJSON('metadata.json'), fetchJSON('coverage_history.json').catch(() => []), fetchJSON('commit_activity.json').catch(() => []), + fetchJSON('star_history.json').catch(() => []), ]); const active = activeRepos(repos); @@ -514,6 +542,7 @@ document.addEventListener('DOMContentLoaded', async () => { renderSummary(active, activePRs, metadata); renderBarChart('chart-stars', active.map(r => ({ name: r.name, value: r.stars }))); + renderStarHistory(starHistory); renderBarChart('chart-forks', active.map(r => ({ name: r.name, value: r.forks }))); renderBarChart('chart-issues', active.map(r => ({ name: r.name, value: r.issues }))); renderPRsBarChart(active, activePRs); diff --git a/gh-pages-template/index.html b/gh-pages-template/index.html index f6ab367f6..5096cd919 100644 --- a/gh-pages-template/index.html +++ b/gh-pages-template/index.html @@ -54,7 +54,10 @@

Star Gazers

+

Current

+

History

+
diff --git a/src/builder.py b/src/builder.py index 4439a2fcf..0fae531f5 100644 --- a/src/builder.py +++ b/src/builder.py @@ -120,6 +120,34 @@ def _get_commit_activity(base_dir: str, name: str) -> list: return [] +def _get_star_history(base_dir: str, name: str) -> list: + """ + Read the cached sampled star history for a repo. + + Parameters + ---------- + base_dir : str + Root directory containing the gh-pages data. + name : str + Repository name. + + Returns + ------- + list + List of dicts with keys ``repo``, ``date``, and ``stars``. + """ + safe = _safe_name(name) + path = os.path.join(base_dir, 'github', 'starHistory', f'{safe}.json') + if not os.path.exists(path): + return [] + try: + with open(path) as f: + data = json.load(f) + return [{'repo': name, 'date': e['date'], 'stars': e['stars']} for e in data] + except Exception: + return [] + + def _build_repo_entry(repo: dict, coverage: float, languages: dict, prs: list, rtd_repos: set) -> dict: name = repo['name'] pr_count = len(prs) @@ -170,6 +198,7 @@ def build(): prs_all = [] coverage_history = [] commit_activity = [] + star_history = [] for repo in raw_repos: if repo.get('private') or repo.get('archived'): @@ -184,6 +213,7 @@ def build(): languages = _get_languages(BASE_DIR, name) prs = _get_prs(BASE_DIR, name) commit_activity.extend(_get_commit_activity(BASE_DIR, name)) + star_history.extend(_get_star_history(BASE_DIR, name)) repos.append(_build_repo_entry(repo, coverage, languages, prs, rtd_repos)) prs_all.extend({'repo': name, **pr} for pr in prs) @@ -201,6 +231,7 @@ def write_json(filename, data): write_json('prs.json', prs_all) write_json('coverage_history.json', coverage_history) write_json('commit_activity.json', commit_activity) + write_json('star_history.json', star_history) write_json('metadata.json', { 'updated_at': datetime.now(timezone.utc).isoformat(), 'repo_count': len(repos), diff --git a/src/updater.py b/src/updater.py index 91b21400a..ce8186b0f 100644 --- a/src/updater.py +++ b/src/updater.py @@ -1,7 +1,9 @@ # standard imports import json +import math import os from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeout +from datetime import datetime, timezone from threading import Thread # lib imports @@ -216,11 +218,189 @@ def _get_stats_with_timeout(repo, timeout=60): return None +def _seed_star_history(repo, total: int, initial_samples: int) -> list[dict]: + """ + Fetch evenly-spaced pages from the stargazers API for a first-time seed. + + Parameters + ---------- + repo : + PyGithub Repository object. + total : int + Current star count (used to calculate page spread). + initial_samples : int + Maximum number of pages to request. + + Returns + ------- + list + Unsorted list of ``{date, stars}`` dicts sampled across the history. + """ + per_page = 100 + total_pages = math.ceil(total / per_page) + + if total_pages <= initial_samples: + pages_to_fetch = list(range(total_pages)) + else: + pages_to_fetch = sorted({ + round(i * (total_pages - 1) / (initial_samples - 1)) + for i in range(initial_samples) + }) + + history = [] + stargazers = repo.get_stargazers_with_dates() + for page_idx in pages_to_fetch: + try: + page = stargazers.get_page(page_idx) + if not page: + continue + history.append({ + 'date': page[0].starred_at.strftime('%Y-%m-%d'), + 'stars': page_idx * per_page + 1, + }) + if len(page) > 1: + history.append({ + 'date': page[-1].starred_at.strftime('%Y-%m-%d'), + 'stars': page_idx * per_page + len(page), + }) + except Exception as e: + log.warning(f'Error fetching star history page {page_idx} for {repo.name}: {e}') + + return history + + +def _collect_star_history(repo, initial_samples: int = 5) -> list: + """ + Build a cumulative star-history time series for a repository. + + On the first call for a repo the function seeds the history by fetching a + small number of evenly-spaced API pages (``initial_samples``). On every + subsequent call it reads the cached file and appends only today's current + star count, so **no additional API requests are made after the initial + seed**. + + Parameters + ---------- + repo : + PyGithub Repository object. + initial_samples : int + Number of pages to fetch when no cached history exists yet. + + Returns + ------- + list + List of dicts with keys ``date`` (YYYY-MM-DD) and ``stars`` + (cumulative star count at that point in time). + """ + total = repo.stargazers_count + if total == 0: + return [] + + today = datetime.now(tz=timezone.utc).strftime('%Y-%m-%d') + cache_path = os.path.join(BASE_DIR, 'github', 'starHistory', f'{repo.name}.json') + + existing = [] + if os.path.exists(cache_path): + try: + with open(cache_path) as f: + existing = json.load(f) + except Exception: + pass + + if existing: + if existing[-1]['date'] == today: + existing[-1]['stars'] = total + else: + existing.append({'date': today, 'stars': total}) + return existing + + history: list[dict] = list(_seed_star_history(repo, total, initial_samples)) + if not history or history[-1]['date'] != today: + history.append({'date': today, 'stars': total}) + return history + + +def _process_github_repo(repo, headers: dict, graphql_url: str) -> None: + """ + Collect and cache all per-repository data for a single GitHub repo. + + Parameters + ---------- + repo : + PyGithub Repository object. + headers : dict + HTTP headers including the GitHub authorisation token. + graphql_url : str + GitHub GraphQL endpoint URL. + """ + # languages + languages = repo.get_languages() + file_path = os.path.join(BASE_DIR, 'github', 'languages', repo.name) + helpers.write_json_files(file_path=file_path, data=languages) + + # commit activity (last year, weekly buckets) + commit_activity = _get_stats_with_timeout(repo) + if commit_activity: + commits = [week.raw_data for week in commit_activity] + file_path = os.path.join(BASE_DIR, 'github', 'commitActivity', repo.name) + helpers.write_json_files(file_path=file_path, data=commits) + + # open pull requests + pulls_data = [] + for pr in repo.get_pulls(state='open'): + pulls_data.append({ + 'number': pr.number, + 'title': pr.title, + 'author': pr.user.login, + 'labels': [label.name for label in pr.labels], + 'assignees': [assignee.login for assignee in pr.assignees], + 'created_at': pr.created_at.isoformat(), + 'updated_at': pr.updated_at.isoformat(), + 'draft': pr.draft, + 'milestone': pr.milestone.title if pr.milestone else None, + }) + file_path = os.path.join(BASE_DIR, 'github', 'pulls', repo.name) + helpers.write_json_files(file_path=file_path, data=pulls_data) + + # star history (sampled to cap API calls) + star_history = _collect_star_history(repo) + if star_history: + file_path = os.path.join(BASE_DIR, 'github', 'starHistory', repo.name) + helpers.write_json_files(file_path=file_path, data=star_history) + + # openGraphImages - uses GraphQL + query = """ + { + repository(owner: "%s", name: "%s") { + openGraphImageUrl + } + } + """ % (repo.owner.login, repo.name) + + response = helpers.s.post(url=graphql_url, json={'query': query}, headers=headers) + repo_data = response.json() + try: + image_url = repo_data['data']['repository']['openGraphImageUrl'] + except KeyError: + log.error(f'Error: update_github: {repo_data}') + raise SystemExit('"GITHUB_TOKEN" is invalid.') + if 'avatars' not in image_url: + file_path = os.path.join(BASE_DIR, 'github', 'openGraphImages', repo.name) + helpers.save_image_from_url( + file_path=file_path, + file_extension='png', + image_url=image_url, + size_x=624, + size_y=312, + ) + + def update_github(): """ Cache and update GitHub Repo banners and data. """ g = Github(auth=Auth.Token(os.environ["GITHUB_TOKEN"]), timeout=30) + g.per_page = 100 # Get the user/organization owner = g.get_user(os.environ["GITHUB_REPOSITORY_OWNER"]) @@ -246,64 +426,9 @@ def update_github(): iterable=repos, desc='Updating GitHub data', ): - # skip archived repos if repo.archived: continue - - # languages - use PyGithub - languages = repo.get_languages() - file_path = os.path.join(BASE_DIR, 'github', 'languages', repo.name) - helpers.write_json_files(file_path=file_path, data=languages) - - # commit activity (last year, weekly buckets) - commit_activity = _get_stats_with_timeout(repo) - if commit_activity: - commits = [week.raw_data for week in commit_activity] - file_path = os.path.join(BASE_DIR, 'github', 'commitActivity', repo.name) - helpers.write_json_files(file_path=file_path, data=commits) - - # open pull requests - pulls_data = [] - for pr in repo.get_pulls(state='open'): - pulls_data.append({ - 'number': pr.number, - 'title': pr.title, - 'author': pr.user.login, - 'labels': [label.name for label in pr.labels], - 'assignees': [assignee.login for assignee in pr.assignees], - 'created_at': pr.created_at.isoformat(), - 'updated_at': pr.updated_at.isoformat(), - 'draft': pr.draft, - 'milestone': pr.milestone.title if pr.milestone else None, - }) - file_path = os.path.join(BASE_DIR, 'github', 'pulls', repo.name) - helpers.write_json_files(file_path=file_path, data=pulls_data) - - # openGraphImages - uses GraphQL - query = """ - { - repository(owner: "%s", name: "%s") { - openGraphImageUrl - } - } - """ % (repo.owner.login, repo.name) - - response = helpers.s.post(url=graphql_url, json={'query': query}, headers=headers) - repo_data = response.json() - try: - image_url = repo_data['data']['repository']['openGraphImageUrl'] - except KeyError: - log.error(f'Error: update_github: {repo_data}') - raise SystemExit('"GITHUB_TOKEN" is invalid.') - if 'avatars' not in image_url: - file_path = os.path.join(BASE_DIR, 'github', 'openGraphImages', repo.name) - helpers.save_image_from_url( - file_path=file_path, - file_extension='png', - image_url=image_url, - size_x=624, - size_y=312, - ) + _process_github_repo(repo, headers, graphql_url) def update_patreon(): From 335c8c0aead17e47dc810043dcc59210bb3e7f2b Mon Sep 17 00:00:00 2001 From: ReenigneArcher <42013603+ReenigneArcher@users.noreply.github.com> Date: Fri, 20 Mar 2026 16:02:05 -0400 Subject: [PATCH 3/3] Enable DataTables export buttons and layout Add DataTables Buttons integration and update table layout for the PR table. dashboard.js was updated to configure the table layout (placing page length and export buttons at topStart, search at topEnd, info and paging at bottom). index.html includes new CSS/JS assets required for Buttons (buttons.bootstrap5, dataTables.buttons, buttons.html5, buttons.print) and jszip for Excel export, plus the buttons stylesheet. This enables copy/csv/excel/print export controls and adjusts the table UI. --- gh-pages-template/assets/js/dashboard.js | 20 ++++++++++++++++++-- gh-pages-template/index.html | 16 +++++++++++++--- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/gh-pages-template/assets/js/dashboard.js b/gh-pages-template/assets/js/dashboard.js index 5df7fc125..8e6930912 100644 --- a/gh-pages-template/assets/js/dashboard.js +++ b/gh-pages-template/assets/js/dashboard.js @@ -199,10 +199,26 @@ function renderPRTable(prs) { }); }); const dt = new DataTable('#pr-datatable', { - pageLength: 25, + pageLength: 10, lengthMenu: [10, 25, 50, 100], order: [[7, 'asc']], orderCellsTop: true, + autoFill: true, + layout: { + topStart: [ + 'pageLength', + { + buttons: [ + 'copy', + 'csv', + 'excel', + ], + }, + ], + topEnd: 'search', + bottomStart: 'info', + bottomEnd: 'paging', + }, }); filterInputs.forEach((input, i) => { input.addEventListener('input', () => { @@ -231,7 +247,7 @@ function renderPRTable(prs) { const opt = document.createElement('option'); opt.value = String(n); opt.textContent = String(n); - if (n === 25) opt.selected = true; + if (n === 10) opt.selected = true; lengthSel.appendChild(opt); }); const lengthLabel = document.createElement('label'); diff --git a/gh-pages-template/index.html b/gh-pages-template/index.html index 5096cd919..0c47fb639 100644 --- a/gh-pages-template/index.html +++ b/gh-pages-template/index.html @@ -6,13 +6,23 @@ - donate.html - support.html +# see: https://datatables.net/download/release ext-css: - - href: https://cdn.datatables.net/2.2.2/css/dataTables.bootstrap5.min.css + - href: https://cdn.datatables.net/2.3.7/css/dataTables.bootstrap5.min.css + - https://cdn.datatables.net/autofill/2.7.1/css/autoFill.bootstrap5.min.css + - href: https://cdn.datatables.net/buttons/3.2.6/css/buttons.bootstrap5.min.css ext-js: - https://cdn.plot.ly/plotly-2.35.3.min.js - - https://cdn.datatables.net/2.2.2/js/dataTables.min.js - - https://cdn.datatables.net/2.2.2/js/dataTables.bootstrap5.min.js + - https://code.jquery.com/jquery-3.7.1.min.js + - https://cdn.datatables.net/2.3.7/js/dataTables.min.js + - https://cdn.datatables.net/2.3.7/js/dataTables.bootstrap5.min.js + - https://cdn.datatables.net/autofill/2.7.1/js/dataTables.autoFill.min.js + - https://cdn.datatables.net/autofill/2.7.1/js/autoFill.bootstrap5.min.js + - https://cdnjs.cloudflare.com/ajax/libs/jszip/3.10.1/jszip.min.js # required for excel export + - https://cdn.datatables.net/buttons/3.2.6/js/dataTables.buttons.min.js + - https://cdn.datatables.net/buttons/3.2.6/js/buttons.bootstrap5.min.js + - https://cdn.datatables.net/buttons/3.2.6/js/buttons.html5.min.js js: - /dashboard/assets/js/dashboard.js