diff --git a/.github/workflows/update-pages.yml b/.github/workflows/update-pages.yml index 33ac5faeb..4a55e517f 100644 --- a/.github/workflows/update-pages.yml +++ b/.github/workflows/update-pages.yml @@ -16,7 +16,7 @@ concurrency: cancel-in-progress: true jobs: - update_pages: + update: permissions: contents: read runs-on: ubuntu-latest @@ -25,48 +25,17 @@ jobs: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Checkout gh-pages - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - ref: gh-pages - path: gh-pages - persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of the personal token - fetch-depth: 0 # otherwise, will fail to push refs to dest repo - - name: Setup python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.14' - - name: Setup node - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 - with: - node-version: 'latest' - - name: Install Python dependencies run: | python -m pip install --upgrade pip setuptools wheel python -m pip install . - - name: Install npm dependencies - run: npm install --ignore-scripts - - - name: Get current date - id: date - run: echo "date=$(date +'%Y-%m-%d')" >> "${GITHUB_OUTPUT}" - - - name: Prepare gh-pages - run: | - # empty contents - rm -f -r ./gh-pages/* - - # no jekyll - touch ./gh-pages/.nojekyll - - # copy dependencies - cp -f ./node_modules/plotly.js/dist/plotly.min.js ./gh-pages/plotly.js - - - name: Convert notebook + - name: Collect data env: DASHBOARD_AUR_REPOS: sunshine,sunshine-bin,sunshine-git CODECOV_TOKEN: ${{ secrets.CODECOV_API_TOKEN }} @@ -78,62 +47,40 @@ jobs: GITHUB_TOKEN: ${{ secrets.GH_BOT_TOKEN || secrets.GITHUB_TOKEN }} PATREON_CAMPAIGN_ID: 6131567 READTHEDOCS_TOKEN: ${{ secrets.READTHEDOCS_TOKEN }} - run: | - jupyter nbconvert \ - --debug \ - --config=./jupyter_nbconvert_config.py \ - --execute \ - --no-input \ - --to=html \ - --output-dir=./gh-pages \ - --output=index \ - ./notebook/dashboard.ipynb + THREADING_EXCEPTION_HANDLER: "true" + run: python -u -m src.updater - name: Cat log + if: always() run: cat ./logs/updater.log - - name: Check notebook for tracebacks - run: | - cat ./gh-pages/index.html - echo "---" - echo "Checking for tracebacks..." - set +e - tracebacks=$(grep -i -E 'Traceback \(most recent call last\):' ./gh-pages/index.html) - set -e - if [ -n "${tracebacks}" ]; then - echo "Tracebacks found:" - echo "${tracebacks}" - exit 1 - fi + - name: Build dashboard data + run: python -u -m src.builder - - name: Archive gh-pages - if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' + - name: Prepare Artifacts # uploading artifacts will fail if not zipped due to very large quantity of files shell: bash - run: | - 7z \ - "-xr!*.git*" \ - a "./gh-pages.zip" "./gh-pages" + run: 7z a build.zip ./gh-pages/* ./gh-pages-template/* - - name: Upload Artifacts - if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' + - name: Upload artifact uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: - name: gh-pages + name: update + path: ${{ github.workspace }}/build.zip if-no-files-found: error - path: | - ${{ github.workspace }}/gh-pages.zip + include-hidden-files: true + retention-days: 1 - - name: Deploy to gh-pages - if: >- - (github.event_name == 'push' && github.ref == 'refs/heads/master') || - github.event_name == 'schedule' || - github.event_name == 'workflow_dispatch' - uses: actions-js/push@5a7cbd780d82c0c937b5977586e641b2fd94acc5 # v1.5 - with: - github_token: ${{ secrets.GH_BOT_TOKEN }} - author_email: ${{ secrets.GH_BOT_EMAIL }} - author_name: ${{ vars.GH_BOT_NAME }} - directory: gh-pages - branch: gh-pages - force: false - message: automatic-update-${{ steps.date.outputs.date }} + call-jekyll-build: + needs: update + permissions: + contents: read + uses: LizardByte/LizardByte.github.io/.github/workflows/jekyll-build.yml@master + secrets: + GH_BOT_EMAIL: ${{ secrets.GH_BOT_EMAIL }} + GH_BOT_TOKEN: ${{ secrets.GH_BOT_TOKEN }} + with: + gh_bot_name: ${{ vars.GH_BOT_NAME }} + site_artifact: 'update' + extract_archive: 'build.zip' + target_branch: 'gh-pages' + clean_gh_pages: true diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 000000000..681f205b8 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,25 @@ +--- +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +version: 2 + +build: + os: ubuntu-24.04 + tools: + ruby: "3.3" + apt_packages: + - 7zip + - jq + jobs: + install: + - | + mkdir -p "./tmp" + branch="master" + base_url="https://raw.githubusercontent.com/LizardByte/LizardByte.github.io" + url="${base_url}/refs/heads/${branch}/scripts/readthedocs_build.sh" + curl -sSL -o "./tmp/readthedocs_build.sh" "${url}" + chmod +x "./tmp/readthedocs_build.sh" + build: + html: + - "./tmp/readthedocs_build.sh" diff --git a/gh-pages-template/_config.yml b/gh-pages-template/_config.yml new file mode 100644 index 000000000..29db11009 --- /dev/null +++ b/gh-pages-template/_config.yml @@ -0,0 +1,2 @@ +--- +site-js: [] # disable crowdin for this site diff --git a/gh-pages-template/assets/js/dashboard.js b/gh-pages-template/assets/js/dashboard.js new file mode 100644 index 000000000..8e6930912 --- /dev/null +++ b/gh-pages-template/assets/js/dashboard.js @@ -0,0 +1,577 @@ +'use strict'; + +// Config +const _cfg = globalThis.DASHBOARD_CONFIG || {}; +const basePath = _cfg.base_path + ? ('/' + _cfg.base_path).replaceAll(/\/+/g, '/').replace(/\/$/, '') + : '/dashboard'; +const baseUrl = globalThis.location.origin + basePath; + +// Helpers +async function fetchJSON(filename) { + const url = `${baseUrl}/assets/data/${filename}`; + const resp = await fetch(url); + if (!resp.ok) throw new Error(`Failed to fetch ${url}: ${resp.status}`); + return resp.json(); +} + +function isDark() { + const bsTheme = document.documentElement.dataset.bsTheme; + if (bsTheme === 'dark') return true; + if (bsTheme === 'light') return false; + return document.documentElement.classList.contains('dark-mode') + || globalThis.matchMedia('(prefers-color-scheme: dark)').matches; +} + +function plotlyTemplate() { + return isDark() ? 'plotly_dark' : 'plotly_white'; +} + +const LAYOUT = { + paper_bgcolor: 'rgba(0,0,0,0)', + plot_bgcolor: 'rgba(0,0,0,0)', + margin: { t: 30, r: 20, b: 120, l: 60 }, + autosize: true, +}; +const CONFIG = { responsive: true }; + +function themeLayout(overrides = {}, applyFontColor = true) { + const base = { ...LAYOUT, template: plotlyTemplate() }; + if (applyFontColor) { + const textColor = getComputedStyle(document.documentElement).getPropertyValue('--bs-body-color').trim(); + if (textColor) base.font = { color: textColor }; + } + return { ...base, ...overrides }; +} + +function activeRepos(repos) { + return repos.filter(r => !r.archived && !(r.topics || []).includes('package-manager')); +} + +// Summary +function renderSummary(active, prs, metadata) { + const el = document.getElementById('summary-stats'); + if (!el) return; + el.classList.add('justify-content-center'); + const updated = new Date(metadata.updated_at).toUTCString(); + const stat = (val, label) => + `
+

${val}

+ ${label} +
`; + el.innerHTML = + stat(active.length, 'Active Repositories') + + stat(active.filter(r => r.fork).length, 'Forked') + + stat(active.reduce((s, r) => s + r.issues, 0), 'Open Issues') + + stat(active.reduce((s, r) => s + r.prs, 0), 'Open PRs') + + `
+ Last updated: ${updated} +
`; +} + +// Bar charts +function renderBarChart(divId, entries) { + const el = document.getElementById(divId); + if (!el || !entries.length) return; + const sorted = [...entries].sort((a, b) => b.value - a.value); + Plotly.newPlot(divId, [{ + x: sorted.map(d => d.name), + y: sorted.map(d => Math.log1p(d.value)), + text: sorted.map(d => String(d.value)), + type: 'bar', + textposition: 'inside', + marker: { color: '#28a9e6' }, + }], themeLayout({ + yaxis: { showticklabels: false }, + xaxis: { tickangle: -45 }, + }), CONFIG); +} + +// Star history line chart +function renderStarHistory(history) { + const el = document.getElementById('chart-star-history'); + if (!el) return; + if (!history?.length) { + el.innerHTML = '

No star history available.

'; + return; + } + const byRepo = {}; + history.forEach(({ repo, date, stars }) => { + if (!byRepo[repo]) byRepo[repo] = { x: [], y: [] }; + byRepo[repo].x.push(date); + byRepo[repo].y.push(stars); + }); + const traces = Object.entries(byRepo) + .sort((a, b) => { + const lastA = a[1].y.at(-1) ?? 0; + const lastB = b[1].y.at(-1) ?? 0; + return lastB - lastA; + }) + .map(([repo, d]) => ({ name: repo, x: d.x, y: d.y, mode: 'lines+markers', type: 'scatter' })); + Plotly.newPlot('chart-star-history', traces, themeLayout({ + yaxis: { title: 'Stars' }, + margin: { t: 30, r: 20, b: 60, l: 60 }, + }), CONFIG); +} + +// PRs bar chart +function renderPRsBarChart(active, prs) { + const el = document.getElementById('chart-prs'); + if (!el) return; + const counts = {}; + active.forEach(r => { counts[r.name] = { Ready: 0, Draft: 0 }; }); + prs.forEach(pr => { + if (counts[pr.repo]) counts[pr.repo][pr.draft ? 'Draft' : 'Ready']++; + }); + const sorted = active + .map(r => ({ name: r.name, ...counts[r.name], total: counts[r.name].Ready + counts[r.name].Draft })) + .filter(r => r.total > 0) + .sort((a, b) => b.total - a.total); + if (!sorted.length) { + el.innerHTML = '

No open PRs.

'; + return; + } + Plotly.newPlot('chart-prs', ['Ready', 'Draft'].map(status => ({ + name: status, + x: sorted.map(r => r.name), + y: sorted.map(r => r[status]), + text: sorted.map(r => String(r[status])), + type: 'bar', + textposition: 'inside', + })), themeLayout({ + barmode: 'stack', + xaxis: { tickangle: -45 }, + }), CONFIG); +} + +function stripTags(html) { + let out = '', inTag = false; + for (const ch of html) { + if (ch === '<') { inTag = true; } + else if (ch === '>') { inTag = false; } + else if (!inTag) { out += ch; } + } + return out; +} + +// PR table +function renderPRTable(prs) { + const container = document.getElementById('table-prs'); + if (!container) return; + const cols = ['Repo', '#', 'Title', 'Author', 'Labels', 'Assignees', 'Status', 'Created', 'Last Activity', 'Milestone']; + let rows = ''; + prs.forEach(pr => { + const link = `#${pr.number}`; + rows += ` + ${pr.repo} + ${link} + ${pr.title} + ${pr.author} + ${(pr.labels || []).join(', ')} + ${(pr.assignees || []).join(', ')} + ${pr.draft ? 'Draft' : 'Ready'} + ${(pr.created_at || '').substring(0, 10)} + ${(pr.updated_at || '').substring(0, 10)} + ${pr.milestone || ''} + `; + }); + const headerCells = cols.map(c => `${c}`).join(''); + const filterCells = cols.map(() => ``).join(''); + container.innerHTML = ` + + + ${headerCells} + ${filterCells} + + ${rows} +
`; + + const filterInputs = Array.from(container.querySelectorAll('thead tr:nth-child(2) input')); + const tbody = container.querySelector('tbody'); + + if (typeof DataTable !== 'undefined') { + const filterValues = new Array(cols.length).fill(''); + DataTable.ext.search.push((settings, rowData) => { + if (settings.nTable.id !== 'pr-datatable') return true; + return filterValues.every((val, i) => { + if (!val) return true; + return stripTags(String(rowData[i] || '')).toLowerCase().includes(val.toLowerCase()); + }); + }); + const dt = new DataTable('#pr-datatable', { + pageLength: 10, + lengthMenu: [10, 25, 50, 100], + order: [[7, 'asc']], + orderCellsTop: true, + autoFill: true, + layout: { + topStart: [ + 'pageLength', + { + buttons: [ + 'copy', + 'csv', + 'excel', + ], + }, + ], + topEnd: 'search', + bottomStart: 'info', + bottomEnd: 'paging', + }, + }); + filterInputs.forEach((input, i) => { + input.addEventListener('input', () => { + filterValues[i] = input.value; + dt.draw(); + }); + }); + return dt; + } + + // Fallback: sort by Created date (col 7) ascending, then add paging + per-column filtering. + const allRows = Array.from(tbody.querySelectorAll('tr')); + allRows.sort((a, b) => { + const aVal = a.querySelectorAll('td')[7]?.textContent || ''; + const bVal = b.querySelectorAll('td')[7]?.textContent || ''; + return aVal.localeCompare(bVal); + }); + allRows.forEach(row => tbody.appendChild(row)); + + let pageSize = 10; + let currentPage = 0; + + const lengthSel = document.createElement('select'); + lengthSel.className = 'form-select form-select-sm d-inline-block w-auto'; + [10, 25, 50, 100].forEach(n => { + const opt = document.createElement('option'); + opt.value = String(n); + opt.textContent = String(n); + if (n === 10) opt.selected = true; + lengthSel.appendChild(opt); + }); + const lengthLabel = document.createElement('label'); + lengthLabel.className = 'text-muted small me-auto'; + lengthLabel.append('Show ', lengthSel, ' entries'); + + const infoEl = document.createElement('small'); + infoEl.className = 'text-muted'; + + const pagUl = document.createElement('ul'); + pagUl.className = 'pagination pagination-sm mb-0'; + + const topBar = document.createElement('div'); + topBar.className = 'd-flex align-items-center mb-2'; + topBar.appendChild(lengthLabel); + + const bottomBar = document.createElement('div'); + bottomBar.className = 'd-flex justify-content-between align-items-center mt-2 flex-wrap gap-2'; + bottomBar.append(infoEl, pagUl); + + const table = container.querySelector('table'); + table.before(topBar); + container.appendChild(bottomBar); + + function getFiltered() { + const filters = filterInputs.map(inp => inp.value.toLowerCase()); + return allRows.filter(row => { + const cells = Array.from(row.querySelectorAll('td')); + return filters.every((f, i) => !f || (cells[i]?.textContent || '').toLowerCase().includes(f)); + }); + } + + function renderPage() { + const filtered = getFiltered(); + const totalPages = Math.max(1, Math.ceil(filtered.length / pageSize)); + currentPage = Math.min(currentPage, totalPages - 1); + const start = currentPage * pageSize; + + allRows.forEach(r => { r.style.display = 'none'; }); + filtered.slice(start, start + pageSize).forEach(r => { r.style.display = ''; }); + + const from = filtered.length === 0 ? 0 : start + 1; + const to = Math.min(start + pageSize, filtered.length); + infoEl.textContent = `Showing ${from} to ${to} of ${filtered.length} entries`; + + pagUl.innerHTML = ''; + const addBtn = (label, page, disabled, active) => { + const li = document.createElement('li'); + li.className = `page-item${disabled ? ' disabled' : ''}${active ? ' active' : ''}`; + const btn = document.createElement('button'); + btn.className = 'page-link'; + btn.textContent = label; + btn.disabled = disabled; + if (!disabled) btn.addEventListener('click', () => { currentPage = page; renderPage(); }); + li.appendChild(btn); + pagUl.appendChild(li); + }; + addBtn('Previous', currentPage - 1, currentPage === 0, false); + const startP = Math.max(0, Math.min(currentPage - 2, totalPages - 5)); + for (let p = startP; p < Math.min(totalPages, startP + 5); p++) { + addBtn(String(p + 1), p, false, p === currentPage); + } + addBtn('Next', currentPage + 1, currentPage >= totalPages - 1, false); + } + + lengthSel.addEventListener('change', () => { + pageSize = Number.parseInt(lengthSel.value, 10); + currentPage = 0; + renderPage(); + }); + filterInputs.forEach(inp => inp.addEventListener('input', () => { currentPage = 0; renderPage(); })); + renderPage(); +} + +// License treemap +function renderLicenseChart(repos) { + const licRepos = {}; + repos.forEach(r => { + const lic = r.license || 'No License'; + if (!licRepos[lic]) licRepos[lic] = []; + licRepos[lic].push(r.name); + }); + const ids = [], labels = [], parents = [], values = []; + Object.entries(licRepos).forEach(([lic, rlist]) => { + ids.push(`lic::${lic}`); + labels.push(lic); + parents.push(''); + values.push(rlist.length); + rlist.forEach(name => { + ids.push(`repo::${lic}::${name}`); + labels.push(name); + parents.push(`lic::${lic}`); + values.push(1); + }); + }); + Plotly.newPlot('chart-license', [{ + type: 'treemap', ids, labels, parents, values, + branchvalues: 'total', + tiling: { squarifyratio: 1.618, pad: 2 }, + pathbar: { visible: false }, + }], themeLayout({ + margin: { t: 5, r: 5, b: 5, l: 5 }, + }, false), CONFIG); +} + +// Coverage scatter +function renderCoverageChart(repos) { + const sorted = [...repos].filter(r => r.coverage > 0).sort((a, b) => b.coverage - a.coverage); + if (!sorted.length) return; + Plotly.newPlot('chart-coverage', [{ + x: sorted.map(r => r.name), + y: sorted.map(r => r.coverage), + mode: 'markers', + type: 'scatter', + marker: { + size: sorted.map(r => Math.max(8, 110 - r.coverage)), + color: sorted.map(r => r.coverage), + colorscale: [[0, 'red'], [0.5, 'yellow'], [1, 'green']], + colorbar: { title: 'Coverage %' }, + showscale: true, + }, + text: sorted.map(r => `${r.name}: ${r.coverage.toFixed(1)}%`), + }], themeLayout({ + yaxis: { title: 'Coverage %', range: [0, 105] }, + xaxis: { tickangle: -45 }, + }), CONFIG); +} + +// Coverage history line +function renderCoverageHistory(history) { + const el = document.getElementById('chart-coverage-history'); + if (!el) return; + if (!history?.length) { + el.innerHTML = '

No coverage history available.

'; + return; + } + const byRepo = {}; + history.forEach(({ repo, date, coverage }) => { + if (!byRepo[repo]) byRepo[repo] = { x: [], y: [] }; + byRepo[repo].x.push(date); + byRepo[repo].y.push(coverage); + }); + const traces = Object.entries(byRepo).map(([repo, d]) => ({ + name: repo, x: d.x, y: d.y, mode: 'lines', type: 'scatter', + })); + Plotly.newPlot('chart-coverage-history', traces, themeLayout({ + yaxis: { title: 'Coverage %', range: [0, 100] }, + }), CONFIG); +} + +// Language treemaps +function treemap(data, valueFor) { + const ids = [], labels = [], parents = [], values = []; + const sorted = Object.entries(data).sort((a, b) => valueFor(b) - valueFor(a)); + sorted.forEach(([lang, info]) => { + ids.push(`L::${lang}`); + labels.push(lang); + parents.push(''); + values.push(valueFor([lang, info])); + const rlist = Array.isArray(info) ? info : Object.keys(info.repos); + rlist.forEach(name => { + ids.push(`R::${lang}::${name}`); + labels.push(name); + parents.push(`L::${lang}`); + values.push(Array.isArray(info) ? 1 : info.repos[name]); + }); + }); + return { ids, labels, parents, values }; +} + +function renderLanguageCharts(repos) { + const byBytes = {}, byCount = {}; + repos.forEach(r => { + Object.entries(r.languages || {}).forEach(([lang, bytes]) => { + if (!byBytes[lang]) byBytes[lang] = { total: 0, repos: {} }; + byBytes[lang].total += bytes; + byBytes[lang].repos[r.name] = (byBytes[lang].repos[r.name] || 0) + bytes; + if (!byCount[lang]) byCount[lang] = []; + if (!byCount[lang].includes(r.name)) byCount[lang].push(r.name); + }); + }); + + const treemapTrace = (data) => ({ + type: 'treemap', ...data, + branchvalues: 'total', + tiling: { squarifyratio: 1.618, pad: 2 }, + pathbar: { visible: false }, + }); + const treemapLayout = { margin: { t: 5, r: 5, b: 5, l: 5 } }; + + Plotly.newPlot('chart-languages-bytes', [treemapTrace(treemap(byBytes, ([, info]) => info.total))], + themeLayout(treemapLayout, false), CONFIG); + + Plotly.newPlot('chart-languages-repos', [treemapTrace(treemap(byCount, ([, arr]) => arr.length))], + themeLayout(treemapLayout, false), CONFIG); +} + +// Commit activity charts +function renderCommitActivityChart(commitActivity, active) { + const activeNames = new Set(active.map(r => r.name)); + + const byWeek = {}; + commitActivity.forEach(({ repo, week, total }) => { + if (!activeNames.has(repo)) return; + byWeek[week] = (byWeek[week] || 0) + total; + }); + const weeks = Object.keys(byWeek).sort((a, b) => a.localeCompare(b)); + if (weeks.length) { + Plotly.newPlot('chart-commit-activity-weekly', [{ + x: weeks, + y: weeks.map(w => byWeek[w]), + mode: 'lines', + type: 'scatter', + fill: 'tozeroy', + line: { color: '#28a9e6' }, + }], themeLayout({ + yaxis: { title: 'Commits' }, + }), CONFIG); + } + + const byRepo = {}; + commitActivity.forEach(({ repo, week, total }) => { + if (!activeNames.has(repo)) return; + if (!byRepo[repo]) byRepo[repo] = { x: [], y: [] }; + byRepo[repo].x.push(week); + byRepo[repo].y.push(total); + }); + const repoTraces = Object.entries(byRepo) + .filter(([, d]) => d.x.length > 0) + .map(([repo, d]) => { + const pairs = d.x.map((w, i) => [w, d.y[i]]).sort((a, b) => a[0].localeCompare(b[0])); + return { + name: repo, + x: pairs.map(([w]) => w), + y: pairs.map(([, v]) => v), + mode: 'lines', + type: 'scatter', + }; + }); + if (repoTraces.length) { + Plotly.newPlot('chart-commit-activity-repos', repoTraces, themeLayout({ + yaxis: { title: 'Commits' }, + margin: { t: 30, r: 20, b: 60, l: 60 }, + }), CONFIG); + } +} + +// Docs treemap +function renderDocsChart(repos) { + const hasRTD = repos.filter(r => r.has_readthedocs); + const noRTD = repos.filter(r => !r.has_readthedocs); + const ids = ['true', 'false', + ...hasRTD.map(r => `y::${r.name}`), + ...noRTD.map(r => `n::${r.name}`)]; + const labels = ['Has ReadTheDocs', 'No ReadTheDocs', + ...hasRTD.map(r => r.name), + ...noRTD.map(r => r.name)]; + const parents = ['', '', + ...hasRTD.map(() => 'true'), + ...noRTD.map(() => 'false')]; + const values = [hasRTD.length, noRTD.length, + ...hasRTD.map(() => 1), + ...noRTD.map(() => 1)]; + Plotly.newPlot('chart-docs', [{ + type: 'treemap', ids, labels, parents, values, + branchvalues: 'total', + tiling: { squarifyratio: 1.618, pad: 2 }, + pathbar: { visible: false }, + }], themeLayout({ + margin: { t: 5, r: 5, b: 5, l: 5 }, + }, false), CONFIG); +} + +// Main +document.addEventListener('DOMContentLoaded', async () => { + const loadingEl = document.getElementById('loading-msg'); + const contentEl = document.getElementById('dashboard-content'); + try { + const [repos, prs, metadata, coverageHistory, commitActivity, starHistory] = await Promise.all([ + fetchJSON('repos.json'), + fetchJSON('prs.json'), + fetchJSON('metadata.json'), + fetchJSON('coverage_history.json').catch(() => []), + fetchJSON('commit_activity.json').catch(() => []), + fetchJSON('star_history.json').catch(() => []), + ]); + + const active = activeRepos(repos); + const activePRs = prs.filter(pr => active.some(r => r.name === pr.repo)); + + // Patch repos whose current coverage is 0 using their latest history value + if (coverageHistory.length) { + const latestByRepo = {}; + coverageHistory.forEach(({ repo, date, coverage }) => { + if (!latestByRepo[repo] || date > latestByRepo[repo].date) { + latestByRepo[repo] = { date, coverage }; + } + }); + active.forEach(r => { + if (!r.coverage && latestByRepo[r.name]) { + r.coverage = latestByRepo[r.name].coverage; + } + }); + } + + if (loadingEl) loadingEl.style.display = 'none'; + if (contentEl) contentEl.style.display = ''; + + renderSummary(active, activePRs, metadata); + renderBarChart('chart-stars', active.map(r => ({ name: r.name, value: r.stars }))); + renderStarHistory(starHistory); + renderBarChart('chart-forks', active.map(r => ({ name: r.name, value: r.forks }))); + renderBarChart('chart-issues', active.map(r => ({ name: r.name, value: r.issues }))); + renderPRsBarChart(active, activePRs); + renderPRTable(activePRs); + renderLicenseChart(active); + renderCoverageChart(active); + renderCoverageHistory(coverageHistory); + renderLanguageCharts(active); + renderCommitActivityChart(commitActivity, active); + renderDocsChart(active); + + } catch (err) { + if (loadingEl) loadingEl.innerHTML = + `
Failed to load dashboard data: ${err.message}
`; + } +}); diff --git a/gh-pages-template/index.html b/gh-pages-template/index.html new file mode 100644 index 000000000..0c47fb639 --- /dev/null +++ b/gh-pages-template/index.html @@ -0,0 +1,133 @@ +--- +title: Dashboard +layout: page +full-width: true +after-content: + - donate.html + - support.html + +# see: https://datatables.net/download/release +ext-css: + - href: https://cdn.datatables.net/2.3.7/css/dataTables.bootstrap5.min.css + - https://cdn.datatables.net/autofill/2.7.1/css/autoFill.bootstrap5.min.css + - href: https://cdn.datatables.net/buttons/3.2.6/css/buttons.bootstrap5.min.css + +ext-js: + - https://cdn.plot.ly/plotly-2.35.3.min.js + - https://code.jquery.com/jquery-3.7.1.min.js + - https://cdn.datatables.net/2.3.7/js/dataTables.min.js + - https://cdn.datatables.net/2.3.7/js/dataTables.bootstrap5.min.js + - https://cdn.datatables.net/autofill/2.7.1/js/dataTables.autoFill.min.js + - https://cdn.datatables.net/autofill/2.7.1/js/autoFill.bootstrap5.min.js + - https://cdnjs.cloudflare.com/ajax/libs/jszip/3.10.1/jszip.min.js # required for excel export + - https://cdn.datatables.net/buttons/3.2.6/js/dataTables.buttons.min.js + - https://cdn.datatables.net/buttons/3.2.6/js/buttons.bootstrap5.min.js + - https://cdn.datatables.net/buttons/3.2.6/js/buttons.html5.min.js + +js: + - /dashboard/assets/js/dashboard.js +--- + + + +
+ +

Loading dashboard data…

+
+ + diff --git a/jupyter_nbconvert_config.py b/jupyter_nbconvert_config.py deleted file mode 100644 index 4cac9eaf1..000000000 --- a/jupyter_nbconvert_config.py +++ /dev/null @@ -1,8 +0,0 @@ -# jupyter imports -from traitlets.config import get_config - -# define the config object -c = get_config() - -c.Templateexporter.exclude_input_prompt = True -c.HTMLExporter.theme = 'dark' diff --git a/notebook/dashboard.ipynb b/notebook/dashboard.ipynb deleted file mode 100644 index 225251dcb..000000000 --- a/notebook/dashboard.ipynb +++ /dev/null @@ -1,328 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "4aed1755a10d75d7", - "metadata": {}, - "source": [ - "
\n", - "

LizardByte Developer Dashboard

\n", - "
\n", - "\n", - "
\n", - "[\n", - " Developer Tools •\n", - " Repository Data •\n", - " Star Gazers •\n", - " Forks •\n", - " Open Issues •\n", - " Open PRs •\n", - " License Distribution •\n", - " Coverage •\n", - " Programming Languages •\n", - " Documentation\n", - "]\n", - "
\n", - "\n", - "
\n", - "

Developer Tools

\n", - " \n", - " \"CodeCov\"\n", - " \n", - " \n", - " \"Crowdin\n", - " \n", - " \n", - " \"Crowdin\n", - " \n", - " \n", - " \"Fedora\n", - " \n", - " \n", - " \"GitHub\n", - " \n", - " \n", - " \"GitHub\n", - " \n", - " \n", - " \"NPM\"\n", - " \n", - " \n", - " \"PyPI\"\n", - " \n", - " \n", - " \"ReadTheDocs\"\n", - " \n", - " \n", - " \"SonarCloud\"\n", - " \n", - "
\n", - "\n", - "
\n", - "

Under Construction

\n", - " \n", - " \"Cloudsmith\"\n", - " \n", - " \n", - " \"renovate\"\n", - " \n", - " \n", - " \"YouTrack\"\n", - " \n", - "
\n", - "\n", - "## Repository Data" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "initial_id", - "metadata": {}, - "outputs": [], - "source": [ - "# Setup the environment\n", - "import os\n", - "import sys\n", - "import time\n", - "\n", - "# disable TQDM output if not in debug mode\n", - "if not os.getenv('ACTIONS_RUNNER_DEBUG') and not os.getenv('ACTIONS_STEP_DEBUG'):\n", - " os.environ['TQDM_DISABLE'] = '1'\n", - "\n", - "# Add the src directory to the path\n", - "sys.path.insert(0, os.path.abspath(os.path.join(os.getcwd(), os.pardir)))\n", - "\n", - "from src import dashboard # noqa: E402" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "55ae039e9f91e18", - "metadata": {}, - "outputs": [], - "source": [ - "# get repo data\n", - "dashboard.init()\n", - "df_all_repos = dashboard.get_repo_data()\n", - "df_repos = dashboard.get_df_repos()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "466d0d2fec2e1828", - "metadata": {}, - "outputs": [], - "source": [ - "# Initial Results\n", - "print(f'Last Updated: {time.strftime(\"%Y-%m-%d %H:%M:%S\", time.gmtime())} UTC')\n", - "print(f'Total Repositories: {len(df_all_repos)}')\n", - "print(f'Archived Repositories: {df_all_repos[\"archived\"].sum()}')\n", - "print(f'Forked Repositories: {df_all_repos[\"fork\"].sum()}')\n", - "print(f'Total Open Issues: {df_all_repos[\"issues\"].apply(len).sum()}')\n", - "print(f'Total Open PRs: {df_all_repos[\"prs\"].apply(len).sum()}')\n", - "print(f'Open issues in active repositories: {df_repos[\"issues\"].apply(len).sum()}')\n", - "print(f'Open PRs in active repositories: {df_repos[\"prs\"].apply(len).sum()}')" - ] - }, - { - "cell_type": "markdown", - "id": "616d87e4b0c3dea5", - "metadata": {}, - "source": [ - "### Star Gazers" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "337aca37b216cc0d", - "metadata": {}, - "outputs": [], - "source": [ - "# Stars\n", - "dashboard.show_star_gazers()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a08398efe666a399", - "metadata": {}, - "outputs": [], - "source": [ - "# Star History Visuals\n", - "dashboard.show_star_history()" - ] - }, - { - "cell_type": "markdown", - "id": "d46c30581ed069fd", - "metadata": {}, - "source": [ - "### Forks" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7c6ffc3f9b56adc6", - "metadata": {}, - "outputs": [], - "source": [ - "# Forks\n", - "dashboard.show_forks()" - ] - }, - { - "cell_type": "markdown", - "id": "89ad81f3285c93d5", - "metadata": {}, - "source": [ - "### Open Issues" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ce4c9d640154b774", - "metadata": {}, - "outputs": [], - "source": [ - "# Open Issues\n", - "dashboard.show_issues()" - ] - }, - { - "cell_type": "markdown", - "id": "63b0eb1841d850be", - "metadata": {}, - "source": [ - "### Open PRs" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4f473fe7a839b01c", - "metadata": {}, - "outputs": [], - "source": [ - "# PR Graph\n", - "dashboard.show_pr_graph()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b416c3c48ae4e103", - "metadata": {}, - "outputs": [], - "source": [ - "# PR Table\n", - "dashboard.show_pr_table()" - ] - }, - { - "cell_type": "markdown", - "id": "3c067ce1b06d52da", - "metadata": {}, - "source": [ - "### License Distribution" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2bd8913a09eb4802", - "metadata": {}, - "outputs": [], - "source": [ - "# License distribution\n", - "dashboard.show_license_distribution()" - ] - }, - { - "cell_type": "markdown", - "id": "1e7ac0186201244b", - "metadata": {}, - "source": [ - "### Coverage" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c8dced4bcfff438", - "metadata": {}, - "outputs": [], - "source": [ - "# Coverage\n", - "dashboard.show_coverage()\n", - "\n", - "# Coverage History\n", - "dashboard.show_coverage_history()" - ] - }, - { - "cell_type": "markdown", - "id": "d9116f5c8e154dae", - "metadata": {}, - "source": [ - "### Programming Languages" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "20e09d93eda0478a", - "metadata": {}, - "outputs": [], - "source": [ - "# Programming Languages\n", - "dashboard.show_language_data()" - ] - }, - { - "cell_type": "markdown", - "id": "a520a320cd823874", - "metadata": {}, - "source": [ - "### Documentation" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ec43a8fd7f9d49bb", - "metadata": {}, - "outputs": [], - "source": [ - "# Docs\n", - "dashboard.show_docs_data()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/package.json b/package.json deleted file mode 100644 index f06816907..000000000 --- a/package.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "dependencies": { - "plotly.js": "2.35.3" - } -} diff --git a/pyproject.toml b/pyproject.toml index 485859c0e..13f39a607 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,12 +14,7 @@ authors = [ dependencies = [ "cloudscraper==1.2.71", - "ipython==9.11.0", - "itables==2.7.1", - "notebook==7.5.5", - "pandas==3.0.1", "pillow==12.1.1", - "plotly==5.24.1", "pygithub==2.8.1", "python-dotenv==1.2.2", "requests==2.32.5", @@ -27,13 +22,7 @@ dependencies = [ "unhandled_exit==1.0.0", ] -[project.optional-dependencies] -dev = [ - "nb-clean==4.0.1", - "nbqa[toolchain]==1.9.1", -] - [project.urls] -Homepage = "https://app.lizardbyte.dev/Sunshine" -Repository = "https://github.com/LizardByte/Sunshine" -Issues = "https://github.com/LizardByte/Sunshine/issues" +Homepage = "https://app.lizardbyte.dev/dashboard" +Repository = "https://github.com/LizardByte/dashboard" +Issues = "https://github.com/LizardByte/dashboard/issues" diff --git a/src/__init__.py b/src/__init__.py index 8ef445a3b..afed78b4c 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -8,3 +8,4 @@ load_dotenv() BASE_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'gh-pages') +TEMPLATE_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'gh-pages-template') diff --git a/src/builder.py b/src/builder.py new file mode 100644 index 000000000..0fae531f5 --- /dev/null +++ b/src/builder.py @@ -0,0 +1,244 @@ +"""Build consolidated dashboard data JSON files from raw collected data.""" + +# standard imports +import json +import os +from datetime import datetime, timezone + +# local imports +from src import BASE_DIR, TEMPLATE_DIR +from src.logger import log + + +def _safe_name(name: str) -> str: + return os.path.basename(name) + + +def _load_rtd_repos(base_dir: str) -> set: + rtd_repos = set() + rtd_path = os.path.join(base_dir, 'readthedocs', 'projects.json') + if not os.path.exists(rtd_path): + return rtd_repos + try: + with open(rtd_path) as f: + rtd_data = json.load(f) + for project in rtd_data: + git_url = project.get('repository', {}).get('url', '') + repo_name = git_url.rsplit('/', 1)[-1].rsplit('.git', 1)[0] + rtd_repos.add(repo_name) + except Exception: + pass + return rtd_repos + + +def _get_coverage(base_dir: str, name: str) -> float: + safe = _safe_name(name) + codecov_path = os.path.join(base_dir, 'codecov', f'{safe}.json') + if not os.path.exists(codecov_path): + return 0.0 + try: + with open(codecov_path) as f: + data = json.load(f) + return float((data.get('totals') or {}).get('coverage', 0) or 0) + except Exception: + return 0.0 + + +def _collect_coverage_history(base_dir: str, name: str) -> list: + safe = _safe_name(name) + trend_path = os.path.join(base_dir, 'codecov', f'{safe}_coverage_trend.json') + if not os.path.exists(trend_path): + return [] + try: + with open(trend_path) as f: + trend_data = json.load(f) + return [ + {'repo': name, 'date': entry.get('timestamp'), 'coverage': float(entry['avg'])} + for entry in trend_data + if entry.get('avg') is not None + ] + except Exception: + return [] + + +def _get_languages(base_dir: str, name: str) -> dict: + safe = _safe_name(name) + lang_path = os.path.join(base_dir, 'github', 'languages', f'{safe}.json') + if not os.path.exists(lang_path): + return {} + try: + with open(lang_path) as f: + return json.load(f) + except Exception: + return {} + + +def _get_prs(base_dir: str, name: str) -> list: + safe = _safe_name(name) + pulls_path = os.path.join(base_dir, 'github', 'pulls', f'{safe}.json') + if not os.path.exists(pulls_path): + return [] + try: + with open(pulls_path) as f: + return json.load(f) + except Exception: + return [] + + +def _get_commit_activity(base_dir: str, name: str) -> list: + """ + Read cached commit activity for a repo and return flat weekly records. + + Parameters + ---------- + base_dir : str + Root directory containing the gh-pages data. + name : str + Repository name. + + Returns + ------- + list + List of dicts with keys ``repo``, ``week`` (ISO date), and ``total``. + """ + safe = _safe_name(name) + path = os.path.join(base_dir, 'github', 'commitActivity', f'{safe}.json') + if not os.path.exists(path): + return [] + try: + with open(path) as f: + data = json.load(f) + result = [] + for entry in data: + week_ts = entry.get('week') + total = entry.get('total', 0) + if week_ts and total > 0: + week_str = datetime.fromtimestamp(week_ts, tz=timezone.utc).strftime('%Y-%m-%d') + result.append({'repo': name, 'week': week_str, 'total': total}) + return result + except Exception: + return [] + + +def _get_star_history(base_dir: str, name: str) -> list: + """ + Read the cached sampled star history for a repo. + + Parameters + ---------- + base_dir : str + Root directory containing the gh-pages data. + name : str + Repository name. + + Returns + ------- + list + List of dicts with keys ``repo``, ``date``, and ``stars``. + """ + safe = _safe_name(name) + path = os.path.join(base_dir, 'github', 'starHistory', f'{safe}.json') + if not os.path.exists(path): + return [] + try: + with open(path) as f: + data = json.load(f) + return [{'repo': name, 'date': e['date'], 'stars': e['stars']} for e in data] + except Exception: + return [] + + +def _build_repo_entry(repo: dict, coverage: float, languages: dict, prs: list, rtd_repos: set) -> dict: + name = repo['name'] + pr_count = len(prs) + open_issues_total = repo.get('open_issues_count', 0) + issue_count = max(0, open_issues_total - pr_count) + + license_info = repo.get('license') + license_name = 'No License' + if license_info: + license_name = license_info.get('name') or license_info.get('spdx_id') or 'No License' + + return { + 'name': name, + 'stars': repo.get('stargazers_count', 0), + 'forks': repo.get('forks_count', 0), + 'issues': issue_count, + 'prs': pr_count, + 'license': license_name, + 'coverage': coverage, + 'language': repo.get('language'), + 'languages': languages, + 'archived': repo.get('archived', False), + 'fork': repo.get('fork', False), + 'topics': repo.get('topics', []), + 'has_readthedocs': name in rtd_repos, + 'created_at': repo.get('created_at'), + 'updated_at': repo.get('updated_at'), + } + + +def build(): + """ + Read raw data collected by updater.py and write dashboard-ready JSON files + into gh-pages-template/assets/data/ for consumption by the Jekyll site's JavaScript. + """ + log.info('Building dashboard data...') + + repos_path = os.path.join(BASE_DIR, 'github', 'repos.json') + if not os.path.exists(repos_path): + log.error(f'Repos file not found: {repos_path}') + return + + with open(repos_path) as f: + raw_repos = json.load(f) + + rtd_repos = _load_rtd_repos(BASE_DIR) + repos = [] + prs_all = [] + coverage_history = [] + commit_activity = [] + star_history = [] + + for repo in raw_repos: + if repo.get('private') or repo.get('archived'): + continue + + name = repo['name'] + coverage = _get_coverage(BASE_DIR, name) + coverage_hist = _collect_coverage_history(BASE_DIR, name) + coverage_history.extend(coverage_hist) + if not coverage and coverage_hist: + coverage = max(coverage_hist, key=lambda e: e.get('date', '')).get('coverage', 0.0) + languages = _get_languages(BASE_DIR, name) + prs = _get_prs(BASE_DIR, name) + commit_activity.extend(_get_commit_activity(BASE_DIR, name)) + star_history.extend(_get_star_history(BASE_DIR, name)) + + repos.append(_build_repo_entry(repo, coverage, languages, prs, rtd_repos)) + prs_all.extend({'repo': name, **pr} for pr in prs) + + data_dir = os.path.join(TEMPLATE_DIR, 'assets', 'data') + os.makedirs(data_dir, exist_ok=True) + + def write_json(filename, data): + path = os.path.join(data_dir, filename) + with open(path, 'w') as f: + json.dump(data, f) + log.info(f'Written: {path}') + + write_json('repos.json', repos) + write_json('prs.json', prs_all) + write_json('coverage_history.json', coverage_history) + write_json('commit_activity.json', commit_activity) + write_json('star_history.json', star_history) + write_json('metadata.json', { + 'updated_at': datetime.now(timezone.utc).isoformat(), + 'repo_count': len(repos), + }) + + log.info('Dashboard build complete.') + + +if __name__ == '__main__': + build() diff --git a/src/dashboard.py b/src/dashboard.py deleted file mode 100644 index 82f38df24..000000000 --- a/src/dashboard.py +++ /dev/null @@ -1,617 +0,0 @@ -# standard imports -import json -import os - -# lib imports -from github import Github, PaginatedList -from IPython.display import HTML, display -from itables import init_notebook_mode, show -import numpy as np -import pandas as pd -import plotly.express as px -import plotly.graph_objects as go -import plotly.io as pio - -# local imports -from src import BASE_DIR -from src import updater - -# Authenticate with GitHub -token = os.getenv("GITHUB_TOKEN") -g = Github(token) - -# set the default plotly template -pio.templates.default = "plotly_dark" - -# Fetch repository data -org_name = "LizardByte" -org = g.get_organization(org_name) - -# constants -text_template = '%{text}' - - -# globals -updated = False -repos = [] -df_all_repos = pd.DataFrame() -df_repos = pd.DataFrame() -df_pr_details = pd.DataFrame() -df_language_data = pd.DataFrame() -df_docs_data = pd.DataFrame() - - -def init(): - global updated - if not updated: - update_html_head_title() - updater.update() - updated = True - - -def update_html_head_title(): - script = """ - - """ - display(HTML(script)) - - -def get_repos() -> PaginatedList: - global repos - repos = org.get_repos() - return repos - - -def get_repo_data() -> pd.DataFrame: - global df_all_repos - - if not repos: - get_repos() - - # all readthedocs projects - readthedocs_path = os.path.join(BASE_DIR, 'readthedocs', 'projects.json') - with open(readthedocs_path, 'r') as f: - readthedocs_data = json.load(f) - - repo_data = [] - for repo in repos: - # skip private repos - if repo.private: - continue - - # get license - license_name = repo.license.name if repo.license else "No License" - - # split open issues and PRs - open_issues = repo.get_issues(state='open') - open_prs = [issue for issue in open_issues if issue.pull_request is not None] - open_issues = [issue for issue in open_issues if issue.pull_request is None] - - # coverage data - coverage = 0 - try: - with open(os.path.join(BASE_DIR, 'codecov', f'{repo.name}.json')) as f: - coverage_data = json.load(f) - coverage = coverage_data['totals']['coverage'] - except Exception: - pass - - # readthedocs data - readthedocs_project = None - for project in readthedocs_data: - if project['repository']['url'] == repo.clone_url: - readthedocs_project = project - - repo_data.append({ - "repo": repo.name, - "stars": repo.stargazers_count, - "archived": repo.archived, - "fork": repo.fork, - "forks": repo.forks_count, - "issues": open_issues, - "topics": repo.get_topics(), - "languages": repo.get_languages(), - "license": license_name, - "prs": open_prs, - "created_at": repo.created_at, - "updated_at": repo.updated_at, - "coverage": coverage, - "readthedocs": readthedocs_project, - "has_readthedocs": readthedocs_project is not None, - "_repo": repo, - }) - - df_all_repos = pd.DataFrame(repo_data) - return df_all_repos - - -def get_df_repos() -> pd.DataFrame: - global df_repos - - if df_all_repos.empty: - get_repo_data() - - df_repos = df_all_repos[ - (~df_all_repos['archived']) & - (~df_all_repos['topics'].apply(lambda topics: 'package-manager' in topics)) - ] - return df_repos - - -def show_star_gazers(): - if df_repos.empty: - get_df_repos() - - df_stars = df_repos.sort_values( - by='stars', - ascending=False, - ) - df_stars['log_stars'] = np.log1p(df_stars['stars']) - fig = px.bar( - df_stars, - x='repo', - y='log_stars', - title='Stars', - text='stars', - ) - fig.update_traces( - texttemplate=text_template, - textposition='inside', - ) - fig.update_layout( - yaxis_title=None, - yaxis_showticklabels=False, - ) - fig.show() - - -def get_stargazer_data() -> list: - if df_repos.empty: - get_df_repos() - - stargazer_data = [] - for repo in df_repos.to_dict('records'): - stargazers = repo['_repo'].get_stargazers_with_dates() - for stargazer in stargazers: - stargazer_data.append({ - "repo": repo['repo'], - "date": stargazer.starred_at, - }) - - return stargazer_data - - -def show_star_history(): - df_stargazers = pd.DataFrame(get_stargazer_data()) - df_stargazers = df_stargazers.sort_values(by="date") - df_stargazers["cumulative_stars"] = df_stargazers.groupby("repo").cumcount() + 1 - - fig = px.line( - df_stargazers, - x="date", - y="cumulative_stars", - color="repo", - title="Star History", - labels={"date": "Date", "cumulative_stars": "Cumulative Stars"}, - ) - fig.show() - - -def show_forks(): - if df_repos.empty: - get_df_repos() - - df_forks = df_repos.sort_values( - by='forks', - ascending=False, - ) - df_forks['log_forks'] = np.log1p(df_forks['forks']) - fig = px.bar( - df_forks, - x='repo', - y='log_forks', - title='Forks', - text='forks', - ) - fig.update_traces( - texttemplate=text_template, - textposition='inside', - ) - fig.update_layout( - yaxis_title=None, - yaxis_showticklabels=False, - ) - fig.show() - - -def show_issues(): - if df_repos.empty: - get_df_repos() - - df_issues = df_repos.copy() - df_issues['issue_count'] = df_issues['issues'].apply(len) - df_issues = df_issues.sort_values(by='issue_count', ascending=False) - df_issues['log_issues'] = np.log1p(df_issues['issue_count']) - fig = px.bar( - df_issues, - x='repo', - y='log_issues', - title='Open Issues', - text='issue_count', - ) - fig.update_traces( - texttemplate=text_template, - textposition='inside', - ) - fig.update_layout( - yaxis_title=None, - yaxis_showticklabels=False, - ) - fig.show() - - -def get_pr_data() -> pd.DataFrame: - global df_pr_details - - if df_repos.empty: - get_df_repos() - - pr_data = [] - for repo in df_repos.to_dict('records'): - for pr in repo['prs']: - pr_details = repo['_repo'].get_pull(pr.number) - - # Check if the PR has been approved - reviews = pr_details.get_reviews() - approved = any(review.state == 'APPROVED' for review in reviews) - - # Get the milestone - milestone = pr_details.milestone.title if pr_details.milestone else None - - pr_data.append({ - "repo": repo['repo'], - "number": pr_details.number, - "title": pr_details.title, - "author": pr_details.user.login, - "labels": [label.name for label in pr_details.labels], - "assignees": [assignee.login for assignee in pr_details.assignees], - "created_at": pr_details.created_at, - "last_activity": pr_details.updated_at, - "status": "Draft" if pr_details.draft else "Ready", - "approved": approved, - "milestone": milestone, - }) - - df_pr_details = pd.DataFrame(pr_data) - return df_pr_details - - -def show_pr_graph(): - if df_pr_details.empty: - get_pr_data() - - # Group by repository and status to get the count of PRs - df_pr_counts = df_pr_details.groupby(['repo', 'status']).size().reset_index(name='pr_count') - - # Sort repositories by total PR count - df_pr_counts['total_prs'] = df_pr_counts.groupby('repo')['pr_count'].transform('sum') - df_pr_counts = df_pr_counts.sort_values(by='total_prs', ascending=False) - - # Create Stacked Bar Chart - fig_bar = px.bar( - df_pr_counts, - x='repo', - y='pr_count', - color='status', - title='Open Pull Requests', - labels={'pr_count': 'Count of PRs', 'repo': 'Repository', 'status': 'PR Status'}, - text='pr_count', - category_orders={'repo': df_pr_counts['repo'].tolist()}, - ) - - fig_bar.update_layout( - yaxis_title='Open PRs', - xaxis_title='Repository', - ) - fig_bar.update_traces( - texttemplate=text_template, - textposition='inside', - ) - fig_bar.show() - - -def show_pr_table(): - if df_pr_details.empty: - get_pr_data() - - # darken the column filter inputs - css = """ - .dt-column-title input[type="text"] { - background-color: var(--jp-layout-color0); - border-color: rgb(64,67,70); - border-width: 1px; - color: var(--jp-ui-font-color1); - } - """ - display(HTML(f"")) - - init_notebook_mode( - all_interactive=True, - connected=False, - ) - - # Display the DataFrame as an interactive table using itables - table_download_name = "LizardByte-Pull-Requests" - show( - df_pr_details, - buttons=[ - "pageLength", - "copyHtml5", - {"extend": "csvHtml5", "title": table_download_name}, - {"extend": "excelHtml5", "title": table_download_name}, - ], - classes="display compact", - column_filters="header", - layout={"topEnd": None}, - ) - - -def show_license_distribution(): - if df_repos.empty: - get_df_repos() - - license_counts = df_repos.groupby(['license', 'repo']).size().reset_index(name='count') - - fig_treemap = px.treemap( - license_counts, - path=['license', 'repo'], - values='count', - title='License Distribution', - hover_data={'repo': True, 'count': False}, - ) - fig_treemap.show() - - -def show_coverage(): - if df_repos.empty: - get_df_repos() - - df_coverage = df_repos.sort_values( - by='coverage', - ascending=False, - ) - - # inverse marker size, so higher coverage has smaller markers - df_coverage['marker_size'] = df_coverage['coverage'].apply(lambda x: 110 - x if x > 0 else 0) - - fig_scatter = px.scatter( - df_coverage, - x='repo', - y='coverage', - title='Coverage (Current)', - size='marker_size', - color='coverage', - color_continuous_scale=['red', 'yellow', 'green'], # red is low, green is high - ) - fig_scatter.update_layout( - yaxis_title='Coverage Percentage', - xaxis_title='Repository', - ) - fig_scatter.show() - - -def get_coverage_trend_data() -> list: - """ - Get coverage trend data for all repositories from codecov. - """ - if df_repos.empty: - get_df_repos() - - coverage_trend_data = [] - for repo in df_repos.to_dict('records'): - trend_file = os.path.join(BASE_DIR, 'codecov', f'{repo["repo"]}_coverage_trend.json') - - if os.path.exists(trend_file): - try: - with open(trend_file, 'r') as f: - trend_data = json.load(f) - - for entry in trend_data: - # Parse the coverage data - use avg coverage value - if 'avg' in entry and entry['avg'] is not None: - coverage_trend_data.append({ - "repo": repo['repo'], - "date": entry.get('timestamp'), - "coverage": entry['avg'], - }) - except Exception: - # Skip repos with no coverage data or errors - pass - - return coverage_trend_data - - -def show_coverage_history(): - """ - Display coverage history over time for all repositories, similar to star history. - """ - df_coverage_trend = pd.DataFrame(get_coverage_trend_data()) - - if df_coverage_trend.empty: - print("No coverage trend data available.") - return - - # Convert date string to datetime - df_coverage_trend['date'] = pd.to_datetime(df_coverage_trend['date']) - df_coverage_trend = df_coverage_trend.sort_values(by="date") - - fig = px.line( - df_coverage_trend, - x="date", - y="coverage", - color="repo", - title="Coverage (History)", - labels={"date": "Date", "coverage": "Coverage %"}, - ) - fig.update_layout( - yaxis_title='Coverage Percentage', - xaxis_title='Date', - yaxis_range=[0, 100], - ) - fig.show() - - -def get_language_data(): - global df_language_data - - language_data = [] - for repo in df_repos.to_dict('records'): - for language, bytes_of_code in repo['languages'].items(): - language_data.append({ - "repo": repo['repo'], - "language": language, - "bytes_of_code": bytes_of_code, - }) - - df_language_data = pd.DataFrame(language_data) - return df_language_data - - -def show_language_data(): - if df_language_data.empty: - get_language_data() - - # Aggregate data by language and repo - language_counts_bytes = df_language_data.groupby(['language', 'repo']).agg({ - 'bytes_of_code': 'sum' - }).reset_index() - language_counts_repos = df_language_data.groupby(['language', 'repo']).size().reset_index(name='repo_count') - - def create_language_figures(counts: pd.DataFrame, path_key: str, value_key: str): - _fig_treemap = px.treemap( - counts, - path=[path_key, 'repo'], - values=value_key, - ) - _fig_sunburst = px.sunburst( - counts, - path=[path_key, 'repo'], - values=value_key, - ) - return _fig_treemap, _fig_sunburst - - # List of tuples containing the data and titles for each figure - figures_data = [ - (language_counts_bytes, 'language', 'bytes_of_code', 'Programming Languages by Bytes of Code'), - (language_counts_repos, 'language', 'repo_count', 'Programming Languages by Repo Count') - ] - - # Loop through the list to create figures and add traces - for _counts, _path_key, value_key, title in figures_data: - fig_treemap, fig_sunburst = create_language_figures(counts=_counts, path_key=_path_key, value_key=value_key) - - fig = go.Figure() - fig.add_trace(fig_treemap.data[0]) - fig.add_trace(fig_sunburst.data[0]) - fig.data[1].visible = False - - fig.update_layout( - title=title, - updatemenus=[ - { - "buttons": [ - { - "label": "Treemap", - "method": "update", - "args": [ - {"visible": [True, False]}, - ], - }, - { - "label": "Sunburst", - "method": "update", - "args": [ - {"visible": [False, True]}, - ], - }, - ], - "direction": "down", - "showactive": True, - } - ] - ) - fig.show() - - -def get_docs_data(): - global df_docs_data - - docs_data = [] - for repo in df_repos.to_dict('records'): - docs_data.append({ - "repo": repo['repo'], - "has_readthedocs": repo['has_readthedocs'], - }) - - df_docs_data = pd.DataFrame(docs_data) - return df_docs_data - - -def show_docs_data(): - if df_docs_data.empty: - get_docs_data() - readthedocs_counts = df_docs_data.groupby(['has_readthedocs', 'repo']).size().reset_index(name='repo_count') - - def create_figures(counts: pd.DataFrame, path_key: str): - _fig_treemap = px.treemap( - counts, - path=[path_key, 'repo'], - values='repo_count', - ) - _fig_sunburst = px.sunburst( - counts, - path=[path_key, 'repo'], - values='repo_count', - ) - return _fig_treemap, _fig_sunburst - - # List of tuples containing the data and titles for each figure - figures_data = [ - (readthedocs_counts, 'has_readthedocs', 'Uses ReadTheDocs') - ] - - # Loop through the list to create figures and add traces - for _counts, _path_key, title in figures_data: - fig_treemap, fig_sunburst = create_figures(counts=_counts, path_key=_path_key) - - fig = go.Figure() - fig.add_trace(fig_treemap.data[0]) - fig.add_trace(fig_sunburst.data[0]) - fig.data[1].visible = False - - fig.update_layout( - title=title, - updatemenus=[ - { - "buttons": [ - { - "label": "Treemap", - "method": "update", - "args": [{"visible": [True, False]}], - }, - { - "label": "Sunburst", - "method": "update", - "args": [{"visible": [False, True]}], - }, - ], - "direction": "down", - "showactive": True, - } - ] - ) - fig.show() diff --git a/src/helpers.py b/src/helpers.py index dc03010ee..87a2a563c 100644 --- a/src/helpers.py +++ b/src/helpers.py @@ -17,22 +17,32 @@ # constants HTTPS = 'https://' +DEFAULT_TIMEOUT = 30 # seconds # setup requests sessions retry_adapter = HTTPAdapter(max_retries=Retry(total=5, backoff_factor=1)) + +class TimeoutSession(requests.Session): + """A requests.Session that applies a default timeout to every request.""" + + def request(self, *args, **kwargs): + kwargs.setdefault('timeout', DEFAULT_TIMEOUT) + return super().request(*args, **kwargs) + + # cloudscraper session cs = cloudscraper.CloudScraper() # CloudScraper inherits from requests.Session cs.mount(HTTPS, retry_adapter) # requests session -s = requests.Session() +s = TimeoutSession() s.mount(HTTPS, retry_adapter) -class RateLimitedSession(requests.Session): +class RateLimitedSession(TimeoutSession): """ - A requests.Session subclass that implements rate limiting. + A TimeoutSession subclass that implements rate limiting. """ def __init__(self, calls_per_minute=60): super().__init__() diff --git a/src/updater.py b/src/updater.py index f3d656318..ce8186b0f 100644 --- a/src/updater.py +++ b/src/updater.py @@ -1,9 +1,13 @@ # standard imports +import json +import math import os +from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeout +from datetime import datetime, timezone from threading import Thread # lib imports -from github import Github +from github import Auth, Github import requests from tqdm import tqdm import unhandled_exit @@ -80,6 +84,15 @@ def update_codecov(): """ Get code coverage data from Codecov API. """ + archived_repos = set() + repos_path = os.path.join(BASE_DIR, 'github', 'repos.json') + if os.path.exists(repos_path): + try: + with open(repos_path) as f: + archived_repos = {r['name'] for r in json.load(f) if r.get('archived')} + except Exception as e: + log.warning(f'Could not load GitHub repos for archived check: {e}') + headers = { 'Accept': 'application/json', 'Authorization': f'bearer {os.environ["CODECOV_TOKEN"]}', @@ -105,6 +118,9 @@ def update_codecov(): iterable=data['results'], desc='Updating Codecov data', ): + if repo['name'] in archived_repos: + continue + # Get repo details url = f'{base_url}/repos/{repo["name"]}' response = helpers.s.get(url=url, headers=headers) @@ -176,12 +192,215 @@ def update_fb(): helpers.write_json_files(file_path=file_path, data=data) +def _get_stats_with_timeout(repo, timeout=60): + """ + Fetch commit activity for a repo, capping total wait time. + + Parameters + ---------- + repo : + PyGithub Repository object. + timeout : int + Maximum seconds to wait before giving up (GitHub may return 202 while + computing stats, causing PyGithub to retry indefinitely without this guard). + + Returns + ------- + list or None + Weekly commit-activity objects, or None on timeout. + """ + with ThreadPoolExecutor(max_workers=1) as pool: + future = pool.submit(repo.get_stats_commit_activity) + try: + return future.result(timeout=timeout) + except FuturesTimeout: + log.warning(f'Timeout fetching commit activity for {repo.name}, skipping.') + return None + + +def _seed_star_history(repo, total: int, initial_samples: int) -> list[dict]: + """ + Fetch evenly-spaced pages from the stargazers API for a first-time seed. + + Parameters + ---------- + repo : + PyGithub Repository object. + total : int + Current star count (used to calculate page spread). + initial_samples : int + Maximum number of pages to request. + + Returns + ------- + list + Unsorted list of ``{date, stars}`` dicts sampled across the history. + """ + per_page = 100 + total_pages = math.ceil(total / per_page) + + if total_pages <= initial_samples: + pages_to_fetch = list(range(total_pages)) + else: + pages_to_fetch = sorted({ + round(i * (total_pages - 1) / (initial_samples - 1)) + for i in range(initial_samples) + }) + + history = [] + stargazers = repo.get_stargazers_with_dates() + for page_idx in pages_to_fetch: + try: + page = stargazers.get_page(page_idx) + if not page: + continue + history.append({ + 'date': page[0].starred_at.strftime('%Y-%m-%d'), + 'stars': page_idx * per_page + 1, + }) + if len(page) > 1: + history.append({ + 'date': page[-1].starred_at.strftime('%Y-%m-%d'), + 'stars': page_idx * per_page + len(page), + }) + except Exception as e: + log.warning(f'Error fetching star history page {page_idx} for {repo.name}: {e}') + + return history + + +def _collect_star_history(repo, initial_samples: int = 5) -> list: + """ + Build a cumulative star-history time series for a repository. + + On the first call for a repo the function seeds the history by fetching a + small number of evenly-spaced API pages (``initial_samples``). On every + subsequent call it reads the cached file and appends only today's current + star count, so **no additional API requests are made after the initial + seed**. + + Parameters + ---------- + repo : + PyGithub Repository object. + initial_samples : int + Number of pages to fetch when no cached history exists yet. + + Returns + ------- + list + List of dicts with keys ``date`` (YYYY-MM-DD) and ``stars`` + (cumulative star count at that point in time). + """ + total = repo.stargazers_count + if total == 0: + return [] + + today = datetime.now(tz=timezone.utc).strftime('%Y-%m-%d') + cache_path = os.path.join(BASE_DIR, 'github', 'starHistory', f'{repo.name}.json') + + existing = [] + if os.path.exists(cache_path): + try: + with open(cache_path) as f: + existing = json.load(f) + except Exception: + pass + + if existing: + if existing[-1]['date'] == today: + existing[-1]['stars'] = total + else: + existing.append({'date': today, 'stars': total}) + return existing + + history: list[dict] = list(_seed_star_history(repo, total, initial_samples)) + if not history or history[-1]['date'] != today: + history.append({'date': today, 'stars': total}) + return history + + +def _process_github_repo(repo, headers: dict, graphql_url: str) -> None: + """ + Collect and cache all per-repository data for a single GitHub repo. + + Parameters + ---------- + repo : + PyGithub Repository object. + headers : dict + HTTP headers including the GitHub authorisation token. + graphql_url : str + GitHub GraphQL endpoint URL. + """ + # languages + languages = repo.get_languages() + file_path = os.path.join(BASE_DIR, 'github', 'languages', repo.name) + helpers.write_json_files(file_path=file_path, data=languages) + + # commit activity (last year, weekly buckets) + commit_activity = _get_stats_with_timeout(repo) + if commit_activity: + commits = [week.raw_data for week in commit_activity] + file_path = os.path.join(BASE_DIR, 'github', 'commitActivity', repo.name) + helpers.write_json_files(file_path=file_path, data=commits) + + # open pull requests + pulls_data = [] + for pr in repo.get_pulls(state='open'): + pulls_data.append({ + 'number': pr.number, + 'title': pr.title, + 'author': pr.user.login, + 'labels': [label.name for label in pr.labels], + 'assignees': [assignee.login for assignee in pr.assignees], + 'created_at': pr.created_at.isoformat(), + 'updated_at': pr.updated_at.isoformat(), + 'draft': pr.draft, + 'milestone': pr.milestone.title if pr.milestone else None, + }) + file_path = os.path.join(BASE_DIR, 'github', 'pulls', repo.name) + helpers.write_json_files(file_path=file_path, data=pulls_data) + + # star history (sampled to cap API calls) + star_history = _collect_star_history(repo) + if star_history: + file_path = os.path.join(BASE_DIR, 'github', 'starHistory', repo.name) + helpers.write_json_files(file_path=file_path, data=star_history) + + # openGraphImages - uses GraphQL + query = """ + { + repository(owner: "%s", name: "%s") { + openGraphImageUrl + } + } + """ % (repo.owner.login, repo.name) + + response = helpers.s.post(url=graphql_url, json={'query': query}, headers=headers) + repo_data = response.json() + try: + image_url = repo_data['data']['repository']['openGraphImageUrl'] + except KeyError: + log.error(f'Error: update_github: {repo_data}') + raise SystemExit('"GITHUB_TOKEN" is invalid.') + if 'avatars' not in image_url: + file_path = os.path.join(BASE_DIR, 'github', 'openGraphImages', repo.name) + helpers.save_image_from_url( + file_path=file_path, + file_extension='png', + image_url=image_url, + size_x=624, + size_y=312, + ) + + def update_github(): """ Cache and update GitHub Repo banners and data. """ - # Initialize PyGithub client - g = Github(os.environ["GITHUB_TOKEN"]) + g = Github(auth=Auth.Token(os.environ["GITHUB_TOKEN"]), timeout=30) + g.per_page = 100 # Get the user/organization owner = g.get_user(os.environ["GITHUB_REPOSITORY_OWNER"]) @@ -207,47 +426,9 @@ def update_github(): iterable=repos, desc='Updating GitHub data', ): - # skip archived repos if repo.archived: continue - - # languages - use PyGithub - languages = repo.get_languages() - file_path = os.path.join(BASE_DIR, 'github', 'languages', repo.name) - helpers.write_json_files(file_path=file_path, data=languages) - - # commit activity (last year of activity) - commit_activity = repo.get_stats_commit_activity() - commits = [week.raw_data for week in commit_activity] # Convert PyGithub objects to dict format - - file_path = os.path.join(BASE_DIR, 'github', 'commitActivity', repo.name) - helpers.write_json_files(file_path=file_path, data=commits) - - # openGraphImages - uses GraphQL - query = """ - { - repository(owner: "%s", name: "%s") { - openGraphImageUrl - } - } - """ % (repo.owner.login, repo.name) - - response = helpers.s.post(url=graphql_url, json={'query': query}, headers=headers) - repo_data = response.json() - try: - image_url = repo_data['data']['repository']['openGraphImageUrl'] - except KeyError: - log.error(f'Error: update_github: {repo_data}') - raise SystemExit('"GITHUB_TOKEN" is invalid.') - if 'avatars' not in image_url: - file_path = os.path.join(BASE_DIR, 'github', 'openGraphImages', repo.name) - helpers.save_image_from_url( - file_path=file_path, - file_extension='png', - image_url=image_url, - size_x=624, - size_y=312, - ) + _process_github_repo(repo, headers, graphql_url) def update_patreon(): @@ -351,69 +532,83 @@ def append_thread_if_env_set( def update(): - threads = [] + # Threads that are fully independent of each other and of GitHub data. + independent_threads = [] append_thread_if_env_set( env_vars=['DASHBOARD_AUR_REPOS'], name='aur', target=update_aur, - threads=threads, + threads=independent_threads, kwargs={'aur_repos': os.getenv('DASHBOARD_AUR_REPOS').split(',')}, ) - append_thread_if_env_set( - env_vars=['CODECOV_TOKEN', 'GITHUB_REPOSITORY_OWNER'], - name='codecov', - target=update_codecov, - threads=threads, - ) append_thread_if_env_set( env_vars=['DISCORD_INVITE'], name='discord', target=update_discord, - threads=threads, + threads=independent_threads, ) append_thread_if_env_set( env_vars=['FACEBOOK_TOKEN', 'FACEBOOK_PAGE_ID'], name='facebook', target=update_fb, - threads=threads, - ) - append_thread_if_env_set( - env_vars=['GITHUB_TOKEN', 'GITHUB_REPOSITORY_OWNER'], - name='github', - target=update_github, - threads=threads, + threads=independent_threads, ) append_thread_if_env_set( env_vars=['PATREON_CAMPAIGN_ID'], name='patreon', target=update_patreon, - threads=threads, + threads=independent_threads, ) append_thread_if_env_set( env_vars=['READTHEDOCS_TOKEN'], name='readthedocs', target=update_readthedocs, - threads=threads, + threads=independent_threads, + ) + + # GitHub must finish before Codecov so that repos.json is up to date and + # update_codecov() can correctly skip archived repos. + github_threads = [] + append_thread_if_env_set( + env_vars=['GITHUB_TOKEN', 'GITHUB_REPOSITORY_OWNER'], + name='github', + target=update_github, + threads=github_threads, + ) + + codecov_threads = [] + append_thread_if_env_set( + env_vars=['CODECOV_TOKEN', 'GITHUB_REPOSITORY_OWNER'], + name='codecov', + target=update_codecov, + threads=codecov_threads, ) # setup threading exception handling if os.getenv('THREADING_EXCEPTION_HANDLER'): unhandled_exit.activate() - for thread in tqdm( - iterable=threads, - desc='Starting threads', - ): + # Phase 1: start independent threads and GitHub in parallel. + for thread in tqdm(iterable=independent_threads + github_threads, desc='Starting threads'): thread.start() - # wait for all threads to finish - for thread in tqdm( - iterable=threads, - desc='Waiting for threads to finish', - ): + # Wait for GitHub before starting Codecov. + for thread in tqdm(iterable=github_threads, desc='Waiting for GitHub thread'): + thread.join() + + # Phase 2: start Codecov now that repos.json is fresh. + for thread in tqdm(iterable=codecov_threads, desc='Starting Codecov thread'): + thread.start() + + # Wait for all remaining threads. + for thread in tqdm(iterable=independent_threads + codecov_threads, desc='Waiting for threads to finish'): thread.join() # deactivate threading exception handling if os.getenv('THREADING_EXCEPTION_HANDLER'): unhandled_exit.deactivate() + + +if __name__ == '__main__': + update()