diff --git a/.github/workflows/scan-for-updates.yml b/.github/workflows/scan-for-updates.yml index 337e6b250..5a5c5458c 100644 --- a/.github/workflows/scan-for-updates.yml +++ b/.github/workflows/scan-for-updates.yml @@ -117,7 +117,8 @@ jobs: - uses: peter-evans/create-pull-request@v3 with: token: ${{ steps.generate-token.outputs.token }} - author: "GAP Package Distribution Bot " + # https://api.github.com/users/gap-package-distribution-bot%5Bbot%5D + author: "gap-package-distribution-bot <100730870+gap-package-distribution-bot[bot]@users.noreply.github.com>" add-paths: packages/${{ matrix.package }} commit-message: "[${{ matrix.package }}] Update to ${{ env.PKG_VERSION }}" body: "" diff --git a/.github/workflows/test-all.yml b/.github/workflows/test-all.yml index af2473513..a7197f4ca 100644 --- a/.github/workflows/test-all.yml +++ b/.github/workflows/test-all.yml @@ -4,9 +4,12 @@ # # It builds a version of GAP (which one depends on its inputs), then builds # all packages (resp. all matching some glob), then runs the tests of all -# packages (resp. all matching some glob) with a testfile. Finally the various -# test results are aggregated into a JSON file which other workflows can -# process. +# packages (resp. all matching some glob) with a testfile. The various +# test results are aggregated into a test-status JSON file and is uploaded +# as an artifact. Finally, the test results are compared against the latest +# "official" test run. A human-readable comparison report is generated as a +# MARKDOWN file and a summary of the changes as a test-status-diff JSON file +# which other workflows can process. Both files are uploaded as artifacts. # name: "Test packages" @@ -188,25 +191,38 @@ jobs: id: get-names run: | MATRIX="{\"package\":[" + SKIPPED="" for PKG in packages/${{ github.event.inputs.pkg-test-glob || inputs.pkg-test-glob }}/meta.json; do PKG=${PKG%"/meta.json"} PKG=${PKG#"packages/"} - # if [[ ${PKG} == anupq ]]; then - # break # FIXME: HACK for faster testing, remove before merge - # fi - # skip packages without a TestFile if ! jq -e -r '.TestFile' < packages/${PKG}/meta.json > /dev/null ; then - echo "Skipping ${PKG}" - continue + echo "Skip ${PKG}: no TestFile" + SKIPPED="${SKIPPED}${PKG} " + elif [[ ${PKG} == xgap ]]; then + echo "Skip xgap: no X11 headers, and no means to test it" + SKIPPED="${SKIPPED}${PKG} " + elif [[ ${PKG} == polycyclic ]]; then + # HACK FIXME TODO: skip polycyclic for now + echo "Skip polycyclic tests for now, as they run in an infinite (?) loop" + echo "Re-enable them once there is a new polycyclic release" + SKIPPED="${SKIPPED}${PKG} " + else + MATRIX="${MATRIX}\"${PKG}\"," fi - if [[ ${PKG} == xgap ]]; then - # skip xgap: no X11 headers, and no means to test it - continue - fi - MATRIX="${MATRIX}\"${PKG}\"," done MATRIX="${MATRIX}]}" echo "::set-output name=matrix::$MATRIX" + mkdir -p test-status-skipped + for PKG in ${SKIPPED}; do + echo "{\"status_skipped\": \"skipped\"}" > test-status-skipped/${PKG}.json + done + + - name: "Upload skipped job status as artifact" + if: always() + uses: actions/upload-artifact@v2 + with: + name: "test-status-${{ github.event.inputs.which-gap || inputs.which-gap }}-skipped" + path: "test-status-skipped/*.json" test-package: name: "${{ matrix.package }}" @@ -222,12 +238,6 @@ jobs: - name: "Install package distribution tools" run: | python -m pip install -r tools/requirements.txt - if [[ ${{ matrix.package }} == polycyclic ]]; then - # HACK FIXME TODO: skip polycyclic for now - echo "SKIPPING polycycylic tests for now, as they run in an infinite (?) loop" - echo "Re-enable them once there is a new polycyclic release" - exit 1 - fi - name: "Download GAP from previous job" uses: actions/download-artifact@v2 @@ -267,8 +277,8 @@ jobs: - name: "Run tests with OnlyNeeded" timeout-minutes: 10 + if: always() id: tests-only-needed - if: ${{ always() }} run: | PKG=${{ matrix.package }} gap -A --quitonbreak -r -c " @@ -298,7 +308,7 @@ jobs: # https://docs.github.com/en/actions/learn-github-actions/contexts#steps-context # steps..outcome possible values are success, failure, cancelled, or skipped. - name: "Create job status json-file" - if: ${{ always() }} + if: always() run: | PKG="${{ matrix.package }}" STATUS_DEFAULT="${{ steps.tests-default.outcome }}" @@ -307,16 +317,16 @@ jobs: cat ${PKG}.json - name: "Upload job status as artifact" - if: ${{ always() }} + if: always() uses: actions/upload-artifact@v2 with: - name: "reports-${{ matrix.package }}" + name: "test-status-${{ github.event.inputs.which-gap || inputs.which-gap }}-${{ matrix.package }}" path: "${{ matrix.package }}.json" report: - name: "Report" + name: "Generate report" needs: test-package - if: ${{ always() }} + if: always() runs-on: ubuntu-latest outputs: test-status: ${{ steps.test-status.outputs.test-status }} @@ -330,25 +340,49 @@ jobs: - name: "Download every job status" uses: elonh/download-artifact-regexp@master # FIXME/TODO: Switch to actions/download-artifact once they officially support wildcards, see https://github.com/actions/download-artifact/issues/6 with: - pattern: reports-* + pattern: test-status-${{ github.event.inputs.which-gap || inputs.which-gap }}-* path: _reports - # warning/error in utils.py throw a syntax error with default version of python - - name: "Set up Python 3.7" - uses: actions/setup-python@v2 - with: - python-version: 3.7 - - name: "Install package distribution tools" - run: | - python -m pip install -r tools/requirements.txt + run: python -m pip install -r tools/requirements.txt - name: "Generate test-status.json" - id: report - run: python tools/generate_test_status.py ${{ github.repository }} "$GITHUB_RUN_ID" "$GITHUB_SHA" $(git rev-parse --short "$GITHUB_SHA") - - - name: "Output test-status" id: test-status run: | - TEST_STATUS=$(cat test-status.json) - echo "::set-output name=test-status::$TEST_STATUS" + ROOT='data/reports' + # relative path (with respect to ROOT) to generated test-status.json, i.e. the "id" entry of the json-file. + DIR_REL=$(python tools/generate_test_status.py ${{ github.repository }} "$GITHUB_RUN_ID" "$GITHUB_SHA" ${{ github.event.inputs.which-gap || inputs.which-gap }}) + DIR="${ROOT}/${DIR_REL}" + echo "::set-output name=dir::${DIR}" + echo "::set-output name=dir-rel::${DIR_REL}" + + - name: "Download latest report" + id: download-latest-report + run: | + ROOT="data/reports" + DIR_SYM_REL=latest-${{ github.event.inputs.which-gap || inputs.which-gap }} + DIR_SYM="${ROOT}/${DIR_SYM_REL}" + URL_SYM="https://raw.githubusercontent.com/${{ github.repository }}/${DIR_SYM}" + # Check if file at url exists, + # so we do not run into errors for the first run of the script + # (when the first report is created and thus no latest report is available) + if wget --spider "${URL_SYM}"; then + # wget downloads the "symbolic link" as a plain file + # containing as content the path that the symlink points to, + # so we need to convert this into a real symbolic link. + DIR_REL=$(wget -O - ${URL_SYM}) + DIR="${ROOT}/${DIR_REL}" + ln -s ${DIR_REL} ${DIR_SYM} + URL="https://raw.githubusercontent.com/${{ github.repository }}/${DIR}/test-status.json" + wget -P ${DIR} ${URL} + fi + + - name: "Generate report" + id: report + run: python tools/generate_report.py ${{ steps.test-status.outputs.dir-rel }} latest-${{ github.event.inputs.which-gap || inputs.which-gap }} + + - name: "Upload report as artifact" + uses: actions/upload-artifact@v2 + with: + name: "report-${{ github.event.inputs.which-gap || inputs.which-gap }}" + path: "${{ steps.test-status.outputs.dir }}" diff --git a/.github/workflows/test-daily.yml b/.github/workflows/test-daily.yml new file mode 100644 index 000000000..9f60bde94 --- /dev/null +++ b/.github/workflows/test-daily.yml @@ -0,0 +1,42 @@ +# +# This workflow is run regularly and tests the official package distribution +# by calling the test-all and update-latest-report YML workflows. +# +name: "Daily tests" + +on: + workflow_dispatch: # for debugging + schedule: + - cron: '00 2 * * *' + + +jobs: + # master + test-all-master: + name: "Test packages" + uses: ./.github/workflows/test-all.yml + with: + which-gap: master + + update-latest-master: + name: "Upload report" + needs: test-all-master + if: always() + uses: ./.github/workflows/update-latest-report.yml + with: + which-gap: master + + # 4.11.1 + test-all-4-11-1: + name: "Test packages" + uses: ./.github/workflows/test-all.yml + with: + which-gap: 4.11.1 + + update-latest-4-11: + name: "Upload report" + needs: test-all-4-11-1 + if: always() + uses: ./.github/workflows/update-latest-report.yml + with: + which-gap: 4.11.1 diff --git a/.github/workflows/update-latest-report.yml b/.github/workflows/update-latest-report.yml new file mode 100644 index 000000000..72f510065 --- /dev/null +++ b/.github/workflows/update-latest-report.yml @@ -0,0 +1,75 @@ +# +# This workflow is run either by a manual workflow dispatch, or +# as part of a daily build. +# +# We assume that we have run the test suite for all packages and +# generated a report by calling the test-all YML workflow beforehand. +# The report is uploaded to the data branch on the repository +# and the symlink pointing to the latest report is updated there. +# In addition, the badge for the latest workflow is generated and +# is uploaded to the data branch as well. Finally, the html +# redirect pointing to the latest report is updated and uploaded +# to the gh-pages branch on the repository. +# +# TODO: Create a Slack notification if a package is failing only +# on the current run. +# +name: "Update latest report" + +on: + workflow_call: + inputs: + which-gap: + description: 'Either a GAP branch name or a GAP version' + required: true + type: string + default: master # or 4.11.1 or ... + +jobs: + update-latest: + name: "Upload report" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: "Install package distribution tools" + run: python -m pip install -r tools/requirements.txt + + - name: "Download report" + uses: actions/download-artifact@v3 + with: + name: report-${{ github.event.inputs.which-gap || inputs.which-gap }} + path: _report + + - name: "Create data and gh-pages worktree" + run: | + git remote set-url origin https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY} + git fetch + git branch --track data origin/data + git worktree add data data + git branch --track gh-pages origin/gh-pages + git worktree add gh-pages gh-pages + + - name: "Update latest report" + run: | + ROOT="data/reports" + DIR_REL=$(jq -r '.id' < '_report/test-status.json') + DIR="${ROOT}/${DIR_REL}" + mkdir -p ${DIR} + mv _report/* ${DIR} + python tools/update_latest_report.py ${DIR_REL} latest-${{ github.event.inputs.which-gap || inputs.which-gap }} + + - name: "Push report" + id: push-report + run: | + git config --global user.name 'gap-package-distribution-bot' + # https://api.github.com/users/gap-package-distribution-bot%5Bbot%5D + git config --global user.email '100730870+gap-package-distribution-bot[bot]@users.noreply.github.com' + cd data + git add -A + git commit -m "Automated report" + git pull --rebase && git push + cd ../gh-pages + git add -A + git commit -m "Automated redirect" + git pull --rebase && git push diff --git a/tools/generate_report.py b/tools/generate_report.py new file mode 100644 index 000000000..f759adddd --- /dev/null +++ b/tools/generate_report.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 + +############################################################################# +## +## This file is part of GAP, a system for computational discrete algebra. +## +## Copyright of GAP belongs to its developers, whose names are too numerous +## to list here. Please refer to the COPYRIGHT file for details. +## +## SPDX-License-Identifier: GPL-2.0-or-later +## + +# Run this script on main branch with data and gh-pages worktree + +""" +This script compares the current test-status.json with a previous version, +and generates a main report.md along with a test-status-diff.json. +""" + +from utils import error + +import sys +import os +import json + +################################################################################ +# Arguments and Paths +num_args = len(sys.argv) + +if num_args <= 1 or num_args > 3: + error('Unknown number of arguments') + +# relative paths to report directories from root +root = 'data/reports' +os.makedirs(root, exist_ok = True) +dir_last_report_rel = 'latest-master' + +if num_args > 1: dir_report_rel = sys.argv[1] +if num_args > 2: dir_last_report_rel = sys.argv[2] + +dir_report = os.path.join(root, dir_report_rel) +dir_last_report_symbolic = os.path.join(root, dir_last_report_rel) +dir_last_report = os.path.realpath(dir_last_report_symbolic) + +report_path = os.path.join(dir_report, 'test-status.json') +last_report_path = os.path.join(dir_last_report, 'test-status.json') + +################################################################################ +# Read current and previous test-status +with open(report_path, 'r') as f: + report = json.load(f) + +if os.path.isfile(last_report_path): + with open(last_report_path, 'r') as f: + last_report = json.load(f) +else: # deal with the first run of this script + last_report = {'pkgs': {}, 'id': 'NULL'} + +repo = report['repo'] + +################################################################################ +# Generate report.md and test-status-diff.json +report_diff = {} +report_diff['current'] = report['id'] +report_diff['last'] = last_report['id'] +report_diff['total'] = report['total'] +report_diff['failure'] = report['failure'] +report_diff['success'] = report['success'] +report_diff['skipped'] = report['skipped'] + +################################################################################ +# Package Dictionaries +pkgs = report['pkgs'] +last_pkgs = last_report['pkgs'] +pkgs_new = pkgs.keys() - last_pkgs.keys() +pkgs_removed = last_pkgs.keys() - pkgs.keys() +status_list = ['failure', 'success', 'skipped'] + +pkgs_changed = {} +for status in status_list: + pkgs_changed[status] = [pkg for pkg in pkgs.keys() if + pkg in last_pkgs.keys() and + pkgs[pkg]['status'] != last_pkgs[pkg]['status'] and + pkgs[pkg]['status'] == status] + +pkgs_same = {} +for status in status_list: + pkgs_same[status] = [pkg for pkg in pkgs.keys() if + pkg in last_pkgs.keys() and + pkgs[pkg]['status'] == last_pkgs[pkg]['status'] and + pkgs[pkg]['status'] == status] + +with open(dir_report+'/report.md', 'w') as f: + ############################################################################ + # Header + f.write('# Package Evaluation Report\n\n') + f.write('## Job Properties\n\n') + f.write('*Testing:* [%s](%s) vs [%s](%s)\n\n' % ( + report_diff['current'], + os.path.join(repo, 'blob', root, report_diff['current']), + report_diff['last'], + os.path.join(repo, 'blob', root, report_diff['last']) + )) + f.write('*Generated by Workflow:* %s\n\n' % report['workflow']) + f.write('In total, %d packages were tested, out of which %d succeeded, %d failed and %d were skipped.\n\n' % (report['total'], report['success'], report['failure'], report['skipped'])) + if len(pkgs_changed['failure']) > 0: + f.write(':bangbang: **Detected package(s) failing only on current version.** :bangbang:\n\n') + + ############################################################################ + # New Packages + report_diff['new'] = len(pkgs_new) + if len(pkgs_new) > 0: + f.write('## New Packages\n\n') + for pkg in pkgs_new: + version = pkgs[pkg]['version'] + status = pkgs[pkg]['status'] + f.write('- %s %s : %s
\n' % (pkg, version, status)) + + f.write('\n') + + ############################################################################ + # Removed Packages + report_diff['removed'] = len(pkgs_removed) + if len(pkgs_removed) > 0: + f.write('## Removed Packages\n\n') + for pkg in pkgs_removed: + version = last_pkgs[pkg]['version'] + status = last_pkgs[pkg]['status'] + f.write('- %s %s : %s
\n' % (pkg, version, status)) + + f.write('\n') + + ############################################################################ + # Changed Status Packages + for status, status_msg, status_header in [ + ('failure', 'failed', ':exclamation: :exclamation: Packages now failing'), + ('success', 'succeeded', ':heavy_check_mark: :heavy_check_mark: Packages now succeeding'), + ('skipped', 'skipped', ':heavy_multiplication_x: :heavy_multiplication_x: Packages that now skipped')]: + + pkgs_filtered = pkgs_changed[status] + report_diff[status+'_changed'] = len(pkgs_filtered) + if len(pkgs_filtered) > 0: + f.write('## %s\n\n' % status_header) + f.write('%d package(s) %s tests only on the current version.' % (len(pkgs_filtered), status_msg)) + f.write('
Click to show package(s)!\n\n') + for pkg in pkgs_filtered: + version = pkgs[pkg]['version'] + last_status = last_pkgs[pkg]['status'] + last_version = last_pkgs[pkg]['version'] + f.write('- %s %s vs %s %s (%s)
\n' % (pkg, version, pkg, last_version, last_status)) + f.write('
\n\n') + + ############################################################################ + # Same Status Packages + for status, status_msg, status_header in [ + ('failure', 'failed', ':exclamation: Packages still failing'), + ('success', 'succeeded', ':heavy_check_mark: Packages still succeeding'), + ('skipped', 'skipped', ':heavy_minus_sign: Packages that still skipped')]: + + pkgs_filtered = pkgs_same[status] + report_diff[status+'_same'] = len(pkgs_filtered) + if len(pkgs_filtered) > 0: + f.write('## %s\n\n' % status_header) + f.write('%d package(s) %s tests also on the previous version.' % (len(pkgs_filtered), status_msg)) + f.write('
Click to show package(s)!\n\n') + for pkg in pkgs_filtered: + version = pkgs[pkg]['version'] + f.write('- %s %s
\n' % (pkg, version)) + f.write('
\n\n') + +# Write test-status-diff.json +with open(dir_report+'/test-status-diff.json', 'w') as f: + json.dump(report_diff, f, ensure_ascii=False, indent=2) + f.write("\n") diff --git a/tools/generate_test_status.py b/tools/generate_test_status.py index 48c057695..2a963fc2a 100644 --- a/tools/generate_test_status.py +++ b/tools/generate_test_status.py @@ -12,7 +12,12 @@ """ This script collects the job-status of each package from _reports/ -and generates a main test-status.json +and generates a main test-status.json. + +The file is written into data/reports/{{id}} where +id={{which_gap}}/{{date}}-{{hash_short}}. + +Prints {{id}} to terminal. """ from utils import error, warning @@ -32,45 +37,51 @@ if num_args > 5: error('Too many arguments') -repo = runID = hash = hash_short ='Unknown' +repo = runID = hash = which_gap = 'Unknown' if num_args > 1: repo = 'https://github.com/'+sys.argv[1] if num_args > 2: runID = sys.argv[2] if num_args > 3: hash = sys.argv[3] -if num_args > 4: hash_short = sys.argv[4] +if num_args > 4: which_gap = sys.argv[4] ################################################################################ # Collect the job-status of each package from _reports/ -FILES = [] -for FILE in glob.glob('_reports/**/*.json', recursive=True): - FILES.append(FILE) +files = [] +for file in glob.glob('_reports/**/*.json', recursive=True): + files.append(file) -FILES.sort() +files.sort() -PKG_STATUS = {} +pkgs = {} -for FILE in FILES: - with open(FILE, 'r', encoding='utf-8', errors='ignore') as f: +for file in files: + with open(file, 'r', encoding='utf-8', errors='ignore') as f: data = json.load(f) - PKG_STATUS[os.path.splitext(os.path.basename(FILE))[0]] = data - + pkgs[os.path.splitext(os.path.basename(file))[0]] = data ################################################################################ # Generate main test-status.json # General Information -REPORT: dict[str,Any] = {} -REPORT['repo'] = repo -REPORT['workflow'] = repo+'/actions/runs/'+runID -REPORT['hash'] = hash -REPORT['hash_short'] = hash_short -REPORT['date'] = str(datetime.now()) +report: dict[str,Any] = {} +report['repo'] = repo +report['workflow'] = repo+'/actions/runs/'+runID +report['hash'] = hash +date = str(datetime.now()).split('.')[0] +report['date'] = date +report['id'] = os.path.join(which_gap, "%s-%s" % (date.replace(' ','-'), hash[:8])) + +# Path +root = 'data/reports' +dir_test_status = os.path.join(root, report['id']) +os.makedirs(dir_test_status, exist_ok = True) # Package Information -for pkg, data in PKG_STATUS.items(): +for pkg, data in pkgs.items(): with open(os.path.join('packages', pkg, 'meta.json'), 'r') as f: meta = json.load(f) + data['version'] = meta['Version'] data['archive_url'] = meta['ArchiveURL'] data['archive_sha256'] = meta['ArchiveSHA256'] @@ -88,25 +99,28 @@ else: # all are 'success' data['status'] = 'success' -REPORT['pkgs'] = PKG_STATUS +report['pkgs'] = pkgs # Summary Information -REPORT['total'] = 0 -REPORT['success'] = 0 -REPORT['failure'] = 0 -REPORT['skipped'] = 0 +report['total'] = 0 +report['success'] = 0 +report['failure'] = 0 +report['skipped'] = 0 -for pkg, data in PKG_STATUS.items(): - REPORT['total'] += 1 +for pkg, data in pkgs.items(): + report['total'] += 1 status = data['status'] if status == 'success': - REPORT['success'] += 1 + report['success'] += 1 elif status == 'failure': - REPORT['failure'] += 1 + report['failure'] += 1 elif status == 'skipped': - REPORT['skipped'] += 1 + report['skipped'] += 1 else: warning('Unknown job status detected for pkg \"'+pkg+'\"') -with open('test-status.json', 'w') as f: - json.dump(REPORT, f, ensure_ascii=False, indent=2) +with open(os.path.join(dir_test_status, 'test-status.json'), 'w') as f: + json.dump(report, f, ensure_ascii=False, indent=2) + f.write("\n") + +print(report['id']) diff --git a/tools/update_latest_report.py b/tools/update_latest_report.py new file mode 100644 index 000000000..678acd288 --- /dev/null +++ b/tools/update_latest_report.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 + +############################################################################# +## +## This file is part of GAP, a system for computational discrete algebra. +## +## Copyright of GAP belongs to its developers, whose names are too numerous +## to list here. Please refer to the COPYRIGHT file for details. +## +## SPDX-License-Identifier: GPL-2.0-or-later +## + +# Run this script on main branch with data and gh-pages worktree + +""" +This script updates the latest symlink for the report, +and generates a html-redirect along with a badge. +""" + +from utils import error, symlink + +import sys +import os +import json + +################################################################################ +# Arguments and Paths +num_args = len(sys.argv) + +if num_args <= 1 or num_args > 3: + error('Unknown number of arguments') + +# relative paths to report directories from root +root = 'data/reports' +os.makedirs(root, exist_ok = True) +dir_last_report_rel = 'latest-master' + +if num_args > 1: dir_report_rel = sys.argv[1] +if num_args > 2: dir_last_report_rel = sys.argv[2] + +dir_report = os.path.join(root, dir_report_rel) +dir_last_report_symbolic = os.path.join(root, dir_last_report_rel) + +report_path = os.path.join(dir_report, 'test-status.json') + +dir_badge = os.path.join('data/badges', dir_last_report_rel) +os.makedirs(dir_badge, exist_ok = True) + +dir_redirect = os.path.join('gh-pages', dir_last_report_rel) +os.makedirs(dir_redirect, exist_ok = True) + +################################################################################ +# Read current test-status +with open(report_path, 'r') as f: + report = json.load(f) + +repo = report['repo'] + +################################################################################ +# Update symlink +symlink(dir_report_rel, dir_last_report_symbolic, overwrite=True) + +############################################################################ +# Generate html redirect +with open(os.path.join(dir_redirect, 'redirect.html'), 'w') as f: + f.write(''' + + + Redirecting to latest report + + + ''' % (repo+'/blob/'+dir_report+'/report.md', repo+'/'+dir_report+'/report.md')) + +############################################################################ +# Generate badge, see https://shields.io/endpoint +relativeFailures = 1 - report['success'] / report['total'] +if relativeFailures > 0.05: + color = 'critical' +elif relativeFailures > 0: + color = 'important' +else: + color = 'success' + +badge = { + 'schemaVersion' : 1, + 'label': 'Tests', + 'message': '%d/%d passing' % (report['success'], report['total']), + 'color': color, + 'namedLogo': "github" +} + +with open(os.path.join(dir_badge, 'badge.json'), 'w') as f: + json.dump(badge, f, ensure_ascii=False, indent=2) + f.write("\n") diff --git a/tools/utils.py b/tools/utils.py index ff3eca30b..dd5d79578 100644 --- a/tools/utils.py +++ b/tools/utils.py @@ -12,6 +12,7 @@ import os import sys import requests +import tempfile from os.path import join @@ -88,3 +89,42 @@ def archive_name(pkg_name: str) -> str: def archive_url(pkg_name: str) -> str: pkg_json = metadata(pkg_name) return pkg_json["ArchiveURL"] + pkg_json["ArchiveFormats"].split(" ")[0] + +# https://stackoverflow.com/questions/8299386/modifying-a-symlink-in-python/55742015#55742015 +def symlink(target, link_name, overwrite=False): + ''' + Create a symbolic link named link_name pointing to target. + If link_name exists then FileExistsError is raised, unless overwrite=True. + When trying to overwrite a directory, IsADirectoryError is raised. + ''' + + if not overwrite: + os.symlink(target, link_name) + return + + # os.replace() may fail if files are on different filesystems + link_dir = os.path.dirname(link_name) + + # Create link to target with temporary filename + while True: + temp_link_name = tempfile.mktemp(dir=link_dir) + + # os.* functions mimic as closely as possible system functions + # The POSIX symlink() returns EEXIST if link_name already exists + # https://pubs.opengroup.org/onlinepubs/9699919799/functions/symlink.html + try: + os.symlink(target, temp_link_name) + break + except FileExistsError: + pass + + # Replace link_name with temp_link_name + try: + # Pre-empt os.replace on a directory with a nicer message + if not os.path.islink(link_name) and os.path.isdir(link_name): + raise IsADirectoryError(f"Cannot symlink over existing directory: '{link_name}'") + os.replace(temp_link_name, link_name) + except: + if os.path.islink(temp_link_name): + os.remove(temp_link_name) + raise