Adding CI for riscv-openocd

.github/workflows/riscv-openocd-ci.yml

@@ -0,0 +1,64 @@
+name: riscv-openocd-ci
+on: [push]
+
+jobs:
+
+  run-riscv-tests:
+    runs-on: ubuntu-18.04
+    steps:
+
+      - name: Install required pkgs
+        run: |
+          sudo apt-get install libtool pkg-config autoconf automake libusb-1.0 libftdi1 lcov device-tree-compiler
+          sudo python3 -m pip install pexpect
+
+      - name: Checkout OpenOCD
+        uses: actions/checkout@v2
+
+      - name: Build OpenOCD
+        run: bash riscv-openocd-ci/build_openocd.sh
+
+      - name: Checkout & build Spike
+        run: bash riscv-openocd-ci/build_spike.sh
+
+      - name: Download RISC-V toolchain
+        run: bash riscv-openocd-ci/download_toolchain.sh
+
+      - name: Update env. variables
+        # Change RISCV and PATH env variables for subsequent steps.
+        run: |
+          echo "`pwd`/riscv-openocd-ci/work/install/bin" >> $GITHUB_PATH
+          echo "RISCV=`pwd`/riscv-openocd-ci/work/install" >> $GITHUB_ENV
+
+      - name: Checkout & run riscv-tests
+        id: run_tests
+        run: bash riscv-openocd-ci/run_tests.sh
+
+      - name: Process test results
+        run: |
+          cd riscv-openocd-ci
+          python3 process_test_results.py --log-dir work/riscv-tests/debug/logs --output-dir work/results/logs
+
+      - name: Store test results
+        # Run if tests were executed.
+        if: steps.run_tests.outputs.exit_code == 0
+        uses: actions/upload-artifact@v2
+        with:
+          name: test-results
+          path: riscv-openocd-ci/work/results/logs
+
+      - name: Collect OpenOCD code coverage
+        id: collect_cov
+        # Run if tests were executed.
+        if: steps.run_tests.outputs.exit_code == 0
+        run: |
+          lcov --capture --directory . --output-file riscv-openocd-ci/work/openocd-coverage.info
+          genhtml riscv-openocd-ci/work/openocd-coverage.info --output-directory riscv-openocd-ci/work/results/openocd-coverage
+
+      - name: Store OpenOCD code coverage
+        # Run if coverage was collected.
+        if: steps.collect_cov.outputs.exit_code == 0
+        uses: actions/upload-artifact@v2
+        with:
+          name: openocd-coverage
+          path: riscv-openocd-ci/work/results/openocd-coverage

riscv-openocd-ci/.gitignore

@@ -0,0 +1 @@
+work

riscv-openocd-ci/README.md

@@ -0,0 +1,21 @@
+# CI for riscv-openocd
+
+This directory contains a set of scripts that automatically 
+run [riscv-tests/debug](https://github.com/riscv/riscv-tests/tree/master/debug)
+against riscv-openocd. 
+
+The scripts are intended to be called automatically by Github
+Actions as a means of testing & continuous integration for riscv-openocd.
+
+The scripts perform these actions:
+
+- Build OpenOCD from source
+- Checkout and build Spike (RISC-V ISA simulator) from source
+- Download a pre-built RISC-V toolchain
+- Use these components together to run 
+  [riscv-tests/debug](https://github.com/riscv/riscv-tests/tree/master/debug)
+- Process the test results
+- Collect code coverage for OpenOCD
+
+See [.github/workflows](../.github/workflows) for an example of how this is 
+used in practice. 

riscv-openocd-ci/build_openocd.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+INSTALL_DIR=`pwd`/riscv-openocd-ci/work/install
+
+# Fail on first error.
+set -e
+
+# Echo commands.
+set -o xtrace
+
+# Assuming OpenOCD source is already checked-out in the current workdir.
+
+./bootstrap
+
+# Enable most frequently used JTAG drivers.
+# Allow for code coverage collection.
+./configure \
+    --enable-remote-bitbang \
+    --enable-jtag_vpi \
+    --enable-ftdi \
+    --prefix=$INSTALL_DIR \
+    CFLAGS="-O0 --coverage -fprofile-arcs -ftest-coverage" \
+    CXXFLAGS="-O0 --coverage -fprofile-arcs -ftest-coverage" \
+    LDFLAGS="-fprofile-arcs -lgcov"
+
+# Patch OpenOCD so that coverage is recorded also when terminated
+# by a signal.
+git apply riscv-openocd-ci/patches/openocd_gcov_flush.patch
+
+# Build and install OpenOCD
+make clean  # safety
+make -j`nproc`
+make install
+
+# Check that OpenOCD runs
+$INSTALL_DIR/bin/openocd --version

riscv-openocd-ci/build_spike.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+CHECKOUT_DIR=`pwd`/riscv-openocd-ci/work/riscv-isa-sim
+INSTALL_DIR=`pwd`/riscv-openocd-ci/work/install
+
+# Fail on first error.
+set -e
+
+# Echo commands.
+set -o xtrace
+
+# Checkout Spike.
+mkdir -p "$CHECKOUT_DIR"
+cd "$CHECKOUT_DIR"
+git clone --recursive https://github.com/riscv/riscv-isa-sim.git .
+
+# Build Spike
+mkdir build
+cd build
+bash ../configure --prefix=$INSTALL_DIR
+make clean  # safety
+make -j`nproc`
+make install
+
+# Check that Spike runs
+$INSTALL_DIR/bin/spike --help

riscv-openocd-ci/download_toolchain.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+TOOLCHAIN_URL="https://buildbot.embecosm.com/job/riscv32-gcc-ubuntu1804/25/artifact/riscv32-embecosm-gcc-ubuntu1804-20201108.tar.gz"
+ARCHIVE_NAME=${TOOLCHAIN_URL##*/}
+DOWNLOAD_DIR=`pwd`/riscv-openocd-ci/work
+INSTALL_DIR=`pwd`/riscv-openocd-ci/work/install
+
+# Fail on first error.
+set -e
+
+# Echo commands.
+set -o xtrace
+
+# Download the toolchain.
+# Use a pre-built toolchain binaries provided by Embecosm: https://buildbot.embecosm.com/
+mkdir -p "$DOWNLOAD_DIR"
+cd "$DOWNLOAD_DIR"
+wget --progress dot:mega "$TOOLCHAIN_URL"
+
+# Extract
+mkdir -p "$INSTALL_DIR"
+cd "$INSTALL_DIR"
+tar xvf "$DOWNLOAD_DIR/$ARCHIVE_NAME" --strip-components=1
+
+# Make symlinks: riscv64-* --> riscv32-*
+cd "$INSTALL_DIR/bin"
+find . -name 'riscv32-*' | while read F; do ln -s $F $(echo $F | sed -e 's/riscv32/riscv64/'); done
+
+# Check that the compiler runs
+./riscv64-unknown-elf-gcc --version

riscv-openocd-ci/patches/openocd_gcov_flush.patch

@@ -0,0 +1,15 @@
+diff --git a/src/server/server.c b/src/server/server.c
+index 4e970fa8f..43bd9fb2e 100644
+--- a/src/server/server.c
++++ b/src/server/server.c
+@@ -725,6 +725,10 @@ void server_free(void)
+
+ void exit_on_signal(int sig)
+ {
++	/* dump coverage before being killed by the signal
++	 * (otherwise gcov's *.gcda files would not be created) */
++	void __gcov_flush(void);
++	__gcov_flush();
+ #ifndef _WIN32
+ 	/* bring back default system handler and kill yourself */
+ 	signal(sig, SIG_DFL);

riscv-openocd-ci/process_test_results.py

@@ -0,0 +1,172 @@
+
+from glob import glob
+import logging
+from logging import info, error
+import os
+import re
+import shutil
+import sys
+
+KNOWN_RESULTS = ["pass", "fail", "not_applicable", "exception"]
+
+
+def info_box(msg):
+    """Display an emphasized message - print an ASCII box around it. """
+    box = " +==" + ("=" * len(msg)) + "==+"
+    info("")
+    info(box)
+    info(" |  " + msg + "  |")
+    info(box)
+    info("")
+
+
+def parse_args():
+    """Process command-line arguments. """
+    import argparse
+    parser = argparse.ArgumentParser("Process logs from riscv-tests/debug")
+    parser.add_argument("--log-dir", required=True, help="Directory where logs from RISC-V debug tests are stored")
+    parser.add_argument("--output-dir", required=True, help="Directory where put post-processed logs")
+    return parser.parse_args()
+
+
+def process_test_logs(log_dir, output_dir):
+    """Process all logs from the testing. """
+
+    assert os.path.isdir(log_dir)
+    os.makedirs(output_dir, exist_ok=True)
+
+    # process log files
+    file_pattern = os.path.join(log_dir, "*.log")
+    log_files = sorted(glob(file_pattern))
+
+    if not len(log_files):
+        # Did not find any *.log.
+        # Either the tests did not start at all or a wrong log directory was specified.
+        raise RuntimeError("No log files (*.log) in directory {}".format(log_dir))
+
+    tests = []
+    for lf in log_files:
+        target, result = process_one_log(lf)
+        copy_one_log(lf, result, output_dir)
+        tests += [{"log": lf, "target": target, "result": result}]
+
+    return tests
+
+
+def process_one_log(log_file):
+    """Parse a single log file, extract required pieces from it. """
+    assert os.path.isfile(log_file)
+    target = None
+    result = None
+    # Find target name and the test result in the log file
+    for line in open(log_file, "r"):
+        target_match = re.match(r"^Target: (\S+)$", line)
+        if target_match is not None:
+            target = target_match.group(1)
+        result_match = re.match(r"^Result: (\S+)$", line)
+        if result_match is not None:
+            result = result_match.group(1)
+            if result not in KNOWN_RESULTS:
+                msg = ("Unknown test result '{}' in file {}. Expected one of: {}"
+                       .format(result, log_file, KNOWN_RESULTS))
+                raise RuntimeError(msg)
+
+    if target is None:
+        raise RuntimeError("Could not find target name in log file {}".format(log_file))
+    if result is None:
+        raise RuntimeError("Could not find test result in log file {}".format(log_file))
+
+    return target, result
+
+
+def copy_one_log(log_file, result, output_dir):
+    """Copy the log to a sub-folder based on the result. """
+    target_dir = os.path.join(output_dir, result)
+    os.makedirs(target_dir, exist_ok=True)
+    assert os.path.isdir(target_dir)
+    shutil.copy2(log_file, target_dir)
+
+
+def print_aggregated_results(tests):
+    """Print the tests grouped by the result. Print also pass/fail/... counts."""
+
+    def _filter_tests(tests, target=None, result=None):
+        tests_out = tests
+        if target is not None:
+            tests_out = filter(lambda t: t["target"] == target, tests_out)
+        if result is not None:
+            tests_out = filter(lambda t: t["result"] == result, tests_out)
+        return list(tests_out)
+
+    # Print lists of passed/failed/... tests
+    outcomes = {
+        "Passed tests": "pass",
+        "Not applicable tests": "not_applicable",
+        "Failed tests": "fail",
+        "Tests ended with exception": "exception",
+    }
+    for caption, result in outcomes.items():
+        info_box(caption)
+        tests_filtered = _filter_tests(tests, result=result)
+        for t in tests_filtered:
+            name = os.path.splitext(os.path.basename(t["log"]))[0]
+            info(name)
+        if not tests_filtered:
+            info("(none)")
+
+    target_names = sorted(set([t["target"] for t in tests]))
+
+    # Print summary - passed/failed/... counts, for each target and total
+
+    info_box("Summary")
+
+    def _print_row(target, total, num_pass, num_na, num_fail, num_exc):
+        info("{:<25} {:<10} {:<10} {:<10} {:<10} {:<10}".format(target, total, num_pass, num_na, num_fail, num_exc))
+
+    _print_row("Target", "# tests", "Pass", "Not_appl.", "Fail", "Exception")
+    _print_row("-----", "-----", "-----", "-----", "-----", "-----")
+    sum_pass = sum_na = sum_fail = sum_exc = 0
+    for tn in target_names:
+        t_pass = len(_filter_tests(tests, target=tn, result="pass"))
+        t_na = len(_filter_tests(tests, target=tn, result="not_applicable"))
+        t_fail = len(_filter_tests(tests, target=tn, result="fail"))
+        t_exc = len(_filter_tests(tests, target=tn, result="exception"))
+        t_sum = len(_filter_tests(tests, target=tn))
+        assert t_sum == t_pass + t_na + t_fail + t_exc  # self-check
+        _print_row(tn, t_sum, t_pass, t_na, t_fail, t_exc)
+        sum_pass += t_pass
+        sum_na += t_na
+        sum_fail += t_fail
+        sum_exc += t_exc
+    assert len(tests) == sum_pass + sum_na + sum_fail + sum_exc  # self-check
+    _print_row("-----", "-----", "-----", "-----", "-----", "-----")
+    _print_row("All targets:", len(tests), sum_pass, sum_na, sum_fail, sum_exc)
+    _print_row("-----", "-----", "-----", "-----", "-----", "-----")
+
+    any_failed = (sum_fail + sum_exc) > 0
+    return any_failed
+
+
+def main():
+    args = parse_args()
+
+    # Use absolute paths.
+    args.log_dir = os.path.abspath(args.log_dir)
+    args.output_dir = os.path.abspath(args.output_dir)
+
+    # Process the log files and print results.
+    tests = process_test_logs(args.log_dir, args.output_dir)
+    any_failed = print_aggregated_results(tests)
+
+    # The overall exit code.
+    exit_code = 1 if any_failed else 0
+    if any_failed:
+        error("Encountered failed test(s). Exiting with non-zero code.")
+    else:
+        info("Success - no failed tests encountered.")
+    return exit_code
+
+
+if __name__ == '__main__':
+    logging.getLogger().setLevel(logging.INFO)
+    sys.exit(main())

riscv-openocd-ci/run_tests.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+CHECKOUT_DIR=`pwd`/riscv-openocd-ci/work/riscv-tests
+
+# Fail on first error.
+set -e
+
+# Echo commands.
+set -o xtrace
+
+# Checkout riscv-tests.
+mkdir -p "$CHECKOUT_DIR"
+cd "$CHECKOUT_DIR"
+git clone --recursive https://github.com/riscv/riscv-tests .
+
+# Run the debug tests.
+# Do not stop even on a failed test.
+# Use slightly more jobs than CPUs. Observed that this still speeds up the testing.
+cd debug
+JOBS=$(($(nproc) + 2))
+make -k -j$JOBS all || true