From 9c0468b80fb52eae3a6ccd9fd6f7d30c61608086 Mon Sep 17 00:00:00 2001 From: Haxy Date: Sat, 6 Jun 2026 00:53:31 +0100 Subject: [PATCH] Build refactoring, all downloads take place before any build attempts and are cached where possible. --- .github/workflows/build.yml | 2 +- .gitignore | 2 ++ Dockerfile | 2 +- README.md | 2 +- depends/check-pv.sh | 3 ++ depends/check-python.sh | 10 ++++-- download-manifest.txt | 5 +++ scripts/000-download-files.sh | 65 +++++++++++++++++++++++++++++++++++ scripts/001-binutils-PPU.sh | 12 ++----- scripts/002-gcc-newlib-PPU.sh | 10 +++--- scripts/005-binutils-SPU.sh | 12 ++----- scripts/006-gcc-newlib-SPU.sh | 10 +++--- scripts/008-psl1ght.sh | 8 ++--- scripts/009-ps3libraries.sh | 22 ++++++++++-- toolchain.sh | 18 +++++++--- 15 files changed, 136 insertions(+), 47 deletions(-) create mode 100755 depends/check-pv.sh create mode 100644 download-manifest.txt create mode 100755 scripts/000-download-files.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a23eaecfb..e062cfe5e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,7 +20,7 @@ jobs: if: matrix.os == 'ubuntu-latest' run: | sudo apt-get update - sudo apt-get -y install autoconf automake bison flex gcc libelf-dev make texinfo libncurses5-dev patch python3 python-is-python3 subversion wget zlib1g-dev libtool-bin python3-dev bzip2 libgmp3-dev pkg-config + sudo apt-get -y install autoconf automake bison flex gcc libelf-dev make texinfo libncurses5-dev patch python3 python-is-python3 subversion wget zlib1g-dev libtool-bin python3-dev bzip2 libgmp3-dev pkg-config pv - name: Runs all the stages in the shell run: | diff --git a/.gitignore b/.gitignore index ae20c71b3..97bccf683 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ build/ .idea/** +downloads/ +ps3dev/ diff --git a/Dockerfile b/Dockerfile index 1676330da..a1b7f382d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ RUN \ apt-get -y install \ autoconf bison build-essential ca-certificates flex git libelf-dev\ libgmp-dev libncurses5-dev libssl-dev libtool-bin pkg-config python-dev-is-python3 \ - texinfo wget zlib1g-dev && \ + texinfo wget zlib1g-dev pv && \ apt-get -y clean autoclean autoremove && \ rm -rf /var/lib/{apt,dpkg,cache,log}/ diff --git a/README.md b/README.md index d03118e70..c91ddc9b5 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ autoconf, automake, bison, flex, gcc, libelf, make, makeinfo, ncurses, patch, python, subversion, wget, zlib, libtool, python, - bzip2, gmp, pkg-config, g++, libssl-dev, clang + bzip2, gmp, pkg-config, g++, libssl-dev, clang, pv ### Linux diff --git a/depends/check-pv.sh b/depends/check-pv.sh new file mode 100755 index 000000000..74741a6a8 --- /dev/null +++ b/depends/check-pv.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +( pv -V ) 1>/dev/null 2>&1 || { echo "ERROR: Install pv before continuing."; exit 1; } diff --git a/depends/check-python.sh b/depends/check-python.sh index 90774e550..6b32953ec 100755 --- a/depends/check-python.sh +++ b/depends/check-python.sh @@ -5,8 +5,14 @@ ( python --version || python -V ) 1>/dev/null 2>&1 || { echo "ERROR: Install python before continuing."; exit 1; } ## Check for python-config -pyprefix=$(python-config --prefix || python3-config --prefix) -[ $? -eq 0 ] || { echo "ERROR: Install python-dev before continuing."; exit 1; } +if command -v python-config >/dev/null 2>&1; then + pyprefix=$(python-config --prefix) +elif command -v python3-config >/dev/null 2>&1; then + pyprefix=$(python3-config --prefix) +else + echo "Neither python-config nor python3-config found" >&2 + exit 1 +fi ## Check for python header files ( ls -1d "${pyprefix}"/include/python[23].*/Python.h || ls -1d /opt/local/include/python[23].*/Python.h ) 1>/dev/null 2>&1 || [ -f "$PYINSTALLDIR/include/Python.h" ] || { echo "ERROR: Install python-dev before continuing."; exit 1; } diff --git a/download-manifest.txt b/download-manifest.txt new file mode 100644 index 000000000..3d2862712 --- /dev/null +++ b/download-manifest.txt @@ -0,0 +1,5 @@ +https://ftpmirror.gnu.org/binutils/binutils-2.22.tar.bz2 binutils-2.22.tar.bz2 6c7af8ed1c8cf9b4b9d6e6fe09a3e1d3d479fe63984ba8b9b26bf356b6313ca9 +https://ftpmirror.gnu.org/gnu/gcc/gcc-7.2.0/gcc-7.2.0.tar.xz gcc-7.2.0.tar.xz 1cf7adf8ff4b5aa49041c8734bbcf1ad18cc4c94d0029aae0f4e48841088479a +https://sourceware.org/pub/newlib/newlib-1.20.0.tar.gz newlib-1.20.0.tar.gz c644b2847244278c57bec2ddda69d8fab5a7c767f3b9af69aa7aa3da823ff692 +https://github.com/ps3dev/PSL1GHT/tarball/master psl1ght-master.tar.gz +https://github.com/ps3dev/ps3libraries/tarball/master ps3libraries-master.tar.gz diff --git a/scripts/000-download-files.sh b/scripts/000-download-files.sh new file mode 100755 index 000000000..a0fcc6f12 --- /dev/null +++ b/scripts/000-download-files.sh @@ -0,0 +1,65 @@ +#!/bin/bash +set -euo pipefail + +MANIFEST="../download-manifest.txt" +DOWNLOAD_DIR="../downloads" + +echo "Downloading files to ${DOWNLOAD_DIR}" + +mkdir -p "$DOWNLOAD_DIR" + +while IFS= read -r line || [[ -n "$line" ]]; do + # handle Windows line endings + line="${line%$'\r'}" + + # skip blanks/comments + [[ -z "$line" || "$line" == \#* ]] && continue + + url=$(awk '{print $1}' <<< "$line") + filename=$(awk '{print $2}' <<< "$line") + expected_sha=$(awk '{print $3}' <<< "$line") + + filepath="$DOWNLOAD_DIR/$filename" + tmpfile="${filepath}.tmp" + + # URL + filename only → always download + if [[ -z "${expected_sha:-}" ]]; then + echo "No SHA provided, force downloading: $filepath" + curl -L -o "$tmpfile" "$url" + mv "$tmpfile" "$filepath" + echo "Downloaded: $filepath" + continue + fi + + # SHA mode → cache check + if [[ -f "$filepath" ]]; then + actual_sha=$(sha256sum "$filepath" | awk '{print $1}') + + if [[ "$actual_sha" == "$expected_sha" ]]; then + echo "OK (cached): $filepath" + continue + else + echo "SHA mismatch, re-downloading: $filepath" + fi + else + echo "Missing file, downloading: $filepath" + fi + + curl -L -o "$tmpfile" "$url" + + actual_sha=$(sha256sum "$tmpfile" | awk '{print $1}') + + if [[ "$actual_sha" == "$expected_sha" ]]; then + mv "$tmpfile" "$filepath" + echo "OK: $filepath" + else + echo "FAIL: $filepath" + echo "Expected: $expected_sha" + echo "Got: $actual_sha" + rm -f "$tmpfile" + exit 1 + fi + +done < "$MANIFEST" + +echo "All files verified." \ No newline at end of file diff --git a/scripts/001-binutils-PPU.sh b/scripts/001-binutils-PPU.sh index 0ff372f22..eb86ef72d 100755 --- a/scripts/001-binutils-PPU.sh +++ b/scripts/001-binutils-PPU.sh @@ -5,21 +5,15 @@ BINUTILS="binutils-2.22" if [ ! -d ${BINUTILS} ]; then - ## Download the source code. - if [ ! -f ${BINUTILS}.tar.bz2 ]; then wget --continue https://ftpmirror.gnu.org/binutils/${BINUTILS}.tar.bz2; fi - - ## Download an up-to-date config.guess and config.sub - if [ ! -f config.guess ]; then wget --continue https://git.savannah.gnu.org/cgit/config.git/plain/config.guess; fi - if [ ! -f config.sub ]; then wget --continue https://git.savannah.gnu.org/cgit/config.git/plain/config.sub; fi - ## Unpack the source code. - tar xfvj ${BINUTILS}.tar.bz2 + echo "Unpacking ${BINUTILS}" + pv -pterab ../downloads/${BINUTILS}.tar.bz2 | tar xjf - ## Patch the source code. cat ../patches/${BINUTILS}-PS3.patch | patch -p1 -d ${BINUTILS} ## Replace config.guess and config.sub - cp config.guess config.sub ${BINUTILS} + cp "$(automake --print-libdir)"/config.guess "$(automake --print-libdir)"/config.sub ${BINUTILS} fi diff --git a/scripts/002-gcc-newlib-PPU.sh b/scripts/002-gcc-newlib-PPU.sh index 705e07315..b3ad383dd 100755 --- a/scripts/002-gcc-newlib-PPU.sh +++ b/scripts/002-gcc-newlib-PPU.sh @@ -6,13 +6,11 @@ NEWLIB="newlib-1.20.0" if [ ! -d ${GCC} ]; then - ## Download the source code. - if [ ! -f ${GCC}.tar.xz ]; then wget --continue https://ftpmirror.gnu.org/gnu/gcc/${GCC}/${GCC}.tar.xz; fi - if [ ! -f ${NEWLIB}.tar.gz ]; then wget --continue https://sourceware.org/pub/newlib/${NEWLIB}.tar.gz; fi - ## Unpack the source code. - rm -Rf ${GCC} && tar xfvJ ${GCC}.tar.xz - rm -Rf ${NEWLIB} && tar xfvz ${NEWLIB}.tar.gz + echo "Unpacking ${GCC}" + pv -pterab ../downloads/${GCC}.tar.xz | tar xJf - + echo "Unpacking ${NEWLIB}" + pv -pterab ../downloads/${NEWLIB}.tar.gz | tar xzf - ## Patch the source code. cat ../patches/${GCC}-PS3.patch | patch -p1 -d ${GCC} diff --git a/scripts/005-binutils-SPU.sh b/scripts/005-binutils-SPU.sh index 99da4b91a..dfbbf6627 100755 --- a/scripts/005-binutils-SPU.sh +++ b/scripts/005-binutils-SPU.sh @@ -5,21 +5,15 @@ BINUTILS="binutils-2.22" if [ ! -d ${BINUTILS} ]; then - ## Download the source code. - if [ ! -f ${BINUTILS}.tar.bz2 ]; then wget --continue https://ftpmirror.gnu.org/binutils/${BINUTILS}.tar.bz2; fi - - ## Download an up-to-date config.guess and config.sub - if [ ! -f config.guess ]; then wget --continue https://git.savannah.gnu.org/cgit/config.git/plain/config.guess; fi - if [ ! -f config.sub ]; then wget --continue https://git.savannah.gnu.org/cgit/config.git/plain/config.sub; fi - ## Unpack the source code. - tar xfvj ${BINUTILS}.tar.bz2 + echo "Unpacking ${BINUTILS}" + pv -pterab ../downloads/${BINUTILS}.tar.bz2 | tar xjf - ## Patch the source code. cat ../patches/${BINUTILS}-PS3.patch | patch -p1 -d ${BINUTILS} ## Replace config.guess and config.sub - cp config.guess config.sub ${BINUTILS} + cp "$(automake --print-libdir)"/config.guess "$(automake --print-libdir)"/config.sub ${BINUTILS} fi diff --git a/scripts/006-gcc-newlib-SPU.sh b/scripts/006-gcc-newlib-SPU.sh index 837da9aff..80a22d7c1 100755 --- a/scripts/006-gcc-newlib-SPU.sh +++ b/scripts/006-gcc-newlib-SPU.sh @@ -6,13 +6,11 @@ NEWLIB="newlib-1.20.0" if [ ! -d ${GCC} ]; then - ## Download the source code. - if [ ! -f ${GCC}.tar.xz ]; then wget --continue https://ftpmirror.gnu.org/gnu/gcc/${GCC}/${GCC}.tar.xz; fi - if [ ! -f ${NEWLIB}.tar.gz ]; then wget --continue https://sourceware.org/pub/newlib/${NEWLIB}.tar.gz; fi - ## Unpack the source code. - rm -Rf ${GCC} && tar xfvJ ${GCC}.tar.xz - rm -Rf ${NEWLIB} && tar xfvz ${NEWLIB}.tar.gz + echo "Unpacking ${GCC}" + pv -pterab ../downloads/${GCC}.tar.xz | tar xJf - + echo "Unpacking ${NEWLIB}" + pv -pterab ../downloads/${NEWLIB}.tar.gz | tar xzf - ## Patch the source code. cat ../patches/${GCC}-PS3.patch | patch -p1 -d ${GCC} diff --git a/scripts/008-psl1ght.sh b/scripts/008-psl1ght.sh index 0f66d90dc..393ff67a8 100755 --- a/scripts/008-psl1ght.sh +++ b/scripts/008-psl1ght.sh @@ -1,11 +1,11 @@ #!/bin/sh -e # psl1ght.sh by Naomi Peori (naomi@peori.ca) -## Download the source code. -wget --no-check-certificate https://github.com/ps3dev/PSL1GHT/tarball/master -O psl1ght.tar.gz - ## Unpack the source code. -rm -Rf psl1ght && mkdir psl1ght && tar --strip-components=1 --directory=psl1ght -xvzf psl1ght.tar.gz +rm -Rf psl1ght +mkdir psl1ght +echo "Unpacking psl1ght" +pv -pterb ../downloads/psl1ght-master.tar.gz | tar --strip-components=1 --directory=psl1ght -xzf - ## Create the build directory. cd psl1ght diff --git a/scripts/009-ps3libraries.sh b/scripts/009-ps3libraries.sh index 67fd2adc7..c72d26719 100755 --- a/scripts/009-ps3libraries.sh +++ b/scripts/009-ps3libraries.sh @@ -1,11 +1,27 @@ #!/bin/sh -e # ps3libraries.sh by Naomi Peori (naomi@peori.ca) -## Download the source code. -wget --no-check-certificate https://github.com/ps3dev/ps3libraries/tarball/master -O ps3libraries.tar.gz +CACHE_DIR="ps3libraries-temp-download-cache" +DOWNLOADS_DIR="ps3libraries/downloads" + +## Preserve download cache +if [ -d "$DOWNLOADS_DIR" ]; then + cp -r "$DOWNLOADS_DIR" "$CACHE_DIR" +fi ## Unpack the source code. -rm -Rf ps3libraries && mkdir ps3libraries && tar --strip-components=1 --directory=ps3libraries -xvzf ps3libraries.tar.gz && cd ps3libraries +rm -Rf ps3libraries +mkdir ps3libraries +echo "Unpacking ps3libraries" +pv -pterb ../downloads/ps3libraries-master.tar.gz | tar --strip-components=1 --directory=ps3libraries -xzf - + +## Restore download cache +if [ -d "$CACHE_DIR" ]; then + mkdir -p "$DOWNLOADS_DIR" + cp -r "$CACHE_DIR/." "$DOWNLOADS_DIR/" +fi + +cd ps3libraries ## Compile and install. ./libraries.sh diff --git a/toolchain.sh b/toolchain.sh index 4a341ef34..b66d979fa 100755 --- a/toolchain.sh +++ b/toolchain.sh @@ -10,11 +10,15 @@ mkdir -p build && cd build || { echo "ERROR: Could not create the build director ## Use gmake if available which gmake 1>/dev/null 2>&1 && export MAKE=gmake -## Fetch the depend scripts. -DEPEND_SCRIPTS=`ls ../depends/*.sh | sort` - ## Run all the depend scripts. -for SCRIPT in $DEPEND_SCRIPTS; do "$SCRIPT" || { echo "$SCRIPT: Failed."; exit 1; } done +failed=0 +for SCRIPT in $(ls ../depends/*.sh | sort); do + "$SCRIPT" || { + echo "$SCRIPT: Failed." + failed=1 + } +done +[ "$failed" -ne 0 ] && exit "$failed" ## Fetch the build scripts. BUILD_SCRIPTS=`ls ../scripts/*.sh | sort` @@ -28,6 +32,7 @@ if [ $1 ]; then for STEP in $@; do SCRIPT="" for i in $BUILD_SCRIPTS; do + echo ">>> Running: $SCRIPT" if [ `basename $i | cut -d'-' -f1` -eq $STEP ]; then SCRIPT=$i break @@ -44,4 +49,7 @@ if [ $1 ]; then fi ## Run the build scripts. -for SCRIPT in $BUILD_SCRIPTS; do "$SCRIPT" || { echo "$SCRIPT: Failed."; exit 1; } done +for SCRIPT in $BUILD_SCRIPTS; do + echo ">>> Running: $SCRIPT" + "$SCRIPT" || { echo "$SCRIPT: Failed."; exit 1; } +done