From f94bcb33d84a0f32c8267690cdddb82f391d7c38 Mon Sep 17 00:00:00 2001 From: Robert O'Rourke Date: Mon, 21 Mar 2022 17:34:42 +0000 Subject: [PATCH 1/4] Ignore .history --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index bccf7dd..2c7c541 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ *.pem .env +.history build/ +build-bin/ node_modules probot/ src/linters/phpcs/vendor From 8072aedc7481c909790e8ccc40e25f204af45c5e Mon Sep 17 00:00:00 2001 From: Robert O'Rourke Date: Mon, 21 Mar 2022 17:40:01 +0000 Subject: [PATCH 2/4] Add PHP binary build script Related to #161 --- package.json | 1 + scripts/Dockerfile.build-php | 42 ++++++++++++++++++++++++++++++++++++ scripts/build-php.sh | 11 ++++++++++ 3 files changed, 54 insertions(+) create mode 100644 scripts/Dockerfile.build-php create mode 100755 scripts/build-php.sh diff --git a/package.json b/package.json index d5490df..9c67397 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "run.env": "^1.1.0" }, "scripts": { + "build:php": "scripts/build-php.sh", "build:babel-watch": "babel src --out-dir build --ignore src/linters/phpcs/vendor -D -w", "build:bin": "aws s3 sync s3://hm-linter/bin ./bin && chmod +x ./bin/*", "build:lib": "aws s3 sync s3://hm-linter/lib ./lib && chmod +x ./lib/*", diff --git a/scripts/Dockerfile.build-php b/scripts/Dockerfile.build-php new file mode 100644 index 0000000..ab71238 --- /dev/null +++ b/scripts/Dockerfile.build-php @@ -0,0 +1,42 @@ +# Lambda base image Amazon linux +FROM public.ecr.aws/lambda/provided as builder + +# Set desired PHP Version +ARG php_version="7.4.27" +RUN echo ${php_version} +RUN yum clean all && \ + yum install -y autoconf \ + bison \ + bzip2-devel \ + gcc \ + gcc-c++ \ + git \ + gzip \ + libcurl-devel \ + libicu-devel \ + libtool \ + libxml2-devel \ + make \ + oniguruma-devel \ + openssl-devel \ + re2c \ + sqlite \ + sqlite-devel \ + tar \ + unzip \ + xz-devel \ + zip + +# Download the PHP source, compile, and install both PHP and Composer +RUN curl -sL https://github.com/php/php-src/archive/php-${php_version}.tar.gz | tar -xvz && \ + cd php-src-php-${php_version} && \ + ./buildconf --force && \ + ./configure --prefix=/opt/php-bin/ --with-openssl --with-curl --with-zlib --without-pear --enable-bcmath --with-bz2 --enable-mbstring --with-mysqli && \ + make -j 5 && \ + make install && \ + /opt/php-bin/bin/php -v + +# Export binary as output +FROM scratch as export +ARG php_version +COPY --from=builder /opt/php-bin/bin/php /php@${php_version} diff --git a/scripts/build-php.sh b/scripts/build-php.sh new file mode 100755 index 0000000..d200cc1 --- /dev/null +++ b/scripts/build-php.sh @@ -0,0 +1,11 @@ +#!/bin/bash -e + +if [[ -z $1 ]]; then + echo "You must provide a valid PHP version, if running via npm use `npm run build:php -- `" + exit 1 +fi + +PHP_VERSION="$1" +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +docker build --build-arg "php_version=${PHP_VERSION}" -o bin - < "$SCRIPT_DIR/Dockerfile.build-php" From 90e2f4f6ccb8998582706ddbaad4b77a2dcd54ec Mon Sep 17 00:00:00 2001 From: Joe Hoyle Date: Thu, 9 Feb 2023 10:25:20 +0200 Subject: [PATCH 3/4] Build libs for PHP too --- scripts/Dockerfile.build-php | 8 +- scripts/build-php.sh | 3 +- scripts/copy-dependencies.php | 88 +++++++++++++ scripts/libs-x86.txt | 242 ++++++++++++++++++++++++++++++++++ 4 files changed, 339 insertions(+), 2 deletions(-) create mode 100644 scripts/copy-dependencies.php create mode 100644 scripts/libs-x86.txt diff --git a/scripts/Dockerfile.build-php b/scripts/Dockerfile.build-php index ab71238..0f0f852 100644 --- a/scripts/Dockerfile.build-php +++ b/scripts/Dockerfile.build-php @@ -36,7 +36,13 @@ RUN curl -sL https://github.com/php/php-src/archive/php-${php_version}.tar.gz | make install && \ /opt/php-bin/bin/php -v +COPY copy-dependencies.php /copy-deps.php +COPY libs-x86.txt /libs-x86.txt +RUN mkdir /opt/php-bin/lib-export +RUN /opt/php-bin/bin/php /copy-deps.php /opt/php-bin/bin/php /opt/php-bin/lib-export +RUN ls /opt/php-bin/lib-export # Export binary as output FROM scratch as export ARG php_version -COPY --from=builder /opt/php-bin/bin/php /php@${php_version} +COPY --from=builder /opt/php-bin/bin/php /bin/php +COPY --from=builder /opt/php-bin/lib-export /lib diff --git a/scripts/build-php.sh b/scripts/build-php.sh index d200cc1..760cfad 100755 --- a/scripts/build-php.sh +++ b/scripts/build-php.sh @@ -8,4 +8,5 @@ fi PHP_VERSION="$1" SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -docker build --build-arg "php_version=${PHP_VERSION}" -o bin - < "$SCRIPT_DIR/Dockerfile.build-php" +cd $SCRIPT_DIR +docker build --platform linux/amd64 --progress plain --build-arg "php_version=${PHP_VERSION}" -f "Dockerfile.build-php" -o ../ . diff --git a/scripts/copy-dependencies.php b/scripts/copy-dependencies.php new file mode 100644 index 0000000..d3ec388 --- /dev/null +++ b/scripts/copy-dependencies.php @@ -0,0 +1,88 @@ + + * + * For example: + * php copy-dependencies.php /opt/bin/php /opt/lib + * + * This is taken from Bref PHP. + * + ********************************************************/ + +if (! ($argv[1] ?? false)) { + echo 'Missing the first argument, check the file to see how to use it' . PHP_EOL; + exit(1); +} +if (! ($argv[2] ?? false)) { + echo 'Missing the second argument, check the file to see how to use it' . PHP_EOL; + exit(1); +} +[$_, $pathToCheck, $targetDirectory] = $argv; + +$arch = 'x86'; +if (php_uname('m') !== 'x86_64') { + $arch = 'arm'; +} + +$librariesThatExistOnLambda = file(__DIR__ . "/libs-$arch.txt"); +$librariesThatExistOnLambda = array_map('trim', $librariesThatExistOnLambda); +// For some reason some libraries are actually not in Lambda, despite being in the docker image 🤷 +$librariesThatExistOnLambda = array_filter($librariesThatExistOnLambda, function ($library) { + return ! str_contains($library, 'libgcrypt.so') && ! str_contains($library, 'libgpg-error.so'); +}); + +$requiredLibraries = listDependencies($pathToCheck); + +print_r($requiredLibraries); +// Exclude existing system libraries +$requiredLibraries = array_filter($requiredLibraries, function (string $lib) use ($librariesThatExistOnLambda) { + // Libraries that we compiled are in /usr/lib or /usr/lib64, we compiled them because they are more + // recent than the ones in Lambda so we definitely want to use them + $isALibraryWeCompiled = str_starts_with($lib, '/usr/lib'); + $doesNotExistInLambda = !in_array(basename($lib), $librariesThatExistOnLambda, true); + $keep = $isALibraryWeCompiled || $doesNotExistInLambda; + if (! $keep) { + echo "Skipping $lib because it's already in Lambda" . PHP_EOL; + } + return $keep; +}); + +// Copy all the libraries +foreach ($requiredLibraries as $libraryPath) { + $targetPath = $targetDirectory . '/' . basename($libraryPath); + echo "Copying $libraryPath to $targetPath" . PHP_EOL; + $success = copy($libraryPath, $targetPath); + if (! $success) { + throw new RuntimeException("Could not copy $libraryPath to $targetPath"); + } +} + + +function listDependencies(string $path): array +{ + // ldd lists the dependencies of a binary or library/extension (.so file) + exec("ldd $path 2>&1", $lines); + if (str_contains(end($lines), 'exited with unknown exit code (139)')) { + // We can't use `ldd` on binaries (like /opt/bin/php) because it fails on cross-platform builds + // so we fall back to `LD_TRACE_LOADED_OBJECTS` (which doesn't work for .so files, that's why we also try `ldd`) + // See https://stackoverflow.com/a/35905007/245552 + $output = shell_exec("LD_TRACE_LOADED_OBJECTS=1 $path 2>&1"); + if (!$output) { + throw new RuntimeException("Could not list dependencies for $path"); + } + $lines = explode(PHP_EOL, $output); + } + $dependencies = []; + foreach ($lines as $line) { + $matches = []; + if (preg_match('/=> (.*) \(0x[0-9a-f]+\)/', $line, $matches)) { + $dependencies[] = $matches[1]; + } + } + return $dependencies; +} diff --git a/scripts/libs-x86.txt b/scripts/libs-x86.txt new file mode 100644 index 0000000..eff633e --- /dev/null +++ b/scripts/libs-x86.txt @@ -0,0 +1,242 @@ +libattr.so.1 +libattr.so.1.1.0 +libblkid.so.1 +libblkid.so.1.1.0 +libBrokenLocale-2.26.so +libBrokenLocale.so.1 +libbz2.so.1 +libbz2.so.1.0.6 +libc-2.26.so +libcap.so.2 +libcap.so.2.54 +libcidn-2.26.so +libcidn.so.1 +libcom_err.so.2 +libcom_err.so.2.1 +libcrypt-2.26.so +libcrypto.so.10 +libcrypto.so.1.0.2k +.libcrypto.so.1.0.2k.hmac +.libcrypto.so.10.hmac +libcrypt.so.1 +libc.so.6 +libcurl.so.4 +libcurl.so.4.7.0 +libdb-5.3.so +libdb-5.so +libdl-2.26.so +libdl.so.2 +libelf-0.176.so +libelf.so.1 +libexpat.so.1 +libexpat.so.1.6.0 +libffi.so.6 +libffi.so.6.0.1 +libfontconfig.so.1 +libfontconfig.so.1.11.1 +libform.so.6 +libform.so.6.0 +libformw.so.6 +libformw.so.6.0 +libfreebl3.chk +libfreebl3.so +libfreeblpriv3.chk +libfreeblpriv3.so +libfreetype.so.6 +libfreetype.so.6.14.0 +libgcc_s-7-20180712.so.1 +libgcc_s.so.1 +libgcrypt.so.11 +libgcrypt.so.11.8.2 +.libgcrypt.so.11.hmac +libgdbm_compat.so.4 +libgdbm_compat.so.4.0.0 +libgdbm.so.4 +libgdbm.so.4.0.0 +libgio-2.0.so.0 +libgio-2.0.so.0.5600.1 +libglib-2.0.so.0 +libglib-2.0.so.0.5600.1 +libgmodule-2.0.so.0 +libgmodule-2.0.so.0.5600.1 +libgmp.so.10 +libgmp.so.10.2.0 +libgmpxx.so.4 +libgmpxx.so.4.4.0 +libgobject-2.0.so.0 +libgobject-2.0.so.0.5600.1 +libgpg-error.so.0 +libgpg-error.so.0.10.0 +libgpgme-pthread.so.11 +libgpgme-pthread.so.11.8.1 +libgpgme.so.11 +libgpgme.so.11.8.1 +libgssapi_krb5.so.2 +libgssapi_krb5.so.2.2 +libgssrpc.so.4 +libgssrpc.so.4.2 +libgthread-2.0.so.0 +libgthread-2.0.so.0.5600.1 +libhistory.so.6 +libhistory.so.6.2 +libicudata.so.50 +libicudata.so.50.2 +libicui18n.so.50 +libicui18n.so.50.2 +libicuio.so.50 +libicuio.so.50.2 +libicule.so.50 +libicule.so.50.2 +libiculx.so.50 +libiculx.so.50.2 +libicutest.so.50 +libicutest.so.50.2 +libicutu.so.50 +libicutu.so.50.2 +libicuuc.so.50 +libicuuc.so.50.2 +libidn2.so.0 +libidn2.so.0.3.7 +libk5crypto.so.3 +libk5crypto.so.3.1 +libkdb5.so.8 +libkdb5.so.8.0 +libkeyutils.so.1 +libkeyutils.so.1.5 +libkrad.so.0 +libkrad.so.0.0 +libkrb5.so.3 +libkrb5.so.3.3 +libkrb5support.so.0 +libkrb5support.so.0.1 +liblber-2.4.so.2 +liblber-2.4.so.2.10.7 +libldap-2.4.so.2 +libldap-2.4.so.2.10.7 +libldap_r-2.4.so.2 +libldap_r-2.4.so.2.10.7 +liblua-5.1.so +liblzma.so.5 +liblzma.so.5.2.2 +libm-2.26.so +libmagic.so.1 +libmagic.so.1.0.0 +libmemusage.so +libmenu.so.6 +libmenu.so.6.0 +libmenuw.so.6 +libmenuw.so.6.0 +libmount.so.1 +libmount.so.1.1.0 +libm.so.6 +libmvec-2.26.so +libmvec.so.1 +libncurses.so.6 +libncurses.so.6.0 +libncursesw.so.6 +libncursesw.so.6.0 +libnghttp2.so.14 +libnghttp2.so.14.20.0 +libnsl-2.26.so +libnsl.so.1 +libnspr4.so +libnss3.so +libnssckbi.so +libnss_compat-2.26.so +libnss_compat.so.2 +libnssdbm3.chk +libnssdbm3.so +libnss_dns-2.26.so +libnss_dns.so.2 +libnss_files-2.26.so +libnss_files.so.2 +libnsspem.so +libnsssysinit.so +libnssutil3.so +libp11-kit.so.0 +libp11-kit.so.0.3.0 +libpanel.so.6 +libpanel.so.6.0 +libpanelw.so.6 +libpanelw.so.6.0 +libpcprofile.so +libpcre16.so.0 +libpcre16.so.0.2.0 +libpcre32.so.0 +libpcre32.so.0.0.0 +libpcrecpp.so.0 +libpcrecpp.so.0.0.0 +libpcreposix.so.0 +libpcreposix.so.0.0.1 +libpcre.so.1 +libpcre.so.1.2.0 +libplc4.so +libplds4.so +libpng15.so.15 +libpng15.so.15.13.0 +libpopt.so.0 +libpopt.so.0.0.0 +libpsx.so.2 +libpsx.so.2.54 +libpthread-2.26.so +libpthread.so.0 +libpth.so.20 +libpth.so.20.0.27 +libpython2.7.so.1.0 +libreadline.so.6 +libreadline.so.6.2 +libresolv-2.26.so +libresolv.so.2 +librpmbuild.so.3 +librpmbuild.so.3.2.2 +librpmio.so.3 +librpmio.so.3.2.2 +librpmsign.so.1 +librpmsign.so.1.2.2 +librpm.so.3 +librpm.so.3.2.2 +librt-2.26.so +librt.so.1 +libsasl2.so.3 +libsasl2.so.3.0.0 +libSegFault.so +libselinux.so.1 +libsepol.so.1 +libslapi-2.4.so.2 +libslapi-2.4.so.2.10.7 +libsmime3.so +libsoftokn3.chk +libsoftokn3.so +libsqlite3.so.0 +libsqlite3.so.0.8.6 +libssh2.so.1 +libssh2.so.1.0.1 +libssl3.so +libssl.so.10 +libssl.so.1.0.2k +.libssl.so.1.0.2k.hmac +.libssl.so.10.hmac +libstdc++.so.6 +libstdc++.so.6.0.24 +libtasn1.so.6 +libtasn1.so.6.5.3 +libthread_db-1.0.so +libthread_db.so.1 +libtic.so.6 +libtic.so.6.0 +libtinfo.so.6 +libtinfo.so.6.0 +libunistring.so.0 +libunistring.so.0.1.2 +libutil-2.26.so +libutil.so.1 +libuuid.so.1 +libuuid.so.1.3.0 +libverto.so.1 +libverto.so.1.0.0 +libxml2.so.2 +libxml2.so.2.9.1 +libz.so.1 +libz.so.1.2.7 +p11-kit-proxy.so +p11-kit-trust.so From ca4fc4c18214c3789d10c48701c8e80c4e2de118 Mon Sep 17 00:00:00 2001 From: Robert O'Rourke Date: Mon, 28 Oct 2024 12:21:36 +0000 Subject: [PATCH 4/4] Add node_modules build step using latest Node v20 lambda docker image --- package.json | 9 ++++++--- scripts/Dockerfile.build-npm | 12 ++++++++++++ scripts/Dockerfile.build-php | 6 +++--- scripts/build-npm.sh | 9 +++++++++ src/config.js | 1 + src/linters/phpcs/index.js | 2 ++ 6 files changed, 33 insertions(+), 6 deletions(-) create mode 100644 scripts/Dockerfile.build-npm create mode 100755 scripts/build-npm.sh diff --git a/package.json b/package.json index 9c67397..d97234b 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,8 @@ { "private": true, + "engines": { + "node" : "20" + }, "dependencies": { "@humanmade/probot-util": "^0.2.0", "lodash.chunk": "^4.2.0", @@ -23,7 +26,7 @@ "build:babel-watch": "babel src --out-dir build --ignore src/linters/phpcs/vendor -D -w", "build:bin": "aws s3 sync s3://hm-linter/bin ./bin && chmod +x ./bin/*", "build:lib": "aws s3 sync s3://hm-linter/lib ./lib && chmod +x ./lib/*", - "build:npm": "docker run --rm -v \"${PWD}\":/var/task lambci/lambda:build-nodejs12.x npm install", + "build:npm": "scripts/build-npm.sh", "build": "npm run build:bin && npm run build:lib && npm run build:npm", "clean:package": "( test -f lambda-function.zip && rm lambda-function.zip ) || true", "deploy:check": "test -f bin/php && test -f lib/libcrypt.so && test -f production.private-key.pem", @@ -35,7 +38,7 @@ "deploy:push-dev": "NODE_ENV=development run.env scripts/deploy.sh", "deploy": "npm run clean:package && npm run build && npm run deploy:check && npm run deploy:package-production && npm run deploy:push", "deploy:dev": "npm run clean:package && npm run build && npm run deploy:check-dev && npm run deploy:package-dev && npm run deploy:push-dev", - "start": "docker run -it --rm --env-file .env.dev -e NO_UPDATE_NOTIFIER=1 -v \"$PWD\":/var/task:ro --entrypoint /var/task/node_modules/.bin/nodemon lambci/lambda:nodejs12.x /var/task/start-development.js", - "test": "docker run -i --rm --env-file .env -e DOCKER_LAMBDA_USE_STDIN=1 -v \"$PWD\":/var/task:ro lambci/lambda:nodejs12.x index.probotHandler" + "start": "docker run -it --rm --env-file .env.dev -e NO_UPDATE_NOTIFIER=1 -v \"$PWD\":/var/task:ro --entrypoint /var/task/node_modules/.bin/nodemon public.ecr.aws/lambda/nodejs:20 /var/task/start-development.js", + "test": "docker run -i --rm --env-file .env -e DOCKER_LAMBDA_USE_STDIN=1 -v \"$PWD\":/var/task:ro public.ecr.aws/lambda/nodejs:20 index.probotHandler" } } diff --git a/scripts/Dockerfile.build-npm b/scripts/Dockerfile.build-npm new file mode 100644 index 0000000..a90231d --- /dev/null +++ b/scripts/Dockerfile.build-npm @@ -0,0 +1,12 @@ +# Lambda base image Amazon linux +FROM public.ecr.aws/lambda/nodejs:20 AS builder + +# Build Node Modules +COPY package.json package.json +COPY package-lock.json package-lock.json +RUN npm ci + +# Export binary as output +FROM scratch AS export + +COPY --from=builder /var/task/node_modules node_modules diff --git a/scripts/Dockerfile.build-php b/scripts/Dockerfile.build-php index 0f0f852..df20a55 100644 --- a/scripts/Dockerfile.build-php +++ b/scripts/Dockerfile.build-php @@ -1,8 +1,8 @@ # Lambda base image Amazon linux -FROM public.ecr.aws/lambda/provided as builder +FROM public.ecr.aws/lambda/provided:al2 AS builder # Set desired PHP Version -ARG php_version="7.4.27" +ARG php_version="8.2.24" RUN echo ${php_version} RUN yum clean all && \ yum install -y autoconf \ @@ -42,7 +42,7 @@ RUN mkdir /opt/php-bin/lib-export RUN /opt/php-bin/bin/php /copy-deps.php /opt/php-bin/bin/php /opt/php-bin/lib-export RUN ls /opt/php-bin/lib-export # Export binary as output -FROM scratch as export +FROM scratch AS export ARG php_version COPY --from=builder /opt/php-bin/bin/php /bin/php COPY --from=builder /opt/php-bin/lib-export /lib diff --git a/scripts/build-npm.sh b/scripts/build-npm.sh new file mode 100755 index 0000000..00267d8 --- /dev/null +++ b/scripts/build-npm.sh @@ -0,0 +1,9 @@ +#!/bin/bash -e + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +cd $SCRIPT_DIR +cp ../package.json ./ +cp ../package-lock.json ./ +docker build --platform linux/amd64 --progress plain -f "Dockerfile.build-npm" -o ../ . +rm package*.json diff --git a/src/config.js b/src/config.js index 347e0a1..f362c8d 100644 --- a/src/config.js +++ b/src/config.js @@ -11,6 +11,7 @@ const DEFAULT_CONFIG = { phpcs: { enabled: true, version: 'inherit', + php: process.env.DEFAULT_PHP || '8.2', }, eslint: { enabled: true, diff --git a/src/linters/phpcs/index.js b/src/linters/phpcs/index.js index 5870325..eb4bb73 100644 --- a/src/linters/phpcs/index.js +++ b/src/linters/phpcs/index.js @@ -10,6 +10,8 @@ const CONFIG_NAMES = [ 'phpcs.ruleset.xml', ]; +const DEFAULT_PHP = process.env.DEFAULT_PHP || '8.2'; + /** * Convert a phpcs error into formatOutput-style results. *