From ee8c2ddbaae04bebc27dbb0b03fe8fdc5b194b64 Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Fri, 22 Aug 2025 09:03:37 -0500 Subject: [PATCH 1/3] Back port Jenkinsfile and associated changes to generator/build (3.24) Ticket: ENT-12581 Changelog: none --- Jenkinsfile | 148 ++++++++++++++++++++++++++++++++++++++++ generator/build/main.sh | 79 ++++++++++----------- generator/build/run.sh | 14 +++- 3 files changed, 200 insertions(+), 41 deletions(-) create mode 100644 Jenkinsfile diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 000000000..551dad4e1 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,148 @@ +pipeline { + agent { label 'CONTAINERS' } + environment { + REPOS = "core enterprise nova masterfiles northerntechhq/nt-docs" + PR_BASE = getPR_BASE() + DOCS_BRANCH = getDOCS_BRANCH() + PACKAGE_JOB = "cf-remote" + PACKAGE_UPLOAD_DIRECTORY = "n/a" + PACKAGE_BUILD = "n/a" + } + parameters { + string(name: "CORE_REV", defaultValue: '', description: 'used for changelog, examples. Use NUMBER or "pull/NUMBER/merge" for pull request (it\'s merged version, THIS DOESN\'T MERGE THE PR) or "pull/NUMBER/head" to build the docs with the non-merged code. Special syntax \'tag:SOME_TAG\' can be used to use a tag as a revision.') + string(name: "NOVA_REV", defaultValue: '', description: 'used for changelog. Use NUMBER or "pull/NUMBER/merge" for pull request (it\'s merged version, THIS DOESN\'T MERGE THE PR) or "pull/NUMBER/head" to build the docs with the non-merged code. Special syntax \'tag:SOME_TAG\' can be used to use a tag as a revision.') + string(name: "ENTERPRISE_REV", defaultValue: '', description: 'used for changelog. Use NUMBER or "pull/NUMBER/merge" for pull request (it\'s merged version, THIS DOESN\'T MERGE THE PR) or "pull/NUMBER/head" to build the docs with the non-merged code. Special syntax \'tag:SOME_TAG\' can be used to use a tag as a revision.') + string(name: "MASTERFILES_REV", defaultValue: '', description: 'used to document masterfiles. Use NUMBER or "pull/NUMBER/merge" for pull request (it\'s merged version, THIS DOESN\'T MERGE THE PR) or "pull/NUMBER/head" to build the docs with the non-merged code. Special syntax \'tag:SOME_TAG\' can be used to use a tag as a revision.') + string(name: "DOCS_REV", defaultValue: '', description: 'Use NUMBER or "pull/NUMBER/merge" for pull request (it\'s merged version, THIS DOESN\'T MERGE THE PR) or "pull/NUMBER/head" to build the docs with the non-merged code. Special syntax \'tag:SOME_TAG\' can be used to use a tag as a revision.') + string(name: "NT_DOCS_REV", defaultValue: '', description: 'Use NUMBER or "pull/NUMBER/merge" for pull request (it\'s merged version, THIS DOESN\'T MERGE THE PR) or "pull/NUMBER/head" to build the docs with the non-merged code. Special syntax \'tag:SOME_TAG\' can be used to use a tag as a revision.') + string(name: "DOCS_BRANCH", defaultValue: '', description: 'Where to upload artifacts - to http://buildcache.cloud.cfengine.com/packages/build-documentation-$DOCS_BRANCH/ and https://docs.cfengine.com/docs/$DOCS_BRANCH/') + string(name: "PACKAGE_JOB", defaultValue: 'cf-remote', description: 'where to get CFEngine HUB package from, either a dir at http://buildcache.cloud.cfengine.com/packages like testing-pr or a keyword cf-remote to use cf-remote download') + string(name: "USE_NIGHTLIES_FOR", defaultValue: '', description: 'branch whose nightlies to use (master, 3.18.x, etc) - will be one of http://buildcache.cloud.cfengine.com/packages/testing-pr/jenkins-$USE_NIGHTLIES_FOR-nightly-pipeline-$NUMBER/') + } + options { + checkoutToSubdirectory('documentation') + } + stages { + stage('Environment check') { + steps { + sh 'env' + sh 'whoami; pwd; ls' + sh 'uname -a; cat /etc/os-release' + } + } + // we clean FIRST and NOT at the end of the job so that we can replay various stages and have the build result from previous runs + stage('Clean workspace') { + steps { + sh 'for r in $REPOS; do rm -rf "$(basename "$r")"; done' + } + } + stage('Checkout repositories'){ + steps { + script { + if (env.CHANGE_ID) { + sh "echo \"${pullRequest.title}\" > pull-request-title" + sh "echo \"${pullRequest.body}\" > pull-request-body" + } + } + sh "curl -O https://raw.githubusercontent.com/cfengine/buildscripts/refs/heads/master/ci/create-revisions-file.sh" + sh "chmod u+x ./create-revisions-file.sh" + sh "./create-revisions-file.sh" + sh "cat revisions" + sh "curl -O https://gitlab.com/Northern.tech/OpenSource/GODS/-/raw/master/parallel_git_rev_fetch.sh" + sh "chmod u+x ./parallel_git_rev_fetch.sh" + + withCredentials([sshUserPrivateKey(credentialsId:"autobuild", keyFileVariable: "key")]) { + sh 'export GIT_SSH_COMMAND="ssh -i $key"; ./parallel_git_rev_fetch.sh revisions' + } + } + } + stage('Build documentation') { + steps { + sh 'bash -x documentation/generator/build/run.sh' + } + } + stage('Publish to buildcache') { + steps { + sshPublisher( + // we must use alwaysPublishFromMaster: true because our CONTAINERS build hosts are not in the private network which has access to buildcache.cloud.cfengine.com + alwaysPublishFromMaster: true, + publishers: [ + sshPublisherDesc( + configName: 'buildcache.cloud.cfengine.com', + transfers: [ + sshTransfer( + cleanRemote: false, + excludes: '', + execCommand: ''' +#!/usr/bin/env bash +set -x +WRKDIR="$(pwd)" +export WRKDIR + +mkdir -p upload +mkdir -p output + +# find two tarballs +archive="$(find upload -name "cfengine-documentation-*.tar.gz")" +tarball=$(findfind upload -name packed-for-shipping.tar.gz) +echo "TARBALL: $tarball" +echo "ARCHIVE: $archive" + +# unpack $tarball +( # shubshell to change directories + cd "$(dirname "$tarball")" || exit + tar zxvf packed-for-shipping.tar.gz + rm packed-for-shipping.tar.gz + + # move $archive to the _site + mv "$WRKDIR/$archive" _site + + ls -la +) + +ls -la upload + +# note: this triggers systemd job to AV-scan new files +# and move them to proper places +mv upload/* output +''', + execTimeout: 120000, + flatten: false, + makeEmptyDirs: false, + noDefaultExcludes: false, + patternSeparator: '[, ]+', + remoteDirectory: getRemoteDirectory(), + remoteDirectorySDF: false, + removePrefix: '', + sourceFiles: 'output/' + ) // sshTransfer + ], // transfers + usePromotionTimestamp: false, + useWorkspaceInPromotion: false, + verbose: false + ) // sshPublisherDesc + ] // publishers + ) // sshPublisher + } // steps + } // stage('Publish to buildcache') + } // stages +} +def getDOCS_BRANCH() { + if (env.DOCS_BRANCH) { + return env.DOCS_BRANCH + } else if (env.CHANGE_ID) { + return "${env.CHANGE_TARGET}" + } else { + return "${env.BRANCH_NAME}" + } +} +def getPR_BASE() { + if (env.CHANGE_ID) { + return "${pullRequest.base}" + } else { + return "" + } +} +def getRemoteDirectory() { + return "upload/${env.BUILD_TAG}/build-documentation-${env.DOCS_BRANCH}/${env.BUILD_TAG}/" +} diff --git a/generator/build/main.sh b/generator/build/main.sh index 08ddd8770..ef99ba956 100644 --- a/generator/build/main.sh +++ b/generator/build/main.sh @@ -9,13 +9,22 @@ if [ "$#" != 4 ]; then exit 1 fi +echo "$(basename "$0"): Diagnostic facts about execution environment:" +echo "======" +whoami +cat /etc/os-release +free -h +df -h +uname -a +echo "======" + + export BRANCH=$1 export PACKAGE_JOB=$2 export PACKAGE_UPLOAD_DIRECTORY=$3 export PACKAGE_BUILD=$4 export JOB_TO_UPLOAD=$PACKAGE_JOB -export FLAG_FILE_URL="http://buildcache.cfengine.com/packages/$PACKAGE_JOB/$PACKAGE_UPLOAD_DIRECTORY/PACKAGES_HUB_x86_64_linux_ubuntu_22/core-commitID" export NO_OUTPUT_DIR=1 env @@ -64,36 +73,46 @@ test ! -z "$PACKAGE_UPLOAD_DIRECTORY" test ! -z "$PACKAGE_BUILD" -echo "Waiting for flag file to appear" -for i in $(seq 30); do - if wget -O- "$FLAG_FILE_URL"; then - break - fi - echo "Waiting 10 sec" - sleep 10 -done -# check if flag file is there - if not, script will fail here -wget -O- "$FLAG_FILE_URL" - -echo "Detecting version" -HUB_DIR_NAME=PACKAGES_HUB_x86_64_linux_ubuntu_22 -HUB_DIR_URL="http://buildcache.cfengine.com/packages/$PACKAGE_JOB/$PACKAGE_UPLOAD_DIRECTORY/$HUB_DIR_NAME/" -HUB_PACKAGE_NAME="$(wget "$HUB_DIR_URL" -O- | sed '/\.deb/!d;s/.*"\([^"]*\.deb\)".*/\1/')" -fetch_file "$HUB_DIR_URL$HUB_PACKAGE_NAME" "cfengine-nova-hub.deb" 12 +echo "Install hub package" +if [ "$PACKAGE_JOB" = "cf-remote" ]; then + echo "Install using cf-remote" + sudo apt update -y + sudo apt install -y python3-venv pipx + pipx install cf-remote + export PATH="$HOME/.local/bin:$PATH" + # shellcheck source=/dev/null + source /etc/os-release + rm -rf ~/.cfengine/cf-remote/packages # to ensure we only get one + cf-remote --version "$BRANCH" download "${ID}$(echo "${VERSION_ID}" | cut -d. -f1)" hub "$(uname -m)" + find "$HOME/.cfengine" # debug + find "$HOME/.cfengine" -name '*.deb' -print0 | xargs -0 -I{} cp {} cfengine-nova-hub.deb +else + echo "Installing with old-style fetch_file function" + HUB_DIR_NAME=PACKAGES_HUB_x86_64_linux_ubuntu_22 + HUB_DIR_URL="http://buildcache.cfengine.com/packages/$PACKAGE_JOB/$PACKAGE_UPLOAD_DIRECTORY/$HUB_DIR_NAME/" + HUB_PACKAGE_NAME="$(wget "$HUB_DIR_URL" -O- | sed '/\.deb/!d;s/.*"\([^"]*\.deb\)".*/\1/')" + + fetch_file "$HUB_DIR_URL$HUB_PACKAGE_NAME" "cfengine-nova-hub.deb" 12 +fi sudo apt-get -y purge cfengine-nova-hub || true sudo rm -rf /*/cfengine -# unpack +# we unpack the hub package instead of installing to get around trouble with the package trying to start up services in a container which doesn't work all that well (yet, 2025) sudo dpkg --unpack cfengine-nova-hub.deb rm cfengine-nova-hub.deb + +# TODO: why copy the masterfiles from the package over the top of one we checked out which could have changes from a PR? sudo cp -a /var/cfengine/share/NovaBase/masterfiles "$WRKDIR" sudo chmod -R a+rX "$WRKDIR"/masterfiles # write current branch into the config.yml echo "branch: $BRANCH" >> "$WRKDIR"/documentation/generator/_config.yml +# replace %branch% placeholder with actual branch name +sed -i "s/%branch%/$BRANCH/g" "$WRKDIR"/documentation/hugo/config.toml + # Generate syntax data ./_regenerate_json.sh || exit 4 @@ -104,25 +123,7 @@ echo "branch: $BRANCH" >> "$WRKDIR"/documentation/generator/_config.yml # so instead of set -x we just echo each command ourselves set +x -# since May 14 2019, we need this to run jekyll. IDK why. -echo "+ source ~/.rvm/scripts/rvm" -# shellcheck disable=SC1090 -source ~/.rvm/scripts/rvm -echo "+ rvm --default use 1.9.3-p551" -rvm --default use 1.9.3-p551 -echo "+ source ~/.profile" -ls -lah ~ -# shellcheck disable=SC1090 -test -f ~/.profile && source ~/.profile -echo "+ source ~/.rvm/scripts/rvm" -# shellcheck disable=SC1090 -source ~/.rvm/scripts/rvm - export LC_ALL=C.UTF-8 - -# finally, run actual jekyll -echo "+ bash -x ./_scripts/_run_jekyll.sh $BRANCH || exit 6" -bash -x ./_scripts/_run_jekyll.sh "$BRANCH" || exit 6 - -cd "$WRKDIR"/documentation/generator -npm run build +bash ./build/install_hugo.sh +echo "+ bash -x ./_scripts/_run.sh $BRANCH || exit 6" +bash -x ./_scripts/_run.sh "$BRANCH" || exit 6 diff --git a/generator/build/run.sh b/generator/build/run.sh index 0e1384ba0..2ad8a3274 100644 --- a/generator/build/run.sh +++ b/generator/build/run.sh @@ -14,12 +14,22 @@ fi # * masterfiles (used to document masterfies) # * documentation (this repo) -# These env vars must be defined: -true "${BRANCH?undefined}" +# The PACKAGE* vars are not needed for fast-build jobs as they use cf-remote --version $BRANCH install +# We still require them to have a value but by current convention (until cf-remote --version testing-pr-build-number works) we set them to cf-remote in documentation/Jenkinsfile true "${PACKAGE_JOB?undefined}" true "${PACKAGE_UPLOAD_DIRECTORY?undefined}" true "${PACKAGE_BUILD?undefined}" +# figure out BRANCH from jenkins environment variables +if [ -n "$PR_BASE" ]; then + # PR_BASE comes from documentation/Jenkinsfile and ${pullRequest.base} + BRANCH="$PR_BASE" +elif [ -n "$BRANCH_NAME" ]; then + # jenkins, for pull requests this will be e.g. PR- so not used + # for non-pull reqeusts this will be master, 3.24.x, etc + BRANCH="$BRANCH_NAME" +fi + c=$(buildah from -v "$PWD":/nt docs-revamp-22) trap 'buildah run "$c" bash -c "sudo chown -R root:root /nt; sudo chmod -R a+rwX /nt"; buildah rm "$c" >/dev/null' EXIT buildah run "$c" bash -x documentation/generator/build/main.sh "$BRANCH" "$PACKAGE_JOB" "$PACKAGE_UPLOAD_DIRECTORY" "$PACKAGE_BUILD" From ae029c8009ed3376655490371d52a1a268407a67 Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Fri, 22 Aug 2025 12:31:02 -0500 Subject: [PATCH 2/3] Adjust generator/build/main.sh to fit 3.21 style jekyll instead of 3.24/master hugo style builds (cherry picked from commit 83a53e136f917ec9c56a1be9c0131a2bb70369d0) --- generator/build/main.sh | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/generator/build/main.sh b/generator/build/main.sh index ef99ba956..9761fe3d6 100644 --- a/generator/build/main.sh +++ b/generator/build/main.sh @@ -110,9 +110,6 @@ sudo chmod -R a+rX "$WRKDIR"/masterfiles # write current branch into the config.yml echo "branch: $BRANCH" >> "$WRKDIR"/documentation/generator/_config.yml -# replace %branch% placeholder with actual branch name -sed -i "s/%branch%/$BRANCH/g" "$WRKDIR"/documentation/hugo/config.toml - # Generate syntax data ./_regenerate_json.sh || exit 4 @@ -123,7 +120,25 @@ sed -i "s/%branch%/$BRANCH/g" "$WRKDIR"/documentation/hugo/config.toml # so instead of set -x we just echo each command ourselves set +x +# since May 14 2019, we need this to run jekyll. IDK why. +echo "+ source ~/.rvm/scripts/rvm" +# shellcheck disable=SC1090 +source ~/.rvm/scripts/rvm +echo "+ rvm --default use 1.9.3-p551" +rvm --default use 1.9.3-p551 +echo "+ source ~/.profile" +ls -lah ~ +# shellcheck disable=SC1090 +test -f ~/.profile && source ~/.profile +echo "+ source ~/.rvm/scripts/rvm" +# shellcheck disable=SC1090 +source ~/.rvm/scripts/rvm + export LC_ALL=C.UTF-8 -bash ./build/install_hugo.sh -echo "+ bash -x ./_scripts/_run.sh $BRANCH || exit 6" -bash -x ./_scripts/_run.sh "$BRANCH" || exit 6 + +# finally, run actual jekyll +echo "+ bash -x ./_scripts/_run_jekyll.sh $BRANCH || exit 6" +bash -x ./_scripts/_run_jekyll.sh "$BRANCH" || exit 6 + +cd "$WRKDIR"/documentation/generator +npm run build From 7ee701e546166c65badd4aa25f6077d425d7f1af Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Fri, 22 Aug 2025 10:45:41 -0500 Subject: [PATCH 3/3] Fixed version installed by cf-remote via fast-build(Jenkinsfile) on LTS branches On LTS branches, BRANCH=3.21 without the .x suffix and cf-remote doesn't understand this. Ticket: ENT-12581 Changelog: none (cherry picked from commit 8c01aab7f4a6170e5a7f22bfbb133ac4e94d0e86) --- generator/build/main.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/generator/build/main.sh b/generator/build/main.sh index 9761fe3d6..16513dbe9 100644 --- a/generator/build/main.sh +++ b/generator/build/main.sh @@ -84,7 +84,17 @@ if [ "$PACKAGE_JOB" = "cf-remote" ]; then # shellcheck source=/dev/null source /etc/os-release rm -rf ~/.cfengine/cf-remote/packages # to ensure we only get one - cf-remote --version "$BRANCH" download "${ID}$(echo "${VERSION_ID}" | cut -d. -f1)" hub "$(uname -m)" + # in case of LTS branches like 3.21 (without .x since we are in documentation repo) need to add on .x + if [ "$(expr "$BRANCH" : ".*.x")" = 0 ]; then + if [ "$BRANCH" = "master" ]; then + _VERSION=master + else + _VERSION="$BRANCH".x + fi + else + _VERSION="$BRANCH" # in case someone copy/pastes this to a repo besides documentation + fi + cf-remote --version "$_VERSION" download "${ID}$(echo "${VERSION_ID}" | cut -d. -f1)" hub "$(uname -m)" find "$HOME/.cfengine" # debug find "$HOME/.cfengine" -name '*.deb' -print0 | xargs -0 -I{} cp {} cfengine-nova-hub.deb else