From be18b72f2ddefbdfc12d3ebe219ed28cb2f1594f Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Tue, 19 Aug 2025 13:04:36 -0500 Subject: [PATCH 1/5] Removed unused flag file in build scripts Ticket: ENT-12581 Changelog: none --- generator/build/main.sh | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/generator/build/main.sh b/generator/build/main.sh index 2a063eef0..050b6e119 100644 --- a/generator/build/main.sh +++ b/generator/build/main.sh @@ -15,7 +15,6 @@ export PACKAGE_UPLOAD_DIRECTORY=$3 export PACKAGE_BUILD=$4 export JOB_TO_UPLOAD=$PACKAGE_JOB -export FLAG_FILE_URL="http://buildcache.cfengine.com/packages/$PACKAGE_JOB/$PACKAGE_UPLOAD_DIRECTORY/PACKAGES_HUB_x86_64_linux_ubuntu_22/core-commitID" export NO_OUTPUT_DIR=1 env @@ -64,17 +63,6 @@ test ! -z "$PACKAGE_UPLOAD_DIRECTORY" test ! -z "$PACKAGE_BUILD" -echo "Waiting for flag file to appear" -for i in $(seq 30); do - if wget -O- "$FLAG_FILE_URL"; then - break - fi - echo "Waiting 10 sec" - sleep 10 -done -# check if flag file is there - if not, script will fail here -wget -O- "$FLAG_FILE_URL" - echo "Detecting version" HUB_DIR_NAME=PACKAGES_HUB_x86_64_linux_ubuntu_22 HUB_DIR_URL="http://buildcache.cfengine.com/packages/$PACKAGE_JOB/$PACKAGE_UPLOAD_DIRECTORY/$HUB_DIR_NAME/" From cb8e5eb85bbd093eb6374eb42aaa9c0e20410452 Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Tue, 19 Aug 2025 13:05:16 -0500 Subject: [PATCH 2/5] Added PACKAGE_JOB=cf-remote option to do fast docs build Ticket: ENT-12581 Changelog: none --- generator/build/main.sh | 30 ++++++++++++++++++++++++------ generator/build/run.sh | 4 ++-- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/generator/build/main.sh b/generator/build/main.sh index 050b6e119..55e046195 100644 --- a/generator/build/main.sh +++ b/generator/build/main.sh @@ -63,19 +63,37 @@ test ! -z "$PACKAGE_UPLOAD_DIRECTORY" test ! -z "$PACKAGE_BUILD" -echo "Detecting version" -HUB_DIR_NAME=PACKAGES_HUB_x86_64_linux_ubuntu_22 -HUB_DIR_URL="http://buildcache.cfengine.com/packages/$PACKAGE_JOB/$PACKAGE_UPLOAD_DIRECTORY/$HUB_DIR_NAME/" -HUB_PACKAGE_NAME="$(wget "$HUB_DIR_URL" -O- | sed '/\.deb/!d;s/.*"\([^"]*\.deb\)".*/\1/')" -fetch_file "$HUB_DIR_URL$HUB_PACKAGE_NAME" "cfengine-nova-hub.deb" 12 +echo "Install hub package" +if [ "$PACKAGE_JOB" = "cf-remote" ]; then + echo "Install using cf-remote" + sudo apt update -y + sudo apt install -y python3-venv pipx + pipx install cf-remote + export PATH="$HOME/.local/bin:$PATH" + # shellcheck source=/dev/null + source /etc/os-release + rm -rf ~/.cfengine/cf-remote/packages # to ensure we only get one + cf-remote --version "$BRANCH" download "${ID}$(echo "${VERSION_ID}" | cut -d. -f1)" hub "$(uname -m)" + find "$HOME/.cfengine" # debug + find "$HOME/.cfengine" -name '*.deb' -print0 | xargs -0 -I{} cp {} cfengine-nova-hub.deb +else + echo "Installing with old-style fetch_file function" + HUB_DIR_NAME=PACKAGES_HUB_x86_64_linux_ubuntu_22 + HUB_DIR_URL="http://buildcache.cfengine.com/packages/$PACKAGE_JOB/$PACKAGE_UPLOAD_DIRECTORY/$HUB_DIR_NAME/" + HUB_PACKAGE_NAME="$(wget "$HUB_DIR_URL" -O- | sed '/\.deb/!d;s/.*"\([^"]*\.deb\)".*/\1/')" + + fetch_file "$HUB_DIR_URL$HUB_PACKAGE_NAME" "cfengine-nova-hub.deb" 12 +fi sudo apt-get -y purge cfengine-nova-hub || true sudo rm -rf /*/cfengine -# unpack +# we unpack the hub package instead of installing to get around trouble with the package trying to start up services in a container which doesn't work all that well (yet, 2025) sudo dpkg --unpack cfengine-nova-hub.deb rm cfengine-nova-hub.deb + +# TODO: why copy the masterfiles from the package over the top of one we checked out which could have changes from a PR? sudo cp -a /var/cfengine/share/NovaBase/masterfiles "$WRKDIR" sudo chmod -R a+rX "$WRKDIR"/masterfiles diff --git a/generator/build/run.sh b/generator/build/run.sh index 0e1384ba0..a7f6c3130 100644 --- a/generator/build/run.sh +++ b/generator/build/run.sh @@ -14,8 +14,8 @@ fi # * masterfiles (used to document masterfies) # * documentation (this repo) -# These env vars must be defined: -true "${BRANCH?undefined}" +# The PACKAGE* vars are not needed for fast-build jobs as they use cf-remote --version $BRANCH install +# We still require them to have a value but by current convention (until cf-remote --version testing-pr-build-number works) we set them to cf-remote in documentation/Jenkinsfile true "${PACKAGE_JOB?undefined}" true "${PACKAGE_UPLOAD_DIRECTORY?undefined}" true "${PACKAGE_BUILD?undefined}" From fdb97ee05e4cbdb75d45f61fdc7711ca85742a95 Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Tue, 19 Aug 2025 13:06:23 -0500 Subject: [PATCH 3/5] Small diagnostic output added during docs build --- generator/build/main.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/generator/build/main.sh b/generator/build/main.sh index 55e046195..ef99ba956 100644 --- a/generator/build/main.sh +++ b/generator/build/main.sh @@ -9,6 +9,16 @@ if [ "$#" != 4 ]; then exit 1 fi +echo "$(basename "$0"): Diagnostic facts about execution environment:" +echo "======" +whoami +cat /etc/os-release +free -h +df -h +uname -a +echo "======" + + export BRANCH=$1 export PACKAGE_JOB=$2 export PACKAGE_UPLOAD_DIRECTORY=$3 From ca2fadaa6199bb1d4a7e8d140c56f92ad37b6acb Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Tue, 19 Aug 2025 13:06:45 -0500 Subject: [PATCH 4/5] Added determining BRANCH from PR_BASE or BRANCH_NAME environment variables provided by jenkins --- generator/build/run.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/generator/build/run.sh b/generator/build/run.sh index a7f6c3130..2ad8a3274 100644 --- a/generator/build/run.sh +++ b/generator/build/run.sh @@ -20,6 +20,16 @@ true "${PACKAGE_JOB?undefined}" true "${PACKAGE_UPLOAD_DIRECTORY?undefined}" true "${PACKAGE_BUILD?undefined}" +# figure out BRANCH from jenkins environment variables +if [ -n "$PR_BASE" ]; then + # PR_BASE comes from documentation/Jenkinsfile and ${pullRequest.base} + BRANCH="$PR_BASE" +elif [ -n "$BRANCH_NAME" ]; then + # jenkins, for pull requests this will be e.g. PR- so not used + # for non-pull reqeusts this will be master, 3.24.x, etc + BRANCH="$BRANCH_NAME" +fi + c=$(buildah from -v "$PWD":/nt docs-revamp-22) trap 'buildah run "$c" bash -c "sudo chown -R root:root /nt; sudo chmod -R a+rwX /nt"; buildah rm "$c" >/dev/null' EXIT buildah run "$c" bash -x documentation/generator/build/main.sh "$BRANCH" "$PACKAGE_JOB" "$PACKAGE_UPLOAD_DIRECTORY" "$PACKAGE_BUILD" From 68f87f8b8394c3742d9543468f0f9d5c359baa79 Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Tue, 19 Aug 2025 13:06:49 -0500 Subject: [PATCH 5/5] Added Jenkinsfile to support fast doc builds via Organization Folders in Jenkins Ticket: ENT-12581 Changelog: none --- Jenkinsfile | 125 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 Jenkinsfile diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 000000000..12f731a04 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,125 @@ +pipeline { + agent { label 'CONTAINERS' } + environment { + REPOS = "core enterprise nova masterfiles northerntechhq/nt-docs" + PR_BASE="${pullRequest.base}" + PACKAGE_JOB = "cf-remote" + PACKAGE_UPLOAD_DIRECTORY = "n/a" + PACKAGE_BUILD = "n/a" + } + parameters { + string(name: "CORE_REV", defaultValue: '', description: 'used for changelog, examples. Use NUMBER or "pull/NUMBER/merge" for pull request (it\'s merged version, THIS DOESN\'T MERGE THE PR) or "pull/NUMBER/head" to build the docs with the non-merged code. Special syntax \'tag:SOME_TAG\' can be used to use a tag as a revision.') + string(name: "NOVA_REV", defaultValue: '', description: 'used for changelog. Use NUMBER or "pull/NUMBER/merge" for pull request (it\'s merged version, THIS DOESN\'T MERGE THE PR) or "pull/NUMBER/head" to build the docs with the non-merged code. Special syntax \'tag:SOME_TAG\' can be used to use a tag as a revision.') + string(name: "ENTERPRISE_REV", defaultValue: '', description: 'used for changelog. Use NUMBER or "pull/NUMBER/merge" for pull request (it\'s merged version, THIS DOESN\'T MERGE THE PR) or "pull/NUMBER/head" to build the docs with the non-merged code. Special syntax \'tag:SOME_TAG\' can be used to use a tag as a revision.') + string(name: "MASTERFILES_REV", defaultValue: '', description: 'used to document masterfiles. Use NUMBER or "pull/NUMBER/merge" for pull request (it\'s merged version, THIS DOESN\'T MERGE THE PR) or "pull/NUMBER/head" to build the docs with the non-merged code. Special syntax \'tag:SOME_TAG\' can be used to use a tag as a revision.') + string(name: "DOCS_REV", defaultValue: '', description: 'Use NUMBER or "pull/NUMBER/merge" for pull request (it\'s merged version, THIS DOESN\'T MERGE THE PR) or "pull/NUMBER/head" to build the docs with the non-merged code. Special syntax \'tag:SOME_TAG\' can be used to use a tag as a revision.') + string(name: "NT_DOCS_REV", defaultValue: '', description: 'Use NUMBER or "pull/NUMBER/merge" for pull request (it\'s merged version, THIS DOESN\'T MERGE THE PR) or "pull/NUMBER/head" to build the docs with the non-merged code. Special syntax \'tag:SOME_TAG\' can be used to use a tag as a revision.') + string(name: "DOCS_BRANCH", defaultValue: '', description: 'Where to upload artifacts - to http://buildcache.cloud.cfengine.com/packages/build-documentation-$DOCS_BRANCH/ and https://docs.cfengine.com/docs/$DOCS_BRANCH/') + string(name: "PACKAGE_JOB", defaultValue: 'cf-remote', description: 'where to get CFEngine HUB package from, either a dir at http://buildcache.cloud.cfengine.com/packages like testing-pr or a keyword cf-remote to use cf-remote download') + string(name: "USE_NIGHTLIES_FOR", defaultValue: '', description: 'branch whose nightlies to use (master, 3.18.x, etc) - will be one of http://buildcache.cloud.cfengine.com/packages/testing-pr/jenkins-$USE_NIGHTLIES_FOR-nightly-pipeline-$NUMBER/') + } + options { + checkoutToSubdirectory('documentation') + } + stages { + stage('Environment check') { + steps { + sh 'env' + sh 'whoami; pwd; ls' + sh 'uname -a; cat /etc/os-release' + } + } + // we clean FIRST and NOT at the end of the job so that we can replay various stages and have the build result from previous runs + stage('Clean workspace') { + steps { + sh 'for r in $REPOS; do rm -rf "$(basename "$r")"; done' + } + } + stage('Checkout repositories'){ + steps { + sh "echo \"${pullRequest.title}\" > pull-request-title" + sh "echo \"${pullRequest.body}\" > pull-request-body" +// TODO: change URL after merging https://github.com/cfengine/buildscripts/pull/1848 + sh "curl -O https://raw.githubusercontent.com/craigcomstock/buildscripts/refs/heads/ENT-12581/ci/create-revisions-file.sh" + sh "chmod u+x ./create-revisions-file.sh" + sh "./create-revisions-file.sh" + sh "cat revisions" + sh "curl -O https://gitlab.com/Northern.tech/OpenSource/GODS/-/raw/master/parallel_git_rev_fetch.sh" + sh "chmod u+x ./parallel_git_rev_fetch.sh" + + withCredentials([sshUserPrivateKey(credentialsId:"autobuild", keyFileVariable: "key")]) { + sh 'export GIT_SSH_COMMAND="ssh -i $key"; ./parallel_git_rev_fetch.sh revisions' + } + } + } + stage('Build documentation') { + steps { + sh 'bash -x documentation/generator/build/run.sh' + } + } + stage('Publish to buildcache') { + steps { + sshPublisher( + // we must use alwaysPublishFromMaster: true because our CONTAINERS build hosts are not in the private network which has access to buildcache.cloud.cfengine.com + alwaysPublishFromMaster: true, + publishers: [ + sshPublisherDesc( + configName: 'buildcache.cloud.cfengine.com', + transfers: [ + sshTransfer( + cleanRemote: false, + excludes: '', + execCommand: ''' +#!/usr/bin/env bash +set -x +WRKDIR="$(pwd)" +export WRKDIR + +mkdir -p upload +mkdir -p output + +# find two tarballs +archive="$(find upload -name "cfengine-documentation-*.tar.gz")" +tarball=$(findfind upload -name packed-for-shipping.tar.gz) +echo "TARBALL: $tarball" +echo "ARCHIVE: $archive" + +# unpack $tarball +( # shubshell to change directories + cd "$(dirname "$tarball")" || exit + tar zxvf packed-for-shipping.tar.gz + rm packed-for-shipping.tar.gz + + # move $archive to the _site + mv "$WRKDIR/$archive" _site + + ls -la +) + +ls -la upload + +# note: this triggers systemd job to AV-scan new files +# and move them to proper places +mv upload/* output +''', + execTimeout: 120000, + flatten: false, + makeEmptyDirs: false, + noDefaultExcludes: false, + patternSeparator: '[, ]+', + remoteDirectory: 'upload/$BUILD_TAG/build-documentation-$DOCS_BRANCH/$BUILD_TAG/', + remoteDirectorySDF: false, + removePrefix: '', + sourceFiles: 'output/' + ) // sshTransfer + ], // transfers + usePromotionTimestamp: false, + useWorkspaceInPromotion: false, + verbose: false + ) // sshPublisherDesc + ] // publishers + ) // sshPublisher + } // steps + } // stage('Publish to buildcache') + } // stages +}