diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 000000000..12f731a04 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,125 @@ +pipeline { + agent { label 'CONTAINERS' } + environment { + REPOS = "core enterprise nova masterfiles northerntechhq/nt-docs" + PR_BASE="${pullRequest.base}" + PACKAGE_JOB = "cf-remote" + PACKAGE_UPLOAD_DIRECTORY = "n/a" + PACKAGE_BUILD = "n/a" + } + parameters { + string(name: "CORE_REV", defaultValue: '', description: 'used for changelog, examples. Use NUMBER or "pull/NUMBER/merge" for pull request (it\'s merged version, THIS DOESN\'T MERGE THE PR) or "pull/NUMBER/head" to build the docs with the non-merged code. Special syntax \'tag:SOME_TAG\' can be used to use a tag as a revision.') + string(name: "NOVA_REV", defaultValue: '', description: 'used for changelog. Use NUMBER or "pull/NUMBER/merge" for pull request (it\'s merged version, THIS DOESN\'T MERGE THE PR) or "pull/NUMBER/head" to build the docs with the non-merged code. Special syntax \'tag:SOME_TAG\' can be used to use a tag as a revision.') + string(name: "ENTERPRISE_REV", defaultValue: '', description: 'used for changelog. Use NUMBER or "pull/NUMBER/merge" for pull request (it\'s merged version, THIS DOESN\'T MERGE THE PR) or "pull/NUMBER/head" to build the docs with the non-merged code. Special syntax \'tag:SOME_TAG\' can be used to use a tag as a revision.') + string(name: "MASTERFILES_REV", defaultValue: '', description: 'used to document masterfiles. Use NUMBER or "pull/NUMBER/merge" for pull request (it\'s merged version, THIS DOESN\'T MERGE THE PR) or "pull/NUMBER/head" to build the docs with the non-merged code. Special syntax \'tag:SOME_TAG\' can be used to use a tag as a revision.') + string(name: "DOCS_REV", defaultValue: '', description: 'Use NUMBER or "pull/NUMBER/merge" for pull request (it\'s merged version, THIS DOESN\'T MERGE THE PR) or "pull/NUMBER/head" to build the docs with the non-merged code. Special syntax \'tag:SOME_TAG\' can be used to use a tag as a revision.') + string(name: "NT_DOCS_REV", defaultValue: '', description: 'Use NUMBER or "pull/NUMBER/merge" for pull request (it\'s merged version, THIS DOESN\'T MERGE THE PR) or "pull/NUMBER/head" to build the docs with the non-merged code. Special syntax \'tag:SOME_TAG\' can be used to use a tag as a revision.') + string(name: "DOCS_BRANCH", defaultValue: '', description: 'Where to upload artifacts - to http://buildcache.cloud.cfengine.com/packages/build-documentation-$DOCS_BRANCH/ and https://docs.cfengine.com/docs/$DOCS_BRANCH/') + string(name: "PACKAGE_JOB", defaultValue: 'cf-remote', description: 'where to get CFEngine HUB package from, either a dir at http://buildcache.cloud.cfengine.com/packages like testing-pr or a keyword cf-remote to use cf-remote download') + string(name: "USE_NIGHTLIES_FOR", defaultValue: '', description: 'branch whose nightlies to use (master, 3.18.x, etc) - will be one of http://buildcache.cloud.cfengine.com/packages/testing-pr/jenkins-$USE_NIGHTLIES_FOR-nightly-pipeline-$NUMBER/') + } + options { + checkoutToSubdirectory('documentation') + } + stages { + stage('Environment check') { + steps { + sh 'env' + sh 'whoami; pwd; ls' + sh 'uname -a; cat /etc/os-release' + } + } + // we clean FIRST and NOT at the end of the job so that we can replay various stages and have the build result from previous runs + stage('Clean workspace') { + steps { + sh 'for r in $REPOS; do rm -rf "$(basename "$r")"; done' + } + } + stage('Checkout repositories'){ + steps { + sh "echo \"${pullRequest.title}\" > pull-request-title" + sh "echo \"${pullRequest.body}\" > pull-request-body" +// TODO: change URL after merging https://github.com/cfengine/buildscripts/pull/1848 + sh "curl -O https://raw.githubusercontent.com/craigcomstock/buildscripts/refs/heads/ENT-12581/ci/create-revisions-file.sh" + sh "chmod u+x ./create-revisions-file.sh" + sh "./create-revisions-file.sh" + sh "cat revisions" + sh "curl -O https://gitlab.com/Northern.tech/OpenSource/GODS/-/raw/master/parallel_git_rev_fetch.sh" + sh "chmod u+x ./parallel_git_rev_fetch.sh" + + withCredentials([sshUserPrivateKey(credentialsId:"autobuild", keyFileVariable: "key")]) { + sh 'export GIT_SSH_COMMAND="ssh -i $key"; ./parallel_git_rev_fetch.sh revisions' + } + } + } + stage('Build documentation') { + steps { + sh 'bash -x documentation/generator/build/run.sh' + } + } + stage('Publish to buildcache') { + steps { + sshPublisher( + // we must use alwaysPublishFromMaster: true because our CONTAINERS build hosts are not in the private network which has access to buildcache.cloud.cfengine.com + alwaysPublishFromMaster: true, + publishers: [ + sshPublisherDesc( + configName: 'buildcache.cloud.cfengine.com', + transfers: [ + sshTransfer( + cleanRemote: false, + excludes: '', + execCommand: ''' +#!/usr/bin/env bash +set -x +WRKDIR="$(pwd)" +export WRKDIR + +mkdir -p upload +mkdir -p output + +# find two tarballs +archive="$(find upload -name "cfengine-documentation-*.tar.gz")" +tarball=$(findfind upload -name packed-for-shipping.tar.gz) +echo "TARBALL: $tarball" +echo "ARCHIVE: $archive" + +# unpack $tarball +( # shubshell to change directories + cd "$(dirname "$tarball")" || exit + tar zxvf packed-for-shipping.tar.gz + rm packed-for-shipping.tar.gz + + # move $archive to the _site + mv "$WRKDIR/$archive" _site + + ls -la +) + +ls -la upload + +# note: this triggers systemd job to AV-scan new files +# and move them to proper places +mv upload/* output +''', + execTimeout: 120000, + flatten: false, + makeEmptyDirs: false, + noDefaultExcludes: false, + patternSeparator: '[, ]+', + remoteDirectory: 'upload/$BUILD_TAG/build-documentation-$DOCS_BRANCH/$BUILD_TAG/', + remoteDirectorySDF: false, + removePrefix: '', + sourceFiles: 'output/' + ) // sshTransfer + ], // transfers + usePromotionTimestamp: false, + useWorkspaceInPromotion: false, + verbose: false + ) // sshPublisherDesc + ] // publishers + ) // sshPublisher + } // steps + } // stage('Publish to buildcache') + } // stages +} diff --git a/generator/build/main.sh b/generator/build/main.sh index 2a063eef0..ef99ba956 100644 --- a/generator/build/main.sh +++ b/generator/build/main.sh @@ -9,13 +9,22 @@ if [ "$#" != 4 ]; then exit 1 fi +echo "$(basename "$0"): Diagnostic facts about execution environment:" +echo "======" +whoami +cat /etc/os-release +free -h +df -h +uname -a +echo "======" + + export BRANCH=$1 export PACKAGE_JOB=$2 export PACKAGE_UPLOAD_DIRECTORY=$3 export PACKAGE_BUILD=$4 export JOB_TO_UPLOAD=$PACKAGE_JOB -export FLAG_FILE_URL="http://buildcache.cfengine.com/packages/$PACKAGE_JOB/$PACKAGE_UPLOAD_DIRECTORY/PACKAGES_HUB_x86_64_linux_ubuntu_22/core-commitID" export NO_OUTPUT_DIR=1 env @@ -64,30 +73,37 @@ test ! -z "$PACKAGE_UPLOAD_DIRECTORY" test ! -z "$PACKAGE_BUILD" -echo "Waiting for flag file to appear" -for i in $(seq 30); do - if wget -O- "$FLAG_FILE_URL"; then - break - fi - echo "Waiting 10 sec" - sleep 10 -done -# check if flag file is there - if not, script will fail here -wget -O- "$FLAG_FILE_URL" - -echo "Detecting version" -HUB_DIR_NAME=PACKAGES_HUB_x86_64_linux_ubuntu_22 -HUB_DIR_URL="http://buildcache.cfengine.com/packages/$PACKAGE_JOB/$PACKAGE_UPLOAD_DIRECTORY/$HUB_DIR_NAME/" -HUB_PACKAGE_NAME="$(wget "$HUB_DIR_URL" -O- | sed '/\.deb/!d;s/.*"\([^"]*\.deb\)".*/\1/')" -fetch_file "$HUB_DIR_URL$HUB_PACKAGE_NAME" "cfengine-nova-hub.deb" 12 +echo "Install hub package" +if [ "$PACKAGE_JOB" = "cf-remote" ]; then + echo "Install using cf-remote" + sudo apt update -y + sudo apt install -y python3-venv pipx + pipx install cf-remote + export PATH="$HOME/.local/bin:$PATH" + # shellcheck source=/dev/null + source /etc/os-release + rm -rf ~/.cfengine/cf-remote/packages # to ensure we only get one + cf-remote --version "$BRANCH" download "${ID}$(echo "${VERSION_ID}" | cut -d. -f1)" hub "$(uname -m)" + find "$HOME/.cfengine" # debug + find "$HOME/.cfengine" -name '*.deb' -print0 | xargs -0 -I{} cp {} cfengine-nova-hub.deb +else + echo "Installing with old-style fetch_file function" + HUB_DIR_NAME=PACKAGES_HUB_x86_64_linux_ubuntu_22 + HUB_DIR_URL="http://buildcache.cfengine.com/packages/$PACKAGE_JOB/$PACKAGE_UPLOAD_DIRECTORY/$HUB_DIR_NAME/" + HUB_PACKAGE_NAME="$(wget "$HUB_DIR_URL" -O- | sed '/\.deb/!d;s/.*"\([^"]*\.deb\)".*/\1/')" + + fetch_file "$HUB_DIR_URL$HUB_PACKAGE_NAME" "cfengine-nova-hub.deb" 12 +fi sudo apt-get -y purge cfengine-nova-hub || true sudo rm -rf /*/cfengine -# unpack +# we unpack the hub package instead of installing to get around trouble with the package trying to start up services in a container which doesn't work all that well (yet, 2025) sudo dpkg --unpack cfengine-nova-hub.deb rm cfengine-nova-hub.deb + +# TODO: why copy the masterfiles from the package over the top of one we checked out which could have changes from a PR? sudo cp -a /var/cfengine/share/NovaBase/masterfiles "$WRKDIR" sudo chmod -R a+rX "$WRKDIR"/masterfiles diff --git a/generator/build/run.sh b/generator/build/run.sh index 0e1384ba0..2ad8a3274 100644 --- a/generator/build/run.sh +++ b/generator/build/run.sh @@ -14,12 +14,22 @@ fi # * masterfiles (used to document masterfies) # * documentation (this repo) -# These env vars must be defined: -true "${BRANCH?undefined}" +# The PACKAGE* vars are not needed for fast-build jobs as they use cf-remote --version $BRANCH install +# We still require them to have a value but by current convention (until cf-remote --version testing-pr-build-number works) we set them to cf-remote in documentation/Jenkinsfile true "${PACKAGE_JOB?undefined}" true "${PACKAGE_UPLOAD_DIRECTORY?undefined}" true "${PACKAGE_BUILD?undefined}" +# figure out BRANCH from jenkins environment variables +if [ -n "$PR_BASE" ]; then + # PR_BASE comes from documentation/Jenkinsfile and ${pullRequest.base} + BRANCH="$PR_BASE" +elif [ -n "$BRANCH_NAME" ]; then + # jenkins, for pull requests this will be e.g. PR- so not used + # for non-pull reqeusts this will be master, 3.24.x, etc + BRANCH="$BRANCH_NAME" +fi + c=$(buildah from -v "$PWD":/nt docs-revamp-22) trap 'buildah run "$c" bash -c "sudo chown -R root:root /nt; sudo chmod -R a+rwX /nt"; buildah rm "$c" >/dev/null' EXIT buildah run "$c" bash -x documentation/generator/build/main.sh "$BRANCH" "$PACKAGE_JOB" "$PACKAGE_UPLOAD_DIRECTORY" "$PACKAGE_BUILD"